From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- third_party/dav1d/src/arm/32/cdef.S | 540 ++ third_party/dav1d/src/arm/32/cdef16.S | 233 + third_party/dav1d/src/arm/32/cdef_tmpl.S | 515 ++ third_party/dav1d/src/arm/32/filmgrain.S | 2039 +++++ third_party/dav1d/src/arm/32/filmgrain16.S | 2137 +++++ third_party/dav1d/src/arm/32/ipred.S | 2937 ++++++ third_party/dav1d/src/arm/32/ipred16.S | 3254 +++++++ third_party/dav1d/src/arm/32/itx.S | 3343 +++++++ third_party/dav1d/src/arm/32/itx16.S | 3625 ++++++++ third_party/dav1d/src/arm/32/loopfilter.S | 868 ++ third_party/dav1d/src/arm/32/loopfilter16.S | 859 ++ third_party/dav1d/src/arm/32/looprestoration.S | 791 ++ third_party/dav1d/src/arm/32/looprestoration16.S | 801 ++ .../dav1d/src/arm/32/looprestoration_common.S | 453 + .../dav1d/src/arm/32/looprestoration_tmpl.S | 600 ++ third_party/dav1d/src/arm/32/mc.S | 3340 +++++++ third_party/dav1d/src/arm/32/mc16.S | 3658 ++++++++ third_party/dav1d/src/arm/32/msac.S | 575 ++ third_party/dav1d/src/arm/32/refmvs.S | 97 + third_party/dav1d/src/arm/32/util.S | 184 + third_party/dav1d/src/arm/64/cdef.S | 520 ++ third_party/dav1d/src/arm/64/cdef16.S | 229 + third_party/dav1d/src/arm/64/cdef_tmpl.S | 511 ++ third_party/dav1d/src/arm/64/filmgrain.S | 2010 ++++ third_party/dav1d/src/arm/64/filmgrain16.S | 1997 ++++ third_party/dav1d/src/arm/64/ipred.S | 3985 ++++++++ third_party/dav1d/src/arm/64/ipred16.S | 4204 +++++++++ third_party/dav1d/src/arm/64/itx.S | 3270 +++++++ third_party/dav1d/src/arm/64/itx16.S | 3648 ++++++++ third_party/dav1d/src/arm/64/loopfilter.S | 1129 +++ third_party/dav1d/src/arm/64/loopfilter16.S | 925 ++ third_party/dav1d/src/arm/64/looprestoration.S | 1336 +++ third_party/dav1d/src/arm/64/looprestoration16.S | 1419 +++ .../dav1d/src/arm/64/looprestoration_common.S | 432 + .../dav1d/src/arm/64/looprestoration_tmpl.S | 597 ++ third_party/dav1d/src/arm/64/mc.S | 3310 +++++++ third_party/dav1d/src/arm/64/mc16.S | 3611 ++++++++ third_party/dav1d/src/arm/64/msac.S | 480 + third_party/dav1d/src/arm/64/refmvs.S | 91 + third_party/dav1d/src/arm/64/util.S | 229 + third_party/dav1d/src/arm/asm-offsets.h | 43 + third_party/dav1d/src/arm/asm.S | 291 + third_party/dav1d/src/arm/cdef.h | 88 + third_party/dav1d/src/arm/cpu.c | 99 + third_party/dav1d/src/arm/cpu.h | 37 + third_party/dav1d/src/arm/filmgrain.h | 204 + third_party/dav1d/src/arm/ipred.h | 222 + third_party/dav1d/src/arm/itx.h | 141 + third_party/dav1d/src/arm/loopfilter.h | 45 + third_party/dav1d/src/arm/looprestoration.h | 265 + third_party/dav1d/src/arm/mc.h | 114 + third_party/dav1d/src/arm/msac.h | 52 + third_party/dav1d/src/arm/refmvs.h | 39 + third_party/dav1d/src/cdef.h | 71 + third_party/dav1d/src/cdef_apply.h | 39 + third_party/dav1d/src/cdef_apply_tmpl.c | 309 + third_party/dav1d/src/cdef_tmpl.c | 331 + third_party/dav1d/src/cdf.c | 4123 +++++++++ third_party/dav1d/src/cdf.h | 150 + third_party/dav1d/src/cpu.c | 101 + third_party/dav1d/src/cpu.h | 102 + third_party/dav1d/src/ctx.h | 91 + third_party/dav1d/src/data.c | 145 + third_party/dav1d/src/data.h | 56 + third_party/dav1d/src/dav1d.rc.in | 32 + third_party/dav1d/src/decode.c | 3910 ++++++++ third_party/dav1d/src/decode.h | 35 + third_party/dav1d/src/dequant_tables.c | 229 + third_party/dav1d/src/dequant_tables.h | 37 + third_party/dav1d/src/env.h | 521 ++ third_party/dav1d/src/ext/x86/x86inc.asm | 1902 ++++ third_party/dav1d/src/fg_apply.h | 58 + third_party/dav1d/src/fg_apply_tmpl.c | 239 + third_party/dav1d/src/filmgrain.h | 84 + third_party/dav1d/src/filmgrain_tmpl.c | 441 + third_party/dav1d/src/getbits.c | 178 + third_party/dav1d/src/getbits.h | 59 + third_party/dav1d/src/internal.h | 467 + third_party/dav1d/src/intra_edge.c | 165 + third_party/dav1d/src/intra_edge.h | 57 + third_party/dav1d/src/ipred.h | 94 + third_party/dav1d/src/ipred_prepare.h | 108 + third_party/dav1d/src/ipred_prepare_tmpl.c | 204 + third_party/dav1d/src/ipred_tmpl.c | 771 ++ third_party/dav1d/src/itx.h | 48 + third_party/dav1d/src/itx_1d.c | 1034 +++ third_party/dav1d/src/itx_1d.h | 59 + third_party/dav1d/src/itx_tmpl.c | 264 + third_party/dav1d/src/levels.h | 289 + third_party/dav1d/src/lf_apply.h | 48 + third_party/dav1d/src/lf_apply_tmpl.c | 466 + third_party/dav1d/src/lf_mask.c | 491 + third_party/dav1d/src/lf_mask.h | 83 + third_party/dav1d/src/lib.c | 802 ++ third_party/dav1d/src/log.c | 57 + third_party/dav1d/src/log.h | 47 + third_party/dav1d/src/loopfilter.h | 57 + third_party/dav1d/src/loopfilter_tmpl.c | 268 + third_party/dav1d/src/looprestoration.h | 79 + third_party/dav1d/src/looprestoration_tmpl.c | 554 ++ third_party/dav1d/src/lr_apply.h | 47 + third_party/dav1d/src/lr_apply_tmpl.c | 201 + third_party/dav1d/src/mc.h | 136 + third_party/dav1d/src/mc_tmpl.c | 953 ++ third_party/dav1d/src/mem.c | 119 + third_party/dav1d/src/mem.h | 103 + third_party/dav1d/src/meson.build | 348 + third_party/dav1d/src/msac.c | 208 + third_party/dav1d/src/msac.h | 108 + third_party/dav1d/src/obu.c | 1702 ++++ third_party/dav1d/src/obu.h | 36 + third_party/dav1d/src/picture.c | 367 + third_party/dav1d/src/picture.h | 115 + third_party/dav1d/src/ppc/cdef.h | 61 + third_party/dav1d/src/ppc/cdef_tmpl.c | 487 + third_party/dav1d/src/ppc/cpu.c | 51 + third_party/dav1d/src/ppc/cpu.h | 37 + third_party/dav1d/src/ppc/dav1d_types.h | 54 + third_party/dav1d/src/ppc/looprestoration.h | 48 + third_party/dav1d/src/ppc/looprestoration_tmpl.c | 321 + third_party/dav1d/src/qm.c | 3148 +++++++ third_party/dav1d/src/qm.h | 37 + third_party/dav1d/src/recon.h | 85 + third_party/dav1d/src/recon_tmpl.c | 2202 +++++ third_party/dav1d/src/ref.c | 107 + third_party/dav1d/src/ref.h | 60 + third_party/dav1d/src/refmvs.c | 940 ++ third_party/dav1d/src/refmvs.h | 176 + third_party/dav1d/src/scan.c | 299 + third_party/dav1d/src/scan.h | 37 + third_party/dav1d/src/tables.c | 1013 +++ third_party/dav1d/src/tables.h | 125 + third_party/dav1d/src/thread.h | 188 + third_party/dav1d/src/thread_data.h | 40 + third_party/dav1d/src/thread_task.c | 936 ++ third_party/dav1d/src/thread_task.h | 53 + third_party/dav1d/src/warpmv.c | 209 + third_party/dav1d/src/warpmv.h | 39 + third_party/dav1d/src/wedge.c | 342 + third_party/dav1d/src/wedge.h | 41 + third_party/dav1d/src/win32/thread.c | 99 + third_party/dav1d/src/x86/cdef.h | 87 + third_party/dav1d/src/x86/cdef16_avx2.asm | 877 ++ third_party/dav1d/src/x86/cdef16_avx512.asm | 622 ++ third_party/dav1d/src/x86/cdef16_sse.asm | 1033 +++ third_party/dav1d/src/x86/cdef_avx2.asm | 1772 ++++ third_party/dav1d/src/x86/cdef_avx512.asm | 860 ++ third_party/dav1d/src/x86/cdef_sse.asm | 1357 +++ third_party/dav1d/src/x86/cpu.c | 100 + third_party/dav1d/src/x86/cpu.h | 44 + third_party/dav1d/src/x86/cpuid.asm | 55 + third_party/dav1d/src/x86/filmgrain.h | 81 + third_party/dav1d/src/x86/filmgrain16_avx2.asm | 2248 +++++ third_party/dav1d/src/x86/filmgrain16_avx512.asm | 932 ++ third_party/dav1d/src/x86/filmgrain16_sse.asm | 3421 +++++++ third_party/dav1d/src/x86/filmgrain_avx2.asm | 2107 +++++ third_party/dav1d/src/x86/filmgrain_avx512.asm | 813 ++ third_party/dav1d/src/x86/filmgrain_common.asm | 46 + third_party/dav1d/src/x86/filmgrain_sse.asm | 3233 +++++++ third_party/dav1d/src/x86/ipred.h | 151 + third_party/dav1d/src/x86/ipred16_avx2.asm | 4992 ++++++++++ third_party/dav1d/src/x86/ipred16_avx512.asm | 833 ++ third_party/dav1d/src/x86/ipred16_sse.asm | 1923 ++++ third_party/dav1d/src/x86/ipred_avx2.asm | 5387 +++++++++++ third_party/dav1d/src/x86/ipred_avx512.asm | 1432 +++ third_party/dav1d/src/x86/ipred_sse.asm | 5409 +++++++++++ third_party/dav1d/src/x86/itx.h | 363 + third_party/dav1d/src/x86/itx16_avx2.asm | 8599 ++++++++++++++++++ third_party/dav1d/src/x86/itx16_avx512.asm | 4133 +++++++++ third_party/dav1d/src/x86/itx16_sse.asm | 8135 +++++++++++++++++ third_party/dav1d/src/x86/itx_avx2.asm | 5542 +++++++++++ third_party/dav1d/src/x86/itx_avx512.asm | 7389 +++++++++++++++ third_party/dav1d/src/x86/itx_sse.asm | 6533 +++++++++++++ third_party/dav1d/src/x86/loopfilter.h | 66 + third_party/dav1d/src/x86/loopfilter16_avx2.asm | 1161 +++ third_party/dav1d/src/x86/loopfilter16_avx512.asm | 912 ++ third_party/dav1d/src/x86/loopfilter16_sse.asm | 1793 ++++ third_party/dav1d/src/x86/loopfilter_avx2.asm | 1569 ++++ third_party/dav1d/src/x86/loopfilter_avx512.asm | 1534 ++++ third_party/dav1d/src/x86/loopfilter_sse.asm | 2348 +++++ third_party/dav1d/src/x86/looprestoration.h | 94 + .../dav1d/src/x86/looprestoration16_avx2.asm | 2540 ++++++ .../dav1d/src/x86/looprestoration16_avx512.asm | 2524 +++++ .../dav1d/src/x86/looprestoration16_sse.asm | 3723 ++++++++ third_party/dav1d/src/x86/looprestoration_avx2.asm | 2237 +++++ .../dav1d/src/x86/looprestoration_avx512.asm | 2122 +++++ third_party/dav1d/src/x86/looprestoration_sse.asm | 3681 ++++++++ third_party/dav1d/src/x86/mc.h | 299 + third_party/dav1d/src/x86/mc16_avx2.asm | 5879 ++++++++++++ third_party/dav1d/src/x86/mc16_avx512.asm | 4858 ++++++++++ third_party/dav1d/src/x86/mc16_sse.asm | 8731 ++++++++++++++++++ third_party/dav1d/src/x86/mc_avx2.asm | 5669 ++++++++++++ third_party/dav1d/src/x86/mc_avx512.asm | 4538 +++++++++ third_party/dav1d/src/x86/mc_sse.asm | 9599 ++++++++++++++++++++ third_party/dav1d/src/x86/msac.asm | 667 ++ third_party/dav1d/src/x86/msac.h | 75 + third_party/dav1d/src/x86/refmvs.asm | 688 ++ third_party/dav1d/src/x86/refmvs.h | 61 + 198 files changed, 249732 insertions(+) create mode 100644 third_party/dav1d/src/arm/32/cdef.S create mode 100644 third_party/dav1d/src/arm/32/cdef16.S create mode 100644 third_party/dav1d/src/arm/32/cdef_tmpl.S create mode 100644 third_party/dav1d/src/arm/32/filmgrain.S create mode 100644 third_party/dav1d/src/arm/32/filmgrain16.S create mode 100644 third_party/dav1d/src/arm/32/ipred.S create mode 100644 third_party/dav1d/src/arm/32/ipred16.S create mode 100644 third_party/dav1d/src/arm/32/itx.S create mode 100644 third_party/dav1d/src/arm/32/itx16.S create mode 100644 third_party/dav1d/src/arm/32/loopfilter.S create mode 100644 third_party/dav1d/src/arm/32/loopfilter16.S create mode 100644 third_party/dav1d/src/arm/32/looprestoration.S create mode 100644 third_party/dav1d/src/arm/32/looprestoration16.S create mode 100644 third_party/dav1d/src/arm/32/looprestoration_common.S create mode 100644 third_party/dav1d/src/arm/32/looprestoration_tmpl.S create mode 100644 third_party/dav1d/src/arm/32/mc.S create mode 100644 third_party/dav1d/src/arm/32/mc16.S create mode 100644 third_party/dav1d/src/arm/32/msac.S create mode 100644 third_party/dav1d/src/arm/32/refmvs.S create mode 100644 third_party/dav1d/src/arm/32/util.S create mode 100644 third_party/dav1d/src/arm/64/cdef.S create mode 100644 third_party/dav1d/src/arm/64/cdef16.S create mode 100644 third_party/dav1d/src/arm/64/cdef_tmpl.S create mode 100644 third_party/dav1d/src/arm/64/filmgrain.S create mode 100644 third_party/dav1d/src/arm/64/filmgrain16.S create mode 100644 third_party/dav1d/src/arm/64/ipred.S create mode 100644 third_party/dav1d/src/arm/64/ipred16.S create mode 100644 third_party/dav1d/src/arm/64/itx.S create mode 100644 third_party/dav1d/src/arm/64/itx16.S create mode 100644 third_party/dav1d/src/arm/64/loopfilter.S create mode 100644 third_party/dav1d/src/arm/64/loopfilter16.S create mode 100644 third_party/dav1d/src/arm/64/looprestoration.S create mode 100644 third_party/dav1d/src/arm/64/looprestoration16.S create mode 100644 third_party/dav1d/src/arm/64/looprestoration_common.S create mode 100644 third_party/dav1d/src/arm/64/looprestoration_tmpl.S create mode 100644 third_party/dav1d/src/arm/64/mc.S create mode 100644 third_party/dav1d/src/arm/64/mc16.S create mode 100644 third_party/dav1d/src/arm/64/msac.S create mode 100644 third_party/dav1d/src/arm/64/refmvs.S create mode 100644 third_party/dav1d/src/arm/64/util.S create mode 100644 third_party/dav1d/src/arm/asm-offsets.h create mode 100644 third_party/dav1d/src/arm/asm.S create mode 100644 third_party/dav1d/src/arm/cdef.h create mode 100644 third_party/dav1d/src/arm/cpu.c create mode 100644 third_party/dav1d/src/arm/cpu.h create mode 100644 third_party/dav1d/src/arm/filmgrain.h create mode 100644 third_party/dav1d/src/arm/ipred.h create mode 100644 third_party/dav1d/src/arm/itx.h create mode 100644 third_party/dav1d/src/arm/loopfilter.h create mode 100644 third_party/dav1d/src/arm/looprestoration.h create mode 100644 third_party/dav1d/src/arm/mc.h create mode 100644 third_party/dav1d/src/arm/msac.h create mode 100644 third_party/dav1d/src/arm/refmvs.h create mode 100644 third_party/dav1d/src/cdef.h create mode 100644 third_party/dav1d/src/cdef_apply.h create mode 100644 third_party/dav1d/src/cdef_apply_tmpl.c create mode 100644 third_party/dav1d/src/cdef_tmpl.c create mode 100644 third_party/dav1d/src/cdf.c create mode 100644 third_party/dav1d/src/cdf.h create mode 100644 third_party/dav1d/src/cpu.c create mode 100644 third_party/dav1d/src/cpu.h create mode 100644 third_party/dav1d/src/ctx.h create mode 100644 third_party/dav1d/src/data.c create mode 100644 third_party/dav1d/src/data.h create mode 100644 third_party/dav1d/src/dav1d.rc.in create mode 100644 third_party/dav1d/src/decode.c create mode 100644 third_party/dav1d/src/decode.h create mode 100644 third_party/dav1d/src/dequant_tables.c create mode 100644 third_party/dav1d/src/dequant_tables.h create mode 100644 third_party/dav1d/src/env.h create mode 100644 third_party/dav1d/src/ext/x86/x86inc.asm create mode 100644 third_party/dav1d/src/fg_apply.h create mode 100644 third_party/dav1d/src/fg_apply_tmpl.c create mode 100644 third_party/dav1d/src/filmgrain.h create mode 100644 third_party/dav1d/src/filmgrain_tmpl.c create mode 100644 third_party/dav1d/src/getbits.c create mode 100644 third_party/dav1d/src/getbits.h create mode 100644 third_party/dav1d/src/internal.h create mode 100644 third_party/dav1d/src/intra_edge.c create mode 100644 third_party/dav1d/src/intra_edge.h create mode 100644 third_party/dav1d/src/ipred.h create mode 100644 third_party/dav1d/src/ipred_prepare.h create mode 100644 third_party/dav1d/src/ipred_prepare_tmpl.c create mode 100644 third_party/dav1d/src/ipred_tmpl.c create mode 100644 third_party/dav1d/src/itx.h create mode 100644 third_party/dav1d/src/itx_1d.c create mode 100644 third_party/dav1d/src/itx_1d.h create mode 100644 third_party/dav1d/src/itx_tmpl.c create mode 100644 third_party/dav1d/src/levels.h create mode 100644 third_party/dav1d/src/lf_apply.h create mode 100644 third_party/dav1d/src/lf_apply_tmpl.c create mode 100644 third_party/dav1d/src/lf_mask.c create mode 100644 third_party/dav1d/src/lf_mask.h create mode 100644 third_party/dav1d/src/lib.c create mode 100644 third_party/dav1d/src/log.c create mode 100644 third_party/dav1d/src/log.h create mode 100644 third_party/dav1d/src/loopfilter.h create mode 100644 third_party/dav1d/src/loopfilter_tmpl.c create mode 100644 third_party/dav1d/src/looprestoration.h create mode 100644 third_party/dav1d/src/looprestoration_tmpl.c create mode 100644 third_party/dav1d/src/lr_apply.h create mode 100644 third_party/dav1d/src/lr_apply_tmpl.c create mode 100644 third_party/dav1d/src/mc.h create mode 100644 third_party/dav1d/src/mc_tmpl.c create mode 100644 third_party/dav1d/src/mem.c create mode 100644 third_party/dav1d/src/mem.h create mode 100644 third_party/dav1d/src/meson.build create mode 100644 third_party/dav1d/src/msac.c create mode 100644 third_party/dav1d/src/msac.h create mode 100644 third_party/dav1d/src/obu.c create mode 100644 third_party/dav1d/src/obu.h create mode 100644 third_party/dav1d/src/picture.c create mode 100644 third_party/dav1d/src/picture.h create mode 100644 third_party/dav1d/src/ppc/cdef.h create mode 100644 third_party/dav1d/src/ppc/cdef_tmpl.c create mode 100644 third_party/dav1d/src/ppc/cpu.c create mode 100644 third_party/dav1d/src/ppc/cpu.h create mode 100644 third_party/dav1d/src/ppc/dav1d_types.h create mode 100644 third_party/dav1d/src/ppc/looprestoration.h create mode 100644 third_party/dav1d/src/ppc/looprestoration_tmpl.c create mode 100644 third_party/dav1d/src/qm.c create mode 100644 third_party/dav1d/src/qm.h create mode 100644 third_party/dav1d/src/recon.h create mode 100644 third_party/dav1d/src/recon_tmpl.c create mode 100644 third_party/dav1d/src/ref.c create mode 100644 third_party/dav1d/src/ref.h create mode 100644 third_party/dav1d/src/refmvs.c create mode 100644 third_party/dav1d/src/refmvs.h create mode 100644 third_party/dav1d/src/scan.c create mode 100644 third_party/dav1d/src/scan.h create mode 100644 third_party/dav1d/src/tables.c create mode 100644 third_party/dav1d/src/tables.h create mode 100644 third_party/dav1d/src/thread.h create mode 100644 third_party/dav1d/src/thread_data.h create mode 100644 third_party/dav1d/src/thread_task.c create mode 100644 third_party/dav1d/src/thread_task.h create mode 100644 third_party/dav1d/src/warpmv.c create mode 100644 third_party/dav1d/src/warpmv.h create mode 100644 third_party/dav1d/src/wedge.c create mode 100644 third_party/dav1d/src/wedge.h create mode 100644 third_party/dav1d/src/win32/thread.c create mode 100644 third_party/dav1d/src/x86/cdef.h create mode 100644 third_party/dav1d/src/x86/cdef16_avx2.asm create mode 100644 third_party/dav1d/src/x86/cdef16_avx512.asm create mode 100644 third_party/dav1d/src/x86/cdef16_sse.asm create mode 100644 third_party/dav1d/src/x86/cdef_avx2.asm create mode 100644 third_party/dav1d/src/x86/cdef_avx512.asm create mode 100644 third_party/dav1d/src/x86/cdef_sse.asm create mode 100644 third_party/dav1d/src/x86/cpu.c create mode 100644 third_party/dav1d/src/x86/cpu.h create mode 100644 third_party/dav1d/src/x86/cpuid.asm create mode 100644 third_party/dav1d/src/x86/filmgrain.h create mode 100644 third_party/dav1d/src/x86/filmgrain16_avx2.asm create mode 100644 third_party/dav1d/src/x86/filmgrain16_avx512.asm create mode 100644 third_party/dav1d/src/x86/filmgrain16_sse.asm create mode 100644 third_party/dav1d/src/x86/filmgrain_avx2.asm create mode 100644 third_party/dav1d/src/x86/filmgrain_avx512.asm create mode 100644 third_party/dav1d/src/x86/filmgrain_common.asm create mode 100644 third_party/dav1d/src/x86/filmgrain_sse.asm create mode 100644 third_party/dav1d/src/x86/ipred.h create mode 100644 third_party/dav1d/src/x86/ipred16_avx2.asm create mode 100644 third_party/dav1d/src/x86/ipred16_avx512.asm create mode 100644 third_party/dav1d/src/x86/ipred16_sse.asm create mode 100644 third_party/dav1d/src/x86/ipred_avx2.asm create mode 100644 third_party/dav1d/src/x86/ipred_avx512.asm create mode 100644 third_party/dav1d/src/x86/ipred_sse.asm create mode 100644 third_party/dav1d/src/x86/itx.h create mode 100644 third_party/dav1d/src/x86/itx16_avx2.asm create mode 100644 third_party/dav1d/src/x86/itx16_avx512.asm create mode 100644 third_party/dav1d/src/x86/itx16_sse.asm create mode 100644 third_party/dav1d/src/x86/itx_avx2.asm create mode 100644 third_party/dav1d/src/x86/itx_avx512.asm create mode 100644 third_party/dav1d/src/x86/itx_sse.asm create mode 100644 third_party/dav1d/src/x86/loopfilter.h create mode 100644 third_party/dav1d/src/x86/loopfilter16_avx2.asm create mode 100644 third_party/dav1d/src/x86/loopfilter16_avx512.asm create mode 100644 third_party/dav1d/src/x86/loopfilter16_sse.asm create mode 100644 third_party/dav1d/src/x86/loopfilter_avx2.asm create mode 100644 third_party/dav1d/src/x86/loopfilter_avx512.asm create mode 100644 third_party/dav1d/src/x86/loopfilter_sse.asm create mode 100644 third_party/dav1d/src/x86/looprestoration.h create mode 100644 third_party/dav1d/src/x86/looprestoration16_avx2.asm create mode 100644 third_party/dav1d/src/x86/looprestoration16_avx512.asm create mode 100644 third_party/dav1d/src/x86/looprestoration16_sse.asm create mode 100644 third_party/dav1d/src/x86/looprestoration_avx2.asm create mode 100644 third_party/dav1d/src/x86/looprestoration_avx512.asm create mode 100644 third_party/dav1d/src/x86/looprestoration_sse.asm create mode 100644 third_party/dav1d/src/x86/mc.h create mode 100644 third_party/dav1d/src/x86/mc16_avx2.asm create mode 100644 third_party/dav1d/src/x86/mc16_avx512.asm create mode 100644 third_party/dav1d/src/x86/mc16_sse.asm create mode 100644 third_party/dav1d/src/x86/mc_avx2.asm create mode 100644 third_party/dav1d/src/x86/mc_avx512.asm create mode 100644 third_party/dav1d/src/x86/mc_sse.asm create mode 100644 third_party/dav1d/src/x86/msac.asm create mode 100644 third_party/dav1d/src/x86/msac.h create mode 100644 third_party/dav1d/src/x86/refmvs.asm create mode 100644 third_party/dav1d/src/x86/refmvs.h (limited to 'third_party/dav1d/src') diff --git a/third_party/dav1d/src/arm/32/cdef.S b/third_party/dav1d/src/arm/32/cdef.S new file mode 100644 index 0000000000..4a0df6eac8 --- /dev/null +++ b/third_party/dav1d/src/arm/32/cdef.S @@ -0,0 +1,540 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "cdef_tmpl.S" + +// n1 = s0/d0 +// w1 = d0/q0 +// n2 = s4/d2 +// w2 = d2/q1 +.macro pad_top_bottom s1, s2, w, stride, n1, w1, n2, w2, align, ret + tst r7, #1 // CDEF_HAVE_LEFT + beq 2f + // CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldrh r12, [\s1, #-2] + vldr \n1, [\s1] + vdup.16 d4, r12 + ldrh r12, [\s1, #\w] + vmov.16 d4[1], r12 + ldrh r12, [\s2, #-2] + vldr \n2, [\s2] + vmov.16 d4[2], r12 + ldrh r12, [\s2, #\w] + vmovl.u8 q0, d0 + vmov.16 d4[3], r12 + vmovl.u8 q1, d2 + vmovl.u8 q2, d4 + vstr s8, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s9, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s10, [r0, #-4] + vst1.16 {\w2}, [r0, :\align] + vstr s11, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldrh r12, [\s1, #-2] + vldr \n1, [\s1] + vdup.16 d4, r12 + ldrh r12, [\s2, #-2] + vldr \n2, [\s2] + vmovl.u8 q0, d0 + vmov.16 d4[1], r12 + vmovl.u8 q1, d2 + vmovl.u8 q2, d4 + vstr s8, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s9, [r0, #-4] + vst1.16 {\w2}, [r0, :\align] + vstr s12, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +2: + // !CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + vldr \n1, [\s1] + ldrh r12, [\s1, #\w] + vldr \n2, [\s2] + vdup.16 d4, r12 + ldrh r12, [\s2, #\w] + vmovl.u8 q0, d0 + vmov.16 d4[1], r12 + vmovl.u8 q1, d2 + vmovl.u8 q2, d4 + vstr s12, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s8, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s12, [r0, #-4] + vst1.16 {\w2}, [r0, :\align] + vstr s9, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vldr \n1, [\s1] + vldr \n2, [\s2] + vmovl.u8 q0, d0 + vmovl.u8 q1, d2 + vstr s12, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s12, [r0, #-4] + vst1.16 {\w2}, [r0, :\align] + vstr s12, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride +.endif +3: +.endm + +.macro load_n_incr dst, src, incr, w +.if \w == 4 + vld1.32 {\dst\()[0]}, [\src, :32], \incr +.else + vld1.8 {\dst\()}, [\src, :64], \incr +.endif +.endm + +// void dav1d_cdef_paddingX_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +// n1 = s0/d0 +// w1 = d0/q0 +// n2 = s4/d2 +// w2 = d2/q1 +.macro padding_func w, stride, n1, w1, n2, w2, align +function cdef_padding\w\()_8bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + cmp r7, #0xf // fully edged + beq cdef_padding\w\()_edged_8bpc_neon + vmov.i16 q3, #0x8000 + tst r7, #4 // CDEF_HAVE_TOP + bne 1f + // !CDEF_HAVE_TOP + sub r12, r0, #2*(2*\stride+2) + vmov.i16 q2, #0x8000 + vst1.16 {q2,q3}, [r12]! +.if \w == 8 + vst1.16 {q2,q3}, [r12]! +.endif + b 3f +1: + // CDEF_HAVE_TOP + add r8, r4, r2 + sub r0, r0, #2*(2*\stride) + pad_top_bottom r4, r8, \w, \stride, \n1, \w1, \n2, \w2, \align, 0 + + // Middle section +3: + tst r7, #1 // CDEF_HAVE_LEFT + beq 2f + // CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + vld1.16 {d2[]}, [r3, :16]! + ldrh r12, [r1, #\w] + load_n_incr d0, r1, r2, \w + subs r6, r6, #1 + vmov.16 d2[1], r12 + vmovl.u8 q0, d0 + vmovl.u8 q1, d2 + vstr s4, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s5, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 0b + b 3f +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vld1.16 {d2[]}, [r3, :16]! + load_n_incr d0, r1, r2, \w + subs r6, r6, #1 + vmovl.u8 q0, d0 + vmovl.u8 q1, d2 + vstr s4, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 1b + b 3f +2: + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ldrh r12, [r1, #\w] + load_n_incr d0, r1, r2, \w + vdup.16 d2, r12 + subs r6, r6, #1 + vmovl.u8 q0, d0 + vmovl.u8 q1, d2 + vstr s12, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s4, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 0b + b 3f +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + load_n_incr d0, r1, r2, \w + subs r6, r6, #1 + vmovl.u8 q0, d0 + vstr s12, [r0, #-4] + vst1.16 {\w1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 1b + +3: + tst r7, #8 // CDEF_HAVE_BOTTOM + bne 1f + // !CDEF_HAVE_BOTTOM + sub r12, r0, #4 + vmov.i16 q2, #0x8000 + vst1.16 {q2,q3}, [r12]! +.if \w == 8 + vst1.16 {q2,q3}, [r12]! +.endif + pop {r4-r8,pc} +1: + // CDEF_HAVE_BOTTOM + add r8, r5, r2 + pad_top_bottom r5, r8, \w, \stride, \n1, \w1, \n2, \w2, \align, 1 +endfunc +.endm + +padding_func 8, 16, d0, q0, d2, q1, 128 +padding_func 4, 8, s0, d0, s4, d2, 64 + +// void cdef_paddingX_edged_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_edged w, stride, reg, align +function cdef_padding\w\()_edged_8bpc_neon + sub r0, r0, #(2*\stride) + + ldrh r12, [r4, #-2] + vldr \reg, [r4] + add r8, r4, r2 + strh r12, [r0, #-2] + ldrh r12, [r4, #\w] + vstr \reg, [r0] + strh r12, [r0, #\w] + + ldrh r12, [r8, #-2] + vldr \reg, [r8] + strh r12, [r0, #\stride-2] + ldrh r12, [r8, #\w] + vstr \reg, [r0, #\stride] + strh r12, [r0, #\stride+\w] + add r0, r0, #2*\stride + +0: + ldrh r12, [r3], #2 + vldr \reg, [r1] + str r12, [r0, #-2] + ldrh r12, [r1, #\w] + add r1, r1, r2 + subs r6, r6, #1 + vstr \reg, [r0] + str r12, [r0, #\w] + add r0, r0, #\stride + bgt 0b + + ldrh r12, [r5, #-2] + vldr \reg, [r5] + add r8, r5, r2 + strh r12, [r0, #-2] + ldrh r12, [r5, #\w] + vstr \reg, [r0] + strh r12, [r0, #\w] + + ldrh r12, [r8, #-2] + vldr \reg, [r8] + strh r12, [r0, #\stride-2] + ldrh r12, [r8, #\w] + vstr \reg, [r0, #\stride] + strh r12, [r0, #\stride+\w] + + pop {r4-r8,pc} +endfunc +.endm + +padding_func_edged 8, 16, d0, 64 +padding_func_edged 4, 8, s0, 32 + +tables + +filter 8, 8 +filter 4, 8 + +find_dir 8 + +.macro load_px_8 d11, d12, d21, d22, w +.if \w == 8 + add r6, r2, r9 // x + off + sub r9, r2, r9 // x - off + vld1.8 {\d11}, [r6] // p0 + add r6, r6, #16 // += stride + vld1.8 {\d21}, [r9] // p1 + add r9, r9, #16 // += stride + vld1.8 {\d12}, [r6] // p0 + vld1.8 {\d22}, [r9] // p1 +.else + add r6, r2, r9 // x + off + sub r9, r2, r9 // x - off + vld1.32 {\d11[0]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d21[0]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d11[1]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d21[1]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d12[0]}, [r6] // p0 + add r6, r6, #8 // += stride + vld1.32 {\d22[0]}, [r9] // p1 + add r9, r9, #8 // += stride + vld1.32 {\d12[1]}, [r6] // p0 + vld1.32 {\d22[1]}, [r9] // p1 +.endif +.endm +.macro handle_pixel_8 s1, s2, thresh_vec, shift, tap, min +.if \min + vmin.u8 q3, q3, \s1 + vmax.u8 q4, q4, \s1 + vmin.u8 q3, q3, \s2 + vmax.u8 q4, q4, \s2 +.endif + vabd.u8 q8, q0, \s1 // abs(diff) + vabd.u8 q11, q0, \s2 // abs(diff) + vshl.u8 q9, q8, \shift // abs(diff) >> shift + vshl.u8 q12, q11, \shift // abs(diff) >> shift + vqsub.u8 q9, \thresh_vec, q9 // clip = imax(0, threshold - (abs(diff) >> shift)) + vqsub.u8 q12, \thresh_vec, q12// clip = imax(0, threshold - (abs(diff) >> shift)) + vcgt.u8 q10, q0, \s1 // px > p0 + vcgt.u8 q13, q0, \s2 // px > p1 + vmin.u8 q9, q9, q8 // imin(abs(diff), clip) + vmin.u8 q12, q12, q11 // imin(abs(diff), clip) + vneg.s8 q8, q9 // -imin() + vneg.s8 q11, q12 // -imin() + vbsl q10, q8, q9 // constrain() = imax(imin(diff, clip), -clip) + vdup.8 d18, \tap // taps[k] + vbsl q13, q11, q12 // constrain() = imax(imin(diff, clip), -clip) + vmlal.s8 q1, d20, d18 // sum += taps[k] * constrain() + vmlal.s8 q1, d26, d18 // sum += taps[k] * constrain() + vmlal.s8 q2, d21, d18 // sum += taps[k] * constrain() + vmlal.s8 q2, d27, d18 // sum += taps[k] * constrain() +.endm + +// void cdef_filterX_edged_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func_8 w, pri, sec, min, suffix +function cdef_filter\w\suffix\()_edged_neon +.if \pri + movrel_local r8, pri_taps + and r9, r3, #1 + add r8, r8, r9, lsl #1 +.endif + movrel_local r9, directions\w + add r5, r9, r5, lsl #1 + vmov.u8 d17, #7 + vdup.8 d16, r6 // damping + + vmov.8 d8[0], r3 + vmov.8 d8[1], r4 + vclz.i8 d8, d8 // clz(threshold) + vsub.i8 d8, d17, d8 // ulog2(threshold) + vqsub.u8 d8, d16, d8 // shift = imax(0, damping - ulog2(threshold)) + vneg.s8 d8, d8 // -shift +.if \sec + vdup.8 q6, d8[1] +.endif +.if \pri + vdup.8 q5, d8[0] +.endif + +1: +.if \w == 8 + add r12, r2, #16 + vld1.8 {d0}, [r2, :64] // px + vld1.8 {d1}, [r12, :64] // px +.else + add r12, r2, #8 + vld1.32 {d0[0]}, [r2, :32] // px + add r9, r2, #2*8 + vld1.32 {d0[1]}, [r12, :32] // px + add r12, r12, #2*8 + vld1.32 {d1[0]}, [r9, :32] // px + vld1.32 {d1[1]}, [r12, :32] // px +.endif + + vmov.u8 q1, #0 // sum + vmov.u8 q2, #0 // sum +.if \min + vmov.u16 q3, q0 // min + vmov.u16 q4, q0 // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov lr, #2 // sec_taps[0] + +2: +.if \pri + ldrsb r9, [r5] // off1 + + load_px_8 d28, d29, d30, d31, \w +.endif + +.if \sec + add r5, r5, #4 // +2*2 + ldrsb r9, [r5] // off2 +.endif + +.if \pri + ldrb r12, [r8] // *pri_taps + vdup.8 q7, r3 // threshold + + handle_pixel_8 q14, q15, q7, q5, r12, \min +.endif + +.if \sec + load_px_8 d28, d29, d30, d31, \w + + add r5, r5, #8 // +2*4 + ldrsb r9, [r5] // off3 + + vdup.8 q7, r4 // threshold + + handle_pixel_8 q14, q15, q7, q6, lr, \min + + load_px_8 d28, d29, d30, d31, \w + + handle_pixel_8 q14, q15, q7, q6, lr, \min + + sub r5, r5, #11 // r5 -= 2*(2+4); r5 += 1; +.else + add r5, r5, #1 // r5 += 1 +.endif + subs lr, lr, #1 // sec_tap-- (value) +.if \pri + add r8, r8, #1 // pri_taps++ (pointer) +.endif + bne 2b + + vshr.s16 q14, q1, #15 // -(sum < 0) + vshr.s16 q15, q2, #15 // -(sum < 0) + vadd.i16 q1, q1, q14 // sum - (sum < 0) + vadd.i16 q2, q2, q15 // sum - (sum < 0) + vrshr.s16 q1, q1, #4 // (8 + sum - (sum < 0)) >> 4 + vrshr.s16 q2, q2, #4 // (8 + sum - (sum < 0)) >> 4 + vaddw.u8 q1, q1, d0 // px + (8 + sum ...) >> 4 + vaddw.u8 q2, q2, d1 // px + (8 + sum ...) >> 4 + vqmovun.s16 d0, q1 + vqmovun.s16 d1, q2 +.if \min + vmin.u8 q0, q0, q4 + vmax.u8 q0, q0, q3 // iclip(px + .., min, max) +.endif +.if \w == 8 + vst1.8 {d0}, [r0, :64], r1 + add r2, r2, #2*16 // tmp += 2*tmp_stride + subs r7, r7, #2 // h -= 2 + vst1.8 {d1}, [r0, :64], r1 +.else + vst1.32 {d0[0]}, [r0, :32], r1 + add r2, r2, #4*8 // tmp += 4*tmp_stride + vst1.32 {d0[1]}, [r0, :32], r1 + subs r7, r7, #4 // h -= 4 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 +.endif + + // Reset pri_taps and directions back to the original point + sub r5, r5, #2 +.if \pri + sub r8, r8, #2 +.endif + + bgt 1b + vpop {q4-q7} + pop {r4-r9,pc} +endfunc +.endm + +.macro filter_8 w +filter_func_8 \w, pri=1, sec=0, min=0, suffix=_pri +filter_func_8 \w, pri=0, sec=1, min=0, suffix=_sec +filter_func_8 \w, pri=1, sec=1, min=1, suffix=_pri_sec +.endm + +filter_8 8 +filter_8 4 diff --git a/third_party/dav1d/src/arm/32/cdef16.S b/third_party/dav1d/src/arm/32/cdef16.S new file mode 100644 index 0000000000..d14525d720 --- /dev/null +++ b/third_party/dav1d/src/arm/32/cdef16.S @@ -0,0 +1,233 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "cdef_tmpl.S" + +// r1 = d0/q0 +// r2 = d2/q1 +.macro pad_top_bot_16 s1, s2, w, stride, r1, r2, align, ret + tst r7, #1 // CDEF_HAVE_LEFT + beq 2f + // CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + vldr s8, [\s1, #-4] + vld1.16 {\r1}, [\s1, :\align] + vldr s9, [\s1, #2*\w] + vldr s10, [\s2, #-4] + vld1.16 {\r2}, [\s2, :\align] + vldr s11, [\s2, #2*\w] + vstr s8, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s9, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s10, [r0, #-4] + vst1.16 {\r2}, [r0, :\align] + vstr s11, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vldr s8, [\s1, #-4] + vld1.16 {\r1}, [\s1, :\align] + vldr s9, [\s2, #-4] + vld1.16 {\r2}, [\s2, :\align] + vstr s8, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s9, [r0, #-4] + vst1.16 {\r2}, [r0, :\align] + vstr s12, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +2: + // !CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + vld1.16 {\r1}, [\s1, :\align] + vldr s8, [\s1, #2*\w] + vld1.16 {\r2}, [\s2, :\align] + vldr s9, [\s2, #2*\w] + vstr s12, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s8, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s12, [r0, #-4] + vst1.16 {\r2}, [r0, :\align] + vstr s9, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride + b 3f +.endif + +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vld1.16 {\r1}, [\s1, :\align] + vld1.16 {\r2}, [\s2, :\align] + vstr s12, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + vstr s12, [r0, #-4] + vst1.16 {\r2}, [r0, :\align] + vstr s12, [r0, #2*\w] +.if \ret + pop {r4-r8,pc} +.else + add r0, r0, #2*\stride +.endif +3: +.endm + +// void dav1d_cdef_paddingX_16bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +// r1 = d0/q0 +// r2 = d2/q1 +.macro padding_func_16 w, stride, r1, r2, align +function cdef_padding\w\()_16bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + vmov.i16 q3, #0x8000 + tst r7, #4 // CDEF_HAVE_TOP + bne 1f + // !CDEF_HAVE_TOP + sub r12, r0, #2*(2*\stride+2) + vmov.i16 q2, #0x8000 + vst1.16 {q2,q3}, [r12]! +.if \w == 8 + vst1.16 {q2,q3}, [r12]! +.endif + b 3f +1: + // CDEF_HAVE_TOP + add r8, r4, r2 + sub r0, r0, #2*(2*\stride) + pad_top_bot_16 r4, r8, \w, \stride, \r1, \r2, \align, 0 + + // Middle section +3: + tst r7, #1 // CDEF_HAVE_LEFT + beq 2f + // CDEF_HAVE_LEFT + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + vld1.32 {d2[]}, [r3, :32]! + vldr s5, [r1, #2*\w] + vld1.16 {\r1}, [r1, :\align], r2 + subs r6, r6, #1 + vstr s4, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s5, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 0b + b 3f +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vld1.32 {d2[]}, [r3, :32]! + vld1.16 {\r1}, [r1, :\align], r2 + subs r6, r6, #1 + vstr s4, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 1b + b 3f +2: + tst r7, #2 // CDEF_HAVE_RIGHT + beq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + vldr s4, [r1, #2*\w] + vld1.16 {\r1}, [r1, :\align], r2 + subs r6, r6, #1 + vstr s12, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s4, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 0b + b 3f +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + vld1.16 {\r1}, [r1, :\align], r2 + subs r6, r6, #1 + vstr s12, [r0, #-4] + vst1.16 {\r1}, [r0, :\align] + vstr s12, [r0, #2*\w] + add r0, r0, #2*\stride + bgt 1b + +3: + tst r7, #8 // CDEF_HAVE_BOTTOM + bne 1f + // !CDEF_HAVE_BOTTOM + sub r12, r0, #4 + vmov.i16 q2, #0x8000 + vst1.16 {q2,q3}, [r12]! +.if \w == 8 + vst1.16 {q2,q3}, [r12]! +.endif + pop {r4-r8,pc} +1: + // CDEF_HAVE_BOTTOM + add r8, r5, r2 + pad_top_bot_16 r5, r8, \w, \stride, \r1, \r2, \align, 1 +endfunc +.endm + +padding_func_16 8, 16, q0, q1, 128 +padding_func_16 4, 8, d0, d2, 64 + +tables + +filter 8, 16 +filter 4, 16 + +find_dir 16 diff --git a/third_party/dav1d/src/arm/32/cdef_tmpl.S b/third_party/dav1d/src/arm/32/cdef_tmpl.S new file mode 100644 index 0000000000..33ff9e5816 --- /dev/null +++ b/third_party/dav1d/src/arm/32/cdef_tmpl.S @@ -0,0 +1,515 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro dir_table w, stride +const directions\w + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 + .byte 1 * \stride + 0, 2 * \stride + 0 + .byte 1 * \stride + 0, 2 * \stride - 1 +// Repeated, to avoid & 7 + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 +endconst +.endm + +.macro tables +dir_table 8, 16 +dir_table 4, 8 + +const pri_taps + .byte 4, 2, 3, 3 +endconst +.endm + +.macro load_px d11, d12, d21, d22, w +.if \w == 8 + add r6, r2, r9, lsl #1 // x + off + sub r9, r2, r9, lsl #1 // x - off + vld1.16 {\d11,\d12}, [r6] // p0 + vld1.16 {\d21,\d22}, [r9] // p1 +.else + add r6, r2, r9, lsl #1 // x + off + sub r9, r2, r9, lsl #1 // x - off + vld1.16 {\d11}, [r6] // p0 + add r6, r6, #2*8 // += stride + vld1.16 {\d21}, [r9] // p1 + add r9, r9, #2*8 // += stride + vld1.16 {\d12}, [r6] // p0 + vld1.16 {\d22}, [r9] // p1 +.endif +.endm +.macro handle_pixel s1, s2, thresh_vec, shift, tap, min +.if \min + vmin.u16 q2, q2, \s1 + vmax.s16 q3, q3, \s1 + vmin.u16 q2, q2, \s2 + vmax.s16 q3, q3, \s2 +.endif + vabd.u16 q8, q0, \s1 // abs(diff) + vabd.u16 q11, q0, \s2 // abs(diff) + vshl.u16 q9, q8, \shift // abs(diff) >> shift + vshl.u16 q12, q11, \shift // abs(diff) >> shift + vqsub.u16 q9, \thresh_vec, q9 // clip = imax(0, threshold - (abs(diff) >> shift)) + vqsub.u16 q12, \thresh_vec, q12// clip = imax(0, threshold - (abs(diff) >> shift)) + vsub.i16 q10, \s1, q0 // diff = p0 - px + vsub.i16 q13, \s2, q0 // diff = p1 - px + vneg.s16 q8, q9 // -clip + vneg.s16 q11, q12 // -clip + vmin.s16 q10, q10, q9 // imin(diff, clip) + vmin.s16 q13, q13, q12 // imin(diff, clip) + vdup.16 q9, \tap // taps[k] + vmax.s16 q10, q10, q8 // constrain() = imax(imin(diff, clip), -clip) + vmax.s16 q13, q13, q11 // constrain() = imax(imin(diff, clip), -clip) + vmla.i16 q1, q10, q9 // sum += taps[k] * constrain() + vmla.i16 q1, q13, q9 // sum += taps[k] * constrain() +.endm + +// void dav1d_cdef_filterX_Ybpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func w, bpc, pri, sec, min, suffix +function cdef_filter\w\suffix\()_\bpc\()bpc_neon +.if \bpc == 8 + cmp r8, #0xf + beq cdef_filter\w\suffix\()_edged_neon +.endif +.if \pri +.if \bpc == 16 + clz r9, r9 + sub r9, r9, #24 // -bitdepth_min_8 + neg r9, r9 // bitdepth_min_8 +.endif + movrel_local r8, pri_taps +.if \bpc == 16 + lsr r9, r3, r9 // pri_strength >> bitdepth_min_8 + and r9, r9, #1 // (pri_strength >> bitdepth_min_8) & 1 +.else + and r9, r3, #1 +.endif + add r8, r8, r9, lsl #1 +.endif + movrel_local r9, directions\w + add r5, r9, r5, lsl #1 + vmov.u16 d17, #15 + vdup.16 d16, r6 // damping + +.if \pri + vdup.16 q5, r3 // threshold +.endif +.if \sec + vdup.16 q7, r4 // threshold +.endif + vmov.16 d8[0], r3 + vmov.16 d8[1], r4 + vclz.i16 d8, d8 // clz(threshold) + vsub.i16 d8, d17, d8 // ulog2(threshold) + vqsub.u16 d8, d16, d8 // shift = imax(0, damping - ulog2(threshold)) + vneg.s16 d8, d8 // -shift +.if \sec + vdup.16 q6, d8[1] +.endif +.if \pri + vdup.16 q4, d8[0] +.endif + +1: +.if \w == 8 + vld1.16 {q0}, [r2, :128] // px +.else + add r12, r2, #2*8 + vld1.16 {d0}, [r2, :64] // px + vld1.16 {d1}, [r12, :64] // px +.endif + + vmov.u16 q1, #0 // sum +.if \min + vmov.u16 q2, q0 // min + vmov.u16 q3, q0 // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov lr, #2 // sec_taps[0] + +2: +.if \pri + ldrsb r9, [r5] // off1 + + load_px d28, d29, d30, d31, \w +.endif + +.if \sec + add r5, r5, #4 // +2*2 + ldrsb r9, [r5] // off2 +.endif + +.if \pri + ldrb r12, [r8] // *pri_taps + + handle_pixel q14, q15, q5, q4, r12, \min +.endif + +.if \sec + load_px d28, d29, d30, d31, \w + + add r5, r5, #8 // +2*4 + ldrsb r9, [r5] // off3 + + handle_pixel q14, q15, q7, q6, lr, \min + + load_px d28, d29, d30, d31, \w + + handle_pixel q14, q15, q7, q6, lr, \min + + sub r5, r5, #11 // r5 -= 2*(2+4); r5 += 1; +.else + add r5, r5, #1 // r5 += 1 +.endif + subs lr, lr, #1 // sec_tap-- (value) +.if \pri + add r8, r8, #1 // pri_taps++ (pointer) +.endif + bne 2b + + vshr.s16 q14, q1, #15 // -(sum < 0) + vadd.i16 q1, q1, q14 // sum - (sum < 0) + vrshr.s16 q1, q1, #4 // (8 + sum - (sum < 0)) >> 4 + vadd.i16 q0, q0, q1 // px + (8 + sum ...) >> 4 +.if \min + vmin.s16 q0, q0, q3 + vmax.s16 q0, q0, q2 // iclip(px + .., min, max) +.endif +.if \bpc == 8 + vmovn.u16 d0, q0 +.endif +.if \w == 8 + add r2, r2, #2*16 // tmp += tmp_stride + subs r7, r7, #1 // h-- +.if \bpc == 8 + vst1.8 {d0}, [r0, :64], r1 +.else + vst1.16 {q0}, [r0, :128], r1 +.endif +.else +.if \bpc == 8 + vst1.32 {d0[0]}, [r0, :32], r1 +.else + vst1.16 {d0}, [r0, :64], r1 +.endif + add r2, r2, #2*16 // tmp += 2*tmp_stride + subs r7, r7, #2 // h -= 2 +.if \bpc == 8 + vst1.32 {d0[1]}, [r0, :32], r1 +.else + vst1.16 {d1}, [r0, :64], r1 +.endif +.endif + + // Reset pri_taps and directions back to the original point + sub r5, r5, #2 +.if \pri + sub r8, r8, #2 +.endif + + bgt 1b + vpop {q4-q7} + pop {r4-r9,pc} +endfunc +.endm + +.macro filter w, bpc +filter_func \w, \bpc, pri=1, sec=0, min=0, suffix=_pri +filter_func \w, \bpc, pri=0, sec=1, min=0, suffix=_sec +filter_func \w, \bpc, pri=1, sec=1, min=1, suffix=_pri_sec + +function cdef_filter\w\()_\bpc\()bpc_neon, export=1 + push {r4-r9,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #92] + ldrd r6, r7, [sp, #100] +.if \bpc == 16 + ldrd r8, r9, [sp, #108] +.else + ldr r8, [sp, #108] +.endif + cmp r3, #0 // pri_strength + bne 1f + b cdef_filter\w\()_sec_\bpc\()bpc_neon // only sec +1: + cmp r4, #0 // sec_strength + bne 1f + b cdef_filter\w\()_pri_\bpc\()bpc_neon // only pri +1: + b cdef_filter\w\()_pri_sec_\bpc\()bpc_neon // both pri and sec +endfunc +.endm + +const div_table, align=4 + .short 840, 420, 280, 210, 168, 140, 120, 105 +endconst + +const alt_fact, align=4 + .short 420, 210, 140, 105, 105, 105, 105, 105, 140, 210, 420, 0 +endconst + +.macro cost_alt dest, s1, s2, s3, s4, s5, s6 + vmull.s16 q1, \s1, \s1 // sum_alt[n]*sum_alt[n] + vmull.s16 q2, \s2, \s2 + vmull.s16 q3, \s3, \s3 + vmull.s16 q5, \s4, \s4 // sum_alt[n]*sum_alt[n] + vmull.s16 q12, \s5, \s5 + vmull.s16 q6, \s6, \s6 // q6 overlaps the first \s1-\s2 here + vmul.i32 q1, q1, q13 // sum_alt[n]^2*fact + vmla.i32 q1, q2, q14 + vmla.i32 q1, q3, q15 + vmul.i32 q5, q5, q13 // sum_alt[n]^2*fact + vmla.i32 q5, q12, q14 + vmla.i32 q5, q6, q15 + vadd.i32 d2, d2, d3 + vadd.i32 d3, d10, d11 + vpadd.i32 \dest, d2, d3 // *cost_ptr +.endm + +.macro find_best s1, s2, s3 +.ifnb \s2 + vmov.32 lr, \s2 +.endif + cmp r12, r1 // cost[n] > best_cost + itt gt + movgt r0, r3 // best_dir = n + movgt r1, r12 // best_cost = cost[n] +.ifnb \s2 + add r3, r3, #1 // n++ + cmp lr, r1 // cost[n] > best_cost + vmov.32 r12, \s3 + itt gt + movgt r0, r3 // best_dir = n + movgt r1, lr // best_cost = cost[n] + add r3, r3, #1 // n++ +.endif +.endm + +// int dav1d_cdef_find_dir_Xbpc_neon(const pixel *img, const ptrdiff_t stride, +// unsigned *const var) +.macro find_dir bpc +function cdef_find_dir_\bpc\()bpc_neon, export=1 + push {lr} + vpush {q4-q7} +.if \bpc == 16 + clz r3, r3 // clz(bitdepth_max) + sub lr, r3, #24 // -bitdepth_min_8 +.endif + sub sp, sp, #32 // cost + mov r3, #8 + vmov.u16 q1, #0 // q0-q1 sum_diag[0] + vmov.u16 q3, #0 // q2-q3 sum_diag[1] + vmov.u16 q5, #0 // q4-q5 sum_hv[0-1] + vmov.u16 q8, #0 // q6,d16 sum_alt[0] + // q7,d17 sum_alt[1] + vmov.u16 q9, #0 // q9,d22 sum_alt[2] + vmov.u16 q11, #0 + vmov.u16 q10, #0 // q10,d23 sum_alt[3] + + +.irpc i, 01234567 +.if \bpc == 8 + vld1.8 {d30}, [r0, :64], r1 + vmov.u8 d31, #128 + vsubl.u8 q15, d30, d31 // img[x] - 128 +.else + vld1.16 {q15}, [r0, :128], r1 + vdup.16 q14, lr // -bitdepth_min_8 + vshl.u16 q15, q15, q14 + vmov.u16 q14, #128 + vsub.i16 q15, q15, q14 // img[x] - 128 +.endif + vmov.u16 q14, #0 + +.if \i == 0 + vmov q0, q15 // sum_diag[0] +.else + vext.8 q12, q14, q15, #(16-2*\i) + vext.8 q13, q15, q14, #(16-2*\i) + vadd.i16 q0, q0, q12 // sum_diag[0] + vadd.i16 q1, q1, q13 // sum_diag[0] +.endif + vrev64.16 q13, q15 + vswp d26, d27 // [-x] +.if \i == 0 + vmov q2, q13 // sum_diag[1] +.else + vext.8 q12, q14, q13, #(16-2*\i) + vext.8 q13, q13, q14, #(16-2*\i) + vadd.i16 q2, q2, q12 // sum_diag[1] + vadd.i16 q3, q3, q13 // sum_diag[1] +.endif + + vpadd.u16 d26, d30, d31 // [(x >> 1)] + vmov.u16 d27, #0 + vpadd.u16 d24, d26, d28 + vpadd.u16 d24, d24, d28 // [y] + vmov.u16 r12, d24[0] + vadd.i16 q5, q5, q15 // sum_hv[1] +.if \i < 4 + vmov.16 d8[\i], r12 // sum_hv[0] +.else + vmov.16 d9[\i-4], r12 // sum_hv[0] +.endif + +.if \i == 0 + vmov.u16 q6, q13 // sum_alt[0] +.else + vext.8 q12, q14, q13, #(16-2*\i) + vext.8 q14, q13, q14, #(16-2*\i) + vadd.i16 q6, q6, q12 // sum_alt[0] + vadd.i16 d16, d16, d28 // sum_alt[0] +.endif + vrev64.16 d26, d26 // [-(x >> 1)] + vmov.u16 q14, #0 +.if \i == 0 + vmov q7, q13 // sum_alt[1] +.else + vext.8 q12, q14, q13, #(16-2*\i) + vext.8 q13, q13, q14, #(16-2*\i) + vadd.i16 q7, q7, q12 // sum_alt[1] + vadd.i16 d17, d17, d26 // sum_alt[1] +.endif + +.if \i < 6 + vext.8 q12, q14, q15, #(16-2*(3-(\i/2))) + vext.8 q13, q15, q14, #(16-2*(3-(\i/2))) + vadd.i16 q9, q9, q12 // sum_alt[2] + vadd.i16 d22, d22, d26 // sum_alt[2] +.else + vadd.i16 q9, q9, q15 // sum_alt[2] +.endif +.if \i == 0 + vmov q10, q15 // sum_alt[3] +.elseif \i == 1 + vadd.i16 q10, q10, q15 // sum_alt[3] +.else + vext.8 q12, q14, q15, #(16-2*(\i/2)) + vext.8 q13, q15, q14, #(16-2*(\i/2)) + vadd.i16 q10, q10, q12 // sum_alt[3] + vadd.i16 d23, d23, d26 // sum_alt[3] +.endif +.endr + + vmov.u32 q15, #105 + + vmull.s16 q12, d8, d8 // sum_hv[0]*sum_hv[0] + vmlal.s16 q12, d9, d9 + vmull.s16 q13, d10, d10 // sum_hv[1]*sum_hv[1] + vmlal.s16 q13, d11, d11 + vadd.s32 d8, d24, d25 + vadd.s32 d9, d26, d27 + vpadd.s32 d8, d8, d9 // cost[2,6] (s16, s17) + vmul.i32 d8, d8, d30 // cost[2,6] *= 105 + + vrev64.16 q1, q1 + vrev64.16 q3, q3 + vext.8 q1, q1, q1, #10 // sum_diag[0][14-n] + vext.8 q3, q3, q3, #10 // sum_diag[1][14-n] + + vstr s16, [sp, #2*4] // cost[2] + vstr s17, [sp, #6*4] // cost[6] + + movrel_local r12, div_table + vld1.16 {q14}, [r12, :128] + + vmull.s16 q5, d0, d0 // sum_diag[0]*sum_diag[0] + vmull.s16 q12, d1, d1 + vmlal.s16 q5, d2, d2 + vmlal.s16 q12, d3, d3 + vmull.s16 q0, d4, d4 // sum_diag[1]*sum_diag[1] + vmull.s16 q1, d5, d5 + vmlal.s16 q0, d6, d6 + vmlal.s16 q1, d7, d7 + vmovl.u16 q13, d28 // div_table + vmovl.u16 q14, d29 + vmul.i32 q5, q5, q13 // cost[0] + vmla.i32 q5, q12, q14 + vmul.i32 q0, q0, q13 // cost[4] + vmla.i32 q0, q1, q14 + vadd.i32 d10, d10, d11 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d10, d0 // cost[0,4] = s0,s1 + + movrel_local r12, alt_fact + vld1.16 {d29, d30, d31}, [r12, :64] // div_table[2*m+1] + 105 + + vstr s0, [sp, #0*4] // cost[0] + vstr s1, [sp, #4*4] // cost[4] + + vmovl.u16 q13, d29 // div_table[2*m+1] + 105 + vmovl.u16 q14, d30 + vmovl.u16 q15, d31 + + cost_alt d14, d12, d13, d16, d14, d15, d17 // cost[1], cost[3] + cost_alt d15, d18, d19, d22, d20, d21, d23 // cost[5], cost[7] + vstr s28, [sp, #1*4] // cost[1] + vstr s29, [sp, #3*4] // cost[3] + + mov r0, #0 // best_dir + vmov.32 r1, d0[0] // best_cost + mov r3, #1 // n + + vstr s30, [sp, #5*4] // cost[5] + vstr s31, [sp, #7*4] // cost[7] + + vmov.32 r12, d14[0] + + find_best d14[0], d8[0], d14[1] + find_best d14[1], d0[1], d15[0] + find_best d15[0], d8[1], d15[1] + find_best d15[1] + + eor r3, r0, #4 // best_dir ^4 + ldr r12, [sp, r3, lsl #2] + sub r1, r1, r12 // best_cost - cost[best_dir ^ 4] + lsr r1, r1, #10 + str r1, [r2] // *var + + add sp, sp, #32 + vpop {q4-q7} + pop {pc} +endfunc +.endm diff --git a/third_party/dav1d/src/arm/32/filmgrain.S b/third_party/dav1d/src/arm/32/filmgrain.S new file mode 100644 index 0000000000..d1f83efb98 --- /dev/null +++ b/third_party/dav1d/src/arm/32/filmgrain.S @@ -0,0 +1,2039 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "src/arm/asm-offsets.h" + +#define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr r11, r2, #3 + lsr r12, r2, #12 + lsr lr, r2, #1 + eor r11, r2, r11 // (r >> 0) ^ (r >> 3) + eor r12, r12, lr // (r >> 12) ^ (r >> 1) + eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr r2, r2, #\steps +.endif + and r11, r11, #((1 << \steps) - 1) // bit +.if \shift + orr r2, r2, r11, lsl #(16 - \steps) // *state +.else + orr r2, r2, r11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, r2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, r2, #17 - \bits, #\bits + lsr r2, r2, #1 +.endm + +// special calling convention: +// r2 holds seed +// r3 holds dav1d_gaussian_sequence +// clobbers r11-r12 +// returns in d0-d1 +function get_gaussian_neon + push {r5-r6,lr} + increment_seed 4 + read_rand r5, 11, 3 + read_rand r6, 11, 2 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d0[0]}, [r5] + read_rand r5, 11, 1 + vld1.16 {d0[1]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 0 + increment_seed 4 + add r6, r3, r6, lsl #1 + vld1.16 {d0[2]}, [r5] + read_rand r5, 11, 3 + vld1.16 {d0[3]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 2 + vld1.16 {d1[0]}, [r5] + add r6, r3, r6, lsl #1 + read_rand r5, 11, 1 + vld1.16 {d1[1]}, [r6] + read_rand r6, 11, 0 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d1[2]}, [r5] + vld1.16 {d1[3]}, [r6] + pop {r5-r6,pc} +endfunc + +.macro get_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r0, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r1, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r2, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r3, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r4, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r5, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r6, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r7, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r8, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r9, q0 + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 \r10, q0 +.endm + +.macro store_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 + vst1.16 {\r0, \r1, \r2, \r3}, [r0]! + vst1.16 {\r4, \r5, \r6, \r7}, [r0]! + vst1.16 {\r8, \r9}, [r0]! + vst1.16 {\r10[0]}, [r0]! +.endm + +.macro get_grain_row_44 r0, r1, r2, r3, r4, r5 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r0, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r1, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r2, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r3, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r4, q0 + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d0[1]}, [r12] + add r11, r3, r11, lsl #1 + read_rand r12, 11, 0 + vld1.16 {d0[2]}, [r11] + add r12, r3, r12, lsl #1 + vld1.16 {d0[3]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 \r5, q0 +.endm + +.macro store_grain_row_44 r0, r1, r2, r3, r4, r5 + vst1.16 {\r0, \r1, \r2, \r3}, [r0]! + vst1.16 {\r4, \r5}, [r0] + add r0, r0, #GRAIN_WIDTH-32 +.endm + +function get_grain_2_neon + push {r11,lr} + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 d0, q0 + pop {r11,pc} +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +// r1 holds the number of entries to produce +// r6, r8 and r10 hold the previous output entries +// q0 holds the vector of produced entries +// q1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon + push {r0, lr} +.if \n == 1 + mov lr, #-128 +.else + mov r0, #1 + mov lr, #1 + sub r7, r7, #1 + sub r9, r9, #1 + lsl r0, r0, r7 + lsl lr, lr, r9 + add r7, r7, #1 + add r9, r9, #1 +.endif +1: + read_shift_rand r12, 11 + vmov.32 r11, d2[0] + lsl r12, r12, #1 + vext.8 q0, q0, q0, #1 + ldrsh r12, [r3, r12] +.if \n == 1 + mla r11, r6, r4, r11 // sum (above) + *coeff * prev output + add r6, r11, r8 // 1 << (ar_coeff_shift - 1) + add r12, r12, r10 + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + cmp r6, r5 +.elseif \n == 2 + mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 + mla r11, r6, r10, r11 // += *coeff * prev output 2 + mov r8, r6 + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mov lr, #-128 +.else + push {r1-r3} + sbfx r1, r4, #0, #8 + sbfx r2, r4, #8, #8 + sbfx r3, r4, #16, #8 + mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 + mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 + mla r11, r6, r3, r11 // += *coeff * prev output 3 + pop {r1-r3} + mov r10, r8 + mov r8, r6 + + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mov lr, #-128 +.endif + it gt + movgt r6, r5 + cmp r6, lr + it lt + movlt r6, lr +.if \n >= 2 + pop {lr} +.endif + subs r1, r1, #1 + vext.8 q1, q1, q1, #4 + vmov.8 d1[7], r6 + bgt 1b + pop {r0, pc} +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + vmull.s8 q2, d6, d28 + vmull.s8 q3, d7, d28 + vmull.s8 q4, d0, d27 + vmull.s8 q5, d1, d27 + + vaddl.s16 q0, d4, d8 + vaddl.s16 q2, d5, d9 + vaddl.s16 q4, d6, d10 + vaddl.s16 q5, d7, d11 + + vmull.s8 q3, d3, d29 + vmull.s8 q1, d2, d29 + + vaddw.s16 q4, q4, d6 + vaddw.s16 q5, q5, d7 + vaddw.s16 q3, q2, d3 + vaddw.s16 q2, q0, d2 + bx lr +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff +.ifc \lag\()_\edge, lag3_left + bl sum_lag3_left_above_neon +.else + bl sum_\lag\()_above_neon +.endif +.ifc \type, uv_420 + vpush {q6-q7} + add r12, r11, #GRAIN_WIDTH + vld1.16 {q0, q1}, [r11]! + vld1.16 {q6, q7}, [r12]! + vpaddl.s8 q0, q0 + vpaddl.s8 q1, q1 + vpaddl.s8 q6, q6 + vpaddl.s8 q7, q7 + vadd.i16 q0, q0, q6 + vadd.i16 q1, q1, q7 + vpop {q6-q7} + vrshrn.s16 d0, q0, #2 + vrshrn.s16 d1, q1, #2 +.endif +.ifc \type, uv_422 + vld1.8 {q0, q1}, [r11]! + vpaddl.s8 q0, q0 + vpaddl.s8 q1, q1 + vrshrn.s16 d0, q0, #1 + vrshrn.s16 d1, q1, #1 +.endif +.ifc \type, uv_444 + vld1.8 {q0}, [r11]! +.endif +.if \uv_layout +.ifnb \uv_coeff + vdup.8 d13, \uv_coeff +.endif + vmull.s8 q1, d0, d13 + vmull.s8 q0, d1, d13 + vaddw.s16 q2, q2, d2 + vaddw.s16 q3, q3, d3 + vaddw.s16 q4, q4, d0 + vaddw.s16 q5, q5, d1 +.endif +.if \uv_layout && \elems == 16 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 15 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 9 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: + push {r11} +.ifc \edge, left + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d1[1]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d1[2]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d1[3]}, [r11] + lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 + vrshl.s16 d1, d1, d30 + vmovn.i16 d1, q0 + vext.8 q2, q2, q2, #12 +.ifc \lag, lag3 + vmov.s8 r10, d1[5] +.endif +.ifnc \lag, lag1 + vmov.s8 r8, d1[6] +.endif + vmov.s8 r6, d1[7] + + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + vmov q1, q3 + mov r1, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + vmov q1, q4 +.if \elems == 9 + mov r1, #1 + bl output_\lag\()_neon + lsr r2, r2, #3 + + read_rand r11, 11, 2 + read_rand r12, 11, 1 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r11] + read_rand r11, 11, 0 + vld1.16 {d2[1]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d2[2]}, [r11] + vrshl.s16 d2, d2, d30 + vmovn.i16 d2, q1 + vext.8 q0, q0, q1, #7 +.else + mov r1, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + vmov q1, q5 + +.ifc \edge, right + mov r1, #3 + bl output_\lag\()_neon + read_shift_rand r11, 11 + add r11, r3, r11, lsl #1 + vld1.16 {d2[0]}, [r11] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #1 +.else + mov r1, #4 + bl output_\lag\()_neon +.endif +.endif +.if \store + vst1.8 {q0}, [r0]! +.endif + pop {r11} + pop {r1, pc} +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag1_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 15 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 15 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 9 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 9 + +.macro sum_lag1 type, dst, left, mid, right, edge=mid + vmov q3, \mid + vext.8 q0, \left, \mid, #15 + vext.8 q1, \mid, \right, #1 + bl sum_\type\()_lag1_\edge\()_neon + vmov \dst, q0 +.endm + +.macro sum_y_lag1 dst, left, mid, right, edge=mid + sum_lag1 y, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_444, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_422, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_420, \dst, \left, \mid, \right, \edge +.endm + + +function sum_lag2_above_neon + push {lr} + sub r12, r0, #2*GRAIN_WIDTH - 16 + sub lr, r0, #1*GRAIN_WIDTH - 16 + vld1.8 {q10}, [r12] // load top right + vld1.8 {q13}, [lr] + + vext.8 q6, q8, q9, #14 // top left, top mid + vdup.8 d14, d28[0] + vext.8 q8, q8, q9, #15 + vdup.8 d15, d28[1] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q2, d0, d12 + vaddl.s16 q3, d1, d13 + vaddl.s16 q4, d2, d16 + vaddl.s16 q5, d3, d17 + + vext.8 q6, q9, q10, #1 // top mid, top right + vdup.8 d14, d28[3] + vext.8 q8, q9, q10, #2 + vdup.8 d15, d28[4] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vext.8 q6, q11, q12, #14 // top left, top mid + vdup.8 d14, d28[5] + vext.8 q8, q11, q12, #15 + vdup.8 d15, d28[6] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vext.8 q6, q12, q13, #1 // top mid, top right + vdup.8 d14, d29[0] + vext.8 q8, q12, q13, #2 + vdup.8 d15, d29[1] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vdup.8 d14, d28[2] + vdup.8 d15, d28[7] + + vmull.s8 q0, d18, d14 + vmull.s8 q1, d19, d14 + vmull.s8 q6, d24, d15 + vmull.s8 q8, d25, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vmov q8, q9 + vmov q9, q10 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vmov q11, q12 + vmov q12, q13 + + pop {pc} +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag2_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #2*GRAIN_WIDTH + sub lr, r0, #1*GRAIN_WIDTH + vld1.8 {q9}, [r12] // load the previous block right above + vld1.8 {q12}, [lr] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[4] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 15 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 15 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 9 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 9 + + +function sum_lag3_left_above_neon + // A separate codepath for the left edge, to avoid reading outside + // of the edge of the buffer. + sub r12, r0, #3*GRAIN_WIDTH + vld1.8 {q11, q12}, [r12] + vext.8 q12, q11, q12, #13 + vext.8 q11, q11, q11, #13 + b sum_lag3_above_start +endfunc + +function sum_lag3_above_neon + sub r12, r0, #3*GRAIN_WIDTH + 3 + vld1.8 {q11, q12}, [r12] + +sum_lag3_above_start: + vdup.8 d20, d26[0] + vext.8 q9, q11, q12, #1 + vdup.8 d21, d26[1] + + vmull.s8 q0, d22, d20 + vmull.s8 q1, d23, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vext.8 q8, q11, q12, #2 + vdup.8 d20, d26[2] + vext.8 q9, q11, q12, #3 + vdup.8 d21, d26[3] + + vaddl.s16 q2, d0, d12 + vaddl.s16 q3, d1, d13 + vaddl.s16 q4, d2, d14 + vaddl.s16 q5, d3, d15 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #4 + vdup.8 d20, d26[4] + vext.8 q7, q11, q12, #5 + vdup.8 d21, d26[5] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + sub r12, r0, #2*GRAIN_WIDTH + 3 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #6 + vld1.8 {q11, q12}, [r12] + vdup.8 d20, d26[6] + vdup.8 d21, d26[7] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d22, d21 + vmull.s8 q7, d23, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #1 + vdup.8 d20, d27[0] + vext.8 q7, q11, q12, #2 + vdup.8 d21, d27[1] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #3 + vdup.8 d20, d27[2] + vext.8 q9, q11, q12, #4 + vdup.8 d21, d27[3] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + sub r12, r0, #1*GRAIN_WIDTH + 3 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #5 + vdup.8 d20, d27[4] + vext.8 q7, q11, q12, #6 + vdup.8 d21, d27[5] + + vld1.8 {q11, q12}, [r12] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vdup.8 d20, d27[6] + vext.8 q9, q11, q12, #1 + vdup.8 d21, d27[7] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d22, d20 + vmull.s8 q1, d23, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #2 + vdup.8 d20, d28[0] + vext.8 q7, q11, q12, #3 + vdup.8 d21, d28[1] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #4 + vdup.8 d20, d28[2] + vext.8 q9, q11, q12, #5 + vdup.8 d21, d28[3] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #6 + vdup.8 d20, d28[4] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + + vaddw.s16 q2, q2, d0 + vaddw.s16 q3, q3, d1 + vaddw.s16 q4, q4, d2 + vaddw.s16 q5, q5, d3 + + bx lr +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag3_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[0] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 15 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 15 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 9 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 9 + +function generate_grain_rows_neon + push {r11,lr} +1: + get_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 + subs r1, r1, #1 + store_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 + bgt 1b + pop {r11,pc} +endfunc + +function generate_grain_rows_44_neon + push {r11,lr} +1: + get_grain_row_44 d16, d17, d18, d19, d20, d21 + subs r1, r1, #1 + store_grain_row_44 d16, d17, d18, d19, d20, d21 + bgt 1b + pop {r11,pc} +endfunc + +function gen_grain_uv_444_lag0_neon + vld1.8 {q3}, [r11]! + push {r11,lr} + bl get_gaussian_neon + vrshl.s16 q8, q0, q15 + bl get_gaussian_neon + vrshl.s16 q9, q0, q15 + vqmovn.s16 d0, q8 + vqmovn.s16 d1, q9 + + vand q3, q3, q1 + vmull.s8 q2, d6, d22 + vmull.s8 q3, d7, d22 + vrshl.s16 q2, q2, q12 + vrshl.s16 q3, q3, q12 + vaddw.s8 q2, q2, d0 + vaddw.s8 q3, q3, d1 + vqmovn.s16 d4, q2 + vqmovn.s16 d5, q3 + vst1.8 {q2}, [r0]! + pop {r11,pc} +endfunc + +function get_grain_row_44_neon + push {r11,lr} + get_grain_row_44 d16, d17, d18, d19, d20, d21 + pop {r11,pc} +endfunc + +function add_uv_420_coeff_lag0_neon + vld1.16 {q2, q3}, [r11]! + vld1.16 {q4, q5}, [r12]! + vpaddl.s8 q2, q2 + vpaddl.s8 q3, q3 + vpaddl.s8 q4, q4 + vpaddl.s8 q5, q5 + vadd.i16 q2, q2, q4 + vadd.i16 q3, q3, q5 + vrshrn.s16 d4, q2, #2 + vrshrn.s16 d5, q3, #2 + b add_coeff_lag0_start +endfunc + +function add_uv_422_coeff_lag0_neon + vld1.16 {q2, q3}, [r11]! + vpaddl.s8 q2, q2 + vpaddl.s8 q3, q3 + vrshrn.s16 d4, q2, #1 + vrshrn.s16 d5, q3, #1 + +add_coeff_lag0_start: + vand q3, q2, q1 + vmull.s8 q2, d6, d22 + vmull.s8 q3, d7, d22 + vrshl.s16 q2, q2, q12 + vrshl.s16 q3, q3, q12 + vaddw.s8 q2, q2, d0 + vaddw.s8 q3, q3, d1 + vqmovn.s16 d4, q2 + vqmovn.s16 d5, q3 + bx lr +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + +.ifc \type, uv_444 + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH + mov r1, r2 + mul r12, r12, lr +.endif + movrel r3, X(gaussian_sequence) + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add r4, r1, #FGD_AR_COEFFS_Y +.else + add r4, r1, #FGD_AR_COEFFS_UV +.endif + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + +.ifc \type, uv_444 + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 +.endif + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, y + mov r1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + + mov r1, #3 + bl generate_grain_rows_neon + mov r1, #GRAIN_HEIGHT-3 + + vdup.16 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vext.8 q13, q0, q1, #13 + vext.8 q14, q1, q0, #1 + vneg.s16 q12, q12 + +1: + vmov q1, q13 + bl gen_grain_uv_444_lag0_neon // 16 + vmov.i8 q1, #255 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 64 + vmov q1, q14 + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 + add r11, r11, #2 + vst1.16 {d16[0]}, [r0]! + bgt 1b +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb r4, [r4, #1] // ar_coeffs_y[3] +.else + add r4, r4, #2 +.endif + + mov r1, #3 +.ifc \type, uv_444 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + sum_\type\()_lag1 q7, q8, q8, q9, left + sum_\type\()_lag1 q8, q8, q9, q10 + sum_\type\()_lag1 q9, q9, q10, q11 + sum_\type\()_lag1 q10, q10, q11, q12 + sum_\type\()_lag1 q12, q11, q12, q13, right + get_grain_2 d26 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + store_grain_row d14, d15, d16, d17, d18, d19, d20, d21, d24, d25, d26 + vmov q11, q10 + vmov q10, q9 + vmov q9, q8 + vmov q8, q7 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + vst1.16 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #127 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + vpush {d26} + bl generate_grain_rows_neon + vpop {d26} + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + vst1.16 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH-(3*32) +.else + sub \reg, \reg, #3*32-GRAIN_WIDTH +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH-3 + mov r1, r2 + mul r12, r12, lr + + movrel r3, X(gaussian_sequence) + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] + add r4, r1, #FGD_AR_COEFFS_UV + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, uv_420 + vpush {q4-q5} +.endif + mov r1, #3 + bl generate_grain_rows_44_neon + set_height r1, \type + + vdup.16 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vext.8 q13, q0, q1, #13 + vext.8 q14, q1, q0, #7 + vneg.s16 q12, q12 + +1: + bl get_grain_row_44_neon +.ifc \type, uv_420 + add r12, r11, #GRAIN_WIDTH +.endif + vmov q1, q13 + vmov q0, q8 + bl add_\type\()_coeff_lag0_neon + vmov.i8 q1, #255 + vmov q0, q9 + vmov q8, q2 + bl add_\type\()_coeff_lag0_neon + vmov.i8 q1, q14 + vmov q0, q10 + vmov q9, q2 + bl add_\type\()_coeff_lag0_neon + vmov q10, q2 + subs r1, r1, #1 + increment_y_ptr r11, \type + store_grain_row_44 d16, d17, d18, d19, d20, d21 + bgt 1b + +.ifc \type, uv_420 + vpop {q4-q5} +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] + add r4, r4, #2 + + mov r1, #3 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + sum_\type\()_lag1 q7, q8, q8, q9, left + sum_\type\()_lag1 q8, q8, q9, q10 + sum_\type\()_lag1 q10, q9, q10, q11, right + subs r1, r1, #1 + increment_y_ptr r11, \type + store_grain_row_44 d14, d15, d16, d17, d20, d21 + vmov q9, q8 + vmov q8, q7 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH-48 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #127 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH-48 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 + +.macro gather_interleaved dst1, dst2, src1, src2, off + vmov.u8 r11, \src1[0+\off] + vmov.u8 r12, \src2[0+\off] + add r11, r11, r3 + vmov.u8 lr, \src1[2+\off] + add r12, r12, r3 + vld1.8 {\dst1[0+\off]}, [r11] + vmov.u8 r11, \src2[2+\off] + add lr, lr, r3 + vld1.8 {\dst2[0+\off]}, [r12] + vmov.u8 r12, \src1[4+\off] + add r11, r11, r3 + vld1.8 {\dst1[2+\off]}, [lr] + vmov.u8 lr, \src2[4+\off] + add r12, r12, r3 + vld1.8 {\dst2[2+\off]}, [r11] + vmov.u8 r11, \src1[6+\off] + add lr, lr, r3 + vld1.8 {\dst1[4+\off]}, [r12] + vmov.u8 r12, \src2[6+\off] + add r11, r11, r3 + vld1.8 {\dst2[4+\off]}, [lr] + add r12, r12, r3 + vld1.8 {\dst1[6+\off]}, [r11] + vld1.8 {\dst2[6+\off]}, [r12] +.endm + +.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4 + gather_interleaved \dst1, \dst3, \src1, \src3, 0 + gather_interleaved \dst1, \dst3, \src1, \src3, 1 + gather_interleaved \dst2, \dst4, \src2, \src4, 0 + gather_interleaved \dst2, \dst4, \src2, \src4, 1 +.endm + +function gather32_neon + push {r11-r12,lr} + gather d8, d9, d10, d11, d0, d1, d2, d3 + pop {r11-r12,pc} +endfunc + +function gather16_neon + push {r11-r12,lr} + gather_interleaved d8, d9, d0, d1, 0 + gather_interleaved d8, d9, d0, d1, 1 + pop {r11-r12,pc} +endfunc + +const overlap_coeffs_0, align=4 + .byte 27, 17, 0, 0, 0, 0, 0, 0 + .byte 17, 27, 32, 32, 32, 32, 32, 32 +endconst + +const overlap_coeffs_1, align=4 + .byte 23, 0, 0, 0, 0, 0, 0, 0 + .byte 22, 32, 32, 32, 32, 32, 32, 32 +endconst + +.macro calc_offset offx, offy, src, sx, sy + and \offy, \src, #0xF // randval & 0xF + lsr \offx, \src, #4 // randval >> 4 +.if \sy == 0 + add \offy, \offy, \offy // 2 * (randval & 0xF) +.endif +.if \sx == 0 + add \offx, \offx, \offx // 2 * (randval >> 4) +.endif +.endm + +.macro add_offset dst, offx, offy, src, stride + mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy + add \dst, \dst, \offx // grain_lut += offx +.endm + +// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const int scaling_shift, +// const entry grain_lut[][GRAIN_WIDTH], +// const int offsets[][2], +// const int h, const ptrdiff_t clip, +// const ptrdiff_t type); +function fgy_32x32_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut + ldrd r6, r7, [sp, #108] // offsets, h + ldr r8, [sp, #116] // clip + mov r9, #GRAIN_WIDTH // grain_lut stride + + neg r4, r4 + vdup.16 q13, r4 // -scaling_shift + cmp r8, #0 + + movrel_local r12, overlap_coeffs_0 + + beq 1f + // clip + vmov.i8 q14, #16 + vmov.i8 q15, #235 + b 2f +1: + // no clip + vmov.i8 q14, #0 + vmov.i8 q15, #255 +2: + + vld1.8 {d24, d25}, [r12, :128] // overlap_coeffs + + add r5, r5, #9 // grain_lut += 9 + add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride + add r5, r5, r9 // grain_lut += grain_stride + + ldr r10, [r6, #8] // offsets[1][0] + calc_offset r10, r4, r10, 0, 0 + add_offset r4, r10, r4, r5, r9 + ldr r10, [r6, #4] // offsets[0][1] + calc_offset r10, r11, r10, 0, 0 + add_offset r11, r10, r11, r5, r9 + ldr r10, [r6, #12] // offsets[1][1] + calc_offset r10, r8, r10, 0, 0 + add_offset r8, r10, r8, r5, r9 + ldr r6, [r6] // offsets[0][0] + calc_offset r6, lr, r6, 0, 0 + add_offset r5, r6, lr, r5, r9 + + add r4, r4, #32 // grain_lut += BLOCK_SIZE * bx + add r6, r11, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + + ldr r10, [sp, #120] // type + adr r11, L(fgy_loop_tbl) + + tst r10, #1 + ldr r10, [r11, r10, lsl #2] + + add r8, r8, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + add r8, r8, #32 // grain_lut += BLOCK_SIZE * bx + + add r11, r11, r10 + + beq 1f + // y overlap + vdup.8 d14, d24[0] + vdup.8 d15, d24[1] + mov r10, r7 // backup actual h + mov r7, #2 +1: + bx r11 +endfunc + +function fgy_loop_neon +L(fgy_loop_tbl): + .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB + +.macro fgy ox, oy +L(loop_\ox\oy): +1: +.if \ox + vld1.8 {d8}, [r4], r9 // grain_lut old +.endif +.if \oy + vld1.8 {q2, q3}, [r6], r9 // grain_lut top +.endif +.if \ox && \oy + vld1.8 {d10}, [r8], r9 // grain_lut top old +.endif + vld1.8 {q0, q1}, [r1, :128], r2 // src + vld1.8 {q10, q11}, [r5], r9 // grain_lut + +.if \ox + vmull.s8 q4, d8, d24 + vmlal.s8 q4, d20, d25 +.endif + +.if \oy +.if \ox + vmull.s8 q5, d10, d24 + vmlal.s8 q5, d4, d25 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d4, q5, #5 +.endif + + vmull.s8 q4, d20, d15 + vmull.s8 q5, d21, d15 + vmull.s8 q8, d22, d15 + vmull.s8 q9, d23, d15 + vmlal.s8 q4, d4, d14 + vmlal.s8 q5, d5, d14 + vmlal.s8 q8, d6, d14 + vmlal.s8 q9, d7, d14 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d21, q5, #5 + vqrshrn.s16 d22, q8, #5 + vqrshrn.s16 d23, q9, #5 +.elseif \ox + vqrshrn.s16 d20, q4, #5 +.endif + + bl gather32_neon + + vmovl.s8 q8, d20 // grain + vmovl.s8 q9, d21 + vmovl.s8 q10, d22 + vmovl.s8 q11, d23 + + vmovl.u8 q2, d8 // scaling + vmovl.u8 q3, d9 + vmovl.u8 q4, d10 + vmovl.u8 q5, d11 + + vmul.i16 q8, q8, q2 // scaling * grain + vmul.i16 q9, q9, q3 + vmul.i16 q10, q10, q4 + vmul.i16 q11, q11, q5 + + vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) + vrshl.s16 q9, q9, q13 + vrshl.s16 q10, q10, q13 + vrshl.s16 q11, q11, q13 + + vaddw.u8 q8, q8, d0 // *src + noise + vaddw.u8 q9, q9, d1 + vaddw.u8 q10, q10, d2 + vaddw.u8 q11, q11, d3 + + vqmovun.s16 d0, q8 + vqmovun.s16 d1, q9 + vqmovun.s16 d2, q10 + vqmovun.s16 d3, q11 + + vmax.u8 q0, q0, q14 + vmax.u8 q1, q1, q14 + vmin.u8 q0, q0, q15 + vmin.u8 q1, q1, q15 + + subs r7, r7, #1 +.if \oy + vdup.8 d14, d25[0] + vdup.8 d15, d25[1] +.endif + vst1.8 {q0, q1}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r10, #2 + sub r7, r10, #2 // restore actual remaining h + bgt L(loop_\ox\()0) +.endif + vpop {q4-q7} + pop {r4-r11,pc} +.endm + + fgy 0, 0 + fgy 0, 1 + fgy 1, 0 + fgy 1, 1 +endfunc + +// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst, +// const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const Dav1dFilmGrainData *const data, +// const entry grain_lut[][GRAIN_WIDTH], +// const pixel *const luma_row, +// const ptrdiff_t luma_stride, +// const int offsets[][2], +// const ptrdiff_t h, const ptrdiff_t uv, +// const ptrdiff_t is_id, +// const ptrdiff_t type); +.macro fguv layout, sx, sy +function fguv_32x32_\layout\()_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] // data, grain_lut + ldrd r6, r7, [sp, #108] // luma_row, luma_stride + ldrd r8, r9, [sp, #116] // offsets, h + ldrd r10, r11, [sp, #124] // uv, is_id + + // !csfl + add r10, r4, r10, lsl #2 // + 4*uv + add r12, r10, #FGD_UV_LUMA_MULT + add lr, r10, #FGD_UV_MULT + add r10, r10, #FGD_UV_OFFSET + vld1.16 {d4[]}, [r12] // uv_luma_mult + vld1.16 {d4[2]}, [r10] // uv_offset + vld1.16 {d4[1]}, [lr] // uv_mult + + ldr lr, [r4, #FGD_SCALING_SHIFT] + ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE] + neg lr, lr // -scaling_shift + + cmp r12, #0 + vdup.16 q13, lr // -scaling_shift + + beq 1f + // clip + cmp r11, #0 + vmov.i8 q14, #16 + vmov.i8 q15, #240 + beq 2f + // is_id + vmov.i8 q15, #235 + b 2f +1: + // no clip + vmov.i8 q14, #0 + vmov.i8 q15, #255 +2: + + mov r10, #GRAIN_WIDTH // grain_lut stride + + add r5, r5, #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6 +.if \sy + add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride + add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride +.else + add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride + add r5, r5, r10 // grain_lut += grain_stride +.endif + + ldr r12, [r8, #8] // offsets[1][0] + calc_offset r12, r4, r12, \sx, \sy + add_offset r4, r12, r4, r5, r10 + + ldr r12, [r8, #4] // offsets[0][1] + calc_offset r12, lr, r12, \sx, \sy + add_offset lr, r12, lr, r5, r10 + + ldr r12, [r8, #12] // offsets[1][1] + calc_offset r12, r11, r12, \sx, \sy + add_offset r11, r12, r11, r5, r10 + + ldr r8, [r8] // offsets[0][0] + calc_offset r8, r12, r8, \sx, \sy + add_offset r5, r8, r12, r5, r10 + + add r4, r4, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add r11, r11, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + + movrel_local r12, overlap_coeffs_\sx + ldr lr, [sp, #132] // type + + vld1.8 {d24, d25}, [r12, :128] // overlap_coeffs + + movrel_local r12, L(fguv_loop_sx\sx\()_tbl) +#if CONFIG_THUMB + // This uses movrel_local instead of adr above, because the target + // can be out of range for adr. But movrel_local leaves the thumb bit + // set on COFF (but probably wouldn't if building for thumb on ELF), + // thus try to clear the bit for robustness. + bic r12, r12, #1 +#endif + + tst lr, #1 + ldr lr, [r12, lr, lsl #2] + + add r12, r12, lr + + beq 1f + // y overlap + sub lr, r9, #(2 >> \sy) // backup remaining h + mov r9, #(2 >> \sy) + +1: + +.if \sy + vmov.i8 d6, #23 + vmov.i8 d7, #22 +.else + vmov.i8 d6, #27 + vmov.i8 d7, #17 +.endif + +.if \sy + add r7, r7, r7 // luma_stride *= 2 +.endif + + bx r12 +endfunc +.endm + +fguv 420, 1, 1 +fguv 422, 1, 0 +fguv 444, 0, 0 + +function fguv_loop_sx0_neon +L(fguv_loop_sx0_tbl): + .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + +.macro fguv_loop_sx0 csfl, ox, oy +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): +.if \oy + mov r12, lr +.endif +1: +.if \ox + vld1.8 {d8}, [r4], r10 // grain_lut old +.endif +.if \oy + vld1.8 {q8, q9}, [r8], r10 // grain_lut top +.endif +.if \ox && \oy + vld1.8 {d10}, [r11], r10 // grain_lut top old +.endif + vld1.8 {q0, q1}, [r6, :128], r7 // luma + vld1.8 {q10, q11}, [r5], r10 // grain_lut + +.if \ox + vmull.s8 q4, d8, d24 + vmlal.s8 q4, d20, d25 +.endif + +.if \oy +.if \ox + vmull.s8 q5, d10, d24 + vmlal.s8 q5, d16, d25 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d16, q5, #5 +.endif + + vmull.s8 q4, d20, d7 + vmull.s8 q5, d21, d7 + vmull.s8 q6, d22, d7 + vmull.s8 q7, d23, d7 + vmlal.s8 q4, d16, d6 + vmlal.s8 q5, d17, d6 + vmlal.s8 q6, d18, d6 + vmlal.s8 q7, d19, d6 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d21, q5, #5 + vqrshrn.s16 d22, q6, #5 + vqrshrn.s16 d23, q7, #5 +.elseif \ox + vqrshrn.s16 d20, q4, #5 +.endif +.if !\csfl + vld1.8 {q8, q9}, [r1, :128] // src + vmovl.u8 q4, d0 + vmovl.u8 q5, d1 + vmovl.u8 q6, d2 + vmovl.u8 q7, d3 + vmovl.u8 q0, d16 + vmovl.u8 q1, d17 + vmovl.u8 q8, d18 + vmovl.u8 q9, d19 + vmul.i16 q4, q4, d4[0] + vmul.i16 q5, q5, d4[0] + vmul.i16 q6, q6, d4[0] + vmul.i16 q7, q7, d4[0] + vmul.i16 q0, q0, d4[1] + vmul.i16 q1, q1, d4[1] + vmul.i16 q8, q8, d4[1] + vmul.i16 q9, q9, d4[1] + vqadd.s16 q4, q4, q0 + vqadd.s16 q5, q5, q1 + vqadd.s16 q6, q6, q8 + vqadd.s16 q7, q7, q9 + vdup.16 q0, d4[2] + vshr.s16 q4, q4, #6 + vshr.s16 q5, q5, #6 + vshr.s16 q6, q6, #6 + vshr.s16 q7, q7, #6 + vadd.i16 q4, q4, q0 + vadd.i16 q5, q5, q0 + vadd.i16 q6, q6, q0 + vadd.i16 q7, q7, q0 + vqmovun.s16 d0, q4 + vqmovun.s16 d1, q5 + vqmovun.s16 d2, q6 + vqmovun.s16 d3, q7 +.endif + + bl gather32_neon + + vld1.8 {q0, q1}, [r1, :128], r2 // src + + vmovl.s8 q8, d20 // grain + vmovl.s8 q9, d21 + vmovl.s8 q10, d22 + vmovl.s8 q11, d23 + + vmovl.u8 q6, d8 // scaling + vmovl.u8 q7, d9 + vmovl.u8 q4, d10 + vmovl.u8 q5, d11 + + vmul.i16 q8, q8, q6 // scaling * grain + vmul.i16 q9, q9, q7 + vmul.i16 q10, q10, q4 + vmul.i16 q11, q11, q5 + + vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) + vrshl.s16 q9, q9, q13 + vrshl.s16 q10, q10, q13 + vrshl.s16 q11, q11, q13 + + vaddw.u8 q8, q8, d0 // *src + noise + vaddw.u8 q9, q9, d1 + vaddw.u8 q10, q10, d2 + vaddw.u8 q11, q11, d3 + + vqmovun.s16 d0, q8 + vqmovun.s16 d1, q9 + vqmovun.s16 d2, q10 + vqmovun.s16 d3, q11 + + vmax.u8 q0, q0, q14 + vmax.u8 q1, q1, q14 + vmin.u8 q0, q0, q15 + vmin.u8 q1, q1, q15 + + subs r9, r9, #1 +.if \oy + vdup.8 d6, d25[0] + vdup.8 d7, d25[1] +.endif + + vst1.8 {q0, q1}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r12, #0 + mov r9, r12 // restore actual remaining h + bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) +.endif + b 9f +.endm + fguv_loop_sx0 0, 0, 0 + fguv_loop_sx0 0, 0, 1 + fguv_loop_sx0 0, 1, 0 + fguv_loop_sx0 0, 1, 1 + fguv_loop_sx0 1, 0, 0 + fguv_loop_sx0 1, 0, 1 + fguv_loop_sx0 1, 1, 0 + fguv_loop_sx0 1, 1, 1 + +9: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function fguv_loop_sx1_neon +L(fguv_loop_sx1_tbl): + .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + +.macro fguv_loop_sx1 csfl, ox, oy +L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): +.if \oy + mov r12, lr +.endif +1: +.if \ox + vld1.8 {d8}, [r4], r10 // grain_lut old +.endif +.if \oy + vld1.8 {q8}, [r8], r10 // grain_lut top +.endif +.if \ox && \oy + vld1.8 {d10}, [r11], r10 // grain_lut top old +.endif + vld1.8 {q0, q1}, [r6, :128], r7 // luma + vld1.8 {q10}, [r5], r10 // grain_lut + vld1.8 {q11}, [r1, :128], r2 // src + +.if \ox + vmull.s8 q4, d8, d24 + vmlal.s8 q4, d20, d25 +.endif + + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 +.if \oy +.if \ox + vmull.s8 q5, d10, d24 + vmlal.s8 q5, d16, d25 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d16, q5, #5 +.endif + + vmull.s8 q4, d20, d7 + vmull.s8 q5, d21, d7 + vmlal.s8 q4, d16, d6 + vmlal.s8 q5, d17, d6 + vqrshrn.s16 d20, q4, #5 + vqrshrn.s16 d21, q5, #5 +.elseif \ox + vqrshrn.s16 d20, q4, #5 +.endif +.if \csfl + vrshrn.u16 d0, q0, #1 + vrshrn.u16 d1, q1, #1 +.else + vrshr.u16 q4, q0, #1 + vrshr.u16 q5, q1, #1 + vmovl.u8 q0, d22 + vmovl.u8 q1, d23 + vmul.i16 q4, q4, d4[0] + vmul.i16 q5, q5, d4[0] + vmul.i16 q0, q0, d4[1] + vmul.i16 q1, q1, d4[1] + vqadd.s16 q4, q4, q0 + vqadd.s16 q5, q5, q1 + vdup.16 q0, d4[2] + vshr.s16 q4, q4, #6 + vshr.s16 q5, q5, #6 + vadd.i16 q4, q4, q0 + vadd.i16 q5, q5, q0 + vqmovun.s16 d0, q4 + vqmovun.s16 d1, q5 +.endif + + bl gather16_neon + + vmovl.s8 q8, d20 // grain + vmovl.s8 q9, d21 + + vmovl.u8 q6, d8 // scaling + vmovl.u8 q7, d9 + + vmul.i16 q8, q8, q6 // scaling * grain + vmul.i16 q9, q9, q7 + + vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) + vrshl.s16 q9, q9, q13 + + vaddw.u8 q8, q8, d22 // *src + noise + vaddw.u8 q9, q9, d23 + + vqmovun.s16 d0, q8 + vqmovun.s16 d1, q9 + + vmax.u8 q0, q0, q14 + vmin.u8 q0, q0, q15 + + subs r9, r9, #1 +.if \oy + vswp d6, d7 +.endif + vst1.8 {q0}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r12, #0 + mov r9, r12 // restore actual remaining h + bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) +.endif + + b 9f +.endm + fguv_loop_sx1 0, 0, 0 + fguv_loop_sx1 0, 0, 1 + fguv_loop_sx1 0, 1, 0 + fguv_loop_sx1 0, 1, 1 + fguv_loop_sx1 1, 0, 0 + fguv_loop_sx1 1, 0, 1 + fguv_loop_sx1 1, 1, 0 + fguv_loop_sx1 1, 1, 1 + +9: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/filmgrain16.S b/third_party/dav1d/src/arm/32/filmgrain16.S new file mode 100644 index 0000000000..6c36caceae --- /dev/null +++ b/third_party/dav1d/src/arm/32/filmgrain16.S @@ -0,0 +1,2137 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "src/arm/asm-offsets.h" + +#define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr r11, r2, #3 + lsr r12, r2, #12 + lsr lr, r2, #1 + eor r11, r2, r11 // (r >> 0) ^ (r >> 3) + eor r12, r12, lr // (r >> 12) ^ (r >> 1) + eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr r2, r2, #\steps +.endif + and r11, r11, #((1 << \steps) - 1) // bit +.if \shift + orr r2, r2, r11, lsl #(16 - \steps) // *state +.else + orr r2, r2, r11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, r2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, r2, #17 - \bits, #\bits + lsr r2, r2, #1 +.endm + +// special calling convention: +// r2 holds seed +// r3 holds dav1d_gaussian_sequence +// clobbers r11-r12 +// returns in d0-d1 +function get_gaussian_neon + push {r5-r6,lr} + increment_seed 4 + read_rand r5, 11, 3 + read_rand r6, 11, 2 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d0[0]}, [r5] + read_rand r5, 11, 1 + vld1.16 {d0[1]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 0 + increment_seed 4 + add r6, r3, r6, lsl #1 + vld1.16 {d0[2]}, [r5] + read_rand r5, 11, 3 + vld1.16 {d0[3]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 2 + vld1.16 {d1[0]}, [r5] + add r6, r3, r6, lsl #1 + read_rand r5, 11, 1 + vld1.16 {d1[1]}, [r6] + read_rand r6, 11, 0 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d1[2]}, [r5] + vld1.16 {d1[3]}, [r6] + pop {r5-r6,pc} +endfunc + +function get_grain_2_neon + push {r11,lr} + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + pop {r11,pc} +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +function get_grain_4_neon + push {r11,lr} + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d0[1]}, [r12] + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[2]}, [r11] + vld1.16 {d0[3]}, [r12] + vrshl.s16 d0, d0, d30 + pop {r11,pc} +endfunc + +.macro get_grain_4 dst + bl get_grain_4_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +// r1 holds the number of entries to produce +// r6, r8 and r10 hold the previous output entries +// q0 holds the vector of produced entries +// q1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon + push {r0, lr} +.if \n == 1 + mvn lr, r5 // grain_min = ~grain_max +.else + mov r0, #1 + mov lr, #1 + sub r7, r7, #1 + sub r9, r9, #1 + lsl r0, r0, r7 + lsl lr, lr, r9 + add r7, r7, #1 + add r9, r9, #1 +.endif +1: + read_shift_rand r12, 11 + vmov.32 r11, d2[0] + lsl r12, r12, #1 + vext.8 q0, q0, q0, #2 + ldrsh r12, [r3, r12] +.if \n == 1 + mla r11, r6, r4, r11 // sum (above) + *coeff * prev output + add r6, r11, r8 // 1 << (ar_coeff_shift - 1) + add r12, r12, r10 + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + cmp r6, r5 +.elseif \n == 2 + mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 + mla r11, r6, r10, r11 // += *coeff * prev output 2 + mov r8, r6 + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mvn lr, r5 // grain_min = ~grain_max +.else + push {r1-r3} + sbfx r1, r4, #0, #8 + sbfx r2, r4, #8, #8 + sbfx r3, r4, #16, #8 + mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 + mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 + mla r11, r6, r3, r11 // += *coeff * prev output 3 + pop {r1-r3} + mov r10, r8 + mov r8, r6 + + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mvn lr, r5 // grain_min = ~grain_max +.endif + it gt + movgt r6, r5 + cmp r6, lr + it lt + movlt r6, lr +.if \n >= 2 + pop {lr} +.endif + subs r1, r1, #1 + vext.8 q1, q1, q1, #4 + vmov.16 d1[3], r6 + bgt 1b + pop {r0, pc} +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + sub r12, r0, #1*GRAIN_WIDTH*2 - 16 + vld1.16 {q10}, [r12] // load top right + + vext.8 q0, q8, q9, #14 // top left, top mid + vext.8 q1, q9, q10, #2 // top left, top mid + + vmull.s16 q2, d18, d28 + vmlal.s16 q2, d0, d27 + vmlal.s16 q2, d2, d29 + vmull.s16 q3, d19, d28 + vmlal.s16 q3, d1, d27 + vmlal.s16 q3, d3, d29 + + vmov q8, q9 + vmov q9, q10 + + bx lr +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff +.ifc \lag\()_\edge, lag3_left + bl sum_lag3_left_above_neon +.else + bl sum_\lag\()_above_neon +.endif +.ifc \type, uv_420 + vpush {q6-q7} + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q0, q1}, [r11]! + vld1.16 {q6, q7}, [r12]! + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d12, d12, d13 + vpadd.i16 d13, d14, d15 + vadd.i16 q0, q0, q6 + vpop {q6-q7} + vrshr.s16 q0, q0, #2 +.endif +.ifc \type, uv_422 + vld1.16 {q0, q1}, [r11]! + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vrshr.s16 q0, q0, #1 +.endif +.ifc \type, uv_444 + vld1.16 {q0}, [r11]! +.endif +.if \uv_layout +.ifnb \uv_coeff + vdup.8 d13, \uv_coeff + vmovl.s8 q6, d13 +.endif + vmlal.s16 q2, d0, d13 + vmlal.s16 q3, d1, d13 +.endif +.if \uv_layout && \elems == 8 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 7 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 1 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: + push {r11} +.if \elems > 4 +.ifc \edge, left + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d1[1]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d1[2]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d1[3]}, [r11] + lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 + vrshl.s16 d1, d1, d30 + vext.8 q2, q2, q2, #12 +.ifc \lag, lag3 + vmov.s16 r10, d1[1] +.endif +.ifnc \lag, lag1 + vmov.s16 r8, d1[2] +.endif + vmov.s16 r6, d1[3] + + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + vmov q1, q3 +.ifc \edge, right + mov r1, #3 + bl output_\lag\()_neon + read_shift_rand r12, 11 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r12] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #2 +.else + mov r1, #4 + bl output_\lag\()_neon +.endif +.else + // elems == 1 + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon + lsr r2, r2, #3 + + read_rand r11, 11, 2 + read_rand r12, 11, 1 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r11] + read_rand r11, 11, 0 + vld1.16 {d2[1]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d2[2]}, [r11] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #14 +.endif + vst1.16 {q0}, [r0]! + pop {r11} + pop {r1, pc} +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag1_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #1*GRAIN_WIDTH*2 + vld1.8 {q9}, [r12] // load the previous block right above +.endif + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 7 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 7 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 1 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 1 + + +function sum_lag2_above_neon + push {lr} + sub r12, r0, #2*GRAIN_WIDTH*2 - 16 + sub lr, r0, #1*GRAIN_WIDTH*2 - 16 + vld1.16 {q10}, [r12] // load top right + vld1.16 {q13}, [lr] + + vdup.8 d10, d28[0] + vext.8 q0, q8, q9, #12 // top left, top mid + vdup.8 d12, d28[1] + vext.8 q1, q8, q9, #14 + vdup.8 d14, d28[3] + vext.8 q4, q9, q10, #2 // top mid, top right + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + + vmull.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d8, d14 + vmull.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d9, d14 + + vdup.8 d10, d28[4] + vext.8 q0, q9, q10, #4 // top mid, top right + vdup.8 d12, d28[5] + vext.8 q1, q11, q12, #12 // top left, top mid + vdup.8 d14, d28[6] + vext.8 q4, q11, q12, #14 + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + + vmlal.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d8, d14 + vmlal.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d9, d14 + + vdup.8 d10, d29[0] + vext.8 q0, q12, q13, #2 // top mid, top right + vdup.8 d12, d29[1] + vext.8 q1, q12, q13, #4 + + vdup.8 d14, d28[2] + vdup.8 d8, d28[7] + + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q4, d8 + + vmlal.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d18, d14 + vmlal.s16 q2, d24, d8 + vmlal.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d19, d14 + vmlal.s16 q3, d25, d8 + + vmov q8, q9 + vmov q9, q10 + + vmov q11, q12 + vmov q12, q13 + + pop {pc} +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag2_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #2*GRAIN_WIDTH*2 + sub lr, r0, #1*GRAIN_WIDTH*2 + vld1.16 {q9}, [r12] // load the previous block right above + vld1.16 {q12}, [lr] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, uv_coeff=d29[4] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 7 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 7 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 1 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 1 + + +function sum_lag3_left_above_neon + // A separate codepath for the left edge, to avoid reading outside + // of the edge of the buffer. + sub r12, r0, #3*GRAIN_WIDTH*2 + vld1.8 {q11, q12}, [r12] + vext.8 q12, q11, q12, #10 + vext.8 q11, q11, q11, #10 + b sum_lag3_above_start +endfunc + +function sum_lag3_above_neon + movw r12, #(3*GRAIN_WIDTH + 3)*2 + sub r12, r0, r12 + vld1.8 {q11, q12}, [r12] + +sum_lag3_above_start: + vdup.8 d12, d26[0] + vext.8 q1, q11, q12, #2 + vdup.8 d14, d26[1] + vext.8 q4, q11, q12, #4 + vdup.8 d16, d26[2] + vext.8 q5, q11, q12, #6 + vdup.8 d18, d26[3] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + movw r12, #(2*GRAIN_WIDTH + 3)*2 + sub r12, r0, r12 + + vmull.s16 q2, d22, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmull.s16 q3, d23, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d26[4] + vext.8 q0, q11, q12, #8 + vdup.8 d14, d26[5] + vext.8 q1, q11, q12, #10 + vdup.8 d16, d26[6] + vext.8 q4, q11, q12, #12 + vld1.8 {q11, q12}, [r12] + vdup.8 d18, d26[7] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d22, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d23, d18 + + vdup.8 d12, d27[0] + vext.8 q0, q11, q12, #2 + vdup.8 d14, d27[1] + vext.8 q1, q11, q12, #4 + vdup.8 d16, d27[2] + vext.8 q4, q11, q12, #6 + vdup.8 d18, d27[3] + vext.8 q5, q11, q12, #8 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + sub r12, r0, #(1*GRAIN_WIDTH + 3)*2 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d27[4] + vext.8 q0, q11, q12, #10 + vdup.8 d14, d27[5] + vext.8 q1, q11, q12, #12 + vld1.8 {q11, q12}, [r12] + vdup.8 d16, d27[6] + vdup.8 d18, d27[7] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vext.8 q5, q11, q12, #2 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d22, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d23, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d28[0] + vext.8 q0, q11, q12, #4 + vdup.8 d14, d28[1] + vext.8 q1, q11, q12, #6 + vdup.8 d16, d28[2] + vext.8 q4, q11, q12, #8 + vdup.8 d18, d28[3] + vext.8 q5, q11, q12, #10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d28[4] + vext.8 q0, q11, q12, #12 + vmovl.s8 q6, d12 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q3, d1, d12 + + bx lr +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag3_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, uv_coeff=d29[0] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 7 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 7 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 1 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 1 + +function generate_grain_rows_neon + push {r10-r11,lr} +1: + mov r10, #80 +2: + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + subs r10, r10, #8 + vst1.16 {q0}, [r0]! + bgt 2b + get_grain_2 d0 + subs r1, r1, #1 + vst1.32 {d0[0]}, [r0]! + bgt 1b + pop {r10-r11,pc} +endfunc + +function generate_grain_rows_44_neon + push {r10-r11,lr} +1: + mov r10, #40 +2: + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + subs r10, r10, #8 + vst1.16 {q0}, [r0]! + bgt 2b + get_grain_4 d0 + subs r1, r1, #1 + vst1.16 {d0}, [r0] + add r0, r0, #GRAIN_WIDTH*2-80 + bgt 1b + pop {r10-r11,pc} +endfunc + +function gen_grain_uv_444_lag0_neon + vld1.16 {q3}, [r11]! +gen_grain_uv_lag0_8_start: + push {r11,lr} + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 +gen_grain_uv_lag0_8_add: + vand q3, q3, q1 + vmull.s16 q2, d6, d22 + vmull.s16 q3, d7, d22 + vrshl.s32 q2, q2, q12 + vrshl.s32 q3, q3, q12 + vqmovn.s32 d4, q2 + vqmovn.s32 d5, q3 + vqadd.s16 q2, q2, q0 + vmin.s16 q2, q2, q9 + vmax.s16 q2, q2, q10 + vst1.16 {q2}, [r0]! + pop {r11,pc} +endfunc + +function gen_grain_uv_420_lag0_8_neon + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q2,q3}, [r11]! + vld1.16 {q4,q5}, [r12] + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vpadd.i16 d8, d8, d9 + vpadd.i16 d9, d10, d11 + vadd.i16 q2, q2, q4 + vrshr.s16 q3, q2, #2 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_422_lag0_8_neon + vld1.16 {q2,q3}, [r11]! + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vrshr.s16 q3, q2, #1 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_420_lag0_4_neon + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q2}, [r11] + vld1.16 {q0}, [r12] + add r11, r11, #32 + vpadd.i16 d4, d4, d5 + vpadd.i16 d0, d0, d1 + vadd.i16 d4, d4, d0 + vrshr.s16 d6, d4, #2 + push {r11,lr} + get_grain_4 d0 + b gen_grain_uv_lag0_8_add +endfunc + +function gen_grain_uv_422_lag0_4_neon + vld1.16 {q2}, [r11] + add r11, r11, #32 + vpadd.i16 d4, d4, d5 + vrshr.s16 d6, d4, #1 + push {r11,lr} + get_grain_4 d0 + b gen_grain_uv_lag0_8_add +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + +.ifc \type, uv_444 + ldr r4, [sp, #36] + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH*2 + mov r1, r2 + mul r12, r12, lr + clz lr, r4 +.else + clz lr, r2 +.endif + movrel r3, X(gaussian_sequence) + sub lr, lr, #24 // -bitdepth_min_8 + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add r4, r1, #FGD_AR_COEFFS_Y +.else + add r4, r1, #FGD_AR_COEFFS_UV +.endif + add r9, r9, lr // grain_scale_shift - bitdepth_min_8 + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + +.ifc \type, uv_444 + push {lr} + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + pop {lr} +.endif + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + neg lr, lr // bitdepth_min_8 + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, y + mov r1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + mvn r6, r5 // grain_min = ~grain_max + + mov r1, #3 + bl generate_grain_rows_neon + mov r1, #GRAIN_HEIGHT-3 + + vdup.32 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vdup.16 q9, r5 + vdup.16 q10, r6 + vext.8 q13, q0, q1, #10 + vext.8 q14, q1, q0, #2 + vneg.s32 q12, q12 + vmovl.s8 q11, d22 + +1: + vmov q1, q13 + bl gen_grain_uv_444_lag0_neon // 8 + vmov.i8 q1, #255 + bl gen_grain_uv_444_lag0_neon // 16 + bl gen_grain_uv_444_lag0_neon // 24 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 40 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 56 + bl gen_grain_uv_444_lag0_neon // 64 + bl gen_grain_uv_444_lag0_neon // 72 + vmov q1, q14 + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 + add r11, r11, #4 + vst1.32 {d16[0]}, [r0]! + bgt 1b +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb r4, [r4, #1] // ar_coeffs_y[3] +.else + add r4, r4, #2 +.endif + + mov r1, #3 +.ifc \type, uv_444 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + vmovl.s8 q13, d27 + vmovl.s8 q12, d29 + vmovl.s8 q14, d28 + vmov d29, d24 +.ifc \type, uv_444 + vmovl.s8 q6, d13 +.endif + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_mid_neon // 48 + bl sum_\type\()_lag1_mid_neon // 56 + bl sum_\type\()_lag1_mid_neon // 64 + bl sum_\type\()_lag1_mid_neon // 72 + bl sum_\type\()_lag1_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_mid_neon // 48 + bl sum_\type\()_lag2_mid_neon // 56 + bl sum_\type\()_lag2_mid_neon // 64 + bl sum_\type\()_lag2_mid_neon // 72 + bl sum_\type\()_lag2_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + vpush {d26} + bl generate_grain_rows_neon + vpop {d26} + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_mid_neon // 48 + bl sum_\type\()_lag3_mid_neon // 56 + bl sum_\type\()_lag3_mid_neon // 64 + bl sum_\type\()_lag3_mid_neon // 72 + bl sum_\type\()_lag3_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) +.else + sub \reg, \reg, #6*32-GRAIN_WIDTH*2 +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + + ldr r4, [sp, #36] + mov r12, r3 + movw r11, #(3*GRAIN_WIDTH-3)*2 + mov lr, #28 + add r11, r1, r11 + mov r1, r2 + mul r12, r12, lr + clz lr, r4 + + movrel r3, X(gaussian_sequence) + sub lr, lr, #24 // -bitdepth_min_8 + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] + add r4, r1, #FGD_AR_COEFFS_UV + add r9, r9, lr // grain_scale_shift - bitdepth_min_8 + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + + push {lr} + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + pop {lr} + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + neg lr, lr + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, uv_420 + vpush {q4-q5} +.endif + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + mvn r6, r5 // grain_min = ~grain_max + + mov r1, #3 + bl generate_grain_rows_44_neon + set_height r1, \type + + vdup.32 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vdup.16 q9, r5 + vdup.16 q10, r6 + vext.8 q13, q0, q1, #10 + vext.8 q14, q1, q0, #14 + vneg.s32 q12, q12 + vmovl.s8 q11, d22 + +1: + vmov q1, q13 + bl gen_grain_\type\()_lag0_8_neon // 8 + vmov.i8 q1, #255 + bl gen_grain_\type\()_lag0_8_neon // 16 + bl gen_grain_\type\()_lag0_8_neon // 24 + bl gen_grain_\type\()_lag0_8_neon // 32 + bl gen_grain_\type\()_lag0_8_neon // 40 + vmov q1, q14 + bl gen_grain_\type\()_lag0_4_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + +.ifc \type, uv_420 + vpop {q4-q5} +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] + add r4, r4, #2 + + mov r1, #3 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + vmovl.s8 q13, d27 + vmovl.s8 q12, d29 + vmovl.s8 q14, d28 + vmov d29, d24 + vmovl.s8 q6, d13 + + set_height r1, \type +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 + +.macro gather_interleaved dst1, dst2, src1, src2, src3, src4, off + vmov.u16 r11, \src1[0+\off] + vmov.u16 r12, \src3[0+\off] + add r11, r11, r3 + vmov.u16 lr, \src1[2+\off] + add r12, r12, r3 + vld1.8 {\dst1[0+\off]}, [r11] + vmov.u16 r11, \src3[2+\off] + add lr, lr, r3 + vld1.8 {\dst2[0+\off]}, [r12] + vmov.u16 r12, \src2[0+\off] + add r11, r11, r3 + vld1.8 {\dst1[2+\off]}, [lr] + vmov.u16 lr, \src4[0+\off] + add r12, r12, r3 + vld1.8 {\dst2[2+\off]}, [r11] + vmov.u16 r11, \src2[2+\off] + add lr, lr, r3 + vld1.8 {\dst1[4+\off]}, [r12] + vmov.u16 r12, \src4[2+\off] + add r11, r11, r3 + vld1.8 {\dst2[4+\off]}, [lr] + add r12, r12, r3 + vld1.8 {\dst1[6+\off]}, [r11] + vld1.8 {\dst2[6+\off]}, [r12] +.endm + +.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, src7, src8 + gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 0 + gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 1 + gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 0 + gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 1 +.endm + +function gather32_neon + push {r11-r12,lr} + gather d8, d9, d10, d11, d0, d1, d2, d3, d4, d5, d6, d7 + pop {r11-r12,pc} +endfunc + +function gather16_neon + push {r11-r12,lr} + gather_interleaved d8, d9, d0, d1, d2, d3, 0 + gather_interleaved d8, d9, d0, d1, d2, d3, 1 + pop {r11-r12,pc} +endfunc + +const overlap_coeffs_0, align=4 + .short 27, 17, 0, 0 + .short 17, 27, 32, 32 +endconst + +const overlap_coeffs_1, align=4 + .short 23, 0, 0, 0 + .short 22, 32, 32, 32 +endconst + +.macro calc_offset offx, offy, src, sx, sy + and \offy, \src, #0xF // randval & 0xF + lsr \offx, \src, #4 // randval >> 4 +.if \sy == 0 + add \offy, \offy, \offy // 2 * (randval & 0xF) +.endif +.if \sx == 0 + add \offx, \offx, \offx // 2 * (randval >> 4) +.endif +.endm + +.macro add_offset dst, offx, offy, src, stride + mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy + add \dst, \dst, \offx, lsl #1 // grain_lut += offx +.endm + +// void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const int scaling_shift, +// const entry grain_lut[][GRAIN_WIDTH], +// const int offsets[][2], +// const int h, const ptrdiff_t clip, +// const ptrdiff_t type, +// const int bitdepth_max); +function fgy_32x32_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut + ldrd r6, r7, [sp, #108] // offsets, h + ldr r8, [sp, #116] // clip + mov r9, #GRAIN_WIDTH*2 // grain_lut stride + ldr r10, [sp, #124] // bitdepth_max + + eor r4, r4, #15 // 15 - scaling_shift + vdup.16 q6, r10 // bitdepth_max + clz r10, r10 + vdup.16 q13, r4 // 15 - scaling_shift + rsb r10, r10, #24 // bitdepth_min_8 + cmp r8, #0 + vdup.16 q12, r10 // bitdepth_min_8 + + movrel_local r12, overlap_coeffs_0 + + beq 1f + // clip + vmov.i16 q14, #16 + vmov.i16 q15, #235 + vshl.s16 q14, q14, q12 + vshl.s16 q15, q15, q12 + b 2f +1: + // no clip + vmov.i16 q14, #0 + vmov q15, q6 +2: + vshr.u16 q6, q6, #1 // grain_max + + vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs + + add r5, r5, #18 // grain_lut += 9 + add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride + add r5, r5, r9 // grain_lut += grain_stride + + ldr r10, [r6, #8] // offsets[1][0] + calc_offset r10, r4, r10, 0, 0 + add_offset r4, r10, r4, r5, r9 + ldr r10, [r6, #4] // offsets[0][1] + calc_offset r10, r11, r10, 0, 0 + add_offset r11, r10, r11, r5, r9 + ldr r10, [r6, #12] // offsets[1][1] + calc_offset r10, r8, r10, 0, 0 + add_offset r8, r10, r8, r5, r9 + ldr r6, [r6] // offsets[0][0] + calc_offset r6, lr, r6, 0, 0 + add_offset r5, r6, lr, r5, r9 + + add r4, r4, #32*2 // grain_lut += BLOCK_SIZE * bx + add r6, r11, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + + ldr r10, [sp, #120] // type + adr r11, L(fgy_loop_tbl) + + tst r10, #1 + ldr r10, [r11, r10, lsl #2] + + add r8, r8, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + add r8, r8, #32*2 // grain_lut += BLOCK_SIZE * bx + + add r11, r11, r10 + + beq 1f + // y overlap + vdup.16 d14, d24[0] + vdup.16 d15, d24[1] + mov r10, r7 // backup actual h + mov r7, #2 +1: + sub r2, r2, #32 // src_stride -= 32 + sub r9, r9, #32 // grain_stride -= 32 + bx r11 +endfunc + +function fgy_loop_neon +L(fgy_loop_tbl): + .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB + .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB + +.macro fgy ox, oy +L(loop_\ox\oy): +1: +.if \ox + vld1.16 {d0}, [r4], r9 // grain_lut old +.endif +.if \oy + vld1.16 {q2, q3}, [r6]! // grain_lut top +.endif +.if \ox && \oy + vld1.16 {d2}, [r8], r9 // grain_lut top old +.endif +.if \oy + vld1.16 {q4, q5}, [r6], r9 // grain_lut top +.endif +.if !\ox && !\oy + vld1.16 {q0, q1}, [r1, :128]! // src +.endif + vld1.16 {q8, q9}, [r5]! // grain_lut +.if !\ox && !\oy + vld1.16 {q2, q3}, [r1, :128], r2 // src +.endif +.if !\oy + vmvn.i16 q5, #0xf000 // 0x0fff +.endif + vld1.16 {q10, q11}, [r5], r9 // grain_lut + +.if \ox + add r4, r4, #32 + vmull.s16 q0, d0, d24 + vmlal.s16 q0, d16, d25 +.endif + +.if \oy +.if \ox + add r8, r8, #32 + vmull.s16 q1, d2, d24 + vmlal.s16 q1, d4, d25 + vqrshrn.s32 d16, q0, #5 + vmvn d0, d12 // grain_min + vqrshrn.s32 d4, q1, #5 + vmin.s16 d16, d16, d12 + vmin.s16 d4, d4, d12 + vmax.s16 d16, d16, d0 + vmax.s16 d4, d4, d0 +.endif + + vmull.s16 q0, d4, d14 + vmull.s16 q1, d5, d14 + vmull.s16 q2, d6, d14 + vmull.s16 q3, d7, d14 + vmlal.s16 q0, d16, d15 + vmlal.s16 q1, d17, d15 + vmlal.s16 q2, d18, d15 + vmlal.s16 q3, d19, d15 + vmull.s16 q8, d20, d15 + vmull.s16 q9, d21, d15 + vmull.s16 q10, d22, d15 + vmull.s16 q11, d23, d15 + vmlal.s16 q8, d8, d14 + vmlal.s16 q9, d9, d14 + vmlal.s16 q10, d10, d14 + vmlal.s16 q11, d11, d14 + vmvn q4, q6 // grain_min + vqrshrn.s32 d0, q0, #5 + vqrshrn.s32 d1, q1, #5 + vqrshrn.s32 d2, q2, #5 + vqrshrn.s32 d3, q3, #5 + vqrshrn.s32 d4, q8, #5 + vqrshrn.s32 d5, q9, #5 + vqrshrn.s32 d6, q10, #5 + vqrshrn.s32 d7, q11, #5 + vmin.s16 q8, q0, q6 + vmin.s16 q9, q1, q6 + vld1.16 {q0, q1}, [r1, :128]! // src + vmin.s16 q10, q2, q6 + vmin.s16 q11, q3, q6 + vmax.s16 q8, q8, q4 + vmax.s16 q9, q9, q4 + vld1.16 {q2, q3}, [r1, :128], r2 // src + vmvn.i16 q5, #0xf000 // 0x0fff + vmax.s16 q10, q10, q4 + vmax.s16 q11, q11, q4 +.elseif \ox + vmvn d4, d12 // grain_min + vqrshrn.s32 d16, q0, #5 + vld1.16 {q0, q1}, [r1, :128]! // src + vmin.s16 d16, d16, d12 + vmax.s16 d16, d16, d4 + vld1.16 {q2, q3}, [r1, :128], r2 // src +.endif + + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + vand q0, q0, q5 + vand q1, q1, q5 + vand q2, q2, q5 + vand q3, q3, q5 + + bl gather32_neon + +.if \ox || \oy + vpush {q6-q7} +.endif + + vmovl.u8 q6, d8 // scaling + vmovl.u8 q7, d9 + vmovl.u8 q4, d10 + vmovl.u8 q5, d11 + + vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) + vshl.u16 q7, q7, q13 + vshl.u16 q4, q4, q13 + vshl.u16 q5, q5, q13 + + vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) + vqrdmulh.s16 q9, q9, q7 + vqrdmulh.s16 q10, q10, q4 + vqrdmulh.s16 q11, q11, q5 + +.if \ox || \oy + vpop {q6-q7} +.endif + + vqadd.s16 q0, q0, q8 // *src + noise + vqadd.s16 q1, q1, q9 + vqadd.s16 q2, q2, q10 + vqadd.s16 q3, q3, q11 + + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q0, q0, q15 + vmin.s16 q1, q1, q15 + vmin.s16 q2, q2, q15 + vmin.s16 q3, q3, q15 + + vst1.16 {q0, q1}, [r0, :128]! // dst + subs r7, r7, #1 +.if \oy + vdup.16 d14, d25[0] + vdup.16 d15, d25[1] +.endif + vst1.16 {q2, q3}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r10, #2 + sub r7, r10, #2 // restore actual remaining h + bgt L(loop_\ox\()0) +.endif + vpop {q4-q7} + pop {r4-r11,pc} +.endm + + fgy 0, 0 + fgy 0, 1 + fgy 1, 0 + fgy 1, 1 +endfunc + +// void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst, +// const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const Dav1dFilmGrainData *const data, +// const entry grain_lut[][GRAIN_WIDTH], +// const pixel *const luma_row, +// const ptrdiff_t luma_stride, +// const int offsets[][2], +// const ptrdiff_t h, const ptrdiff_t uv, +// const ptrdiff_t is_id, +// const ptrdiff_t type, +// const int bitdepth_max); +.macro fguv layout, sx, sy +function fguv_32x32_\layout\()_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] // data, grain_lut + ldrd r10, r11, [sp, #124] // uv, is_id + ldr r6, [sp, #136] // bitdepth_max + + clz r7, r6 + rsb r7, r7, #24 // bitdepth_min_8 + + // !csfl + add r10, r4, r10, lsl #2 // + 4*uv + add r12, r10, #FGD_UV_LUMA_MULT + add lr, r10, #FGD_UV_MULT + ldrh r10, [r10, #FGD_UV_OFFSET] // uv_offset + vld1.16 {d30[]}, [r12] // uv_luma_mult + lsl r10, r10, r7 // uv_offset << bitdepth_min_8 + vld1.16 {d30[1]}, [lr] // uv_mult + + ldr lr, [r4, #FGD_SCALING_SHIFT] + ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE] + eor lr, lr, #15 // 15 - scaling_shift + + vmov.16 d30[2], r10 // uv_offset << bitdepth_min_8 + + cmp r12, #0 + vdup.16 q13, lr // 15 - scaling_shift + + beq 1f + // clip + cmp r11, #0 + mov r8, #16 + mov r9, #240 + lsl r8, r8, r7 + lsl r9, r9, r7 + beq 2f + // is_id + mov r9, #235 + lsl r9, r9, r7 + b 2f +1: + // no clip + mov r8, #0 + mov r9, r6 // bitdepth_max +2: + vmov.16 d30[3], r6 // bitdepth_max + vdup.16 d31, r8 // clip_min + + mov r10, #GRAIN_WIDTH*2 // grain_lut stride + +.if \sy + mov r6, #23 + mov r7, #22 +.else + mov r6, #27 + mov r7, #17 +.endif + vmov.16 d31[1], r9 // clip_max + + ldrd r8, r9, [sp, #116] // offsets, h + + add r5, r5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6 +.if \sy + add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride + add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride +.else + add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride + add r5, r5, r10 // grain_lut += grain_stride +.endif + vmov.16 d31[2], r6 // overlap y [0] + + ldr r12, [r8, #8] // offsets[1][0] + calc_offset r12, r4, r12, \sx, \sy + add_offset r4, r12, r4, r5, r10 + + ldr r12, [r8, #4] // offsets[0][1] + calc_offset r12, lr, r12, \sx, \sy + add_offset lr, r12, lr, r5, r10 + + ldr r12, [r8, #12] // offsets[1][1] + calc_offset r12, r11, r12, \sx, \sy + add_offset r11, r12, r11, r5, r10 + + ldr r8, [r8] // offsets[0][0] + calc_offset r8, r12, r8, \sx, \sy + add_offset r5, r8, r12, r5, r10 + + vmov.16 d31[3], r7 // overlap y [1] + + add r4, r4, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add r11, r11, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + + movrel_local r12, overlap_coeffs_\sx + ldr lr, [sp, #132] // type + ldrd r6, r7, [sp, #108] // luma_row, luma_stride + + vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs + + movrel_local r12, L(fguv_loop_sx\sx\()_tbl) +#if CONFIG_THUMB + // This uses movrel_local instead of adr above, because the target + // can be out of range for adr. But movrel_local leaves the thumb bit + // set on COFF (but probably wouldn't if building for thumb on ELF), + // thus try to clear the bit for robustness. + bic r12, r12, #1 +#endif + + tst lr, #1 + ldr lr, [r12, lr, lsl #2] + + add r12, r12, lr + + beq 1f + // y overlap + sub lr, r9, #(2 >> \sy) // backup remaining h + mov r9, #(2 >> \sy) + +1: +.if \sy + add r7, r7, r7 // luma_stride *= 2 +.endif + sub r7, r7, #32 // luma_stride -= 32 + + bx r12 +endfunc +.endm + +fguv 420, 1, 1 +fguv 422, 1, 0 +fguv 444, 0, 0 + +function fguv_loop_sx0_neon +L(fguv_loop_sx0_tbl): + .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB + +.macro fguv_loop_sx0 csfl, ox, oy +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): + sub r2, r2, #32 // src_stride -= 32 + sub r10, r10, #32 // grain_stride -= 32 +.if \oy + mov r12, lr +.endif +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy\()_loopstart): +1: +.if \ox + vld1.16 {d0}, [r4], r10 // grain_lut old +.endif +.if \oy + vld1.16 {q2, q3}, [r8]! // grain_lut top +.endif +.if \ox && \oy + vld1.16 {d2}, [r11], r10 // grain_lut top old +.endif +.if !\ox && !\oy + vld1.16 {q0, q1}, [r6, :128]! // luma +.endif + vld1.16 {q8, q9}, [r5]! // grain_lut +.if \oy + vld1.16 {q4, q5}, [r8], r10 // grain_lut top +.endif +.if !\ox && !\oy + vld1.16 {q2, q3}, [r6, :128], r7 // luma +.endif +.if \oy + vdup.16 d28, d31[2] // overlap y coeff + vdup.16 d29, d31[3] // overlap y coeff +.endif + vld1.16 {q10, q11}, [r5], r10 // grain_lut + +.if \ox + vdup.16 q7, d30[3] // bitdepth_max + add r4, r4, #32 + vmull.s16 q0, d0, d24 + vshr.u16 q7, q7, #1 // grain_max + vmlal.s16 q0, d16, d25 + vmvn q6, q7 // grain_min +.endif + +.if \oy +.if \ox + add r11, r11, #32 + vmull.s16 q1, d2, d24 + vmlal.s16 q1, d4, d25 + vqrshrn.s32 d16, q0, #5 + vqrshrn.s32 d4, q1, #5 + vmin.s16 d4, d4, d14 + vmin.s16 d16, d16, d14 + vmax.s16 d4, d4, d12 + vmax.s16 d16, d16, d12 +.endif + + vmull.s16 q0, d4, d28 + vmull.s16 q1, d5, d28 + vmull.s16 q2, d6, d28 + vmull.s16 q3, d7, d28 +.if !\ox + vdup.16 q7, d30[3] // bitdepth_max +.endif + vmlal.s16 q0, d16, d29 + vmlal.s16 q1, d17, d29 + vmlal.s16 q2, d18, d29 + vmlal.s16 q3, d19, d29 +.if !\ox + vshr.u16 q7, q7, #1 // grain_max +.endif + vmull.s16 q8, d20, d29 + vmull.s16 q9, d21, d29 + vmull.s16 q10, d22, d29 + vmull.s16 q11, d23, d29 +.if !\ox + vmvn q6, q7 // grain_min +.endif + vmlal.s16 q8, d8, d28 + vmlal.s16 q9, d9, d28 + vmlal.s16 q10, d10, d28 + vmlal.s16 q11, d11, d28 + vqrshrn.s32 d0, q0, #5 + vqrshrn.s32 d1, q1, #5 + vqrshrn.s32 d2, q2, #5 + vqrshrn.s32 d3, q3, #5 + vqrshrn.s32 d4, q8, #5 + vqrshrn.s32 d5, q9, #5 + vqrshrn.s32 d6, q10, #5 + vqrshrn.s32 d7, q11, #5 + vmin.s16 q8, q0, q7 + vmin.s16 q9, q1, q7 + vld1.16 {q0, q1}, [r6, :128]! // luma + vmin.s16 q10, q2, q7 + vmin.s16 q11, q3, q7 + vmax.s16 q8, q8, q6 + vmax.s16 q9, q9, q6 + vld1.16 {q2, q3}, [r6, :128], r7 // luma + vmax.s16 q10, q10, q6 + vmax.s16 q11, q11, q6 +.elseif \ox + vqrshrn.s32 d16, q0, #5 + vld1.16 {q0, q1}, [r6, :128]! // luma + vmin.s16 d16, d16, d14 + vld1.16 {q2, q3}, [r6, :128], r7 // luma + vmax.s16 d16, d16, d12 +.endif + +.if !\csfl + vdup.16 d28, d30[0] // uv_luma_mult + vld1.16 {q4, q5}, [r1, :128]! // src + vdup.16 d29, d30[1] // uv_mult + vmull.s16 q6, d0, d28 + vmull.s16 q7, d1, d28 + vmull.s16 q0, d2, d28 + vmull.s16 q1, d3, d28 + vmlal.s16 q6, d8, d29 + vmlal.s16 q7, d9, d29 + vmlal.s16 q0, d10, d29 + vmlal.s16 q1, d11, d29 + vld1.16 {q4, q5}, [r1, :128] // src + sub r1, r1, #32 + vshrn.s32 d12, q6, #6 + vshrn.s32 d13, q7, #6 + vshrn.s32 d14, q0, #6 + vshrn.s32 d15, q1, #6 + vmull.s16 q0, d4, d28 + vmull.s16 q1, d5, d28 + vmull.s16 q2, d6, d28 + vmull.s16 q3, d7, d28 + vmlal.s16 q0, d8, d29 + vmlal.s16 q1, d9, d29 + vmlal.s16 q2, d10, d29 + vmlal.s16 q3, d11, d29 + vdup.16 q14, d30[2] // uv_offset + vshrn.s32 d0, q0, #6 + vshrn.s32 d1, q1, #6 + vshrn.s32 d2, q2, #6 + vshrn.s32 d3, q3, #6 + vdup.16 q4, d30[3] // bitdepth_max + vmov.i16 q5, #0 + vadd.i16 q6, q6, q14 + vadd.i16 q7, q7, q14 + vadd.i16 q2, q0, q14 + vadd.i16 q3, q1, q14 + vmin.s16 q0, q6, q4 + vmin.s16 q1, q7, q4 + vmin.s16 q2, q2, q4 + vmin.s16 q3, q3, q4 + vmax.s16 q0, q0, q5 + vmax.s16 q1, q1, q5 + vmax.s16 q2, q2, q5 + vmax.s16 q3, q3, q5 +.else + vdup.16 q14, d30[3] // bitdepth_max + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + vand q0, q0, q14 + vand q1, q1, q14 + vand q2, q2, q14 + vand q3, q3, q14 +.endif + + bl gather32_neon + + vld1.16 {q0, q1}, [r1, :128]! // src + + vmovl.u8 q6, d8 // scaling + vmovl.u8 q7, d9 + vmovl.u8 q4, d10 + vmovl.u8 q5, d11 + + vld1.16 {q2, q3}, [r1, :128], r2 // src + + vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) + vshl.u16 q7, q7, q13 + vshl.u16 q4, q4, q13 + vshl.u16 q5, q5, q13 + + vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) + vqrdmulh.s16 q9, q9, q7 + vqrdmulh.s16 q10, q10, q4 + vqrdmulh.s16 q11, q11, q5 + + + vdup.16 q4, d31[0] // clip_min + vdup.16 q5, d31[1] // clip_max + + vqadd.s16 q0, q0, q8 // *src + noise + vqadd.s16 q1, q1, q9 + vqadd.s16 q2, q2, q10 + vqadd.s16 q3, q3, q11 + +.if \oy + vmov.32 lr, d25[0] // 2 first 16 bit coeffs from overlap x +.endif + + vmax.s16 q0, q0, q4 + vmax.s16 q1, q1, q4 + vmax.s16 q2, q2, q4 + vmax.s16 q3, q3, q4 + vmin.s16 q0, q0, q5 + vmin.s16 q1, q1, q5 + vmin.s16 q2, q2, q5 + vmin.s16 q3, q3, q5 + + vst1.16 {q0, q1}, [r0, :128]! // dst + + subs r9, r9, #1 +.if \oy + vmov.32 d31[1], lr // new coeffs for overlap y +.endif + + vst1.16 {q2, q3}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r12, #0 + mov r9, r12 // restore actual remaining h + bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0_loopstart) +.endif + b 9f +.endm + fguv_loop_sx0 0, 0, 0 + fguv_loop_sx0 0, 0, 1 + fguv_loop_sx0 0, 1, 0 + fguv_loop_sx0 0, 1, 1 + fguv_loop_sx0 1, 0, 0 + fguv_loop_sx0 1, 0, 1 + fguv_loop_sx0 1, 1, 0 + fguv_loop_sx0 1, 1, 1 + +9: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function fguv_loop_sx1_neon +L(fguv_loop_sx1_tbl): + .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB + +.macro fguv_loop_sx1 csfl, ox, oy +L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): +.if \oy + mov r12, lr +.endif +1: +.if \ox + vld1.16 {d0}, [r4], r10 // grain_lut old +.endif +.if \ox && \oy + vld1.16 {d2}, [r11], r10 // grain_lut top old +.endif +.if \oy + vld1.16 {q2, q3}, [r8], r10 // grain_lut top +.endif +.if !\ox && !\oy + vld1.16 {q0, q1}, [r6, :128]! // luma +.endif + vld1.16 {q8, q9}, [r5], r10 // grain_lut +.if \oy + vdup.16 d28, d31[2] // overlap y coeff + vdup.16 d29, d31[3] // overlap y coeff +.endif +.if !\ox && !\oy + vld1.16 {q2, q3}, [r6, :128], r7 // luma +.endif + +.if \ox + vdup.16 q7, d30[3] // bitdepth_max + vmull.s16 q0, d0, d24 + vshr.u16 q7, q7, #1 // grain_max + vmlal.s16 q0, d16, d25 + vmvn q6, q7 // grain_min +.endif + +.if \oy +.if \ox + vmull.s16 q1, d2, d24 + vmlal.s16 q1, d4, d25 + vqrshrn.s32 d16, q0, #5 + vqrshrn.s32 d4, q1, #5 + vmin.s16 d4, d4, d14 + vmin.s16 d16, d16, d14 + vmax.s16 d4, d4, d12 + vmax.s16 d16, d16, d12 +.endif + + vmull.s16 q0, d4, d28 + vmull.s16 q1, d5, d28 + vmull.s16 q2, d6, d28 + vmull.s16 q3, d7, d28 +.if !\ox + vdup.16 q7, d30[3] // bitdepth_max +.endif + vmlal.s16 q0, d16, d29 + vmlal.s16 q1, d17, d29 + vmlal.s16 q2, d18, d29 + vmlal.s16 q3, d19, d29 +.if !\ox + vshr.u16 q7, q7, #1 // grain_max +.endif + vqrshrn.s32 d16, q0, #5 + vqrshrn.s32 d17, q1, #5 + vqrshrn.s32 d18, q2, #5 + vqrshrn.s32 d19, q3, #5 +.if !\ox + vmvn q6, q7 // grain_min +.endif + vld1.16 {q0, q1}, [r6, :128]! // luma + vmin.s16 q8, q8, q7 + vmin.s16 q9, q9, q7 + vmax.s16 q8, q8, q6 + vmax.s16 q9, q9, q6 + vld1.16 {q2, q3}, [r6, :128], r7 // luma +.elseif \ox + vqrshrn.s32 d16, q0, #5 + vld1.16 {q0, q1}, [r6, :128]! // luma + vmin.s16 d16, d16, d14 + vld1.16 {q2, q3}, [r6, :128], r7 // luma + vmax.s16 d16, d16, d12 +.endif + + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d4, d5 + vpadd.i16 d3, d6, d7 + vrshr.u16 q0, q0, #1 + vrshr.u16 q1, q1, #1 +.if !\csfl + vdup.16 d28, d30[0] // uv_luma_mult + vld1.16 {q2, q3}, [r1, :128], r2 // src + vdup.16 d29, d30[1] // uv_mult + vmull.s16 q6, d0, d28 + vmull.s16 q7, d1, d28 + vmull.s16 q0, d2, d28 + vmull.s16 q1, d3, d28 + vmlal.s16 q6, d4, d29 + vmlal.s16 q7, d5, d29 + vmlal.s16 q0, d6, d29 + vmlal.s16 q1, d7, d29 + vshrn.s32 d12, q6, #6 + vshrn.s32 d13, q7, #6 + vshrn.s32 d14, q0, #6 + vshrn.s32 d15, q1, #6 + vdup.16 q14, d30[2] // uv_offset + vdup.16 q4, d30[3] // bitdepth_max + vmov.i16 q5, #0 + vadd.i16 q6, q6, q14 + vadd.i16 q7, q7, q14 + vmin.s16 q0, q6, q4 + vmin.s16 q1, q7, q4 + vmax.s16 q0, q0, q5 + vmax.s16 q1, q1, q5 +.else + vdup.16 q14, d30[3] // bitdepth_max + vld1.16 {q2, q3}, [r1, :128], r2 // src + + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + vand q0, q0, q14 + vand q1, q1, q14 +.endif + + bl gather16_neon + + vmovl.u8 q6, d8 // scaling + vmovl.u8 q7, d9 + + vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) + vshl.u16 q7, q7, q13 + + vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) + vqrdmulh.s16 q9, q9, q7 + + + vdup.16 q4, d31[0] // clip_min + vdup.16 q5, d31[1] // clip_max + + vqadd.s16 q0, q2, q8 // *src + noise + vqadd.s16 q1, q3, q9 + +.if \oy + // Swap the two last coefficients of d31, place them first in d28 + vrev64.16 d28, d31 +.endif + + vmax.s16 q0, q0, q4 + vmax.s16 q1, q1, q4 + vmin.s16 q0, q0, q5 + vmin.s16 q1, q1, q5 + + subs r9, r9, #1 +.if \oy + // Take the first two 16 bit coefficients of d28 and place them at the + // end of d31 + vtrn.32 d31, d28 +.endif + + vst1.16 {q0, q1}, [r0, :128], r2 // dst + bgt 1b + +.if \oy + cmp r12, #0 + mov r9, r12 // restore actual remaining h + bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) +.endif + + b 9f +.endm + fguv_loop_sx1 0, 0, 0 + fguv_loop_sx1 0, 0, 1 + fguv_loop_sx1 0, 1, 0 + fguv_loop_sx1 0, 1, 1 + fguv_loop_sx1 1, 0, 0 + fguv_loop_sx1 1, 0, 1 + fguv_loop_sx1 1, 1, 0 + fguv_loop_sx1 1, 1, 1 + +9: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/ipred.S b/third_party/dav1d/src/arm/32/ipred.S new file mode 100644 index 0000000000..ff55d95d4a --- /dev/null +++ b/third_party/dav1d/src/arm/32/ipred.S @@ -0,0 +1,2937 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * Copyright © 2019, B Krishnan Iyer + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 + push {r4, lr} + ldr r4, [sp, #8] + clz r3, r3 + adr r2, L(ipred_dc_128_tbl) + sub r3, r3, #25 + ldr r3, [r2, r3, lsl #2] + vmov.i8 q0, #128 + add r2, r2, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r2 + + .align 2 +L(ipred_dc_128_tbl): + .word 640f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 320f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 16f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 8f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 4f - L(ipred_dc_128_tbl) + CONFIG_THUMB +4: + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + subs r4, r4, #4 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt 4b + pop {r4, pc} +8: + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt 8b + pop {r4, pc} +16: + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + bgt 16b + pop {r4, pc} +320: + vmov.i8 q1, #128 +32: + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 32b + pop {r4, pc} +640: + vmov.i8 q1, #128 + sub r1, r1, #32 +64: + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 64b + pop {r4, pc} +endfunc + +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 + push {r4, lr} + ldr lr, [sp, #8] + clz r3, r3 + adr r4, L(ipred_v_tbl) + sub r3, r3, #25 + ldr r3, [r4, r3, lsl #2] + add r2, r2, #1 + add r4, r4, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r4 + + .align 2 +L(ipred_v_tbl): + .word 640f - L(ipred_v_tbl) + CONFIG_THUMB + .word 320f - L(ipred_v_tbl) + CONFIG_THUMB + .word 160f - L(ipred_v_tbl) + CONFIG_THUMB + .word 80f - L(ipred_v_tbl) + CONFIG_THUMB + .word 40f - L(ipred_v_tbl) + CONFIG_THUMB +40: + vld1.32 {d0[]}, [r2] +4: + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + subs lr, lr, #4 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt 4b + pop {r4, pc} +80: + vld1.8 {d0}, [r2] +8: + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + subs lr, lr, #4 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt 8b + pop {r4, pc} +160: + vld1.8 {q0}, [r2] +16: + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + subs lr, lr, #4 + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + bgt 16b + pop {r4, pc} +320: + vld1.8 {q0, q1}, [r2] +32: + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs lr, lr, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 32b + pop {r4, pc} +640: + vld1.8 {q0, q1}, [r2]! + sub r1, r1, #32 + vld1.8 {q2, q3}, [r2] +64: + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 + vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 + subs lr, lr, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 + vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 + bgt 64b + pop {r4, pc} +endfunc + +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + clz r3, r3 + adr r5, L(ipred_h_tbl) + sub r3, r3, #25 + ldr r3, [r5, r3, lsl #2] + sub r2, r2, #4 + mov lr, #-4 + add r5, r5, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_h_tbl): + .word 640f - L(ipred_h_tbl) + CONFIG_THUMB + .word 320f - L(ipred_h_tbl) + CONFIG_THUMB + .word 160f - L(ipred_h_tbl) + CONFIG_THUMB + .word 8f - L(ipred_h_tbl) + CONFIG_THUMB + .word 4f - L(ipred_h_tbl) + CONFIG_THUMB +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr + vst1.32 {d3[0]}, [r0, :32], r1 + vst1.32 {d2[0]}, [r12, :32], r1 + subs r4, r4, #4 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt 4b + pop {r4-r5, pc} +8: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr + vst1.8 {d3}, [r0, :64], r1 + vst1.8 {d2}, [r12, :64], r1 + subs r4, r4, #4 + vst1.8 {d1}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt 8b + pop {r4-r5, pc} +160: + add r2, r2, #3 + mov lr, #-1 +16: + vld1.8 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.8 {d2[], d3[]}, [r2], lr + vst1.8 {q0}, [r0, :128], r1 + vld1.8 {d4[], d5[]}, [r2], lr + vst1.8 {q1}, [r12, :128], r1 + vld1.8 {d6[], d7[]}, [r2], lr + vst1.8 {q2}, [r0, :128], r1 + vst1.8 {q3}, [r12, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + add r2, r2, #3 + mov lr, #-1 + sub r1, r1, #16 +32: + vld1.8 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.8 {d2[], d3[]}, [r2], lr + vst1.8 {q0}, [r0, :128]! + vld1.8 {d4[], d5[]}, [r2], lr + vst1.8 {q1}, [r12, :128]! + vld1.8 {d6[], d7[]}, [r2], lr + vst1.8 {q0}, [r0, :128], r1 + vst1.8 {q1}, [r12, :128], r1 + vst1.8 {q2}, [r0, :128]! + vst1.8 {q3}, [r12, :128]! + vst1.8 {q2}, [r0, :128], r1 + vst1.8 {q3}, [r12, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + add r2, r2, #3 + mov lr, #-1 + sub r1, r1, #48 +64: + vld1.8 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.8 {d2[], d3[]}, [r2], lr + vst1.8 {q0}, [r0, :128]! + vld1.8 {d4[], d5[]}, [r2], lr + vst1.8 {q1}, [r12, :128]! + vld1.8 {d6[], d7[]}, [r2], lr + vst1.8 {q0}, [r0, :128]! + vst1.8 {q1}, [r12, :128]! + vst1.8 {q0}, [r0, :128]! + vst1.8 {q1}, [r12, :128]! + vst1.8 {q0}, [r0, :128], r1 + vst1.8 {q1}, [r12, :128], r1 + vst1.8 {q2}, [r0, :128]! + vst1.8 {q3}, [r12, :128]! + vst1.8 {q2}, [r0, :128]! + vst1.8 {q3}, [r12, :128]! + vst1.8 {q2}, [r0, :128]! + vst1.8 {q3}, [r12, :128]! + vst1.8 {q2}, [r0, :128], r1 + vst1.8 {q3}, [r12, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + clz r3, r3 + adr r5, L(ipred_dc_top_tbl) + sub r3, r3, #25 + ldr r3, [r5, r3, lsl #2] + add r2, r2, #1 + add r5, r5, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_top_tbl): + .word 640f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 320f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 160f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 80f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 40f - L(ipred_dc_top_tbl) + CONFIG_THUMB +40: + vld1.32 {d0[]}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #2 + vdup.8 d0, d0[0] +4: + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + subs r4, r4, #4 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt 4b + pop {r4-r5, pc} +80: + vld1.8 {d0}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #3 + vdup.8 d0, d0[0] +8: + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt 8b + pop {r4-r5, pc} +160: + vld1.8 {d0, d1}, [r2] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #4 + vdup.8 q0, d0[0] +16: + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + vld1.8 {d0, d1, d2, d3}, [r2] + vaddl.u8 q0, d0, d1 + vaddl.u8 q1, d2, d3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d4, q0, #5 + vdup.8 q0, d4[0] + vdup.8 q1, d4[0] +32: + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + vld1.8 {d0, d1, d2, d3}, [r2]! + vaddl.u8 q0, d0, d1 + vld1.8 {d4, d5, d6, d7}, [r2] + vaddl.u8 q1, d2, d3 + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q0, q1 + vadd.u16 q1, q2, q3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d18, q0, #6 + vdup.8 q0, d18[0] + vdup.8 q1, d18[0] + sub r1, r1, #32 +64: + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + sub r2, r2, r4 + clz r3, r3 + clz lr, r4 + sub lr, lr, #25 + adr r5, L(ipred_dc_left_tbl) + sub r3, r3, #20 + ldr r3, [r5, r3, lsl #2] + ldr lr, [r5, lr, lsl #2] + add r3, r5, r3 + add r5, r5, lr + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_left_tbl): + .word L(ipred_dc_left_h64) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h32) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h16) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h8) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h4) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w64) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w32) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w8) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w4) - L(ipred_dc_left_tbl) + CONFIG_THUMB + +L(ipred_dc_left_h4): + vld1.32 {d0[]}, [r2, :32] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #2 + vdup.8 q0, d0[0] + bx r3 +L(ipred_dc_left_w4): + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + subs r4, r4, #4 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt L(ipred_dc_left_w4) + pop {r4-r5, pc} +L(ipred_dc_left_h8): + vld1.8 {d0}, [r2, :64] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #3 + vdup.8 q0, d0[0] + bx r3 +L(ipred_dc_left_w8): + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt L(ipred_dc_left_w8) + pop {r4-r5, pc} +L(ipred_dc_left_h16): + vld1.8 {d0, d1}, [r2, :128] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #4 + vdup.8 q0, d0[0] + bx r3 +L(ipred_dc_left_w16): + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + bgt L(ipred_dc_left_w16) + pop {r4-r5, pc} +L(ipred_dc_left_h32): + vld1.8 {d0, d1, d2, d3}, [r2, :128] + vaddl.u8 q0, d0, d1 + vaddl.u8 q1, d2, d3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #5 + vdup.8 q0, d0[0] + bx r3 +L(ipred_dc_left_w32): + vmov.8 q1, q0 +1: + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 1b + pop {r4-r5, pc} +L(ipred_dc_left_h64): + vld1.8 {d0, d1, d2, d3}, [r2, :128]! + vld1.8 {d4, d5, d6, d7}, [r2, :128] + vaddl.u8 q0, d0, d1 + vaddl.u8 q1, d2, d3 + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q0, q1 + vadd.u16 q1, q2, q3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshrn.u16 d0, q0, #6 + vdup.8 q0, d0[0] + bx r3 +L(ipred_dc_left_w64): + vmov.8 q1, q0 + sub r1, r1, #32 +1: + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 1b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 + push {r4-r6, lr} + ldr r4, [sp, #16] + sub r2, r2, r4 + add lr, r3, r4 // width + height + clz r3, r3 + clz r12, r4 + vdup.16 q15, lr // width + height + adr r5, L(ipred_dc_tbl) + rbit lr, lr // rbit(width + height) + sub r3, r3, #20 // 25 leading bits, minus table offset 5 + sub r12, r12, #25 + clz lr, lr // ctz(width + height) + ldr r3, [r5, r3, lsl #2] + ldr r12, [r5, r12, lsl #2] + neg lr, lr // -ctz(width + height) + add r3, r5, r3 + add r5, r5, r12 + vshr.u16 q15, q15, #1 // (width + height) >> 1 + vdup.16 q14, lr // -ctz(width + height) + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_tbl): + .word L(ipred_dc_h64) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h32) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h16) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h8) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h4) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w64) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w32) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w8) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w4) - L(ipred_dc_tbl) + CONFIG_THUMB + +L(ipred_dc_h4): + vld1.32 {d0[]}, [r2, :32]! + vpaddl.u8 d0, d0 + add r2, r2, #1 + vpadd.u16 d0, d0 + bx r3 +L(ipred_dc_w4): + vld1.32 {d1[]}, [r2] + vadd.s16 d0, d0, d30 + vpaddl.u8 d1, d1 + vpadd.u16 d1, d1 + cmp r4, #4 + vadd.s16 d0, d0, d1 + vshl.u16 d0, d0, d28 + beq 1f + // h = 8/16 + movw lr, #(0x3334/2) + movw r5, #(0x5556/2) + cmp r4, #16 + it ne + movne lr, r5 + vdup.16 d30, lr + vqdmulh.s16 d0, d0, d30 +1: + vdup.8 d0, d0[0] +2: + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + subs r4, r4, #4 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[0]}, [r12, :32], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h8): + vld1.8 {d0}, [r2, :64]! + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + add r2, r2, #1 + vpadd.u16 d0, d0 + bx r3 +L(ipred_dc_w8): + vld1.8 {d2}, [r2] + vadd.s16 d0, d0, d30 + vpaddl.u8 d2, d2 + vpadd.u16 d2, d2 + vpadd.u16 d2, d2 + cmp r4, #8 + vadd.s16 d0, d0, d2 + vshl.u16 d0, d0, d28 + beq 1f + // h = 4/16/32 + cmp r4, #32 + movw lr, #(0x3334/2) + movw r5, #(0x5556/2) + it ne + movne lr, r5 + vdup.16 d24, lr + vqdmulh.s16 d0, d0, d24 +1: + vdup.8 d0, d0[0] +2: + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d0}, [r12, :64], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h16): + vld1.8 {d0, d1}, [r2, :128]! + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + add r2, r2, #1 + vpadd.u16 d0, d0 + bx r3 +L(ipred_dc_w16): + vld1.8 {d2, d3}, [r2] + vadd.s16 d0, d0, d30 + vaddl.u8 q1, d2, d3 + vadd.u16 d2, d2, d3 + vpadd.u16 d2, d2 + vpadd.u16 d2, d2 + cmp r4, #16 + vadd.s16 d0, d0, d2 + vshl.u16 d0, d0, d28 + beq 1f + // h = 4/8/32/64 + tst r4, #(32+16+8) // 16 added to make a consecutive bitmask + movw lr, #(0x3334/2) + movw r5, #(0x5556/2) + it ne + movne lr, r5 + vdup.16 d24, lr + vqdmulh.s16 d0, d0, d24 +1: + vdup.8 q0, d0[0] +2: + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1}, [r0, :128], r1 + vst1.8 {d0, d1}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h32): + vld1.8 {d0, d1, d2, d3}, [r2, :128]! + vaddl.u8 q0, d0, d1 + vaddl.u8 q1, d2, d3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + add r2, r2, #1 + vpadd.u16 d0, d0 + bx r3 +L(ipred_dc_w32): + vld1.8 {d2, d3, d4, d5}, [r2] + vadd.s16 d0, d0, d30 + vaddl.u8 q1, d2, d3 + vaddl.u8 q2, d4, d5 + vadd.u16 q1, q1, q2 + vadd.u16 d2, d2, d3 + vpadd.u16 d2, d2 + vpadd.u16 d2, d2 + cmp r4, #32 + vadd.s16 d0, d0, d2 + vshl.u16 d4, d0, d28 + beq 1f + // h = 8/16/64 + cmp r4, #8 + movw lr, #(0x3334/2) + movw r5, #(0x5556/2) + it ne + movne lr, r5 + vdup.16 d24, lr + vqdmulh.s16 d4, d4, d24 +1: + vdup.8 q0, d4[0] + vdup.8 q1, d4[0] +2: + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h64): + vld1.8 {d0, d1, d2, d3}, [r2, :128]! + vaddl.u8 q0, d0, d1 + vld1.8 {d4, d5, d6, d7}, [r2, :128]! + vaddl.u8 q1, d2, d3 + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q0, q1 + vadd.u16 q1, q2, q3 + vadd.u16 q0, q0, q1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + add r2, r2, #1 + vpadd.u16 d0, d0 + bx r3 +L(ipred_dc_w64): + vld1.8 {d2, d3, d4, d5}, [r2]! + vadd.s16 d0, d0, d30 + vaddl.u8 q2, d4, d5 + vaddl.u8 q1, d2, d3 + vadd.u16 d4, d4, d5 + vadd.u16 d2, d2, d3 + vld1.8 {d16, d17, d18, d19}, [r2] + vpadd.u16 d4, d4 + vpadd.u16 d2, d2 + vpadd.u16 d4, d4 + vpadd.u16 d2, d2 + vaddl.u8 q8, d16, d17 + vaddl.u8 q9, d18, d19 + vadd.u16 d16, d16, d17 + vadd.u16 d18, d18, d19 + vpadd.u16 d16, d16 + vpadd.u16 d18, d18 + vpadd.u16 d16, d16 + vpadd.u16 d18, d18 + vadd.u16 d2, d2, d4 + vadd.u16 d3, d16, d18 + cmp r4, #64 + vadd.s16 d0, d0, d2 + vadd.s16 d0, d0, d3 + vshl.u16 d18, d0, d28 + beq 1f + // h = 16/32 + movw lr, #(0x5556/2) + movt lr, #(0x3334/2) + and r5, r4, #31 + lsr lr, lr, r5 + vdup.16 d30, lr + vqdmulh.s16 d18, d18, d30 +1: + sub r1, r1, #32 + vdup.8 q0, d18[0] + vdup.8 q1, d18[0] +2: + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.8 {d0, d1, d2, d3}, [r0, :128]! + vst1.8 {d0, d1, d2, d3}, [r12, :128]! + vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} +endfunc + +// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + clz lr, r3 + adr r5, L(ipred_paeth_tbl) + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[], d5[]}, [r2] + add r8, r2, #1 + sub r2, r2, #4 + add r5, r5, lr + mov r7, #-4 + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_paeth_tbl): + .word 640f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_paeth_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[], d7[]}, [r8] + vsubl.u8 q8, d6, d4 // top - topleft +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 + vzip.32 d0, d1 + vzip.32 d2, d3 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d2 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vmov d1, d2 + vabd.u8 q10, q3, q9 // tdiff + vabd.u8 q11, q2, q9 // tldiff + vabd.u8 q9, q0, q9 // ldiff + vmin.u8 q12, q10, q11 // min(tdiff, tldiff) + vcge.u8 q10, q11, q10 // tldiff >= tdiff + vcge.u8 q9, q12, q9 // min(tdiff, tldiff) >= ldiff + vbsl q10, q3, q2 // tdiff <= tldiff ? top : topleft + vbit q10, q0, q9 // ldiff <= min ? left : ... + vst1.32 {d21[1]}, [r0, :32], r1 + vst1.32 {d21[0]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d20[1]}, [r0, :32], r1 + vst1.32 {d20[0]}, [r6, :32], r1 + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d6}, [r8] + vsubl.u8 q8, d6, d4 // top - topleft + vmov d7, d6 +8: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d1 + vaddw.u8 q11, q8, d2 + vaddw.u8 q12, q8, d3 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vqmovun.s16 d20, q11 + vqmovun.s16 d21, q12 + vabd.u8 q11, q3, q9 // tdiff + vabd.u8 q12, q3, q10 + vabd.u8 q13, q2, q9 // tldiff + vabd.u8 q14, q2, q10 + vabd.u8 q10, q1, q10 // ldiff + vabd.u8 q9, q0, q9 + vmin.u8 q15, q12, q14 // min(tdiff, tldiff) + vcge.u8 q12, q14, q12 // tldiff >= tdiff + vmin.u8 q14, q11, q13 // min(tdiff, tldiff) + vcge.u8 q11, q13, q11 // tldiff >= tdiff + vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff + vcge.u8 q9, q14, q9 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + vst1.8 {d25}, [r0, :64], r1 + vst1.8 {d24}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d23}, [r0, :64], r1 + vst1.8 {d22}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: +640: + vld1.8 {d6}, [r8]! + mov r12, r3 + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 +1: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 +2: + vsubl.u8 q8, d6, d4 // top - topleft + vmov d7, d6 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d1 + vaddw.u8 q11, q8, d2 + vaddw.u8 q12, q8, d3 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vqmovun.s16 d20, q11 + vqmovun.s16 d21, q12 + vabd.u8 q11, q3, q9 // tdiff + vabd.u8 q12, q3, q10 + vabd.u8 q13, q2, q9 // tldiff + vabd.u8 q14, q2, q10 + vabd.u8 q10, q1, q10 // ldiff + vabd.u8 q9, q0, q9 + vmin.u8 q15, q12, q14 // min(tdiff, tldiff) + vcge.u8 q12, q14, q12 // tldiff >= tdiff + vmin.u8 q14, q11, q13 // min(tdiff, tldiff) + vcge.u8 q11, q13, q11 // tldiff >= tdiff + vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff + vcge.u8 q9, q14, q9 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + subs r3, r3, #8 + vst1.8 {d25}, [r0, :64]! + vst1.8 {d24}, [r6, :64]! + vst1.8 {d23}, [r5, :64]! + vst1.8 {d22}, [lr, :64]! + ble 8f + vld1.8 {d6}, [r8]! + b 2b +8: + subs r4, r4, #4 + ble 9f + // End of horizontal loop, move pointers to next four rows + sub r8, r8, r12 + add r0, r0, r1 + add r6, r6, r1 + vld1.8 {d6}, [r8]! + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + pop {r4-r8, pc} +endfunc + +// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_8bpc_neon, export=1 + push {r4-r10, lr} + ldr r4, [sp, #32] + movrel r10, X(sm_weights) + add r12, r10, r4 + add r10, r10, r3 + clz r9, r3 + adr r5, L(ipred_smooth_tbl) + sub lr, r2, r4 + sub r9, r9, #25 + ldr r9, [r5, r9, lsl #2] + vld1.8 {d4[]}, [lr] // bottom + add r8, r2, #1 + add r5, r5, r9 + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_tbl): + .word 640f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_tbl) + CONFIG_THUMB + +40: + vld1.32 {d16[]}, [r8] // top + vld1.32 {d18[]}, [r10, :32] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vdup.8 q3, d16[3] // right + vsubl.u8 q8, d16, d4 // top-bottom + vmovl.u8 q9, d18 // weights_hor +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left + vld4.8 {d20[], d21[], d22[], d23[]}, [r12, :32]! // weights_ver + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vzip.32 d1, d0 // left, flipped + vzip.32 d3, d2 + vzip.32 d20, d21 // weights_ver + vzip.32 d22, d23 + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q0, d1, d6 // left-right + vsubl.u8 q1, d3, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmla.i16 q12, q1, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q0, q9 // (left flipped) + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vst1.32 {d24[0]}, [r0, :32], r1 + vst1.32 {d24[1]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d25[0]}, [r0, :32], r1 + vst1.32 {d25[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r10, pc} +80: + vld1.8 {d16}, [r8] // top + vld1.8 {d18}, [r10, :64] // weights_hor + sub r2, r2, #2 + mov r7, #-2 + vdup.8 q3, d16[7] // right + vsubl.u8 q8, d16, d4 // top-bottom + vmovl.u8 q9, d18 // weights_hor +8: + vld2.8 {d0[], d1[]}, [r2, :16], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q1, d0, d6 // left-right (left flipped) + vsubl.u8 q0, d1, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q1, q9 + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + subs r4, r4, #2 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + bgt 8b + pop {r4-r10, pc} +160: +320: +640: + add lr, r2, r3 + sub r2, r2, #2 + mov r7, #-2 + vld1.8 {d6[], d7[]}, [lr] // right + sub r1, r1, r3 + mov r9, r3 + +1: + vld2.8 {d0[], d1[]}, [r2, :16], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vsubl.u8 q1, d0, d6 // left-right (left flipped) + vsubl.u8 q0, d1, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 +2: + vld1.8 {d16}, [r8]! // top + vld1.8 {d18}, [r10, :64]! // weights_hor + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vmovl.u8 q9, d18 // weights_hor + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q8, d16, d4 // top-bottom + vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q1, q9 + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + subs r3, r3, #8 + vst1.8 {d24}, [r0, :64]! + vst1.8 {d25}, [r6, :64]! + bgt 2b + subs r4, r4, #2 + ble 9f + sub r8, r8, r9 + sub r10, r10, r9 + add r0, r0, r1 + add r6, r6, r1 + mov r3, r9 + b 1b +9: + pop {r4-r10, pc} +endfunc + +// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_8bpc_neon, export=1 + push {r4-r7, lr} + ldr r4, [sp, #20] + movrel r7, X(sm_weights) + add r7, r7, r4 + clz lr, r3 + adr r5, L(ipred_smooth_v_tbl) + sub r12, r2, r4 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[]}, [r12] // bottom + add r2, r2, #1 + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_v_tbl): + .word 640f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[]}, [r2] // top + vsubl.u8 q3, d6, d4 // top-bottom +4: + vld4.8 {d16[], d17[], d18[], d19[]}, [r7, :32]! // weights_ver + vshll.i8 q10, d4, #8 // bottom*256 + vshll.i8 q11, d4, #8 + vzip.32 d16, d17 // weights_ver + vzip.32 d18, d19 + vmovl.u8 q8, d16 // weights_ver + vmovl.u8 q9, d18 + subs r4, r4, #4 + vmla.i16 q10, q3, q8 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q11, q3, q9 + vrshrn.i16 d20, q10, #8 + vrshrn.i16 d21, q11, #8 + vst1.32 {d20[0]}, [r0, :32], r1 + vst1.32 {d20[1]}, [r6, :32], r1 + vst1.32 {d21[0]}, [r0, :32], r1 + vst1.32 {d21[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r7, pc} +80: + vld1.8 {d6}, [r2] // top + vsubl.u8 q3, d6, d4 // top-bottom +8: + vld4.8 {d16[], d18[], d20[], d22[]}, [r7, :32]! // weights_ver + vshll.i8 q12, d4, #8 // bottom*256 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmovl.u8 q8, d16 // weights_ver + vmovl.u8 q9, d18 + vmovl.u8 q10, d20 + vmovl.u8 q11, d22 + vmla.i16 q12, q3, q8 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q13, q3, q9 + vmla.i16 q14, q3, q10 + vmla.i16 q15, q3, q11 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vrshrn.i16 d26, q14, #8 + vrshrn.i16 d27, q15, #8 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d26}, [r0, :64], r1 + vst1.8 {d27}, [r6, :64], r1 + bgt 8b + pop {r4-r7, pc} +160: +320: +640: + vpush {q4-q7} + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 + mov r12, r3 + +1: + vld4.8 {d8[], d10[], d12[], d14[]}, [r7, :32]! // weights_ver + vmovl.u8 q4, d8 // weights_ver + vmovl.u8 q5, d10 + vmovl.u8 q6, d12 + vmovl.u8 q7, d14 +2: + vld1.8 {q3}, [r2]! // top + vshll.i8 q8, d4, #8 // bottom*256 + vshll.i8 q9, d4, #8 + vshll.i8 q10, d4, #8 + vshll.i8 q11, d4, #8 + vsubl.u8 q0, d6, d4 // top-bottom + vsubl.u8 q1, d7, d4 + vshll.i8 q12, d4, #8 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmla.i16 q8, q0, q4 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q9, q1, q4 + vmla.i16 q10, q0, q5 + vmla.i16 q11, q1, q5 + vmla.i16 q12, q0, q6 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q13, q1, q6 + vmla.i16 q14, q0, q7 + vmla.i16 q15, q1, q7 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vrshrn.i16 d18, q10, #8 + vrshrn.i16 d19, q11, #8 + vrshrn.i16 d20, q12, #8 + vrshrn.i16 d21, q13, #8 + vrshrn.i16 d22, q14, #8 + vrshrn.i16 d23, q15, #8 + subs r3, r3, #16 + vst1.8 {q8}, [r0, :128]! + vst1.8 {q9}, [r6, :128]! + vst1.8 {q10}, [r5, :128]! + vst1.8 {q11}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r2, r2, r12 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r7, pc} +endfunc + +// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + movrel r8, X(sm_weights) + add r8, r8, r3 + clz lr, r3 + adr r5, L(ipred_smooth_h_tbl) + add r12, r2, r3 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[]}, [r12] // right + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_h_tbl): + .word 640f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[]}, [r8, :32] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vmovl.u8 q3, d6 // weights_hor +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left + vshll.i8 q8, d4, #8 // right*256 + vshll.i8 q9, d4, #8 + vzip.32 d3, d2 // left, flipped + vzip.32 d1, d0 + vsubl.u8 q1, d3, d4 // left-right + vsubl.u8 q0, d1, d4 + subs r4, r4, #4 + vmla.i16 q8, q1, q3 // right*256 + (left-right)*weights_hor + vmla.i16 q9, q0, q3 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vst1.32 {d16[0]}, [r0, :32], r1 + vst1.32 {d16[1]}, [r6, :32], r1 + vst1.32 {d17[0]}, [r0, :32], r1 + vst1.32 {d17[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d6}, [r8, :64] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vmovl.u8 q3, d6 // weights_hor +8: + vld4.8 {d16[], d18[], d20[], d22[]}, [r2, :32], r7 // left + vshll.i8 q12, d4, #8 // right*256 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vsubl.u8 q11, d22, d4 // left-right + vsubl.u8 q10, d20, d4 + vsubl.u8 q9, d18, d4 + vsubl.u8 q8, d16, d4 + vmla.i16 q12, q11, q3 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q10, q3 // (left flipped) + vmla.i16 q14, q9, q3 + vmla.i16 q15, q8, q3 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vrshrn.i16 d26, q14, #8 + vrshrn.i16 d27, q15, #8 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d26}, [r0, :64], r1 + vst1.8 {d27}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: +640: + vpush {q4-q7} + sub r2, r2, #4 + mov r7, #-4 + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 + mov r12, r3 + +1: + vld4.8 {d8[], d10[], d12[], d14[]}, [r2, :32], r7 // left + vsubl.u8 q4, d8, d4 // left-right + vsubl.u8 q5, d10, d4 + vsubl.u8 q6, d12, d4 + vsubl.u8 q7, d14, d4 +2: + vld1.8 {q1}, [r8, :128]! // weights_hor + vshll.i8 q8, d4, #8 // right*256 + vshll.i8 q9, d4, #8 + vshll.i8 q10, d4, #8 + vshll.i8 q11, d4, #8 + vmovl.u8 q0, d2 // weights_hor + vmovl.u8 q1, d3 + vshll.i8 q12, d4, #8 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmla.i16 q8, q7, q0 // right*256 + (left-right)*weights_hor + vmla.i16 q9, q7, q1 // (left flipped) + vmla.i16 q10, q6, q0 + vmla.i16 q11, q6, q1 + vmla.i16 q12, q5, q0 + vmla.i16 q13, q5, q1 + vmla.i16 q14, q4, q0 + vmla.i16 q15, q4, q1 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vrshrn.i16 d18, q10, #8 + vrshrn.i16 d19, q11, #8 + vrshrn.i16 d20, q12, #8 + vrshrn.i16 d21, q13, #8 + vrshrn.i16 d22, q14, #8 + vrshrn.i16 d23, q15, #8 + subs r3, r3, #16 + vst1.8 {q8}, [r0, :128]! + vst1.8 {q9}, [r6, :128]! + vst1.8 {q10}, [r5, :128]! + vst1.8 {q11}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r8, r8, r12 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r8, pc} +endfunc + +// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height); +function ipred_filter_8bpc_neon, export=1 + push {r4-r8, lr} + movw r12, #511 + ldrd r4, r5, [sp, #24] + and r5, r5, r12 // 511 + movrel r6, X(filter_intra_taps) + lsl r5, r5, #6 + add r6, r6, r5 + vld1.8 {d20, d21, d22, d23}, [r6, :128]! + clz lr, r3 + adr r5, L(ipred_filter_tbl) + vld1.8 {d27, d28, d29}, [r6, :64] + sub lr, lr, #26 + ldr lr, [r5, lr, lsl #2] + vmovl.s8 q8, d20 + vmovl.s8 q9, d21 + add r5, r5, lr + vmovl.s8 q10, d22 + vmovl.s8 q11, d23 + add r6, r0, r1 + lsl r1, r1, #1 + vmovl.s8 q12, d27 + vmovl.s8 q13, d28 + vmovl.s8 q14, d29 + add r8, r2, #1 + sub r2, r2, #2 + mov r7, #-2 + bx r5 + + .align 2 +L(ipred_filter_tbl): + .word 320f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 160f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 80f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 40f - L(ipred_filter_tbl) + CONFIG_THUMB + +40: + vld1.32 {d0[]}, [r8] // top (0-3) + vmovl.u8 q0, d0 // top (0-3) +4: + vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmovl.u8 q1, d2 // left (0-1) + topleft (2) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vqrshrun.s16 d4, q2, #4 + subs r4, r4, #2 + vst1.32 {d4[0]}, [r0, :32], r1 + vmovl.u8 q0, d4 + vst1.32 {d4[1]}, [r6, :32], r1 + vmov d0, d1 // move top from [4-7] to [0-3] + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d0}, [r8] // top (0-7) + vmovl.u8 q0, d0 // top (0-7) +8: + vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmovl.u8 q1, d2 // left (0-1) + topleft (2) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vmul.i16 q3, q9, d1[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d1[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d1[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d4, q2, #4 + vmovl.u8 q1, d4 // first block, in 16 bit + vmla.i16 q3, q12, d1[3] // p4(top[3]) * filter(4) + vmla.i16 q3, q8, d0[3] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d2[3] // p5(left[0]) * filter(5) + vmla.i16 q3, q14, d3[3] // p6(left[1]) * filter(6) + vqrshrun.s16 d5, q3, #4 + vzip.32 d4, d5 + subs r4, r4, #2 + vst1.8 {d4}, [r0, :64], r1 + vmovl.u8 q0, d5 + vst1.8 {d5}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: + vpush {q4-q5} + sub r1, r1, r3 + mov lr, r3 + +1: + vld1.32 {d0[]}, [r2], r7 // left (0-1) + topleft (2) + vmovl.u8 q0, d0 // left (0-1) + topleft (2) +2: + vld1.8 {q2}, [r8]! // top(0-15) + vmul.i16 q3, q8, d0[2] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d0[1] // p5(left[0]) * filter(5) + vmovl.u8 q1, d4 // top(0-7) + vmovl.u8 q2, d5 // top(8-15) + vmla.i16 q3, q14, d0[0] // p6(left[1]) * filter(6) + vmla.i16 q3, q9, d2[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d2[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d2[2] // p3(top[2]) * filter(3) + vmla.i16 q3, q12, d2[3] // p4(top[3]) * filter(4) + + vmul.i16 q4, q9, d3[0] // p1(top[0]) * filter(1) + vmla.i16 q4, q10, d3[1] // p2(top[1]) * filter(2) + vmla.i16 q4, q11, d3[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d6, q3, #4 + vmovl.u8 q0, d6 // first block, in 16 bit + vmla.i16 q4, q12, d3[3] // p4(top[3]) * filter(4) + vmla.i16 q4, q8, d2[3] // p0(topleft) * filter(0) + vmla.i16 q4, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q4, q14, d1[3] // p6(left[1]) * filter(6) + + vmul.i16 q5, q9, d4[0] // p1(top[0]) * filter(1) + vmla.i16 q5, q10, d4[1] // p2(top[1]) * filter(2) + vmla.i16 q5, q11, d4[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d7, q4, #4 + vmovl.u8 q0, d7 // second block, in 16 bit + vmla.i16 q5, q12, d4[3] // p4(top[3]) * filter(4) + vmla.i16 q5, q8, d3[3] // p0(topleft) * filter(0) + vmla.i16 q5, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q5, q14, d1[3] // p6(left[1]) * filter(6) + + vmul.i16 q15, q9, d5[0] // p1(top[0]) * filter(1) + vmla.i16 q15, q10, d5[1] // p2(top[1]) * filter(2) + vmla.i16 q15, q11, d5[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d8, q5, #4 + vmovl.u8 q0, d8 // third block, in 16 bit + vmov.u8 r12, d5[6] + vmla.i16 q15, q12, d5[3] // p4(top[3]) * filter(4) + vmla.i16 q15, q8, d4[3] // p0(topleft) * filter(0) + vmla.i16 q15, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q15, q14, d1[3] // p6(left[1]) * filter(6) + vmov.8 d0[4], r12 + + subs r3, r3, #16 + vqrshrun.s16 d9, q15, #4 + + vst4.32 {d6[0], d7[0], d8[0], d9[0]}, [r0, :128]! + vst4.32 {d6[1], d7[1], d8[1], d9[1]}, [r6, :128]! + ble 8f + vmov.u8 r12, d9[7] + vmov.8 d0[0], r12 + vmov.u8 r12, d9[3] + vmov.8 d0[2], r12 + b 2b +8: + subs r4, r4, #2 + + ble 9f + sub r8, r6, lr + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + b 1b +9: + vpop {q4-q5} + pop {r4-r8, pc} +endfunc + +// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_8bpc_neon, export=1 + push {r4-r5, lr} + ldrd r4, r5, [sp, #12] + vld1.16 {q0}, [r2, :128] + clz lr, r4 + adr r12, L(pal_pred_tbl) + sub lr, lr, #25 + ldr lr, [r12, lr, lsl #2] + vmovn.i16 d0, q0 + add r12, r12, lr + add r2, r0, r1 + bx r12 + + .align 2 +L(pal_pred_tbl): + .word 640f - L(pal_pred_tbl) + CONFIG_THUMB + .word 320f - L(pal_pred_tbl) + CONFIG_THUMB + .word 160f - L(pal_pred_tbl) + CONFIG_THUMB + .word 80f - L(pal_pred_tbl) + CONFIG_THUMB + .word 40f - L(pal_pred_tbl) + CONFIG_THUMB + +40: + lsl r1, r1, #1 +4: + vld1.8 {q1}, [r3, :128]! + subs r5, r5, #4 + vtbl.8 d2, {d0}, d2 + vtbl.8 d3, {d0}, d3 + vst1.32 {d2[0]}, [r0, :32], r1 + vst1.32 {d2[1]}, [r2, :32], r1 + vst1.32 {d3[0]}, [r0, :32], r1 + vst1.32 {d3[1]}, [r2, :32], r1 + bgt 4b + pop {r4-r5, pc} +80: + lsl r1, r1, #1 +8: + vld1.8 {q1, q2}, [r3, :128]! + subs r5, r5, #4 + vtbl.8 d2, {d0}, d2 + vtbl.8 d3, {d0}, d3 + vst1.8 {d2}, [r0, :64], r1 + vtbl.8 d4, {d0}, d4 + vst1.8 {d3}, [r2, :64], r1 + vtbl.8 d5, {d0}, d5 + vst1.8 {d4}, [r0, :64], r1 + vst1.8 {d5}, [r2, :64], r1 + bgt 8b + pop {r4-r5, pc} +160: + lsl r1, r1, #1 +16: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #4 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8}, [r0, :128], r1 + vtbl.8 d22, {d0}, d22 + vst1.8 {q9}, [r2, :128], r1 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10}, [r0, :128], r1 + vst1.8 {q11}, [r2, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + lsl r1, r1, #1 +32: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #2 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8, q9}, [r0, :128], r1 + vtbl.8 d22, {d0}, d22 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10, q11}, [r2, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + sub r1, r1, #32 +64: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #1 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8, q9}, [r0, :128]! + vtbl.8 d22, {d0}, d22 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10, q11}, [r0, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_128_8bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz lr, r3 + adr r12, L(ipred_cfl_128_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vmov.i16 q0, #128 // dc + vdup.i16 q1, r6 // alpha + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r12 + + .align 2 +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w8) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w4) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + +L(ipred_cfl_splat_w4): + vld1.16 {q2, q3}, [r5, :128]! + vmul.i16 q2, q2, q1 // diff = ac * alpha + vmul.i16 q3, q3, q1 + vshr.s16 q8, q2, #15 // sign = diff >> 15 + vshr.s16 q9, q3, #15 + vadd.i16 q2, q2, q8 // diff + sign + vadd.i16 q3, q3, q9 + vrshr.s16 q2, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q3, q3, #6 + vadd.i16 q2, q2, q0 // dc + apply_sign() + vadd.i16 q3, q3, q0 + vqmovun.s16 d4, q2 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d5, q3 + vst1.32 {d4[0]}, [r0, :32], r1 + vst1.32 {d4[1]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d5[0]}, [r0, :32], r1 + vst1.32 {d5[1]}, [r6, :32], r1 + bgt L(ipred_cfl_splat_w4) + pop {r4-r8, pc} +L(ipred_cfl_splat_w8): + vld1.16 {q8, q9}, [r5, :128]! + vld1.16 {q10, q11}, [r5, :128]! + vmul.i16 q8, q8, q1 // diff = ac * alpha + vmul.i16 q9, q9, q1 + vmul.i16 q10, q10, q1 + vmul.i16 q11, q11, q1 + vshr.s16 q12, q8, #15 // sign = diff >> 15 + vshr.s16 q13, q9, #15 + vshr.s16 q14, q10, #15 + vshr.s16 q15, q11, #15 + vadd.i16 q8, q8, q12 // diff + sign + vadd.i16 q9, q9, q13 + vadd.i16 q10, q10, q14 + vadd.i16 q11, q11, q15 + vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q9, q9, #6 + vrshr.s16 q10, q10, #6 + vrshr.s16 q11, q11, #6 + vadd.i16 q8, q8, q0 // dc + apply_sign() + vadd.i16 q9, q9, q0 + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q0 + vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d17, q9 + vqmovun.s16 d18, q10 + vqmovun.s16 d19, q11 + vst1.8 {d16}, [r0, :64], r1 + vst1.8 {d17}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d18}, [r0, :64], r1 + vst1.8 {d19}, [r6, :64], r1 + bgt L(ipred_cfl_splat_w8) + pop {r4-r8, pc} +L(ipred_cfl_splat_w16): + add r12, r5, r3, lsl #1 + sub r1, r1, r3 + mov lr, r3 +1: + vld1.16 {q8, q9}, [r5, :128]! + vmul.i16 q8, q8, q1 // diff = ac * alpha + vld1.16 {q10, q11}, [r12, :128]! + vmul.i16 q9, q9, q1 + vmul.i16 q10, q10, q1 + vmul.i16 q11, q11, q1 + vshr.s16 q12, q8, #15 // sign = diff >> 15 + vshr.s16 q13, q9, #15 + vshr.s16 q14, q10, #15 + vshr.s16 q15, q11, #15 + vadd.i16 q8, q8, q12 // diff + sign + vadd.i16 q9, q9, q13 + vadd.i16 q10, q10, q14 + vadd.i16 q11, q11, q15 + vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q9, q9, #6 + vrshr.s16 q10, q10, #6 + vrshr.s16 q11, q11, #6 + vadd.i16 q8, q8, q0 // dc + apply_sign() + vadd.i16 q9, q9, q0 + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q0 + vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d17, q9 + vqmovun.s16 d18, q10 + vqmovun.s16 d19, q11 + subs r3, r3, #16 + vst1.16 {q8}, [r0, :128]! + vst1.16 {q9}, [r6, :128]! + bgt 1b + subs r4, r4, #2 + add r5, r5, lr, lsl #1 + add r12, r12, lr, lsl #1 + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + bgt 1b + pop {r4-r8, pc} +endfunc + +// void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_top_8bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz lr, r3 + adr r12, L(ipred_cfl_top_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vdup.16 q1, r6 // alpha + add r2, r2, #1 + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r12 + + .align 2 +L(ipred_cfl_top_tbl): + .word 32f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 16f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 8f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 4f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + +4: + vld1.32 {d0[]}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) +8: + vld1.8 {d0}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) +16: + vld1.8 {q0}, [r2] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +32: + vld1.8 {q2, q3}, [r2] + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q2, q3 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #5 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_left_8bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + sub r2, r2, r4 + clz lr, r3 + clz r8, r4 + adr r12, L(ipred_cfl_splat_tbl) + adr r7, L(ipred_cfl_left_tbl) + sub lr, lr, #26 + sub r8, r8, #26 + ldr lr, [r12, lr, lsl #2] + ldr r8, [r7, r8, lsl #2] + vdup.16 q1, r6 // alpha + add r12, r12, lr + add r7, r7, r8 + add r6, r0, r1 + lsl r1, r1, #1 + bx r7 + + .align 2 +L(ipred_cfl_left_tbl): + .word L(ipred_cfl_left_h32) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h16) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h8) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h4) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + +L(ipred_cfl_left_h4): + vld1.32 {d0[]}, [r2, :32] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h8): + vld1.8 {d0}, [r2, :64] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h16): + vld1.8 {q0}, [r2, :128] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h32): + vld1.8 {q2, q3}, [r2, :128] + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q2, q3 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #5 + vdup.16 q0, d0[0] + bx r12 +endfunc + +// void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_8bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + sub r2, r2, r4 + add r8, r3, r4 // width + height + vdup.16 q1, r6 // alpha + clz lr, r3 + clz r6, r4 + vdup.16 d16, r8 // width + height + adr r7, L(ipred_cfl_tbl) + rbit r8, r8 // rbit(width + height) + sub lr, lr, #22 // 26 leading bits, minus table offset 4 + sub r6, r6, #26 + clz r8, r8 // ctz(width + height) + ldr lr, [r7, lr, lsl #2] + ldr r6, [r7, r6, lsl #2] + neg r8, r8 // -ctz(width + height) + add r12, r7, lr + add r7, r7, r6 + vshr.u16 d16, d16, #1 // (width + height) >> 1 + vdup.16 d17, r8 // -ctz(width + height) + add r6, r0, r1 + lsl r1, r1, #1 + bx r7 + + .align 2 +L(ipred_cfl_tbl): + .word L(ipred_cfl_h32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h4) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w4) - L(ipred_cfl_tbl) + CONFIG_THUMB + +L(ipred_cfl_h4): + vld1.32 {d0[]}, [r2, :32]! + vpaddl.u8 d0, d0 + add r2, r2, #1 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w4): + vld1.32 {d1[]}, [r2] + vadd.i16 d0, d0, d16 + vpaddl.u8 d1, d1 + vpadd.u16 d1, d1 + cmp r4, #4 + vadd.i16 d0, d0, d1 + vshl.u16 d0, d0, d17 + beq 1f + // h = 8/16 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + cmp r4, #16 + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + vld1.8 {d0}, [r2, :64]! + vpaddl.u8 d0, d0 + vpadd.i16 d0, d0 + add r2, r2, #1 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w8): + vld1.8 {d1}, [r2] + vadd.i16 d0, d0, d16 + vpaddl.u8 d1, d1 + vpadd.i16 d1, d1 + vpadd.i16 d1, d1 + cmp r4, #8 + vadd.i16 d0, d0, d1 + vshl.u16 d0, d0, d17 + beq 1f + // h = 4/16/32 + cmp r4, #32 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + vld1.8 {q0}, [r2, :128]! + vaddl.u8 q0, d0, d1 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0 + add r2, r2, #1 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w16): + vld1.8 {q2}, [r2] + vadd.i16 d0, d0, d16 + vaddl.u8 q2, d4, d5 + vadd.i16 d4, d4, d5 + vpadd.i16 d4, d4 + vpadd.i16 d4, d4 + cmp r4, #16 + vadd.i16 d0, d0, d4 + vshl.u16 d0, d0, d17 + beq 1f + // h = 4/8/32/64 + tst r4, #(32+16+8) // 16 added to make a consecutive bitmask + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + vld1.8 {q2, q3}, [r2, :128]! + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.i16 q0, q2, q3 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0 + add r2, r2, #1 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w32): + vld1.8 {q2, q3}, [r2] + vadd.i16 d0, d0, d16 + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.i16 q2, q2, q3 + vadd.i16 d4, d4, d5 + vpadd.i16 d4, d4 + vpadd.i16 d4, d4 + cmp r4, #32 + vadd.i16 d0, d0, d4 + vshl.u16 d0, d0, d17 + beq 1f + // h = 8/16/64 + cmp r4, #8 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_8bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_420_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_420_tbl): + .word L(ipred_cfl_ac_420_w16) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w8) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w4) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w4): +1: // Copy and subsample input + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d2}, [r12, :64], r2 + vld1.8 {d1}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vadd.i16 q0, q0, q1 + vshl.i16 q0, q0, #1 + subs r8, r8, #2 + vst1.16 {q0}, [r0, :128]! + vadd.i16 q8, q8, q0 + bgt 1b + cmp r4, #0 + vmov d0, d1 + vmov d2, d1 + vmov d3, d1 +L(ipred_cfl_ac_420_w4_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q8, q8, q1 + bgt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + vadd.i16 q0, q8, q9 + vadd.i16 q1, q10, q11 + vpaddl.u16 q0, q0 + vpaddl.u16 q1, q1 + vadd.i32 q0, q1 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d0, d0 // sum + sub r0, r0, r6, lsl #3 + vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz + vdup.16 q8, d16[0] +L(ipred_cfl_ac_420_w4_subtract_dc): +6: // Subtract dc from ac + vld1.16 {q0, q1}, [r0, :128] + subs r6, r6, #4 + vsub.i16 q0, q0, q8 + vsub.i16 q1, q1, q8 + vst1.16 {q0, q1}, [r0, :128]! + bgt 6b + pop {r4-r8, pc} + +L(ipred_cfl_ac_420_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q2, #1 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov q0, q1 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.16 {d0}, [r1, :64], r2 + vld1.16 {d2}, [r12, :64], r2 + vld1.16 {d1}, [r1, :64], r2 + vld1.16 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vadd.i16 q0, q0, q1 + vshl.i16 q0, q0, #1 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov q0, q1 + +L(ipred_cfl_ac_420_w8_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl r6, r6, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + adr r7, L(ipred_cfl_ac_420_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_420_w16_tbl): + .word L(ipred_cfl_ac_420_w16_wpad0) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad1) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad2) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad3) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w16_wpad0): +1: // Copy and subsample input, without padding + vld1.8 {q0, q1}, [r1, :128], r2 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q12, q13}, [r1, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q12, q12 + vpaddl.u8 q13, q13 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q12, q12, q2 + vadd.i16 q13, q13, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q1, #1 + vshl.i16 q2, q12, #1 + vshl.i16 q3, q13, #1 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): +1: // Copy and subsample input, padding 4 + vldr d2, [r1, #16] + vld1.8 {q0}, [r1, :128], r2 + vldr d6, [r12, #16] + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 d2, d2 + vldr d26, [r1, #16] + vpaddl.u8 q0, q0 + vld1.8 {q12}, [r1, :128], r2 + vpaddl.u8 d6, d6 + vldr d30, [r12, #16] + vpaddl.u8 q2, q2 + vld1.8 {q14}, [r12, :128], r2 + vpaddl.u8 d26, d26 + vpaddl.u8 q12, q12 + vpaddl.u8 d30, d30 + vpaddl.u8 q14, q14 + vadd.i16 d2, d2, d6 + vadd.i16 q0, q0, q2 + vadd.i16 d26, d26, d30 + vadd.i16 q12, q12, q14 + vshl.i16 d2, d2, #1 + vshl.i16 q0, q0, #1 + vshl.i16 d6, d26, #1 + vshl.i16 q2, q12, #1 + vdup.16 d3, d2[3] + vdup.16 d7, d6[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q2, q2, #1 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d4}, [r1, :64], r2 + vpaddl.u8 q0, q0 + vld1.8 {d5}, [r12, :64], r2 + vpaddl.u8 q2, q2 + vadd.i16 d0, d0, d1 + vadd.i16 d4, d4, d5 + vshl.i16 d0, d0, #1 + vshl.i16 d4, d4, #1 + vdup.16 q1, d0[3] + vdup.16 q3, d4[3] + vdup.16 d1, d0[3] + vdup.16 d5, d4[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl r6, r6, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) +endfunc + +// void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_8bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_422_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_422_tbl): + .word L(ipred_cfl_ac_422_w16) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w8) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w4) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w4): +1: // Copy and subsample input + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d2}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q2, #2 + vshl.i16 q3, q3, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d2}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vdup.16 d7, d3[3] + vmov d6, d3 + vdup.16 d5, d2[3] + vmov d4, d2 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + adr r7, L(ipred_cfl_ac_422_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_422_w16_tbl): + .word L(ipred_cfl_ac_422_w16_wpad0) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad1) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad2) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad3) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w16_wpad0): +1: // Copy and subsample input, without padding + vld1.8 {q0, q1}, [r1, :128], r2 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q2, #2 + vshl.i16 q3, q3, #2 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): +1: // Copy and subsample input, padding 4 + vldr d2, [r1, #16] + vld1.8 {q0}, [r1, :128], r2 + vldr d6, [r12, #16] + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 d2, d2 + vpaddl.u8 q0, q0 + vpaddl.u8 d6, d6 + vpaddl.u8 q2, q2 + vshl.i16 d2, d2, #2 + vshl.i16 q0, q0, #2 + vshl.i16 d6, d6, #2 + vshl.i16 q2, q2, #2 + vdup.16 d3, d2[3] + vdup.16 d7, d6[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q2, q2 + vshl.i16 q0, q0, #2 + vshl.i16 q2, q2, #2 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vshl.i16 q0, q0, #2 + vdup.16 q3, d1[3] + vdup.16 q1, d0[3] + vdup.16 d5, d1[3] + vmov d4, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) +endfunc + +// void cfl_ac_444_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_8bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_444_tbl) + sub r8, r8, #26 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_444_tbl): + .word L(ipred_cfl_ac_444_w32) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w16) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w8) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w4) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w4): +1: // Copy and expand input + vld1.32 {d0[]}, [r1, :32], r2 + vld1.32 {d0[1]}, [r12, :32], r2 + vld1.32 {d2[]}, [r1, :32], r2 + vld1.32 {d2[1]}, [r12, :32], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q1, d2, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): +1: // Copy and expand input + vld1.16 {d0}, [r1, :64], r2 + vld1.16 {d2}, [r12, :64], r2 + vld1.16 {d4}, [r1, :64], r2 + vshll.u8 q0, d0, #3 + vld1.16 {d6}, [r12, :64], r2 + vshll.u8 q1, d2, #3 + vshll.u8 q2, d4, #3 + vshll.u8 q3, d6, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + cmp r3, #0 + bne L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + vld1.8 {q1}, [r1, :128], r2 + vld1.8 {q3}, [r12, :128], r2 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q2, d6, #3 + vshll.u8 q3, d7, #3 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d4}, [r12, :64], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q2, d4, #3 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + adr r7, L(ipred_cfl_ac_444_w32_tbl) + ldr r3, [r7, r3, lsl #1] // (w3>>1) << 2 + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_444_w32_tbl): + .word L(ipred_cfl_ac_444_w32_wpad0) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad2) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad4) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad6) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w32_wpad0): +1: // Copy and expand input, without padding + vld1.8 {q2, q3}, [r1, :128], r2 + vld1.8 {q13, q14}, [r12, :128], r2 + vshll.u8 q0, d4, #3 + vshll.u8 q1, d5, #3 + vshll.u8 q2, d6, #3 + vshll.u8 q3, d7, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vshll.u8 q0, d28, #3 + vshll.u8 q1, d29, #3 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): +1: // Copy and expand input, padding 8 + vldr d4, [r1, #16] + vld1.8 {q1}, [r1, :128], r2 + vldr d28, [r12, #16] + vld1.8 {q13}, [r12, :128], r2 + vshll.u8 q2, d4, #3 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vshll.u8 q0, d28, #3 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vdup.16 q1, d1[3] + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): +1: // Copy and expand input, padding 16 + vld1.8 {q1}, [r1, :128], r2 + vld1.8 {q13}, [r12, :128], r2 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + vdup.16 q2, d3[3] + vdup.16 q3, d3[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vdup.16 q0, d27[3] + vdup.16 q1, d27[3] + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): +1: // Copy and expand input, padding 24 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d24}, [r12, :64], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q12, d24, #3 + subs r8, r8, #2 + vdup.16 q1, d1[3] + vdup.16 q2, d1[3] + vdup.16 q3, d1[3] + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vdup.16 q13, d25[3] + vdup.16 q0, d25[3] + vdup.16 q1, d25[3] + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + +L(ipred_cfl_ac_444_w32_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #1 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 2b +3: + + // Multiply the height by eight and reuse the w4 subtracting + lsl r6, r6, #3 + // Aggregate the sums, with wider intermediates earlier than in + // ipred_cfl_ac_420_w8_calc_subtract_dc. + vpaddl.u16 q0, q8 + vpaddl.u16 q1, q9 + vpaddl.u16 q2, q10 + vpaddl.u16 q3, q11 + vadd.i32 q0, q0, q1 + vadd.i32 q2, q2, q3 + vadd.i32 q0, q0, q2 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d0, d0 // sum + sub r0, r0, r6, lsl #3 + vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz + vdup.16 q8, d16[0] + b L(ipred_cfl_ac_420_w4_subtract_dc) +endfunc diff --git a/third_party/dav1d/src/arm/32/ipred16.S b/third_party/dav1d/src/arm/32/ipred16.S new file mode 100644 index 0000000000..993d9500aa --- /dev/null +++ b/third_party/dav1d/src/arm/32/ipred16.S @@ -0,0 +1,3254 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, B Krishnan Iyer + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height, +// const int bitdepth_max); +function ipred_dc_128_16bpc_neon, export=1 + push {r4, lr} + ldr r4, [sp, #8] + ldr r12, [sp, #24] + clz r3, r3 + adr r2, L(ipred_dc_128_tbl) + sub r3, r3, #25 + vdup.16 q0, r12 + ldr r3, [r2, r3, lsl #2] + add r12, r0, r1 + vrshr.u16 q0, q0, #1 + add r2, r2, r3 + lsl r1, r1, #1 + bx r2 + + .align 2 +L(ipred_dc_128_tbl): + .word 640f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 320f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 160f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 8f - L(ipred_dc_128_tbl) + CONFIG_THUMB + .word 4f - L(ipred_dc_128_tbl) + CONFIG_THUMB +4: + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt 4b + pop {r4, pc} +8: + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + bgt 8b + pop {r4, pc} +160: + vmov q1, q0 +16: + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 16b + pop {r4, pc} +320: + vmov q1, q0 + sub r1, r1, #32 +32: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 32b + pop {r4, pc} +640: + vmov q1, q0 + sub r1, r1, #96 +64: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + subs r4, r4, #2 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 64b + pop {r4, pc} +endfunc + +// void ipred_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_16bpc_neon, export=1 + push {r4, lr} + ldr lr, [sp, #8] + clz r3, r3 + adr r4, L(ipred_v_tbl) + sub r3, r3, #25 + ldr r3, [r4, r3, lsl #2] + add r2, r2, #2 + add r4, r4, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r4 + + .align 2 +L(ipred_v_tbl): + .word 640f - L(ipred_v_tbl) + CONFIG_THUMB + .word 320f - L(ipred_v_tbl) + CONFIG_THUMB + .word 160f - L(ipred_v_tbl) + CONFIG_THUMB + .word 80f - L(ipred_v_tbl) + CONFIG_THUMB + .word 40f - L(ipred_v_tbl) + CONFIG_THUMB + +40: + vld1.16 {d0}, [r2] +4: + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + subs lr, lr, #4 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt 4b + pop {r4, pc} +80: + vld1.16 {q0}, [r2] +8: + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + subs lr, lr, #4 + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + bgt 8b + pop {r4, pc} +160: + vld1.16 {q0, q1}, [r2] +16: + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs lr, lr, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 16b + pop {r4, pc} +320: + vld1.16 {q0, q1}, [r2]! + sub r1, r1, #32 + vld1.16 {q2, q3}, [r2] +32: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d4, d5, d6, d7}, [r0, :128], r1 + vst1.16 {d4, d5, d6, d7}, [r12, :128], r1 + subs lr, lr, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d4, d5, d6, d7}, [r0, :128], r1 + vst1.16 {d4, d5, d6, d7}, [r12, :128], r1 + bgt 32b + pop {r4, pc} +640: + vld1.16 {q0, q1}, [r2]! + sub r1, r1, #96 + vld1.16 {q2, q3}, [r2]! + vld1.16 {q8, q9}, [r2]! + vld1.16 {q10, q11}, [r2]! +64: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d4, d5, d6, d7}, [r0, :128]! + vst1.16 {d4, d5, d6, d7}, [r12, :128]! + subs lr, lr, #2 + vst1.16 {d16, d17, d18, d19}, [r0, :128]! + vst1.16 {d16, d17, d18, d19}, [r12, :128]! + vst1.16 {d20, d21, d22, d23}, [r0, :128], r1 + vst1.16 {d20, d21, d22, d23}, [r12, :128], r1 + bgt 64b + pop {r4, pc} +endfunc + +// void ipred_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_16bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + clz r3, r3 + adr r5, L(ipred_h_tbl) + sub r3, r3, #25 + ldr r3, [r5, r3, lsl #2] + sub r2, r2, #2 + mov lr, #-2 + add r5, r5, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_h_tbl): + .word 640f - L(ipred_h_tbl) + CONFIG_THUMB + .word 320f - L(ipred_h_tbl) + CONFIG_THUMB + .word 160f - L(ipred_h_tbl) + CONFIG_THUMB + .word 8f - L(ipred_h_tbl) + CONFIG_THUMB + .word 40f - L(ipred_h_tbl) + CONFIG_THUMB +40: + sub r2, r2, #6 + mov lr, #-8 +4: + vld4.16 {d0[], d1[], d2[], d3[]}, [r2], lr + vst1.16 {d3}, [r0, :64], r1 + vst1.16 {d2}, [r12, :64], r1 + subs r4, r4, #4 + vst1.16 {d1}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt 4b + pop {r4-r5, pc} +8: + vld1.16 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.16 {d2[], d3[]}, [r2], lr + vst1.16 {q0}, [r0, :128], r1 + vld1.16 {d4[], d5[]}, [r2], lr + vst1.16 {q1}, [r12, :128], r1 + vld1.16 {d6[], d7[]}, [r2], lr + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r12, :128], r1 + bgt 8b + pop {r4-r5, pc} +160: + sub r1, r1, #16 +16: + vld1.16 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.16 {d2[], d3[]}, [r2], lr + vst1.16 {q0}, [r0, :128]! + vld1.16 {d4[], d5[]}, [r2], lr + vst1.16 {q1}, [r12, :128]! + vld1.16 {d6[], d7[]}, [r2], lr + vst1.16 {q0}, [r0, :128], r1 + vst1.16 {q1}, [r12, :128], r1 + vst1.16 {q2}, [r0, :128]! + vst1.16 {q3}, [r12, :128]! + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r12, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + sub r1, r1, #48 +32: + vld1.16 {d0[], d1[]}, [r2], lr + subs r4, r4, #4 + vld1.16 {d2[], d3[]}, [r2], lr + vst1.16 {q0}, [r0, :128]! + vld1.16 {d4[], d5[]}, [r2], lr + vst1.16 {q1}, [r12, :128]! + vld1.16 {d6[], d7[]}, [r2], lr + vst1.16 {q0}, [r0, :128]! + vst1.16 {q1}, [r12, :128]! + vst1.16 {q0}, [r0, :128]! + vst1.16 {q1}, [r12, :128]! + vst1.16 {q0}, [r0, :128], r1 + vst1.16 {q1}, [r12, :128], r1 + vst1.16 {q2}, [r0, :128]! + vst1.16 {q3}, [r12, :128]! + vst1.16 {q2}, [r0, :128]! + vst1.16 {q3}, [r12, :128]! + vst1.16 {q2}, [r0, :128]! + vst1.16 {q3}, [r12, :128]! + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r12, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + sub r1, r1, #96 +64: + vld1.16 {d0[], d1[]}, [r2], lr + subs r4, r4, #2 + vld1.16 {d4[], d5[]}, [r2], lr + vmov q1, q0 + vmov q3, q2 + vst1.16 {q0, q1}, [r0, :128]! + vst1.16 {q2, q3}, [r12, :128]! + vst1.16 {q0, q1}, [r0, :128]! + vst1.16 {q2, q3}, [r12, :128]! + vst1.16 {q0, q1}, [r0, :128]! + vst1.16 {q2, q3}, [r12, :128]! + vst1.16 {q0, q1}, [r0, :128], r1 + vst1.16 {q2, q3}, [r12, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_16bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + clz r3, r3 + adr r5, L(ipred_dc_top_tbl) + sub r3, r3, #25 + ldr r3, [r5, r3, lsl #2] + add r2, r2, #2 + add r5, r5, r3 + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_top_tbl): + .word 640f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 320f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 160f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 80f - L(ipred_dc_top_tbl) + CONFIG_THUMB + .word 40f - L(ipred_dc_top_tbl) + CONFIG_THUMB + +40: + vld1.16 {d0}, [r2] + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 d0, d0[0] +4: + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt 4b + pop {r4-r5, pc} +80: + vld1.16 {d0, d1}, [r2] + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] +8: + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + bgt 8b + pop {r4-r5, pc} +160: + vld1.16 {d0, d1, d2, d3}, [r2] + vadd.i16 q0, q0, q1 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d4, d0, #4 + vdup.16 q0, d4[0] + vdup.16 q1, d4[0] +16: + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + vld1.16 {d0, d1, d2, d3}, [r2]! + vld1.16 {d4, d5, d6, d7}, [r2] + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vadd.i16 q0, q0, q2 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpaddl.u16 d0, d0 + vrshrn.i32 d18, q0, #5 + vdup.16 q0, d18[0] + vdup.16 q1, d18[0] + sub r1, r1, #32 +32: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + vld1.16 {d0, d1, d2, d3}, [r2]! + vld1.16 {d4, d5, d6, d7}, [r2]! + vadd.i16 q0, q0, q1 + vld1.16 {d16, d17, d18, d19}, [r2]! + vadd.i16 q2, q2, q3 + vld1.16 {d20, d21, d22, d23}, [r2] + vadd.i16 q8, q8, q9 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q0, q2 + vadd.i16 q8, q8, q10 + vadd.i16 q0, q0, q8 + vadd.i16 d0, d0, d1 + vpaddl.u16 d0, d0 + vpadd.i32 d0, d0, d0 + vrshrn.i32 d18, q0, #6 + vdup.16 q0, d18[0] + vdup.16 q1, d18[0] + sub r1, r1, #96 +64: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + subs r4, r4, #2 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_16bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + sub r2, r2, r4, lsl #1 + clz r3, r3 + clz lr, r4 + sub lr, lr, #25 + adr r5, L(ipred_dc_left_tbl) + sub r3, r3, #20 + ldr r3, [r5, r3, lsl #2] + ldr lr, [r5, lr, lsl #2] + add r3, r5, r3 + add r5, r5, lr + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_left_tbl): + .word L(ipred_dc_left_h64) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h32) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h16) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h8) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_h4) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w64) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w32) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w8) - L(ipred_dc_left_tbl) + CONFIG_THUMB + .word L(ipred_dc_left_w4) - L(ipred_dc_left_tbl) + CONFIG_THUMB + +L(ipred_dc_left_h4): + vld1.16 {d0}, [r2, :64] + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + bx r3 +L(ipred_dc_left_w4): + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt L(ipred_dc_left_w4) + pop {r4-r5, pc} +L(ipred_dc_left_h8): + vld1.16 {d0, d1}, [r2, :128] + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + bx r3 +L(ipred_dc_left_w8): + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + bgt L(ipred_dc_left_w8) + pop {r4-r5, pc} +L(ipred_dc_left_h16): + vld1.16 {d0, d1, d2, d3}, [r2, :128] + vadd.i16 q0, q0, q1 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + bx r3 +L(ipred_dc_left_w16): + vmov q1, q0 +1: + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 1b + pop {r4-r5, pc} +L(ipred_dc_left_h32): + vld1.16 {d0, d1, d2, d3}, [r2, :128]! + vld1.16 {d4, d5, d6, d7}, [r2, :128] + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vadd.i16 q0, q0, q2 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpaddl.u16 d0, d0 + vrshrn.i32 d0, q0, #5 + vdup.16 q0, d0[0] + bx r3 +L(ipred_dc_left_w32): + sub r1, r1, #32 + vmov q1, q0 +1: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 1b + pop {r4-r5, pc} +L(ipred_dc_left_h64): + vld1.16 {d0, d1, d2, d3}, [r2, :128]! + vld1.16 {d4, d5, d6, d7}, [r2, :128]! + vadd.i16 q0, q0, q1 + vld1.16 {d16, d17, d18, d19}, [r2, :128]! + vadd.i16 q2, q2, q3 + vld1.16 {d20, d21, d22, d23}, [r2, :128] + vadd.i16 q8, q8, q9 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q0, q2 + vadd.i16 q8, q8, q10 + vadd.i16 q0, q0, q8 + vadd.i16 d0, d0, d1 + vpaddl.u16 d0, d0 + vpadd.i32 d0, d0, d0 + vrshrn.i32 d0, q0, #6 + vdup.16 q0, d0[0] + bx r3 +L(ipred_dc_left_w64): + sub r1, r1, #96 + vmov q1, q0 +1: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + subs r4, r4, #2 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 1b + pop {r4-r5, pc} +endfunc + +// void ipred_dc_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_16bpc_neon, export=1 + push {r4-r6, lr} + ldr r4, [sp, #16] + sub r2, r2, r4, lsl #1 + add lr, r3, r4 // width + height + clz r3, r3 + clz r12, r4 + vdup.32 q15, lr // width + height + adr r5, L(ipred_dc_tbl) + rbit lr, lr // rbit(width + height) + sub r3, r3, #20 // 25 leading bits, minus table offset 5 + sub r12, r12, #25 + clz lr, lr // ctz(width + height) + ldr r3, [r5, r3, lsl #2] + ldr r12, [r5, r12, lsl #2] + neg lr, lr // -ctz(width + height) + add r3, r5, r3 + add r5, r5, r12 + vshr.u32 q15, q15, #1 // (width + height) >> 1 + vdup.32 q14, lr // -ctz(width + height) + add r12, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_dc_tbl): + .word L(ipred_dc_h64) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h32) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h16) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h8) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_h4) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w64) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w32) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w8) - L(ipred_dc_tbl) + CONFIG_THUMB + .word L(ipred_dc_w4) - L(ipred_dc_tbl) + CONFIG_THUMB + +L(ipred_dc_h4): + vld1.16 {d0}, [r2, :64]! + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r3 +L(ipred_dc_w4): + vld1.16 {d2}, [r2] + vadd.i32 d0, d0, d30 + vpadd.i16 d2, d2, d2 + vpaddl.u16 d2, d2 + cmp r4, #4 + vadd.i32 d0, d0, d2 + vshl.u32 d0, d0, d28 + beq 1f + // h = 8/16 + cmp r4, #16 + movw lr, #0x6667 + movw r5, #0xAAAB + it ne + movne lr, r5 + vdup.32 d24, lr + vmul.i32 d0, d0, d24 + vshr.u32 d0, d0, #17 +1: + vdup.16 d0, d0[0] +2: + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + subs r4, r4, #4 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d0}, [r12, :64], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h8): + vld1.16 {d0, d1}, [r2, :128]! + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r3 +L(ipred_dc_w8): + vld1.16 {d2, d3}, [r2] + vadd.i32 d0, d0, d30 + vadd.i16 d2, d2, d3 + vpadd.i16 d2, d2, d2 + vpaddl.u16 d2, d2 + cmp r4, #8 + vadd.i32 d0, d0, d2 + vshl.u32 d0, d0, d28 + beq 1f + // h = 4/16/32 + cmp r4, #32 + movw lr, #0x6667 + movw r5, #0xAAAB + it ne + movne lr, r5 + vdup.32 d24, lr + vmul.i32 d0, d0, d24 + vshr.u32 d0, d0, #17 +1: + vdup.16 q0, d0[0] +2: + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1}, [r0, :128], r1 + vst1.16 {d0, d1}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h16): + vld1.16 {d0, d1, d2, d3}, [r2, :128]! + vadd.i16 q0, q0, q1 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r3 +L(ipred_dc_w16): + vld1.16 {d2, d3, d4, d5}, [r2] + vadd.i32 d0, d0, d30 + vadd.i16 q1, q1, q2 + vadd.i16 d2, d2, d3 + vpadd.i16 d2, d2, d1 + vpaddl.u16 d2, d2 + cmp r4, #16 + vadd.i32 d0, d0, d2 + vshl.u32 d4, d0, d28 + beq 1f + // h = 4/8/32/64 + tst r4, #(32+16+8) // 16 added to make a consecutive bitmask + movw lr, #0x6667 + movw r5, #0xAAAB + it ne + movne lr, r5 + vdup.32 d24, lr + vmul.i32 d4, d4, d24 + vshr.u32 d4, d4, #17 +1: + vdup.16 q0, d4[0] + vdup.16 q1, d4[0] +2: + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} + +L(ipred_dc_h32): + vld1.16 {d0, d1, d2, d3}, [r2, :128]! + vld1.16 {d4, d5, d6, d7}, [r2, :128]! + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vadd.i16 q0, q0, q2 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r3 +L(ipred_dc_w32): + vld1.16 {d2, d3, d4, d5}, [r2]! + vadd.i32 d0, d0, d30 + vld1.16 {d16, d17, d18, d19}, [r2] + vadd.i16 q1, q1, q2 + vadd.i16 q8, q8, q9 + vadd.i16 q1, q1, q8 + vadd.i16 d2, d2, d3 + vpadd.i16 d2, d2, d2 + vpaddl.u16 d2, d2 + cmp r4, #32 + vadd.i32 d0, d0, d2 + vshl.u32 d4, d0, d28 + beq 1f + // h = 8/16/64 + cmp r4, #8 + movw lr, #0x6667 + movw r5, #0xAAAB + it ne + movne lr, r5 + vdup.32 d24, lr + vmul.i32 d4, d4, d24 + vshr.u32 d4, d4, #17 +1: + sub r1, r1, #32 + vdup.16 q0, d4[0] + vdup.16 q1, d4[0] +2: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + subs r4, r4, #4 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} +L(ipred_dc_h64): + vld1.16 {d0, d1, d2, d3}, [r2, :128]! + vld1.16 {d4, d5, d6, d7}, [r2, :128]! + vadd.i16 q0, q0, q1 + vld1.16 {d16, d17, d18, d19}, [r2, :128]! + vadd.i16 q2, q2, q3 + vld1.16 {d20, d21, d22, d23}, [r2, :128]! + vadd.i16 q8, q8, q9 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q0, q2 + vadd.i16 q8, q8, q10 + vadd.i16 q0, q0, q8 + vadd.i16 d0, d0, d1 + vpaddl.u16 d0, d0 + add r2, r2, #2 + vpadd.i32 d0, d0, d0 + bx r3 +L(ipred_dc_w64): + vld1.16 {d2, d3, d4, d5}, [r2]! + vadd.i32 d0, d0, d30 + vld1.16 {d16, d17, d18, d19}, [r2]! + vadd.i16 q1, q1, q2 + vld1.16 {d20, d21, d22, d23}, [r2]! + vadd.i16 q8, q8, q9 + vld1.16 {d24, d25, d26, d27}, [r2]! + vadd.i16 q10, q10, q11 + vadd.i16 q12, q12, q13 + vadd.i16 q1, q1, q8 + vadd.i16 q10, q10, q12 + vadd.i16 q1, q1, q10 + vadd.i16 d2, d2, d3 + vpaddl.u16 d2, d2 + vpadd.i32 d2, d2, d2 + cmp r4, #64 + vadd.i32 d0, d0, d2 + vshl.u32 d4, d0, d28 + beq 1f + // h = 16/32 + cmp r4, #16 + movw lr, #0x6667 + movw r5, #0xAAAB + it ne + movne lr, r5 + vdup.32 d24, lr + vmul.i32 d4, d4, d24 + vshr.u32 d4, d4, #17 +1: + sub r1, r1, #96 + vdup.16 q0, d4[0] + vdup.16 q1, d4[0] +2: + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + subs r4, r4, #2 + vst1.16 {d0, d1, d2, d3}, [r0, :128]! + vst1.16 {d0, d1, d2, d3}, [r12, :128]! + vst1.16 {d0, d1, d2, d3}, [r0, :128], r1 + vst1.16 {d0, d1, d2, d3}, [r12, :128], r1 + bgt 2b + pop {r4-r6, pc} +endfunc + +// void ipred_paeth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_16bpc_neon, export=1 + push {r4-r6, lr} + vpush {q4} + ldr r4, [sp, #32] + clz lr, r3 + adr r12, L(ipred_paeth_tbl) + sub lr, lr, #25 + ldr lr, [r12, lr, lsl #2] + vld1.16 {d4[], d5[]}, [r2] + add r6, r2, #2 + sub r2, r2, #4 + add r12, r12, lr + mov r5, #-4 + add lr, r0, r1 + lsl r1, r1, #1 + bx r12 + + .align 2 +L(ipred_paeth_tbl): + .word 640f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_paeth_tbl) + CONFIG_THUMB + +40: + sub r2, r2, #4 + mov r5, #-8 + vld1.16 {d6}, [r6] + vsub.i16 d16, d6, d4 // top - topleft + vmov d7, d6 + vmov d17, d16 +4: + vld4.16 {d0[], d1[], d2[], d3[]}, [r2, :64], r5 + vadd.i16 q9, q8, q0 // base + vadd.i16 q10, q8, q1 + vabd.s16 q11, q3, q9 // tdiff + vabd.s16 q12, q3, q10 + vabd.s16 q13, q2, q9 // tldiff + vabd.s16 q14, q2, q10 + vabd.s16 q9, q0, q9 // ldiff + vabd.s16 q10, q1, q10 + vmin.u16 q15, q11, q13 // min(tdiff, tldiff) + vmin.u16 q4, q12, q14 + vcge.u16 q11, q13, q11 // tldiff >= tdiff + vcge.u16 q12, q14, q12 + vcge.u16 q9, q15, q9 // min(tdiff, tldiff) >= ldiff + vcge.u16 q10, q4, q10 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + vst1.16 {d25}, [r0, :64], r1 + vst1.16 {d24}, [lr, :64], r1 + subs r4, r4, #4 + vst1.16 {d23}, [r0, :64], r1 + vst1.16 {d22}, [lr, :64], r1 + bgt 4b + vpop {q4} + pop {r4-r6, pc} +80: +160: +320: +640: + vld1.16 {q3}, [r6]! + mov r12, r3 + sub r1, r1, r3, lsl #1 +1: + vld2.16 {d0[], d2[]}, [r2, :32], r5 + vmov d1, d0 + vmov d3, d2 +2: + vsub.i16 q8, q3, q2 // top - topleft + vadd.i16 q9, q8, q0 // base + vadd.i16 q10, q8, q1 + vabd.s16 q11, q3, q9 // tdiff + vabd.s16 q12, q3, q10 + vabd.s16 q13, q2, q9 // tldiff + vabd.s16 q14, q2, q10 + vabd.s16 q9, q0, q9 // ldiff + vabd.s16 q10, q1, q10 + vmin.u16 q15, q11, q13 // min(tdiff, tldiff) + vmin.u16 q4, q12, q14 + vcge.u16 q11, q13, q11 // tldiff >= tdiff + vcge.u16 q12, q14, q12 + vcge.u16 q9, q15, q9 // min(tdiff, tldiff) >= ldiff + vcge.u16 q10, q4, q10 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + subs r3, r3, #8 + vst1.16 {q12}, [r0, :128]! + vst1.16 {q11}, [lr, :128]! + ble 8f + vld1.16 {q3}, [r6]! + b 2b +8: + subs r4, r4, #2 + ble 9f + // End of horizontal loop, move pointers to next two rows + sub r6, r6, r12, lsl #1 + add r0, r0, r1 + add lr, lr, r1 + vld1.16 {q3}, [r6]! + mov r3, r12 + b 1b +9: + vpop {q4} + pop {r4-r6, pc} +endfunc + +// void ipred_smooth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_16bpc_neon, export=1 + push {r4-r10, lr} + ldr r4, [sp, #32] + movrel r10, X(sm_weights) + add r12, r10, r4 + add r10, r10, r3 + clz r9, r3 + adr r5, L(ipred_smooth_tbl) + sub lr, r2, r4, lsl #1 + sub r9, r9, #25 + ldr r9, [r5, r9, lsl #2] + vld1.16 {d4[], d5[]}, [lr] // bottom + add r8, r2, #2 + add r5, r5, r9 + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_tbl): + .word 640f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_tbl) + CONFIG_THUMB + +40: + vld1.16 {d16}, [r8] // top + vld1.32 {d18[]}, [r10, :32] // weights_hor + sub r2, r2, #8 + mov r7, #-8 + vdup.16 q3, d16[3] // right + vsub.i16 q8, q8, q2 // top-bottom + vmovl.u8 q9, d18 // weights_hor + vadd.i16 d19, d4, d6 // bottom+right +4: + vld4.16 {d0[], d1[], d2[], d3[]}, [r2, :64], r7 // left + vld4.8 {d20[], d21[], d22[], d23[]}, [r12, :32]! // weights_ver + vshll.u16 q12, d19, #8 // (bottom+right)*256 + vshll.u16 q13, d19, #8 + vshll.u16 q14, d19, #8 + vshll.u16 q15, d19, #8 + vzip.32 d20, d21 // weights_ver + vzip.32 d22, d23 + vsub.i16 q1, q1, q3 // left-right + vsub.i16 q0, q0, q3 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmlal.s16 q12, d3, d18 // += (left-right)*weights_hor + vmlal.s16 q13, d2, d18 // (left flipped) + vmlal.s16 q14, d1, d18 + vmlal.s16 q15, d0, d18 + vmlal.s16 q12, d16, d20 // += (top-bottom)*weights_ver + vmlal.s16 q13, d16, d21 + vmlal.s16 q14, d16, d22 + vmlal.s16 q15, d16, d23 + vrshrn.i32 d24, q12, #9 + vrshrn.i32 d25, q13, #9 + vrshrn.i32 d26, q14, #9 + vrshrn.i32 d27, q15, #9 + vst1.16 {d24}, [r0, :64], r1 + vst1.16 {d25}, [r6, :64], r1 + subs r4, r4, #4 + vst1.16 {d26}, [r0, :64], r1 + vst1.16 {d27}, [r6, :64], r1 + bgt 4b + pop {r4-r10, pc} +80: + vld1.16 {q8}, [r8] // top + vld1.8 {d18}, [r10, :64] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vdup.16 q3, d17[3] // right + vsub.i16 q8, q8, q2 // top-bottom + vmovl.u8 q9, d18 // weights_hor + vadd.i16 d3, d4, d6 // bottom+right +8: + vld2.16 {d0[], d1[]}, [r2, :32], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vshll.u16 q12, d3, #8 // (bottom+right)*256 + vshll.u16 q13, d3, #8 + vshll.u16 q14, d3, #8 + vshll.u16 q15, d3, #8 + vsub.i16 q0, q0, q3 // left-right + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmlal.s16 q12, d1, d18 // += (left-right)*weights_hor + vmlal.s16 q13, d1, d19 // (left flipped) + vmlal.s16 q14, d0, d18 + vmlal.s16 q15, d0, d19 + vmlal.s16 q12, d16, d20 // += (top-bottom)*weights_ver + vmlal.s16 q13, d17, d20 + vmlal.s16 q14, d16, d22 + vmlal.s16 q15, d17, d22 + vrshrn.i32 d24, q12, #9 + vrshrn.i32 d25, q13, #9 + vrshrn.i32 d26, q14, #9 + vrshrn.i32 d27, q15, #9 + subs r4, r4, #2 + vst1.16 {q12}, [r0, :128], r1 + vst1.16 {q13}, [r6, :128], r1 + bgt 8b + pop {r4-r10, pc} +160: +320: +640: + add lr, r2, r3, lsl #1 + sub r2, r2, #4 + mov r7, #-4 + vld1.16 {d6[], d7[]}, [lr] // right + sub r1, r1, r3, lsl #1 + mov r9, r3 + vadd.i16 d3, d4, d6 // bottom+right + +1: + vld2.16 {d0[], d1[]}, [r2, :32], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vsub.i16 q0, q0, q3 // left-right + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 +2: + vld1.8 {d18}, [r10, :64]! // weights_hor + vld1.16 {q8}, [r8]! // top + vshll.u16 q12, d3, #8 // (bottom+right)*256 + vshll.u16 q13, d3, #8 + vmovl.u8 q9, d18 // weights_hor + vshll.u16 q14, d3, #8 + vshll.u16 q15, d3, #8 + vsub.i16 q8, q8, q2 // top-bottom + vmlal.s16 q12, d1, d18 // += (left-right)*weights_hor + vmlal.s16 q13, d1, d19 // (left flipped) + vmlal.s16 q14, d0, d18 + vmlal.s16 q15, d0, d19 + vmlal.s16 q12, d16, d20 // += (top-bottom)*weights_ver + vmlal.s16 q13, d17, d20 + vmlal.s16 q14, d16, d22 + vmlal.s16 q15, d17, d22 + vrshrn.i32 d24, q12, #9 + vrshrn.i32 d25, q13, #9 + vrshrn.i32 d26, q14, #9 + vrshrn.i32 d27, q15, #9 + subs r3, r3, #8 + vst1.16 {q12}, [r0, :128]! + vst1.16 {q13}, [r6, :128]! + bgt 2b + subs r4, r4, #2 + ble 9f + sub r8, r8, r9, lsl #1 + sub r10, r10, r9 + add r0, r0, r1 + add r6, r6, r1 + mov r3, r9 + b 1b +9: + pop {r4-r10, pc} +endfunc + +// void ipred_smooth_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_16bpc_neon, export=1 + push {r4-r7, lr} + ldr r4, [sp, #20] + movrel r7, X(sm_weights) + add r7, r7, r4 + clz lr, r3 + adr r5, L(ipred_smooth_v_tbl) + sub r12, r2, r4, lsl #1 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.16 {d4[], d5[]}, [r12] // bottom + add r2, r2, #2 + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_v_tbl): + .word 640f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + +40: + vld1.16 {d6}, [r2] // top + vsub.i16 d6, d6, d4 // top-bottom + vmov d7, d6 +4: + vld4.8 {d16[], d17[], d18[], d19[]}, [r7, :32]! // weights_ver + vzip.32 d16, d17 // weights_ver + vzip.32 d18, d19 + vshll.u8 q8, d16, #7 // weights_ver << 7 + vshll.u8 q9, d18, #7 + vqrdmulh.s16 q10, q3, q8 // ((top-bottom)*weights_ver + 128) >> 8 + vqrdmulh.s16 q11, q3, q9 + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q2 + vst1.16 {d20}, [r0, :64], r1 + vst1.16 {d21}, [r6, :64], r1 + subs r4, r4, #4 + vst1.16 {d22}, [r0, :64], r1 + vst1.16 {d23}, [r6, :64], r1 + bgt 4b + pop {r4-r7, pc} +80: + vld1.16 {q3}, [r2] // top + vsub.i16 q3, q3, q2 // top-bottom +8: + vld4.8 {d16[], d18[], d20[], d22[]}, [r7, :32]! // weights_ver + vshll.u8 q8, d16, #7 // weights_ver << 7 + vshll.u8 q9, d18, #7 + vshll.u8 q10, d20, #7 + vshll.u8 q11, d22, #7 + vqrdmulh.s16 q8, q3, q8 // ((top-bottom)*weights_ver + 128) >> 8 + vqrdmulh.s16 q9, q3, q9 + vqrdmulh.s16 q10, q3, q10 + vqrdmulh.s16 q11, q3, q11 + vadd.i16 q8, q8, q2 + vadd.i16 q9, q9, q2 + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q2 + vst1.16 {q8}, [r0, :128], r1 + vst1.16 {q9}, [r6, :128], r1 + subs r4, r4, #4 + vst1.16 {q10}, [r0, :128], r1 + vst1.16 {q11}, [r6, :128], r1 + bgt 8b + pop {r4-r7, pc} +160: +320: +640: + vpush {q4-q7} + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3, lsl #1 + mov r12, r3 + +1: + vld4.8 {d8[], d10[], d12[], d14[]}, [r7, :32]! // weights_ver + vshll.u8 q4, d8, #7 // weights_ver << 7 + vshll.u8 q5, d10, #7 + vshll.u8 q6, d12, #7 + vshll.u8 q7, d14, #7 +2: + vld1.16 {q0, q1}, [r2]! // top + vsub.i16 q0, q0, q2 // top-bottom + vsub.i16 q1, q1, q2 + vqrdmulh.s16 q8, q0, q4 // ((top-bottom)*weights_ver + 128) >> 8 + vqrdmulh.s16 q9, q1, q4 + vqrdmulh.s16 q10, q0, q5 + vqrdmulh.s16 q11, q1, q5 + vqrdmulh.s16 q12, q0, q6 + vqrdmulh.s16 q13, q1, q6 + vqrdmulh.s16 q14, q0, q7 + vqrdmulh.s16 q15, q1, q7 + vadd.i16 q8, q8, q2 + vadd.i16 q9, q9, q2 + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q2 + vadd.i16 q12, q12, q2 + vadd.i16 q13, q13, q2 + vadd.i16 q14, q14, q2 + vadd.i16 q15, q15, q2 + subs r3, r3, #16 + vst1.16 {q8, q9}, [r0, :128]! + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r5, :128]! + vst1.16 {q14, q15}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r2, r2, r12, lsl #1 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r7, pc} +endfunc + +// void ipred_smooth_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_16bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + movrel r8, X(sm_weights) + add r8, r8, r3 + clz lr, r3 + adr r5, L(ipred_smooth_h_tbl) + add r12, r2, r3, lsl #1 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.16 {d4[], d5[]}, [r12] // right + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_h_tbl): + .word 640f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[]}, [r8, :32] // weights_hor + sub r2, r2, #8 + mov r7, #-8 + vshll.u8 q3, d6, #7 // weights_hor << 7 +4: + vld4.16 {d0[], d1[], d2[], d3[]}, [r2, :64], r7 // left + vsub.i16 q0, q0, q2 // left-right + vsub.i16 q1, q1, q2 + subs r4, r4, #4 + vqrdmulh.s16 q8, q1, q3 // ((left-right)*weights_hor + 128) >> 8 + vqrdmulh.s16 q9, q0, q3 // (left flipped) + vadd.i16 q8, q8, q2 + vadd.i16 q9, q9, q2 + vst1.16 {d17}, [r0, :64], r1 + vst1.16 {d16}, [r6, :64], r1 + vst1.16 {d19}, [r0, :64], r1 + vst1.16 {d18}, [r6, :64], r1 + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d6}, [r8, :64] // weights_hor + sub r2, r2, #8 + mov r7, #-8 + vshll.u8 q3, d6, #7 // weights_hor << 7 +8: + vld1.16 {d23}, [r2, :64], r7 // left + subs r4, r4, #4 + vsub.i16 d23, d23, d4 // left-right + vdup.16 q8, d23[3] // flip left + vdup.16 q9, d23[2] + vdup.16 q10, d23[1] + vdup.16 q11, d23[0] + vqrdmulh.s16 q8, q8, q3 // ((left-right)*weights_hor + 128) >> 8 + vqrdmulh.s16 q9, q9, q3 + vqrdmulh.s16 q10, q10, q3 + vqrdmulh.s16 q11, q11, q3 + vadd.i16 q8, q8, q2 + vadd.i16 q9, q9, q2 + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q2 + vst1.16 {q8}, [r0, :128], r1 + vst1.16 {q9}, [r6, :128], r1 + vst1.16 {q10}, [r0, :128], r1 + vst1.16 {q11}, [r6, :128], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: +640: + vpush {q4-q7} + sub r2, r2, #8 + mov r7, #-8 + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3, lsl #1 + mov r12, r3 + +1: + vld1.16 {d15}, [r2, :64], r7 // left + vsub.i16 d15, d15, d4 // left-right + vdup.16 q4, d15[3] // flip left + vdup.16 q5, d15[2] + vdup.16 q6, d15[1] + vdup.16 q7, d15[0] +2: + vld1.8 {q1}, [r8, :128]! // weights_hor + subs r3, r3, #16 + vshll.u8 q0, d2, #7 // weights_hor << 7 + vshll.u8 q1, d3, #7 + vqrdmulh.s16 q8, q0, q4 // ((left-right)*weights_hor + 128) >> 8 + vqrdmulh.s16 q9, q1, q4 + vqrdmulh.s16 q10, q0, q5 + vqrdmulh.s16 q11, q1, q5 + vqrdmulh.s16 q12, q0, q6 + vqrdmulh.s16 q13, q1, q6 + vqrdmulh.s16 q14, q0, q7 + vqrdmulh.s16 q15, q1, q7 + vadd.i16 q8, q8, q2 + vadd.i16 q9, q9, q2 + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q2 + vadd.i16 q12, q12, q2 + vadd.i16 q13, q13, q2 + vadd.i16 q14, q14, q2 + vadd.i16 q15, q15, q2 + vst1.16 {q8, q9}, [r0, :128]! + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r5, :128]! + vst1.16 {q14, q15}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r8, r8, r12 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r8, pc} +endfunc + +// void ipred_filter_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height, +// const int bitdepth_max); +.macro filter_fn bpc +function ipred_filter_\bpc\()bpc_neon, export=1 + movw r12, #511 + ldrd r4, r5, [sp, #88] + and r5, r5, r12 // 511 + movrel r6, X(filter_intra_taps) + lsl r5, r5, #6 + add r6, r6, r5 + vld1.8 {d20, d21, d22, d23}, [r6, :128]! + clz lr, r3 + adr r5, L(ipred_filter\bpc\()_tbl) + vld1.8 {d27, d28, d29}, [r6, :64] + sub lr, lr, #26 + ldr lr, [r5, lr, lsl #2] + vmovl.s8 q8, d20 + vmovl.s8 q9, d21 + add r5, r5, lr + vmovl.s8 q10, d22 + vmovl.s8 q11, d23 + add r6, r0, r1 + lsl r1, r1, #1 + vmovl.s8 q12, d27 + vmovl.s8 q13, d28 + vmovl.s8 q14, d29 + mov r7, #-4 + vdup.16 q15, r8 + add r8, r2, #2 + sub r2, r2, #4 +.if \bpc == 10 + vmov.i16 q7, #0 +.endif + bx r5 + + .align 2 +L(ipred_filter\bpc\()_tbl): + .word 320f - L(ipred_filter\bpc\()_tbl) + CONFIG_THUMB + .word 160f - L(ipred_filter\bpc\()_tbl) + CONFIG_THUMB + .word 80f - L(ipred_filter\bpc\()_tbl) + CONFIG_THUMB + .word 40f - L(ipred_filter\bpc\()_tbl) + CONFIG_THUMB + +40: + vld1.16 {d0}, [r8] // top (0-3) +4: + vld1.16 {d2}, [r2], r7 // left (0-1) + topleft (2) +.if \bpc == 10 + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vrshr.s16 q2, q2, #4 + vmax.s16 q2, q2, q7 +.else + vmull.s16 q2, d18, d0[0] // p1(top[0]) * filter(1) + vmlal.s16 q2, d20, d0[1] // p2(top[1]) * filter(2) + vmlal.s16 q2, d22, d0[2] // p3(top[2]) * filter(3) + vmlal.s16 q2, d24, d0[3] // p4(top[3]) * filter(4) + vmlal.s16 q2, d16, d2[2] // p0(topleft) * filter(0) + vmlal.s16 q2, d26, d2[1] // p5(left[0]) * filter(5) + vmlal.s16 q2, d28, d2[0] // p6(left[1]) * filter(6) + vmull.s16 q3, d19, d0[0] // p1(top[0]) * filter(1) + vmlal.s16 q3, d21, d0[1] // p2(top[1]) * filter(2) + vmlal.s16 q3, d23, d0[2] // p3(top[2]) * filter(3) + vmlal.s16 q3, d25, d0[3] // p4(top[3]) * filter(4) + vmlal.s16 q3, d17, d2[2] // p0(topleft) * filter(0) + vmlal.s16 q3, d27, d2[1] // p5(left[0]) * filter(5) + vmlal.s16 q3, d29, d2[0] // p6(left[1]) * filter(6) + vqrshrun.s32 d4, q2, #4 + vqrshrun.s32 d5, q3, #4 +.endif + vmin.s16 q2, q2, q15 + subs r4, r4, #2 + vst1.16 {d4}, [r0, :64], r1 + vst1.16 {d5}, [r6, :64], r1 + vmov d0, d5 // move top from [4-7] to [0-3] + bgt 4b + vpop {q4-q7} + pop {r4-r8, pc} +80: + vld1.16 {q0}, [r8] // top (0-7) +8: + vld1.16 {d2}, [r2], r7 // left (0-1) + topleft (2) +.if \bpc == 10 + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vmul.i16 q3, q9, d1[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d1[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d1[2] // p3(top[2]) * filter(3) + vrshr.s16 q2, q2, #4 + vmax.s16 q2, q2, q7 + vmin.s16 q2, q2, q15 + vmla.i16 q3, q12, d1[3] // p4(top[3]) * filter(4) + vmla.i16 q3, q8, d0[3] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d4[3] // p5(left[0]) * filter(5) + vmla.i16 q3, q14, d5[3] // p6(left[1]) * filter(6) + vrshr.s16 q3, q3, #4 + vmax.s16 q3, q3, q7 +.else + vmull.s16 q2, d18, d0[0] // p1(top[0]) * filter(1) + vmlal.s16 q2, d20, d0[1] // p2(top[1]) * filter(2) + vmlal.s16 q2, d22, d0[2] // p3(top[2]) * filter(3) + vmlal.s16 q2, d24, d0[3] // p4(top[3]) * filter(4) + vmlal.s16 q2, d16, d2[2] // p0(topleft) * filter(0) + vmlal.s16 q2, d26, d2[1] // p5(left[0]) * filter(5) + vmlal.s16 q2, d28, d2[0] // p6(left[1]) * filter(6) + vmull.s16 q3, d19, d0[0] // p1(top[0]) * filter(1) + vmlal.s16 q3, d21, d0[1] // p2(top[1]) * filter(2) + vmlal.s16 q3, d23, d0[2] // p3(top[2]) * filter(3) + vmlal.s16 q3, d25, d0[3] // p4(top[3]) * filter(4) + vmlal.s16 q3, d17, d2[2] // p0(topleft) * filter(0) + vmlal.s16 q3, d27, d2[1] // p5(left[0]) * filter(5) + vmlal.s16 q3, d29, d2[0] // p6(left[1]) * filter(6) + vqrshrun.s32 d4, q2, #4 + vmull.s16 q4, d18, d1[0] // p1(top[0]) * filter(1) + vmlal.s16 q4, d20, d1[1] // p2(top[1]) * filter(2) + vmlal.s16 q4, d22, d1[2] // p3(top[2]) * filter(3) + vqrshrun.s32 d5, q3, #4 + vmin.s16 q2, q2, q15 + vmlal.s16 q4, d24, d1[3] // p4(top[3]) * filter(4) + vmlal.s16 q4, d16, d0[3] // p0(topleft) * filter(0) + vmlal.s16 q4, d26, d4[3] // p5(left[0]) * filter(5) + vmlal.s16 q4, d28, d5[3] // p6(left[1]) * filter(6) + vmull.s16 q5, d19, d1[0] // p1(top[0]) * filter(1) + vmlal.s16 q5, d21, d1[1] // p2(top[1]) * filter(2) + vmlal.s16 q5, d23, d1[2] // p3(top[2]) * filter(3) + vmlal.s16 q5, d25, d1[3] // p4(top[3]) * filter(4) + vmlal.s16 q5, d17, d0[3] // p0(topleft) * filter(0) + vmlal.s16 q5, d27, d4[3] // p5(left[0]) * filter(5) + vmlal.s16 q5, d29, d5[3] // p6(left[1]) * filter(6) + vqrshrun.s32 d6, q4, #4 + vqrshrun.s32 d7, q5, #4 +.endif + vmin.s16 q3, q3, q15 + vswp d5, d6 + subs r4, r4, #2 + vst1.16 {q2}, [r0, :128], r1 + vmov q0, q3 + vst1.16 {q3}, [r6, :128], r1 + bgt 8b + vpop {q4-q7} + pop {r4-r8, pc} +160: +320: + sub r1, r1, r3, lsl #1 + mov lr, r3 + +1: + vld1.16 {d0}, [r2], r7 // left (0-1) + topleft (2) +2: + vld1.16 {q1, q2}, [r8]! // top(0-15) +.if \bpc == 10 + vmul.i16 q3, q8, d0[2] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d0[1] // p5(left[0]) * filter(5) + vmla.i16 q3, q14, d0[0] // p6(left[1]) * filter(6) + vmla.i16 q3, q9, d2[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d2[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d2[2] // p3(top[2]) * filter(3) + vmla.i16 q3, q12, d2[3] // p4(top[3]) * filter(4) + + vmul.i16 q4, q9, d3[0] // p1(top[0]) * filter(1) + vmla.i16 q4, q10, d3[1] // p2(top[1]) * filter(2) + vmla.i16 q4, q11, d3[2] // p3(top[2]) * filter(3) + vrshr.s16 q3, q3, #4 + vmax.s16 q3, q3, q7 + vmin.s16 q3, q3, q15 + vmla.i16 q4, q12, d3[3] // p4(top[3]) * filter(4) + vmla.i16 q4, q8, d2[3] // p0(topleft) * filter(0) + vmla.i16 q4, q13, d6[3] // p5(left[0]) * filter(5) + vmla.i16 q4, q14, d7[3] // p6(left[1]) * filter(6) + + vmul.i16 q5, q9, d4[0] // p1(top[0]) * filter(1) + vmla.i16 q5, q10, d4[1] // p2(top[1]) * filter(2) + vmla.i16 q5, q11, d4[2] // p3(top[2]) * filter(3) + vrshr.s16 q4, q4, #4 + vmax.s16 q4, q4, q7 + vmin.s16 q4, q4, q15 + vmov q0, q4 + vmla.i16 q5, q12, d4[3] // p4(top[3]) * filter(4) + vmla.i16 q5, q8, d3[3] // p0(topleft) * filter(0) + vmla.i16 q5, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q5, q14, d1[3] // p6(left[1]) * filter(6) + + vmul.i16 q6, q9, d5[0] // p1(top[0]) * filter(1) + vmla.i16 q6, q10, d5[1] // p2(top[1]) * filter(2) + vmla.i16 q6, q11, d5[2] // p3(top[2]) * filter(3) + vrshr.s16 q5, q5, #4 + vmax.s16 q5, q5, q7 + vmin.s16 q5, q5, q15 + vmov q0, q5 + vmov.u16 r12, d5[3] + vmla.i16 q6, q12, d5[3] // p4(top[3]) * filter(4) + vmla.i16 q6, q8, d4[3] // p0(topleft) * filter(0) + vmla.i16 q6, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q6, q14, d1[3] // p6(left[1]) * filter(6) + vmov.16 d0[2], r12 + subs r3, r3, #16 + vrshr.s16 q6, q6, #4 +.else + vmull.s16 q3, d16, d0[2] // p0(topleft) * filter(0) + vmlal.s16 q3, d26, d0[1] // p5(left[0]) * filter(5) + vmlal.s16 q3, d28, d0[0] // p6(left[1]) * filter(6) + vmlal.s16 q3, d18, d2[0] // p1(top[0]) * filter(1) + vmlal.s16 q3, d20, d2[1] // p2(top[1]) * filter(2) + vmlal.s16 q3, d22, d2[2] // p3(top[2]) * filter(3) + vmlal.s16 q3, d24, d2[3] // p4(top[3]) * filter(4) + vmull.s16 q4, d17, d0[2] // p0(topleft) * filter(0) + vmlal.s16 q4, d27, d0[1] // p5(left[0]) * filter(5) + vmlal.s16 q4, d29, d0[0] // p6(left[1]) * filter(6) + vmlal.s16 q4, d19, d2[0] // p1(top[0]) * filter(1) + vmlal.s16 q4, d21, d2[1] // p2(top[1]) * filter(2) + vmlal.s16 q4, d23, d2[2] // p3(top[2]) * filter(3) + vmlal.s16 q4, d25, d2[3] // p4(top[3]) * filter(4) + vqrshrun.s32 d6, q3, #4 + vmull.s16 q5, d18, d3[0] // p1(top[0]) * filter(1) + vmlal.s16 q5, d20, d3[1] // p2(top[1]) * filter(2) + vqrshrun.s32 d7, q4, #4 + vmin.s16 q3, q3, q15 + vmlal.s16 q5, d22, d3[2] // p3(top[2]) * filter(3) + vmlal.s16 q5, d24, d3[3] // p4(top[3]) * filter(4) + vmlal.s16 q5, d16, d2[3] // p0(topleft) * filter(0) + vmlal.s16 q5, d26, d6[3] // p5(left[0]) * filter(5) + vmlal.s16 q5, d28, d7[3] // p6(left[1]) * filter(6) + vmull.s16 q6, d19, d3[0] // p1(top[0]) * filter(1) + vmlal.s16 q6, d21, d3[1] // p2(top[1]) * filter(2) + vmlal.s16 q6, d23, d3[2] // p3(top[2]) * filter(3) + vmlal.s16 q6, d25, d3[3] // p4(top[3]) * filter(4) + vmlal.s16 q6, d17, d2[3] // p0(topleft) * filter(0) + vmlal.s16 q6, d27, d6[3] // p5(left[0]) * filter(5) + vmlal.s16 q6, d29, d7[3] // p6(left[1]) * filter(6) + vqrshrun.s32 d8, q5, #4 + vmull.s16 q7, d18, d4[0] // p1(top[0]) * filter(1) + vmlal.s16 q7, d20, d4[1] // p2(top[1]) * filter(2) + vmlal.s16 q7, d22, d4[2] // p3(top[2]) * filter(3) + vqrshrun.s32 d9, q6, #4 + vmin.s16 q0, q4, q15 + vmlal.s16 q7, d24, d4[3] // p4(top[3]) * filter(4) + vmlal.s16 q7, d16, d3[3] // p0(topleft) * filter(0) + vmlal.s16 q7, d26, d0[3] // p5(left[0]) * filter(5) + vmlal.s16 q7, d28, d1[3] // p6(left[1]) * filter(6) + vmin.s16 q4, q4, q15 + vmull.s16 q6, d19, d4[0] // p1(top[0]) * filter(1) + vmlal.s16 q6, d21, d4[1] // p2(top[1]) * filter(2) + vmlal.s16 q6, d23, d4[2] // p3(top[2]) * filter(3) + vmlal.s16 q6, d25, d4[3] // p4(top[3]) * filter(4) + vmlal.s16 q6, d17, d3[3] // p0(topleft) * filter(0) + vmlal.s16 q6, d27, d0[3] // p5(left[0]) * filter(5) + vmlal.s16 q6, d29, d1[3] // p6(left[1]) * filter(6) + vqrshrun.s32 d10, q7, #4 + vmull.s16 q1, d18, d5[0] // p1(top[0]) * filter(1) + vmlal.s16 q1, d20, d5[1] // p2(top[1]) * filter(2) + vmlal.s16 q1, d22, d5[2] // p3(top[2]) * filter(3) + vqrshrun.s32 d11, q6, #4 + vmin.s16 q0, q5, q15 + vmlal.s16 q1, d24, d5[3] // p4(top[3]) * filter(4) + vmlal.s16 q1, d16, d4[3] // p0(topleft) * filter(0) + vmlal.s16 q1, d26, d0[3] // p5(left[0]) * filter(5) + vmlal.s16 q1, d28, d1[3] // p6(left[1]) * filter(6) + vmin.s16 q5, q5, q15 + vmov.u16 r12, d5[3] + vmull.s16 q7, d19, d5[0] // p1(top[0]) * filter(1) + vmlal.s16 q7, d21, d5[1] // p2(top[1]) * filter(2) + vmlal.s16 q7, d23, d5[2] // p3(top[2]) * filter(3) + vmlal.s16 q7, d25, d5[3] // p4(top[3]) * filter(4) + vmlal.s16 q7, d17, d4[3] // p0(topleft) * filter(0) + vmlal.s16 q7, d27, d0[3] // p5(left[0]) * filter(5) + vmlal.s16 q7, d29, d1[3] // p6(left[1]) * filter(6) + vmov.16 d0[2], r12 + vqrshrun.s32 d12, q1, #4 + subs r3, r3, #16 + vqrshrun.s32 d13, q7, #4 +.endif + vswp q4, q5 +.if \bpc == 10 + vmax.s16 q6, q6, q7 +.endif + vswp d7, d10 + vmin.s16 q6, q6, q15 + + vswp d9, d12 + + vst1.16 {q3, q4}, [r0, :128]! + vst1.16 {q5, q6}, [r6, :128]! + ble 8f + vmov.u16 r12, d13[3] + vmov.16 d0[0], r12 + vmov.u16 r12, d9[3] + vmov.16 d0[1], r12 + b 2b +8: + subs r4, r4, #2 + + ble 9f + sub r8, r6, lr, lsl #1 + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + b 1b +9: + vpop {q4-q7} + pop {r4-r8, pc} +endfunc +.endm + +filter_fn 10 +filter_fn 12 + +function ipred_filter_16bpc_neon, export=1 + push {r4-r8, lr} + vpush {q4-q7} + movw r12, 0x3ff + ldr r8, [sp, #104] + cmp r8, r12 + ble ipred_filter_10bpc_neon + b ipred_filter_12bpc_neon +endfunc + +// void pal_pred_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_16bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + ldr r5, [sp, #16] + vld1.16 {q14}, [r2, :128] + clz lr, r4 + adr r12, L(pal_pred_tbl) + sub lr, lr, #25 + ldr lr, [r12, lr, lsl #2] + vmov.i16 q15, #0x100 + add r12, r12, lr + add r2, r0, r1 + bx r12 + + .align 2 +L(pal_pred_tbl): + .word 640f - L(pal_pred_tbl) + CONFIG_THUMB + .word 320f - L(pal_pred_tbl) + CONFIG_THUMB + .word 160f - L(pal_pred_tbl) + CONFIG_THUMB + .word 80f - L(pal_pred_tbl) + CONFIG_THUMB + .word 40f - L(pal_pred_tbl) + CONFIG_THUMB + +40: + lsl r1, r1, #1 +4: + vld1.8 {q1}, [r3, :128]! + subs r5, r5, #4 + // Restructure q1 from a, b, c, ... into 2*a, 2*a+1, 2*b, 2*b+1, 2*c, 2*c+1, ... + vadd.i8 q0, q1, q1 + vadd.i8 q1, q1, q1 + vzip.8 q0, q1 + vadd.i16 q0, q0, q15 + vadd.i16 q1, q1, q15 + vtbl.8 d0, {q14}, d0 + vtbl.8 d1, {q14}, d1 + vst1.16 {d0}, [r0, :64], r1 + vtbl.8 d2, {q14}, d2 + vst1.16 {d1}, [r2, :64], r1 + vtbl.8 d3, {q14}, d3 + vst1.16 {d2}, [r0, :64], r1 + vst1.16 {d3}, [r2, :64], r1 + bgt 4b + pop {r4-r5, pc} +80: + lsl r1, r1, #1 +8: + vld1.8 {q1, q2}, [r3, :128]! + subs r5, r5, #4 + // Prefer doing the adds twice, instead of chaining a vmov after + // the add. + vadd.i8 q0, q1, q1 + vadd.i8 q1, q1, q1 + vadd.i8 q3, q2, q2 + vadd.i8 q2, q2, q2 + vzip.8 q0, q1 + vzip.8 q2, q3 + vadd.i16 q0, q0, q15 + vadd.i16 q1, q1, q15 + vtbl.8 d0, {q14}, d0 + vadd.i16 q2, q2, q15 + vtbl.8 d1, {q14}, d1 + vadd.i16 q3, q3, q15 + vtbl.8 d2, {q14}, d2 + vtbl.8 d3, {q14}, d3 + vtbl.8 d4, {q14}, d4 + vtbl.8 d5, {q14}, d5 + vst1.16 {q0}, [r0, :128], r1 + vtbl.8 d6, {q14}, d6 + vst1.16 {q1}, [r2, :128], r1 + vtbl.8 d7, {q14}, d7 + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r2, :128], r1 + bgt 8b + pop {r4-r5, pc} +160: + lsl r1, r1, #1 +16: + vld1.8 {q2, q3}, [r3, :128]! + subs r5, r5, #4 + vld1.8 {q10, q11}, [r3, :128]! + vadd.i8 q0, q2, q2 + vadd.i8 q1, q2, q2 + vadd.i8 q2, q3, q3 + vadd.i8 q3, q3, q3 + vadd.i8 q8, q10, q10 + vadd.i8 q9, q10, q10 + vadd.i8 q10, q11, q11 + vzip.8 q0, q1 + vadd.i8 q11, q11, q11 + vzip.8 q2, q3 + vzip.8 q8, q9 + vadd.i16 q0, q0, q15 + vzip.8 q10, q11 + vadd.i16 q1, q1, q15 + vadd.i16 q2, q2, q15 + vadd.i16 q3, q3, q15 + vadd.i16 q8, q8, q15 + vadd.i16 q9, q9, q15 + vadd.i16 q10, q10, q15 + vtbl.8 d0, {q14}, d0 + vadd.i16 q11, q11, q15 + vtbl.8 d1, {q14}, d1 + vtbl.8 d2, {q14}, d2 + vtbl.8 d3, {q14}, d3 + vtbl.8 d4, {q14}, d4 + vtbl.8 d5, {q14}, d5 + vtbl.8 d6, {q14}, d6 + vtbl.8 d7, {q14}, d7 + vtbl.8 d16, {q14}, d16 + vtbl.8 d17, {q14}, d17 + vtbl.8 d18, {q14}, d18 + vst1.16 {q0, q1}, [r0, :128], r1 + vtbl.8 d19, {q14}, d19 + vtbl.8 d20, {q14}, d20 + vst1.16 {q2, q3}, [r2, :128], r1 + vtbl.8 d21, {q14}, d21 + vtbl.8 d22, {q14}, d22 + vst1.16 {q8, q9}, [r0, :128], r1 + vtbl.8 d23, {q14}, d23 + vst1.16 {q10, q11}, [r2, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + lsl r1, r1, #1 + sub r1, r1, #32 +32: + vld1.8 {q2, q3}, [r3, :128]! + subs r5, r5, #2 + vld1.8 {q10, q11}, [r3, :128]! + vadd.i8 q0, q2, q2 + vadd.i8 q1, q2, q2 + vadd.i8 q2, q3, q3 + vadd.i8 q3, q3, q3 + vadd.i8 q8, q10, q10 + vadd.i8 q9, q10, q10 + vadd.i8 q10, q11, q11 + vzip.8 q0, q1 + vadd.i8 q11, q11, q11 + vzip.8 q2, q3 + vzip.8 q8, q9 + vadd.i16 q0, q0, q15 + vzip.8 q10, q11 + vadd.i16 q1, q1, q15 + vadd.i16 q2, q2, q15 + vadd.i16 q3, q3, q15 + vadd.i16 q8, q8, q15 + vadd.i16 q9, q9, q15 + vadd.i16 q10, q10, q15 + vtbl.8 d0, {q14}, d0 + vadd.i16 q11, q11, q15 + vtbl.8 d1, {q14}, d1 + vtbl.8 d2, {q14}, d2 + vtbl.8 d3, {q14}, d3 + vtbl.8 d4, {q14}, d4 + vtbl.8 d5, {q14}, d5 + vtbl.8 d6, {q14}, d6 + vtbl.8 d7, {q14}, d7 + vtbl.8 d16, {q14}, d16 + vtbl.8 d17, {q14}, d17 + vtbl.8 d18, {q14}, d18 + vst1.16 {q0, q1}, [r0, :128]! + vtbl.8 d19, {q14}, d19 + vtbl.8 d20, {q14}, d20 + vst1.16 {q2, q3}, [r0, :128], r1 + vtbl.8 d21, {q14}, d21 + vtbl.8 d22, {q14}, d22 + vst1.16 {q8, q9}, [r2, :128]! + vtbl.8 d23, {q14}, d23 + vst1.16 {q10, q11}, [r2, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + sub r1, r1, #96 +64: + vld1.8 {q2, q3}, [r3, :128]! + subs r5, r5, #1 + vld1.8 {q10, q11}, [r3, :128]! + vadd.i8 q0, q2, q2 + vadd.i8 q1, q2, q2 + vadd.i8 q2, q3, q3 + vadd.i8 q3, q3, q3 + vadd.i8 q8, q10, q10 + vadd.i8 q9, q10, q10 + vadd.i8 q10, q11, q11 + vzip.8 q0, q1 + vadd.i8 q11, q11, q11 + vzip.8 q2, q3 + vzip.8 q8, q9 + vadd.i16 q0, q0, q15 + vzip.8 q10, q11 + vadd.i16 q1, q1, q15 + vadd.i16 q2, q2, q15 + vadd.i16 q3, q3, q15 + vadd.i16 q8, q8, q15 + vadd.i16 q9, q9, q15 + vadd.i16 q10, q10, q15 + vtbl.8 d0, {q14}, d0 + vadd.i16 q11, q11, q15 + vtbl.8 d1, {q14}, d1 + vtbl.8 d2, {q14}, d2 + vtbl.8 d3, {q14}, d3 + vtbl.8 d4, {q14}, d4 + vtbl.8 d5, {q14}, d5 + vtbl.8 d6, {q14}, d6 + vtbl.8 d7, {q14}, d7 + vtbl.8 d16, {q14}, d16 + vtbl.8 d17, {q14}, d17 + vtbl.8 d18, {q14}, d18 + vst1.16 {q0, q1}, [r0, :128]! + vtbl.8 d19, {q14}, d19 + vtbl.8 d20, {q14}, d20 + vst1.16 {q2, q3}, [r0, :128]! + vtbl.8 d21, {q14}, d21 + vtbl.8 d22, {q14}, d22 + vst1.16 {q8, q9}, [r0, :128]! + vtbl.8 d23, {q14}, d23 + vst1.16 {q10, q11}, [r0, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_cfl_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_128_16bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + clz lr, r3 + vdup.16 q15, r7 // bitdepth_max + adr r12, L(ipred_cfl_128_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vrshr.u16 q0, q15, #1 + vdup.16 q1, r6 // alpha + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + vmov.i16 q14, #0 + bx r12 + + .align 2 +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w8) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w4) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + +L(ipred_cfl_splat_w4): + vld1.16 {q8, q9}, [r5, :128]! + vmull.s16 q2, d16, d2 // diff = ac * alpha + vmull.s16 q3, d17, d3 + vmull.s16 q8, d18, d2 + vmull.s16 q9, d19, d3 + vshr.s32 q10, q2, #31 // sign = diff >> 15 + vshr.s32 q11, q3, #31 + vshr.s32 q12, q8, #31 + vshr.s32 q13, q9, #31 + vadd.i32 q2, q2, q10 // diff + sign + vadd.i32 q3, q3, q11 + vadd.i32 q8, q8, q12 + vadd.i32 q9, q9, q13 + vrshrn.i32 d4, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshrn.i32 d5, q3, #6 + vrshrn.i32 d6, q8, #6 + vrshrn.i32 d7, q9, #6 + vadd.i16 q2, q2, q0 // dc + apply_sign() + vadd.i16 q3, q3, q0 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q2, q2, q15 + vmin.s16 q3, q3, q15 + vst1.16 {d4}, [r0, :64], r1 + vst1.16 {d5}, [r6, :64], r1 + subs r4, r4, #4 + vst1.16 {d6}, [r0, :64], r1 + vst1.16 {d7}, [r6, :64], r1 + bgt L(ipred_cfl_splat_w4) + pop {r4-r8, pc} +L(ipred_cfl_splat_w8): + vld1.16 {q8, q9}, [r5, :128]! + subs r4, r4, #2 + vmull.s16 q2, d16, d2 // diff = ac * alpha + vmull.s16 q3, d17, d3 + vmull.s16 q8, d18, d2 + vmull.s16 q9, d19, d3 + vshr.s32 q10, q2, #31 // sign = diff >> 15 + vshr.s32 q11, q3, #31 + vshr.s32 q12, q8, #31 + vshr.s32 q13, q9, #31 + vadd.i32 q2, q2, q10 // diff + sign + vadd.i32 q3, q3, q11 + vadd.i32 q8, q8, q12 + vadd.i32 q9, q9, q13 + vrshrn.i32 d4, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshrn.i32 d5, q3, #6 + vrshrn.i32 d6, q8, #6 + vrshrn.i32 d7, q9, #6 + vadd.i16 q2, q2, q0 // dc + apply_sign() + vadd.i16 q3, q3, q0 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q2, q2, q15 + vmin.s16 q3, q3, q15 + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r6, :128], r1 + bgt L(ipred_cfl_splat_w8) + pop {r4-r8, pc} +L(ipred_cfl_splat_w16): + vpush {q4-q7} + add r12, r5, r3, lsl #1 + sub r1, r1, r3, lsl #1 + mov lr, r3 +1: + vld1.16 {q6, q7}, [r5, :128]! + vmull.s16 q2, d12, d2 // diff = ac * alpha + vld1.16 {q8, q9}, [r12, :128]! + vmull.s16 q3, d13, d3 + vmull.s16 q4, d14, d2 + vmull.s16 q5, d15, d3 + vmull.s16 q6, d16, d2 + vmull.s16 q7, d17, d3 + vmull.s16 q8, d18, d2 + vmull.s16 q9, d19, d3 + vshr.s32 q10, q2, #31 // sign = diff >> 15 + vshr.s32 q11, q3, #31 + vshr.s32 q12, q4, #31 + vshr.s32 q13, q5, #31 + vadd.i32 q2, q2, q10 // diff + sign + vshr.s32 q10, q6, #31 + vadd.i32 q3, q3, q11 + vshr.s32 q11, q7, #31 + vadd.i32 q4, q4, q12 + vshr.s32 q12, q8, #31 + vadd.i32 q5, q5, q13 + vshr.s32 q13, q9, #31 + vadd.i32 q6, q6, q10 + vadd.i32 q7, q7, q11 + vadd.i32 q8, q8, q12 + vadd.i32 q9, q9, q13 + vrshrn.i32 d4, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshrn.i32 d5, q3, #6 + vrshrn.i32 d6, q4, #6 + vrshrn.i32 d7, q5, #6 + vadd.i16 q2, q2, q0 // dc + apply_sign() + vrshrn.i32 d8, q6, #6 + vrshrn.i32 d9, q7, #6 + vadd.i16 q3, q3, q0 + vrshrn.i32 d10, q8, #6 + vrshrn.i32 d11, q9, #6 + vadd.i16 q4, q4, q0 + vadd.i16 q5, q5, q0 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmax.s16 q4, q4, q14 + vmax.s16 q5, q5, q14 + vmin.s16 q2, q2, q15 + vmin.s16 q3, q3, q15 + vmin.s16 q4, q4, q15 + vmin.s16 q5, q5, q15 + subs r3, r3, #16 + vst1.16 {q2, q3}, [r0, :128]! + vst1.16 {q4, q5}, [r6, :128]! + bgt 1b + subs r4, r4, #2 + add r5, r5, lr, lsl #1 + add r12, r12, lr, lsl #1 + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + bgt 1b + vpop {q4-q7} + pop {r4-r8, pc} +endfunc + +// void ipred_cfl_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_top_16bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + clz lr, r3 + vdup.16 q15, r7 // bitdepth_max + adr r12, L(ipred_cfl_top_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vdup.16 q1, r6 // alpha + add r2, r2, #2 + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + vmov.i16 q14, #0 + bx r12 + + .align 2 +L(ipred_cfl_top_tbl): + .word 32f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 16f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 8f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 4f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + +4: + vld1.16 {d0}, [r2] + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) +8: + vld1.16 {q0}, [r2] + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) +16: + vld1.16 {q2, q3}, [r2] + vadd.i16 q0, q2, q3 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +32: + vld1.16 {q8, q9}, [r2]! + vld1.16 {q10, q11}, [r2] + vadd.i16 q8, q8, q9 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q8, q10 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpaddl.u16 d0, d0 + vrshrn.i32 d0, q0, #5 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void ipred_cfl_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_left_16bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + sub r2, r2, r4, lsl #1 + clz lr, r3 + clz r8, r4 + vdup.16 q15, r7 // bitdepth_max + adr r12, L(ipred_cfl_splat_tbl) + adr r7, L(ipred_cfl_left_tbl) + sub lr, lr, #26 + sub r8, r8, #26 + ldr lr, [r12, lr, lsl #2] + ldr r8, [r7, r8, lsl #2] + vdup.16 q1, r6 // alpha + add r12, r12, lr + add r7, r7, r8 + add r6, r0, r1 + lsl r1, r1, #1 + vmov.i16 q14, #0 + bx r7 + + .align 2 +L(ipred_cfl_left_tbl): + .word L(ipred_cfl_left_h32) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h16) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h8) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h4) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + +L(ipred_cfl_left_h4): + vld1.16 {d0}, [r2, :64] + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h8): + vld1.16 {q0}, [r2, :128] + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h16): + vld1.16 {q2, q3}, [r2, :128] + vadd.i16 q0, q2, q3 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpadd.i16 d0, d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h32): + vld1.16 {q8, q9}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128] + vadd.i16 q8, q8, q9 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q8, q10 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + vpaddl.u16 d0, d0 + vrshrn.i32 d0, q0, #5 + vdup.16 q0, d0[0] + bx r12 +endfunc + +// void ipred_cfl_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_16bpc_neon, export=1 + push {r4-r8, lr} + ldrd r4, r5, [sp, #24] + ldrd r6, r7, [sp, #32] + sub r2, r2, r4, lsl #1 + add r8, r3, r4 // width + height + vdup.16 q1, r6 // alpha + clz lr, r3 + clz r6, r4 + vdup.32 d16, r8 // width + height + vdup.16 q15, r7 // bitdepth_max + adr r7, L(ipred_cfl_tbl) + rbit r8, r8 // rbit(width + height) + sub lr, lr, #22 // 26 leading bits, minus table offset 4 + sub r6, r6, #26 + clz r8, r8 // ctz(width + height) + ldr lr, [r7, lr, lsl #2] + ldr r6, [r7, r6, lsl #2] + neg r8, r8 // -ctz(width + height) + add r12, r7, lr + add r7, r7, r6 + vshr.u32 d16, d16, #1 // (width + height) >> 1 + vdup.32 d17, r8 // -ctz(width + height) + add r6, r0, r1 + lsl r1, r1, #1 + vmov.i16 q14, #0 + bx r7 + + .align 2 +L(ipred_cfl_tbl): + .word L(ipred_cfl_h32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h4) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w4) - L(ipred_cfl_tbl) + CONFIG_THUMB + +L(ipred_cfl_h4): + vld1.16 {d0}, [r2, :64]! + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r12 +L(ipred_cfl_w4): + vld1.16 {d1}, [r2] + vadd.i32 d0, d0, d16 + vpadd.i16 d1, d1, d1 + vpaddl.u16 d1, d1 + cmp r4, #4 + vadd.i32 d0, d0, d1 + vshl.u32 d0, d0, d17 + beq 1f + // h = 8/16 + cmp r4, #16 + movw lr, #0x6667 + movw r8, #0xAAAB + it ne + movne lr, r8 + vdup.32 d18, lr + vmul.i32 d0, d0, d18 + vshr.u32 d0, d0, #17 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + vld1.16 {q0}, [r2, :128]! + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r12 +L(ipred_cfl_w8): + vld1.16 {q2}, [r2] + vadd.i32 d0, d0, d16 + vadd.i16 d1, d4, d5 + vpadd.i16 d1, d1, d1 + vpaddl.u16 d1, d1 + cmp r4, #8 + vadd.i32 d0, d0, d1 + vshl.u32 d0, d0, d17 + beq 1f + // h = 4/16/32 + cmp r4, #32 + movw lr, #0x6667 + movw r8, #0xAAAB + it ne + movne lr, r8 + vdup.32 d18, lr + vmul.i32 d0, d0, d18 + vshr.u32 d0, d0, #17 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + vld1.16 {q2, q3}, [r2, :128]! + vadd.i16 q0, q2, q3 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r12 +L(ipred_cfl_w16): + vld1.16 {q2, q3}, [r2] + vadd.i32 d0, d0, d16 + vadd.i16 q2, q2, q3 + vadd.i16 d1, d4, d5 + vpadd.i16 d1, d1, d1 + vpaddl.u16 d1, d1 + cmp r4, #16 + vadd.i32 d0, d0, d1 + vshl.u32 d0, d0, d17 + beq 1f + // h = 4/8/32/64 + tst r4, #(32+16+8) // 16 added to make a consecutive bitmask + movw lr, #0x6667 + movw r8, #0xAAAB + it ne + movne lr, r8 + vdup.32 d18, lr + vmul.i32 d0, d0, d18 + vshr.u32 d0, d0, #17 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + vld1.16 {q2, q3}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128]! + vadd.i16 q2, q2, q3 + vadd.i16 q10, q10, q11 + vadd.i16 q0, q2, q10 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0, d0 + add r2, r2, #2 + vpaddl.u16 d0, d0 + bx r12 +L(ipred_cfl_w32): + vld1.16 {q2, q3}, [r2]! + vadd.i32 d0, d0, d16 + vld1.16 {q10, q11}, [r2]! + vadd.i16 q2, q2, q3 + vadd.i16 q10, q10, q11 + vadd.i16 q2, q2, q10 + vadd.i16 d1, d4, d5 + vpadd.i16 d1, d1, d1 + vpaddl.u16 d1, d1 + cmp r4, #32 + vadd.i32 d0, d0, d1 + vshl.u32 d0, d0, d17 + beq 1f + // h = 8/16/64 + cmp r4, #8 + movw lr, #0x6667 + movw r8, #0xAAAB + it ne + movne lr, r8 + vdup.32 d18, lr + vmul.i32 d0, d0, d18 + vshr.u32 d0, d0, #17 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void cfl_ac_420_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_16bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_420_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i32 q8, #0 + vmov.i32 q9, #0 + vmov.i32 q10, #0 + vmov.i32 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_420_tbl): + .word L(ipred_cfl_ac_420_w16) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w8) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w4) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w4): +1: // Copy and subsample input + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q1}, [r12, :128], r2 + vld1.16 {q2}, [r1, :128], r2 + vld1.16 {q3}, [r12, :128], r2 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d4, d5 + vshl.i16 q0, q0, #1 + subs r8, r8, #2 + vst1.16 {q0}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + bgt 1b + cmp r4, #0 + vmov d0, d1 + vmov d2, d1 + vmov d3, d1 +L(ipred_cfl_ac_420_w4_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + vadd.i32 q8, q8, q9 + vadd.i32 q10, q10, q11 + vadd.i32 q0, q8, q10 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d0, d0 // sum + sub r0, r0, r6, lsl #3 + vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz + vdup.16 q8, d16[0] +6: // Subtract dc from ac + vld1.16 {q0, q1}, [r0, :128] + subs r6, r6, #4 + vsub.i16 q0, q0, q8 + vsub.i16 q1, q1, q8 + vst1.16 {q0, q1}, [r0, :128]! + bgt 6b + pop {r4-r8, pc} + +L(ipred_cfl_ac_420_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + vld1.16 {q0, q1}, [r1, :128], r2 + vld1.16 {q2, q3}, [r12, :128], r2 + vld1.16 {q12, q13}, [r1, :128], r2 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vld1.16 {q2, q3}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vadd.i16 q12, q12, q2 + vadd.i16 q13, q13, q3 + vpadd.i16 d2, d24, d25 + vpadd.i16 d3, d26, d27 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q1, #1 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + vmov q0, q1 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q1}, [r12, :128], r2 + vld1.16 {q2}, [r1, :128], r2 + vld1.16 {q3}, [r12, :128], r2 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d4, d5 + vshl.i16 q0, q0, #1 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + vmov q0, q1 + +L(ipred_cfl_ac_420_w8_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl r6, r6, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + adr r7, L(ipred_cfl_ac_420_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_420_w16_tbl): + .word L(ipred_cfl_ac_420_w16_wpad0) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad1) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad2) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad3) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w16_wpad0): + sub r2, r2, #32 +1: // Copy and subsample input, without padding + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q12, q13}, [r12, :128]! + vld1.16 {q2, q3}, [r1, :128], r2 + vadd.i16 q0, q0, q12 + vadd.i16 q1, q1, q13 + vld1.16 {q12, q13}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vadd.i16 q2, q2, q12 + vadd.i16 q3, q3, q13 + vpadd.i16 d2, d4, d5 + vpadd.i16 d3, d6, d7 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q1, #1 + subs r8, r8, #1 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): + sub r2, r2, #32 +1: // Copy and subsample input, padding 4 + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q12, q13}, [r12, :128]! + vld1.16 {q2}, [r1, :128], r2 + vadd.i16 q0, q0, q12 + vadd.i16 q1, q1, q13 + vld1.16 {q12}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vadd.i16 q2, q2, q12 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d4, d5 + vshl.i16 q0, q0, #1 + vshl.i16 d2, d2, #1 + subs r8, r8, #1 + vdup.16 d3, d2[3] + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.16 {q0, q1}, [r1, :128], r2 + vld1.16 {q12, q13}, [r12, :128], r2 + vadd.i16 q0, q0, q12 + vadd.i16 q1, q1, q13 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vshl.i16 q0, q0, #1 + subs r8, r8, #1 + vdup.16 q1, d1[3] + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q12}, [r12, :128], r2 + vadd.i16 q0, q0, q12 + vpadd.i16 d0, d0, d1 + vshl.i16 d0, d0, #1 + subs r8, r8, #1 + vdup.16 q1, d0[3] + vdup.16 d1, d0[3] + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl r6, r6, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) +endfunc + +// void cfl_ac_422_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_16bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_422_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_422_tbl): + .word L(ipred_cfl_ac_422_w16) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w8) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w4) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w4): +1: // Copy and subsample input + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q1}, [r12, :128], r2 + vld1.16 {q2}, [r1, :128], r2 + vld1.16 {q3}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d4, d5 + vpadd.i16 d3, d6, d7 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + vld1.16 {q0, q1}, [r1, :128], r2 + vld1.16 {q2, q3}, [r12, :128], r2 + vld1.16 {q12, q13}, [r1, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d4, d5 + vpadd.i16 d3, d6, d7 + vld1.16 {q2, q3}, [r12, :128], r2 + vpadd.i16 d24, d24, d25 + vpadd.i16 d25, d26, d27 + vpadd.i16 d26, d4, d5 + vpadd.i16 d27, d6, d7 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q12, #2 + vshl.i16 q3, q13, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q2}, [r12, :128], r2 + vld1.16 {q12}, [r1, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d4, d5 + vld1.16 {q2, q3}, [r12, :128], r2 + vpadd.i16 d24, d24, d25 + vpadd.i16 d25, d4, d5 + vshl.i16 q0, q0, #2 + vshl.i16 q12, q12, #2 + vdup.16 d7, d25[3] + vmov d6, d25 + vdup.16 d5, d24[3] + vmov d4, d24 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + adr r7, L(ipred_cfl_ac_422_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_422_w16_tbl): + .word L(ipred_cfl_ac_422_w16_wpad0) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad1) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad2) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad3) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w16_wpad0): + sub r2, r2, #32 +1: // Copy and subsample input, without padding + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q2, q3}, [r12, :128]! + vld1.16 {q12, q13}, [r1, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d24, d25 + vpadd.i16 d3, d26, d27 + vld1.16 {q12, q13}, [r12, :128], r2 + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vpadd.i16 d6, d24, d25 + vpadd.i16 d7, d26, d27 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q2, #2 + vshl.i16 q3, q3, #2 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): + sub r2, r2, #32 +1: // Copy and subsample input, padding 4 + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q2, q3}, [r12, :128]! + vld1.16 {q12}, [r1, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d2, d24, d25 + vld1.16 {q12}, [r12, :128], r2 + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vpadd.i16 d6, d24, d25 + vshl.i16 q0, q0, #2 + vshl.i16 d2, d2, #2 + vshl.i16 q2, q2, #2 + vshl.i16 d6, d6, #2 + vdup.16 d3, d2[3] + vdup.16 d7, d6[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.16 {q0, q1}, [r1, :128], r2 + vld1.16 {q2, q3}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vshl.i16 q0, q0, #2 + vshl.i16 q2, q2, #2 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q2}, [r12, :128], r2 + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d4, d5 + vshl.i16 q0, q0, #2 + vdup.16 q3, d1[3] + vdup.16 q1, d0[3] + vdup.16 d5, d1[3] + vmov d4, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) +endfunc + +// void cfl_ac_444_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_16bpc_neon, export=1 + push {r4-r8,lr} + ldrd r4, r5, [sp, #24] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_444_tbl) + sub r8, r8, #26 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_444_tbl): + .word L(ipred_cfl_ac_444_w32) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w16) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w8) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w4) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w4): +1: // Copy and expand input + vld1.16 {d0}, [r1, :64], r2 + vld1.16 {d1}, [r12, :64], r2 + vld1.16 {d2}, [r1, :64], r2 + vld1.16 {d3}, [r12, :64], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): +1: // Copy and expand input + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q1}, [r12, :128], r2 + vld1.16 {q2}, [r1, :128], r2 + vld1.16 {q3}, [r12, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + vshl.i16 q2, q2, #3 + vshl.i16 q3, q3, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + cmp r3, #0 + bne L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + vld1.16 {q0, q1}, [r1, :128], r2 + vld1.16 {q2, q3}, [r12, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + vshl.i16 q2, q2, #3 + vshl.i16 q3, q3, #3 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + vld1.16 {q0}, [r1, :128], r2 + vld1.16 {q2}, [r12, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q2, q2, #3 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + adr r7, L(ipred_cfl_ac_444_w32_tbl) + ldr r3, [r7, r3, lsl #1] // (w3>>1) << 2 + asr r2, r2, #1 + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_444_w32_tbl): + .word L(ipred_cfl_ac_444_w32_wpad0) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad2) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad4) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad6) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w32_wpad0): + sub r2, r2, #32 +1: // Copy and expand input, without padding + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q2, q3}, [r1, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + vshl.i16 q2, q2, #3 + vshl.i16 q3, q3, #3 + subs r8, r8, #1 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): + sub r2, r2, #32 +1: // Copy and expand input, padding 8 + vld1.16 {q0, q1}, [r1, :128]! + vld1.16 {q2}, [r1, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + vshl.i16 q2, q2, #3 + subs r8, r8, #1 + vst1.16 {q0, q1}, [r0, :128]! + vdup.16 q3, d5[3] + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): +1: // Copy and expand input, padding 16 + vld1.16 {q0, q1}, [r1, :128], r2 + vshl.i16 q0, q0, #3 + vshl.i16 q1, q1, #3 + subs r8, r8, #1 + vst1.16 {q0, q1}, [r0, :128]! + vdup.16 q2, d3[3] + vdup.16 q3, d3[3] + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): +1: // Copy and expand input, padding 24 + vld1.16 {q0}, [r1, :128], r2 + vshl.i16 q0, q0, #3 + subs r8, r8, #1 + vdup.16 q1, d1[3] + vst1.16 {q0, q1}, [r0, :128]! + vdup.16 q2, d1[3] + vdup.16 q3, d1[3] + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 1b + cmp r4, #0 + +L(ipred_cfl_ac_444_w32_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #1 + vst1.16 {q0, q1}, [r0, :128]! + vaddw.u16 q8, q8, d0 + vaddw.u16 q9, q9, d1 + vaddw.u16 q10, q10, d2 + vaddw.u16 q11, q11, d3 + vst1.16 {q2, q3}, [r0, :128]! + vaddw.u16 q8, q8, d4 + vaddw.u16 q9, q9, d5 + vaddw.u16 q10, q10, d6 + vaddw.u16 q11, q11, d7 + bgt 2b +3: + + // Multiply the height by eight and reuse the w4 subtracting + lsl r6, r6, #3 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) +endfunc diff --git a/third_party/dav1d/src/arm/32/itx.S b/third_party/dav1d/src/arm/32/itx.S new file mode 100644 index 0000000000..ceea025e45 --- /dev/null +++ b/third_party/dav1d/src/arm/32/itx.S @@ -0,0 +1,3343 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob); + +// Most of the functions use the following register layout: +// r0-r3 external parameters +// r4 function pointer to first transform +// r5 function pointer to second transform +// r6 output parameter for helper function +// r7 input parameter for helper function +// r8 input stride for helper function +// r9 scratch variable for helper functions +// r10-r11 pointer to list of eob thresholds, eob threshold value, +// scratch variables within helper functions (backed up) + +// The SIMD registers most often use the following layout: +// d0-d3 multiplication coefficients +// d4-d7 scratch registers +// d8-d15 unused in some transforms, used for scratch registers in others +// d16-v31 inputs/outputs of transforms + +// Potential further optimizations, that are left unimplemented for now: +// - Trying to keep multiplication coefficients in registers across multiple +// transform functions. (The register layout is designed to potentially +// allow this.) +// - Use a simplified version of the transforms themselves for cases where +// we know a significant number of inputs are zero. E.g. if the eob value +// indicates only a quarter of input values are set, for idct16 and up, +// a significant amount of calculation can be skipped, at the cost of more +// code duplication and special casing. + +const idct_coeffs, align=4 + // idct4 + .short 2896, 2896*8, 1567, 3784 + // idct8 + .short 799, 4017, 3406, 2276 + // idct16 + .short 401, 4076, 3166, 2598 + .short 1931, 3612, 3920, 1189 + // idct32 + .short 201, 4091, 3035, 2751 + .short 1751, 3703, 3857, 1380 + .short 995, 3973, 3513, 2106 + .short 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .short 101*8, 4095*8, 2967*8, -2824*8 + .short 1660*8, 3745*8, 3822*8, -1474*8 + .short 4076, 401, 4017, 799 + + .short 4036*8, -700*8, 2359*8, 3349*8 + .short 3461*8, -2191*8, 897*8, 3996*8 + .short -3166, -2598, -799, -4017 + + .short 501*8, 4065*8, 3229*8, -2520*8 + .short 2019*8, 3564*8, 3948*8, -1092*8 + .short 3612, 1931, 2276, 3406 + + .short 4085*8, -301*8, 2675*8, 3102*8 + .short 3659*8, -1842*8, 1285*8, 3889*8 + .short -3920, -1189, -3406, -2276 +endconst + +const iadst4_coeffs, align=4 + // .h[4-5] can be interpreted as .s[2] + .short 1321, 3803, 2482, 3344, 3344, 0 +endconst + +const iadst8_coeffs, align=4 + .short 4076, 401, 3612, 1931 + .short 2598, 3166, 1189, 3920 + // idct_coeffs + .short 2896, 0, 1567, 3784, 0, 0, 0, 0 +endconst + +const iadst16_coeffs, align=4 + .short 4091, 201, 3973, 995 + .short 3703, 1751, 3290, 2440 + .short 2751, 3035, 2106, 3513 + .short 1380, 3857, 601, 4052 +endconst + +.macro vmull_vmlal d0, s0, s1, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlal.s16 \d0, \s1, \c1 +.endm + +.macro vmull_vmlal_8h d0, d1, s0, s1, s2, s3, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlal.s16 \d0, \s2, \c1 + vmull.s16 \d1, \s1, \c0 + vmlal.s16 \d1, \s3, \c1 +.endm + +.macro vmull_vmlsl d0, s0, s1, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlsl.s16 \d0, \s1, \c1 +.endm + +.macro vmull_vmlsl_8h d0, d1, s0, s1, s2, s3, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlsl.s16 \d0, \s2, \c1 + vmull.s16 \d1, \s1, \c0 + vmlsl.s16 \d1, \s3, \c1 +.endm + +.macro vqrshrn_8h d0, d1, s0, s1, shift + vqrshrn.s32 \d0, \s0, \shift + vqrshrn.s32 \d1, \s1, \shift +.endm + +.macro scale_input c, r0, r1, r2 r3, r4, r5, r6, r7 + vqrdmulh.s16 \r0, \r0, \c + vqrdmulh.s16 \r1, \r1, \c +.ifnb \r2 + vqrdmulh.s16 \r2, \r2, \c + vqrdmulh.s16 \r3, \r3, \c +.endif +.ifnb \r4 + vqrdmulh.s16 \r4, \r4, \c + vqrdmulh.s16 \r5, \r5, \c + vqrdmulh.s16 \r6, \r6, \c + vqrdmulh.s16 \r7, \r7, \c +.endif +.endm + +.macro load_add_store load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src, shiftbits=4 +.ifnb \load + vld1.8 {\load}, [\src, :64], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + vaddw.u8 \adddst, \adddst, \addsrc +.endif +.ifnb \narrowsrc + vqmovun.s16 \narrowdst, \narrowsrc +.endif +.ifnb \store + vst1.8 {\store}, [\dst, :64], r1 +.endif +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + load_add_store d2, q8, , , , , , \dst, \src, \shiftbits + load_add_store d3, q9, , , , , , \dst, \src, \shiftbits + load_add_store d4, q10, d2, q8, , , , \dst, \src, \shiftbits + load_add_store d5, q11, d3, q9, q8, d2, , \dst, \src, \shiftbits + load_add_store d6, q12, d4, q10, q9, d3, d2, \dst, \src, \shiftbits + load_add_store d7, q13, d5, q11, q10, d4, d3, \dst, \src, \shiftbits + load_add_store d2, q14, d6, q12, q11, d5, d4, \dst, \src, \shiftbits + load_add_store d3, q15, d7, q13, q12, d6, d5, \dst, \src, \shiftbits + load_add_store , , d2, q14, q13, d7, d6, \dst, \src, \shiftbits + load_add_store , , d3, q15, q14, d2, d7, \dst, \src, \shiftbits + load_add_store , , , , q15, d3, d2, \dst, \src, \shiftbits + load_add_store , , , , , , d3, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src + mov \src, \dst + load_add_store d2, q8, , , , , , \dst, \src + load_add_store d3, q9, , , , , , \dst, \src + load_add_store d4, q10, d2, q8, , , , \dst, \src + load_add_store d5, q11, d3, q9, q8, d2, , \dst, \src + load_add_store , , d4, q10, q9, d3, d2, \dst, \src + load_add_store , , d5, q11, q10, d4, d3, \dst, \src + load_add_store , , , , q11, d5, d4, \dst, \src + load_add_store , , , , , , d5, \dst, \src +.endm +.macro load_add_store4 load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src +.ifnb \load + vld1.32 {\load[0]}, [\src, :32], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #4 +.endif +.ifnb \load + vld1.32 {\load[1]}, [\src, :32], r1 +.endif +.ifnb \addsrc + vaddw.u8 \adddst, \adddst, \addsrc +.endif +.ifnb \store + vst1.32 {\store[0]}, [\dst, :32], r1 +.endif +.ifnb \narrowsrc + vqmovun.s16 \narrowdst, \narrowsrc +.endif +.ifnb \store + vst1.32 {\store[1]}, [\dst, :32], r1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + load_add_store4 d0, , , , , , , \dst, \src + load_add_store4 d1, q8, , , , , , \dst, \src + load_add_store4 d2, q9, d0, q8, , , , \dst, \src + load_add_store4 d3, q10, d1, q9, q8, d0, , \dst, \src + load_add_store4 d4, q11, d2, q10, q9, d1, d0, \dst, \src + load_add_store4 d5, q12, d3, q11, q10, d2, d1, \dst, \src + load_add_store4 d6, q13, d4, q12, q11, d3, d2, \dst, \src + load_add_store4 d7, q14, d5, q13, q12, d4, d3, \dst, \src + load_add_store4 , q15, d6, q14, q13, d5, d4, \dst, \src + load_add_store4 , , d7, q15, q14, d6, d5, \dst, \src + load_add_store4 , , , , q15, d7, d6, \dst, \src + load_add_store4 , , , , , , d7, \dst, \src +.endm +.macro load_add_store_4x8 dst, src + mov \src, \dst + load_add_store4 d0, , , , , , , \dst, \src + load_add_store4 d1, q8, , , , , , \dst, \src + load_add_store4 d2, q9, d0, q8, , , , \dst, \src + load_add_store4 d3, q10, d1, q9, q8, d0, , \dst, \src + load_add_store4 , q11, d2, q10, q9, d1, d0, \dst, \src + load_add_store4 , , d3, q11, q10, d2, d1, \dst, \src + load_add_store4 , , , , q11, d3, d2, \dst, \src + load_add_store4 , , , , , , d3, \dst, \src +.endm + +.macro idct_dc w, h, shift + cmp r3, #0 + bne 1f + vmov.i16 d30, #0 + movw r12, #2896*8 + vld1.16 {d16[]}, [r2, :16] + vdup.16 d0, r12 + vqrdmulh.s16 d16, d16, d0[0] + vst1.16 {d30[0]}, [r2, :16] +.if (\w == 2*\h) || (2*\w == \h) + vqrdmulh.s16 d16, d16, d0[0] +.endif +.if \shift > 0 + vrshr.s16 d16, d16, #\shift +.endif + vqrdmulh.s16 d20, d16, d0[0] + mov r3, #\h + vrshr.s16 d16, d20, #4 + vrshr.s16 d17, d20, #4 + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon +1: + vld1.32 {d0[0]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[0]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + subs r3, r3, #4 + sub r0, r0, r1, lsl #2 + vaddw.u8 q10, q8, d0 + vqmovun.s16 d0, q10 + vaddw.u8 q11, q8, d1 + vst1.32 {d0[0]}, [r0, :32], r1 + vqmovun.s16 d1, q11 + vst1.32 {d0[1]}, [r0, :32], r1 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w8_neon +1: + vld1.8 {d0}, [r0, :64], r1 + vld1.8 {d1}, [r0, :64], r1 + vld1.8 {d2}, [r0, :64], r1 + vaddw.u8 q10, q8, d0 + vld1.8 {d3}, [r0, :64], r1 + sub r0, r0, r1, lsl #2 + subs r3, r3, #4 + vaddw.u8 q11, q8, d1 + vqmovun.s16 d0, q10 + vaddw.u8 q12, q8, d2 + vqmovun.s16 d1, q11 + vaddw.u8 q13, q8, d3 + vst1.8 {d0}, [r0, :64], r1 + vqmovun.s16 d2, q12 + vst1.8 {d1}, [r0, :64], r1 + vqmovun.s16 d3, q13 + vst1.8 {d2}, [r0, :64], r1 + vst1.8 {d3}, [r0, :64], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w16_neon +1: + vld1.8 {q0}, [r0, :128], r1 + vld1.8 {q1}, [r0, :128], r1 + vld1.8 {q2}, [r0, :128], r1 + subs r3, r3, #4 + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vld1.8 {q3}, [r0, :128], r1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, r1, lsl #2 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0}, [r0, :128], r1 + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q1}, [r0, :128], r1 + vst1.8 {q2}, [r0, :128], r1 + vst1.8 {q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w32_neon +1: + vld1.8 {q0, q1}, [r0, :128], r1 + subs r3, r3, #2 + vld1.8 {q2, q3}, [r0, :128], r1 + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, r1, lsl #1 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0, q1}, [r0, :128], r1 + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w64_neon + sub r1, r1, #32 +1: + vld1.8 {q0, q1}, [r0, :128]! + subs r3, r3, #1 + vld1.8 {q2, q3}, [r0, :128] + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, #32 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0, q1}, [r0, :128]! + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +.macro iwht4 + vadd.i16 d16, d16, d17 + vsub.i16 d21, d18, d19 + vsub.i16 d20, d16, d21 + vshr.s16 d20, d20, #1 + vsub.i16 d18, d20, d17 + vsub.i16 d17, d20, d19 + vadd.i16 d19, d21, d18 + vsub.i16 d16, d16, d17 +.endm + +.macro idct_4h_x4 r0, r1, r2, r3 + vmull_vmlal q3, \r1, \r3, d0[3], d0[2] + vmull_vmlsl q2, \r1, \r3, d0[2], d0[3] + vmull_vmlal q1, \r0, \r2, d0[0], d0[0] + vqrshrn.s32 d6, q3, #12 + vqrshrn.s32 d7, q2, #12 + vmull_vmlsl q2, \r0, \r2, d0[0], d0[0] + vqrshrn.s32 d2, q1, #12 + vqrshrn.s32 d3, q2, #12 + vqadd.s16 \r0, d2, d6 + vqsub.s16 \r3, d2, d6 + vqadd.s16 \r1, d3, d7 + vqsub.s16 \r2, d3, d7 +.endm + +.macro idct_8h_x4 q0, q1, q2, q3, r0, r1, r2, r3, r4, r5, r6, r7 + vmull_vmlal_8h q6, q7, \r2, \r3, \r6, \r7, d0[3], d0[2] + vmull_vmlsl_8h q4, q5, \r2, \r3, \r6, \r7, d0[2], d0[3] + vmull_vmlal_8h q2, q3, \r0, \r1, \r4, \r5, d0[0], d0[0] + vqrshrn_8h d12, d13, q6, q7, #12 + vqrshrn_8h d14, d15, q4, q5, #12 + vmull_vmlsl_8h q4, q5, \r0, \r1, \r4, \r5, d0[0], d0[0] + vqrshrn_8h d4, d5, q2, q3, #12 + vqrshrn_8h d6, d7, q4, q5, #12 + vqadd.s16 \q0, q2, q6 + vqsub.s16 \q3, q2, q6 + vqadd.s16 \q1, q3, q7 + vqsub.s16 \q2, q3, q7 +.endm + +function inv_dct_4h_x4_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] + idct_4h_x4 d16, d17, d18, d19 + bx lr +endfunc + +function inv_dct_8h_x4_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] + idct_8h_x4 q8, q9, q10, q11, d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel_local r12, iadst4_coeffs + vld1.16 {d0, d1}, [r12, :128] + + vsubl.s16 q1, d16, d18 + vmull.s16 q2, d16, d0[0] + vmlal.s16 q2, d18, d0[1] + vmlal.s16 q2, d19, d0[2] + vmull.s16 q10, d17, d0[3] + vaddw.s16 q1, q1, d19 + vmull.s16 q3, d16, d0[2] + vmlsl.s16 q3, d18, d0[0] + vmlsl.s16 q3, d19, d0[1] + + vadd.s32 q11, q2, q3 + vmul.s32 q1, q1, d1[0] + vadd.s32 q2, q2, q10 + vadd.s32 q3, q3, q10 + vsub.s32 q11, q11, q10 + + vqrshrn.s32 \o0, q2, #12 + vqrshrn.s32 \o2, q1, #12 + vqrshrn.s32 \o1, q3, #12 + vqrshrn.s32 \o3, q11, #12 +.endm + +function inv_adst_4h_x4_neon, export=1 + iadst_4x4 d16, d17, d18, d19 + bx lr +endfunc + +function inv_flipadst_4h_x4_neon, export=1 + iadst_4x4 d19, d18, d17, d16 + bx lr +endfunc + +.macro iadst_8x4 o0, o1, o2, o3, o4, o5, o6, o7 + movrel_local r12, iadst4_coeffs + vld1.16 {d0, d1}, [r12, :128] + + vsubl.s16 q2, d16, d20 + vsubl.s16 q3, d17, d21 + vmull.s16 q4, d16, d0[0] + vmlal.s16 q4, d20, d0[1] + vmlal.s16 q4, d22, d0[2] + vmull.s16 q5, d17, d0[0] + vmlal.s16 q5, d21, d0[1] + vmlal.s16 q5, d23, d0[2] + vaddw.s16 q2, q2, d22 + vaddw.s16 q3, q3, d23 + vmull.s16 q6, d16, d0[2] + vmlsl.s16 q6, d20, d0[0] + vmlsl.s16 q6, d22, d0[1] + vmull.s16 q7, d17, d0[2] + vmlsl.s16 q7, d21, d0[0] + vmlsl.s16 q7, d23, d0[1] + + vmul.s32 q10, q2, d1[0] + vmul.s32 q11, q3, d1[0] + + vmull.s16 q2, d18, d0[3] + vmull.s16 q3, d19, d0[3] + + vadd.s32 q8, q4, q2 // out0 + vadd.s32 q9, q5, q3 + + vadd.s32 q4, q4, q6 // out3 + vadd.s32 q5, q5, q7 + + vadd.s32 q6, q6, q2 // out1 + vadd.s32 q7, q7, q3 + + vsub.s32 q4, q4, q2 // out3 + vsub.s32 q5, q5, q3 + + vqrshrn.s32 d20, q10, #12 + vqrshrn.s32 d21, q11, #12 + + vqrshrn.s32 \o0, q8, #12 + vqrshrn.s32 \o1, q9, #12 + +.ifc \o4, d18 + vmov q9, q10 +.endif + + vqrshrn.s32 \o2, q6, #12 + vqrshrn.s32 \o3, q7, #12 + + vqrshrn.s32 \o6, q4, #12 + vqrshrn.s32 \o7, q5, #12 +.endm + +function inv_adst_8h_x4_neon, export=1 + iadst_8x4 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +function inv_flipadst_8h_x4_neon, export=1 + iadst_8x4 d22, d23, d20, d21, d18, d19, d16, d17 + bx lr +endfunc + +function inv_identity_4h_x4_neon, export=1 + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + vqrdmulh.s16 q2, q8, d0[0] + vqrdmulh.s16 q3, q9, d0[0] + vqadd.s16 q8, q8, q2 + vqadd.s16 q9, q9, q3 + bx lr +endfunc + +function inv_identity_8h_x4_neon, export=1 + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + vqrdmulh.s16 q1, q8, d0[0] + vqrdmulh.s16 q2, q9, d0[0] + vqrdmulh.s16 q3, q10, d0[0] + vqadd.s16 q8, q8, q1 + vqrdmulh.s16 q1, q11, d0[0] + vqadd.s16 q9, q9, q2 + vqadd.s16 q10, q10, q3 + vqadd.s16 q11, q11, q1 + bx lr +endfunc + +.macro identity_8x4_shift1 r0, r1, r2, r3, c +.irp i, \r0, \r1, \r2, \r3 + vqrdmulh.s16 q1, \i, \c + vrhadd.s16 \i, \i, q1 +.endr +.endm + +function inv_txfm_add_wht_wht_4x4_8bpc_neon, export=1 + push {r4-r5,lr} + vmov.i16 q15, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q15}, [r2, :128]! + + vshr.s16 q8, q8, #2 + vshr.s16 q9, q9, #2 + + iwht4 + + vst1.16 {q15}, [r2, :128]! + transpose_4x4h q8, q9, d16, d17, d18, d19 + + iwht4 + + vld1.32 {d0[]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + vmov.i16 q15, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q15}, [r2, :128]! + + blx r4 + + vst1.16 {q15}, [r2, :128]! + transpose_4x4h q8, q9, d16, d17, d18, d19 + + blx r5 + + vld1.32 {d0[]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + vrshr.s16 q8, q8, #4 + vrshr.s16 q9, q9, #4 + +L(itx_4x4_end): + sub r0, r0, r1, lsl #2 + vaddw.u8 q8, q8, d0 + vqmovun.s16 d0, q8 + vaddw.u8 q9, q9, d1 + vst1.32 {d0[0]}, [r0, :32], r1 + vqmovun.s16 d1, q9 + vst1.32 {d0[1]}, [r0, :32], r1 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 + + pop {r4-r5,pc} +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_8bpc_neon, export=1 + push {r4-r5,lr} + +.ifc \txfm1\()_\txfm2, dct_dct + cmp r3, #0 + bne 1f + vmov.i16 d30, #0 + movw r12, #2896*8 + vld1.16 {d16[]}, [r2, :16] + vdup.16 d4, r12 + vst1.16 {d30[0]}, [r2, :16] + vqrdmulh.s16 d16, d16, d4[0] + vld1.32 {d0[0]}, [r0, :32], r1 + vqrdmulh.s16 d20, d16, d4[0] + vld1.32 {d0[1]}, [r0, :32], r1 + vrshr.s16 d16, d20, #4 + vrshr.s16 d17, d20, #4 + vld1.32 {d1[0]}, [r0, :32], r1 + vmov q9, q8 + vld1.32 {d1[1]}, [r0, :32], r1 + b L(itx_4x4_end) +1: +.endif + movrel_local r4, inv_\txfm1\()_4h_x4_neon + movrel_local r5, inv_\txfm2\()_4h_x4_neon + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_8h_x8 q0, q1, q2, q3, q4, q5, q6, q7, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15 + idct_8h_x4 \q0, \q2, \q4, \q6, \r0, \r1, \r4, \r5, \r8, \r9, \r12, \r13 + + vmull_vmlsl_8h q2, q3, \r2, \r3, \r14, \r15, d1[0], d1[1] // -> t4a + vmull_vmlal_8h q4, q5, \r2, \r3, \r14, \r15, d1[1], d1[0] // -> t7a + vmull_vmlsl_8h q6, q7, \r10, \r11, \r6, \r7, d1[2], d1[3] // -> t5a + vqrshrn_8h \r2, \r3, q2, q3, #12 // t4a + vqrshrn_8h \r14, \r15, q4, q5, #12 // t7a + vmull_vmlal_8h q2, q3, \r10, \r11, \r6, \r7, d1[3], d1[2] // -> t6a + vqrshrn_8h \r6, \r7, q6, q7, #12 // t5a + vqrshrn_8h \r10, \r11, q2, q3, #12 // t6a + + vqadd.s16 q2, \q1, \q3 // t4 + vqsub.s16 \q1, \q1, \q3 // t5a + vqadd.s16 q3, \q7, \q5 // t7 + vqsub.s16 \q3, \q7, \q5 // t6a + + vmull_vmlsl_8h q4, q5, \r6, \r7, \r2, \r3, d0[0], d0[0] // -> t5 + vmull_vmlal_8h q6, q7, \r6, \r7, \r2, \r3, d0[0], d0[0] // -> t6 + vqrshrn_8h d8, d9, q4, q5, #12 // t5 + vqrshrn_8h d10, d11, q6, q7, #12 // t6 + + vqsub.s16 \q7, \q0, q3 // out7 + vqadd.s16 \q0, \q0, q3 // out0 + vqadd.s16 \q1, \q2, q5 // out1 + vqsub.s16 q6, \q2, q5 // out6 + vqadd.s16 \q2, \q4, q4 // out2 + vqsub.s16 \q5, \q4, q4 // out5 + vqadd.s16 \q3, \q6, q2 // out3 + vqsub.s16 \q4, \q6, q2 // out4 + vmov \q6, q6 // out6 +.endm + +.macro idct_4h_x8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_4h_x4 \r0, \r2, \r4, \r6 + + vmull_vmlsl q1, \r1, \r7, d1[0], d1[1] // -> t4a + vmull_vmlal q2, \r1, \r7, d1[1], d1[0] // -> t7a + vmull_vmlsl q3, \r5, \r3, d1[2], d1[3] // -> t5a + vqrshrn.s32 \r1, q1, #12 // t4a + vmull_vmlal q1, \r5, \r3, d1[3], d1[2] // -> t6a + vqrshrn.s32 \r7, q2, #12 // t7a + vqrshrn.s32 \r3, q3, #12 // t5a + vqrshrn.s32 \r5, q1, #12 // taa + + vqadd.s16 d2, \r1, \r3 // t4 + vqsub.s16 \r1, \r1, \r3 // t5a + vqadd.s16 d3, \r7, \r5 // t7 + vqsub.s16 \r3, \r7, \r5 // t6a + + vmull_vmlsl q2, \r3, \r1, d0[0], d0[0] // -> t5 + vmull_vmlal q3, \r3, \r1, d0[0], d0[0] // -> t6 + vqrshrn.s32 d4, q2, #12 // t5 + vqrshrn.s32 d5, q3, #12 // t6 + + vqsub.s16 \r7, \r0, d3 // out7 + vqadd.s16 \r0, \r0, d3 // out0 + vqadd.s16 \r1, \r2, d5 // out1 + vqsub.s16 d6, \r2, d5 // out6 + vqadd.s16 \r2, \r4, d4 // out2 + vqsub.s16 \r5, \r4, d4 // out5 + vqadd.s16 \r3, \r6, d2 // out3 + vqsub.s16 \r4, \r6, d2 // out4 + vmov \r6, d6 // out6 +.endm + +function inv_dct_8h_x8_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0}, [r12, :128] + idct_8h_x8 q8, q9, q10, q11, q12, q13, q14, q15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_dct_4h_x8_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0}, [r12, :128] + idct_4h_x8 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +.macro iadst_8h_x8 q0, q1, q2, q3, q4, q5, q6, q7, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15 + movrel_local r12, iadst8_coeffs + vld1.16 {d0, d1, d2}, [r12, :64] + + vmull_vmlal_8h q2, q3, d30, d31, d16, d17, d0[0], d0[1] + vmull_vmlsl_8h q4, q5, d30, d31, d16, d17, d0[1], d0[0] + vmull_vmlal_8h q6, q7, d26, d27, d20, d21, d0[2], d0[3] + vqrshrn_8h d16, d17, q2, q3, #12 // t0a + vqrshrn_8h d30, d31, q4, q5, #12 // t1a + vmull_vmlsl_8h q2, q3, d26, d27, d20, d21, d0[3], d0[2] + vmull_vmlal_8h q4, q5, d22, d23, d24, d25, d1[0], d1[1] + vqrshrn_8h d20, d21, q6, q7, #12 // t2a + vqrshrn_8h d26, d27, q2, q3, #12 // t3a + vmull_vmlsl_8h q6, q7, d22, d23, d24, d25, d1[1], d1[0] + vmull_vmlal_8h q2, q3, d18, d19, d28, d29, d1[2], d1[3] + vqrshrn_8h d24, d25, q4, q5, #12 // t4a + vqrshrn_8h d22, d23, q6, q7, #12 // t5a + vmull_vmlsl_8h q4, q5, d18, d19, d28, d29, d1[3], d1[2] + vqrshrn_8h d28, d29, q2, q3, #12 // t6a + vqrshrn_8h d18, d19, q4, q5, #12 // t7a + + vqadd.s16 q2, q8, q12 // t0 + vqsub.s16 q3, q8, q12 // t4 + vqadd.s16 q4, q15, q11 // t1 + vqsub.s16 q5, q15, q11 // t5 + vqadd.s16 q6, q10, q14 // t2 + vqsub.s16 q7, q10, q14 // t6 + vqadd.s16 q10, q13, q9 // t3 + vqsub.s16 q11, q13, q9 // t7 + + vmull_vmlal_8h q8, q9, d6, d7, d10, d11, d2[3], d2[2] + vmull_vmlsl_8h q12, q13, d6, d7, d10, d11, d2[2], d2[3] + vmull_vmlsl_8h q14, q15, d22, d23, d14, d15, d2[3], d2[2] + + vqrshrn_8h d6, d7, q8, q9, #12 // t4a + vqrshrn_8h d10, d11, q12, q13, #12 // t5a + + vmull_vmlal_8h q8, q9, d22, d23, d14, d15, d2[2], d2[3] + + vqrshrn_8h d14, d15, q14, q15, #12 // t6a + vqrshrn_8h d22, d23, q8, q9, #12 // t7a + + vqadd.s16 \q0, q2, q6 // out0 + vqsub.s16 q2, q2, q6 // t2 + vqadd.s16 \q7, q4, q10 // out7 + vqsub.s16 q4, q4, q10 // t3 + vqneg.s16 \q7, \q7 // out7 + + vqadd.s16 \q1, q3, q7 // out1 + vqsub.s16 q3, q3, q7 // t6 + vqadd.s16 \q6, q5, q11 // out6 + vqsub.s16 q5, q5, q11 // t7 + vqneg.s16 \q1, \q1 // out1 + + vmull_vmlal_8h q10, q11, d4, d5, d8, d9, d2[0], d2[0] // -> out3 (q11 or q12) + vmull_vmlsl_8h q6, q7, d4, d5, d8, d9, d2[0], d2[0] // -> out4 (q12 or q11) + vmull_vmlsl_8h q12, q13, d6, d7, d10, d11, d2[0], d2[0] // -> out5 (q13 or q10) + vqrshrn_8h d4, d5, q10, q11, #12 // out3 + vmull_vmlal_8h q10, q11, d6, d7, d10, d11, d2[0], d2[0] // -> out2 (q10 or q13) + vqrshrn_8h d6, d7, q12, q13, #12 // out5 + vqrshrn_8h \r4, \r5, q10, q11, #12 // out2 (q10 or q13) + vqrshrn_8h \r8, \r9, q6, q7, #12 // out4 (q12 or q11) + + vqneg.s16 \q3, q2 // out3 + vqneg.s16 \q5, q3 // out5 +.endm + +.macro iadst_4h_x8 r0, r1, r2, r3, r4, r5, r6, r7 + movrel_local r12, iadst8_coeffs + vld1.16 {d0, d1, d2}, [r12, :64] + + vmull_vmlal q2, d23, d16, d0[0], d0[1] + vmull_vmlsl q3, d23, d16, d0[1], d0[0] + vmull_vmlal q4, d21, d18, d0[2], d0[3] + vqrshrn.s32 d16, q2, #12 // t0a + vqrshrn.s32 d23, q3, #12 // t1a + vmull_vmlsl q5, d21, d18, d0[3], d0[2] + vmull_vmlal q6, d19, d20, d1[0], d1[1] + vqrshrn.s32 d18, q4, #12 // t2a + vqrshrn.s32 d21, q5, #12 // t3a + vmull_vmlsl q7, d19, d20, d1[1], d1[0] + vmull_vmlal q2, d17, d22, d1[2], d1[3] + vqrshrn.s32 d20, q6, #12 // t4a + vqrshrn.s32 d19, q7, #12 // t5a + vmull_vmlsl q3, d17, d22, d1[3], d1[2] + vqrshrn.s32 d22, q2, #12 // t6a + vqrshrn.s32 d17, q3, #12 // t7a + + vqadd.s16 d4, d16, d20 // t0 + vqsub.s16 d5, d16, d20 // t4 + vqadd.s16 d6, d23, d19 // t1 + vqsub.s16 d7, d23, d19 // t5 + vqadd.s16 d8, d18, d22 // t2 + vqsub.s16 d9, d18, d22 // t6 + vqadd.s16 d18, d21, d17 // t3 + vqsub.s16 d19, d21, d17 // t7 + + vmull_vmlal q8, d5, d7, d2[3], d2[2] + vmull_vmlsl q10, d5, d7, d2[2], d2[3] + vmull_vmlsl q11, d19, d9, d2[3], d2[2] + + vqrshrn.s32 d5, q8, #12 // t4a + vqrshrn.s32 d7, q10, #12 // t5a + + vmull_vmlal q8, d19, d9, d2[2], d2[3] + + vqrshrn.s32 d9, q11, #12 // t6a + vqrshrn.s32 d19, q8, #12 // t7a + + vqadd.s16 \r0, d4, d8 // out0 + vqsub.s16 d4, d4, d8 // t2 + vqadd.s16 \r7, d6, d18 // out7 + vqsub.s16 d6, d6, d18 // t3 + vqneg.s16 \r7, \r7 // out7 + + vqadd.s16 \r1, d5, d9 // out1 + vqsub.s16 d5, d5, d9 // t6 + vqadd.s16 \r6, d7, d19 // out6 + vqsub.s16 d7, d7, d19 // t7 + vqneg.s16 \r1, \r1 // out1 + + vmull_vmlal q9, d4, d6, d2[0], d2[0] // -> out3 (d19 or d20) + vmull_vmlsl q4, d4, d6, d2[0], d2[0] // -> out4 (d20 or d19) + vmull_vmlsl q10, d5, d7, d2[0], d2[0] // -> out5 (d21 or d18) + vqrshrn.s32 d4, q9, #12 // out3 + vmull_vmlal q9, d5, d7, d2[0], d2[0] // -> out2 (d18 or d21) + vqrshrn.s32 d5, q10, #12 // out5 + vqrshrn.s32 \r2, q9, #12 // out2 (d18 or d21) + vqrshrn.s32 \r4, q4, #12 // out4 (d20 or d19) + + vqneg.s16 \r3, d4 // out3 + vqneg.s16 \r5, d5 // out5 +.endm + +function inv_adst_8h_x8_neon, export=1 + iadst_8h_x8 q8, q9, q10, q11, q12, q13, q14, q15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_flipadst_8h_x8_neon, export=1 + iadst_8h_x8 q15, q14, q13, q12, q11, q10, q9, q8, d30, d31, d28, d29, d26, d27, d24, d25, d22, d23, d20, d21, d18, d19, d16, d17 + bx lr +endfunc + +function inv_adst_4h_x8_neon, export=1 + iadst_4h_x8 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +function inv_flipadst_4h_x8_neon, export=1 + iadst_4h_x8 d23, d22, d21, d20, d19, d18, d17, d16 + bx lr +endfunc + +function inv_identity_8h_x8_neon, export=1 + vqshl.s16 q8, q8, #1 + vqshl.s16 q9, q9, #1 + vqshl.s16 q10, q10, #1 + vqshl.s16 q11, q11, #1 + vqshl.s16 q12, q12, #1 + vqshl.s16 q13, q13, #1 + vqshl.s16 q14, q14, #1 + vqshl.s16 q15, q15, #1 + bx lr +endfunc + +function inv_identity_4h_x8_neon, export=1 + vqshl.s16 q8, q8, #1 + vqshl.s16 q9, q9, #1 + vqshl.s16 q10, q10, #1 + vqshl.s16 q11, q11, #1 + bx lr +endfunc + +.macro def_fn_8x8_base variant +function inv_txfm_\variant\()add_8x8_neon + vmov.i16 q0, #0 + vmov.i16 q1, #0 + vld1.16 {q8, q9}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q12, q13}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q14, q15}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128] + +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else + blx r4 + + vrshr.s16 q8, q8, #1 + vrshr.s16 q9, q9, #1 + vrshr.s16 q10, q10, #1 + vrshr.s16 q11, q11, #1 + vrshr.s16 q12, q12, #1 + vrshr.s16 q13, q13, #1 + vrshr.s16 q14, q14, #1 + vrshr.s16 q15, q15, #1 +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + blx r5 + + load_add_store_8x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc +.endm + +def_fn_8x8_base +def_fn_8x8_base identity_ + +.macro def_fn_8x8 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + push {r4-r5,r7,lr} + vpush {q4-q7} + movrel_local r5, inv_\txfm2\()_8h_x8_neon +.ifc \txfm1, identity + b inv_txfm_identity_add_8x8_neon +.else + movrel_local r4, inv_\txfm1\()_8h_x8_neon + b inv_txfm_add_8x8_neon +.endif +endfunc +.endm + +def_fn_8x8 dct, dct +def_fn_8x8 identity, identity +def_fn_8x8 dct, adst +def_fn_8x8 dct, flipadst +def_fn_8x8 dct, identity +def_fn_8x8 adst, dct +def_fn_8x8 adst, adst +def_fn_8x8 adst, flipadst +def_fn_8x8 flipadst, dct +def_fn_8x8 flipadst, adst +def_fn_8x8 flipadst, flipadst +def_fn_8x8 identity, dct +def_fn_8x8 adst, identity +def_fn_8x8 flipadst, identity +def_fn_8x8 identity, adst +def_fn_8x8 identity, flipadst + +function inv_txfm_add_8x4_neon + vmov.i16 q14, #0 + vmov.i16 q15, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128]! + vld1.16 {d20, d21, d22, d23}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128] + + scale_input d0[0], q8, q9, q10, q11 + + blx r4 + + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + vswp d17, d20 + vswp d19, d21 + vswp d18, d20 + vswp d21, d22 + + blx r5 + + load_add_store_8x4 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc + +function inv_txfm_add_4x8_neon + vmov.i16 q14, #0 + vmov.i16 q15, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + vld1.16 {q8, q9}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128] + + scale_input d0[0], q8, q9, q10, q11 + + blx r4 + + transpose_4x8h q8, q9, q10, q11 + vswp d17, d20 + vswp d19, d21 + vswp d17, d18 + vswp d19, d22 + + blx r5 + + load_add_store_4x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + push {r4-r5,r7,lr} + vpush {q4-q7} + movrel_local r4, inv_\txfm1\()_\h\()h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_\w\()h_x\h\()_neon + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct +def_fn_48 \w, \h, identity, identity +def_fn_48 \w, \h, dct, adst +def_fn_48 \w, \h, dct, flipadst +def_fn_48 \w, \h, dct, identity +def_fn_48 \w, \h, adst, dct +def_fn_48 \w, \h, adst, adst +def_fn_48 \w, \h, adst, flipadst +def_fn_48 \w, \h, flipadst, dct +def_fn_48 \w, \h, flipadst, adst +def_fn_48 \w, \h, flipadst, flipadst +def_fn_48 \w, \h, identity, dct +def_fn_48 \w, \h, adst, identity +def_fn_48 \w, \h, flipadst, identity +def_fn_48 \w, \h, identity, adst +def_fn_48 \w, \h, identity, flipadst +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + +function inv_dct_4h_x16_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0, q1}, [r12, :128] + + vmull_vmlsl q2, d17, d31, d2[0], d2[1] // -> t8a + vmull_vmlal q3, d17, d31, d2[1], d2[0] // -> t15a + vmull_vmlsl q4, d25, d23, d2[2], d2[3] // -> t9a + vqrshrn.s32 d17, q2, #12 // t8a + vqrshrn.s32 d31, q3, #12 // t15a + vmull_vmlal q2, d25, d23, d2[3], d2[2] // -> t14a + vmull_vmlsl q3, d21, d27, d3[0], d3[1] // -> t10a + vqrshrn.s32 d23, q4, #12 // t9a + vqrshrn.s32 d25, q2, #12 // t14a + vmull_vmlal q4, d21, d27, d3[1], d3[0] // -> t13a + vmull_vmlsl q2, d29, d19, d3[2], d3[3] // -> t11a + vqrshrn.s32 d21, q3, #12 // t10a + vqrshrn.s32 d27, q4, #12 // t13a + vmull_vmlal q3, d29, d19, d3[3], d3[2] // -> t12a + vqrshrn.s32 d19, q2, #12 // t11a + vqrshrn.s32 d29, q3, #12 // t12a + + idct_4h_x8 d16, d18, d20, d22, d24, d26, d28, d30 + + vqsub.s16 d4, d17, d23 // t9 + vqadd.s16 d17, d17, d23 // t8 + vqsub.s16 d5, d31, d25 // t14 + vqadd.s16 d31, d31, d25 // t15 + vqsub.s16 d23, d19, d21 // t10 + vqadd.s16 d19, d19, d21 // t11 + vqadd.s16 d25, d29, d27 // t12 + vqsub.s16 d29, d29, d27 // t13 + + vmull_vmlsl q3, d5, d4, d0[2], d0[3] // -> t9a + vmull_vmlal q4, d5, d4, d0[3], d0[2] // -> t14a + vqrshrn.s32 d21, q3, #12 // t9a + vqrshrn.s32 d27, q4, #12 // t14a + + vmull_vmlsl q3, d29, d23, d0[2], d0[3] // -> t13a + vmull_vmlal q4, d29, d23, d0[3], d0[2] // -> t10a + vqrshrn.s32 d29, q3, #12 // t13a + vneg.s32 q4, q4 + vqrshrn.s32 d23, q4, #12 // t10a + + vqsub.s16 d4, d17, d19 // t11a + vqadd.s16 d17, d17, d19 // t8a + vqsub.s16 d5, d31, d25 // t12a + vqadd.s16 d31, d31, d25 // t15a + vqadd.s16 d19, d21, d23 // t9 + vqsub.s16 d21, d21, d23 // t10 + vqsub.s16 d25, d27, d29 // t13 + vqadd.s16 d27, d27, d29 // t14 + + vmull_vmlsl q3, d5, d4, d0[0], d0[0] // -> t11 + vmull_vmlal q4, d5, d4, d0[0], d0[0] // -> t12 + vmull_vmlsl q2, d25, d21, d0[0], d0[0] // -> t10a + + vqrshrn.s32 d6, q3, #12 // t11 + vqrshrn.s32 d7, q4, #12 // t12 + vmull_vmlal q4, d25, d21, d0[0], d0[0] // -> t13a + vqrshrn.s32 d4, q2, #12 // t10a + vqrshrn.s32 d5, q4, #12 // t13a + + vqadd.s16 d8, d16, d31 // out0 + vqsub.s16 d31, d16, d31 // out15 + vmov d16, d8 + vqadd.s16 d23, d30, d17 // out7 + vqsub.s16 d9, d30, d17 // out8 + vqadd.s16 d17, d18, d27 // out1 + vqsub.s16 d30, d18, d27 // out14 + vqadd.s16 d18, d20, d5 // out2 + vqsub.s16 d29, d20, d5 // out13 + vqadd.s16 d5, d28, d19 // out6 + vqsub.s16 d25, d28, d19 // out9 + vqadd.s16 d19, d22, d7 // out3 + vqsub.s16 d28, d22, d7 // out12 + vqadd.s16 d20, d24, d6 // out4 + vqsub.s16 d27, d24, d6 // out11 + vqadd.s16 d21, d26, d4 // out5 + vqsub.s16 d26, d26, d4 // out10 + vmov d24, d9 + vmov d22, d5 + + bx lr +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15 + movrel_local r12, iadst16_coeffs + vld1.16 {q0, q1}, [r12, :128] + movrel_local r12, idct_coeffs + + vmull_vmlal q2, d31, d16, d0[0], d0[1] // -> t0 + vmull_vmlsl q3, d31, d16, d0[1], d0[0] // -> t1 + vmull_vmlal q4, d29, d18, d0[2], d0[3] // -> t2 + vqrshrn.s32 d16, q2, #12 // t0 + vqrshrn.s32 d31, q3, #12 // t1 + vmull_vmlsl q2, d29, d18, d0[3], d0[2] // -> t3 + vmull_vmlal q3, d27, d20, d1[0], d1[1] // -> t4 + vqrshrn.s32 d18, q4, #12 // t2 + vqrshrn.s32 d29, q2, #12 // t3 + vmull_vmlsl q4, d27, d20, d1[1], d1[0] // -> t5 + vmull_vmlal q2, d25, d22, d1[2], d1[3] // -> t6 + vqrshrn.s32 d20, q3, #12 // t4 + vqrshrn.s32 d27, q4, #12 // t5 + vmull_vmlsl q3, d25, d22, d1[3], d1[2] // -> t7 + vmull_vmlal q4, d23, d24, d2[0], d2[1] // -> t8 + vqrshrn.s32 d22, q2, #12 // t6 + vqrshrn.s32 d25, q3, #12 // t7 + vmull_vmlsl q2, d23, d24, d2[1], d2[0] // -> t9 + vmull_vmlal q3, d21, d26, d2[2], d2[3] // -> t10 + vqrshrn.s32 d23, q4, #12 // t8 + vqrshrn.s32 d24, q2, #12 // t9 + vmull_vmlsl q4, d21, d26, d2[3], d2[2] // -> t11 + vmull_vmlal q2, d19, d28, d3[0], d3[1] // -> t12 + vqrshrn.s32 d21, q3, #12 // t10 + vqrshrn.s32 d26, q4, #12 // t11 + vmull_vmlsl q3, d19, d28, d3[1], d3[0] // -> t13 + vmull_vmlal q4, d17, d30, d3[2], d3[3] // -> t14 + vqrshrn.s32 d19, q2, #12 // t12 + vqrshrn.s32 d28, q3, #12 // t13 + vmull_vmlsl q2, d17, d30, d3[3], d3[2] // -> t15 + vqrshrn.s32 d17, q4, #12 // t14 + vqrshrn.s32 d30, q2, #12 // t15 + + vld1.16 {q0}, [r12, :128] + + vqsub.s16 d2, d16, d23 // t8a + vqadd.s16 d16, d16, d23 // t0a + vqsub.s16 d3, d31, d24 // t9a + vqadd.s16 d31, d31, d24 // t1a + vqadd.s16 d23, d18, d21 // t2a + vqsub.s16 d18, d18, d21 // t10a + vqadd.s16 d24, d29, d26 // t3a + vqsub.s16 d29, d29, d26 // t11a + vqadd.s16 d21, d20, d19 // t4a + vqsub.s16 d20, d20, d19 // t12a + vqadd.s16 d26, d27, d28 // t5a + vqsub.s16 d27, d27, d28 // t13a + vqadd.s16 d19, d22, d17 // t6a + vqsub.s16 d22, d22, d17 // t14a + vqadd.s16 d28, d25, d30 // t7a + vqsub.s16 d25, d25, d30 // t15a + + vmull_vmlal q2, d2, d3, d1[1], d1[0] // -> t8 + vmull_vmlsl q3, d2, d3, d1[0], d1[1] // -> t9 + vmull_vmlal q4, d18, d29, d1[3], d1[2] // -> t10 + vqrshrn.s32 d17, q2, #12 // t8 + vqrshrn.s32 d30, q3, #12 // t9 + vmull_vmlsl q2, d18, d29, d1[2], d1[3] // -> t11 + vmull_vmlsl q3, d27, d20, d1[1], d1[0] // -> t12 + vqrshrn.s32 d18, q4, #12 // t10 + vqrshrn.s32 d29, q2, #12 // t11 + vmull_vmlal q4, d27, d20, d1[0], d1[1] // -> t13 + vmull_vmlsl q2, d25, d22, d1[3], d1[2] // -> t14 + vqrshrn.s32 d27, q3, #12 // t12 + vqrshrn.s32 d20, q4, #12 // t13 + vmull_vmlal q3, d25, d22, d1[2], d1[3] // -> t15 + vqrshrn.s32 d25, q2, #12 // t14 + vqrshrn.s32 d22, q3, #12 // t15 + + vqsub.s16 d2, d16, d21 // t4 + vqadd.s16 d16, d16, d21 // t0 + vqsub.s16 d3, d31, d26 // t5 + vqadd.s16 d31, d31, d26 // t1 + vqadd.s16 d21, d23, d19 // t2 + vqsub.s16 d23, d23, d19 // t6 + vqadd.s16 d26, d24, d28 // t3 + vqsub.s16 d24, d24, d28 // t7 + vqadd.s16 d19, d17, d27 // t8a + vqsub.s16 d17, d17, d27 // t12a + vqadd.s16 d28, d30, d20 // t9a + vqsub.s16 d30, d30, d20 // t13a + vqadd.s16 d27, d18, d25 // t10a + vqsub.s16 d18, d18, d25 // t14a + vqadd.s16 d20, d29, d22 // t11a + vqsub.s16 d29, d29, d22 // t15a + + vmull_vmlal q2, d2, d3, d0[3], d0[2] // -> t4a + vmull_vmlsl q3, d2, d3, d0[2], d0[3] // -> t5a + vmull_vmlsl q4, d24, d23, d0[3], d0[2] // -> t6a + vqrshrn.s32 d22, q2, #12 // t4a + vqrshrn.s32 d25, q3, #12 // t5a + vmull_vmlal q2, d24, d23, d0[2], d0[3] // -> t7a + vmull_vmlal q3, d17, d30, d0[3], d0[2] // -> t12 + vqrshrn.s32 d24, q4, #12 // t6a + vqrshrn.s32 d23, q2, #12 // t7a + vmull_vmlsl q4, d17, d30, d0[2], d0[3] // -> t13 + vmull_vmlsl q2, d29, d18, d0[3], d0[2] // -> t14 + vqrshrn.s32 d17, q3, #12 // t12 + vmull_vmlal q3, d29, d18, d0[2], d0[3] // -> t15 + vqrshrn.s32 d29, q4, #12 // t13 + vqrshrn.s32 d30, q2, #12 // t14 + vqrshrn.s32 d18, q3, #12 // t15 + + vqsub.s16 d2, d16, d21 // t2a +.ifc \o0, d16 + vqadd.s16 \o0, d16, d21 // out0 + vqsub.s16 d21, d31, d26 // t3a + vqadd.s16 \o15,d31, d26 // out15 +.else + vqadd.s16 d4, d16, d21 // out0 + vqsub.s16 d21, d31, d26 // t3a + vqadd.s16 \o15,d31, d26 // out15 + vmov \o0, d4 +.endif + vqneg.s16 \o15, \o15 // out15 + + vqsub.s16 d3, d29, d18 // t15a + vqadd.s16 \o13,d29, d18 // out13 + vqadd.s16 \o2, d17, d30 // out2 + vqsub.s16 d26, d17, d30 // t14a + vqneg.s16 \o13,\o13 // out13 + + vqadd.s16 \o1, d19, d27 // out1 + vqsub.s16 d27, d19, d27 // t10 + vqadd.s16 \o14,d28, d20 // out14 + vqsub.s16 d20, d28, d20 // t11 + vqneg.s16 \o1, \o1 // out1 + + vqadd.s16 \o3, d22, d24 // out3 + vqsub.s16 d22, d22, d24 // t6 + vqadd.s16 \o12,d25, d23 // out12 + vqsub.s16 d23, d25, d23 // t7 + vqneg.s16 \o3, \o3 // out3 + + vmull_vmlsl q12, d2, d21, d0[0], d0[0] // -> out8 (d24 or d23) + vmull_vmlal q2, d2, d21, d0[0], d0[0] // -> out7 (d23 or d24) + vmull_vmlal q3, d26, d3, d0[0], d0[0] // -> out5 (d21 or d26) + + vqrshrn.s32 d24, q12, #12 // out8 + vqrshrn.s32 d4, q2, #12 // out7 + vqrshrn.s32 d5, q3, #12 // out5 + vmull_vmlsl q4, d26, d3, d0[0], d0[0] // -> out10 (d26 or d21) + vmull_vmlal q1, d22, d23, d0[0], d0[0] // -> out4 (d20 or d27) + vqrshrn.s32 d26, q4, #12 // out10 + + vmull_vmlsl q4, d22, d23, d0[0], d0[0] // -> out11 (d27 or d20) + vmull_vmlal q11, d27, d20, d0[0], d0[0] // -> out6 (d22 or d25) + vmull_vmlsl q3, d27, d20, d0[0], d0[0] // -> out9 (d25 or d22) + + vqrshrn.s32 \o4, q1, #12 // out4 + vqrshrn.s32 d7, q3, #12 // out9 + vqrshrn.s32 d6, q4, #12 // out11 + vqrshrn.s32 \o6, q11, #12 // out6 + +.ifc \o8, d23 + vmov \o8, d24 + vmov \o10,d26 +.endif + + vqneg.s16 \o7, d4 // out7 + vqneg.s16 \o5, d5 // out5 + vqneg.s16 \o11,d6 // out11 + vqneg.s16 \o9, d7 // out9 +.endm + +function inv_adst_4h_x16_neon, export=1 + iadst_16 d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_flipadst_4h_x16_neon, export=1 + iadst_16 d31, d30, d29, d28, d27, d26, d25, d24, d23, d22, d21, d20, d19, d18, d17, d16 + bx lr +endfunc + +function inv_identity_4h_x16_neon, export=1 + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q1, \i, d0[0] + vqadd.s16 \i, \i, \i + vqadd.s16 \i, \i, q1 +.endr + bx lr +endfunc + +.macro identity_4x16_shift2 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vshr.s16 q2, q2, #1 + vrhadd.s16 \i, \i, q2 +.endr +.endm + +.macro identity_4x16_shift1 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vrshr.s16 q2, q2, #1 + vqadd.s16 \i, \i, q2 +.endr +.endm + +.macro identity_8x8_shift1 c + identity_4x16_shift1 \c +.endm + +.macro identity_8x8 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vqadd.s16 \i, \i, \i + vqadd.s16 \i, \i, q2 +.endr +.endm + +.macro def_horz_16 scale=0, identity=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x4_neon + push {lr} + vmov.i16 d7, #0 +.if \identity + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.endif +.if \scale + movw r12, #2896*8 + vdup.16 d1, r12 +.endif +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr +.if \scale + scale_input d1[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif +.if \identity +.if \shift == -2 + identity_4x16_shift2 d0[0] +.else + identity_4x16_shift1 d0[0] +.endif +.else + blx r4 +.endif +.if \shift > 0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \i, \i, #\shift +.endr +.endif + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + transpose_4x4h q14, q15, d28, d29, d30, d31 + +.irp i, d16, d20, d24, d28, d17, d21, d25, d29, d18, d22, d26, d30, d19, d23, d27, d31 + vst1.16 {\i}, [r6, :64]! +.endr + + pop {pc} +endfunc +.endm + +def_horz_16 scale=0, identity=0, shift=2 +def_horz_16 scale=1, identity=0, shift=1, suffix=_scale +def_horz_16 scale=0, identity=1, shift=-2, suffix=_identity +def_horz_16 scale=1, identity=1, shift=-1, suffix=_scale_identity + +function inv_txfm_add_vert_4x16_neon + push {lr} +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + blx r5 + load_add_store_4x16 r6, r7 + pop {pc} +endfunc + +function inv_txfm_add_16x16_neon + sub_sp_align 512 + ldrh r11, [r10], #2 +.irp i, 0, 4, 8, 12 + add r6, sp, #(\i*16*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 12 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + blx r9 +.endr + b 3f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 512 + vpop {q4} + pop {r4-r11,pc} +endfunc + +const eob_16x16 + .short 10, 36, 78, 256 +endconst + +const eob_16x16_identity + .short 4, 8, 12, 256 +endconst + +.macro def_fn_16x16 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif + push {r4-r11,lr} + vpush {q4} +.ifc \txfm1, identity + movrel_local r9, inv_txfm_horz_identity_16x4_neon +.else + movrel_local r9, inv_txfm_horz_16x4_neon + movrel_local r4, inv_\txfm1\()_4h_x16_neon +.endif + movrel_local r5, inv_\txfm2\()_4h_x16_neon +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel_local r10, eob_16x16 +.else + movrel_local r10, eob_16x16_identity +.endif +.else +.ifc \txfm2, identity + movrel_local r10, eob_16x16_identity +.else + movrel_local r10, eob_16x16 +.endif +.endif + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct +def_fn_16x16 identity, identity +def_fn_16x16 dct, adst +def_fn_16x16 dct, flipadst +def_fn_16x16 dct, identity +def_fn_16x16 adst, dct +def_fn_16x16 adst, adst +def_fn_16x16 adst, flipadst +def_fn_16x16 flipadst, dct +def_fn_16x16 flipadst, adst +def_fn_16x16 flipadst, flipadst +def_fn_16x16 identity, dct + +.macro def_fn_416_base variant +function inv_txfm_\variant\()add_16x4_neon + +.ifc \variant, identity_ + vmov.i16 d4, #0 +.irp i, d16, d18, d20, d22 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr +.irp i, d17, d19, d21, d23 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.irp i, d24, d26, d28, d30 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr +.irp i, d25, d27, d29, d31 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr + + identity_4x16_shift1 d0[0] +.else + vmov.i16 q2, #0 + vmov.i16 q3, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d20, d21, d22, d23}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d24, d25, d26, d27}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d28, d29, d30, d31}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + + blx r4 + + vswp d17, d20 + vswp d19, d22 + vswp d18, d20 + vswp d19, d21 +.irp i, q8, q9, q10, q11 + vrshr.s16 \i, \i, #1 +.endr +.endif + transpose_4x8h q8, q9, q10, q11 + blx r5 + mov r6, r0 + load_add_store_8x4 r6, r7 + +.ifc \variant, identity_ + vmov q8, q12 + vmov q9, q13 + vmov q10, q14 + vmov q11, q15 +.else + vswp d25, d28 + vswp d27, d30 + vswp d26, d28 + vswp d27, d29 + vrshr.s16 q8, q12, #1 + vrshr.s16 q9, q13, #1 + vrshr.s16 q10, q14, #1 + vrshr.s16 q11, q15, #1 +.endif + transpose_4x8h q8, q9, q10, q11 + blx r5 + add r6, r0, #8 + load_add_store_8x4 r6, r7 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_\variant\()add_4x16_neon + vmov.i16 q2, #0 + + mov r11, #32 + cmp r3, r10 + blt 1f + + add r6, r2, #16 +.ifc \variant, identity_ +.irp i, q12, q13, q14, q15 + vld1.16 {\i}, [r6, :128] + vst1.16 {q2}, [r6, :128], r11 +.endr + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + identity_8x4_shift1 q12, q13, q14, q15, d0[0] +.else +.irp i, q8, q9, q10, q11 + vld1.16 {\i}, [r6, :128] + vst1.16 {q2}, [r6, :128], r11 +.endr + blx r4 + vrshr.s16 q12, q8, #1 + vrshr.s16 q13, q9, #1 + vrshr.s16 q14, q10, #1 + vrshr.s16 q15, q11, #1 +.endif + transpose_4x8h q12, q13, q14, q15 + vswp d27, d29 + vswp d26, d28 + vswp d27, d30 + vswp d25, d28 + + b 2f +1: +.irp i, q12, q13, q14, q15 + vmov.i16 \i, #0 +.endr +2: + vmov.i16 q2, #0 +.irp i, q8, q9, q10, q11 + vld1.16 {\i}, [r2, :128] + vst1.16 {q2}, [r2, :128], r11 +.endr +.ifc \variant, identity_ + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + identity_8x4_shift1 q8, q9, q10, q11, d0[0] +.else + blx r4 +.irp i, q8, q9, q10, q11 + vrshr.s16 \i, \i, #1 +.endr +.endif + transpose_4x8h q8, q9, q10, q11 + vswp d19, d21 + vswp d18, d20 + vswp d19, d22 + vswp d17, d20 + + blx r5 + + load_add_store_4x16 r0, r6 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +def_fn_416_base +def_fn_416_base identity_ + +.macro def_fn_416 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 4 + movrel_local r4, inv_\txfm1\()_8h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_4h_x\h\()_neon + mov r10, #\eob_half +.else + movrel_local r4, inv_\txfm1\()_4h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_8h_x\h\()_neon +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct, 29 +def_fn_416 \w, \h, identity, identity, 29 +def_fn_416 \w, \h, dct, adst, 29 +def_fn_416 \w, \h, dct, flipadst, 29 +def_fn_416 \w, \h, dct, identity, 8 +def_fn_416 \w, \h, adst, dct, 29 +def_fn_416 \w, \h, adst, adst, 29 +def_fn_416 \w, \h, adst, flipadst, 29 +def_fn_416 \w, \h, flipadst, dct, 29 +def_fn_416 \w, \h, flipadst, adst, 29 +def_fn_416 \w, \h, flipadst, flipadst, 29 +def_fn_416 \w, \h, identity, dct, 32 +def_fn_416 \w, \h, adst, identity, 8 +def_fn_416 \w, \h, flipadst, identity, 8 +def_fn_416 \w, \h, identity, adst, 32 +def_fn_416 \w, \h, identity, flipadst, 32 +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + +.macro def_fn_816_base variant +function inv_txfm_\variant\()add_16x8_neon + sub_sp_align 256 + +.irp i, 0, 4 + add r6, sp, #(\i*16*2) +.if \i > 0 + cmp r3, r10 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #8*2 + blx r9 +.endr + b 2f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr +2: + +.irp i, 0, 8 + add r7, sp, #(\i*2) + mov r8, #32 +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\j}, [r7, :128], r8 +.endr + blx r5 + + add r6, r0, #(\i) + load_add_store_8x8 r6, r7 +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_\variant\()add_8x16_neon + sub_sp_align 256 + +.irp i, 0, 8 + add r6, sp, #(\i*8*2) +.if \i > 0 + cmp r3, r10 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + + vmov.i16 q2, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\j}, [r7, :128] + vst1.16 {q2}, [r7, :128], r8 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 +.ifc \variant, identity_ + // The identity shl #1 and downshift vrshr #1 cancel out +.else + blx r4 +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \j, \j, #1 +.endr +.endif + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + vst1.16 {q8, q9}, [r6, :128]! + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r6, :128]! + vst1.16 {q14, q15}, [r6, :128]! +.endr + b 2f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr +2: + +.irp i, 0, 4 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #16 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +def_fn_816_base +def_fn_816_base identity_ + +.macro def_fn_816 w, h, txfm1, txfm2, eob_8x8, eob_4x4 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 8 + movrel_local r4, inv_\txfm1\()_8h_x8_neon + movrel_local r5, inv_\txfm2\()_4h_x16_neon +.else +.ifc \txfm1, identity + movrel_local r9, inv_txfm_horz_scale_identity_16x4_neon +.else + movrel_local r4, inv_\txfm1\()_4h_x16_neon + movrel_local r9, inv_txfm_horz_scale_16x4_neon +.endif + movrel_local r5, inv_\txfm2\()_8h_x8_neon +.endif +.if \w == 8 + mov r10, #\eob_8x8 +.else + mov r10, #\eob_4x4 +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct, 43, 10 +def_fn_816 \w, \h, identity, identity, 43, 10 +def_fn_816 \w, \h, dct, adst, 43, 10 +def_fn_816 \w, \h, dct, flipadst, 43, 10 +def_fn_816 \w, \h, dct, identity, 8, 4 +def_fn_816 \w, \h, adst, dct, 43, 10 +def_fn_816 \w, \h, adst, adst, 43, 10 +def_fn_816 \w, \h, adst, flipadst, 43, 10 +def_fn_816 \w, \h, flipadst, dct, 43, 10 +def_fn_816 \w, \h, flipadst, adst, 43, 10 +def_fn_816 \w, \h, flipadst, flipadst, 43, 10 +def_fn_816 \w, \h, identity, dct, 64, 4 +def_fn_816 \w, \h, adst, identity, 8, 4 +def_fn_816 \w, \h, flipadst, identity, 8, 4 +def_fn_816 \w, \h, identity, adst, 64, 4 +def_fn_816 \w, \h, identity, flipadst, 64, 4 +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_4h_x16_neon, export=1 + movrel_local r12, idct_coeffs, 2*16 + vld1.16 {q0, q1}, [r12, :128] + sub r12, r12, #2*16 + + vmull_vmlsl q2, d16, d31, d0[0], d0[1] // -> t16a + vmull_vmlal q3, d16, d31, d0[1], d0[0] // -> t31a + vmull_vmlsl q4, d24, d23, d0[2], d0[3] // -> t17a + vqrshrn.s32 d16, q2, #12 // t16a + vqrshrn.s32 d31, q3, #12 // t31a + vmull_vmlal q2, d24, d23, d0[3], d0[2] // -> t30a + vmull_vmlsl q3, d20, d27, d1[0], d1[1] // -> t18a + vqrshrn.s32 d24, q4, #12 // t17a + vqrshrn.s32 d23, q2, #12 // t30a + vmull_vmlal q4, d20, d27, d1[1], d1[0] // -> t29a + vmull_vmlsl q2, d28, d19, d1[2], d1[3] // -> t19a + vqrshrn.s32 d20, q3, #12 // t18a + vqrshrn.s32 d27, q4, #12 // t29a + vmull_vmlal q3, d28, d19, d1[3], d1[2] // -> t28a + vmull_vmlsl q4, d18, d29, d2[0], d2[1] // -> t20a + vqrshrn.s32 d28, q2, #12 // t19a + vqrshrn.s32 d19, q3, #12 // t28a + vmull_vmlal q2, d18, d29, d2[1], d2[0] // -> t27a + vmull_vmlsl q3, d26, d21, d2[2], d2[3] // -> t21a + vqrshrn.s32 d18, q4, #12 // t20a + vqrshrn.s32 d29, q2, #12 // t27a + vmull_vmlal q4, d26, d21, d2[3], d2[2] // -> t26a + vmull_vmlsl q2, d22, d25, d3[0], d3[1] // -> t22a + vqrshrn.s32 d26, q3, #12 // t21a + vqrshrn.s32 d21, q4, #12 // t26a + vmull_vmlal q3, d22, d25, d3[1], d3[0] // -> t25a + vmull_vmlsl q4, d30, d17, d3[2], d3[3] // -> t23a + vqrshrn.s32 d22, q2, #12 // t22a + vqrshrn.s32 d25, q3, #12 // t25a + vmull_vmlal q2, d30, d17, d3[3], d3[2] // -> t24a + vqrshrn.s32 d30, q4, #12 // t23a + vqrshrn.s32 d17, q2, #12 // t24a + + vld1.16 {q0}, [r12, :128] + + vqsub.s16 d2, d16, d24 // t17 + vqadd.s16 d16, d16, d24 // t16 + vqsub.s16 d3, d31, d23 // t30 + vqadd.s16 d31, d31, d23 // t31 + vqsub.s16 d24, d28, d20 // t18 + vqadd.s16 d28, d28, d20 // t19 + vqadd.s16 d23, d18, d26 // t20 + vqsub.s16 d18, d18, d26 // t21 + vqsub.s16 d20, d30, d22 // t22 + vqadd.s16 d30, d30, d22 // t23 + vqadd.s16 d26, d17, d25 // t24 + vqsub.s16 d17, d17, d25 // t25 + vqsub.s16 d22, d29, d21 // t26 + vqadd.s16 d29, d29, d21 // t27 + vqadd.s16 d25, d19, d27 // t28 + vqsub.s16 d19, d19, d27 // t29 + + vmull_vmlsl q2, d3, d2, d1[0], d1[1] // -> t17a + vmull_vmlal q3, d3, d2, d1[1], d1[0] // -> t30a + vmull_vmlal q4, d19, d24, d1[1], d1[0] // -> t18a + vqrshrn.s32 d21, q2, #12 // t17a + vqrshrn.s32 d27, q3, #12 // t30a + vneg.s32 q4, q4 // -> t18a + vmull_vmlsl q1, d19, d24, d1[0], d1[1] // -> t29a + vmull_vmlsl q2, d22, d18, d1[2], d1[3] // -> t21a + vqrshrn.s32 d19, q4, #12 // t18a + vqrshrn.s32 d24, q1, #12 // t29a + vmull_vmlal q3, d22, d18, d1[3], d1[2] // -> t26a + vmull_vmlal q4, d17, d20, d1[3], d1[2] // -> t22a + vqrshrn.s32 d22, q2, #12 // t21a + vqrshrn.s32 d18, q3, #12 // t26a + vneg.s32 q4, q4 // -> t22a + vmull_vmlsl q1, d17, d20, d1[2], d1[3] // -> t25a + vqrshrn.s32 d17, q4, #12 // t22a + vqrshrn.s32 d20, q1, #12 // t25a + + vqsub.s16 d2, d27, d24 // t29 + vqadd.s16 d27, d27, d24 // t30 + vqsub.s16 d3, d21, d19 // t18 + vqadd.s16 d21, d21, d19 // t17 + vqsub.s16 d24, d16, d28 // t19a + vqadd.s16 d16, d16, d28 // t16a + vqsub.s16 d19, d30, d23 // t20a + vqadd.s16 d30, d30, d23 // t23a + vqsub.s16 d28, d17, d22 // t21 + vqadd.s16 d17, d17, d22 // t22 + vqadd.s16 d23, d26, d29 // t24a + vqsub.s16 d26, d26, d29 // t27a + vqadd.s16 d22, d20, d18 // t25 + vqsub.s16 d20, d20, d18 // t26 + vqsub.s16 d29, d31, d25 // t28a + vqadd.s16 d31, d31, d25 // t31a + + vmull_vmlsl q2, d2, d3, d0[2], d0[3] // -> t18a + vmull_vmlal q3, d2, d3, d0[3], d0[2] // -> t29a + vmull_vmlsl q4, d29, d24, d0[2], d0[3] // -> t19 + vqrshrn.s32 d18, q2, #12 // t18a + vqrshrn.s32 d25, q3, #12 // t29a + vmull_vmlal q1, d29, d24, d0[3], d0[2] // -> t28 + vmull_vmlal q2, d26, d19, d0[3], d0[2] // -> t20 + vqrshrn.s32 d29, q4, #12 // t19 + vqrshrn.s32 d24, q1, #12 // t28 + vneg.s32 q2, q2 // -> t20 + vmull_vmlsl q3, d26, d19, d0[2], d0[3] // -> t27 + vmull_vmlal q4, d20, d28, d0[3], d0[2] // -> t21a + vqrshrn.s32 d26, q2, #12 // t20 + vqrshrn.s32 d19, q3, #12 // t27 + vneg.s32 q4, q4 // -> t21a + vmull_vmlsl q1, d20, d28, d0[2], d0[3] // -> t26a + vqrshrn.s32 d20, q4, #12 // t21a + vqrshrn.s32 d28, q1, #12 // t26a + + vqsub.s16 d2, d16, d30 // t23 + vqadd.s16 d16, d16, d30 // t16 = out16 + vqsub.s16 d3, d31, d23 // t24 + vqadd.s16 d31, d31, d23 // t31 = out31 + vqsub.s16 d23, d21, d17 // t22a + vqadd.s16 d17, d21, d17 // t17a = out17 + vqadd.s16 d30, d27, d22 // t30a = out30 + vqsub.s16 d21, d27, d22 // t25a + vqsub.s16 d27, d18, d20 // t21 + vqadd.s16 d18, d18, d20 // t18 = out18 + vqadd.s16 d4, d29, d26 // t19a = out19 + vqsub.s16 d26, d29, d26 // t20a + vqadd.s16 d29, d25, d28 // t29 = out29 + vqsub.s16 d25, d25, d28 // t26 + vqadd.s16 d28, d24, d19 // t28a = out28 + vqsub.s16 d24, d24, d19 // t27a + vmov d19, d4 // out19 + + vmull_vmlsl q2, d24, d26, d0[0], d0[0] // -> t20 + vmull_vmlal q3, d24, d26, d0[0], d0[0] // -> t27 + vqrshrn.s32 d20, q2, #12 // t20 + vqrshrn.s32 d22, q3, #12 // t27 + + vmull_vmlal q2, d25, d27, d0[0], d0[0] // -> t26a + vmull_vmlsl q3, d25, d27, d0[0], d0[0] // -> t21a + vmov d27, d22 // t27 + vqrshrn.s32 d26, q2, #12 // t26a + + vmull_vmlsl q12, d21, d23, d0[0], d0[0] // -> t22 + vmull_vmlal q2, d21, d23, d0[0], d0[0] // -> t25 + vqrshrn.s32 d21, q3, #12 // t21a + vqrshrn.s32 d22, q12, #12 // t22 + vqrshrn.s32 d25, q2, #12 // t25 + + vmull_vmlsl q2, d3, d2, d0[0], d0[0] // -> t23a + vmull_vmlal q3, d3, d2, d0[0], d0[0] // -> t24a + vqrshrn.s32 d23, q2, #12 // t23a + vqrshrn.s32 d24, q3, #12 // t24a + + bx lr +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x4_neon + push {lr} + vmov.i16 d7, #0 + lsl r8, r8, #1 +.if \scale + movw r12, #2896*8 + vdup.16 d0, r12 +.endif + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 +.if \scale + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct_4h_x16_neon + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + transpose_4x4h q14, q15, d28, d29, d30, d31 + +.macro store1 r0, r1, r2, r3 + vst1.16 {\r0}, [r6, :64]! + vst1.16 {\r1}, [r6, :64]! + vst1.16 {\r2}, [r6, :64]! + vst1.16 {\r3}, [r6, :64]! + add r6, r6, #32 +.endm + store1 d16, d20, d24, d28 + store1 d17, d21, d25, d29 + store1 d18, d22, d26, d30 + store1 d19, d23, d27, d31 +.purgem store1 + sub r6, r6, #64*4 + + vmov.i16 d7, #0 +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in d0[1] + scale_input d0[1], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct32_odd_4h_x16_neon + transpose_4x4h q15, q14, d31, d30, d29, d28 + transpose_4x4h q13, q12, d27, d26, d25, d24 + transpose_4x4h q11, q10, d23, d22, d21, d20 + transpose_4x4h q9, q8, d19, d18, d17, d16 +.macro store2 r0, r1, r2, r3, shift + vld1.16 {q0, q1}, [r6, :128] + vqsub.s16 d7, d0, \r0 + vqadd.s16 d0, d0, \r0 + vqsub.s16 d6, d1, \r1 + vqadd.s16 d1, d1, \r1 + vqsub.s16 d5, d2, \r2 + vqadd.s16 d2, d2, \r2 + vqsub.s16 d4, d3, \r3 + vqadd.s16 d3, d3, \r3 + vrev64.16 q2, q2 + vrev64.16 q3, q3 + vrshr.s16 q0, q0, #\shift + vrshr.s16 q1, q1, #\shift + vrshr.s16 q2, q2, #\shift + vrshr.s16 q3, q3, #\shift + vst1.16 {q0, q1}, [r6, :128]! + vst1.16 {q2, q3}, [r6, :128]! +.endm + + store2 d31, d27, d23, d19, \shift + store2 d30, d26, d22, d18, \shift + store2 d29, d25, d21, d17, \shift + store2 d28, d24, d20, d16, \shift +.purgem store2 + pop {pc} +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_4x32_neon + push {r10-r11,lr} + lsl r8, r8, #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + + bl inv_dct_4h_x16_neon + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vst1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + sub r7, r7, r8, lsr #1 + bl inv_dct32_odd_4h_x16_neon + + neg r9, r8 + mov r10, r6 +.macro combine r0, r1, r2, r3, op, stride + vld1.16 {d4}, [r7, :64], \stride + vld1.32 {d2[0]}, [r10, :32], r1 + vld1.16 {d5}, [r7, :64], \stride + vld1.32 {d2[1]}, [r10, :32], r1 + \op\().s16 d4, d4, \r0 + vld1.16 {d6}, [r7, :64], \stride + vld1.32 {d3[0]}, [r10, :32], r1 + \op\().s16 d5, d5, \r1 + vld1.32 {d3[1]}, [r10, :32], r1 + vrshr.s16 q2, q2, #4 + \op\().s16 d6, d6, \r2 + vld1.16 {d7}, [r7, :64], \stride + vaddw.u8 q2, q2, d2 + \op\().s16 d7, d7, \r3 + vqmovun.s16 d2, q2 + vrshr.s16 q3, q3, #4 + vst1.32 {d2[0]}, [r6, :32], r1 + vaddw.u8 q3, q3, d3 + vst1.32 {d2[1]}, [r6, :32], r1 + vqmovun.s16 d3, q3 + vst1.32 {d3[0]}, [r6, :32], r1 + vst1.32 {d3[1]}, [r6, :32], r1 +.endm + combine d31, d30, d29, d28, vqadd, r8 + combine d27, d26, d25, d24, vqadd, r8 + combine d23, d22, d21, d20, vqadd, r8 + combine d19, d18, d17, d16, vqadd, r8 + sub r7, r7, r8 + combine d16, d17, d18, d19, vqsub, r9 + combine d20, d21, d22, d23, vqsub, r9 + combine d24, d25, d26, d27, vqsub, r9 + combine d28, d29, d30, d31, vqsub, r9 +.purgem combine + + pop {r10-r11,pc} +endfunc + +const eob_32x32 + .short 10, 36, 78, 136, 210, 300, 406, 1024 +endconst + +const eob_16x32 + .short 10, 36, 78, 151, 215, 279, 343, 512 +endconst + +const eob_16x32_shortside + .short 10, 36, 78, 512 +endconst + +const eob_8x32 + // Contrary to the others, this one is only ever used in increments of 8x8 + .short 43, 107, 171, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_8bpc_neon, export=1 + push {r4-r7,lr} + vmov.i16 q0, #0 + movrel_local r5, eob_32x32, 2 + + mov r6, #2*32 +1: + mov r12, #0 + movrel_local r4, eob_32x32, 2 +2: + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r6 +.endr + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + load_add_store_8x8 r0, r7, shiftbits=2 + ldrh lr, [r4], #4 + sub r0, r0, r1, lsl #3 + cmp r3, lr + add r0, r0, #8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12 + add r0, r0, r1, lsl #3 + mls r2, r6, r12, r2 + add r2, r2, #2*8 + b 1b +9: + pop {r4-r7,pc} +endfunc + +.macro shift_8_regs op, shift +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + push {r4-r7,lr} + movw r6, #2896*8 + movw r7, #2*(5793-4096)*8 + vdup.i16 d0, r6 + movrel_local r5, eob_16x32\hshort, 2 + vmov.16 d0[1], r7 + + mov r6, #2*\h +1: + mov r12, #0 + movrel_local r4, eob_16x32\wshort, 2 +2: + vmov.i16 q1, #0 + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q1}, [r2, :128], r6 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 + +.if \w == 16 + // 16x32 + identity_8x8_shift1 d0[1] +.else + // 32x16 + shift_8_regs vqshl.s16, 1 + identity_8x8 d0[1] +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + +.if \w == 16 + load_add_store_8x8 r0, r7, shiftbits=2 +.else + load_add_store_8x8 r0, r7, shiftbits=4 +.endif + ldrh lr, [r4], #4 + sub r0, r0, r1, lsl #3 + cmp r3, lr + add r0, r0, #8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12 + add r0, r0, r1, lsl #3 + mls r2, r6, r12, r2 + add r2, r2, #2*8 + b 1b +9: + pop {r4-r7,pc} +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + push {r4-r5,lr} + vmov.i16 q0, #0 + movrel_local r4, eob_8x32 + + mov r12, #2*\h +1: + ldrh lr, [r4], #2 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r12 +.endr + +.if \w == 8 + // 8x32 + shift_8_regs vrshr.s16, 1 +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + cmp r3, lr +.if \w == 8 + load_add_store_8x8 r0, r5, shiftbits=2 +.else + load_add_store_8x8 r0, r5, shiftbits=3 +.endif + + blt 9f +.if \w == 8 + sub r2, r2, r12, lsl #3 + add r2, r2, #2*8 +.else + sub r0, r0, r1, lsl #3 + add r0, r0, #8 +.endif + b 1b + +9: + pop {r4-r5,pc} +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_8bpc_neon, export=1 + idct_dc 32, 32, 2 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 2048 + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, sp, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 2048 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x32_8bpc_neon, export=1 + idct_dc 16, 32, 1 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel_local r4, inv_dct_4h_x16_neon + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, sp, #(\i*16*2) + add r7, r2, #(\i*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #2*32 + bl inv_txfm_horz_scale_16x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #16*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 1024 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x16_8bpc_neon, export=1 + idct_dc 32, 16, 1 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel_local r5, inv_dct_4h_x16_neon + +.irp i, 0, 4, 8, 12 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 12 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #2*16 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 1024 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_8x32_8bpc_neon, export=1 + idct_dc 8, 32, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 512 + + movrel_local r10, eob_8x32 + + mov r8, #2*32 + mov r9, #32 + mov r6, sp +1: + vmov.i16 q0, #0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r8 +.endr + ldrh r11, [r10], #2 + sub r2, r2, r8, lsl #3 + sub r9, r9, #8 + add r2, r2, #2*8 + + bl inv_dct_8h_x8_neon + +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \i, \i, #2 +.endr + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + vst1.16 {q8, q9}, [r6, :128]! + cmp r3, r11 + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r6, :128]! + vst1.16 {q14, q15}, [r6, :128]! + + bge 1b + cmp r9, #0 + beq 3f + + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r9, r9, #8 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #8*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x8_8bpc_neon, export=1 + idct_dc 32, 8, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 512 + +.irp i, 0, 4 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*2) +.if \i > 0 + cmp r3, #10 + blt 1f +.endif + mov r8, #8*2 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 2f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + +2: + mov r8, #2*32 + mov r9, #0 +1: + add r6, r0, r9 + add r7, sp, r9, lsl #1 // #(\i*2) + +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r7, :128], r8 +.endr + add r9, r9, #8 + + bl inv_dct_8h_x8_neon + + cmp r9, #32 + + load_add_store_8x8 r6, r7 + + blt 1b + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + vld1.16 {d0, d1, d2}, [r12, :64]! + + vqrdmulh.s16 d23, d16, d0[1] // t63a + vqrdmulh.s16 d16, d16, d0[0] // t32a + vqrdmulh.s16 d22, d17, d0[2] // t62a + vqrdmulh.s16 d17, d17, d0[3] // t33a + vqrdmulh.s16 d21, d18, d1[1] // t61a + vqrdmulh.s16 d18, d18, d1[0] // t34a + vqrdmulh.s16 d20, d19, d1[2] // t60a + vqrdmulh.s16 d19, d19, d1[3] // t35a + + vqadd.s16 d24, d16, d17 // t32 + vqsub.s16 d25, d16, d17 // t33 + vqsub.s16 d26, d19, d18 // t34 + vqadd.s16 d27, d19, d18 // t35 + vqadd.s16 d28, d20, d21 // t60 + vqsub.s16 d29, d20, d21 // t61 + vqsub.s16 d30, d23, d22 // t62 + vqadd.s16 d31, d23, d22 // t63 + + vmull_vmlal q2, d29, d26, d2[0], d2[1] // -> t34a + vmull_vmlsl q3, d29, d26, d2[1], d2[0] // -> t61a + vneg.s32 q2, q2 // t34a + vmull_vmlsl q4, d30, d25, d2[1], d2[0] // -> t33a + vqrshrn.s32 d26, q2, #12 // t34a + vmull_vmlal q2, d30, d25, d2[0], d2[1] // -> t62a + vqrshrn.s32 d29, q3, #12 // t61a + vqrshrn.s32 d25, q4, #12 // t33a + vqrshrn.s32 d30, q2, #12 // t62a + + vqadd.s16 d16, d24, d27 // t32a + vqsub.s16 d19, d24, d27 // t35a + vqadd.s16 d17, d25, d26 // t33 + vqsub.s16 d18, d25, d26 // t34 + vqsub.s16 d20, d31, d28 // t60a + vqadd.s16 d23, d31, d28 // t63a + vqsub.s16 d21, d30, d29 // t61 + vqadd.s16 d22, d30, d29 // t62 + + vmull_vmlal q2, d21, d18, d2[2], d2[3] // -> t61a + vmull_vmlsl q3, d21, d18, d2[3], d2[2] // -> t34a + vmull_vmlal q4, d20, d19, d2[2], d2[3] // -> t60 + vqrshrn.s32 d21, q2, #12 // t61a + vqrshrn.s32 d18, q3, #12 // t34a + vmull_vmlsl q2, d20, d19, d2[3], d2[2] // -> t35 + vqrshrn.s32 d20, q4, #12 // t60 + vqrshrn.s32 d19, q2, #12 // t35 + + vst1.16 {d16, d17, d18, d19}, [r6, :128]! + vst1.16 {d20, d21, d22, d23}, [r6, :128]! + + bx lr +endfunc + +function inv_dct64_step2_neon + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + vldr d16, [r6, #2*4*0] // t32a + vldr d17, [r9, #2*4*8] // t39a + vldr d18, [r9, #2*4*0] // t63a + vldr d19, [r6, #2*4*8] // t56a + vldr d20, [r6, #2*4*16] // t40a + vldr d21, [r9, #2*4*24] // t47a + vldr d22, [r9, #2*4*16] // t55a + vldr d23, [r6, #2*4*24] // t48a + + vqadd.s16 d24, d16, d17 // t32 + vqsub.s16 d25, d16, d17 // t39 + vqadd.s16 d26, d18, d19 // t63 + vqsub.s16 d27, d18, d19 // t56 + vqsub.s16 d28, d21, d20 // t40 + vqadd.s16 d29, d21, d20 // t47 + vqadd.s16 d30, d23, d22 // t48 + vqsub.s16 d31, d23, d22 // t55 + + vmull_vmlal q2, d27, d25, d0[3], d0[2] // -> t56a + vmull_vmlsl q3, d27, d25, d0[2], d0[3] // -> t39a + vmull_vmlal q4, d31, d28, d0[3], d0[2] // -> t40a + vqrshrn.s32 d25, q2, #12 // t56a + vqrshrn.s32 d27, q3, #12 // t39a + vneg.s32 q4, q4 // t40a + vmull_vmlsl q2, d31, d28, d0[2], d0[3] // -> t55a + vqrshrn.s32 d31, q4, #12 // t40a + vqrshrn.s32 d28, q2, #12 // t55a + + vqadd.s16 d16, d24, d29 // t32a + vqsub.s16 d19, d24, d29 // t47a + vqadd.s16 d17, d27, d31 // t39 + vqsub.s16 d18, d27, d31 // t40 + vqsub.s16 d20, d26, d30 // t48a + vqadd.s16 d23, d26, d30 // t63a + vqsub.s16 d21, d25, d28 // t55 + vqadd.s16 d22, d25, d28 // t56 + + vmull_vmlsl q2, d21, d18, d0[0], d0[0] // -> t40a + vmull_vmlal q3, d21, d18, d0[0], d0[0] // -> t55a + vmull_vmlsl q4, d20, d19, d0[0], d0[0] // -> t47 + vqrshrn.s32 d18, q2, #12 // t40a + vqrshrn.s32 d21, q3, #12 // t55a + vmull_vmlal q2, d20, d19, d0[0], d0[0] // -> t48 + vqrshrn.s32 d19, q4, #12 // t47 + vqrshrn.s32 d20, q2, #12 // t48 + + vstr d16, [r6, #2*4*0] // t32a + vstr d17, [r9, #2*4*0] // t39 + vstr d18, [r6, #2*4*8] // t40a + vstr d19, [r9, #2*4*8] // t47 + vstr d20, [r6, #2*4*16] // t48 + vstr d21, [r9, #2*4*16] // t55a + vstr d22, [r6, #2*4*24] // t56 + vstr d23, [r9, #2*4*24] // t63a + + add r6, r6, #2*4 + sub r9, r9, #2*4 + cmp r6, r9 + blt 1b + bx lr +endfunc + +.macro load8 src, strd, zero, clear +.irp i, d16, d17, d18, d19, d20, d21, d22, d23 +.if \clear + vld1.16 {\i}, [\src, :64] + vst1.16 {\zero}, [\src, :64], \strd +.else + vld1.16 {\i}, [\src, :64], \strd +.endif +.endr +.endm + +.macro store16 dst + vst1.16 {q8, q9}, [\dst, :128]! + vst1.16 {q10, q11}, [\dst, :128]! + vst1.16 {q12, q13}, [\dst, :128]! + vst1.16 {q14, q15}, [\dst, :128]! +.endm + +.macro clear_upper8 +.irp i, q12, q13, q14, q15 + vmov.i16 \i, #0 +.endr +.endm + +.macro vmov_if reg, val, cond +.if \cond + vmov.i16 \reg, \val +.endif +.endm + +.macro movdup_if reg, gpr, val, cond +.if \cond + movw \gpr, \val + vdup.16 \reg, \gpr +.endif +.endm + +.macro vst1_if regs, dst, dstalign, cond +.if \cond + vst1.16 \regs, \dst, \dstalign +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_4h_x64_neon, export=1 + mov r6, sp + + push {r10-r11,lr} + + lsl r8, r8, #2 + + movdup_if d0, r12, #2896*8, \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + add r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct_4h_x16_neon + + store16 r6 + + movdup_if d0, r12, #2896*8, \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + lsr r8, r8, #1 + sub r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct32_odd_4h_x16_neon + + add r10, r6, #8*15 + sub r6, r6, #8*16 + + mov r9, #-8 + +.macro store_addsub r0, r1, r2, r3 + vld1.16 {d2}, [r6, :64]! + vld1.16 {d3}, [r6, :64]! + vqadd.s16 d6, d2, \r0 + vqsub.s16 \r0, d2, \r0 + vld1.16 {d4}, [r6, :64]! + vqadd.s16 d7, d3, \r1 + vqsub.s16 \r1, d3, \r1 + vld1.16 {d5}, [r6, :64]! + vqadd.s16 d2, d4, \r2 + sub r6, r6, #8*4 + vqsub.s16 \r2, d4, \r2 + vst1.16 {d6}, [r6, :64]! + vst1.16 {\r0}, [r10, :64], r9 + vqadd.s16 d3, d5, \r3 + vqsub.s16 \r3, d5, \r3 + vst1.16 {d7}, [r6, :64]! + vst1.16 {\r1}, [r10, :64], r9 + vst1.16 {d2}, [r6, :64]! + vst1.16 {\r2}, [r10, :64], r9 + vst1.16 {d3}, [r6, :64]! + vst1.16 {\r3}, [r10, :64], r9 +.endm + store_addsub d31, d30, d29, d28 + store_addsub d27, d26, d25, d24 + store_addsub d23, d22, d21, d20 + store_addsub d19, d18, d17, d16 +.purgem store_addsub + + add r6, r6, #2*4*16 + + movrel_local r12, idct64_coeffs + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + add r9, r7, r8, lsl #4 // offset 16 + add r10, r7, r8, lsl #3 // offset 8 + sub r9, r9, r8 // offset 15 + sub r11, r10, r8 // offset 7 + vld1.16 {d16}, [r7, :64] // in1 (offset 0) + vld1.16 {d17}, [r9, :64] // in31 (offset 15) + vld1.16 {d18}, [r10, :64] // in17 (offset 8) + vld1.16 {d19}, [r11, :64] // in15 (offset 7) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + add r7, r7, r8, lsl #2 // offset 4 + sub r9, r9, r8, lsl #2 // offset 11 + sub r10, r7, r8 // offset 3 + add r11, r9, r8 // offset 12 + vld1.16 {d16}, [r10, :64] // in7 (offset 3) + vld1.16 {d17}, [r11, :64] // in25 (offset 12) + vld1.16 {d18}, [r9, :64] // in23 (offset 11) + vld1.16 {d19}, [r7, :64] // in9 (offset 4) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + sub r10, r10, r8, lsl #1 // offset 1 + sub r9, r9, r8, lsl #1 // offset 9 + add r10, r10, r8 // offset 2 + add r9, r9, r8 // offset 10 + add r7, r7, r8 // offset 5 + add r11, r11, r8 // offset 13 + vld1.16 d16, [r10, :64] // in5 (offset 2) + vld1.16 d17, [r11, :64] // in27 (offset 13) + vld1.16 d18, [r9, :64] // in21 (offset 10) + vld1.16 d19, [r7, :64] // in11 (offset 5) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + sub r10, r10, r8 // offset 1 + sub r9, r9, r8 // offset 9 + add r11, r11, r8 // offset 14 + add r7, r7, r8 // offset 6 + vld1.16 d16, [r10, :64] // in3 (offset 1) + vld1.16 d17, [r11, :64] // in29 (offset 14) + vld1.16 d18, [r9, :64] // in19 (offset 9) + vld1.16 d19, [r7, :64] // in13 (offset 6) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + + sub r6, r6, #2*4*32 + add r9, r6, #2*4*7 + + bl inv_dct64_step2_neon + + pop {r10-r11,pc} +endfunc +.endm + +def_dct64_func +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + +function inv_txfm_horz_dct_64x4_neon + vdup.16 q3, r9 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, #2*56 + + push {r10-r11,lr} + + mov r10, #2*64 + mov r11, #-2*4*4 + +1: + vld1.16 {d16, d17, d18, d19}, [r7, :128]! + vld1.16 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.16 {d20, d21, d22, d23}, [r7, :128]! + vld1.16 {d24, d25, d26, d27}, [r8, :128], r11 + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q15, q14, d31, d30, d29, d28 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q13, q12, d27, d26, d25, d24 + +.macro store_addsub src0, src1, src2, src3 + vqsub.s16 d3, \src0, \src1 + vqsub.s16 d2, \src2, \src3 + vqadd.s16 d0, \src0, \src1 + vqadd.s16 d1, \src2, \src3 + vrshl.s16 q1, q1, q3 + vrshl.s16 q0, q0, q3 + vrev64.16 q1, q1 + vst1.16 {q0}, [r6, :128], r10 + vst1.16 {q1}, [r9, :128], r10 +.endm + store_addsub d16, d31, d20, d27 + store_addsub d17, d30, d21, d26 + store_addsub d18, d29, d22, d25 + store_addsub d19, d28, d23, d24 +.purgem store_addsub + sub r6, r6, r10, lsl #2 + sub r9, r9, r10, lsl #2 + add r6, r6, #16 + sub r9, r9, #16 + + cmp r7, r8 + blt 1b + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_vert_dct_4x64_neon + lsl r8, r8, #1 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, r1, lsl #6 + sub r9, r9, r1 + + push {r10-r11,lr} + + neg r10, r1 + mov r11, #-2*4*4 + +1: + vld1.16 {d16, d17, d18, d19}, [r7, :128]! + vld1.16 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.16 {d20, d21, d22, d23}, [r7, :128]! + vld1.16 {d24, d25, d26, d27}, [r8, :128], r11 + +.macro add_dest_addsub src0, src1, src2, src3 + vld1.32 {d0[0]}, [r6, :32], r1 + vld1.32 {d1[0]}, [r9, :32], r10 + vqadd.s16 d4, \src0, \src1 + vld1.32 {d0[1]}, [r6, :32] + vqadd.s16 d5, \src2, \src3 + vld1.32 {d1[1]}, [r9, :32] + vqsub.s16 d6, \src0, \src1 + vqsub.s16 d7, \src2, \src3 + sub r6, r6, r1 + sub r9, r9, r10 + vrshr.s16 q2, q2, #4 + vrshr.s16 q3, q3, #4 + vaddw.u8 q2, q2, d0 + vaddw.u8 q3, q3, d1 + vqmovun.s16 d0, q2 + vqmovun.s16 d1, q3 + vst1.32 {d0[0]}, [r6, :32], r1 + vst1.32 {d1[0]}, [r9, :32], r10 + vst1.32 {d0[1]}, [r6, :32], r1 + vst1.32 {d1[1]}, [r9, :32], r10 +.endm + add_dest_addsub d16, d31, d17, d30 + add_dest_addsub d18, d29, d19, d28 + add_dest_addsub d20, d27, d21, d26 + add_dest_addsub d22, d25, d23, d24 +.purgem add_dest_addsub + cmp r7, r8 + blt 1b + + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x64_8bpc_neon, export=1 + idct_dc 64, 64, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_clear_4h_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r7, r5, #(\i*2) + mov r8, #64*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x32_8bpc_neon, export=1 + idct_dc 64, 32, 1 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_clear_scale_4h_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-1 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i) + add r7, r5, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x64_8bpc_neon, export=1 + idct_dc 32, 64, 1 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 32*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r7, r5, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 32*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x16_8bpc_neon, export=1 + idct_dc 64, 16, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*16*2+64*4*2 + add r4, sp, #64*4*2 + + movrel_local r10, eob_16x32 + +.irp i, 0, 4, 8, 12 + add r6, r4, #(\i*64*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + bl inv_txfm_dct_clear_4h_x64_neon + add r6, r4, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 12 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: + movrel_local r5, inv_dct_4h_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i) + add r7, r4, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 64*16*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x64_8bpc_neon, export=1 + idct_dc 16, 64, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 16*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + + movrel_local r4, inv_dct_4h_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*16*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_16x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r7, r5, #(\i*2) + mov r8, #16*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 16*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/itx16.S b/third_party/dav1d/src/arm/32/itx16.S new file mode 100644 index 0000000000..aa6c272e71 --- /dev/null +++ b/third_party/dav1d/src/arm/32/itx16.S @@ -0,0 +1,3625 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob); + +// Most of the functions use the following register layout: +// r0-r3 external parameters +// r4 function pointer to first transform +// r5 function pointer to second transform +// r6 output parameter for helper function +// r7 input parameter for helper function +// r8 input stride for helper function +// r9 scratch variable for helper functions +// r10-r11 pointer to list of eob thresholds, eob threshold value, +// scratch variables within helper functions (backed up) + +// The SIMD registers most often use the following layout: +// d0-d3 multiplication coefficients +// d4-d7 scratch registers +// d8-d15 unused in some transforms, used for scratch registers in others +// d16-v31 inputs/outputs of transforms + +// Potential further optimizations, that are left unimplemented for now: +// - Trying to keep multiplication coefficients in registers across multiple +// transform functions. (The register layout is designed to potentially +// allow this.) +// - Use a simplified version of the transforms themselves for cases where +// we know a significant number of inputs are zero. E.g. if the eob value +// indicates only a quarter of input values are set, for idct16 and up, +// a significant amount of calculation can be skipped, at the cost of more +// code duplication and special casing. + +// A macro for cases where a thumb mov can express the constant in one +// instruction, while arm mode requires two separate movw+movt pairs. +.macro mov_const reg, val +#if CONFIG_THUMB + mov.w \reg, #\val +#else + movw \reg, #((\val) & 0xffff) + movt \reg, #(((\val) >> 16) & 0xffff) +#endif +.endm + +const idct_coeffs, align=4 + // idct4 + .int 2896, 2896*8*(1<<16), 1567, 3784 + // idct8 + .int 799, 4017, 3406, 2276 + // idct16 + .int 401, 4076, 3166, 2598 + .int 1931, 3612, 3920, 1189 + // idct32 + .int 201, 4091, 3035, 2751 + .int 1751, 3703, 3857, 1380 + .int 995, 3973, 3513, 2106 + .int 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .int 101*8*(1<<16), 4095*8*(1<<16), 2967*8*(1<<16), -2824*8*(1<<16) + .int 1660*8*(1<<16), 3745*8*(1<<16), 3822*8*(1<<16), -1474*8*(1<<16) + .int 4076, 401, 4017, 799 + + .int 4036*8*(1<<16), -700*8*(1<<16), 2359*8*(1<<16), 3349*8*(1<<16) + .int 3461*8*(1<<16), -2191*8*(1<<16), 897*8*(1<<16), 3996*8*(1<<16) + .int -3166, -2598, -799, -4017 + + .int 501*8*(1<<16), 4065*8*(1<<16), 3229*8*(1<<16), -2520*8*(1<<16) + .int 2019*8*(1<<16), 3564*8*(1<<16), 3948*8*(1<<16), -1092*8*(1<<16) + .int 3612, 1931, 2276, 3406 + + .int 4085*8*(1<<16), -301*8*(1<<16), 2675*8*(1<<16), 3102*8*(1<<16) + .int 3659*8*(1<<16), -1842*8*(1<<16), 1285*8*(1<<16), 3889*8*(1<<16) + .int -3920, -1189, -3406, -2276 +endconst + +const iadst4_coeffs, align=4 + .int 1321, 3803, 2482, 3344 +endconst + +const iadst8_coeffs, align=4 + .int 4076, 401, 3612, 1931 + .int 2598, 3166, 1189, 3920 + // idct_coeffs + .int 2896, 0, 1567, 3784 +endconst + +const iadst16_coeffs, align=4 + .int 4091, 201, 3973, 995 + .int 3703, 1751, 3290, 2440 + .int 2751, 3035, 2106, 3513 + .int 1380, 3857, 601, 4052 +endconst + +.macro vmul_vmla d0, s0, s1, c0, c1 + vmul.i32 \d0, \s0, \c0 + vmla.i32 \d0, \s1, \c1 +.endm + +.macro vmul_vmls d0, s0, s1, c0, c1 + vmul.i32 \d0, \s0, \c0 + vmls.i32 \d0, \s1, \c1 +.endm + +.macro scale_input c, r0, r1, r2 r3, r4, r5, r6, r7 + vqrdmulh.s32 \r0, \r0, \c + vqrdmulh.s32 \r1, \r1, \c +.ifnb \r2 + vqrdmulh.s32 \r2, \r2, \c + vqrdmulh.s32 \r3, \r3, \c +.endif +.ifnb \r4 + vqrdmulh.s32 \r4, \r4, \c + vqrdmulh.s32 \r5, \r5, \c + vqrdmulh.s32 \r6, \r6, \c + vqrdmulh.s32 \r7, \r7, \c +.endif +.endm + +.macro load_add_store load, shift, addsrc, adddst, max, min, store, dst, src, shiftbits=4 +.ifnb \load + vld1.16 {\load}, [\src, :128], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + vqadd.s16 \adddst, \adddst, \addsrc +.endif +.ifnb \max + vmax.s16 \max, \max, q6 +.endif +.ifnb \min + vmin.s16 \min, \min, q7 +.endif +.ifnb \store + vst1.16 {\store}, [\dst, :128], r1 +.endif +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff + load_add_store q0, q8, , , , , , \dst, \src, \shiftbits + load_add_store q1, q9, , , , , , \dst, \src, \shiftbits + load_add_store q2, q10, q0, q8, , , , \dst, \src, \shiftbits + load_add_store q3, q11, q1, q9, q8, , , \dst, \src, \shiftbits + load_add_store q4, q12, q2, q10, q9, q8, , \dst, \src, \shiftbits + load_add_store q5, q13, q3, q11, q10, q9, q8, \dst, \src, \shiftbits + load_add_store q0, q14, q4, q12, q11, q10, q9, \dst, \src, \shiftbits + load_add_store q1, q15, q5, q13, q12, q11, q10, \dst, \src, \shiftbits + load_add_store , , q0, q14, q13, q12, q11, \dst, \src, \shiftbits + load_add_store , , q1, q15, q14, q13, q12, \dst, \src, \shiftbits + load_add_store , , , , q15, q14, q13, \dst, \src, \shiftbits + load_add_store , , , , , q15, q14, \dst, \src, \shiftbits + load_add_store , , , , , , q15, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src, shiftbits=4 + mov \src, \dst + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff + load_add_store q0, q8, , , , , , \dst, \src, \shiftbits + load_add_store q1, q9, , , , , , \dst, \src, \shiftbits + load_add_store q2, q10, q0, q8, , , , \dst, \src, \shiftbits + load_add_store q3, q11, q1, q9, q8, , , \dst, \src, \shiftbits + load_add_store , , q2, q10, q9, q8, , \dst, \src, \shiftbits + load_add_store , , q3, q11, q10, q9, q8, \dst, \src, \shiftbits + load_add_store , , , , q11, q10, q9, \dst, \src, \shiftbits + load_add_store , , , , , q11, q10, \dst, \src, \shiftbits + load_add_store , , , , , , q11, \dst, \src, \shiftbits +.endm +.macro load_add_store4 load1, load2, shift, addsrc, adddst, max, min, store1, store2, dst, src, shiftbits=4 +.ifnb \load1 + vld1.16 {\load1}, [\src, :64], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #\shiftbits +.endif +.ifnb \load2 + vld1.16 {\load2}, [\src, :64], r1 +.endif +.ifnb \addsrc + vqadd.s16 \adddst, \adddst, \addsrc +.endif +.ifnb \max + vmax.s16 \max, \max, q6 +.endif +.ifnb \store1 + vst1.16 {\store1}, [\dst, :64], r1 +.endif +.ifnb \min + vmin.s16 \min, \min, q7 +.endif +.ifnb \store2 + vst1.16 {\store2}, [\dst, :64], r1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff + mov \src, \dst + load_add_store4 d0, d1, q8, , , , , , , \dst, \src + load_add_store4 d2, d3, q9, , , , , , , \dst, \src + load_add_store4 d4, d5, q10, q0, q8, , , , , \dst, \src + load_add_store4 d6, d7, q11, q1, q9, q8, , , , \dst, \src + load_add_store4 d8, d9, q12, q2, q10, q9, q8, , , \dst, \src + load_add_store4 d10, d11, q13, q3, q11, q10, q9, d16, d17, \dst, \src + load_add_store4 d0, d1, q14, q4, q12, q11, q10, d18, d19, \dst, \src + load_add_store4 d2, d3, q15, q5, q13, q12, q11, d20, d21, \dst, \src + load_add_store4 , , , q0, q14, q13, q12, d22, d23, \dst, \src + load_add_store4 , , , q1, q15, q14, q13, d24, d25, \dst, \src + load_add_store4 , , , , , q15, q14, d26, d27, \dst, \src + load_add_store4 , , , , , , q15, d28, d29, \dst, \src + load_add_store4 , , , , , , , d30, d31, \dst, \src +.endm +.macro load_add_store_4x8 dst, src, shiftbits=4 + mov \src, \dst + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff + mov \src, \dst + load_add_store4 d0, d1, q8, , , , , , , \dst, \src, \shiftbits + load_add_store4 d2, d3, q9, , , , , , , \dst, \src, \shiftbits + load_add_store4 d4, d5, q10, q0, q8, , , , , \dst, \src, \shiftbits + load_add_store4 d6, d7, q11, q1, q9, q8, , , , \dst, \src, \shiftbits + load_add_store4 , , , q2, q10, q9, q8, , , \dst, \src, \shiftbits + load_add_store4 , , , q3, q11, q10, q9, d16, d17, \dst, \src, \shiftbits + load_add_store4 , , , , , q11, q10, d18, d19, \dst, \src, \shiftbits + load_add_store4 , , , , , , q11, d20, d21, \dst, \src, \shiftbits + load_add_store4 , , , , , , , d22, d23, \dst, \src, \shiftbits +.endm +.macro load_add_store_4x4 dst, src, shiftbits=4 + mov \src, \dst + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff + mov \src, \dst + load_add_store4 d0, d1, q8, , , , , , , \dst, \src, \shiftbits + load_add_store4 d2, d3, q9, q0, q8, , , , , \dst, \src, \shiftbits + load_add_store4 , , , q1, q9, q8, , , , \dst, \src, \shiftbits + load_add_store4 , , , , , q9, q8, , , \dst, \src, \shiftbits + load_add_store4 , , , , , , q9, d16, d17, \dst, \src, \shiftbits + load_add_store4 , , , , , , , d18, d19, \dst, \src, \shiftbits +.endm + +.macro idct_dc w, h, shift + cmp r3, #0 + bne 1f + vmov.i16 q14, #0 + mov_const r12, 2896*8*(1<<16) + vld1.32 {d24[], d25[]}, [r2, :32] + vdup.32 d0, r12 + vqrdmulh.s32 q13, q12, d0[0] + vst1.32 {d28[0]}, [r2, :32] +.if (\w == 2*\h) || (2*\w == \h) + vqrdmulh.s32 q13, q13, d0[0] +.endif +.if \shift > 0 + vqrshrn.s32 d24, q13, #\shift + vqrshrn.s32 d25, q13, #\shift +.else + vqmovn.s32 d24, q13 + vqmovn.s32 d25, q13 +.endif + vqrdmulh.s16 q12, q12, d0[1] + mov r3, #\h + vrshr.s16 q12, q12, #4 + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon + vmvn.i16 q15, #0xfc00 // 0x3ff +1: + vld1.16 {d0}, [r0, :64], r1 + vld1.16 {d1}, [r0, :64], r1 + vld1.16 {d2}, [r0, :64], r1 + vld1.16 {d3}, [r0, :64], r1 + subs r3, r3, #4 + vqadd.s16 q0, q0, q12 + sub r0, r0, r1, lsl #2 + vqadd.s16 q1, q1, q12 + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmin.s16 q0, q0, q15 + vst1.16 {d0}, [r0, :64], r1 + vmin.s16 q1, q1, q15 + vst1.16 {d1}, [r0, :64], r1 + vst1.16 {d2}, [r0, :64], r1 + vst1.16 {d3}, [r0, :64], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w8_neon + vmvn.i16 q15, #0xfc00 // 0x3ff +1: + vld1.16 {q0}, [r0, :128], r1 + subs r3, r3, #4 + vld1.16 {q1}, [r0, :128], r1 + vqadd.s16 q0, q0, q12 + vld1.16 {q2}, [r0, :128], r1 + vqadd.s16 q1, q1, q12 + vld1.16 {q3}, [r0, :128], r1 + vqadd.s16 q2, q2, q12 + vqadd.s16 q3, q3, q12 + sub r0, r0, r1, lsl #2 + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q0, q0, q15 + vmin.s16 q1, q1, q15 + vst1.16 {q0}, [r0, :128], r1 + vmin.s16 q2, q2, q15 + vst1.16 {q1}, [r0, :128], r1 + vmin.s16 q3, q3, q15 + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w16_neon + vmvn.i16 q15, #0xfc00 // 0x3ff +1: + vld1.16 {q0, q1}, [r0, :128], r1 + subs r3, r3, #2 + vld1.16 {q2, q3}, [r0, :128], r1 + vqadd.s16 q0, q0, q12 + vqadd.s16 q1, q1, q12 + vqadd.s16 q2, q2, q12 + vqadd.s16 q3, q3, q12 + sub r0, r0, r1, lsl #1 + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q0, q0, q15 + vmin.s16 q1, q1, q15 + vmin.s16 q2, q2, q15 + vst1.16 {q0, q1}, [r0, :128], r1 + vmin.s16 q3, q3, q15 + vst1.16 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w32_neon + sub r1, r1, #32 + vmvn.i16 q15, #0xfc00 // 0x3ff +1: + vld1.16 {q0, q1}, [r0, :128]! + subs r3, r3, #1 + vld1.16 {q2, q3}, [r0, :128] + vqadd.s16 q0, q0, q12 + vqadd.s16 q1, q1, q12 + vqadd.s16 q2, q2, q12 + vqadd.s16 q3, q3, q12 + sub r0, r0, #32 + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmin.s16 q0, q0, q15 + vmin.s16 q1, q1, q15 + vmin.s16 q2, q2, q15 + vst1.16 {q0, q1}, [r0, :128]! + vmin.s16 q3, q3, q15 + vst1.16 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w64_neon + sub r1, r1, #96 + vmvn.i16 q15, #0xfc00 // 0x3ff +1: + vld1.16 {q0, q1}, [r0, :128]! + subs r3, r3, #1 + vld1.16 {q2, q3}, [r0, :128]! + vqadd.s16 q0, q0, q12 + vld1.16 {q8, q9}, [r0, :128]! + vqadd.s16 q1, q1, q12 + vld1.16 {q10, q11}, [r0, :128] + vqadd.s16 q2, q2, q12 + vqadd.s16 q3, q3, q12 + vqadd.s16 q8, q8, q12 + vqadd.s16 q9, q9, q12 + vqadd.s16 q10, q10, q12 + vqadd.s16 q11, q11, q12 + sub r0, r0, #96 + vmax.s16 q0, q0, q14 + vmax.s16 q1, q1, q14 + vmax.s16 q2, q2, q14 + vmax.s16 q3, q3, q14 + vmax.s16 q8, q8, q14 + vmax.s16 q9, q9, q14 + vmax.s16 q10, q10, q14 + vmax.s16 q11, q11, q14 + vmin.s16 q0, q0, q15 + vmin.s16 q1, q1, q15 + vmin.s16 q2, q2, q15 + vmin.s16 q3, q3, q15 + vmin.s16 q8, q8, q15 + vst1.16 {q0, q1}, [r0, :128]! + vmin.s16 q9, q9, q15 + vst1.16 {q2, q3}, [r0, :128]! + vmin.s16 q10, q10, q15 + vst1.16 {q8, q9}, [r0, :128]! + vmin.s16 q11, q11, q15 + vst1.16 {q10, q11}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +.macro iwht4 + vadd.i32 q8, q8, q9 + vsub.i32 q13, q10, q11 + vsub.i32 q12, q8, q13 + vshr.s32 q12, q12, #1 + vsub.i32 q10, q12, q9 + vsub.i32 q9, q12, q11 + vadd.i32 q11, q13, q10 + vsub.i32 q8, q8, q9 +.endm + +.macro idct_4s_x4 r0, r1, r2, r3 + vmul_vmla q4, \r1, \r3, d1[1], d1[0] + vmul_vmla q2, \r0, \r2, d0[0], d0[0] + vmul_vmls q3, \r1, \r3, d1[0], d1[1] + vmul_vmls q5, \r0, \r2, d0[0], d0[0] + vrshr.s32 q4, q4, #12 + vrshr.s32 q2, q2, #12 + vrshr.s32 q3, q3, #12 + vrshr.s32 q5, q5, #12 + vqadd.s32 \r0, q2, q4 + vqsub.s32 \r3, q2, q4 + vqadd.s32 \r1, q5, q3 + vqsub.s32 \r2, q5, q3 +.endm + +.macro idct_2s_x4 r0, r1, r2, r3 + vmul_vmla d6, \r1, \r3, d1[1], d1[0] + vmul_vmla d4, \r0, \r2, d0[0], d0[0] + vmul_vmls d5, \r1, \r3, d1[0], d1[1] + vmul_vmls d7, \r0, \r2, d0[0], d0[0] + vrshr.s32 d6, d6, #12 + vrshr.s32 d4, d4, #12 + vrshr.s32 d5, d5, #12 + vrshr.s32 d7, d7, #12 + vqadd.s32 \r0, d4, d6 + vqsub.s32 \r3, d4, d6 + vqadd.s32 \r1, d7, d5 + vqsub.s32 \r2, d7, d5 +.endm + +function inv_dct_4s_x4_neon + movrel_local r12, idct_coeffs + vld1.32 {d0, d1}, [r12, :128] + idct_4s_x4 q8, q9, q10, q11 + bx lr +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel_local r12, iadst4_coeffs + vld1.32 {d0, d1}, [r12, :128] + + vsub.i32 q1, q8, q10 + vmul.i32 q2, q8, d0[0] + vmla.i32 q2, q10, d0[1] + vmla.i32 q2, q11, d1[0] + vmul.i32 q4, q9, d1[1] + vadd.i32 q1, q1, q11 + vmul.i32 q3, q8, d1[0] + vmls.i32 q3, q10, d0[0] + vmls.i32 q3, q11, d0[1] + + vadd.i32 \o3, q2, q3 + vmul.i32 \o2, q1, d1[1] + vadd.i32 \o0, q2, q4 + vadd.i32 \o1, q3, q4 + vsub.i32 \o3, \o3, q4 + + vrshr.s32 \o0, \o0, #12 + vrshr.s32 \o2, \o2, #12 + vrshr.s32 \o1, \o1, #12 + vrshr.s32 \o3, \o3, #12 +.endm + +function inv_adst_4s_x4_neon + iadst_4x4 q8, q9, q10, q11 + bx lr +endfunc + +function inv_flipadst_4s_x4_neon + iadst_4x4 q11, q10, q9, q8 + bx lr +endfunc + +function inv_identity_4s_x4_neon + mov r12, #0 + movt r12, #(5793-4096)*8 + vdup.32 d0, r12 + vqrdmulh.s32 q1, q8, d0[0] + vqrdmulh.s32 q2, q9, d0[0] + vqrdmulh.s32 q3, q10, d0[0] + vqrdmulh.s32 q4, q11, d0[0] + vqadd.s32 q8, q8, q1 + vqadd.s32 q9, q9, q2 + vqadd.s32 q10, q10, q3 + vqadd.s32 q11, q11, q4 + bx lr +endfunc + +function inv_txfm_add_wht_wht_4x4_16bpc_neon, export=1 + push {r4-r5,lr} + vpush {q4-q5} + vmov.i16 q14, #0 + vmov.i16 q15, #0 + vld1.32 {q8, q9}, [r2, :128] + vst1.32 {q14, q15}, [r2, :128]! + vshr.s16 q8, q8, #2 + vld1.32 {q10, q11}, [r2, :128] + vshr.s16 q9, q9, #2 + vshr.s16 q10, q10, #2 + vshr.s16 q11, q11, #2 + + iwht4 + + vst1.32 {q14, q15}, [r2, :128] + transpose_4x4s q8, q9, q10, q11, d16, d17, d18, d19, d20, d21, d22, d23 + + iwht4 + + vld1.16 {d0}, [r0, :64], r1 + vqmovn.s32 d16, q8 + vld1.16 {d1}, [r0, :64], r1 + vqmovn.s32 d17, q9 + vld1.16 {d2}, [r0, :64], r1 + vqmovn.s32 d18, q10 + vld1.16 {d3}, [r0, :64], r1 + vqmovn.s32 d19, q11 + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + vmov.i16 q14, #0 + vmov.i16 q15, #0 + vld1.32 {q8, q9}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128]! + vld1.32 {q10, q11}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128] + + blx r4 + + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q9 + vqmovn.s32 d18, q10 + vqmovn.s32 d19, q11 + transpose_4x4h q8, q9, d16, d17, d18, d19 + + blx r5 + + vld1.16 {d0}, [r0, :64], r1 + vld1.16 {d1}, [r0, :64], r1 + vrshr.s16 q8, q8, #4 + vld1.16 {d2}, [r0, :64], r1 + vrshr.s16 q9, q9, #4 + vld1.16 {d3}, [r0, :64], r1 + +L(itx_4x4_end): + vmvn.i16 q15, #0xfc00 // 0x3ff + sub r0, r0, r1, lsl #2 + vqadd.s16 q8, q8, q0 + vqadd.s16 q9, q9, q1 + vmax.s16 q8, q8, q14 + vmax.s16 q9, q9, q14 + vmin.s16 q8, q8, q15 + vmin.s16 q9, q9, q15 + vst1.16 {d16}, [r0, :64], r1 + vst1.16 {d17}, [r0, :64], r1 + vst1.16 {d18}, [r0, :64], r1 + vst1.16 {d19}, [r0, :64], r1 + + vpop {q4-q5} + pop {r4-r5,pc} +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_16bpc_neon, export=1 + push {r4-r5,lr} + vpush {q4-q5} + +.ifc \txfm1\()_\txfm2, dct_dct + cmp r3, #0 + bne 1f + vmov.i16 q14, #0 + mov_const r12, 2896*8*(1<<16) + vld1.32 {d16[], d17[]}, [r2, :32] + vdup.32 d4, r12 + vst1.32 {d28[0]}, [r2, :32] + vqrdmulh.s32 q8, q8, d4[0] + vld1.16 {d0}, [r0, :64], r1 + vqmovn.s32 d20, q8 + vqmovn.s32 d21, q8 + vld1.16 {d1}, [r0, :64], r1 + vqrdmulh.s16 q10, q10, d4[1] + vld1.16 {d2}, [r0, :64], r1 + vrshr.s16 q8, q10, #4 + vld1.16 {d3}, [r0, :64], r1 + vrshr.s16 q9, q10, #4 + b L(itx_4x4_end) +1: +.endif + movrel_local r4, inv_\txfm1\()_4s_x4_neon + movrel r5, X(inv_\txfm2\()_4h_x4_neon) + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_4s_x8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_4s_x4 \r0, \r2, \r4, \r6 + + vmov.i32 q5, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 q4, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 +.irp r, \r0, \r2, \r4, \r6 + vmin.s32 \r, \r, q5 +.endr +.irp r, \r0, \r2, \r4, \r6 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmls q2, \r1, \r7, d2[0], d2[1] // -> t4a + vmul_vmla q3, \r1, \r7, d2[1], d2[0] // -> t7a + vmul_vmls q6, \r5, \r3, d3[0], d3[1] // -> t5a + vmul_vmla q7, \r5, \r3, d3[1], d3[0] // -> t6a + vrshr.s32 \r1, q2, #12 // t4a + vrshr.s32 \r7, q3, #12 // t7a + vrshr.s32 \r3, q6, #12 // t5a + vrshr.s32 \r5, q7, #12 // t6a + + vqadd.s32 q2, \r1, \r3 // t4 + vqsub.s32 \r1, \r1, \r3 // t5a + vqadd.s32 q3, \r7, \r5 // t7 + vqsub.s32 \r3, \r7, \r5 // t6a + +.irp r, q2, \r1, q3, \r3 + vmin.s32 \r, \r, q5 +.endr +.irp r, q2, \r1, q3, \r3 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmls q7, \r3, \r1, d0[0], d0[0] // -> t5 + vmul_vmla q6, \r3, \r1, d0[0], d0[0] // -> t6 + vrshr.s32 q7, q7, #12 // t5 + vrshr.s32 q5, q6, #12 // t6 + + vqsub.s32 \r7, \r0, q3 // out7 + vqadd.s32 \r0, \r0, q3 // out0 + vqadd.s32 \r1, \r2, q5 // out1 + vqsub.s32 q6, \r2, q5 // out6 + vqadd.s32 \r2, \r4, q7 // out2 + vqsub.s32 \r5, \r4, q7 // out5 + vqadd.s32 \r3, \r6, q2 // out3 + vqsub.s32 \r4, \r6, q2 // out4 + vmov \r6, q6 // out6 +.endm + +.macro idct_2s_x8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_2s_x4 \r0, \r2, \r4, \r6 + + vmov.i32 d9, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 d8, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 +.irp r, \r0, \r2, \r4, \r6 + vmin.s32 \r, \r, d9 +.endr +.irp r, \r0, \r2, \r4, \r6 + vmax.s32 \r, \r, d8 +.endr + + vmul_vmls d4, \r1, \r7, d2[0], d2[1] // -> t4a + vmul_vmla d5, \r1, \r7, d2[1], d2[0] // -> t7a + vmul_vmls d6, \r5, \r3, d3[0], d3[1] // -> t5a + vmul_vmla d7, \r5, \r3, d3[1], d3[0] // -> t6a + vrshr.s32 \r1, d4, #12 // t4a + vrshr.s32 \r7, d5, #12 // t7a + vrshr.s32 \r3, d6, #12 // t5a + vrshr.s32 \r5, d7, #12 // t6a + + vqadd.s32 d4, \r1, \r3 // t4 + vqsub.s32 \r1, \r1, \r3 // t5a + vqadd.s32 d5, \r7, \r5 // t7 + vqsub.s32 \r3, \r7, \r5 // t6a + +.irp r, d4, \r1, d5, \r3 + vmin.s32 \r, \r, d9 +.endr +.irp r, d4, \r1, d5, \r3 + vmax.s32 \r, \r, d8 +.endr + + vmul_vmls d6, \r3, \r1, d0[0], d0[0] // -> t5 + vmul_vmla d7, \r3, \r1, d0[0], d0[0] // -> t6 + vrshr.s32 d6, d6, #12 // t5 + vrshr.s32 d7, d7, #12 // t6 + + vqsub.s32 \r7, \r0, d5 // out7 + vqadd.s32 \r0, \r0, d5 // out0 + vqadd.s32 \r1, \r2, d7 // out1 + vqsub.s32 d7, \r2, d7 // out6 + vqadd.s32 \r2, \r4, d6 // out2 + vqsub.s32 \r5, \r4, d6 // out5 + vqadd.s32 \r3, \r6, d4 // out3 + vqsub.s32 \r4, \r6, d4 // out4 + vmov \r6, d7 // out6 +.endm + +function inv_dct_4s_x8_neon + movrel_local r12, idct_coeffs + vld1.32 {q0, q1}, [r12, :128] + idct_4s_x8 q8, q9, q10, q11, q12, q13, q14, q15 + bx lr +endfunc + +.macro iadst_4s_x8 r0, r1, r2, r3, r4, r5, r6, r7 + movrel_local r12, iadst8_coeffs + vld1.32 {q0, q1}, [r12, :128]! + + vmul_vmla q2, q15, q8, d0[0], d0[1] + vmul_vmls q3, q15, q8, d0[1], d0[0] + vmul_vmla q4, q13, q10, d1[0], d1[1] + vrshr.s32 q8, q2, #12 // t0a + vrshr.s32 q15, q3, #12 // t1a + vmul_vmls q5, q13, q10, d1[1], d1[0] + vmul_vmla q6, q11, q12, d2[0], d2[1] + vrshr.s32 q10, q4, #12 // t2a + vrshr.s32 q13, q5, #12 // t3a + vmul_vmls q7, q11, q12, d2[1], d2[0] + vmul_vmla q2, q9, q14, d3[0], d3[1] + vrshr.s32 q12, q6, #12 // t4a + vrshr.s32 q11, q7, #12 // t5a + vmul_vmls q3, q9, q14, d3[1], d3[0] + vrshr.s32 q14, q2, #12 // t6a + vrshr.s32 q9, q3, #12 // t7a + + vld1.32 {q0}, [r12] + + vqadd.s32 q2, q8, q12 // t0 + vqsub.s32 q3, q8, q12 // t4 + vmov.i32 q12, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vqadd.s32 q4, q15, q11 // t1 + vqsub.s32 q5, q15, q11 // t5 + vqadd.s32 q6, q10, q14 // t2 + vqsub.s32 q7, q10, q14 // t6 + vmvn.i32 q14, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 + vqadd.s32 q10, q13, q9 // t3 + vqsub.s32 q11, q13, q9 // t7 + +.irp r, q2, q3, q4, q5, q6, q7, q10, q11 + vmin.s32 \r, \r, q12 +.endr +.irp r, q2, q3, q4, q5, q6, q7, q10, q11 + vmax.s32 \r, \r, q14 +.endr + + vmul_vmla q8, q3, q5, d1[1], d1[0] + vmul_vmls q13, q3, q5, d1[0], d1[1] + vmul_vmls q14, q11, q7, d1[1], d1[0] + + vrshr.s32 q3, q8, #12 // t4a + vrshr.s32 q5, q13, #12 // t5a + + vmul_vmla q8, q11, q7, d1[0], d1[1] + + vrshr.s32 q7, q14, #12 // t6a + vrshr.s32 q11, q8, #12 // t7a + + vqadd.s32 \r0, q2, q6 // out0 + vqsub.s32 q2, q2, q6 // t2 + vqadd.s32 \r7, q4, q10 // out7 + vqsub.s32 q4, q4, q10 // t3 + + vmvn.i32 q10, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 + + vqadd.s32 \r1, q3, q7 // out1 + vqsub.s32 q3, q3, q7 // t6 + vqadd.s32 \r6, q5, q11 // out6 + vqsub.s32 q5, q5, q11 // t7 + + // Not clipping the output registers, as they will be downshifted and + // narrowed afterwards anyway. +.irp r, q2, q4, q3, q5 + vmin.s32 \r, \r, q12 +.endr +.irp r, q2, q4, q3, q5 + vmax.s32 \r, \r, q10 +.endr + + vqneg.s32 \r7, \r7 // out7 + vqneg.s32 \r1, \r1 // out1 + + vmul_vmla q10, q2, q4, d0[0], d0[0] // -> out3 (q11 or q12) + vmul_vmls q6, q2, q4, d0[0], d0[0] // -> out4 (q12 or q11) + vmul_vmls q12, q3, q5, d0[0], d0[0] // -> out5 (q13 or q10) + vrshr.s32 q2, q10, #12 // out3 + vmul_vmla q10, q3, q5, d0[0], d0[0] // -> out2 (q10 or q13) + vrshr.s32 q3, q12, #12 // out5 + vrshr.s32 \r2, q10, #12 // out2 (q10 or q13) + vrshr.s32 \r4, q6, #12 // out4 (q12 or q11) + + vqneg.s32 \r3, q2 // out3 + vqneg.s32 \r5, q3 // out5 +.endm + +function inv_adst_4s_x8_neon + iadst_4s_x8 q8, q9, q10, q11, q12, q13, q14, q15 + bx lr +endfunc + +function inv_flipadst_4s_x8_neon + iadst_4s_x8 q15, q14, q13, q12, q11, q10, q9, q8 + bx lr +endfunc + +function inv_identity_4s_x8_neon + vqshl.s32 q8, q8, #1 + vqshl.s32 q9, q9, #1 + vqshl.s32 q10, q10, #1 + vqshl.s32 q11, q11, #1 + vqshl.s32 q12, q12, #1 + vqshl.s32 q13, q13, #1 + vqshl.s32 q14, q14, #1 + vqshl.s32 q15, q15, #1 + bx lr +endfunc + +function inv_txfm_add_8x8_neon + vmov.i32 q0, #0 + mov r7, #8*4 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r7 +.endr + + blx r4 + + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q12, #1 + vqrshrn.s32 d18, q9, #1 + vqrshrn.s32 d19, q13, #1 + vqrshrn.s32 d20, q10, #1 + vqrshrn.s32 d21, q14, #1 + vqrshrn.s32 d22, q11, #1 + vqrshrn.s32 d23, q15, #1 + + cmp r3, r10 + transpose_4x8h q8, q9, q10, q11 + + blt 1f + + sub r2, r2, r7, lsl #3 + vpush {q8-q11} + + add r2, r2, #16 + vmov.i32 q0, #0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r7 +.endr + + blx r4 + + vqrshrn.s32 d31, q15, #1 + vqrshrn.s32 d30, q11, #1 + vqrshrn.s32 d29, q14, #1 + vqrshrn.s32 d28, q10, #1 + vqrshrn.s32 d27, q13, #1 + vqrshrn.s32 d26, q9, #1 + vqrshrn.s32 d25, q12, #1 + vqrshrn.s32 d24, q8, #1 + vpop {q8-q11} + + transpose_4x8h q12, q13, q14, q15 + + b 2f + +1: + vmov.i16 q12, #0 + vmov.i16 q13, #0 + vmov.i16 q14, #0 + vmov.i16 q15, #0 + +2: + blx r5 + + load_add_store_8x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,r10,pc} +endfunc + +.macro def_fn_8x8 txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + push {r4-r5,r7,r10,lr} + vpush {q4-q7} + mov r10, #\eob_half + movrel_local r4, inv_\txfm1\()_4s_x8_neon + movrel r5, X(inv_\txfm2\()_8h_x8_neon) + b inv_txfm_add_8x8_neon +endfunc +.endm + +def_fn_8x8 dct, dct, 10 +def_fn_8x8 identity, identity, 10 +def_fn_8x8 dct, adst, 10 +def_fn_8x8 dct, flipadst, 10 +def_fn_8x8 dct, identity, 4 +def_fn_8x8 adst, dct, 10 +def_fn_8x8 adst, adst, 10 +def_fn_8x8 adst, flipadst, 10 +def_fn_8x8 flipadst, dct, 10 +def_fn_8x8 flipadst, adst, 10 +def_fn_8x8 flipadst, flipadst, 10 +def_fn_8x8 identity, dct, 4 +def_fn_8x8 adst, identity, 4 +def_fn_8x8 flipadst, identity, 4 +def_fn_8x8 identity, adst, 4 +def_fn_8x8 identity, flipadst, 4 + +function inv_txfm_add_8x4_neon + mov_const r12, 2896*8*(1<<16) + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vld1.16 {q8, q9}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vdup.32 d4, r12 + vld1.16 {q10, q11}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q12, q13}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q14, q15}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + + scale_input d4[0], q8, q9, q10, q11, q12, q13, q14, q15 + + blx r4 + + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q9 + vqmovn.s32 d18, q10 + vqmovn.s32 d19, q11 + vqmovn.s32 d20, q12 + vqmovn.s32 d21, q13 + vqmovn.s32 d22, q14 + vqmovn.s32 d23, q15 + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + vswp d17, d20 + vswp d19, d21 + vswp d18, d20 + vswp d21, d22 + + blx r5 + + load_add_store_8x4 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,r10,pc} +endfunc + +function inv_txfm_add_4x8_neon + mov_const r12, 2896*8*(1<<16) + vmov.i32 q0, #0 + cmp r3, r10 + mov r7, #32 + blt 1f + + add r2, r2, #16 + vdup.32 d2, r12 +.irp i, q8, q9, q10, q11 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r7 +.endr + + scale_input d2[0], q8, q9, q10, q11 + sub r2, r2, r7, lsl #2 + + blx r4 + + sub r2, r2, #16 + + vqmovn.s32 d24, q8 + vqmovn.s32 d25, q9 + vqmovn.s32 d26, q10 + vqmovn.s32 d27, q11 + transpose_4x4h q12, q13, d24, d25, d26, d27 + + b 2f + +1: + vmov.i16 q12, #0 + vmov.i16 q13, #0 + +2: + mov_const r12, 2896*8*(1<<16) + vmov.i32 q0, #0 + vdup.32 d2, r12 +.irp i, q8, q9, q10, q11 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r7 +.endr + scale_input d2[0], q8, q9, q10, q11 + blx r4 + + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q9 + vqmovn.s32 d18, q10 + vqmovn.s32 d19, q11 + transpose_4x4h q8, q9, d16, d17, d18, d19 + + vmov q10, q12 + vmov q11, q13 + + blx r5 + + load_add_store_4x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,r10,pc} +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + push {r4-r5,r7,r10,lr} + vpush {q4-q7} + movrel_local r4, inv_\txfm1\()_4s_x\w\()_neon +.if \w == 4 + mov r10, #\eob_half +.endif + movrel r5, X(inv_\txfm2\()_\w\()h_x\h\()_neon) + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct, 13 +def_fn_48 \w, \h, identity, identity, 13 +def_fn_48 \w, \h, dct, adst, 13 +def_fn_48 \w, \h, dct, flipadst, 13 +def_fn_48 \w, \h, dct, identity, 4 +def_fn_48 \w, \h, adst, dct, 13 +def_fn_48 \w, \h, adst, adst, 13 +def_fn_48 \w, \h, adst, flipadst, 13 +def_fn_48 \w, \h, flipadst, dct, 13 +def_fn_48 \w, \h, flipadst, adst, 13 +def_fn_48 \w, \h, flipadst, flipadst, 13 +def_fn_48 \w, \h, identity, dct, 16 +def_fn_48 \w, \h, adst, identity, 4 +def_fn_48 \w, \h, flipadst, identity, 4 +def_fn_48 \w, \h, identity, adst, 16 +def_fn_48 \w, \h, identity, flipadst, 16 +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + +function inv_dct_2s_x16_neon + movrel_local r12, idct_coeffs + vld1.32 {q0, q1}, [r12, :128]! + + idct_2s_x8 d16, d18, d20, d22, d24, d26, d28, d30 + + // idct_8 leaves the row_clip_max/min constants in d9 and d8 +.irp r, d16, d18, d20, d22, d24, d26, d28, d30 + vmin.s32 \r, \r, d9 +.endr +.irp r, d16, d18, d20, d22, d24, d26, d28, d30 + vmax.s32 \r, \r, d8 +.endr + + vld1.32 {q0, q1}, [r12, :128] + sub r12, r12, #32 + + vmul_vmls d4, d17, d31, d0[0], d0[1] // -> t8a + vmul_vmla d5, d17, d31, d0[1], d0[0] // -> t15a + vmul_vmls d6, d25, d23, d1[0], d1[1] // -> t9a + vrshr.s32 d17, d4, #12 // t8a + vrshr.s32 d31, d5, #12 // t15a + vmul_vmla d4, d25, d23, d1[1], d1[0] // -> t14a + vmul_vmls d5, d21, d27, d2[0], d2[1] // -> t10a + vrshr.s32 d23, d6, #12 // t9a + vrshr.s32 d25, d4, #12 // t14a + vmul_vmla d6, d21, d27, d2[1], d2[0] // -> t13a + vmul_vmls d4, d29, d19, d3[0], d3[1] // -> t11a + vrshr.s32 d21, d5, #12 // t10a + vrshr.s32 d27, d6, #12 // t13a + vmul_vmla d5, d29, d19, d3[1], d3[0] // -> t12a + vrshr.s32 d19, d4, #12 // t11a + vrshr.s32 d29, d5, #12 // t12a + + vld1.32 {q0}, [r12, :128] + + vqsub.s32 d4, d17, d23 // t9 + vqadd.s32 d17, d17, d23 // t8 + vqsub.s32 d5, d31, d25 // t14 + vqadd.s32 d31, d31, d25 // t15 + vqsub.s32 d23, d19, d21 // t10 + vqadd.s32 d19, d19, d21 // t11 + vqadd.s32 d25, d29, d27 // t12 + vqsub.s32 d29, d29, d27 // t13 + +.irp r, d4, d17, d5, d31, d23, d19, d25, d29 + vmin.s32 \r, \r, d9 +.endr +.irp r, d4, d17, d5, d31, d23, d19, d25, d29 + vmax.s32 \r, \r, d8 +.endr + + vmul_vmls d6, d5, d4, d1[0], d1[1] // -> t9a + vmul_vmla d7, d5, d4, d1[1], d1[0] // -> t14a + vrshr.s32 d21, d6, #12 // t9a + vrshr.s32 d27, d7, #12 // t14a + + vmul_vmls d6, d29, d23, d1[0], d1[1] // -> t13a + vmul_vmla d7, d29, d23, d1[1], d1[0] // -> t10a + vrshr.s32 d29, d6, #12 // t13a + vneg.s32 d7, d7 + vrshr.s32 d23, d7, #12 // t10a + + vqsub.s32 d4, d17, d19 // t11a + vqadd.s32 d17, d17, d19 // t8a + vqsub.s32 d5, d31, d25 // t12a + vqadd.s32 d31, d31, d25 // t15a + vqadd.s32 d19, d21, d23 // t9 + vqsub.s32 d21, d21, d23 // t10 + vqsub.s32 d25, d27, d29 // t13 + vqadd.s32 d27, d27, d29 // t14 + +.irp r, d4, d17, d5, d31, d19, d21, d25, d27 + vmin.s32 \r, \r, d9 +.endr +.irp r, d4, d17, d5, d31, d19, d21, d25, d27 + vmax.s32 \r, \r, d8 +.endr + + vmul_vmls d6, d5, d4, d0[0], d0[0] // -> t11 + vmul_vmla d7, d5, d4, d0[0], d0[0] // -> t12 + vmul_vmls d4, d25, d21, d0[0], d0[0] // -> t10a + + vrshr.s32 d6, d6, #12 // t11 + vrshr.s32 d7, d7, #12 // t12 + vmul_vmla d5, d25, d21, d0[0], d0[0] // -> t13a + vrshr.s32 d4, d4, #12 // t10a + vrshr.s32 d5, d5, #12 // t13a + + vqadd.s32 d8, d16, d31 // out0 + vqsub.s32 d31, d16, d31 // out15 + vmov d16, d8 + vqadd.s32 d23, d30, d17 // out7 + vqsub.s32 d9, d30, d17 // out8 + vqadd.s32 d17, d18, d27 // out1 + vqsub.s32 d30, d18, d27 // out14 + vqadd.s32 d18, d20, d5 // out2 + vqsub.s32 d29, d20, d5 // out13 + vqadd.s32 d5, d28, d19 // out6 + vqsub.s32 d25, d28, d19 // out9 + vqadd.s32 d19, d22, d7 // out3 + vqsub.s32 d28, d22, d7 // out12 + vqadd.s32 d20, d24, d6 // out4 + vqsub.s32 d27, d24, d6 // out11 + vqadd.s32 d21, d26, d4 // out5 + vqsub.s32 d26, d26, d4 // out10 + vmov d24, d9 + vmov d22, d5 + + bx lr +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15 + movrel_local r12, iadst16_coeffs + vld1.32 {q0, q1}, [r12, :128]! + + vmul_vmla d4, d31, d16, d0[0], d0[1] // -> t0 + vmul_vmls d6, d31, d16, d0[1], d0[0] // -> t1 + vmul_vmla d8, d29, d18, d1[0], d1[1] // -> t2 + vrshr.s32 d16, d4, #12 // t0 + vrshr.s32 d31, d6, #12 // t1 + vmul_vmls d4, d29, d18, d1[1], d1[0] // -> t3 + vmul_vmla d6, d27, d20, d2[0], d2[1] // -> t4 + vrshr.s32 d18, d8, #12 // t2 + vrshr.s32 d29, d4, #12 // t3 + vmul_vmls d8, d27, d20, d2[1], d2[0] // -> t5 + vmul_vmla d4, d25, d22, d3[0], d3[1] // -> t6 + vrshr.s32 d20, d6, #12 // t4 + vrshr.s32 d27, d8, #12 // t5 + vmul_vmls d6, d25, d22, d3[1], d3[0] // -> t7 + vld1.32 {q0, q1}, [r12, :128] + movrel_local r12, idct_coeffs + vmul_vmla d8, d23, d24, d0[0], d0[1] // -> t8 + vrshr.s32 d22, d4, #12 // t6 + vrshr.s32 d25, d6, #12 // t7 + vmul_vmls d4, d23, d24, d0[1], d0[0] // -> t9 + vmul_vmla d6, d21, d26, d1[0], d1[1] // -> t10 + vrshr.s32 d23, d8, #12 // t8 + vrshr.s32 d24, d4, #12 // t9 + vmul_vmls d8, d21, d26, d1[1], d1[0] // -> t11 + vmul_vmla d4, d19, d28, d2[0], d2[1] // -> t12 + vrshr.s32 d21, d6, #12 // t10 + vrshr.s32 d26, d8, #12 // t11 + vmul_vmls d6, d19, d28, d2[1], d2[0] // -> t13 + vmul_vmla d8, d17, d30, d3[0], d3[1] // -> t14 + vrshr.s32 d19, d4, #12 // t12 + vrshr.s32 d28, d6, #12 // t13 + vmul_vmls d4, d17, d30, d3[1], d3[0] // -> t15 + vrshr.s32 d17, d8, #12 // t14 + vrshr.s32 d30, d4, #12 // t15 + + vld1.32 {q0, q1}, [r12, :128] + + vmov.i32 d11, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 d10, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 + + vqsub.s32 d5, d16, d23 // t8a + vqadd.s32 d16, d16, d23 // t0a + vqsub.s32 d7, d31, d24 // t9a + vqadd.s32 d31, d31, d24 // t1a + vqadd.s32 d23, d18, d21 // t2a + vqsub.s32 d18, d18, d21 // t10a + vqadd.s32 d24, d29, d26 // t3a + vqsub.s32 d29, d29, d26 // t11a + vqadd.s32 d21, d20, d19 // t4a + vqsub.s32 d20, d20, d19 // t12a + vqadd.s32 d26, d27, d28 // t5a + vqsub.s32 d27, d27, d28 // t13a + vqadd.s32 d19, d22, d17 // t6a + vqsub.s32 d22, d22, d17 // t14a + vqadd.s32 d28, d25, d30 // t7a + vqsub.s32 d25, d25, d30 // t15a + +.irp r, d5, d16, d7, d31, d23, d18, d24, d29, d21, d20, d26, d27, d19, d22, d28, d25 + vmin.s32 \r, \r, d11 +.endr +.irp r, d5, d16, d7, d31, d23, d18, d24, d29, d21, d20, d26, d27, d19, d22, d28, d25 + vmax.s32 \r, \r, d10 +.endr + + vmul_vmla d4, d5, d7, d2[1], d2[0] // -> t8 + vmul_vmls d6, d5, d7, d2[0], d2[1] // -> t9 + vmul_vmla d8, d18, d29, d3[1], d3[0] // -> t10 + vrshr.s32 d17, d4, #12 // t8 + vrshr.s32 d30, d6, #12 // t9 + vmul_vmls d4, d18, d29, d3[0], d3[1] // -> t11 + vmul_vmls d6, d27, d20, d2[1], d2[0] // -> t12 + vrshr.s32 d18, d8, #12 // t10 + vrshr.s32 d29, d4, #12 // t11 + vmul_vmla d8, d27, d20, d2[0], d2[1] // -> t13 + vmul_vmls d4, d25, d22, d3[1], d3[0] // -> t14 + vrshr.s32 d27, d6, #12 // t12 + vrshr.s32 d20, d8, #12 // t13 + vmul_vmla d6, d25, d22, d3[0], d3[1] // -> t15 + vrshr.s32 d25, d4, #12 // t14 + vrshr.s32 d22, d6, #12 // t15 + + vqsub.s32 d2, d16, d21 // t4 + vqadd.s32 d16, d16, d21 // t0 + vqsub.s32 d3, d31, d26 // t5 + vqadd.s32 d31, d31, d26 // t1 + vqadd.s32 d21, d23, d19 // t2 + vqsub.s32 d23, d23, d19 // t6 + vqadd.s32 d26, d24, d28 // t3 + vqsub.s32 d24, d24, d28 // t7 + vqadd.s32 d19, d17, d27 // t8a + vqsub.s32 d17, d17, d27 // t12a + vqadd.s32 d28, d30, d20 // t9a + vqsub.s32 d30, d30, d20 // t13a + vqadd.s32 d27, d18, d25 // t10a + vqsub.s32 d18, d18, d25 // t14a + vqadd.s32 d20, d29, d22 // t11a + vqsub.s32 d29, d29, d22 // t15a + +.irp r, d2, d16, d3, d31, d21, d23, d26, d24, d19, d17, d28, d30, d27, d18, d20, d29 + vmin.s32 \r, \r, d11 +.endr +.irp r, d2, d16, d3, d31, d21, d23, d26, d24, d19, d17, d28, d30, d27, d18, d20, d29 + vmax.s32 \r, \r, d10 +.endr + + vmul_vmla d4, d2, d3, d1[1], d1[0] // -> t4a + vmul_vmls d6, d2, d3, d1[0], d1[1] // -> t5a + vmul_vmls d8, d24, d23, d1[1], d1[0] // -> t6a + vrshr.s32 d22, d4, #12 // t4a + vrshr.s32 d25, d6, #12 // t5a + vmul_vmla d4, d24, d23, d1[0], d1[1] // -> t7a + vmul_vmla d6, d17, d30, d1[1], d1[0] // -> t12 + vrshr.s32 d24, d8, #12 // t6a + vrshr.s32 d23, d4, #12 // t7a + vmul_vmls d8, d17, d30, d1[0], d1[1] // -> t13 + vmul_vmls d4, d29, d18, d1[1], d1[0] // -> t14 + vrshr.s32 d17, d6, #12 // t12 + vmul_vmla d6, d29, d18, d1[0], d1[1] // -> t15 + vrshr.s32 d29, d8, #12 // t13 + vrshr.s32 d30, d4, #12 // t14 + vrshr.s32 d18, d6, #12 // t15 + + vqsub.s32 d2, d16, d21 // t2a +.ifc \o0, d16 + vqadd.s32 \o0, d16, d21 // out0 + vqsub.s32 d21, d31, d26 // t3a + vqadd.s32 \o15,d31, d26 // out15 +.else + vqadd.s32 d4, d16, d21 // out0 + vqsub.s32 d21, d31, d26 // t3a + vqadd.s32 \o15,d31, d26 // out15 + vmov \o0, d4 +.endif + + vqsub.s32 d3, d29, d18 // t15a + vqadd.s32 \o13,d29, d18 // out13 + vqadd.s32 \o2, d17, d30 // out2 + vqsub.s32 d26, d17, d30 // t14a + + vqadd.s32 \o1, d19, d27 // out1 + vqsub.s32 d27, d19, d27 // t10 + vqadd.s32 \o14,d28, d20 // out14 + vqsub.s32 d20, d28, d20 // t11 + + vqadd.s32 \o3, d22, d24 // out3 + vqsub.s32 d22, d22, d24 // t6 + vqadd.s32 \o12,d25, d23 // out12 + vqsub.s32 d23, d25, d23 // t7 + + // Not clipping the output registers, as they will be downshifted and + // narrowed afterwards anyway. +.irp r, d2, d21, d3, d26, d27, d20, d22, d23 + vmin.s32 \r, \r, d11 +.endr +.irp r, d2, d21, d3, d26, d27, d20, d22, d23 + vmax.s32 \r, \r, d10 +.endr + + vqneg.s32 \o15, \o15 // out15 + vqneg.s32 \o13,\o13 // out13 + vqneg.s32 \o1, \o1 // out1 + vqneg.s32 \o3, \o3 // out3 + + vmul_vmls d24, d2, d21, d0[0], d0[0] // -> out8 (d24 or d23) + vmul_vmla d4, d2, d21, d0[0], d0[0] // -> out7 (d23 or d24) + vmul_vmla d6, d26, d3, d0[0], d0[0] // -> out5 (d21 or d26) + + vrshr.s32 d24, d24, #12 // out8 + vrshr.s32 d4, d4, #12 // out7 + vrshr.s32 d5, d6, #12 // out5 + vmul_vmls d8, d26, d3, d0[0], d0[0] // -> out10 (d26 or d21) + vmul_vmla d2, d22, d23, d0[0], d0[0] // -> out4 (d20 or d27) + vrshr.s32 d26, d8, #12 // out10 + + vmul_vmls d8, d22, d23, d0[0], d0[0] // -> out11 (d27 or d20) + vmul_vmla d22, d27, d20, d0[0], d0[0] // -> out6 (d22 or d25) + vmul_vmls d6, d27, d20, d0[0], d0[0] // -> out9 (d25 or d22) + + vrshr.s32 \o4, d2, #12 // out4 + vrshr.s32 d7, d6, #12 // out9 + vrshr.s32 d6, d8, #12 // out11 + vrshr.s32 \o6, d22, #12 // out6 + +.ifc \o8, d23 + vmov \o8, d24 + vmov \o10,d26 +.endif + + vqneg.s32 \o7, d4 // out7 + vqneg.s32 \o5, d5 // out5 + vqneg.s32 \o11,d6 // out11 + vqneg.s32 \o9, d7 // out9 +.endm + +function inv_adst_2s_x16_neon + iadst_16 d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_flipadst_2s_x16_neon + iadst_16 d31, d30, d29, d28, d27, d26, d25, d24, d23, d22, d21, d20, d19, d18, d17, d16 + bx lr +endfunc + +function inv_identity_2s_x16_neon + mov r12, #0 + movt r12, #2*(5793-4096)*8 + vdup.32 d0, r12 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s32 q1, \i, d0[0] + vqadd.s32 \i, \i, \i + vqadd.s32 \i, \i, q1 +.endr + bx lr +endfunc + +.macro identity_8x4_shift1 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s32 q2, \i, \c + vrshr.s32 q2, q2, #1 + vqadd.s32 \i, \i, q2 +.endr +.endm + +.macro identity_8x4 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s32 q2, \i, \c + vqadd.s32 \i, \i, \i + vqadd.s32 \i, \i, q2 +.endr +.endm + +.macro def_horz_16 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x2_neon + push {lr} + vmov.i32 d7, #0 +.if \scale + mov_const r12, 2896*8*(1<<16) + vdup.32 d1, r12 +.endif +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.32 {\i}, [r7, :64] + vst1.32 {d7}, [r7, :64], r8 +.endr +.if \scale + scale_input d1[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + blx r4 + vqrshrn.s32 d16, q8, #\shift + vqrshrn.s32 d17, q9, #\shift + vqrshrn.s32 d18, q10, #\shift + vqrshrn.s32 d19, q11, #\shift + vqrshrn.s32 d20, q12, #\shift + vqrshrn.s32 d21, q13, #\shift + vqrshrn.s32 d22, q14, #\shift + vqrshrn.s32 d23, q15, #\shift + vuzp.16 q8, q9 + vuzp.16 q10, q11 + +.irp i, q8, q10, q9, q11 + vst1.16 {\i}, [r6, :128]! +.endr + + pop {pc} +endfunc +.endm + +def_horz_16 scale=0, shift=2 +def_horz_16 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_4x16_neon + push {lr} +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + blx r5 + load_add_store_4x16 r6, r7 + pop {pc} +endfunc + +function inv_txfm_add_16x16_neon + sub_sp_align 512 + ldrh r11, [r10], #2 +.irp i, 0, 2, 4, 6, 8, 10, 12, 14 + add r6, sp, #(\i*16*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 14 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #16*4 + bl inv_txfm_horz_16x2_neon +.endr + b 3f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #32 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +const eob_16x16 + .short 3, 10, 21, 36, 55, 78, 105, 256 +endconst + +const eob_16x16_identity + .short 2, 4, 6, 8, 10, 12, 14, 256 +endconst + +.macro def_fn_16x16 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif + push {r4-r11,lr} + vpush {q4-q7} + movrel_local r4, inv_\txfm1\()_2s_x16_neon + movrel r5, X(inv_\txfm2\()_4h_x16_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel_local r10, eob_16x16 +.else + movrel_local r10, eob_16x16_identity +.endif +.else +.ifc \txfm2, identity + movrel_local r10, eob_16x16_identity +.else + movrel_local r10, eob_16x16 +.endif +.endif + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct +def_fn_16x16 identity, identity +def_fn_16x16 dct, adst +def_fn_16x16 dct, flipadst +def_fn_16x16 dct, identity +def_fn_16x16 adst, dct +def_fn_16x16 adst, adst +def_fn_16x16 adst, flipadst +def_fn_16x16 flipadst, dct +def_fn_16x16 flipadst, adst +def_fn_16x16 flipadst, flipadst +def_fn_16x16 identity, dct + +function inv_txfm_add_16x4_neon + cmp r3, r10 + mov r11, #16 + blt 1f + + add r6, r2, #8 + vmov.i32 d4, #0 +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.32 {\i}, [r6, :64] + vst1.32 {d4}, [r6, :64], r11 +.endr + blx r4 + + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q9, #1 + vqrshrn.s32 d18, q10, #1 + vqrshrn.s32 d19, q11, #1 + vqrshrn.s32 d20, q12, #1 + vqrshrn.s32 d21, q13, #1 + vqrshrn.s32 d22, q14, #1 + vqrshrn.s32 d23, q15, #1 + vuzp.16 q8, q9 + mov r6, sp + vuzp.16 q10, q11 + vpush {q8-q11} + + b 2f + +1: + vmov.i16 q8, #0 + vmov.i16 q9, #0 + mov r6, sp + vpush {q8-q9} + vpush {q8-q9} + +2: + vmov.i32 d4, #0 +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.32 {\i}, [r2, :64] + vst1.32 {d4}, [r2, :64], r11 +.endr + + blx r4 + + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q9, #1 + vqrshrn.s32 d18, q10, #1 + vqrshrn.s32 d19, q11, #1 + vqrshrn.s32 d20, q12, #1 + vqrshrn.s32 d21, q13, #1 + vqrshrn.s32 d22, q14, #1 + vqrshrn.s32 d23, q15, #1 + vuzp.16 q8, q9 + mov r6, sp + vuzp.16 q10, q11 + + vmov q12, q10 + vmov q13, q11 + + vpop {q10-q11} + blx r5 + mov r6, r0 + load_add_store_8x4 r6, r7 + + vpop {q10-q11} + vmov q8, q12 + vmov q9, q13 + blx r5 + add r6, r0, #16 + load_add_store_8x4 r6, r7 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_4x16_neon + ldrh r9, [r10, #4] + + mov r11, #64 + cmp r3, r9 + ldrh r9, [r10, #2] + blt 1f + + add r6, r2, #48 + vmov.i32 q2, #0 +.irp i, q8, q9, q10, q11 + vld1.32 {\i}, [r6, :128] + vst1.32 {q2}, [r6, :128], r11 +.endr + blx r4 + vqrshrn.s32 d28, q8, #1 + vqrshrn.s32 d29, q9, #1 + vqrshrn.s32 d30, q10, #1 + vqrshrn.s32 d31, q11, #1 + transpose_4x4h q14, q15, d28, d29, d30, d31 + + b 2f +1: + vmov.i16 q14, #0 + vmov.i16 q15, #0 +2: + cmp r3, r9 + ldrh r9, [r10] + blt 1f + + add r6, r2, #32 + vmov.i32 q2, #0 +.irp i, q8, q9, q10, q11 + vld1.32 {\i}, [r6, :128] + vst1.32 {q2}, [r6, :128], r11 +.endr + blx r4 + vqrshrn.s32 d24, q8, #1 + vqrshrn.s32 d25, q9, #1 + vqrshrn.s32 d26, q10, #1 + vqrshrn.s32 d27, q11, #1 + transpose_4x4h q12, q13, d24, d25, d26, d27 + + b 2f +1: + vmov.i16 q12, #0 + vmov.i16 q13, #0 +2: + cmp r3, r9 + blt 1f + + add r6, r2, #16 + vmov.i32 q2, #0 +.irp i, q8, q9, q10, q11 + vld1.32 {\i}, [r6, :128] + vst1.32 {q2}, [r6, :128], r11 +.endr + blx r4 + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q9, #1 + vqrshrn.s32 d18, q10, #1 + vqrshrn.s32 d19, q11, #1 + transpose_4x4h q8, q9, d16, d17, d18, d19 + + b 2f +1: + vmov.i16 q8, #0 + vmov.i16 q9, #0 +2: + vmov.i16 q2, #0 + vpush {q8-q9} +.irp i, q8, q9, q10, q11 + vld1.16 {\i}, [r2, :128] + vst1.16 {q2}, [r2, :128], r11 +.endr + blx r4 + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q9, #1 + vqrshrn.s32 d18, q10, #1 + vqrshrn.s32 d19, q11, #1 + transpose_4x4h q8, q9, d16, d17, d18, d19 + vpop {q10-q11} + + blx r5 + + load_add_store_4x16 r0, r6 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +const eob_4x16 + .short 13, 29, 45, 64 +endconst + +const eob_4x16_identity1 + .short 16, 32, 48, 64 +endconst + +const eob_4x16_identity2 + .short 4, 8, 12, 64 +endconst + +.macro def_fn_416 w, h, txfm1, txfm2, eob_16x4 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 4 + movrel_local r4, inv_\txfm1\()_4s_x\w\()_neon + movrel r5, X(inv_\txfm2\()_4h_x\h\()_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel_local r10, eob_4x16 +.else + movrel_local r10, eob_4x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel_local r10, eob_4x16_identity2 +.else + movrel_local r10, eob_4x16 +.endif +.endif +.else + mov r10, #\eob_16x4 + movrel_local r4, inv_\txfm1\()_2s_x\w\()_neon + movrel r5, X(inv_\txfm2\()_8h_x\h\()_neon) +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct, 3 +def_fn_416 \w, \h, identity, identity, 3 +def_fn_416 \w, \h, dct, adst, 3 +def_fn_416 \w, \h, dct, flipadst, 3 +def_fn_416 \w, \h, dct, identity, 2 +def_fn_416 \w, \h, adst, dct, 3 +def_fn_416 \w, \h, adst, adst, 3 +def_fn_416 \w, \h, adst, flipadst, 3 +def_fn_416 \w, \h, flipadst, dct, 3 +def_fn_416 \w, \h, flipadst, adst, 3 +def_fn_416 \w, \h, flipadst, flipadst, 3 +def_fn_416 \w, \h, identity, dct, 2 +def_fn_416 \w, \h, adst, identity, 2 +def_fn_416 \w, \h, flipadst, identity, 2 +def_fn_416 \w, \h, identity, adst, 2 +def_fn_416 \w, \h, identity, flipadst, 2 +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + +function inv_txfm_add_16x8_neon + sub_sp_align 256 + ldrh r11, [r10], #2 + +.irp i, 0, 2, 4, 6 + add r6, sp, #(\i*16*2) +.if \i > 0 + mov r8, #(8 - \i) + cmp r3, r11 + blt 1f +.if \i < 6 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #8*4 + bl inv_txfm_horz_scale_16x2_neon +.endr + b 3f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b +3: + +.irp i, 0, 8 + add r7, sp, #(\i*2) + mov r8, #32 +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\j}, [r7, :128], r8 +.endr + blx r5 + + add r6, r0, #(\i*2) + load_add_store_8x8 r6, r7 +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_8x16_neon + add r10, r10, #2 + sub_sp_align 256 + ldrh r11, [r10], #4 + +.irp i, 0, 4, 8, 12 + add r6, sp, #(\i*8*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 12 + ldrh r11, [r10], #4 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #16*4 + + mov_const r12, 2896*8*(1<<16) + vmov.i32 q2, #0 + vdup.32 d0, r12 + +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\j}, [r7, :128] + vst1.32 {q2}, [r7, :128], r8 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 + blx r4 + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q9, #1 + vqrshrn.s32 d18, q10, #1 + vqrshrn.s32 d19, q11, #1 + vqrshrn.s32 d20, q12, #1 + vqrshrn.s32 d21, q13, #1 + vqrshrn.s32 d22, q14, #1 + vqrshrn.s32 d23, q15, #1 + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 +.irp j, d16, d20, d17, d21, d18, d22, d19, d23 + vst1.16 {\j}, [r6, :64]! +.endr +.endr + b 3f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b +3: + +.irp i, 0, 4 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #16 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +const eob_8x16 + .short 3, 10, 21, 43, 59, 75, 91, 128 +endconst + +const eob_8x16_identity1 + .short 2, 4, 6, 64, 80, 96, 112, 128 +endconst + +const eob_8x16_identity2 + .short 2, 4, 6, 8, 10, 12, 14, 128 +endconst + +.macro def_fn_816 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 8 + movrel_local r4, inv_\txfm1\()_4s_x8_neon + movrel r5, X(inv_\txfm2\()_4h_x16_neon) +.else + movrel_local r4, inv_\txfm1\()_2s_x16_neon + movrel r5, X(inv_\txfm2\()_8h_x8_neon) +.endif +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel_local r10, eob_8x16 +.else + movrel_local r10, eob_8x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel_local r10, eob_8x16_identity2 +.else + movrel_local r10, eob_8x16 +.endif +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct +def_fn_816 \w, \h, identity, identity +def_fn_816 \w, \h, dct, adst +def_fn_816 \w, \h, dct, flipadst +def_fn_816 \w, \h, dct, identity +def_fn_816 \w, \h, adst, dct +def_fn_816 \w, \h, adst, adst +def_fn_816 \w, \h, adst, flipadst +def_fn_816 \w, \h, flipadst, dct +def_fn_816 \w, \h, flipadst, adst +def_fn_816 \w, \h, flipadst, flipadst +def_fn_816 \w, \h, identity, dct +def_fn_816 \w, \h, adst, identity +def_fn_816 \w, \h, flipadst, identity +def_fn_816 \w, \h, identity, adst +def_fn_816 \w, \h, identity, flipadst +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_2s_x16_neon + movrel_local r12, idct_coeffs, 4*16 + vld1.32 {q0, q1}, [r12, :128]! + + vmul_vmls d4, d16, d31, d0[0], d0[1] // -> t16a + vmul_vmla d6, d16, d31, d0[1], d0[0] // -> t31a + vmul_vmls d8, d24, d23, d1[0], d1[1] // -> t17a + vrshr.s32 d16, d4, #12 // t16a + vrshr.s32 d31, d6, #12 // t31a + vmul_vmla d4, d24, d23, d1[1], d1[0] // -> t30a + vmul_vmls d6, d20, d27, d2[0], d2[1] // -> t18a + vrshr.s32 d24, d8, #12 // t17a + vrshr.s32 d23, d4, #12 // t30a + vmul_vmla d8, d20, d27, d2[1], d2[0] // -> t29a + vmul_vmls d4, d28, d19, d3[0], d3[1] // -> t19a + vrshr.s32 d20, d6, #12 // t18a + vrshr.s32 d27, d8, #12 // t29a + vmul_vmla d6, d28, d19, d3[1], d3[0] // -> t28a + vld1.32 {q0, q1}, [r12, :128] + sub r12, r12, #4*24 + vmul_vmls d8, d18, d29, d0[0], d0[1] // -> t20a + vrshr.s32 d28, d4, #12 // t19a + vrshr.s32 d19, d6, #12 // t28a + vmul_vmla d4, d18, d29, d0[1], d0[0] // -> t27a + vmul_vmls d6, d26, d21, d1[0], d1[1] // -> t21a + vrshr.s32 d18, d8, #12 // t20a + vrshr.s32 d29, d4, #12 // t27a + vmul_vmla d8, d26, d21, d1[1], d1[0] // -> t26a + vmul_vmls d4, d22, d25, d2[0], d2[1] // -> t22a + vrshr.s32 d26, d6, #12 // t21a + vrshr.s32 d21, d8, #12 // t26a + vmul_vmla d6, d22, d25, d2[1], d2[0] // -> t25a + vmul_vmls d8, d30, d17, d3[0], d3[1] // -> t23a + vrshr.s32 d22, d4, #12 // t22a + vrshr.s32 d25, d6, #12 // t25a + vmul_vmla d4, d30, d17, d3[1], d3[0] // -> t24a + vrshr.s32 d30, d8, #12 // t23a + vrshr.s32 d17, d4, #12 // t24a + + vld1.32 {q0, q1}, [r12, :128] + + vmov.i32 d11, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 d10, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 + + vqsub.s32 d5, d16, d24 // t17 + vqadd.s32 d16, d16, d24 // t16 + vqsub.s32 d7, d31, d23 // t30 + vqadd.s32 d31, d31, d23 // t31 + vqsub.s32 d24, d28, d20 // t18 + vqadd.s32 d28, d28, d20 // t19 + vqadd.s32 d23, d18, d26 // t20 + vqsub.s32 d18, d18, d26 // t21 + vqsub.s32 d20, d30, d22 // t22 + vqadd.s32 d30, d30, d22 // t23 + vqadd.s32 d26, d17, d25 // t24 + vqsub.s32 d17, d17, d25 // t25 + vqsub.s32 d22, d29, d21 // t26 + vqadd.s32 d29, d29, d21 // t27 + vqadd.s32 d25, d19, d27 // t28 + vqsub.s32 d19, d19, d27 // t29 + +.irp r, d5, d16, d7, d31, d24, d28, d23, d18, d20, d30, d26, d17, d22, d29, d25, d19 + vmin.s32 \r, \r, d11 +.endr +.irp r, d5, d16, d7, d31, d24, d28, d23, d18, d20, d30, d26, d17, d22, d29, d25, d19 + vmax.s32 \r, \r, d10 +.endr + + vmul_vmls d4, d7, d5, d2[0], d2[1] // -> t17a + vmul_vmla d6, d7, d5, d2[1], d2[0] // -> t30a + vmul_vmla d8, d19, d24, d2[1], d2[0] // -> t18a + vrshr.s32 d21, d4, #12 // t17a + vrshr.s32 d27, d6, #12 // t30a + vneg.s32 d8, d8 // -> t18a + vmul_vmls d5, d19, d24, d2[0], d2[1] // -> t29a + vmul_vmls d4, d22, d18, d3[0], d3[1] // -> t21a + vrshr.s32 d19, d8, #12 // t18a + vrshr.s32 d24, d5, #12 // t29a + vmul_vmla d6, d22, d18, d3[1], d3[0] // -> t26a + vmul_vmla d8, d17, d20, d3[1], d3[0] // -> t22a + vrshr.s32 d22, d4, #12 // t21a + vrshr.s32 d18, d6, #12 // t26a + vneg.s32 d8, d8 // -> t22a + vmul_vmls d5, d17, d20, d3[0], d3[1] // -> t25a + vrshr.s32 d17, d8, #12 // t22a + vrshr.s32 d20, d5, #12 // t25a + + vqsub.s32 d2, d27, d24 // t29 + vqadd.s32 d27, d27, d24 // t30 + vqsub.s32 d3, d21, d19 // t18 + vqadd.s32 d21, d21, d19 // t17 + vqsub.s32 d24, d16, d28 // t19a + vqadd.s32 d16, d16, d28 // t16a + vqsub.s32 d19, d30, d23 // t20a + vqadd.s32 d30, d30, d23 // t23a + vqsub.s32 d28, d17, d22 // t21 + vqadd.s32 d17, d17, d22 // t22 + vqadd.s32 d23, d26, d29 // t24a + vqsub.s32 d26, d26, d29 // t27a + vqadd.s32 d22, d20, d18 // t25 + vqsub.s32 d20, d20, d18 // t26 + vqsub.s32 d29, d31, d25 // t28a + vqadd.s32 d31, d31, d25 // t31a + +.irp r, d2, d27, d3, d21, d24, d16, d19, d30, d28, d17, d23, d26, d22, d20, d29, d31 + vmin.s32 \r, \r, d11 +.endr +.irp r, d2, d27, d3, d21, d24, d16, d19, d30, d28, d17, d23, d26, d22, d20, d29, d31 + vmax.s32 \r, \r, d10 +.endr + + vmul_vmls d4, d2, d3, d1[0], d1[1] // -> t18a + vmul_vmla d6, d2, d3, d1[1], d1[0] // -> t29a + vmul_vmls d8, d29, d24, d1[0], d1[1] // -> t19 + vrshr.s32 d18, d4, #12 // t18a + vrshr.s32 d25, d6, #12 // t29a + vmul_vmla d5, d29, d24, d1[1], d1[0] // -> t28 + vmul_vmla d4, d26, d19, d1[1], d1[0] // -> t20 + vrshr.s32 d29, d8, #12 // t19 + vrshr.s32 d24, d5, #12 // t28 + vneg.s32 d4, d4 // -> t20 + vmul_vmls d6, d26, d19, d1[0], d1[1] // -> t27 + vmul_vmla d8, d20, d28, d1[1], d1[0] // -> t21a + vrshr.s32 d26, d4, #12 // t20 + vrshr.s32 d19, d6, #12 // t27 + vneg.s32 d8, d8 // -> t21a + vmul_vmls d5, d20, d28, d1[0], d1[1] // -> t26a + vrshr.s32 d20, d8, #12 // t21a + vrshr.s32 d28, d5, #12 // t26a + + vqsub.s32 d2, d16, d30 // t23 + vqadd.s32 d16, d16, d30 // t16 = out16 + vqsub.s32 d3, d31, d23 // t24 + vqadd.s32 d31, d31, d23 // t31 = out31 + vqsub.s32 d23, d21, d17 // t22a + vqadd.s32 d17, d21, d17 // t17a = out17 + vqadd.s32 d30, d27, d22 // t30a = out30 + vqsub.s32 d21, d27, d22 // t25a + vqsub.s32 d27, d18, d20 // t21 + vqadd.s32 d18, d18, d20 // t18 = out18 + vqadd.s32 d4, d29, d26 // t19a = out19 + vqsub.s32 d26, d29, d26 // t20a + vqadd.s32 d29, d25, d28 // t29 = out29 + vqsub.s32 d25, d25, d28 // t26 + vqadd.s32 d28, d24, d19 // t28a = out28 + vqsub.s32 d24, d24, d19 // t27a + vmov d19, d4 // out19 + +.irp r, d2, d16, d3, d31, d23, d17, d30, d21, d27, d18, d19, d26, d29, d25, d28, d24 + vmin.s32 \r, \r, d11 +.endr +.irp r, d2, d16, d3, d31, d23, d17, d30, d21, d27, d18, d19, d26, d29, d25, d28, d24 + vmax.s32 \r, \r, d10 +.endr + + vmul_vmls d4, d24, d26, d0[0], d0[0] // -> t20 + vmul_vmla d6, d24, d26, d0[0], d0[0] // -> t27 + vrshr.s32 d20, d4, #12 // t20 + vrshr.s32 d22, d6, #12 // t27 + + vmul_vmla d4, d25, d27, d0[0], d0[0] // -> t26a + vmul_vmls d6, d25, d27, d0[0], d0[0] // -> t21a + vmov d27, d22 // t27 + vrshr.s32 d26, d4, #12 // t26a + + vmul_vmls d24, d21, d23, d0[0], d0[0] // -> t22 + vmul_vmla d4, d21, d23, d0[0], d0[0] // -> t25 + vrshr.s32 d21, d6, #12 // t21a + vrshr.s32 d22, d24, #12 // t22 + vrshr.s32 d25, d4, #12 // t25 + + vmul_vmls d4, d3, d2, d0[0], d0[0] // -> t23a + vmul_vmla d6, d3, d2, d0[0], d0[0] // -> t24a + vrshr.s32 d23, d4, #12 // t23a + vrshr.s32 d24, d6, #12 // t24a + + bx lr +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x2_neon + push {lr} + vmov.i32 d7, #0 + lsl r8, r8, #1 +.if \scale + mov_const r12, 2896*8*(1<<16) + vdup.32 d0, r12 +.endif + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.32 {\i}, [r7, :64] + vst1.32 {d7}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 +.if \scale + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct_2s_x16_neon + + // idct_16 leaves the row_clip_max/min constants in d9 and d8, + // but here we want to use full q registers for clipping. + vmov.i32 q3, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 q2, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 +.irp r, q8, q9, q10, q11, q12, q13, q14, q15 + vmin.s32 \r, \r, q3 +.endr +.irp r, q8, q9, q10, q11, q12, q13, q14, q15 + vmax.s32 \r, \r, q2 +.endr + + vtrn.32 d16, d17 + vtrn.32 d18, d19 + vtrn.32 d20, d21 + vtrn.32 d22, d23 + vtrn.32 d24, d25 + vtrn.32 d26, d27 + vtrn.32 d28, d29 + vtrn.32 d30, d31 + +.macro store1 r0, r1, r2, r3 + vst1.16 {\r0}, [r6, :64]! + vst1.16 {\r1}, [r6, :64]! + vst1.16 {\r2}, [r6, :64]! + vst1.16 {\r3}, [r6, :64]! +.endm + store1 d16, d18, d20, d22 + store1 d24, d26, d28, d30 + store1 d17, d19, d21, d23 + store1 d25, d27, d29, d31 +.purgem store1 + sub r6, r6, #64*2 + + vmov.i32 d7, #0 +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.32 {\i}, [r7, :64] + vst1.32 {d7}, [r7, :64], r8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in d0[1] + scale_input d0[1], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct32_odd_2s_x16_neon + vtrn.32 d31, d30 + vtrn.32 d29, d28 + vtrn.32 d27, d26 + vtrn.32 d25, d24 + vtrn.32 d23, d22 + vtrn.32 d21, d20 + vtrn.32 d19, d18 + vtrn.32 d17, d16 +.macro store2 r0, r1, r2, r3, r4, r5, r6, r7, shift + vld1.32 {q0, q1}, [r6, :128]! + vld1.32 {q2, q3}, [r6, :128] + sub r6, r6, #32 + vqsub.s32 d15, d0, \r0 + vqadd.s32 d0, d0, \r0 + vqsub.s32 d14, d1, \r1 + vqadd.s32 d1, d1, \r1 + vqsub.s32 d13, d2, \r2 + vqadd.s32 d2, d2, \r2 + vqsub.s32 d12, d3, \r3 + vqadd.s32 d3, d3, \r3 + vqsub.s32 d11, d4, \r4 + vqadd.s32 d4, d4, \r4 + vqsub.s32 d10, d5, \r5 + vqadd.s32 d5, d5, \r5 + vqsub.s32 d9, d6, \r6 + vqadd.s32 d6, d6, \r6 + vqsub.s32 d8, d7, \r7 + vqadd.s32 d7, d7, \r7 + vqrshrn.s32 d0, q0, #\shift + vqrshrn.s32 d1, q1, #\shift + vqrshrn.s32 d2, q2, #\shift + vqrshrn.s32 d3, q3, #\shift + vqrshrn.s32 d4, q4, #\shift + vqrshrn.s32 d5, q5, #\shift + vqrshrn.s32 d6, q6, #\shift + vqrshrn.s32 d7, q7, #\shift + vrev32.16 q2, q2 + vrev32.16 q3, q3 + vst1.16 {q0, q1}, [r6, :128]! + vst1.16 {q2, q3}, [r6, :128]! +.endm + + store2 d31, d29, d27, d25, d23, d21, d19, d17, \shift + store2 d30, d28, d26, d24, d22, d20, d18, d16, \shift +.purgem store2 + pop {pc} +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_4x32_neon + push {r10-r11,lr} + lsl r8, r8, #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + + bl X(inv_dct_4h_x16_neon) + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vst1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + sub r7, r7, r8, lsr #1 + bl X(inv_dct32_odd_4h_x16_neon) + + neg r9, r8 + mov r10, r6 + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff +.macro combine r0, r1, r2, r3, op, stride + vld1.16 {d4}, [r7, :64], \stride + vld1.16 {d0}, [r10, :64], r1 + vld1.16 {d5}, [r7, :64], \stride + vld1.16 {d1}, [r10, :64], r1 + \op\().s16 d4, d4, \r0 + vld1.16 {d6}, [r7, :64], \stride + vld1.16 {d2}, [r10, :64], r1 + \op\().s16 d5, d5, \r1 + vld1.16 {d3}, [r10, :64], r1 + vrshr.s16 q2, q2, #4 + \op\().s16 d6, d6, \r2 + vld1.16 {d7}, [r7, :64], \stride + vqadd.s16 q0, q0, q2 + \op\().s16 d7, d7, \r3 + vmax.s16 q0, q0, q6 + vrshr.s16 q3, q3, #4 + vmin.s16 q0, q0, q7 + vqadd.s16 q1, q1, q3 + vst1.16 {d0}, [r6, :64], r1 + vmax.s16 q1, q1, q6 + vst1.16 {d1}, [r6, :64], r1 + vmin.s16 q1, q1, q7 + vst1.16 {d2}, [r6, :64], r1 + vst1.16 {d3}, [r6, :64], r1 +.endm + combine d31, d30, d29, d28, vqadd, r8 + combine d27, d26, d25, d24, vqadd, r8 + combine d23, d22, d21, d20, vqadd, r8 + combine d19, d18, d17, d16, vqadd, r8 + sub r7, r7, r8 + combine d16, d17, d18, d19, vqsub, r9 + combine d20, d21, d22, d23, vqsub, r9 + combine d24, d25, d26, d27, vqsub, r9 + combine d28, d29, d30, d31, vqsub, r9 +.purgem combine + + pop {r10-r11,pc} +endfunc + +const eob_32x32 + .short 3, 10, 21, 36, 55, 78, 105, 136, 171, 210, 253, 300, 351, 406, 465, 1024 +endconst + +const eob_16x32 + .short 3, 10, 21, 36, 55, 78, 105, 151, 183, 215, 247, 279, 311, 343, 375, 512 +endconst + +const eob_16x32_shortside + .short 3, 10, 21, 36, 55, 78, 105, 512 +endconst + +const eob_8x32 + .short 3, 10, 21, 43, 59, 75, 91, 107, 123, 139, 155, 171, 187, 203, 219, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_16bpc_neon, export=1 + push {r4-r7,lr} + vpush {q6-q7} + movrel_local r5, eob_32x32, 2 + + mov r6, #4*32 +1: + mov r12, #0 + movrel_local r4, eob_32x32, 6 +2: + vmov.i32 q0, #0 + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r6 +.endr + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q12 + vqmovn.s32 d18, q9 + vqmovn.s32 d19, q13 + vqmovn.s32 d20, q10 + vqmovn.s32 d21, q14 + vqmovn.s32 d22, q11 + vqmovn.s32 d23, q15 + transpose_4x8h q8, q9, q10, q11 + + load_add_store_8x4 r0, r7, shiftbits=2 + ldrh lr, [r4], #8 + sub r0, r0, r1, lsl #2 + cmp r3, lr + add r0, r0, #2*8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12, lsl #1 + add r0, r0, r1, lsl #2 + mls r2, r6, r12, r2 + add r2, r2, #4*4 + b 1b +9: + vpop {q6-q7} + pop {r4-r7,pc} +endfunc + +.macro shift_8_regs op, shift +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + push {r4-r9,lr} + vpush {q6-q7} + mov r9, #0 + mov_const r8, 2896*8*(1<<16) + movt r9, #2*(5793-4096)*8 + movrel_local r5, eob_16x32\hshort, 2 + + mov r6, #4*\h +1: + mov r12, #0 + movrel_local r4, eob_16x32\wshort, 6 +2: + vdup.i32 d0, r8 + vmov.i32 q1, #0 + vmov.32 d0[1], r9 + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q1}, [r2, :128], r6 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 + +.if \w == 16 + // 16x32 + identity_8x4_shift1 d0[1] +.else + // 32x16 + shift_8_regs vqshl.s32, 1 + identity_8x4 d0[1] +.endif + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q12 + vqmovn.s32 d18, q9 + vqmovn.s32 d19, q13 + vqmovn.s32 d20, q10 + vqmovn.s32 d21, q14 + vqmovn.s32 d22, q11 + vqmovn.s32 d23, q15 + transpose_4x8h q8, q9, q10, q11 + +.if \w == 16 + load_add_store_8x4 r0, r7, shiftbits=2 +.else + load_add_store_8x4 r0, r7, shiftbits=4 +.endif + ldrh lr, [r4], #8 + sub r0, r0, r1, lsl #2 + cmp r3, lr + add r0, r0, #2*8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12, lsl #1 + add r0, r0, r1, lsl #2 + mls r2, r6, r12, r2 + add r2, r2, #4*4 + b 1b +9: + vpop {q6-q7} + pop {r4-r9,pc} +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + push {r4-r5,lr} + vpush {q6-q7} + movrel_local r4, eob_8x32, 2 + + mov r12, #4*\h +1: + ldrh lr, [r4], #4 +.if \w == 8 + vmov.i32 q0, #0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r12 +.endr + + vqrshrn.s32 d16, q8, #1 + vqrshrn.s32 d17, q12, #1 + vqrshrn.s32 d18, q9, #1 + vqrshrn.s32 d19, q13, #1 + vqrshrn.s32 d20, q10, #1 + vqrshrn.s32 d21, q14, #1 + vqrshrn.s32 d22, q11, #1 + vqrshrn.s32 d23, q15, #1 + + transpose_4x8h q8, q9, q10, q11 + + cmp r3, lr + load_add_store_8x4 r0, r5, shiftbits=2 + blt 9f + sub r2, r2, r12, lsl #3 + add r2, r2, #4*4 +.else + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vld1.32 {q8, q9}, [r2, :128] + vst1.32 {q0, q1}, [r2, :128], r12 + vld1.32 {q10, q11}, [r2, :128] + vst1.32 {q0, q1}, [r2, :128], r12 + vld1.32 {q12, q13}, [r2, :128] + vst1.32 {q0, q1}, [r2, :128], r12 + vld1.32 {q14, q15}, [r2, :128] + vst1.32 {q0, q1}, [r2, :128], r12 + vqmovn.s32 d16, q8 + vqmovn.s32 d17, q10 + vqmovn.s32 d20, q9 + vqmovn.s32 d21, q11 + vqmovn.s32 d18, q12 + vqmovn.s32 d19, q14 + vqmovn.s32 d22, q13 + vqmovn.s32 d23, q15 + + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + + cmp r3, lr + load_add_store_4x8 r0, r5, shiftbits=3 + blt 9f + sub r0, r0, r1, lsl #3 + add r0, r0, #2*4 +.endif + b 1b + +9: + vpop {q6-q7} + pop {r4-r5,pc} +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_16bpc_neon, export=1 + idct_dc 32, 32, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 2048 + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, sp, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #32*4 + bl inv_txfm_horz_dct_32x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 2048 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x32_16bpc_neon, export=1 + idct_dc 16, 32, 1 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel_local r4, inv_dct_2s_x16_neon + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, sp, #(\i*16*2) + add r7, r2, #(\i*4) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #4*32 + bl inv_txfm_horz_scale_16x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #16*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 1024 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x16_16bpc_neon, export=1 + idct_dc 32, 16, 1 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel r5, X(inv_dct_4h_x16_neon) + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*4) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 14 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #4*16 + bl inv_txfm_horz_scale_dct_32x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 1024 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_8x32_16bpc_neon, export=1 + idct_dc 8, 32, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 512 + + movrel_local r10, eob_8x32, 2 + + mov r8, #4*32 + mov r9, #32 + mov r6, sp +1: + vmov.i32 q0, #0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.32 {\i}, [r2, :128] + vst1.32 {q0}, [r2, :128], r8 +.endr + ldrh r11, [r10], #4 + sub r2, r2, r8, lsl #3 + sub r9, r9, #4 + add r2, r2, #4*4 + + bl inv_dct_4s_x8_neon + + vqrshrn.s32 d16, q8, #2 + vqrshrn.s32 d18, q9, #2 + vqrshrn.s32 d20, q10, #2 + vqrshrn.s32 d22, q11, #2 + vqrshrn.s32 d17, q12, #2 + vqrshrn.s32 d19, q13, #2 + vqrshrn.s32 d21, q14, #2 + vqrshrn.s32 d23, q15, #2 + + transpose_4x8h q8, q9, q10, q11 + + vst1.16 {q8, q9}, [r6, :128]! + cmp r3, r11 + vst1.16 {q10, q11}, [r6, :128]! + + bge 1b + cmp r9, #0 + beq 3f + + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r9, r9, #4 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4 + add r6, r0, #(\i*2) + add r7, sp, #(\i*2) + mov r8, #8*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x8_16bpc_neon, export=1 + idct_dc 32, 8, 2 + + push {r4-r11,lr} + vpush {q4-q7} + movrel_local r10, eob_8x32 + sub_sp_align 512 + ldrh r11, [r10], #2 + +.irp i, 0, 2, 4, 6 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*4) +.if \i > 0 + cmp r3, r11 + mov r8, #(8 - \i) + blt 1f +.if \i < 6 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #8*4 + bl inv_txfm_horz_dct_32x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: + mov r8, #2*32 + mov r9, #0 +1: + add r6, r0, r9, lsl #1 + add r7, sp, r9, lsl #1 // #(\i*2) + +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r7, :128], r8 +.endr + add r9, r9, #8 + + bl X(inv_dct_8h_x8_neon) + + cmp r9, #32 + + load_add_store_8x8 r6, r7 + + blt 1b + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + vld1.32 {q0, q1}, [r12, :128]! + + vqrdmulh.s32 d23, d16, d0[1] // t63a + vqrdmulh.s32 d16, d16, d0[0] // t32a + vqrdmulh.s32 d22, d17, d1[0] // t62a + vqrdmulh.s32 d17, d17, d1[1] // t33a + vqrdmulh.s32 d21, d18, d2[1] // t61a + vqrdmulh.s32 d18, d18, d2[0] // t34a + vqrdmulh.s32 d20, d19, d3[0] // t60a + vqrdmulh.s32 d19, d19, d3[1] // t35a + + vld1.32 {q0}, [r12, :128]! + + vqadd.s32 d24, d16, d17 // t32 + vqsub.s32 d25, d16, d17 // t33 + vqsub.s32 d26, d19, d18 // t34 + vqadd.s32 d27, d19, d18 // t35 + vqadd.s32 d28, d20, d21 // t60 + vqsub.s32 d29, d20, d21 // t61 + vqsub.s32 d30, d23, d22 // t62 + vqadd.s32 d31, d23, d22 // t63 + +.irp r, q12, q13, q14, q15 + vmin.s32 \r, \r, q5 +.endr +.irp r, q12, q13, q14, q15 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmla d4, d29, d26, d0[0], d0[1] // -> t34a + vmul_vmls d6, d29, d26, d0[1], d0[0] // -> t61a + vneg.s32 d4, d4 // t34a + vmul_vmls d7, d30, d25, d0[1], d0[0] // -> t33a + vrshr.s32 d26, d4, #12 // t34a + vmul_vmla d4, d30, d25, d0[0], d0[1] // -> t62a + vrshr.s32 d29, d6, #12 // t61a + vrshr.s32 d25, d7, #12 // t33a + vrshr.s32 d30, d4, #12 // t62a + + vqadd.s32 d16, d24, d27 // t32a + vqsub.s32 d19, d24, d27 // t35a + vqadd.s32 d17, d25, d26 // t33 + vqsub.s32 d18, d25, d26 // t34 + vqsub.s32 d20, d31, d28 // t60a + vqadd.s32 d23, d31, d28 // t63a + vqsub.s32 d21, d30, d29 // t61 + vqadd.s32 d22, d30, d29 // t62 + +.irp r, q8, q9, q10, q11 + vmin.s32 \r, \r, q5 +.endr +.irp r, q8, q9, q10, q11 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmla d4, d21, d18, d1[0], d1[1] // -> t61a + vmul_vmls d6, d21, d18, d1[1], d1[0] // -> t34a + vmul_vmla d7, d20, d19, d1[0], d1[1] // -> t60 + vrshr.s32 d21, d4, #12 // t61a + vrshr.s32 d18, d6, #12 // t34a + vmul_vmls d4, d20, d19, d1[1], d1[0] // -> t35 + vrshr.s32 d20, d7, #12 // t60 + vrshr.s32 d19, d4, #12 // t35 + + vst1.32 {d16, d17, d18, d19}, [r6, :128]! + vst1.32 {d20, d21, d22, d23}, [r6, :128]! + + bx lr +endfunc + +function inv_dct64_step2_neon + movrel_local r12, idct_coeffs + vld1.32 {q0}, [r12, :128] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + vldr d16, [r6, #4*2*0] // t32a + vldr d17, [r9, #4*2*8] // t39a + vldr d18, [r9, #4*2*0] // t63a + vldr d19, [r6, #4*2*8] // t56a + vldr d20, [r6, #4*2*16] // t40a + vldr d21, [r9, #4*2*24] // t47a + vldr d22, [r9, #4*2*16] // t55a + vldr d23, [r6, #4*2*24] // t48a + + vqadd.s32 d24, d16, d17 // t32 + vqsub.s32 d25, d16, d17 // t39 + vqadd.s32 d26, d18, d19 // t63 + vqsub.s32 d27, d18, d19 // t56 + vqsub.s32 d28, d21, d20 // t40 + vqadd.s32 d29, d21, d20 // t47 + vqadd.s32 d30, d23, d22 // t48 + vqsub.s32 d31, d23, d22 // t55 + +.irp r, q12, q13, q14, q15 + vmin.s32 \r, \r, q5 +.endr +.irp r, q12, q13, q14, q15 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmla d4, d27, d25, d1[1], d1[0] // -> t56a + vmul_vmls d6, d27, d25, d1[0], d1[1] // -> t39a + vmul_vmla d7, d31, d28, d1[1], d1[0] // -> t40a + vrshr.s32 d25, d4, #12 // t56a + vrshr.s32 d27, d6, #12 // t39a + vneg.s32 d7, d7 // t40a + vmul_vmls d4, d31, d28, d1[0], d1[1] // -> t55a + vrshr.s32 d31, d7, #12 // t40a + vrshr.s32 d28, d4, #12 // t55a + + vqadd.s32 d16, d24, d29 // t32a + vqsub.s32 d19, d24, d29 // t47a + vqadd.s32 d17, d27, d31 // t39 + vqsub.s32 d18, d27, d31 // t40 + vqsub.s32 d20, d26, d30 // t48a + vqadd.s32 d23, d26, d30 // t63a + vqsub.s32 d21, d25, d28 // t55 + vqadd.s32 d22, d25, d28 // t56 + +.irp r, q8, q9, q10, q11 + vmin.s32 \r, \r, q5 +.endr +.irp r, q8, q9, q10, q11 + vmax.s32 \r, \r, q4 +.endr + + vmul_vmls d4, d21, d18, d0[0], d0[0] // -> t40a + vmul_vmla d6, d21, d18, d0[0], d0[0] // -> t55a + vmul_vmls d7, d20, d19, d0[0], d0[0] // -> t47 + vrshr.s32 d18, d4, #12 // t40a + vrshr.s32 d21, d6, #12 // t55a + vmul_vmla d4, d20, d19, d0[0], d0[0] // -> t48 + vrshr.s32 d19, d7, #12 // t47 + vrshr.s32 d20, d4, #12 // t48 + + vstr d16, [r6, #4*2*0] // t32a + vstr d17, [r9, #4*2*0] // t39 + vstr d18, [r6, #4*2*8] // t40a + vstr d19, [r9, #4*2*8] // t47 + vstr d20, [r6, #4*2*16] // t48 + vstr d21, [r9, #4*2*16] // t55a + vstr d22, [r6, #4*2*24] // t56 + vstr d23, [r9, #4*2*24] // t63a + + add r6, r6, #4*2 + sub r9, r9, #4*2 + cmp r6, r9 + blt 1b + bx lr +endfunc + +.macro load8 src, strd, zero, clear +.irp i, d16, d17, d18, d19, d20, d21, d22, d23 +.if \clear + vld1.32 {\i}, [\src, :64] + vst1.32 {\zero}, [\src, :64], \strd +.else + vld1.32 {\i}, [\src, :64], \strd +.endif +.endr +.endm + +.macro store16 dst + vst1.32 {q8, q9}, [\dst, :128]! + vst1.32 {q10, q11}, [\dst, :128]! + vst1.32 {q12, q13}, [\dst, :128]! + vst1.32 {q14, q15}, [\dst, :128]! +.endm + +.macro clear_upper8 +.irp i, q12, q13, q14, q15 + vmov.i32 \i, #0 +.endr +.endm + +.macro vmov_if reg, val, cond +.if \cond + vmov.i32 \reg, \val +.endif +.endm + +.macro movdup_if reg, gpr, val, cond +.if \cond + mov_const \gpr, \val + vdup.32 \reg, \gpr +.endif +.endm + +.macro vst1_if regs, dst, dstalign, cond +.if \cond + vst1.32 \regs, \dst, \dstalign +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_2s_x64_neon + mov r6, sp + + push {r10-r11,lr} + + lsl r8, r8, #2 + + movdup_if d0, r12, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + add r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct_2s_x16_neon + + // idct_16 leaves the row_clip_max/min constants in d9 and d8, + // but here we want to use full q registers for clipping. + vmov.i32 q3, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 q2, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 +.irp r, q8, q9, q10, q11, q12, q13, q14, q15 + vmin.s32 \r, \r, q3 +.endr +.irp r, q8, q9, q10, q11, q12, q13, q14, q15 + vmax.s32 \r, \r, q2 +.endr + + store16 r6 + + movdup_if d0, r12, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + lsr r8, r8, #1 + sub r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct32_odd_2s_x16_neon + + add r10, r6, #8*15 + sub r6, r6, #8*16 + + mov r9, #-8 + + vmov.i32 d1, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 d0, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 +.macro store_addsub r0, r1, r2, r3 + vld1.32 {d2}, [r6, :64]! + vld1.32 {d3}, [r6, :64]! + vqadd.s32 d6, d2, \r0 + vqsub.s32 \r0, d2, \r0 + vld1.32 {d4}, [r6, :64]! + vqadd.s32 d7, d3, \r1 + vqsub.s32 \r1, d3, \r1 + vmin.s32 d6, d6, d1 + vmin.s32 \r0, \r0, d1 + vld1.32 {d5}, [r6, :64]! + vqadd.s32 d2, d4, \r2 + sub r6, r6, #8*4 + vmax.s32 d6, d6, d0 + vmax.s32 \r0, \r0, d0 + vqsub.s32 \r2, d4, \r2 + vmin.s32 d7, d7, d1 + vmin.s32 \r1, \r1, d1 + vst1.32 {d6}, [r6, :64]! + vst1.32 {\r0}, [r10, :64], r9 + vmin.s32 d2, d2, d1 + vmin.s32 \r2, \r2, d1 + vmax.s32 d7, d7, d0 + vmax.s32 \r1, \r1, d0 + vqadd.s32 d3, d5, \r3 + vqsub.s32 \r3, d5, \r3 + vmax.s32 d2, d2, d0 + vmax.s32 \r2, \r2, d0 + vmin.s32 d3, d3, d1 + vmin.s32 \r3, \r3, d1 + vst1.32 {d7}, [r6, :64]! + vst1.32 {\r1}, [r10, :64], r9 + vmax.s32 d3, d3, d0 + vmax.s32 \r3, \r3, d0 + vst1.32 {d2}, [r6, :64]! + vst1.32 {\r2}, [r10, :64], r9 + vst1.32 {d3}, [r6, :64]! + vst1.32 {\r3}, [r10, :64], r9 +.endm + store_addsub d31, d30, d29, d28 + store_addsub d27, d26, d25, d24 + store_addsub d23, d22, d21, d20 + store_addsub d19, d18, d17, d16 +.purgem store_addsub + + add r6, r6, #2*4*16 + + movrel_local r12, idct64_coeffs + vmov.i32 q5, #0x1ffff // row_clip_max = ~(~bdmax << 7), 0x1ffff + vmvn.i32 q4, #0x1ffff // row_clip_min = (~bdmax << 7), 0xfffe0000 + movdup_if d0, lr, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + add r9, r7, r8, lsl #4 // offset 16 + add r10, r7, r8, lsl #3 // offset 8 + sub r9, r9, r8 // offset 15 + sub r11, r10, r8 // offset 7 + vld1.32 {d16}, [r7, :64] // in1 (offset 0) + vld1.32 {d17}, [r9, :64] // in31 (offset 15) + vld1.32 {d18}, [r10, :64] // in17 (offset 8) + vld1.32 {d19}, [r11, :64] // in15 (offset 7) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + add r7, r7, r8, lsl #2 // offset 4 + sub r9, r9, r8, lsl #2 // offset 11 + sub r10, r7, r8 // offset 3 + add r11, r9, r8 // offset 12 + vld1.32 {d16}, [r10, :64] // in7 (offset 3) + vld1.32 {d17}, [r11, :64] // in25 (offset 12) + vld1.32 {d18}, [r9, :64] // in23 (offset 11) + vld1.32 {d19}, [r7, :64] // in9 (offset 4) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + sub r10, r10, r8, lsl #1 // offset 1 + sub r9, r9, r8, lsl #1 // offset 9 + add r10, r10, r8 // offset 2 + add r9, r9, r8 // offset 10 + add r7, r7, r8 // offset 5 + add r11, r11, r8 // offset 13 + vld1.32 d16, [r10, :64] // in5 (offset 2) + vld1.32 d17, [r11, :64] // in27 (offset 13) + vld1.32 d18, [r9, :64] // in21 (offset 10) + vld1.32 d19, [r7, :64] // in11 (offset 5) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, 2896*8*(1<<16), \scale + vmov_if d7, #0, \clear + sub r10, r10, r8 // offset 1 + sub r9, r9, r8 // offset 9 + add r11, r11, r8 // offset 14 + add r7, r7, r8 // offset 6 + vld1.32 d16, [r10, :64] // in3 (offset 1) + vld1.32 d17, [r11, :64] // in29 (offset 14) + vld1.32 d18, [r9, :64] // in19 (offset 9) + vld1.32 d19, [r7, :64] // in13 (offset 6) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + + sub r6, r6, #2*4*32 + add r9, r6, #2*4*7 + + bl inv_dct64_step2_neon + + pop {r10-r11,pc} +endfunc +.endm + +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + +function inv_txfm_horz_dct_64x2_neon + vdup.32 q4, r9 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, #2*56 + + push {r10-r11,lr} + + mov r10, #2*64 + mov r11, #-2*4*4 + +1: + vld1.32 {d16, d17, d18, d19}, [r7, :128]! + vld1.32 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.32 {d20, d21, d22, d23}, [r7, :128]! + vld1.32 {d24, d25, d26, d27}, [r8, :128], r11 + vtrn.32 d16, d17 + vtrn.32 d18, d19 + vtrn.32 d20, d21 + vtrn.32 d22, d23 + vtrn.32 d31, d30 + vtrn.32 d29, d28 + vtrn.32 d27, d26 + vtrn.32 d25, d24 + +.macro store_addsub src0, src1, src2, src3, src4, src5, src6, src7 + vqsub.s32 d7, \src0, \src1 + vqsub.s32 d6, \src2, \src3 + vqsub.s32 d5, \src4, \src5 + vqsub.s32 d4, \src6, \src7 + vqadd.s32 d0, \src0, \src1 + vqadd.s32 d1, \src2, \src3 + vqadd.s32 d2, \src4, \src5 + vqadd.s32 d3, \src6, \src7 + vrshl.s32 q3, q3, q4 + vrshl.s32 q2, q2, q4 + vrshl.s32 q0, q0, q4 + vrshl.s32 q1, q1, q4 + vqmovn.s32 d7, q3 + vqmovn.s32 d6, q2 + vqmovn.s32 d0, q0 + vqmovn.s32 d1, q1 + vrev32.16 q3, q3 + vst1.16 {q0}, [r6, :128], r10 + vst1.16 {q3}, [r9, :128], r10 +.endm + store_addsub d16, d31, d18, d29, d20, d27, d22, d25 + store_addsub d17, d30, d19, d28, d21, d26, d23, d24 +.purgem store_addsub + sub r6, r6, r10, lsl #1 + sub r9, r9, r10, lsl #1 + add r6, r6, #16 + sub r9, r9, #16 + + cmp r7, r8 + blt 1b + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_vert_dct_4x64_neon + lsl r8, r8, #1 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, r1, lsl #6 + sub r9, r9, r1 + + push {r10-r11,lr} + + neg r10, r1 + mov r11, #-2*4*4 + +1: + vld1.16 {d16, d17, d18, d19}, [r7, :128]! + vld1.16 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.16 {d20, d21, d22, d23}, [r7, :128]! + vld1.16 {d24, d25, d26, d27}, [r8, :128], r11 + + vmov.i16 q6, #0 + vmvn.i16 q7, #0xfc00 // 0x3ff +.macro add_dest_addsub src0, src1, src2, src3 + vld1.16 {d0}, [r6, :64], r1 + vld1.16 {d1}, [r9, :64], r10 + vqadd.s16 d4, \src0, \src1 + vld1.16 {d2}, [r6, :64] + vqsub.s16 d5, \src0, \src1 + vld1.16 {d3}, [r9, :64] + vqadd.s16 d6, \src2, \src3 + vqsub.s16 d7, \src2, \src3 + sub r6, r6, r1 + sub r9, r9, r10 + vrshr.s16 q2, q2, #4 + vrshr.s16 q3, q3, #4 + vqadd.s16 q2, q2, q0 + vqadd.s16 q3, q3, q1 + vmax.s16 q2, q2, q6 + vmax.s16 q3, q3, q6 + vmin.s16 q2, q2, q7 + vmin.s16 q3, q3, q7 + vst1.16 {d4}, [r6, :64], r1 + vst1.16 {d5}, [r9, :64], r10 + vst1.16 {d6}, [r6, :64], r1 + vst1.16 {d7}, [r9, :64], r10 +.endm + add_dest_addsub d16, d31, d17, d30 + add_dest_addsub d18, d29, d19, d28 + add_dest_addsub d20, d27, d21, d26 + add_dest_addsub d22, d25, d23, d24 +.purgem add_dest_addsub + cmp r7, r8 + blt 1b + + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x64_16bpc_neon, export=1 + idct_dc 64, 64, 2 + + push {r4-r11,lr} + vpush {q4-q7} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*4) + mov r8, #32*4 + bl inv_txfm_dct_clear_2s_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x2_neon +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r7, r5, #(\i*2) + mov r8, #64*2 + bl X(inv_txfm_dct_4h_x64_neon) + add r6, r0, #(\i*2) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x32_16bpc_neon, export=1 + idct_dc 64, 32, 1 + + push {r4-r11,lr} + vpush {q4-q7} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*4) + mov r8, #32*4 + bl inv_txfm_dct_clear_scale_2s_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-1 // shift + bl inv_txfm_horz_dct_64x2_neon +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i*2) + add r7, r5, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x64_16bpc_neon, export=1 + idct_dc 32, 64, 1 + + push {r4-r11,lr} + vpush {q4-q7} + + sub_sp_align 32*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, r5, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #32*4 + bl inv_txfm_horz_scale_dct_32x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r7, r5, #(\i*2) + mov r8, #32*2 + bl X(inv_txfm_dct_4h_x64_neon) + add r6, r0, #(\i*2) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 32*32*2+64*4*2 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x16_16bpc_neon, export=1 + idct_dc 64, 16, 2 + + push {r4-r11,lr} + vpush {q4-q7} + + sub_sp_align 64*16*2+64*4*2 + add r4, sp, #64*4*2 + + movrel_local r10, eob_16x32 + +.irp i, 0, 2, 4, 6, 8, 10, 12, 14 + add r6, r4, #(\i*64*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*4) + mov r8, #16*4 + bl inv_txfm_dct_clear_2s_x64_neon + add r6, r4, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x2_neon +.if \i < 8 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: + movrel r5, X(inv_dct_4h_x16_neon) +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i*2) + add r7, r4, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 64*16*2+64*4*2 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x64_16bpc_neon, export=1 + idct_dc 16, 64, 2 + + push {r4-r11,lr} + vpush {q4-q7} + + sub_sp_align 16*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + + movrel_local r4, inv_dct_2s_x16_neon +.irp i, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + add r6, r5, #(\i*16*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 30 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*4) + mov r8, #32*4 + bl inv_txfm_horz_16x2_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 2 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r7, r5, #(\i*2) + mov r8, #16*2 + bl X(inv_txfm_dct_4h_x64_neon) + add r6, r0, #(\i*2) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 16*32*2+64*4*2 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/loopfilter.S b/third_party/dav1d/src/arm/32/loopfilter.S new file mode 100644 index 0000000000..97b960534f --- /dev/null +++ b/third_party/dav1d/src/arm/32/loopfilter.S @@ -0,0 +1,868 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro loop_filter wd +function lpf_8_wd\wd\()_neon + vabd.u8 d0, d22, d23 // abs(p1 - p0) + vabd.u8 d1, d25, d24 // abs(q1 - q0) + vabd.u8 d2, d23, d24 // abs(p0 - q0) + vabd.u8 d3, d22, d25 // abs(p1 - q1) +.if \wd >= 6 + vabd.u8 d4, d21, d22 // abs(p2 - p1) + vabd.u8 d5, d26, d25 // abs(q2 - q1) +.endif +.if \wd >= 8 + vabd.u8 d6, d20, d21 // abs(p3 - p2) + vabd.u8 d7, d27, d26 // abs(q3 - q3) +.endif +.if \wd >= 6 + vmax.u8 d4, d4, d5 +.endif + vqadd.u8 d2, d2, d2 // abs(p0 - q0) * 2 +.if \wd >= 8 + vmax.u8 d6, d6, d7 +.endif + vshr.u8 d3, d3, #1 +.if \wd >= 8 + vmax.u8 d4, d4, d6 +.endif +.if \wd >= 6 + vand d4, d4, d14 +.endif + vmax.u8 d0, d0, d1 // max(abs(p1 - p0), abs(q1 - q0)) + vqadd.u8 d2, d2, d3 // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 +.if \wd >= 6 + vmax.u8 d4, d0, d4 + vcge.u8 d1, d11, d4 // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I +.else + vcge.u8 d1, d11, d0 // max(abs(p1 - p0), abs(q1 - q0)) <= I +.endif + vcge.u8 d2, d10, d2 // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E + vand d1, d1, d2 // fm + vand d1, d1, d13 // fm && wd >= 4 +.if \wd >= 6 + vand d14, d14, d1 // fm && wd > 4 +.endif +.if \wd >= 16 + vand d15, d15, d1 // fm && wd == 16 +.endif + + vmov r10, r11, d1 + orrs r10, r10, r11 + beq 9f // if (!fm || wd < 4) return; + +.if \wd >= 6 + vmov.i8 d10, #1 + vabd.u8 d2, d21, d23 // abs(p2 - p0) + vabd.u8 d3, d22, d23 // abs(p1 - p0) + vabd.u8 d4, d25, d24 // abs(q1 - q0) + vabd.u8 d5, d26, d24 // abs(q2 - q0) +.if \wd >= 8 + vabd.u8 d6, d20, d23 // abs(p3 - p0) + vabd.u8 d7, d27, d24 // abs(q3 - q0) +.endif + vmax.u8 d2, d2, d3 + vmax.u8 d4, d4, d5 +.if \wd >= 8 + vmax.u8 d6, d6, d7 +.endif + vmax.u8 d2, d2, d4 +.if \wd >= 8 + vmax.u8 d2, d2, d6 +.endif + +.if \wd == 16 + vabd.u8 d3, d17, d23 // abs(p6 - p0) + vabd.u8 d4, d18, d23 // abs(p5 - p0) + vabd.u8 d5, d19, d23 // abs(p4 - p0) +.endif + vcge.u8 d2, d10, d2 // flat8in +.if \wd == 16 + vabd.u8 d6, d28, d24 // abs(q4 - q0) + vabd.u8 d7, d29, d24 // abs(q5 - q0) + vabd.u8 d8, d30, d24 // abs(q6 - q0) +.endif + vand d14, d2, d14 // flat8in && fm && wd > 4 + vbic d1, d1, d14 // fm && wd >= 4 && !flat8in +.if \wd == 16 + vmax.u8 d3, d3, d4 + vmax.u8 d5, d5, d6 +.endif + vmov r10, r11, d1 +.if \wd == 16 + vmax.u8 d7, d7, d8 + vmax.u8 d3, d3, d5 + vmax.u8 d3, d3, d7 + vcge.u8 d3, d10, d3 // flat8out +.endif + orrs r10, r10, r11 +.if \wd == 16 + vand d15, d15, d3 // flat8out && fm && wd == 16 + vand d15, d15, d14 // flat8out && flat8in && fm && wd == 16 + vbic d14, d14, d15 // flat8in && fm && wd >= 4 && !flat8out +.endif + beq 1f // skip wd == 4 case +.endif + + vsubl.u8 q1, d22, d25 // p1 - q1 + vcgt.u8 d0, d0, d12 // hev + vqmovn.s16 d2, q1 + vand d4, d2, d0 // if (hev) iclip_diff(p1 - q1) + vbic d0, d1, d0 // (fm && wd >= 4 && !hev) + vsubl.u8 q1, d24, d23 + vmov.i16 q3, #3 + vmul.i16 q1, q1, q3 + vmov.i8 d6, #4 + vaddw.s8 q1, q1, d4 + vmov.i8 d7, #3 + vqmovn.s16 d2, q1 // f + vqadd.s8 d4, d6, d2 // imin(f + 4, 127) + vqadd.s8 d5, d7, d2 // imin(f + 3, 127) + vshr.s8 d4, d4, #3 // f1 + vshr.s8 d5, d5, #3 // f2 + vmovl.u8 q1, d23 // p0 + vmovl.u8 q3, d24 // q0 + vaddw.s8 q1, q1, d5 + vsubw.s8 q3, q3, d4 + vrshr.s8 d4, d4, #1 // (f1 + 1) >> 1 + vqmovun.s16 d2, q1 // out p0 + vqmovun.s16 d6, q3 // out q0 + vbit d23, d2, d1 // if (fm && wd >= 4) + vmovl.u8 q1, d22 // p1 + vbit d24, d6, d1 // if (fm && wd >= 4) + vmovl.u8 q3, d25 // q1 + vaddw.s8 q1, q1, d4 + vsubw.s8 q3, q3, d4 + vqmovun.s16 d2, q1 // out p1 + vqmovun.s16 d6, q3 // out q1 + vbit d22, d2, d0 // if (fm && wd >= 4 && !hev) + vbit d25, d6, d0 // if (fm && wd >= 4 && !hev) +1: + +.if \wd == 6 + vmov r10, r11, d14 + orrs r10, r10, r11 + beq 2f // skip if there's no flat8in + + vaddl.u8 q0, d21, d21 // p2 * 2 + vaddl.u8 q1, d21, d22 // p2 + p1 + vaddl.u8 q2, d22, d23 // p1 + p0 + vaddl.u8 q3, d23, d24 // p0 + q0 + vadd.i16 q4, q0, q1 + vadd.i16 q5, q2, q3 + vaddl.u8 q6, d24, d25 // q0 + q1 + vadd.i16 q4, q4, q5 + vsub.i16 q6, q6, q0 + vaddl.u8 q5, d25, d26 // q1 + q2 + vrshrn.i16 d0, q4, #3 // out p1 + + vadd.i16 q4, q4, q6 + vsub.i16 q5, q5, q1 + vaddl.u8 q6, d26, d26 // q2 + q2 + vrshrn.i16 d1, q4, #3 // out p0 + + vadd.i16 q4, q4, q5 + vsub.i16 q6, q6, q2 + vrshrn.i16 d2, q4, #3 // out q0 + + vbit d22, d0, d14 // p1 if (flat8in) + vadd.i16 q4, q4, q6 + vbit d23, d1, d14 // p0 if (flat8in) + vrshrn.i16 d3, q4, #3 // out q1 + vbit d24, d2, d14 // q0 if (flat8in) + vbit d25, d3, d14 // q1 if (flat8in) +.elseif \wd >= 8 + vmov r10, r11, d14 + orrs r10, r10, r11 +.if \wd == 8 + beq 8f // skip if there's no flat8in +.else + beq 2f // skip if there's no flat8in +.endif + + vaddl.u8 q0, d20, d21 // p3 + p2 + vaddl.u8 q1, d22, d25 // p1 + q1 + vaddl.u8 q2, d20, d22 // p3 + p1 + vaddl.u8 q3, d23, d26 // p0 + q2 + vadd.i16 q4, q0, q0 // 2 * (p3 + p2) + vaddw.u8 q4, q4, d23 // + p0 + vaddw.u8 q4, q4, d24 // + q0 + vadd.i16 q4, q4, q2 // + p3 + p1 + vsub.i16 q1, q1, q0 // p1 + q1 - p3 - p2 + vsub.i16 q3, q3, q2 // p0 + q2 - p3 - p1 + vrshrn.i16 d10, q4, #3 // out p2 + + vadd.i16 q4, q4, q1 + vaddl.u8 q0, d20, d23 // p3 + p0 + vaddl.u8 q1, d24, d27 // q0 + q3 + vrshrn.i16 d11, q4, #3 // out p1 + + vadd.i16 q4, q4, q3 + vsub.i16 q1, q1, q0 // q0 + q3 - p3 - p0 + vaddl.u8 q2, d21, d24 // p2 + q0 + vaddl.u8 q3, d25, d27 // q1 + q3 + vrshrn.i16 d12, q4, #3 // out p0 + + vadd.i16 q4, q4, q1 + vsub.i16 q3, q3, q2 // q1 + q3 - p2 - q0 + vaddl.u8 q0, d22, d25 // p1 + q1 + vaddl.u8 q1, d26, d27 // q2 + q3 + vrshrn.i16 d13, q4, #3 // out q0 + + vadd.i16 q4, q4, q3 + vsub.i16 q1, q1, q0 // q2 + q3 - p1 - q1 + vrshrn.i16 d0, q4, #3 // out q1 + + vadd.i16 q4, q4, q1 + + vbit d21, d10, d14 + vbit d22, d11, d14 + vbit d23, d12, d14 + vrshrn.i16 d1, q4, #3 // out q2 + vbit d24, d13, d14 + vbit d25, d0, d14 + vbit d26, d1, d14 +.endif +2: +.if \wd == 16 + vmov r10, r11, d15 + orrs r10, r10, r11 + bne 1f // check if flat8out is needed + vmov r10, r11, d14 + orrs r10, r10, r11 + beq 8f // if there was no flat8in, just write the inner 4 pixels + b 7f // if flat8in was used, write the inner 6 pixels +1: + + vaddl.u8 q1, d17, d17 // p6 + p6 + vaddl.u8 q2, d17, d18 // p6 + p5 + vaddl.u8 q3, d17, d19 // p6 + p4 + vaddl.u8 q4, d17, d20 // p6 + p3 + vadd.i16 q6, q1, q2 + vadd.i16 q5, q3, q4 + vaddl.u8 q3, d17, d21 // p6 + p2 + vadd.i16 q6, q6, q5 + vaddl.u8 q4, d17, d22 // p6 + p1 + vaddl.u8 q5, d18, d23 // p5 + p0 + vadd.i16 q3, q3, q4 + vaddl.u8 q4, d19, d24 // p4 + q0 + vadd.i16 q6, q6, q3 + vadd.i16 q5, q5, q4 + vaddl.u8 q3, d20, d25 // p3 + q1 + vadd.i16 q6, q6, q5 + vsub.i16 q3, q3, q1 + vaddl.u8 q1, d21, d26 // p2 + q2 + vrshrn.i16 d0, q6, #4 // out p5 + vadd.i16 q6, q6, q3 // - (p6 + p6) + (p3 + q1) + vsub.i16 q1, q1, q2 + vaddl.u8 q2, d22, d27 // p1 + q3 + vaddl.u8 q3, d17, d19 // p6 + p4 + vrshrn.i16 d1, q6, #4 // out p4 + vadd.i16 q6, q6, q1 // - (p6 + p5) + (p2 + q2) + vsub.i16 q2, q2, q3 + vaddl.u8 q3, d23, d28 // p0 + q4 + vaddl.u8 q4, d17, d20 // p6 + p3 + vrshrn.i16 d2, q6, #4 // out p3 + vadd.i16 q6, q6, q2 // - (p6 + p4) + (p1 + q3) + vsub.i16 q3, q3, q4 + vaddl.u8 q4, d24, d29 // q0 + q5 + vaddl.u8 q2, d17, d21 // p6 + p2 + vrshrn.i16 d3, q6, #4 // out p2 + vadd.i16 q6, q6, q3 // - (p6 + p3) + (p0 + q4) + vsub.i16 q4, q4, q2 + vaddl.u8 q3, d25, d30 // q1 + q6 + vaddl.u8 q5, d17, d22 // p6 + p1 + vrshrn.i16 d4, q6, #4 // out p1 + vadd.i16 q6, q6, q4 // - (p6 + p2) + (q0 + q5) + vsub.i16 q3, q3, q5 + vaddl.u8 q4, d26, d30 // q2 + q6 + vbif d0, d18, d15 // out p5 + vaddl.u8 q5, d18, d23 // p5 + p0 + vrshrn.i16 d5, q6, #4 // out p0 + vadd.i16 q6, q6, q3 // - (p6 + p1) + (q1 + q6) + vsub.i16 q4, q4, q5 + vaddl.u8 q5, d27, d30 // q3 + q6 + vbif d1, d19, d15 // out p4 + vaddl.u8 q9, d19, d24 // p4 + q0 + vrshrn.i16 d6, q6, #4 // out q0 + vadd.i16 q6, q6, q4 // - (p5 + p0) + (q2 + q6) + vsub.i16 q5, q5, q9 + vaddl.u8 q4, d28, d30 // q4 + q6 + vbif d2, d20, d15 // out p3 + vaddl.u8 q9, d20, d25 // p3 + q1 + vrshrn.i16 d7, q6, #4 // out q1 + vadd.i16 q6, q6, q5 // - (p4 + q0) + (q3 + q6) + vsub.i16 q9, q4, q9 + vaddl.u8 q5, d29, d30 // q5 + q6 + vbif d3, d21, d15 // out p2 + vaddl.u8 q10, d21, d26 // p2 + q2 + vrshrn.i16 d8, q6, #4 // out q2 + vadd.i16 q6, q6, q9 // - (p3 + q1) + (q4 + q6) + vsub.i16 q5, q5, q10 + vaddl.u8 q9, d30, d30 // q6 + q6 + vbif d4, d22, d15 // out p1 + vaddl.u8 q10, d22, d27 // p1 + q3 + vrshrn.i16 d9, q6, #4 // out q3 + vadd.i16 q6, q6, q5 // - (p2 + q2) + (q5 + q6) + vsub.i16 q9, q9, q10 + vbif d5, d23, d15 // out p0 + vrshrn.i16 d10, q6, #4 // out q4 + vadd.i16 q6, q6, q9 // - (p1 + q3) + (q6 + q6) + vrshrn.i16 d11, q6, #4 // out q5 + vbif d6, d24, d15 // out q0 + vbif d7, d25, d15 // out q1 + vbif d8, d26, d15 // out q2 + vbif d9, d27, d15 // out q3 + vbif d10, d28, d15 // out q4 + vbif d11, d29, d15 // out q5 +.endif + + bx lr +.if \wd == 16 +7: + // Return to a shorter epilogue, writing only the inner 6 pixels + bx r8 +.endif +.if \wd >= 8 +8: + // Return to a shorter epilogue, writing only the inner 4 pixels + bx r9 +.endif +9: + // Return directly without writing back any pixels + bx r12 +endfunc +.endm + +loop_filter 16 +loop_filter 8 +loop_filter 6 +loop_filter 4 + +.macro lpf_8_wd16 + adr r8, 7f + CONFIG_THUMB + adr r9, 8f + CONFIG_THUMB + bl lpf_8_wd16_neon +.endm + +.macro lpf_8_wd8 + adr r9, 8f + CONFIG_THUMB + bl lpf_8_wd8_neon +.endm + +.macro lpf_8_wd6 + bl lpf_8_wd6_neon +.endm + +.macro lpf_8_wd4 + bl lpf_8_wd4_neon +.endm + +function lpf_v_4_8_neon + mov r12, lr + sub r10, r0, r1, lsl #1 + vld1.8 {d22}, [r10, :64], r1 // p1 + vld1.8 {d24}, [r0, :64], r1 // q0 + vld1.8 {d23}, [r10, :64], r1 // p0 + vld1.8 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + + lpf_8_wd4 + + sub r10, r0, r1, lsl #1 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_4_8_neon + mov r12, lr + sub r10, r0, #2 + add r0, r10, r1, lsl #2 + vld1.32 {d22[0]}, [r10], r1 + vld1.32 {d22[1]}, [r0], r1 + vld1.32 {d23[0]}, [r10], r1 + vld1.32 {d23[1]}, [r0], r1 + vld1.32 {d24[0]}, [r10], r1 + vld1.32 {d24[1]}, [r0], r1 + vld1.32 {d25[0]}, [r10], r1 + vld1.32 {d25[1]}, [r0], r1 + add r0, r0, #2 + + transpose_4x8b q11, q12, d22, d23, d24, d25 + + lpf_8_wd4 + + sub r10, r0, r1, lsl #3 + sub r10, r10, #2 + transpose_4x8b q11, q12, d22, d23, d24, d25 + add r0, r10, r1, lsl #2 + + vst1.32 {d22[0]}, [r10], r1 + vst1.32 {d22[1]}, [r0], r1 + vst1.32 {d23[0]}, [r10], r1 + vst1.32 {d23[1]}, [r0], r1 + vst1.32 {d24[0]}, [r10], r1 + vst1.32 {d24[1]}, [r0], r1 + vst1.32 {d25[0]}, [r10], r1 + vst1.32 {d25[1]}, [r0], r1 + add r0, r0, #2 + bx r12 +endfunc + +function lpf_v_6_8_neon + mov r12, lr + sub r10, r0, r1, lsl #1 + sub r10, r10, r1 + vld1.8 {d21}, [r10, :64], r1 // p2 + vld1.8 {d24}, [r0, :64], r1 // q0 + vld1.8 {d22}, [r10, :64], r1 // p1 + vld1.8 {d25}, [r0, :64], r1 // q1 + vld1.8 {d23}, [r10, :64], r1 // p0 + vld1.8 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + + lpf_8_wd6 + + sub r10, r0, r1, lsl #1 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_6_8_neon + mov r12, lr + sub r10, r0, #4 + add r0, r10, r1, lsl #2 + vld1.8 {d20}, [r10], r1 + vld1.8 {d24}, [r0], r1 + vld1.8 {d21}, [r10], r1 + vld1.8 {d25}, [r0], r1 + vld1.8 {d22}, [r10], r1 + vld1.8 {d26}, [r0], r1 + vld1.8 {d23}, [r10], r1 + vld1.8 {d27}, [r0], r1 + add r0, r0, #4 + + transpose_8x8b q10, q11, q12, q13, d20, d21, d22, d23, d24, d25, d26, d27 + + lpf_8_wd6 + + sub r10, r0, r1, lsl #3 + sub r10, r10, #2 + transpose_4x8b q11, q12, d22, d23, d24, d25 + add r0, r10, r1, lsl #2 + + vst1.32 {d22[0]}, [r10], r1 + vst1.32 {d22[1]}, [r0], r1 + vst1.32 {d23[0]}, [r10], r1 + vst1.32 {d23[1]}, [r0], r1 + vst1.32 {d24[0]}, [r10], r1 + vst1.32 {d24[1]}, [r0], r1 + vst1.32 {d25[0]}, [r10], r1 + vst1.32 {d25[1]}, [r0], r1 + add r0, r0, #2 + bx r12 +endfunc + +function lpf_v_8_8_neon + mov r12, lr + sub r10, r0, r1, lsl #2 + vld1.8 {d20}, [r10, :64], r1 // p3 + vld1.8 {d24}, [r0, :64], r1 // q0 + vld1.8 {d21}, [r10, :64], r1 // p2 + vld1.8 {d25}, [r0, :64], r1 // q1 + vld1.8 {d22}, [r10, :64], r1 // p1 + vld1.8 {d26}, [r0, :64], r1 // q2 + vld1.8 {d23}, [r10, :64], r1 // p0 + vld1.8 {d27}, [r0, :64], r1 // q3 + sub r0, r0, r1, lsl #2 + + lpf_8_wd8 + + sub r10, r0, r1, lsl #1 + sub r10, r10, r1 + vst1.8 {d21}, [r10, :64], r1 // p2 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d25}, [r0, :64], r1 // q1 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + bx r12 + +8: + sub r10, r0, r1, lsl #1 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_8_8_neon + mov r12, lr + sub r10, r0, #4 + add r0, r10, r1, lsl #2 + vld1.8 {d20}, [r10], r1 + vld1.8 {d24}, [r0], r1 + vld1.8 {d21}, [r10], r1 + vld1.8 {d25}, [r0], r1 + vld1.8 {d22}, [r10], r1 + vld1.8 {d26}, [r0], r1 + vld1.8 {d23}, [r10], r1 + vld1.8 {d27}, [r0], r1 + add r0, r0, #4 + + transpose_8x8b q10, q11, q12, q13, d20, d21, d22, d23, d24, d25, d26, d27 + + lpf_8_wd8 + + sub r10, r0, r1, lsl #3 + sub r10, r10, #4 + transpose_8x8b q10, q11, q12, q13, d20, d21, d22, d23, d24, d25, d26, d27 + add r0, r10, r1, lsl #2 + + vst1.8 {d20}, [r10], r1 + vst1.8 {d24}, [r0], r1 + vst1.8 {d21}, [r10], r1 + vst1.8 {d25}, [r0], r1 + vst1.8 {d22}, [r10], r1 + vst1.8 {d26}, [r0], r1 + vst1.8 {d23}, [r10], r1 + vst1.8 {d27}, [r0], r1 + add r0, r0, #4 + bx r12 +8: + sub r10, r0, r1, lsl #3 + sub r10, r10, #2 + transpose_4x8b q11, q12, d22, d23, d24, d25 + add r0, r10, r1, lsl #2 + + vst1.32 {d22[0]}, [r10], r1 + vst1.32 {d22[1]}, [r0], r1 + vst1.32 {d23[0]}, [r10], r1 + vst1.32 {d23[1]}, [r0], r1 + vst1.32 {d24[0]}, [r10], r1 + vst1.32 {d24[1]}, [r0], r1 + vst1.32 {d25[0]}, [r10], r1 + vst1.32 {d25[1]}, [r0], r1 + add r0, r0, #2 + bx r12 +endfunc + +function lpf_v_16_8_neon + mov r12, lr + + sub r10, r0, r1, lsl #3 + add r10, r10, r1 + vld1.8 {d17}, [r10, :64], r1 // p6 + vld1.8 {d24}, [r0, :64], r1 // q0 + vld1.8 {d18}, [r10, :64], r1 // p5 + vld1.8 {d25}, [r0, :64], r1 // q1 + vld1.8 {d19}, [r10, :64], r1 // p4 + vld1.8 {d26}, [r0, :64], r1 // q2 + vld1.8 {d20}, [r10, :64], r1 // p3 + vld1.8 {d27}, [r0, :64], r1 // q3 + vld1.8 {d21}, [r10, :64], r1 // p2 + vld1.8 {d28}, [r0, :64], r1 // q4 + vld1.8 {d22}, [r10, :64], r1 // p1 + vld1.8 {d29}, [r0, :64], r1 // q5 + vld1.8 {d23}, [r10, :64], r1 // p0 + vld1.8 {d30}, [r0, :64], r1 // q6 + sub r0, r0, r1, lsl #3 + add r0, r0, r1 + + lpf_8_wd16 + + sub r10, r0, r1, lsl #2 + sub r10, r10, r1, lsl #1 + vst1.8 {d0}, [r10, :64], r1 // p5 + vst1.8 {d6}, [r0, :64], r1 // q0 + vst1.8 {d1}, [r10, :64], r1 // p4 + vst1.8 {d7}, [r0, :64], r1 // q1 + vst1.8 {d2}, [r10, :64], r1 // p3 + vst1.8 {d8}, [r0, :64], r1 // q2 + vst1.8 {d3}, [r10, :64], r1 // p2 + vst1.8 {d9}, [r0, :64], r1 // q3 + vst1.8 {d4}, [r10, :64], r1 // p1 + vst1.8 {d10}, [r0, :64], r1 // q4 + vst1.8 {d5}, [r10, :64], r1 // p0 + vst1.8 {d11}, [r0, :64], r1 // q5 + sub r0, r0, r1, lsl #2 + sub r0, r0, r1, lsl #1 + bx r12 +7: + sub r10, r0, r1 + sub r10, r10, r1, lsl #1 + vst1.8 {d21}, [r10, :64], r1 // p2 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d25}, [r0, :64], r1 // q1 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + bx r12 + +8: + sub r10, r0, r1, lsl #1 + vst1.8 {d22}, [r10, :64], r1 // p1 + vst1.8 {d24}, [r0, :64], r1 // q0 + vst1.8 {d23}, [r10, :64], r1 // p0 + vst1.8 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_16_8_neon + mov r12, lr + sub r10, r0, #8 + vld1.8 {d16}, [r10, :64], r1 + vld1.8 {d24}, [r0, :64], r1 + vld1.8 {d17}, [r10, :64], r1 + vld1.8 {d25}, [r0, :64], r1 + vld1.8 {d18}, [r10, :64], r1 + vld1.8 {d26}, [r0, :64], r1 + vld1.8 {d19}, [r10, :64], r1 + vld1.8 {d27}, [r0, :64], r1 + vld1.8 {d20}, [r10, :64], r1 + vld1.8 {d28}, [r0, :64], r1 + vld1.8 {d21}, [r10, :64], r1 + vld1.8 {d29}, [r0, :64], r1 + vld1.8 {d22}, [r10, :64], r1 + vld1.8 {d30}, [r0, :64], r1 + vld1.8 {d23}, [r10, :64], r1 + vld1.8 {d31}, [r0, :64], r1 + + transpose_8x8b q8, q9, q10, q11, d16, d17, d18, d19, d20, d21, d22, d23 + transpose_8x8b q12, q13, q14, q15, d24, d25, d26, d27, d28, d29, d30, d31 + + lpf_8_wd16 + + sub r0, r0, r1, lsl #3 + sub r10, r0, #8 + + transpose_8x8b q8, q0, q1, q2, d16, d17, d0, d1, d2, d3, d4, d5 + transpose_8x8b q3, q4, q5, q15, d6, d7, d8, d9, d10, d11, d30, d31 + + vst1.8 {d16}, [r10, :64], r1 + vst1.8 {d6}, [r0, :64], r1 + vst1.8 {d17}, [r10, :64], r1 + vst1.8 {d7}, [r0, :64], r1 + vst1.8 {d0}, [r10, :64], r1 + vst1.8 {d8}, [r0, :64], r1 + vst1.8 {d1}, [r10, :64], r1 + vst1.8 {d9}, [r0, :64], r1 + vst1.8 {d2}, [r10, :64], r1 + vst1.8 {d10}, [r0, :64], r1 + vst1.8 {d3}, [r10, :64], r1 + vst1.8 {d11}, [r0, :64], r1 + vst1.8 {d4}, [r10, :64], r1 + vst1.8 {d30}, [r0, :64], r1 + vst1.8 {d5}, [r10, :64], r1 + vst1.8 {d31}, [r0, :64], r1 + bx r12 + +7: + sub r10, r0, r1, lsl #3 + sub r10, r10, #4 + transpose_8x8b q10, q11, q12, q13, d20, d21, d22, d23, d24, d25, d26, d27 + add r0, r10, r1, lsl #2 + + vst1.8 {d20}, [r10], r1 + vst1.8 {d24}, [r0], r1 + vst1.8 {d21}, [r10], r1 + vst1.8 {d25}, [r0], r1 + vst1.8 {d22}, [r10], r1 + vst1.8 {d26}, [r0], r1 + vst1.8 {d23}, [r10], r1 + vst1.8 {d27}, [r0], r1 + add r0, r0, #4 + bx r12 +8: + sub r10, r0, r1, lsl #3 + sub r10, r10, #2 + transpose_4x8b q11, q12, d22, d23, d24, d25 + add r0, r10, r1, lsl #2 + + vst1.32 {d22[0]}, [r10], r1 + vst1.32 {d22[1]}, [r0], r1 + vst1.32 {d23[0]}, [r10], r1 + vst1.32 {d23[1]}, [r0], r1 + vst1.32 {d24[0]}, [r10], r1 + vst1.32 {d24[1]}, [r0], r1 + vst1.32 {d25[0]}, [r10], r1 + vst1.32 {d25[1]}, [r0], r1 + add r0, r0, #2 + bx r12 +endfunc + +// void dav1d_lpf_v_sb_y_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w) + +.macro lpf_func dir, type +function lpf_\dir\()_sb_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [r2] // vmask[0], vmask[1] +.ifc \type, y + ldr r2, [r2, #8] // vmask[2] +.endif + add r5, r5, #128 // Move to sharp part of lut +.ifc \type, y + orr r7, r7, r2 // vmask[1] |= vmask[2] +.endif +.ifc \dir, v + sub r4, r3, r4, lsl #2 +.else + sub r3, r3, #4 + lsl r4, r4, #2 +.endif + orr r6, r6, r7 // vmask[0] |= vmask[1] + +1: + tst r6, #0x03 +.ifc \dir, v + vld1.8 {d0}, [r4]! + vld1.8 {d1}, [r3]! +.else + vld2.32 {d0[0], d1[0]}, [r3], r4 + vld2.32 {d0[1], d1[1]}, [r3], r4 +.endif + beq 7f // if (!(vm & bits)) continue; + + vld1.8 {d5[]}, [r5] // sharp[0] + add r5, r5, #8 + vmov.i32 d2, #0xff + vdup.32 d13, r6 // vmask[0] + + vand d0, d0, d2 // Keep only lowest byte in each 32 bit word + vand d1, d1, d2 + vtst.8 d3, d1, d2 // Check for nonzero values in l[0][0] + vmov.i8 d4, #1 + vld1.8 {d6[]}, [r5] // sharp[1] + sub r5, r5, #8 + vbif d1, d0, d3 // if (!l[0][0]) L = l[offset][0] + vtst.32 d2, d1, d2 // L != 0 + vmul.i32 d1, d1, d4 // L +.ifc \type, y + vdup.32 d15, r2 // vmask[2] +.endif + vdup.32 d14, r7 // vmask[1] + vmov r10, r11, d2 + orrs r10, r10, r11 + beq 7f // if (!L) continue; + vneg.s8 d5, d5 // -sharp[0] + movrel_local r10, word_12 + vshr.u8 d12, d1, #4 // H + vld1.32 {d16}, [r10, :64] + vshl.s8 d3, d1, d5 // L >> sharp[0] +.ifc \type, y + vtst.32 d15, d15, d16 // if (vmask[2] & bits) +.endif + vmov.i8 d7, #2 + vmin.u8 d3, d3, d6 // imin(L >> sharp[0], sharp[1]) + vadd.i8 d0, d1, d7 // L + 2 + vmax.u8 d11, d3, d4 // imax(imin(), 1) = limit = I + vadd.u8 d0, d0, d0 // 2*(L + 2) + vtst.32 d14, d14, d16 // if (vmask[1] & bits) + vadd.i8 d10, d0, d11 // 2*(L + 2) + limit = E + vtst.32 d13, d13, d16 // if (vmask[0] & bits) + vand d13, d13, d2 // vmask[0] &= L != 0 + +.ifc \type, y + tst r2, #0x03 + beq 2f + // wd16 + bl lpf_\dir\()_16_8_neon + b 8f +2: +.endif + tst r7, #0x03 + beq 3f +.ifc \type, y + // wd8 + bl lpf_\dir\()_8_8_neon +.else + // wd6 + bl lpf_\dir\()_6_8_neon +.endif + b 8f +3: + // wd4 + bl lpf_\dir\()_4_8_neon +.ifc \dir, h + b 8f +7: + // For dir h, the functions above increment r0. + // If the whole function is skipped, increment it here instead. + add r0, r0, r1, lsl #3 +.else +7: +.endif +8: + lsrs r6, r6, #2 // vmask[0] >>= 2 + lsr r7, r7, #2 // vmask[1] >>= 2 +.ifc \type, y + lsr r2, r2, #2 // vmask[2] >>= 2 +.endif +.ifc \dir, v + add r0, r0, #8 +.else + // For dir h, r0 is returned incremented +.endif + bne 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +lpf_func v, y +lpf_func h, y +lpf_func v, uv +lpf_func h, uv + +const word_12, align=4 + .word 1, 2 +endconst diff --git a/third_party/dav1d/src/arm/32/loopfilter16.S b/third_party/dav1d/src/arm/32/loopfilter16.S new file mode 100644 index 0000000000..d7daf21f1a --- /dev/null +++ b/third_party/dav1d/src/arm/32/loopfilter16.S @@ -0,0 +1,859 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro loop_filter wd +function lpf_4_wd\wd\()_neon + vabd.u16 d0, d22, d23 // abs(p1 - p0) + vabd.u16 d1, d25, d24 // abs(q1 - q0) + vabd.u16 d2, d23, d24 // abs(p0 - q0) + vabd.u16 d3, d22, d25 // abs(p1 - q1) +.if \wd >= 6 + vabd.u16 d4, d21, d22 // abs(p2 - p1) + vabd.u16 d5, d26, d25 // abs(q2 - q1) +.endif +.if \wd >= 8 + vabd.u16 d6, d20, d21 // abs(p3 - p2) + vabd.u16 d7, d27, d26 // abs(q3 - q3) +.endif +.if \wd >= 6 + vmax.u16 d4, d4, d5 +.endif + vqadd.u16 d2, d2, d2 // abs(p0 - q0) * 2 +.if \wd >= 8 + vmax.u16 d6, d6, d7 +.endif + vshr.u16 d3, d3, #1 +.if \wd >= 8 + vmax.u16 d4, d4, d6 +.endif + vmax.u16 d0, d0, d1 // max(abs(p1 - p0), abs(q1 - q0)) + vqadd.u16 d2, d2, d3 // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 +.if \wd >= 6 + vmax.u16 d4, d0, d4 + vcge.u16 d1, d11, d4 // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I +.else + vcge.u16 d1, d11, d0 // max(abs(p1 - p0), abs(q1 - q0)) <= I +.endif + vcge.u16 d2, d10, d2 // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E + vand d1, d1, d2 // fm && wd >= 4 (implicit) +.if \wd >= 6 + vmov d14, d1 // fm && wd > 4 (implicit) +.endif +.if \wd >= 16 + vmov d15, d1 // fm && wd == 16 (implicit) +.endif + + vmov r10, r11, d1 + orrs r10, r10, r11 + beq 9f // if (!fm || wd < 4) return; + +.if \wd >= 6 + vmov.i16 d10, #1 + vabd.u16 d2, d21, d23 // abs(p2 - p0) + vabd.u16 d3, d22, d23 // abs(p1 - p0) + vabd.u16 d4, d25, d24 // abs(q1 - q0) + vabd.u16 d5, d26, d24 // abs(q2 - q0) + vdup.16 d9, r9 // bitdepth_min_8 +.if \wd >= 8 + vabd.u16 d6, d20, d23 // abs(p3 - p0) + vabd.u16 d7, d27, d24 // abs(q3 - q0) +.endif + vmax.u16 d2, d2, d3 + vmax.u16 d4, d4, d5 +.if \wd >= 8 + vmax.u16 d6, d6, d7 +.endif + vmax.u16 d2, d2, d4 + vshl.u16 d10, d10, d9 // F = 1 << bitdepth_min_8 +.if \wd >= 8 + vmax.u16 d2, d2, d6 +.endif + +.if \wd == 16 + vabd.u16 d3, d17, d23 // abs(p6 - p0) + vabd.u16 d4, d18, d23 // abs(p5 - p0) + vabd.u16 d5, d19, d23 // abs(p4 - p0) +.endif + vcge.u16 d2, d10, d2 // flat8in +.if \wd == 16 + vabd.u16 d6, d28, d24 // abs(q4 - q0) + vabd.u16 d7, d29, d24 // abs(q5 - q0) + vabd.u16 d8, d30, d24 // abs(q6 - q0) +.endif + vand d14, d2, d14 // flat8in && fm && wd > 4 + vbic d1, d1, d14 // fm && wd >= 4 && !flat8in +.if \wd == 16 + vmax.u16 d3, d3, d4 + vmax.u16 d5, d5, d6 +.endif + vmov r10, r11, d1 +.if \wd == 16 + vmax.u16 d7, d7, d8 + vmax.u16 d3, d3, d5 + vmax.u16 d3, d3, d7 + vcge.u16 d3, d10, d3 // flat8out +.endif + orrs r10, r10, r11 +.if \wd == 16 + vand d15, d15, d3 // flat8out && fm && wd == 16 + vand d15, d15, d14 // flat8out && flat8in && fm && wd == 16 + vbic d14, d14, d15 // flat8in && fm && wd >= 4 && !flat8out +.endif + beq 1f // skip wd == 4 case +.endif + + vdup.16 d3, r8 // bitdepth_max + vsub.u16 d2, d22, d25 // p1 - q1 + vshr.u16 d3, d3, #1 // 128 << bitdepth_min_8 - 1 + vcgt.u16 d0, d0, d12 // hev + vmvn d9, d3 // - 128 * (1 << bitdepth_min_8) + vmin.s16 d2, d2, d3 // iclip_diff(p1 - q1) + vmax.s16 d2, d2, d9 // iclip_diff(p1 - q1) + vand d4, d2, d0 // if (hev) iclip_diff(p1 - q1) + vsub.u16 d2, d24, d23 + vmov.i16 d6, #3 + vbic d0, d1, d0 // (fm && wd >= 4 && !hev) + vmul.i16 d2, d2, d6 + vmov.i16 d7, #4 + vadd.i16 d2, d2, d4 + vmin.s16 d2, d2, d3 // f = iclip_diff() + vmax.s16 d2, d2, d9 // f = iclip_diff() + vqadd.s16 d4, d7, d2 // f + 4 + vqadd.s16 d5, d6, d2 // f + 3 + vmin.s16 d4, d4, d3 // imin(f + 4, 128 << bitdepth_min_8 - 1) + vmin.s16 d5, d5, d3 // imin(f + 3, 128 << bitdepth_min_8 - 1) + vshr.s16 d4, d4, #3 // f1 + vshr.s16 d5, d5, #3 // f2 + vmov.i16 d9, #0 + vdup.16 d3, r8 // bitdepth_max + vqadd.s16 d2, d23, d5 // p0 + f2 + vqsub.s16 d6, d24, d4 // q0 - f1 + vrshr.s16 d4, d4, #1 // (f1 + 1) >> 1 + vmin.s16 d2, d2, d3 // out p0 = iclip_pixel() + vmin.s16 d6, d6, d3 // out q0 = iclip_pixel() + vmax.s16 d2, d2, d9 // out p0 = iclip_pixel() + vmax.s16 d6, d6, d9 // out q0 = iclip_pixel() + vbit d23, d2, d1 // if (fm && wd >= 4) + vbit d24, d6, d1 // if (fm && wd >= 4) + vqadd.s16 d2, d22, d4 // p1 + f + vqsub.s16 d6, d25, d4 // q1 - f + vmin.s16 d2, d2, d3 // out p1 = iclip_pixel() + vmin.s16 d6, d6, d3 // out q1 = iclip_pixel() + vmax.s16 d2, d2, d9 // out p1 = iclip_pixel() + vmax.s16 d6, d6, d9 // out q1 = iclip_pixel() + vbit d22, d2, d0 // if (fm && wd >= 4 && !hev) + vbit d25, d6, d0 // if (fm && wd >= 4 && !hev) +1: + +.if \wd == 6 + vmov r10, r11, d14 + orrs r10, r10, r11 + beq 2f // skip if there's no flat8in + + vadd.i16 d0, d21, d21 // p2 * 2 + vadd.i16 d2, d21, d22 // p2 + p1 + vadd.i16 d4, d22, d23 // p1 + p0 + vadd.i16 d6, d23, d24 // p0 + q0 + vadd.i16 d8, d0, d2 + vadd.i16 d10, d4, d6 + vadd.i16 d12, d24, d25 // q0 + q1 + vadd.i16 d8, d8, d10 + vsub.i16 d12, d12, d0 + vadd.i16 d10, d25, d26 // q1 + q2 + vrshr.u16 d0, d8, #3 // out p1 + + vadd.i16 d8, d8, d12 + vsub.i16 d10, d10, d2 + vadd.i16 d12, d26, d26 // q2 + q2 + vrshr.u16 d1, d8, #3 // out p0 + + vadd.i16 d8, d8, d10 + vsub.i16 d12, d12, d4 + vrshr.u16 d2, d8, #3 // out q0 + + vbit d22, d0, d14 // p1 if (flat8in) + vadd.i16 d8, d8, d12 + vbit d23, d1, d14 // p0 if (flat8in) + vrshr.u16 d3, d8, #3 // out q1 + vbit d24, d2, d14 // q0 if (flat8in) + vbit d25, d3, d14 // q1 if (flat8in) +.elseif \wd >= 8 + vmov r10, r11, d14 + orrs r10, r10, r11 +.if \wd == 8 + beq 8f // skip if there's no flat8in +.else + beq 2f // skip if there's no flat8in +.endif + + vadd.i16 d0, d20, d21 // p3 + p2 + vadd.i16 d2, d22, d25 // p1 + q1 + vadd.i16 d4, d20, d22 // p3 + p1 + vadd.i16 d6, d23, d26 // p0 + q2 + vadd.i16 d8, d0, d0 // 2 * (p3 + p2) + vadd.i16 d9, d23, d24 // p0 + q0 + vadd.i16 d8, d8, d4 // + p3 + p1 + vsub.i16 d2, d2, d0 // p1 + q1 - p3 - p2 + vadd.i16 d8, d8, d9 // + p0 + q0 + vsub.i16 d6, d6, d4 // p0 + q2 - p3 - p1 + vrshr.u16 d10, d8, #3 // out p2 + + vadd.i16 d8, d8, d2 + vadd.i16 d0, d20, d23 // p3 + p0 + vadd.i16 d2, d24, d27 // q0 + q3 + vrshr.u16 d11, d8, #3 // out p1 + + vadd.i16 d8, d8, d6 + vsub.i16 d2, d2, d0 // q0 + q3 - p3 - p0 + vadd.i16 d4, d21, d24 // p2 + q0 + vadd.i16 d6, d25, d27 // q1 + q3 + vrshr.u16 d12, d8, #3 // out p0 + + vadd.i16 d8, d8, d2 + vsub.i16 d6, d6, d4 // q1 + q3 - p2 - q0 + vadd.i16 d0, d22, d25 // p1 + q1 + vadd.i16 d2, d26, d27 // q2 + q3 + vrshr.u16 d13, d8, #3 // out q0 + + vadd.i16 d8, d8, d6 + vsub.i16 d2, d2, d0 // q2 + q3 - p1 - q1 + vrshr.u16 d0, d8, #3 // out q1 + + vadd.i16 d8, d8, d2 + + vbit d21, d10, d14 + vbit d22, d11, d14 + vbit d23, d12, d14 + vrshr.u16 d1, d8, #3 // out q2 + vbit d24, d13, d14 + vbit d25, d0, d14 + vbit d26, d1, d14 +.endif +2: +.if \wd == 16 + vmov r10, r11, d15 + orrs r10, r10, r11 + bne 1f // check if flat8out is needed + vmov r10, r11, d14 + orrs r10, r10, r11 + beq 8f // if there was no flat8in, just write the inner 4 pixels + b 7f // if flat8in was used, write the inner 6 pixels +1: + + vadd.i16 d2, d17, d17 // p6 + p6 + vadd.i16 d4, d17, d18 // p6 + p5 + vadd.i16 d6, d17, d19 // p6 + p4 + vadd.i16 d8, d17, d20 // p6 + p3 + vadd.i16 d12, d2, d4 + vadd.i16 d10, d6, d8 + vadd.i16 d6, d17, d21 // p6 + p2 + vadd.i16 d12, d12, d10 + vadd.i16 d8, d17, d22 // p6 + p1 + vadd.i16 d10, d18, d23 // p5 + p0 + vadd.i16 d6, d6, d8 + vadd.i16 d8, d19, d24 // p4 + q0 + vadd.i16 d12, d12, d6 + vadd.i16 d10, d10, d8 + vadd.i16 d6, d20, d25 // p3 + q1 + vadd.i16 d12, d12, d10 + vsub.i16 d6, d6, d2 + vadd.i16 d2, d21, d26 // p2 + q2 + vrshr.u16 d0, d12, #4 // out p5 + vadd.i16 d12, d12, d6 // - (p6 + p6) + (p3 + q1) + vsub.i16 d2, d2, d4 + vadd.i16 d4, d22, d27 // p1 + q3 + vadd.i16 d6, d17, d19 // p6 + p4 + vrshr.u16 d1, d12, #4 // out p4 + vadd.i16 d12, d12, d2 // - (p6 + p5) + (p2 + q2) + vsub.i16 d4, d4, d6 + vadd.i16 d6, d23, d28 // p0 + q4 + vadd.i16 d8, d17, d20 // p6 + p3 + vrshr.u16 d2, d12, #4 // out p3 + vadd.i16 d12, d12, d4 // - (p6 + p4) + (p1 + q3) + vsub.i16 d6, d6, d8 + vadd.i16 d8, d24, d29 // q0 + q5 + vadd.i16 d4, d17, d21 // p6 + p2 + vrshr.u16 d3, d12, #4 // out p2 + vadd.i16 d12, d12, d6 // - (p6 + p3) + (p0 + q4) + vsub.i16 d8, d8, d4 + vadd.i16 d6, d25, d30 // q1 + q6 + vadd.i16 d10, d17, d22 // p6 + p1 + vrshr.u16 d4, d12, #4 // out p1 + vadd.i16 d12, d12, d8 // - (p6 + p2) + (q0 + q5) + vsub.i16 d6, d6, d10 + vadd.i16 d8, d26, d30 // q2 + q6 + vbif d0, d18, d15 // out p5 + vadd.i16 d10, d18, d23 // p5 + p0 + vrshr.u16 d5, d12, #4 // out p0 + vadd.i16 d12, d12, d6 // - (p6 + p1) + (q1 + q6) + vsub.i16 d8, d8, d10 + vadd.i16 d10, d27, d30 // q3 + q6 + vbif d1, d19, d15 // out p4 + vadd.i16 d18, d19, d24 // p4 + q0 + vrshr.u16 d6, d12, #4 // out q0 + vadd.i16 d12, d12, d8 // - (p5 + p0) + (q2 + q6) + vsub.i16 d10, d10, d18 + vadd.i16 d8, d28, d30 // q4 + q6 + vbif d2, d20, d15 // out p3 + vadd.i16 d18, d20, d25 // p3 + q1 + vrshr.u16 d7, d12, #4 // out q1 + vadd.i16 d12, d12, d10 // - (p4 + q0) + (q3 + q6) + vsub.i16 d18, d8, d18 + vadd.i16 d10, d29, d30 // q5 + q6 + vbif d3, d21, d15 // out p2 + vadd.i16 d20, d21, d26 // p2 + q2 + vrshr.u16 d8, d12, #4 // out q2 + vadd.i16 d12, d12, d18 // - (p3 + q1) + (q4 + q6) + vsub.i16 d10, d10, d20 + vadd.i16 d18, d30, d30 // q6 + q6 + vbif d4, d22, d15 // out p1 + vadd.i16 d20, d22, d27 // p1 + q3 + vrshr.u16 d9, d12, #4 // out q3 + vadd.i16 d12, d12, d10 // - (p2 + q2) + (q5 + q6) + vsub.i16 d18, d18, d20 + vbif d5, d23, d15 // out p0 + vrshr.u16 d10, d12, #4 // out q4 + vadd.i16 d12, d12, d18 // - (p1 + q3) + (q6 + q6) + vrshr.u16 d11, d12, #4 // out q5 + vbif d6, d24, d15 // out q0 + vbif d7, d25, d15 // out q1 + vbif d8, d26, d15 // out q2 + vbif d9, d27, d15 // out q3 + vbif d10, d28, d15 // out q4 + vbif d11, d29, d15 // out q5 +.endif + + bx lr +.if \wd == 16 +7: + // Return to a shorter epilogue, writing only the inner 6 pixels + bx r6 +.endif +.if \wd >= 8 +8: + // Return to a shorter epilogue, writing only the inner 4 pixels + bx r7 +.endif +9: + // Return directly without writing back any pixels + bx r12 +endfunc +.endm + +loop_filter 16 +loop_filter 8 +loop_filter 6 +loop_filter 4 + +.macro lpf_4_wd16 + adr r6, 7f + CONFIG_THUMB + adr r7, 8f + CONFIG_THUMB + bl lpf_4_wd16_neon +.endm + +.macro lpf_4_wd8 + adr r7, 8f + CONFIG_THUMB + bl lpf_4_wd8_neon +.endm + +.macro lpf_4_wd6 + bl lpf_4_wd6_neon +.endm + +.macro lpf_4_wd4 + bl lpf_4_wd4_neon +.endm + +function lpf_v_4_4_neon + mov r12, lr + sub r10, r0, r1, lsl #1 + vld1.16 {d22}, [r10, :64], r1 // p1 + vld1.16 {d24}, [r0, :64], r1 // q0 + vld1.16 {d23}, [r10, :64], r1 // p0 + vld1.16 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + + lpf_4_wd4 + + sub r10, r0, r1, lsl #1 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_4_4_neon + mov r12, lr + sub r10, r0, #4 + add r0, r10, r1, lsl #1 + vld1.16 {d22}, [r10], r1 + vld1.16 {d24}, [r0], r1 + vld1.16 {d23}, [r10], r1 + vld1.16 {d25}, [r0], r1 + add r0, r0, #4 + + transpose_4x4h q11, q12, d22, d23, d24, d25 + + lpf_4_wd4 + + sub r10, r0, r1, lsl #2 + sub r10, r10, #4 + transpose_4x4h q11, q12, d22, d23, d24, d25 + add r0, r10, r1, lsl #1 + + vst1.16 {d22}, [r10], r1 + vst1.16 {d24}, [r0], r1 + vst1.16 {d23}, [r10], r1 + vst1.16 {d25}, [r0], r1 + add r0, r0, #4 + bx r12 +endfunc + +function lpf_v_6_4_neon + mov r12, lr + sub r10, r0, r1, lsl #1 + sub r10, r10, r1 + vld1.16 {d21}, [r10, :64], r1 // p2 + vld1.16 {d24}, [r0, :64], r1 // q0 + vld1.16 {d22}, [r10, :64], r1 // p1 + vld1.16 {d25}, [r0, :64], r1 // q1 + vld1.16 {d23}, [r10, :64], r1 // p0 + vld1.16 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + + lpf_4_wd6 + + sub r10, r0, r1, lsl #1 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_6_4_neon + mov r12, lr + sub r10, r0, #8 + vld1.16 {d20}, [r10, :64], r1 + vld1.16 {d24}, [r0, :64], r1 + vld1.16 {d21}, [r10, :64], r1 + vld1.16 {d25}, [r0, :64], r1 + vld1.16 {d22}, [r10, :64], r1 + vld1.16 {d26}, [r0, :64], r1 + vld1.16 {d23}, [r10, :64], r1 + vld1.16 {d27}, [r0, :64], r1 + + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + + lpf_4_wd6 + + sub r0, r0, #4 + transpose_4x4h q11, q12, d22, d23, d24, d25 + sub r10, r0, r1, lsl #2 + sub r0, r0, r1, lsl #1 + + vst1.16 {d22}, [r10], r1 + vst1.16 {d24}, [r0], r1 + vst1.16 {d23}, [r10], r1 + vst1.16 {d25}, [r0], r1 + add r0, r0, #4 + bx r12 +endfunc + +function lpf_v_8_4_neon + mov r12, lr + sub r10, r0, r1, lsl #2 + vld1.16 {d20}, [r10, :64], r1 // p3 + vld1.16 {d24}, [r0, :64], r1 // q0 + vld1.16 {d21}, [r10, :64], r1 // p2 + vld1.16 {d25}, [r0, :64], r1 // q1 + vld1.16 {d22}, [r10, :64], r1 // p1 + vld1.16 {d26}, [r0, :64], r1 // q2 + vld1.16 {d23}, [r10, :64], r1 // p0 + vld1.16 {d27}, [r0, :64], r1 // q3 + sub r0, r0, r1, lsl #2 + + lpf_4_wd8 + + sub r10, r0, r1, lsl #1 + sub r10, r10, r1 + vst1.16 {d21}, [r10, :64], r1 // p2 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d25}, [r0, :64], r1 // q1 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + bx r12 + +8: + sub r10, r0, r1, lsl #1 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_8_4_neon + mov r12, lr + sub r10, r0, #8 + vld1.16 {d20}, [r10, :64], r1 + vld1.16 {d24}, [r0, :64], r1 + vld1.16 {d21}, [r10, :64], r1 + vld1.16 {d25}, [r0, :64], r1 + vld1.16 {d22}, [r10, :64], r1 + vld1.16 {d26}, [r0, :64], r1 + vld1.16 {d23}, [r10, :64], r1 + vld1.16 {d27}, [r0, :64], r1 + + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + + lpf_4_wd8 + + sub r0, r0, r1, lsl #2 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + sub r10, r0, #8 + + vst1.16 {d20}, [r10, :64], r1 + vst1.16 {d24}, [r0, :64], r1 + vst1.16 {d21}, [r10, :64], r1 + vst1.16 {d25}, [r0, :64], r1 + vst1.16 {d22}, [r10, :64], r1 + vst1.16 {d26}, [r0, :64], r1 + vst1.16 {d23}, [r10, :64], r1 + vst1.16 {d27}, [r0, :64], r1 + bx r12 +8: + sub r0, r0, #4 + transpose_4x4h q11, q12, d22, d23, d24, d25 + sub r10, r0, r1, lsl #2 + sub r0, r0, r1, lsl #1 + + vst1.16 {d22}, [r10], r1 + vst1.16 {d24}, [r0], r1 + vst1.16 {d23}, [r10], r1 + vst1.16 {d25}, [r0], r1 + add r0, r0, #4 + bx r12 +endfunc + +function lpf_v_16_4_neon + mov r12, lr + + sub r10, r0, r1, lsl #3 + add r10, r10, r1 + vld1.16 {d17}, [r10, :64], r1 // p6 + vld1.16 {d24}, [r0, :64], r1 // q0 + vld1.16 {d18}, [r10, :64], r1 // p5 + vld1.16 {d25}, [r0, :64], r1 // q1 + vld1.16 {d19}, [r10, :64], r1 // p4 + vld1.16 {d26}, [r0, :64], r1 // q2 + vld1.16 {d20}, [r10, :64], r1 // p3 + vld1.16 {d27}, [r0, :64], r1 // q3 + vld1.16 {d21}, [r10, :64], r1 // p2 + vld1.16 {d28}, [r0, :64], r1 // q4 + vld1.16 {d22}, [r10, :64], r1 // p1 + vld1.16 {d29}, [r0, :64], r1 // q5 + vld1.16 {d23}, [r10, :64], r1 // p0 + vld1.16 {d30}, [r0, :64], r1 // q6 + sub r0, r0, r1, lsl #3 + add r0, r0, r1 + + lpf_4_wd16 + + sub r10, r0, r1, lsl #2 + sub r10, r10, r1, lsl #1 + vst1.16 {d0}, [r10, :64], r1 // p5 + vst1.16 {d6}, [r0, :64], r1 // q0 + vst1.16 {d1}, [r10, :64], r1 // p4 + vst1.16 {d7}, [r0, :64], r1 // q1 + vst1.16 {d2}, [r10, :64], r1 // p3 + vst1.16 {d8}, [r0, :64], r1 // q2 + vst1.16 {d3}, [r10, :64], r1 // p2 + vst1.16 {d9}, [r0, :64], r1 // q3 + vst1.16 {d4}, [r10, :64], r1 // p1 + vst1.16 {d10}, [r0, :64], r1 // q4 + vst1.16 {d5}, [r10, :64], r1 // p0 + vst1.16 {d11}, [r0, :64], r1 // q5 + sub r0, r0, r1, lsl #2 + sub r0, r0, r1, lsl #1 + bx r12 +7: + sub r10, r0, r1 + sub r10, r10, r1, lsl #1 + vst1.16 {d21}, [r10, :64], r1 // p2 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d25}, [r0, :64], r1 // q1 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d26}, [r0, :64], r1 // q2 + sub r0, r0, r1, lsl #1 + sub r0, r0, r1 + bx r12 + +8: + sub r10, r0, r1, lsl #1 + vst1.16 {d22}, [r10, :64], r1 // p1 + vst1.16 {d24}, [r0, :64], r1 // q0 + vst1.16 {d23}, [r10, :64], r1 // p0 + vst1.16 {d25}, [r0, :64], r1 // q1 + sub r0, r0, r1, lsl #1 + bx r12 +endfunc + +function lpf_h_16_4_neon + mov r12, lr + sub r10, r0, #16 + sub r0, r0, #8 + vld1.16 {d16}, [r10, :64], r1 + vld1.16 {d20}, [r0, :64], r1 + vld1.16 {d17}, [r10, :64], r1 + vld1.16 {d21}, [r0, :64], r1 + vld1.16 {d18}, [r10, :64], r1 + vld1.16 {d22}, [r0, :64], r1 + vld1.16 {d19}, [r10, :64], r1 + vld1.16 {d23}, [r0, :64], r1 + sub r10, r10, r1, lsl #2 + sub r0, r0, r1, lsl #2 + add r10, r10, #16 + add r0, r0, #16 + vld1.16 {d24}, [r10, :64], r1 + vld1.16 {d28}, [r0, :64], r1 + vld1.16 {d25}, [r10, :64], r1 + vld1.16 {d29}, [r0, :64], r1 + vld1.16 {d26}, [r10, :64], r1 + vld1.16 {d30}, [r0, :64], r1 + vld1.16 {d27}, [r10, :64], r1 + vld1.16 {d31}, [r0, :64], r1 + sub r0, r0, #8 + + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + transpose_4x4h q14, q15, d28, d29, d30, d31 + + lpf_4_wd16 + + sub r0, r0, r1, lsl #2 + transpose_4x4h q8, q0, d16, d17, d0, d1 + transpose_4x4h q1, q2, d2, d3, d4, d5 + transpose_4x4h q3, q4, d6, d7, d8, d9 + transpose_4x4h q5, q15, d10, d11, d30, d31 + sub r10, r0, #16 + sub r0, r0, #8 + + vst1.16 {d16}, [r10, :64], r1 + vst1.16 {d2}, [r0, :64], r1 + vst1.16 {d17}, [r10, :64], r1 + vst1.16 {d3}, [r0, :64], r1 + vst1.16 {d0}, [r10, :64], r1 + vst1.16 {d4}, [r0, :64], r1 + vst1.16 {d1}, [r10, :64], r1 + vst1.16 {d5}, [r0, :64], r1 + sub r10, r10, r1, lsl #2 + sub r0, r0, r1, lsl #2 + add r10, r10, #16 + add r0, r0, #16 + vst1.16 {d6}, [r10, :64], r1 + vst1.16 {d10}, [r0, :64], r1 + vst1.16 {d7}, [r10, :64], r1 + vst1.16 {d11}, [r0, :64], r1 + vst1.16 {d8}, [r10, :64], r1 + vst1.16 {d30}, [r0, :64], r1 + vst1.16 {d9}, [r10, :64], r1 + vst1.16 {d31}, [r0, :64], r1 + sub r0, r0, #8 + + bx r12 + +7: + sub r0, r0, r1, lsl #2 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + sub r10, r0, #8 + + vst1.16 {d20}, [r10, :64], r1 + vst1.16 {d24}, [r0, :64], r1 + vst1.16 {d21}, [r10, :64], r1 + vst1.16 {d25}, [r0, :64], r1 + vst1.16 {d22}, [r10, :64], r1 + vst1.16 {d26}, [r0, :64], r1 + vst1.16 {d23}, [r10, :64], r1 + vst1.16 {d27}, [r0, :64], r1 + bx r12 +8: + sub r0, r0, #4 + transpose_4x4h q11, q12, d22, d23, d24, d25 + sub r10, r0, r1, lsl #2 + sub r0, r0, r1, lsl #1 + + vst1.16 {d22}, [r10], r1 + vst1.16 {d24}, [r0], r1 + vst1.16 {d23}, [r10], r1 + vst1.16 {d25}, [r0], r1 + add r0, r0, #4 + bx r12 +endfunc + +// void dav1d_lpf_v_sb_y_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w, +// const int bitdepth_max) + +.macro lpf_func dir, type +function lpf_\dir\()_sb_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldr r8, [sp, #112] // bitdepth_max; the 'w' parameter isn't loaded + sub sp, sp, #8 + clz r9, r8 + rsb r9, r9, #24 // bitdepth_min_8 + ldrd r6, r7, [r2] // vmask[0], vmask[1] +.ifc \type, y + ldr r2, [r2, #8] // vmask[2] +.endif + add r5, r5, #128 // Move to sharp part of lut +.ifc \type, y + orr r7, r7, r2 // vmask[1] |= vmask[2] +.endif +.ifc \dir, v + sub r4, r3, r4, lsl #2 +.else + sub r3, r3, #4 + lsl r4, r4, #2 +.endif + orr r6, r6, r7 // vmask[0] |= vmask[1] + +1: + tst r6, #0x01 + strd r6, r7, [sp] +.ifc \dir, v + ldrb r10, [r4], #4 + ldrb r11, [r3], #4 +.else + ldrb r10, [r3] + ldrb r11, [r3, #4] + add r3, r3, r4 +.endif + beq 7f // if (!(vm & bits)) continue; + + orrs r12, r10, r11 + vdup.16 d31, r9 // bitdepth_min_8 + beq 7f // if (!(l[0][0] | l[offset][0])) continue; + cmp r11, #0 // Check for nonzero values in l[0][0] + ldrb r6, [r5], #8 // sharp[0] + it eq + moveq r11, r10 // if (!l[0][0]) L = l[offset][0] + ldrb r12, [r5] // sharp[1] + lsr r6, r11, r6 // L >> sharp[0] + sub r5, r5, #8 + cmp r12, r6 + lsr r10, r11, #4 // H + add r11, r11, #2 // L + 2 + it lt + movlt r6, r12 // imin(L >> sharp[0], sharp[1]) + add r11, r11, r11 // 2*(L + 2) + cmp r6, #1 + lsl r10, r10, r9 // H << bitdepth_min_8 + it lt + movlt r6, #1 // imax(imin(), 1) = limit = I + vdup.16 d12, r10 // H << bitdepth_min_8 + add r11, r11, r6 // 2*(L + 2) + limit = E + lsl r6, r6, r9 // I << bitdepth_min_8 + lsl r11, r11, r9 // E << bitdepth_min_8 + vdup.16 d11, r6 // I << bitdepth_min_8 + vdup.16 d10, r11 // E << bitdepth_min_8 + +.ifc \type, y + tst r2, #0x01 + beq 2f + // wd16 + bl lpf_\dir\()_16_4_neon + b 8f +2: +.endif + tst r7, #0x01 + beq 3f +.ifc \type, y + // wd8 + bl lpf_\dir\()_8_4_neon +.else + // wd6 + bl lpf_\dir\()_6_4_neon +.endif + b 8f +3: + // wd4 + bl lpf_\dir\()_4_4_neon +.ifc \dir, h + b 8f +7: + // For dir h, the functions above increment r0. + // If the whole function is skipped, increment it here instead. + add r0, r0, r1, lsl #2 +.else +7: +.endif +8: + ldrd r6, r7, [sp] +.ifc \type, y + lsr r2, r2, #1 // vmask[2] >>= 1 +.endif +.ifc \dir, v + add r0, r0, #8 +.else + // For dir h, r0 is returned incremented +.endif + lsrs r6, r6, #1 // vmask[0] >>= 1 + lsr r7, r7, #1 // vmask[1] >>= 1 + bne 1b + + add sp, sp, #8 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +lpf_func v, y +lpf_func h, y +lpf_func v, uv +lpf_func h, uv diff --git a/third_party/dav1d/src/arm/32/looprestoration.S b/third_party/dav1d/src/arm/32/looprestoration.S new file mode 100644 index 0000000000..be5c658d6d --- /dev/null +++ b/third_party/dav1d/src/arm/32/looprestoration.S @@ -0,0 +1,791 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +const right_ext_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +right_ext_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void dav1d_wiener_filter_h_8bpc_neon(int16_t *dst, const pixel (*left)[4], +// const pixel *src, ptrdiff_t stride, +// const int16_t fh[8], intptr_t w, +// int h, enum LrEdgeFlags edges); +function wiener_filter_h_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + mov r8, r5 + vld1.16 {q0}, [r4, :128] + movw r9, #(1 << 14) - (1 << 2) + vdup.16 q14, r9 + vmov.s16 q15, #2048 + // Calculate mid_stride + add r10, r5, #7 + bic r10, r10, #7 + lsl r10, r10, #1 + + // Set up pointers for reading/writing alternate rows + add r12, r0, r10 + lsl r10, r10, #1 + add lr, r2, r3 + lsl r3, r3, #1 + + // Subtract the aligned width from mid_stride + add r11, r5, #7 + bic r11, r11, #7 + sub r10, r10, r11, lsl #1 + + // Subtract the number of pixels read from the source stride + add r11, r11, #8 + sub r3, r3, r11 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r1, #0 + bne 0f + // left == NULL + sub r2, r2, #3 + sub lr, lr, #3 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add r3, r3, #3 + + +1: // Loop vertically + vld1.8 {q2}, [r2]! + vld1.8 {q9}, [lr]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r1, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.32 {d3[1]}, [r1]! + // Move r2/lr back to account for the last 3 bytes we loaded earlier, + // which we'll shift out. + sub r2, r2, #3 + sub lr, lr, #3 + vld1.32 {d17[1]}, [r1]! + vext.8 q2, q1, q2, #13 + vext.8 q9, q8, q9, #13 + b 2f +0: + // !LR_HAVE_LEFT, fill q1 with the leftmost byte + // and shift q2 to have 3x the first byte at the front. + vdup.8 q1, d4[0] + vdup.8 q8, d18[0] + // Move r2 back to account for the last 3 bytes we loaded before, + // which we shifted out. + sub r2, r2, #3 + sub lr, lr, #3 + vext.8 q2, q1, q2, #13 + vext.8 q9, q8, q9, #13 + +2: + vmovl.u8 q1, d4 + vmovl.u8 q2, d5 + vmovl.u8 q8, d18 + vmovl.u8 q9, d19 + + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub r9, r5, #14 + ldrb r11, [r2, r9] + ldrb r9, [lr, r9] + // Fill q12/q13 with the right padding pixel + vdup.16 q12, r11 + vdup.16 q13, r9 +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #11 + bge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+3 pixels valid in q1-q2. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in q1/2.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel_local r4, right_ext_mask, -6 + sub r4, r4, r5, lsl #1 + vld1.8 {q10, q11}, [r4] + + vbit q1, q12, q10 + vbit q2, q12, q11 + vbit q8, q13, q10 + vbit q9, q13, q11 + +4: // Loop horizontally + vext.8 q11, q1, q2, #4 + vext.8 q5, q1, q2, #8 + vext.8 q10, q1, q2, #2 + vext.8 q6, q1, q2, #10 + vext.8 q7, q1, q2, #12 + vext.8 q4, q1, q2, #6 + vadd.i16 q5, q5, q11 + vadd.i16 q6, q6, q10 + vadd.i16 q7, q7, q1 + vmul.s16 q3, q4, d0[3] + vmla.s16 q3, q5, d1[0] + vmla.s16 q3, q6, d1[1] + vmla.s16 q3, q7, d1[2] + + vext.8 q4, q8, q9, #4 + vext.8 q6, q8, q9, #8 + vext.8 q11, q8, q9, #2 + vext.8 q7, q8, q9, #10 + vadd.i16 q6, q6, q4 + vext.8 q4, q8, q9, #12 + vext.8 q5, q8, q9, #6 + vadd.i16 q7, q7, q11 + vadd.i16 q4, q4, q8 + vmul.s16 q10, q5, d0[3] + vmla.s16 q10, q6, d1[0] + vmla.s16 q10, q7, d1[1] + vmla.s16 q10, q4, d1[2] + + vext.8 q1, q1, q2, #6 + vext.8 q8, q8, q9, #6 + vshl.s16 q1, q1, #7 + vshl.s16 q8, q8, #7 + vsub.s16 q1, q1, q14 + vsub.s16 q8, q8, q14 + vqadd.s16 q3, q3, q1 + vqadd.s16 q10, q10, q8 + vshr.s16 q3, q3, #3 + vshr.s16 q10, q10, #3 + vadd.s16 q3, q3, q15 + vadd.s16 q10, q10, q15 + subs r5, r5, #8 + vst1.16 {q3}, [r0, :128]! + vst1.16 {q10}, [r12, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vmov q1, q2 + vmov q8, q9 + vld1.8 {d4}, [r2]! + vld1.8 {d18}, [lr]! + vmovl.u8 q2, d4 + vmovl.u8 q9, d18 + bne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r10 + add r12, r12, r10 + add r2, r2, r3 + add lr, lr, r3 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_wiener_filter_v_8bpc_neon(pixel *dst, ptrdiff_t stride, +// const int16_t *mid, int w, int h, +// const int16_t fv[8], enum LrEdgeFlags edges, +// ptrdiff_t mid_stride); +function wiener_filter_v_8bpc_neon, export=1 + push {r4-r7,lr} + vpush {q4-q6} + ldrd r4, r5, [sp, #68] + ldrd r6, r7, [sp, #76] + mov lr, r4 + vld1.16 {q0}, [r5, :128] + + // Calculate the number of rows to move back when looping vertically + mov r12, r4 + tst r6, #4 // LR_HAVE_TOP + beq 0f + sub r2, r2, r7, lsl #1 + add r12, r12, #2 +0: + tst r6, #8 // LR_HAVE_BOTTOM + beq 1f + add r12, r12, #2 + +1: // Start of horizontal loop; start one vertical filter slice. + // Load rows into q8-q11 and pad properly. + tst r6, #4 // LR_HAVE_TOP + vld1.16 {q8}, [r2, :128], r7 + beq 2f + // LR_HAVE_TOP + vld1.16 {q10}, [r2, :128], r7 + vmov q9, q8 + vld1.16 {q11}, [r2, :128], r7 + b 3f +2: // !LR_HAVE_TOP + vmov q9, q8 + vmov q10, q8 + vmov q11, q8 + +3: + cmp r4, #4 + blt 5f + // Start filtering normally; fill in q12-q14 with unique rows. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + vld1.16 {q14}, [r2, :128], r7 + +4: +.macro filter compare + subs r4, r4, #1 + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + vadd.i16 q4, q10, q12 + vadd.i16 q5, q9, q13 + vadd.i16 q6, q8, q14 + vmull.s16 q2, d22, d0[3] + vmlal.s16 q2, d8, d1[0] + vmlal.s16 q2, d10, d1[1] + vmlal.s16 q2, d12, d1[2] + vmull.s16 q3, d23, d0[3] + vmlal.s16 q3, d9, d1[0] + vmlal.s16 q3, d11, d1[1] + vmlal.s16 q3, d13, d1[2] + vqrshrun.s32 d4, q2, #11 + vqrshrun.s32 d5, q3, #11 + vqmovun.s16 d4, q2 + vst1.8 {d4}, [r0, :64], r1 +.if \compare + cmp r4, #4 +.else + ble 9f +.endif + vmov q8, q9 + vmov q9, q10 + vmov q10, q11 + vmov q11, q12 + vmov q12, q13 + vmov q13, q14 +.endm + filter 1 + blt 7f + vld1.16 {q14}, [r2, :128], r7 + b 4b + +5: // Less than 4 rows in total; not all of q12-q13 are filled yet. + tst r6, #8 // LR_HAVE_BOTTOM + beq 6f + // LR_HAVE_BOTTOM + cmp r4, #2 + // We load at least 2 rows in all cases. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + bgt 53f // 3 rows in total + beq 52f // 2 rows in total +51: // 1 row in total, q11 already loaded, load edge into q12-q14. + vmov q13, q12 + b 8f +52: // 2 rows in total, q11 already loaded, load q12 with content data + // and 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vmov q15, q14 + b 8f +53: + // 3 rows in total, q11 already loaded, load q12 and q13 with content + // and 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vld1.16 {q15}, [r2, :128], r7 + vmov q1, q15 + b 8f + +6: + // !LR_HAVE_BOTTOM + cmp r4, #2 + bgt 63f // 3 rows in total + beq 62f // 2 rows in total +61: // 1 row in total, q11 already loaded, pad that into q12-q14. + vmov q12, q11 + vmov q13, q11 + vmov q14, q11 + b 8f +62: // 2 rows in total, q11 already loaded, load q12 and pad that into q12-q15. + vld1.16 {q12}, [r2, :128], r7 + vmov q13, q12 + vmov q14, q12 + vmov q15, q12 + b 8f +63: + // 3 rows in total, q11 already loaded, load q12 and q13 and pad q13 into q14-q15,q1. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + vmov q14, q13 + vmov q15, q13 + vmov q1, q13 + b 8f + +7: + // All registers up to q13 are filled already, 3 valid rows left. + // < 4 valid rows left; fill in padding and filter the last + // few rows. + tst r6, #8 // LR_HAVE_BOTTOM + beq 71f + // LR_HAVE_BOTTOM; load 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vld1.16 {q15}, [r2, :128], r7 + vmov q1, q15 + b 8f +71: + // !LR_HAVE_BOTTOM, pad 3 rows + vmov q14, q13 + vmov q15, q13 + vmov q1, q13 + +8: // At this point, all registers up to q14-15,q1 are loaded with + // edge/padding (depending on how many rows are left). + filter 0 // This branches to 9f when done + vmov q14, q15 + vmov q15, q1 + b 8b + +9: // End of one vertical slice. + subs r3, r3, #8 + ble 0f + // Move pointers back up to the top and loop horizontally. + mls r0, r1, lr, r0 + mls r2, r7, r12, r2 + add r0, r0, #8 + add r2, r2, #16 + mov r4, lr + b 1b + +0: + vpop {q4-q6} + pop {r4-r7,pc} +.purgem filter +endfunc + +#define SUM_STRIDE (384+16) + +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + add r5, r5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add r10, r0, #(4*SUM_STRIDE) // sumsq + add r11, r1, #(2*SUM_STRIDE) // sum + add r12, r3, r4 // src + lsl r4, r4, #1 + mov r9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add lr, r5, #7 + bic lr, lr, #7 + sub r9, r9, lr, lsl #1 + + // Store the width for the vertical loop + mov r8, r5 + + // Subtract the number of pixels read from the input from the stride + add lr, lr, #8 + sub r4, r4, lr + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r2, #0 + bne 0f + // left == NULL + sub r3, r3, #2 + sub r12, r12, #2 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 2 pixels from the src pointer, + // but shift it as if we had done that. + add r4, r4, #2 + + +1: // Loop vertically + vld1.8 {q0}, [r3]! + vld1.8 {q4}, [r12]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r2, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.32 {d3[]}, [r2]! + // Move r3/r12 back to account for the last 2 bytes we loaded earlier, + // which we'll shift out. + sub r3, r3, #2 + sub r12, r12, #2 + vld1.32 {d11[]}, [r2]! + vext.8 q0, q1, q0, #14 + vext.8 q4, q5, q4, #14 + b 2f +0: + // !LR_HAVE_LEFT, fill q1 with the leftmost byte + // and shift q0 to have 2x the first byte at the front. + vdup.8 q1, d0[0] + vdup.8 q5, d8[0] + // Move r3 back to account for the last 2 bytes we loaded before, + // which we shifted out. + sub r3, r3, #2 + sub r12, r12, #2 + vext.8 q0, q1, q0, #14 + vext.8 q4, q5, q4, #14 + +2: + vmull.u8 q1, d0, d0 + vmull.u8 q2, d1, d1 + vmull.u8 q5, d8, d8 + vmull.u8 q6, d9, d9 + + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub lr, r5, #(2 + 16 - 2 + 1) + ldrb r11, [r3, lr] + ldrb lr, [r12, lr] + // Fill q14/q15 with the right padding pixel + vdup.8 q14, r11 + vdup.8 q15, lr + // Restore r11 after using it for a temporary value + add r11, r1, #(2*SUM_STRIDE) +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #10 + bge 4f // If w >= 10, all used input pixels are valid + + // 1 <= w < 10, w pixels valid in q0. For w=9, this ends up called + // again; it's not strictly needed in those cases (we pad enough here), + // but keeping the code as simple as possible. + + // Insert padding in q0/4.b[w] onwards + movrel_local lr, right_ext_mask + sub lr, lr, r5 + vld1.8 {q13}, [lr] + + vbit q0, q14, q13 + vbit q4, q15, q13 + + // Update the precalculated squares + vmull.u8 q1, d0, d0 + vmull.u8 q2, d1, d1 + vmull.u8 q5, d8, d8 + vmull.u8 q6, d9, d9 + +4: // Loop horizontally + vext.8 d16, d0, d1, #1 + vext.8 d17, d0, d1, #2 + vext.8 d18, d8, d9, #1 + vext.8 d19, d8, d9, #2 + vaddl.u8 q3, d0, d16 + vaddw.u8 q3, q3, d17 + vaddl.u8 q7, d8, d18 + vaddw.u8 q7, q7, d19 + + vext.8 q8, q1, q2, #2 + vext.8 q9, q1, q2, #4 + vext.8 q10, q5, q6, #2 + vext.8 q11, q5, q6, #4 + + vaddl.u16 q12, d2, d16 + vaddl.u16 q13, d3, d17 + vaddw.u16 q12, q12, d18 + vaddw.u16 q13, q13, d19 + + vaddl.u16 q8, d10, d20 + vaddl.u16 q9, d11, d21 + vaddw.u16 q8, q8, d22 + vaddw.u16 q9, q9, d23 + + subs r5, r5, #8 + vst1.16 {q3}, [r1, :128]! + vst1.16 {q7}, [r11, :128]! + vst1.32 {q12, q13}, [r0, :128]! + vst1.32 {q8, q9}, [r10, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vld1.8 {d6}, [r3]! + vld1.8 {d14}, [r12]! + vmov q1, q2 + vmov q5, q6 + vext.8 q0, q0, q3, #8 + vext.8 q4, q4, q7, #8 + vmull.u8 q2, d6, d6 + vmull.u8 q6, d14, d14 + + bne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r9, lsl #1 + add r10, r10, r9, lsl #1 + add r1, r1, r9 + add r11, r11, r9 + add r3, r3, r4 + add r12, r12, r4 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_sgr_box5_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + add r5, r5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add r10, r0, #(4*SUM_STRIDE) // sumsq + add r11, r1, #(2*SUM_STRIDE) // sum + add r12, r3, r4 // src + lsl r4, r4, #1 + mov r9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add lr, r5, #7 + bic lr, lr, #7 + sub r9, r9, lr, lsl #1 + add lr, lr, #8 + sub r4, r4, lr + + // Store the width for the vertical loop + mov r8, r5 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r2, #0 + bne 0f + // left == NULL + sub r3, r3, #3 + sub r12, r12, #3 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add r4, r4, #3 + +1: // Loop vertically + vld1.8 {q0}, [r3]! + vld1.8 {q4}, [r12]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r2, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.32 {d3[]}, [r2]! + // Move r3/r12 back to account for the last 3 bytes we loaded earlier, + // which we'll shift out. + sub r3, r3, #3 + sub r12, r12, #3 + vld1.32 {d11[]}, [r2]! + vext.8 q0, q1, q0, #13 + vext.8 q4, q5, q4, #13 + b 2f +0: + // !LR_HAVE_LEFT, fill q1 with the leftmost byte + // and shift q0 to have 3x the first byte at the front. + vdup.8 q1, d0[0] + vdup.8 q5, d8[0] + // Move r3 back to account for the last 3 bytes we loaded before, + // which we shifted out. + sub r3, r3, #3 + sub r12, r12, #3 + vext.8 q0, q1, q0, #13 + vext.8 q4, q5, q4, #13 + +2: + vmull.u8 q1, d0, d0 + vmull.u8 q2, d1, d1 + vmull.u8 q5, d8, d8 + vmull.u8 q6, d9, d9 + + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub lr, r5, #(2 + 16 - 3 + 1) + ldrb r11, [r3, lr] + ldrb lr, [r12, lr] + // Fill q14/q15 with the right padding pixel + vdup.8 q14, r11 + vdup.8 q15, lr + // Restore r11 after using it for a temporary value + add r11, r1, #(2*SUM_STRIDE) +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #11 + bge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+1 pixels valid in q0. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in q0/4.b[w+1] onwards; fuse the +1 into the + // buffer pointer. + movrel_local lr, right_ext_mask, -1 + sub lr, lr, r5 + vld1.8 {q13}, [lr] + + vbit q0, q14, q13 + vbit q4, q15, q13 + + // Update the precalculated squares + vmull.u8 q1, d0, d0 + vmull.u8 q2, d1, d1 + vmull.u8 q5, d8, d8 + vmull.u8 q6, d9, d9 + +4: // Loop horizontally + vext.8 d16, d0, d1, #1 + vext.8 d17, d0, d1, #2 + vext.8 d18, d0, d1, #3 + vext.8 d19, d0, d1, #4 + vext.8 d20, d8, d9, #1 + vext.8 d21, d8, d9, #2 + vext.8 d22, d8, d9, #3 + vext.8 d23, d8, d9, #4 + vaddl.u8 q3, d0, d16 + vaddl.u8 q12, d17, d18 + vaddl.u8 q7, d8, d20 + vaddl.u8 q13, d21, d22 + vaddw.u8 q3, q3, d19 + vaddw.u8 q7, q7, d23 + vadd.u16 q3, q3, q12 + vadd.u16 q7, q7, q13 + + vext.8 q8, q1, q2, #2 + vext.8 q9, q1, q2, #4 + vext.8 q10, q1, q2, #6 + vext.8 q11, q1, q2, #8 + vaddl.u16 q12, d2, d16 + vaddl.u16 q13, d3, d17 + vaddl.u16 q8, d18, d20 + vaddl.u16 q9, d19, d21 + vaddw.u16 q12, q12, d22 + vaddw.u16 q13, q13, d23 + vadd.i32 q12, q12, q8 + vadd.i32 q13, q13, q9 + vext.8 q8, q5, q6, #2 + vext.8 q9, q5, q6, #4 + vext.8 q10, q5, q6, #6 + vext.8 q11, q5, q6, #8 + vaddl.u16 q1, d10, d16 + vaddl.u16 q5, d11, d17 + vaddl.u16 q8, d18, d20 + vaddl.u16 q9, d19, d21 + vaddw.u16 q1, q1, d22 + vaddw.u16 q5, q5, d23 + vadd.i32 q10, q1, q8 + vadd.i32 q11, q5, q9 + + subs r5, r5, #8 + vst1.16 {q3}, [r1, :128]! + vst1.16 {q7}, [r11, :128]! + vst1.32 {q12, q13}, [r0, :128]! + vst1.32 {q10, q11}, [r10, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vld1.8 {d6}, [r3]! + vld1.8 {d14}, [r12]! + vmov q1, q2 + vmov q5, q6 + vext.8 q0, q0, q3, #8 + vext.8 q4, q4, q7, #8 + vmull.u8 q2, d6, d6 + vmull.u8 q6, d14, d14 + bne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r9, lsl #1 + add r10, r10, r9, lsl #1 + add r1, r1, r9 + add r11, r11, r9 + add r3, r3, r4 + add r12, r12, r4 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +sgr_funcs 8 diff --git a/third_party/dav1d/src/arm/32/looprestoration16.S b/third_party/dav1d/src/arm/32/looprestoration16.S new file mode 100644 index 0000000000..d699617a87 --- /dev/null +++ b/third_party/dav1d/src/arm/32/looprestoration16.S @@ -0,0 +1,801 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +const right_ext_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +right_ext_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void dav1d_wiener_filter_h_16bpc_neon(int16_t *dst, const pixel (*left)[4], +// const pixel *src, ptrdiff_t stride, +// const int16_t fh[7], const intptr_t w, +// int h, enum LrEdgeFlags edges, +// const int bitdepth_max); +function wiener_filter_h_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + ldr r8, [sp, #116] // bitdepth_max + vld1.16 {q0}, [r4, :128] + clz r8, r8 + vmov.i32 q14, #1 + sub r9, r8, #38 // -(bitdepth + 6) + sub r8, r8, #25 // -round_bits_h + neg r9, r9 // bitdepth + 6 + vdup.32 q1, r9 + vdup.32 q13, r8 // -round_bits_h + vmov.i16 q15, #8192 + vshl.u32 q14, q14, q1 // 1 << (bitdepth + 6) + mov r8, r5 + // Calculate mid_stride + add r10, r5, #7 + bic r10, r10, #7 + lsl r10, r10, #1 + + // Set up pointers for reading/writing alternate rows + add r12, r0, r10 + lsl r10, r10, #1 + add lr, r2, r3 + lsl r3, r3, #1 + + // Subtract the aligned width from mid_stride + add r11, r5, #7 + bic r11, r11, #7 + sub r10, r10, r11, lsl #1 + + // Subtract the number of pixels read from the source stride + add r11, r11, #8 + sub r3, r3, r11, lsl #1 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r1, #0 + bne 0f + // left == NULL + sub r2, r2, #6 + sub lr, lr, #6 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add r3, r3, #6 + + +1: // Loop vertically + vld1.16 {q2, q3}, [r2]! + vld1.16 {q4, q5}, [lr]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r1, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.16 {d3}, [r1]! + // Move r2/lr back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub r2, r2, #6 + sub lr, lr, #6 + vld1.16 {d13}, [r1]! + vext.8 q3, q2, q3, #10 + vext.8 q2, q1, q2, #10 + vext.8 q5, q4, q5, #10 + vext.8 q4, q6, q4, #10 + b 2f +0: + // !LR_HAVE_LEFT, fill q1 with the leftmost pixel + // and shift q2/q3 to have 3x the first pixel at the front. + vdup.16 q1, d4[0] + vdup.16 q6, d8[0] + // Move r2 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub r2, r2, #6 + sub lr, lr, #6 + vext.8 q3, q2, q3, #10 + vext.8 q2, q1, q2, #10 + vext.8 q5, q4, q5, #10 + vext.8 q4, q6, q4, #10 + +2: + + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that pixel to pad with + // here since we can find it pretty easily from here. + sub r9, r5, #14 + lsl r9, r9, #1 + ldrh r11, [r2, r9] + ldrh r9, [lr, r9] + // Fill q11/q12 with the right padding pixel + vdup.16 q11, r11 + vdup.16 q12, r9 +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #11 + bge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+3 pixels valid in q2-q3. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in q2/3.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel_local r4, right_ext_mask, -6 + sub r4, r4, r5, lsl #1 + vld1.8 {q9, q10}, [r4] + + vbit q2, q11, q9 + vbit q3, q11, q10 + vbit q4, q12, q9 + vbit q5, q12, q10 + +4: // Loop horizontally + vext.8 q7, q2, q3, #4 + vext.8 q8, q2, q3, #8 + vext.8 q6, q2, q3, #2 + vext.8 q9, q2, q3, #10 + vadd.i16 q8, q8, q7 + vadd.i16 q9, q9, q6 + vext.8 q6, q2, q3, #12 + vext.8 q7, q2, q3, #6 + vadd.i16 q2, q2, q6 + vmull.s16 q6, d14, d0[3] + vmlal.s16 q6, d16, d1[0] + vmlal.s16 q6, d18, d1[1] + vmlal.s16 q6, d4, d1[2] + vmull.s16 q7, d15, d0[3] + vmlal.s16 q7, d17, d1[0] + vmlal.s16 q7, d19, d1[1] + vmlal.s16 q7, d5, d1[2] + + vext.8 q8, q4, q5, #4 + vext.8 q10, q4, q5, #8 + vext.8 q9, q4, q5, #2 + vext.8 q2, q4, q5, #10 + vadd.i16 q10, q10, q8 + vadd.i16 q2, q2, q9 + vext.8 q8, q4, q5, #12 + vext.8 q9, q4, q5, #6 + vadd.i16 q4, q4, q8 + vmull.s16 q8, d18, d0[3] + vmlal.s16 q8, d20, d1[0] + vmlal.s16 q8, d4, d1[1] + vmlal.s16 q8, d8, d1[2] + vmull.s16 q9, d19, d0[3] + vmlal.s16 q9, d21, d1[0] + vmlal.s16 q9, d5, d1[1] + vmlal.s16 q9, d9, d1[2] + + vmvn.i16 q10, #0x8000 // 0x7fff = (1 << 15) - 1 + vadd.i32 q6, q6, q14 + vadd.i32 q7, q7, q14 + vadd.i32 q8, q8, q14 + vadd.i32 q9, q9, q14 + vrshl.s32 q6, q6, q13 + vrshl.s32 q7, q7, q13 + vrshl.s32 q8, q8, q13 + vrshl.s32 q9, q9, q13 + vqmovun.s32 d12, q6 + vqmovun.s32 d13, q7 + vqmovun.s32 d14, q8 + vqmovun.s32 d15, q9 + vmin.u16 q6, q6, q10 + vmin.u16 q7, q7, q10 + vsub.i16 q6, q6, q15 + vsub.i16 q7, q7, q15 + subs r5, r5, #8 + vst1.16 {q6}, [r0, :128]! + vst1.16 {q7}, [r12, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vmov q2, q3 + vmov q4, q5 + vld1.16 {q3}, [r2]! + vld1.16 {q5}, [lr]! + bne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r10 + add r12, r12, r10 + add r2, r2, r3 + add lr, lr, r3 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_wiener_filter_v_16bpc_neon(pixel *dst, ptrdiff_t stride, +// const int16_t *mid, int w, int h, +// const int16_t fv[7], enum LrEdgeFlags edges, +// ptrdiff_t mid_stride, const int bitdepth_max); +function wiener_filter_v_16bpc_neon, export=1 + push {r4-r7,lr} + vpush {q4-q5} + ldrd r4, r5, [sp, #52] + ldrd r6, r7, [sp, #60] + ldr lr, [sp, #68] // bitdepth_max + vld1.16 {q0}, [r5, :128] + vdup.16 q5, lr + clz lr, lr + sub lr, lr, #11 // round_bits_v + vdup.32 q4, lr + mov lr, r4 + vneg.s32 q4, q4 // -round_bits_v + + // Calculate the number of rows to move back when looping vertically + mov r12, r4 + tst r6, #4 // LR_HAVE_TOP + beq 0f + sub r2, r2, r7, lsl #1 + add r12, r12, #2 +0: + tst r6, #8 // LR_HAVE_BOTTOM + beq 1f + add r12, r12, #2 + +1: // Start of horizontal loop; start one vertical filter slice. + // Load rows into q8-q11 and pad properly. + tst r6, #4 // LR_HAVE_TOP + vld1.16 {q8}, [r2, :128], r7 + beq 2f + // LR_HAVE_TOP + vld1.16 {q10}, [r2, :128], r7 + vmov q9, q8 + vld1.16 {q11}, [r2, :128], r7 + b 3f +2: // !LR_HAVE_TOP + vmov q9, q8 + vmov q10, q8 + vmov q11, q8 + +3: + cmp r4, #4 + blt 5f + // Start filtering normally; fill in q12-q14 with unique rows. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + vld1.16 {q14}, [r2, :128], r7 + +4: +.macro filter compare + subs r4, r4, #1 + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + vmull.s16 q2, d16, d0[0] + vmlal.s16 q2, d18, d0[1] + vmlal.s16 q2, d20, d0[2] + vmlal.s16 q2, d22, d0[3] + vmlal.s16 q2, d24, d1[0] + vmlal.s16 q2, d26, d1[1] + vmlal.s16 q2, d28, d1[2] + vmull.s16 q3, d17, d0[0] + vmlal.s16 q3, d19, d0[1] + vmlal.s16 q3, d21, d0[2] + vmlal.s16 q3, d23, d0[3] + vmlal.s16 q3, d25, d1[0] + vmlal.s16 q3, d27, d1[1] + vmlal.s16 q3, d29, d1[2] + vrshl.s32 q2, q2, q4 // round_bits_v + vrshl.s32 q3, q3, q4 + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vmin.u16 q2, q2, q5 // bitdepth_max + vst1.16 {q2}, [r0, :128], r1 +.if \compare + cmp r4, #4 +.else + ble 9f +.endif + vmov q8, q9 + vmov q9, q10 + vmov q10, q11 + vmov q11, q12 + vmov q12, q13 + vmov q13, q14 +.endm + filter 1 + blt 7f + vld1.16 {q14}, [r2, :128], r7 + b 4b + +5: // Less than 4 rows in total; not all of q12-q13 are filled yet. + tst r6, #8 // LR_HAVE_BOTTOM + beq 6f + // LR_HAVE_BOTTOM + cmp r4, #2 + // We load at least 2 rows in all cases. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + bgt 53f // 3 rows in total + beq 52f // 2 rows in total +51: // 1 row in total, q11 already loaded, load edge into q12-q14. + vmov q13, q12 + b 8f +52: // 2 rows in total, q11 already loaded, load q12 with content data + // and 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vmov q15, q14 + b 8f +53: + // 3 rows in total, q11 already loaded, load q12 and q13 with content + // and 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vld1.16 {q15}, [r2, :128], r7 + vmov q1, q15 + b 8f + +6: + // !LR_HAVE_BOTTOM + cmp r4, #2 + bgt 63f // 3 rows in total + beq 62f // 2 rows in total +61: // 1 row in total, q11 already loaded, pad that into q12-q14. + vmov q12, q11 + vmov q13, q11 + vmov q14, q11 + b 8f +62: // 2 rows in total, q11 already loaded, load q12 and pad that into q12-q15. + vld1.16 {q12}, [r2, :128], r7 + vmov q13, q12 + vmov q14, q12 + vmov q15, q12 + b 8f +63: + // 3 rows in total, q11 already loaded, load q12 and q13 and pad q13 into q14-q15,q1. + vld1.16 {q12}, [r2, :128], r7 + vld1.16 {q13}, [r2, :128], r7 + vmov q14, q13 + vmov q15, q13 + vmov q1, q13 + b 8f + +7: + // All registers up to q13 are filled already, 3 valid rows left. + // < 4 valid rows left; fill in padding and filter the last + // few rows. + tst r6, #8 // LR_HAVE_BOTTOM + beq 71f + // LR_HAVE_BOTTOM; load 2 rows of edge. + vld1.16 {q14}, [r2, :128], r7 + vld1.16 {q15}, [r2, :128], r7 + vmov q1, q15 + b 8f +71: + // !LR_HAVE_BOTTOM, pad 3 rows + vmov q14, q13 + vmov q15, q13 + vmov q1, q13 + +8: // At this point, all registers up to q14-q15,q1 are loaded with + // edge/padding (depending on how many rows are left). + filter 0 // This branches to 9f when done + vmov q14, q15 + vmov q15, q1 + b 8b + +9: // End of one vertical slice. + subs r3, r3, #8 + ble 0f + // Move pointers back up to the top and loop horizontally. + mls r0, r1, lr, r0 + mls r2, r7, r12, r2 + add r0, r0, #16 + add r2, r2, #16 + mov r4, lr + b 1b + +0: + vpop {q4-q5} + pop {r4-r7,pc} +.purgem filter +endfunc + +#define SUM_STRIDE (384+16) + +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + add r5, r5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add r10, r0, #(4*SUM_STRIDE) // sumsq + add r11, r1, #(2*SUM_STRIDE) // sum + add r12, r3, r4 // src + lsl r4, r4, #1 + mov r9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add lr, r5, #7 + bic lr, lr, #7 + sub r9, r9, lr, lsl #1 + + // Store the width for the vertical loop + mov r8, r5 + + // Subtract the number of pixels read from the input from the stride + add lr, lr, #8 + sub r4, r4, lr, lsl #1 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r2, #0 + bne 0f + // left == NULL + sub r3, r3, #4 + sub r12, r12, #4 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 2 pixels from the src pointer, + // but shift it as if we had done that. + add r4, r4, #4 + + +1: // Loop vertically + vld1.16 {q0, q1}, [r3]! + vld1.16 {q4, q5}, [r12]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r2, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.16 {d5}, [r2]! + // Move r3/r12 back to account for the last 2 pixels we loaded earlier, + // which we'll shift out. + sub r3, r3, #4 + sub r12, r12, #4 + vld1.16 {d13}, [r2]! + vext.8 q1, q0, q1, #12 + vext.8 q0, q2, q0, #12 + vext.8 q5, q4, q5, #12 + vext.8 q4, q6, q4, #12 + b 2f +0: + // !LR_HAVE_LEFT, fill q2 with the leftmost pixel + // and shift q0 to have 2x the first byte at the front. + vdup.16 q2, d0[0] + vdup.16 q6, d8[0] + // Move r3 back to account for the last 2 pixels we loaded before, + // which we shifted out. + sub r3, r3, #4 + sub r12, r12, #4 + vext.8 q1, q0, q1, #12 + vext.8 q0, q2, q0, #12 + vext.8 q5, q4, q5, #12 + vext.8 q4, q6, q4, #12 + +2: + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that pixel to pad with + // here since we can find it pretty easily from here. + sub lr, r5, #(2 + 16 - 2 + 1) + lsl lr, lr, #1 + ldrh r11, [r3, lr] + ldrh lr, [r12, lr] + // Fill q14/q15 with the right padding pixel + vdup.16 q14, r11 + vdup.16 q15, lr + // Restore r11 after using it for a temporary value + add r11, r1, #(2*SUM_STRIDE) +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #10 + bge 4f // If w >= 10, all used input pixels are valid + + // 1 <= w < 10, w pixels valid in q0-q1. For w=9, this ends up called + // again; it's not strictly needed in those cases (we pad enough here), + // but keeping the code as simple as possible. + + // Insert padding in q0/1.h[w] onwards + movrel_local lr, right_ext_mask + sub lr, lr, r5, lsl #1 + vld1.8 {q12, q13}, [lr] + + vbit q0, q14, q12 + vbit q1, q14, q13 + vbit q4, q15, q12 + vbit q5, q15, q13 + +4: // Loop horizontally + vext.8 q8, q0, q1, #2 + vext.8 q10, q4, q5, #2 + vext.8 q9, q0, q1, #4 + vext.8 q11, q4, q5, #4 + vadd.i16 q2, q0, q8 + vadd.i16 q3, q4, q10 + vadd.i16 q2, q2, q9 + vadd.i16 q3, q3, q11 + + vmull.u16 q6, d0, d0 + vmlal.u16 q6, d16, d16 + vmlal.u16 q6, d18, d18 + vmull.u16 q12, d8, d8 + vmlal.u16 q12, d20, d20 + vmlal.u16 q12, d22, d22 + vmull.u16 q7, d1, d1 + vmlal.u16 q7, d17, d17 + vmlal.u16 q7, d19, d19 + vmull.u16 q13, d9, d9 + vmlal.u16 q13, d21, d21 + vmlal.u16 q13, d23, d23 + subs r5, r5, #8 + vst1.16 {q2}, [r1, :128]! + vst1.16 {q3}, [r11, :128]! + vst1.32 {q6, q7}, [r0, :128]! + vst1.32 {q12, q13}, [r10, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vmov q0, q1 + vmov q4, q5 + vld1.16 {q1}, [r3]! + vld1.16 {q5}, [r12]! + + bne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r9, lsl #1 + add r10, r10, r9, lsl #1 + add r1, r1, r9 + add r11, r11, r9 + add r3, r3, r4 + add r12, r12, r4 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + add r5, r5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add r10, r0, #(4*SUM_STRIDE) // sumsq + add r11, r1, #(2*SUM_STRIDE) // sum + add r12, r3, r4 // src + lsl r4, r4, #1 + mov r9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add lr, r5, #7 + bic lr, lr, #7 + sub r9, r9, lr, lsl #1 + add lr, lr, #8 + sub r4, r4, lr, lsl #1 + + // Store the width for the vertical loop + mov r8, r5 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst r7, #1 // LR_HAVE_LEFT + beq 2f + // LR_HAVE_LEFT + cmp r2, #0 + bne 0f + // left == NULL + sub r3, r3, #6 + sub r12, r12, #6 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add r4, r4, #6 + +1: // Loop vertically + vld1.16 {q0, q1}, [r3]! + vld1.16 {q4, q5}, [r12]! + + tst r7, #1 // LR_HAVE_LEFT + beq 0f + cmp r2, #0 + beq 2f + // LR_HAVE_LEFT, left != NULL + vld1.16 {d5}, [r2]! + // Move r3/r12 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub r3, r3, #6 + sub r12, r12, #6 + vld1.16 {d13}, [r2]! + vext.8 q1, q0, q1, #10 + vext.8 q0, q2, q0, #10 + vext.8 q5, q4, q5, #10 + vext.8 q4, q6, q4, #10 + b 2f +0: + // !LR_HAVE_LEFT, fill q2 with the leftmost pixel + // and shift q0 to have 3x the first pixel at the front. + vdup.16 q2, d0[0] + vdup.16 q6, d8[0] + // Move r3 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub r3, r3, #6 + sub r12, r12, #6 + vext.8 q1, q0, q1, #10 + vext.8 q0, q2, q0, #10 + vext.8 q5, q4, q5, #10 + vext.8 q4, q6, q4, #10 + +2: + tst r7, #2 // LR_HAVE_RIGHT + bne 4f + // If we'll need to pad the right edge, load that pixel to pad with + // here since we can find it pretty easily from here. + sub lr, r5, #(2 + 16 - 3 + 1) + lsl lr, lr, #1 + ldrh r11, [r3, lr] + ldrh lr, [r12, lr] + // Fill q14/q15 with the right padding pixel + vdup.16 q14, r11 + vdup.16 q15, lr + // Restore r11 after using it for a temporary value + add r11, r1, #(2*SUM_STRIDE) +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp r5, #11 + bge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+1 pixels valid in q0-q1. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in q0/1.h[w+1] onwards; fuse the +1 into the + // buffer pointer. + movrel_local lr, right_ext_mask, -2 + sub lr, lr, r5, lsl #1 + vld1.8 {q12, q13}, [lr] + + vbit q0, q14, q12 + vbit q1, q14, q13 + vbit q4, q15, q12 + vbit q5, q15, q13 + +4: // Loop horizontally + vext.8 q8, q0, q1, #2 + vext.8 q10, q4, q5, #2 + vext.8 q9, q0, q1, #4 + vext.8 q11, q4, q5, #4 + vadd.i16 q2, q0, q8 + vadd.i16 q3, q4, q10 + vadd.i16 q2, q2, q9 + vadd.i16 q3, q3, q11 + + vmull.u16 q6, d0, d0 + vmlal.u16 q6, d16, d16 + vmlal.u16 q6, d18, d18 + vmull.u16 q12, d8, d8 + vmlal.u16 q12, d20, d20 + vmlal.u16 q12, d22, d22 + vmull.u16 q7, d1, d1 + vmlal.u16 q7, d17, d17 + vmlal.u16 q7, d19, d19 + vmull.u16 q13, d9, d9 + vmlal.u16 q13, d21, d21 + vmlal.u16 q13, d23, d23 + + vext.8 q8, q0, q1, #6 + vext.8 q10, q4, q5, #6 + vext.8 q9, q0, q1, #8 + vext.8 q11, q4, q5, #8 + vadd.i16 q2, q2, q8 + vadd.i16 q3, q3, q10 + vadd.i16 q2, q2, q9 + vadd.i16 q3, q3, q11 + + vmlal.u16 q6, d16, d16 + vmlal.u16 q6, d1, d1 + vmlal.u16 q12, d20, d20 + vmlal.u16 q12, d9, d9 + vmlal.u16 q7, d17, d17 + vmlal.u16 q7, d19, d19 + vmlal.u16 q13, d21, d21 + vmlal.u16 q13, d23, d23 + + subs r5, r5, #8 + vst1.16 {q2}, [r1, :128]! + vst1.16 {q3}, [r11, :128]! + vst1.32 {q6, q7}, [r0, :128]! + vst1.32 {q12, q13}, [r10, :128]! + + ble 9f + tst r7, #2 // LR_HAVE_RIGHT + vmov q0, q1 + vmov q4, q5 + vld1.16 {q1}, [r3]! + vld1.16 {q5}, [r12]! + bne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs r6, r6, #2 + ble 0f + // Jump to the next row and loop horizontally + add r0, r0, r9, lsl #1 + add r10, r10, r9, lsl #1 + add r1, r1, r9 + add r11, r11, r9 + add r3, r3, r4 + add r12, r12, r4 + mov r5, r8 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +sgr_funcs 16 diff --git a/third_party/dav1d/src/arm/32/looprestoration_common.S b/third_party/dav1d/src/arm/32/looprestoration_common.S new file mode 100644 index 0000000000..b080bb5115 --- /dev/null +++ b/third_party/dav1d/src/arm/32/looprestoration_common.S @@ -0,0 +1,453 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define SUM_STRIDE (384+16) + +// void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_v_neon, export=1 + push {r4-r9,lr} + ldr r4, [sp, #28] + add r12, r3, #2 // Number of output rows to move back + mov lr, r3 // Number of input rows to move back + add r2, r2, #2 // Actual summed width + mov r7, #(4*SUM_STRIDE) // sumsq stride + mov r8, #(2*SUM_STRIDE) // sum stride + sub r0, r0, #(4*SUM_STRIDE) // sumsq -= stride + sub r1, r1, #(2*SUM_STRIDE) // sum -= stride + + tst r4, #4 // LR_HAVE_TOP + beq 0f + // If have top, read from row -2. + sub r5, r0, #(4*SUM_STRIDE) + sub r6, r1, #(2*SUM_STRIDE) + add lr, lr, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add r5, r0, #(4*SUM_STRIDE) + add r6, r1, #(2*SUM_STRIDE) +1: + + tst r4, #8 // LR_HAVE_BOTTOM + beq 1f + // LR_HAVE_BOTTOM + add r3, r3, #2 // Sum all h+2 lines with the main loop + add lr, lr, #2 +1: + mov r9, r3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into q8-q13 and q0-q2 taking top + // padding into consideration. + tst r4, #4 // LR_HAVE_TOP + vld1.32 {q8, q9}, [r5, :128], r7 + vld1.16 {q0}, [r6, :128], r8 + beq 2f + // LR_HAVE_TOP + vld1.32 {q10, q11}, [r5, :128], r7 + vld1.16 {q1}, [r6, :128], r8 + vld1.32 {q12, q13}, [r5, :128], r7 + vld1.16 {q2}, [r6, :128], r8 + b 3f +2: // !LR_HAVE_TOP + vmov q10, q8 + vmov q11, q9 + vmov q1, q0 + vmov q12, q8 + vmov q13, q9 + vmov q2, q0 + +3: + subs r3, r3, #1 +.macro add3 + vadd.i32 q8, q8, q10 + vadd.i32 q9, q9, q11 + vadd.i16 q0, q0, q1 + vadd.i32 q8, q8, q12 + vadd.i32 q9, q9, q13 + vadd.i16 q0, q0, q2 + vst1.32 {q8, q9}, [r0, :128], r7 + vst1.16 {q0}, [r1, :128], r8 +.endm + add3 + vmov q8, q10 + vmov q9, q11 + vmov q0, q1 + vmov q10, q12 + vmov q11, q13 + vmov q1, q2 + ble 4f + vld1.32 {q12, q13}, [r5, :128], r7 + vld1.16 {q2}, [r6, :128], r8 + b 3b + +4: + tst r4, #8 // LR_HAVE_BOTTOM + bne 5f + // !LR_HAVE_BOTTOM + // Produce two more rows, extending the already loaded rows. + add3 + vmov q8, q10 + vmov q9, q11 + vmov q0, q1 + add3 + +5: // End of one vertical slice. + subs r2, r2, #8 + ble 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + mls r5, r7, lr, r5 + mls r6, r8, lr, r6 + // Output pointers + mls r0, r7, r12, r0 + mls r1, r8, r12, r1 + add r0, r0, #32 + add r1, r1, #16 + add r5, r5, #32 + add r6, r6, #16 + mov r3, r9 + b 1b + +0: + pop {r4-r9,pc} +.purgem add3 +endfunc + +// void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_v_neon, export=1 + push {r4-r9,lr} + vpush {q5-q7} + ldr r4, [sp, #76] + add r12, r3, #2 // Number of output rows to move back + mov lr, r3 // Number of input rows to move back + add r2, r2, #8 // Actual summed width + mov r7, #(4*SUM_STRIDE) // sumsq stride + mov r8, #(2*SUM_STRIDE) // sum stride + sub r0, r0, #(4*SUM_STRIDE) // sumsq -= stride + sub r1, r1, #(2*SUM_STRIDE) // sum -= stride + + tst r4, #4 // LR_HAVE_TOP + beq 0f + // If have top, read from row -2. + sub r5, r0, #(4*SUM_STRIDE) + sub r6, r1, #(2*SUM_STRIDE) + add lr, lr, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add r5, r0, #(4*SUM_STRIDE) + add r6, r1, #(2*SUM_STRIDE) +1: + + tst r4, #8 // LR_HAVE_BOTTOM + beq 0f + // LR_HAVE_BOTTOM + add r3, r3, #2 // Handle h+2 lines with the main loop + add lr, lr, #2 + b 1f +0: + // !LR_HAVE_BOTTOM + sub r3, r3, #1 // Handle h-1 lines with the main loop +1: + mov r9, r3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into q6-q15 and q0-q3,q5 taking top + // padding into consideration. + tst r4, #4 // LR_HAVE_TOP + vld1.32 {q6, q7}, [r5, :128], r7 + vld1.16 {q0}, [r6, :128], r8 + beq 2f + // LR_HAVE_TOP + vld1.32 {q10, q11}, [r5, :128], r7 + vld1.16 {q2}, [r6, :128], r8 + vmov q8, q6 + vmov q9, q7 + vmov q1, q0 + vld1.32 {q12, q13}, [r5, :128], r7 + vld1.16 {q3}, [r6, :128], r8 + b 3f +2: // !LR_HAVE_TOP + vmov q8, q6 + vmov q9, q7 + vmov q1, q0 + vmov q10, q6 + vmov q11, q7 + vmov q2, q0 + vmov q12, q6 + vmov q13, q7 + vmov q3, q0 + +3: + cmp r3, #0 + beq 4f + vld1.32 {q14, q15}, [r5, :128], r7 + vld1.16 {q5}, [r6, :128], r8 + +3: + // Start of vertical loop + subs r3, r3, #2 +.macro add5 + vadd.i32 q6, q6, q8 + vadd.i32 q7, q7, q9 + vadd.i16 q0, q0, q1 + vadd.i32 q6, q6, q10 + vadd.i32 q7, q7, q11 + vadd.i16 q0, q0, q2 + vadd.i32 q6, q6, q12 + vadd.i32 q7, q7, q13 + vadd.i16 q0, q0, q3 + vadd.i32 q6, q6, q14 + vadd.i32 q7, q7, q15 + vadd.i16 q0, q0, q5 + vst1.32 {q6, q7}, [r0, :128], r7 + vst1.16 {q0}, [r1, :128], r8 +.endm + add5 +.macro shift2 + vmov q6, q10 + vmov q7, q11 + vmov q0, q2 + vmov q8, q12 + vmov q9, q13 + vmov q1, q3 + vmov q10, q14 + vmov q11, q15 + vmov q2, q5 +.endm + shift2 + add r0, r0, r7 + add r1, r1, r8 + ble 5f + vld1.32 {q12, q13}, [r5, :128], r7 + vld1.16 {q3}, [r6, :128], r8 + vld1.32 {q14, q15}, [r5, :128], r7 + vld1.16 {q5}, [r6, :128], r8 + b 3b + +4: + // h == 1, !LR_HAVE_BOTTOM. + // Pad the last row with the only content row, and add. + vmov q14, q12 + vmov q15, q13 + vmov q5, q3 + add5 + shift2 + add r0, r0, r7 + add r1, r1, r8 + add5 + b 6f + +5: + tst r4, #8 // LR_HAVE_BOTTOM + bne 6f + // !LR_HAVE_BOTTOM + cmp r3, #0 + bne 5f + // The intended three edge rows left; output the one at h-2 and + // the past edge one at h. + vld1.32 {q12, q13}, [r5, :128], r7 + vld1.16 {q3}, [r6, :128], r8 + // Pad the past-edge row from the last content row. + vmov q14, q12 + vmov q15, q13 + vmov q5, q3 + add5 + shift2 + add r0, r0, r7 + add r1, r1, r8 + // The last two rows are already padded properly here. + add5 + b 6f + +5: + // r3 == -1, two rows left, output one. + // Pad the last two rows from the mid one. + vmov q12, q10 + vmov q13, q11 + vmov q3, q2 + vmov q14, q10 + vmov q15, q11 + vmov q5, q2 + add5 + add r0, r0, r7 + add r1, r1, r8 + b 6f + +6: // End of one vertical slice. + subs r2, r2, #8 + ble 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + mls r5, r7, lr, r5 + mls r6, r8, lr, r6 + // Output pointers + mls r0, r7, r12, r0 + mls r1, r8, r12, r1 + add r0, r0, #32 + add r1, r1, #16 + add r5, r5, #32 + add r6, r6, #16 + mov r3, r9 + b 1b + +0: + vpop {q5-q7} + pop {r4-r9,pc} +.purgem add5 +endfunc + +// void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +// void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +function sgr_calc_ab1_neon, export=1 + push {r4-r7,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #84] + add r3, r3, #2 // h += 2 + clz r6, r5 + vmov.i32 q15, #9 // n + movw r5, #455 + mov lr, #SUM_STRIDE + b sgr_calc_ab_neon +endfunc + +function sgr_calc_ab2_neon, export=1 + push {r4-r7,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #84] + add r3, r3, #3 // h += 3 + clz r6, r5 + asr r3, r3, #1 // h /= 2 + vmov.i32 q15, #25 // n + mov r5, #164 + mov lr, #(2*SUM_STRIDE) +endfunc + +function sgr_calc_ab_neon + movrel r12, X(sgr_x_by_x) + sub r6, r6, #24 // -bitdepth_min_8 + vld1.8 {q8, q9}, [r12, :128]! + add r7, r6, r6 // -2*bitdepth_min_8 + vmov.i8 q11, #5 + vmov.i8 d10, #55 // idx of last 5 + vld1.8 {q10}, [r12, :128] + vmov.i8 d11, #72 // idx of last 4 + vmov.i8 d12, #101 // idx of last 3 + vmov.i8 d13, #169 // idx of last 2 + vmov.i8 d14, #254 // idx of last 1 + vmov.i8 d15, #32 // elements consumed in first vtbl + add r2, r2, #2 // w += 2 + add r12, r2, #7 + bic r12, r12, #7 // aligned w + sub r12, lr, r12 // increment between rows + vdup.32 q12, r4 + sub r0, r0, #(4*(SUM_STRIDE)) + sub r1, r1, #(2*(SUM_STRIDE)) + mov r4, r2 // backup of w + vsub.i8 q8, q8, q11 + vsub.i8 q9, q9, q11 + vsub.i8 q10, q10, q11 +1: + vld1.32 {q0, q1}, [r0, :128] // a + vld1.16 {q2}, [r1, :128] // b + vdup.32 q13, r7 // -2*bitdepth_min_8 + vdup.16 q14, r6 // -bitdepth_min_8 + subs r2, r2, #8 + vrshl.s32 q0, q0, q13 + vrshl.s32 q1, q1, q13 + vrshl.s16 q4, q2, q14 + vmul.i32 q0, q0, q15 // a * n + vmul.i32 q1, q1, q15 // a * n + vmull.u16 q3, d8, d8 // b * b + vmull.u16 q4, d9, d9 // b * b + vqsub.u32 q0, q0, q3 // imax(a * n - b * b, 0) + vqsub.u32 q1, q1, q4 // imax(a * n - b * b, 0) + vmul.i32 q0, q0, q12 // p * s + vmul.i32 q1, q1, q12 // p * s + vqshrn.u32 d0, q0, #16 + vqshrn.u32 d1, q1, #16 + vqrshrn.u16 d0, q0, #4 // imin(z, 255) + + vcgt.u8 d2, d0, d10 // = -1 if sgr_x_by_x[d0] < 5 + vcgt.u8 d3, d0, d11 // = -1 if sgr_x_by_x[d0] < 4 + vtbl.8 d1, {q8, q9}, d0 + vcgt.u8 d6, d0, d12 // = -1 if sgr_x_by_x[d0] < 3 + vsub.i8 d9, d0, d15 // indices for vtbx + vcgt.u8 d7, d0, d13 // = -1 if sgr_x_by_x[d0] < 2 + vadd.i8 d2, d2, d3 + vtbx.8 d1, {q10}, d9 + vcgt.u8 d8, d0, d14 // = -1 if sgr_x_by_x[d0] < 1 + vadd.i8 d6, d6, d7 + vadd.i8 d8, d8, d22 + vadd.i8 d2, d2, d6 + vadd.i8 d1, d1, d8 + vadd.i8 d1, d1, d2 + vmovl.u8 q0, d1 // x + + vmov.i16 q13, #256 + vdup.32 q14, r5 // one_by_x + + vmull.u16 q1, d0, d4 // x * BB[i] + vmull.u16 q2, d1, d5 // x * BB[i] + vmul.i32 q1, q1, q14 // x * BB[i] * sgr_one_by_x + vmul.i32 q2, q2, q14 // x * BB[i] * sgr_one_by_x + vrshr.s32 q1, q1, #12 // AA[i] + vrshr.s32 q2, q2, #12 // AA[i] + vsub.i16 q0, q13, q0 // 256 - x + + vst1.32 {q1, q2}, [r0, :128]! + vst1.16 {q0}, [r1, :128]! + bgt 1b + + subs r3, r3, #1 + ble 0f + add r0, r0, r12, lsl #2 + add r1, r1, r12, lsl #1 + mov r2, r4 + b 1b +0: + vpop {q4-q7} + pop {r4-r7,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/looprestoration_tmpl.S b/third_party/dav1d/src/arm/32/looprestoration_tmpl.S new file mode 100644 index 0000000000..8a9940bb3a --- /dev/null +++ b/third_party/dav1d/src/arm/32/looprestoration_tmpl.S @@ -0,0 +1,600 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" + +#define FILTER_OUT_STRIDE 384 + +.macro sgr_funcs bpc +// void dav1d_sgr_finish_filter1_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter1_\bpc\()bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldr r6, [sp, #108] + sub r7, r3, #(4*SUM_STRIDE) + add r8, r3, #(4*SUM_STRIDE) + sub r9, r4, #(2*SUM_STRIDE) + add r10, r4, #(2*SUM_STRIDE) + mov r11, #SUM_STRIDE + mov r12, #FILTER_OUT_STRIDE + add lr, r5, #3 + bic lr, lr, #3 // Aligned width +.if \bpc == 8 + sub r2, r2, lr +.else + sub r2, r2, lr, lsl #1 +.endif + sub r12, r12, lr + sub r11, r11, lr + sub r11, r11, #4 // We read 4 extra elements from both a and b + mov lr, r5 + vmov.i16 q14, #3 + vmov.i32 q15, #3 +1: + vld1.16 {q0}, [r9, :128]! + vld1.16 {q1}, [r4, :128]! + vld1.16 {q2}, [r10, :128]! + vld1.32 {q8, q9}, [r7, :128]! + vld1.32 {q10, q11}, [r3, :128]! + vld1.32 {q12, q13}, [r8, :128]! + +2: + subs r5, r5, #4 + vext.8 d6, d0, d1, #2 // -stride + vext.8 d7, d2, d3, #2 // 0 + vext.8 d8, d4, d5, #2 // +stride + vext.8 d9, d0, d1, #4 // +1-stride + vext.8 d10, d2, d3, #4 // +1 + vext.8 d11, d4, d5, #4 // +1+stride + vadd.i16 d2, d2, d6 // -1, -stride + vadd.i16 d7, d7, d8 // 0, +stride + vadd.i16 d0, d0, d9 // -1-stride, +1-stride + vadd.i16 d2, d2, d7 + vadd.i16 d4, d4, d11 // -1+stride, +1+stride + vadd.i16 d2, d2, d10 // +1 + vadd.i16 d0, d0, d4 + + vext.8 q3, q8, q9, #4 // -stride + vshl.i16 d2, d2, #2 + vext.8 q4, q8, q9, #8 // +1-stride + vext.8 q5, q10, q11, #4 // 0 + vext.8 q6, q10, q11, #8 // +1 + vmla.i16 d2, d0, d28 // * 3 -> a + vadd.i32 q3, q3, q10 // -stride, -1 + vadd.i32 q8, q8, q4 // -1-stride, +1-stride + vadd.i32 q5, q5, q6 // 0, +1 + vadd.i32 q8, q8, q12 // -1+stride + vadd.i32 q3, q3, q5 + vext.8 q7, q12, q13, #4 // +stride + vext.8 q10, q12, q13, #8 // +1+stride +.if \bpc == 8 + vld1.32 {d24[0]}, [r1, :32]! // src +.else + vld1.16 {d24}, [r1, :64]! // src +.endif + vadd.i32 q3, q3, q7 // +stride + vadd.i32 q8, q8, q10 // +1+stride + vshl.i32 q3, q3, #2 + vmla.i32 q3, q8, q15 // * 3 -> b +.if \bpc == 8 + vmovl.u8 q12, d24 // src +.endif + vmov d0, d1 + vmlal.u16 q3, d2, d24 // b + a * src + vmov d2, d3 + vrshrn.i32 d6, q3, #9 + vmov d4, d5 + vst1.16 {d6}, [r0]! + + ble 3f + vmov q8, q9 + vmov q10, q11 + vmov q12, q13 + vld1.16 {d1}, [r9, :64]! + vld1.16 {d3}, [r4, :64]! + vld1.16 {d5}, [r10, :64]! + vld1.32 {q9}, [r7, :128]! + vld1.32 {q11}, [r3, :128]! + vld1.32 {q13}, [r8, :128]! + b 2b + +3: + subs r6, r6, #1 + ble 0f + mov r5, lr + add r0, r0, r12, lsl #1 + add r1, r1, r2 + add r3, r3, r11, lsl #2 + add r7, r7, r11, lsl #2 + add r8, r8, r11, lsl #2 + add r4, r4, r11, lsl #1 + add r9, r9, r11, lsl #1 + add r10, r10, r11, lsl #1 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_sgr_finish_filter2_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter2_\bpc\()bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldr r6, [sp, #108] + add r7, r3, #(4*(SUM_STRIDE)) + sub r3, r3, #(4*(SUM_STRIDE)) + add r8, r4, #(2*(SUM_STRIDE)) + sub r4, r4, #(2*(SUM_STRIDE)) + mov r9, #(2*SUM_STRIDE) + mov r10, #FILTER_OUT_STRIDE + add r11, r5, #7 + bic r11, r11, #7 // Aligned width +.if \bpc == 8 + sub r2, r2, r11 +.else + sub r2, r2, r11, lsl #1 +.endif + sub r10, r10, r11 + sub r9, r9, r11 + sub r9, r9, #4 // We read 4 extra elements from a + sub r12, r9, #4 // We read 8 extra elements from b + mov lr, r5 + +1: + vld1.16 {q0, q1}, [r4, :128]! + vld1.16 {q2, q3}, [r8, :128]! + vld1.32 {q8, q9}, [r3, :128]! + vld1.32 {q11, q12}, [r7, :128]! + vld1.32 {q10}, [r3, :128]! + vld1.32 {q13}, [r7, :128]! + +2: + vmov.i16 q14, #5 + vmov.i16 q15, #6 + subs r5, r5, #8 + vext.8 q4, q0, q1, #4 // +1-stride + vext.8 q5, q2, q3, #4 // +1+stride + vext.8 q6, q0, q1, #2 // -stride + vext.8 q7, q2, q3, #2 // +stride + vadd.i16 q0, q0, q4 // -1-stride, +1-stride + vadd.i16 q5, q2, q5 // -1+stride, +1+stride + vadd.i16 q2, q6, q7 // -stride, +stride + vadd.i16 q0, q0, q5 + + vext.8 q4, q8, q9, #8 // +1-stride + vext.8 q5, q9, q10, #8 + vext.8 q6, q11, q12, #8 // +1+stride + vext.8 q7, q12, q13, #8 + vmul.i16 q0, q0, q14 // * 5 + vmla.i16 q0, q2, q15 // * 6 + vadd.i32 q4, q4, q8 // -1-stride, +1-stride + vadd.i32 q5, q5, q9 + vadd.i32 q6, q6, q11 // -1+stride, +1+stride + vadd.i32 q7, q7, q12 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q7 + vext.8 q6, q8, q9, #4 // -stride + vext.8 q7, q9, q10, #4 + vext.8 q8, q11, q12, #4 // +stride + vext.8 q11, q12, q13, #4 + +.if \bpc == 8 + vld1.8 {d4}, [r1, :64]! +.else + vld1.8 {q2}, [r1, :128]! +.endif + + vmov.i32 q14, #5 + vmov.i32 q15, #6 + + vadd.i32 q6, q6, q8 // -stride, +stride + vadd.i32 q7, q7, q11 + vmul.i32 q4, q4, q14 // * 5 + vmla.i32 q4, q6, q15 // * 6 + vmul.i32 q5, q5, q14 // * 5 + vmla.i32 q5, q7, q15 // * 6 + +.if \bpc == 8 + vmovl.u8 q2, d4 +.endif + vmlal.u16 q4, d0, d4 // b + a * src + vmlal.u16 q5, d1, d5 // b + a * src + vmov q0, q1 + vrshrn.i32 d8, q4, #9 + vrshrn.i32 d9, q5, #9 + vmov q2, q3 + vst1.16 {q4}, [r0, :128]! + + ble 3f + vmov q8, q10 + vmov q11, q13 + vld1.16 {q1}, [r4, :128]! + vld1.16 {q3}, [r8, :128]! + vld1.32 {q9, q10}, [r3, :128]! + vld1.32 {q12, q13}, [r7, :128]! + b 2b + +3: + subs r6, r6, #1 + ble 0f + mov r5, lr + add r0, r0, r10, lsl #1 + add r1, r1, r2 + add r3, r3, r9, lsl #2 + add r7, r7, r9, lsl #2 + add r4, r4, r12, lsl #1 + add r8, r8, r12, lsl #1 + + vld1.32 {q8, q9}, [r3, :128]! + vld1.16 {q0, q1}, [r4, :128]! + vld1.32 {q10}, [r3, :128]! + + vmov.i16 q12, #5 + vmov.i16 q13, #6 + +4: + subs r5, r5, #8 + vext.8 q3, q0, q1, #4 // +1 + vext.8 q2, q0, q1, #2 // 0 + vadd.i16 q0, q0, q3 // -1, +1 + + vext.8 q4, q8, q9, #4 // 0 + vext.8 q5, q9, q10, #4 + vext.8 q6, q8, q9, #8 // +1 + vext.8 q7, q9, q10, #8 + vmul.i16 q2, q2, q13 // * 6 + vmla.i16 q2, q0, q12 // * 5 -> a +.if \bpc == 8 + vld1.8 {d22}, [r1, :64]! +.else + vld1.16 {q11}, [r1, :128]! +.endif + vadd.i32 q8, q8, q6 // -1, +1 + vadd.i32 q9, q9, q7 +.if \bpc == 8 + vmovl.u8 q11, d22 +.endif + vmul.i32 q4, q4, q15 // * 6 + vmla.i32 q4, q8, q14 // * 5 -> b + vmul.i32 q5, q5, q15 // * 6 + vmla.i32 q5, q9, q14 // * 5 -> b + + vmlal.u16 q4, d4, d22 // b + a * src + vmlal.u16 q5, d5, d23 + vmov q0, q1 + vrshrn.i32 d8, q4, #8 + vrshrn.i32 d9, q5, #8 + vmov q8, q10 + vst1.16 {q4}, [r0, :128]! + + ble 5f + vld1.16 {q1}, [r4, :128]! + vld1.32 {q9, q10}, [r3, :128]! + b 4b + +5: + subs r6, r6, #1 + ble 0f + mov r5, lr + sub r3, r3, r11, lsl #2 // Rewind r3/r4 to where they started + sub r4, r4, r11, lsl #1 + add r0, r0, r10, lsl #1 + add r1, r1, r2 + sub r3, r3, #16 + sub r4, r4, #16 + b 1b +0: + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +// void dav1d_sgr_weighted1_Xbpc_neon(pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int w, const int h, +// const int wt, const int bitdepth_max); +function sgr_weighted1_\bpc\()bpc_neon, export=1 + push {r4-r9,lr} + ldrd r4, r5, [sp, #28] + ldrd r6, r7, [sp, #36] +.if \bpc == 16 + ldr r8, [sp, #44] +.endif + vdup.16 d31, r7 + cmp r6, #2 +.if \bpc == 16 + vdup.16 q14, r8 +.endif + add r9, r0, r1 + add r12, r2, r3 + add lr, r4, #2*FILTER_OUT_STRIDE + mov r7, #(4*FILTER_OUT_STRIDE) + lsl r1, r1, #1 + lsl r3, r3, #1 + add r8, r5, #7 + bic r8, r8, #7 // Aligned width +.if \bpc == 8 + sub r1, r1, r8 + sub r3, r3, r8 +.else + sub r1, r1, r8, lsl #1 + sub r3, r3, r8, lsl #1 +.endif + sub r7, r7, r8, lsl #1 + mov r8, r5 + blt 2f +1: +.if \bpc == 8 + vld1.8 {d0}, [r2, :64]! + vld1.8 {d16}, [r12, :64]! +.else + vld1.16 {q0}, [r2, :128]! + vld1.16 {q8}, [r12, :128]! +.endif + vld1.16 {q1}, [r4, :128]! + vld1.16 {q9}, [lr, :128]! + subs r5, r5, #8 +.if \bpc == 8 + vshll.u8 q0, d0, #4 // u + vshll.u8 q8, d16, #4 // u +.else + vshl.i16 q0, q0, #4 // u + vshl.i16 q8, q8, #4 // u +.endif + vsub.i16 q1, q1, q0 // t1 - u + vsub.i16 q9, q9, q8 // t1 - u + vshll.u16 q2, d0, #7 // u << 7 + vshll.u16 q3, d1, #7 // u << 7 + vshll.u16 q10, d16, #7 // u << 7 + vshll.u16 q11, d17, #7 // u << 7 + vmlal.s16 q2, d2, d31 // v + vmlal.s16 q3, d3, d31 // v + vmlal.s16 q10, d18, d31 // v + vmlal.s16 q11, d19, d31 // v +.if \bpc == 8 + vrshrn.i32 d4, q2, #11 + vrshrn.i32 d5, q3, #11 + vrshrn.i32 d20, q10, #11 + vrshrn.i32 d21, q11, #11 + vqmovun.s16 d4, q2 + vqmovun.s16 d20, q10 + vst1.8 {d4}, [r0, :64]! + vst1.8 {d20}, [r9, :64]! +.else + vqrshrun.s32 d4, q2, #11 + vqrshrun.s32 d5, q3, #11 + vqrshrun.s32 d20, q10, #11 + vqrshrun.s32 d21, q11, #11 + vmin.u16 q2, q2, q14 + vmin.u16 q10, q10, q14 + vst1.16 {q2}, [r0, :128]! + vst1.16 {q10}, [r9, :128]! +.endif + bgt 1b + + sub r6, r6, #2 + cmp r6, #1 + blt 0f + mov r5, r8 + add r0, r0, r1 + add r9, r9, r1 + add r2, r2, r3 + add r12, r12, r3 + add r4, r4, r7 + add lr, lr, r7 + beq 2f + b 1b + +2: +.if \bpc == 8 + vld1.8 {d0}, [r2, :64]! +.else + vld1.16 {q0}, [r2, :128]! +.endif + vld1.16 {q1}, [r4, :128]! + subs r5, r5, #8 +.if \bpc == 8 + vshll.u8 q0, d0, #4 // u +.else + vshl.i16 q0, q0, #4 // u +.endif + vsub.i16 q1, q1, q0 // t1 - u + vshll.u16 q2, d0, #7 // u << 7 + vshll.u16 q3, d1, #7 // u << 7 + vmlal.s16 q2, d2, d31 // v + vmlal.s16 q3, d3, d31 // v +.if \bpc == 8 + vrshrn.i32 d4, q2, #11 + vrshrn.i32 d5, q3, #11 + vqmovun.s16 d2, q2 + vst1.8 {d2}, [r0, :64]! +.else + vqrshrun.s32 d4, q2, #11 + vqrshrun.s32 d5, q3, #11 + vmin.u16 q2, q2, q14 + vst1.16 {q2}, [r0, :128]! +.endif + bgt 2b +0: + pop {r4-r9,pc} +endfunc + +// void dav1d_sgr_weighted2_Xbpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int16_t *t2, +// const int w, const int h, +// const int16_t wt[2], const int bitdepth_max); +function sgr_weighted2_\bpc\()bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] +.if \bpc == 8 + ldr r8, [sp, #52] +.else + ldrd r8, r9, [sp, #52] +.endif + cmp r7, #2 + add r10, r0, r1 + add r11, r2, r3 + add r12, r4, #2*FILTER_OUT_STRIDE + add lr, r5, #2*FILTER_OUT_STRIDE + vld2.16 {d30[], d31[]}, [r8] // wt[0], wt[1] +.if \bpc == 16 + vdup.16 q14, r9 +.endif + mov r8, #4*FILTER_OUT_STRIDE + lsl r1, r1, #1 + lsl r3, r3, #1 + add r9, r6, #7 + bic r9, r9, #7 // Aligned width +.if \bpc == 8 + sub r1, r1, r9 + sub r3, r3, r9 +.else + sub r1, r1, r9, lsl #1 + sub r3, r3, r9, lsl #1 +.endif + sub r8, r8, r9, lsl #1 + mov r9, r6 + blt 2f +1: +.if \bpc == 8 + vld1.8 {d0}, [r2, :64]! + vld1.8 {d16}, [r11, :64]! +.else + vld1.16 {q0}, [r2, :128]! + vld1.16 {q8}, [r11, :128]! +.endif + vld1.16 {q1}, [r4, :128]! + vld1.16 {q9}, [r12, :128]! + vld1.16 {q2}, [r5, :128]! + vld1.16 {q10}, [lr, :128]! + subs r6, r6, #8 +.if \bpc == 8 + vshll.u8 q0, d0, #4 // u + vshll.u8 q8, d16, #4 // u +.else + vshl.i16 q0, q0, #4 // u + vshl.i16 q8, q8, #4 // u +.endif + vsub.i16 q1, q1, q0 // t1 - u + vsub.i16 q2, q2, q0 // t2 - u + vsub.i16 q9, q9, q8 // t1 - u + vsub.i16 q10, q10, q8 // t2 - u + vshll.u16 q3, d0, #7 // u << 7 + vshll.u16 q0, d1, #7 // u << 7 + vshll.u16 q11, d16, #7 // u << 7 + vshll.u16 q8, d17, #7 // u << 7 + vmlal.s16 q3, d2, d30 // wt[0] * (t1 - u) + vmlal.s16 q3, d4, d31 // wt[1] * (t2 - u) + vmlal.s16 q0, d3, d30 // wt[0] * (t1 - u) + vmlal.s16 q0, d5, d31 // wt[1] * (t2 - u) + vmlal.s16 q11, d18, d30 // wt[0] * (t1 - u) + vmlal.s16 q11, d20, d31 // wt[1] * (t2 - u) + vmlal.s16 q8, d19, d30 // wt[0] * (t1 - u) + vmlal.s16 q8, d21, d31 // wt[1] * (t2 - u) +.if \bpc == 8 + vrshrn.i32 d6, q3, #11 + vrshrn.i32 d7, q0, #11 + vrshrn.i32 d22, q11, #11 + vrshrn.i32 d23, q8, #11 + vqmovun.s16 d6, q3 + vqmovun.s16 d22, q11 + vst1.8 {d6}, [r0, :64]! + vst1.8 {d22}, [r10, :64]! +.else + vqrshrun.s32 d6, q3, #11 + vqrshrun.s32 d7, q0, #11 + vqrshrun.s32 d22, q11, #11 + vqrshrun.s32 d23, q8, #11 + vmin.u16 q3, q3, q14 + vmin.u16 q11, q11, q14 + vst1.16 {q3}, [r0, :128]! + vst1.16 {q11}, [r10, :128]! +.endif + bgt 1b + + subs r7, r7, #2 + cmp r7, #1 + blt 0f + mov r6, r9 + add r0, r0, r1 + add r10, r10, r1 + add r2, r2, r3 + add r11, r11, r3 + add r4, r4, r8 + add r12, r12, r8 + add r5, r5, r8 + add lr, lr, r8 + beq 2f + b 1b + +2: +.if \bpc == 8 + vld1.8 {d0}, [r2, :64]! +.else + vld1.16 {q0}, [r2, :128]! +.endif + vld1.16 {q1}, [r4, :128]! + vld1.16 {q2}, [r5, :128]! + subs r6, r6, #8 +.if \bpc == 8 + vshll.u8 q0, d0, #4 // u +.else + vshl.i16 q0, q0, #4 // u +.endif + vsub.i16 q1, q1, q0 // t1 - u + vsub.i16 q2, q2, q0 // t2 - u + vshll.u16 q3, d0, #7 // u << 7 + vshll.u16 q0, d1, #7 // u << 7 + vmlal.s16 q3, d2, d30 // wt[0] * (t1 - u) + vmlal.s16 q3, d4, d31 // wt[1] * (t2 - u) + vmlal.s16 q0, d3, d30 // wt[0] * (t1 - u) + vmlal.s16 q0, d5, d31 // wt[1] * (t2 - u) +.if \bpc == 8 + vrshrn.i32 d6, q3, #11 + vrshrn.i32 d7, q0, #11 + vqmovun.s16 d6, q3 + vst1.8 {d6}, [r0, :64]! +.else + vqrshrun.s32 d6, q3, #11 + vqrshrun.s32 d7, q0, #11 + vmin.u16 q3, q3, q14 + vst1.16 {q3}, [r0, :128]! +.endif + bgt 1b +0: + pop {r4-r11,pc} +endfunc +.endm diff --git a/third_party/dav1d/src/arm/32/mc.S b/third_party/dav1d/src/arm/32/mc.S new file mode 100644 index 0000000000..1b60a7bdb3 --- /dev/null +++ b/third_party/dav1d/src/arm/32/mc.S @@ -0,0 +1,3340 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro avg dst0, dst1, t0, t1, t2, t3 + vld1.16 {\t0,\t1}, [r2, :128]! + vld1.16 {\t2,\t3}, [r3, :128]! + vadd.i16 \t0, \t0, \t2 + vadd.i16 \t1, \t1, \t3 + vqrshrun.s16 \dst0, \t0, #5 + vqrshrun.s16 \dst1, \t1, #5 +.endm + +.macro w_avg dst0, dst1, t0, t1, t2, t3 + vld1.16 {\t0,\t1}, [r2, :128]! + vld1.16 {\t2,\t3}, [r3, :128]! + vsub.i16 \t0, \t2, \t0 + vsub.i16 \t1, \t3, \t1 + vqdmulh.s16 \t0, \t0, q15 + vqdmulh.s16 \t1, \t1, q15 + vadd.i16 \t0, \t2, \t0 + vadd.i16 \t1, \t3, \t1 + vqrshrun.s16 \dst0, \t0, #4 + vqrshrun.s16 \dst1, \t1, #4 +.endm + +.macro mask dst0, dst1, t0, t1, t2, t3 + vld1.8 {q14}, [lr, :128]! + vld1.16 {\t0,\t1}, [r2, :128]! + vmul.i8 q14, q14, q15 + vld1.16 {\t2,\t3}, [r3, :128]! + vshll.i8 q13, d28, #8 + vshll.i8 q14, d29, #8 + vsub.i16 \t0, \t2, \t0 + vsub.i16 \t1, \t3, \t1 + vqdmulh.s16 \t0, \t0, q13 + vqdmulh.s16 \t1, \t1, q14 + vadd.i16 \t0, \t2, \t0 + vadd.i16 \t1, \t3, \t1 + vqrshrun.s16 \dst0, \t0, #4 + vqrshrun.s16 \dst1, \t1, #4 +.endm + +.macro bidir_fn type +function \type\()_8bpc_neon, export=1 + push {r4-r6,lr} + ldrd r4, r5, [sp, #16] + clz r4, r4 +.ifnc \type, avg + ldr lr, [sp, #24] +.endif +.ifc \type, w_avg + vdup.s16 q15, lr + vneg.s16 q15, q15 + vshl.i16 q15, q15, #11 +.endif +.ifc \type, mask + vmov.i8 q15, #256-2 +.endif + adr r12, L(\type\()_tbl) + sub r4, r4, #24 + ldr r4, [r12, r4, lsl #2] + \type d16, d17, q0, q1, q2, q3 + add r12, r12, r4 + bx r12 + + .align 2 +L(\type\()_tbl): + .word 1280f - L(\type\()_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_tbl) + CONFIG_THUMB + .word 4f - L(\type\()_tbl) + CONFIG_THUMB + +4: + add r6, r0, r1 + lsl r1, r1, #1 + cmp r5, #4 + vst1.32 {d16[0]}, [r0, :32], r1 + vst1.32 {d16[1]}, [r6, :32], r1 + vst1.32 {d17[0]}, [r0, :32], r1 + vst1.32 {d17[1]}, [r6, :32], r1 + beq 0f + \type d18, d19, q0, q1, q2, q3 + cmp r5, #8 + vst1.32 {d18[0]}, [r0, :32], r1 + vst1.32 {d18[1]}, [r6, :32], r1 + vst1.32 {d19[0]}, [r0, :32], r1 + vst1.32 {d19[1]}, [r6, :32], r1 + beq 0f + \type d16, d17, q0, q1, q2, q3 + vst1.32 {d16[0]}, [r0, :32], r1 + vst1.32 {d16[1]}, [r6, :32], r1 + \type d18, d19, q0, q1, q2, q3 + vst1.32 {d17[0]}, [r0, :32], r1 + vst1.32 {d17[1]}, [r6, :32], r1 + vst1.32 {d18[0]}, [r0, :32], r1 + vst1.32 {d18[1]}, [r6, :32], r1 + vst1.32 {d19[0]}, [r0, :32], r1 + vst1.32 {d19[1]}, [r6, :32], r1 + pop {r4-r6,pc} +80: + add r6, r0, r1 + lsl r1, r1, #1 +8: + vst1.8 {d16}, [r0, :64], r1 + \type d18, d19, q0, q1, q2, q3 + vst1.8 {d17}, [r6, :64], r1 + vst1.8 {d18}, [r0, :64], r1 + subs r5, r5, #4 + vst1.8 {d19}, [r6, :64], r1 + ble 0f + \type d16, d17, q0, q1, q2, q3 + b 8b +160: + add r6, r0, r1 + lsl r1, r1, #1 +16: + \type d18, d19, q0, q1, q2, q3 + vst1.8 {q8}, [r0, :128], r1 + \type d20, d21, q0, q1, q2, q3 + vst1.8 {q9}, [r6, :128], r1 + \type d22, d23, q0, q1, q2, q3 + vst1.8 {q10}, [r0, :128], r1 + subs r5, r5, #4 + vst1.8 {q11}, [r6, :128], r1 + ble 0f + \type d16, d17, q0, q1, q2, q3 + b 16b +320: + add r6, r0, r1 + lsl r1, r1, #1 +32: + \type d18, d19, q0, q1, q2, q3 + \type d20, d21, q0, q1, q2, q3 + vst1.8 {q8, q9}, [r0, :128], r1 + \type d22, d23, q0, q1, q2, q3 + subs r5, r5, #2 + vst1.8 {q10, q11}, [r6, :128], r1 + ble 0f + \type d16, d17, q0, q1, q2, q3 + b 32b +640: + add r6, r0, #32 +64: + \type d18, d19, q0, q1, q2, q3 + \type d20, d21, q0, q1, q2, q3 + \type d22, d23, q0, q1, q2, q3 + vst1.8 {q8, q9}, [r0, :128], r1 + \type d16, d17, q0, q1, q2, q3 + vst1.8 {q10, q11}, [r6, :128], r1 + \type d18, d19, q0, q1, q2, q3 + \type d20, d21, q0, q1, q2, q3 + vst1.8 {q8, q9}, [r0, :128], r1 + \type d22, d23, q0, q1, q2, q3 + subs r5, r5, #2 + vst1.8 {q10, q11}, [r6, :128], r1 + ble 0f + \type d16, d17, q0, q1, q2, q3 + b 64b +1280: + sub r1, r1, #32 + add r6, r0, #64 +128: + \type d18, d19, q0, q1, q2, q3 + \type d20, d21, q0, q1, q2, q3 + \type d22, d23, q0, q1, q2, q3 + vst1.8 {q8, q9}, [r0, :128]! + \type d16, d17, q0, q1, q2, q3 + vst1.8 {q10, q11}, [r0, :128], r1 + \type d18, d19, q0, q1, q2, q3 + \type d20, d21, q0, q1, q2, q3 + vst1.8 {q8, q9}, [r6, :128]! + \type d22, d23, q0, q1, q2, q3 + subs r5, r5, #1 + vst1.8 {q10, q11}, [r6, :128], r1 + ble 0f + \type d16, d17, q0, q1, q2, q3 + b 128b + +0: + pop {r4-r6,pc} +endfunc +.endm + +bidir_fn avg +bidir_fn w_avg +bidir_fn mask + + +.macro w_mask_fn type +function w_mask_\type\()_8bpc_neon, export=1 + push {r4-r9,lr} + ldrd r4, r5, [sp, #28] + ldrd r6, r7, [sp, #36] + clz r8, r4 + adr r9, L(w_mask_\type\()_tbl) + sub r8, r8, #24 + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + movw r12, #6903 + vdup.16 q14, r12 +.if \type == 444 + vmov.i8 q15, #64 +.elseif \type == 422 + vdup.8 d0, r7 // d0[] <- sign + vmov.i8 d30, #129 + vsub.i8 d30, d30, d0 // 129 - sign +.elseif \type == 420 + vdup.16 q0, r7 // d0[] <- sign + vmov.i16 q15, #256 + vsub.i16 q15, q15, q0 // 256 - sign +.endif + add r12, r0, r1 + lsl r1, r1, #1 + bx r9 + + .align 2 +L(w_mask_\type\()_tbl): + .word 1280f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + .word 640f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + .word 320f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + .word 160f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + .word 8f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + .word 4f - L(w_mask_\type\()_tbl) + CONFIG_THUMB + +4: + vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1 (four rows at once) + vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2 (four rows at once) + subs r5, r5, #4 + vsub.i16 q8, q2, q0 // tmp2-tmp1 + vsub.i16 q9, q3, q1 + vabd.s16 q10, q0, q2 // (abs(tmp1[x] - tmp2[x])) + vabd.s16 q11, q1, q3 + vqsub.u16 q10, q14, q10 // 6903 - abs () + vqsub.u16 q11, q14, q11 + vshr.s16 q10, q10, #8 // 64-m = (6903 - abs()) >> 8 + vshr.s16 q11, q11, #8 + vshl.s16 q12, q10, #9 // (64-m)<<9 + vshl.s16 q13, q11, #9 + vqdmulh.s16 q12, q12, q8 // ((tmp2-tmp1)*(64-m)<<9)>>15 + vqdmulh.s16 q13, q13, q9 + vadd.i16 q12, q12, q0 // (((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1 + vadd.i16 q13, q13, q1 + vqrshrun.s16 d24, q12, #4 // (((((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1) + 8) >> 4 + vqrshrun.s16 d25, q13, #4 +.if \type == 444 + vmovn.u16 d20, q10 // 64 - m + vmovn.u16 d21, q11 + vsub.i8 q10, q15, q10 // m + vst1.8 {d20, d21}, [r6, :128]! +.elseif \type == 422 + vpadd.s16 d20, d20, d21 // (64 - m) + (64 - n) (column wise addition) + vpadd.s16 d21, d22, d23 + vmovn.s16 d6, q10 + vhsub.u8 d6, d30, d6 // ((129 - sign) - ((64 - m) + (64 - n))) >> 1 + vst1.8 {d6}, [r6, :64]! +.elseif \type == 420 + vadd.s16 d20, d20, d21 // (64 - my1) + (64 - my2) (row wise addition) + vadd.s16 d21, d22, d23 + vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) + vsub.s16 d20, d30, d20 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.32 {d20[0]}, [r6, :32]! +.endif + vst1.32 {d24[0]}, [r0, :32], r1 + vst1.32 {d24[1]}, [r12, :32], r1 + vst1.32 {d25[0]}, [r0, :32], r1 + vst1.32 {d25[1]}, [r12, :32], r1 + bgt 4b + pop {r4-r9,pc} +8: + vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1y1, tmp1y2 + vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2y1, tmp2y2 + subs r5, r5, #2 + vsub.i16 q8, q2, q0 // tmp2y1 - tmp1y1 + vsub.i16 q9, q3, q1 // tmp2y2 - tmp1y2 + vabd.s16 q10, q0, q2 // abs(tmp1y1 - tmp2y1) + vabd.s16 q11, q1, q3 // abs(tmp1y2 - tmp2y2) + vqsub.u16 q10, q14, q10 // 6903 - abs(tmp1y1 - tmp2y1) + vqsub.u16 q11, q14, q11 // 6903 - abs(tmp1y2 - tmp2y2) + vshr.s16 q10, q10, #8 // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8 + vshr.s16 q11, q11, #8 // 64 - my2 = 6903 - abs(tmp1y2 - tmp2y2) >> 8 + vshl.s16 q12, q10, #9 // (64 - my1) << 9 + vshl.s16 q13, q11, #9 // (64 - my2) << 9 + vqdmulh.s16 q12, q12, q8 // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15 + vqdmulh.s16 q13, q13, q9 // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15 + vadd.s16 q12, q12, q0 // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1 + vadd.s16 q13, q13, q1 // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2 + vqrshrun.s16 d24, q12, #4 // (((((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4 + vqrshrun.s16 d25, q13, #4 // (((((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4 +.if \type == 444 + vmovn.u16 d20, q10 // 64 - m + vmovn.u16 d21, q11 + vsub.i8 q10, q15, q10 // m + vst1.8 {d20, d21}, [r6, :128]! +.elseif \type == 422 + vpadd.s16 d20, d20, d21 // (64 - my1) + (64 - ny1) (column wise addition) + vpadd.s16 d21, d22, d23 // (64 - my2) + (64 - ny2) + vmovn.s16 d20, q10 + vhsub.u8 d20, d30, d20 // ((129 - sign) - ((64 - my1/y2) + (64 - ny1/y2))) >> 1 + vst1.8 {d20}, [r6, :64]! +.elseif \type == 420 + vadd.s16 q10, q10, q11 // (64 - my1) + (64 - my2) (row wise addition) + vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) + vsub.s16 d20, d30, d20 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.32 {d20[0]}, [r6, :32]! +.endif + vst1.16 {d24}, [r0, :64], r1 + vst1.16 {d25}, [r12, :64], r1 + bgt 8b + pop {r4-r9,pc} +1280: +640: +320: +160: + sub r1, r1, r4 +.if \type == 444 + add lr, r6, r4 +.elseif \type == 422 + add lr, r6, r4, lsr #1 +.endif + add r9, r3, r4, lsl #1 + add r7, r2, r4, lsl #1 +161: + mov r8, r4 +16: + vld1.16 {d0, d1, d2, d3}, [r2, :128]! // tmp1y1 + vld1.16 {d4, d5, d6, d7}, [r3, :128]! // tmp2y1 + vld1.16 {d16, d17, d18, d19}, [r7, :128]! // tmp1y2 + subs r8, r8, #16 + vsub.i16 q2, q2, q0 // tmp2y1 - tmp1y1 + vsub.i16 q3, q3, q1 + vabs.s16 q10, q2 // abs(tm2y1 - tmp1y1) + vabs.s16 q11, q3 + vqsub.u16 q10, q14, q10 // 6903 - abs(tmp1y1 - tmp2y1) + vqsub.u16 q11, q14, q11 + vshr.s16 q10, q10, #8 // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8 + vshr.s16 q11, q11, #8 + vshl.s16 q12, q10, #9 // (64 - my1) << 9 + vshl.s16 q13, q11, #9 + vqdmulh.s16 q12, q12, q2 // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15 + vqdmulh.s16 q13, q13, q3 + vadd.i16 q12, q12, q0 // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1 + vadd.i16 q13, q13, q1 + vld1.16 {d0, d1, d2, d3}, [r9, :128]! // tmp2h2 +.if \type == 444 + vmovn.u16 d20, q10 // 64 - my1 + vmovn.u16 d21, q11 + vsub.i8 q10, q15, q10 // my1 + vst1.8 {d20, d21}, [r6, :128]! +.elseif \type == 422 + vpadd.s16 d20, d20, d21 // (64 - my1) + (64 - ny1) (column wise addition) + vpadd.s16 d21, d22, d23 + vmovn.s16 d20, q10 + vhsub.u8 d20, d30, d20 // ((129 - sign) - ((64 - my1) + (64 - ny1))) >> 1 + vst1.8 {d20}, [r6, :64]! +.endif + vqrshrun.s16 d24, q12, #4 // (((((tmp2y1 - tmp1y1)*(64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4 + vqrshrun.s16 d25, q13, #4 + vsub.i16 q0, q0, q8 // tmp2y2 - tmp1y2 + vsub.i16 q1, q1, q9 + vst1.16 {d24, d25}, [r0, :128]! // store dsty1 + vabs.s16 q2, q0 // abs(tmp2y2 - tmp1y2) + vabs.s16 q3, q1 + vqsub.u16 q2, q14, q2 // 6903 - abs(tmp2y2 - tmp1y2) + vqsub.u16 q3, q14, q3 + vshr.s16 q2, q2, #8 // (6903 - abs(tmp2y2 - tmp1y2)) >> 8 + vshr.s16 q3, q3, #8 + vshl.s16 q12, q2, #9 // (64 - my2) << 9 + vshl.s16 q13, q3, #9 +.if \type == 444 + vmovn.u16 d4, q2 // 64 - my2 + vmovn.u16 d5, q3 + vsub.i8 q2, q15, q2 // my2 + vst1.8 {d4, d5}, [lr, :128]! +.elseif \type == 422 + vpadd.s16 d4, d4, d5 // (64 - my2) + (64 - ny2) (column wise addition) + vpadd.s16 d5, d6, d7 + vmovn.s16 d4, q2 + vhsub.u8 d4, d30, d4 // ((129 - sign) - ((64 - my2) + (64 - ny2))) >> 1 + vst1.8 {d4}, [lr, :64]! +.elseif \type == 420 + vadd.s16 q10, q10, q2 // (64 - my1) + (64 - my2) (row wise addition) + vadd.s16 q11, q11, q3 + vpadd.s16 d20, d20, d21 // (128 - m) + (128 - n) (column wise addition) + vpadd.s16 d21, d22, d23 + vsub.s16 q10, q15, q10 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.u16 d20, q10, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.8 {d20}, [r6, :64]! +.endif + vqdmulh.s16 q12, q12, q0 // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15 + vqdmulh.s16 q13, q13, q1 + vadd.i16 q12, q12, q8 // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2 + vadd.i16 q13, q13, q9 + vqrshrun.s16 d24, q12, #4 // (((((tmp2y2 - tmp1y2)*(64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4 + vqrshrun.s16 d25, q13, #4 + vst1.16 {d24, d25}, [r12, :128]! // store dsty2 + bgt 16b + subs r5, r5, #2 + add r2, r2, r4, lsl #1 + add r3, r3, r4, lsl #1 + add r7, r7, r4, lsl #1 + add r9, r9, r4, lsl #1 +.if \type == 444 + add r6, r6, r4 + add lr, lr, r4 +.elseif \type == 422 + add r6, r6, r4, lsr #1 + add lr, lr, r4, lsr #1 +.endif + add r0, r0, r1 + add r12, r12, r1 + bgt 161b + pop {r4-r9,pc} +endfunc +.endm + +w_mask_fn 444 +w_mask_fn 422 +w_mask_fn 420 + + +function blend_8bpc_neon, export=1 + push {r4-r5,lr} + ldrd r4, r5, [sp, #12] + clz lr, r3 + adr r3, L(blend_tbl) + sub lr, lr, #26 + ldr lr, [r3, lr, lsl #2] + add r3, r3, lr + bx r3 + + .align 2 +L(blend_tbl): + .word 320f - L(blend_tbl) + CONFIG_THUMB + .word 160f - L(blend_tbl) + CONFIG_THUMB + .word 80f - L(blend_tbl) + CONFIG_THUMB + .word 40f - L(blend_tbl) + CONFIG_THUMB + +40: + vmov.i8 d22, #64 + add r12, r0, r1 + lsl r1, r1, #1 +4: + vld1.u8 {d2}, [r5, :64]! + vld1.u8 {d1}, [r2, :64]! + vld1.32 {d0[]}, [r0, :32] + subs r4, r4, #2 + vld1.32 {d0[1]}, [r12, :32] + vsub.i8 d3, d22, d2 + vmull.u8 q8, d1, d2 + vmlal.u8 q8, d0, d3 + vrshrn.i16 d20, q8, #6 + vst1.32 {d20[0]}, [r0, :32], r1 + vst1.32 {d20[1]}, [r12, :32], r1 + bgt 4b + pop {r4-r5,pc} +80: + vmov.i8 d16, #64 + add r12, r0, r1 + lsl r1, r1, #1 +8: + vld1.u8 {q1}, [r5, :128]! + vld1.u8 {q2}, [r2, :128]! + vld1.u8 {d0}, [r0, :64] + vsub.i8 d17, d16, d2 + vld1.u8 {d1}, [r12, :64] + subs r4, r4, #2 + vsub.i8 d18, d16, d3 + vmull.u8 q3, d2, d4 + vmlal.u8 q3, d0, d17 + vmull.u8 q10, d3, d5 + vmlal.u8 q10, d1, d18 + vrshrn.i16 d22, q3, #6 + vrshrn.i16 d23, q10, #6 + vst1.u8 {d22}, [r0, :64], r1 + vst1.u8 {d23}, [r12, :64], r1 + bgt 8b + pop {r4-r5,pc} +160: + vmov.i8 q12, #64 + add r12, r0, r1 + lsl r1, r1, #1 +16: + vld1.u8 {q1, q2}, [r5, :128]! + vld1.u8 {q8, q9}, [r2, :128]! + vld1.u8 {q0}, [r0, :128] + subs r4, r4, #2 + vsub.i8 q15, q12, q1 + vld1.u8 {q13}, [r12, :128] + vmull.u8 q3, d16, d2 + vmlal.u8 q3, d0, d30 + vmull.u8 q14, d17, d3 + vmlal.u8 q14, d1, d31 + vsub.i8 q15, q12, q2 + vrshrn.i16 d20, q3, #6 + vrshrn.i16 d21, q14, #6 + vmull.u8 q3, d18, d4 + vmlal.u8 q3, d26, d30 + vmull.u8 q14, d19, d5 + vmlal.u8 q14, d27, d31 + vrshrn.i16 d22, q3, #6 + vrshrn.i16 d23, q14, #6 + vst1.u8 {q10}, [r0, :128], r1 + vst1.u8 {q11}, [r12, :128], r1 + bgt 16b + pop {r4-r5,pc} +320: + vmov.i8 q10, #64 +32: + vld1.u8 {q2, q3}, [r5, :128]! + vld1.u8 {q8, q9}, [r2, :128]! + vld1.u8 {q0, q1}, [r0, :128] + subs r4, r4, #1 + vsub.i8 q11, q10, q2 + vmull.u8 q15, d16, d4 + vmlal.u8 q15, d0, d22 + vmull.u8 q14, d17, d5 + vmlal.u8 q14, d1, d23 + vsub.i8 q11, q10, q3 + vrshrn.i16 d24, q15, #6 + vrshrn.i16 d25, q14, #6 + vmull.u8 q15, d18, d6 + vmlal.u8 q15, d2, d22 + vmull.u8 q14, d19, d7 + vmlal.u8 q14, d3, d23 + vrshrn.i16 d26, q15, #6 + vrshrn.i16 d27, q14, #6 + vst1.u8 {q12, q13}, [r0, :128], r1 + bgt 32b + pop {r4-r5,pc} +endfunc + +function blend_h_8bpc_neon, export=1 + push {r4-r5,lr} + ldr r4, [sp, #12] + movrel r5, X(obmc_masks) + add r5, r5, r4 + sub r4, r4, r4, lsr #2 + clz lr, r3 + adr r12, L(blend_h_tbl) + sub lr, lr, #24 + ldr lr, [r12, lr, lsl #2] + add r12, r12, lr + bx r12 + + .align 2 +L(blend_h_tbl): + .word 1280f - L(blend_h_tbl) + CONFIG_THUMB + .word 640f - L(blend_h_tbl) + CONFIG_THUMB + .word 320f - L(blend_h_tbl) + CONFIG_THUMB + .word 160f - L(blend_h_tbl) + CONFIG_THUMB + .word 80f - L(blend_h_tbl) + CONFIG_THUMB + .word 40f - L(blend_h_tbl) + CONFIG_THUMB + .word 20f - L(blend_h_tbl) + CONFIG_THUMB + +20: + vmov.i8 d22, #64 + add r12, r0, r1 + lsl r1, r1, #1 +2: + vld1.16 {d2[], d3[]}, [r5, :16]! + vld1.32 {d1[]}, [r2, :32]! + subs r4, r4, #2 + vld1.16 {d0[]}, [r0, :16] + vzip.8 d2, d3 + vsub.i8 d4, d22, d2 + vld1.16 {d0[1]}, [r12, :16] + vmull.u8 q8, d1, d2 + vmlal.u8 q8, d0, d4 + vrshrn.i16 d20, q8, #6 + vst1.16 {d20[0]}, [r0, :16], r1 + vst1.16 {d20[1]}, [r12, :16], r1 + bgt 2b + pop {r4-r5,pc} +40: + vmov.i8 d22, #64 + add r12, r0, r1 + lsl r1, r1, #1 +4: + vld2.u8 {d2[], d3[]}, [r5, :16]! + vld1.u8 {d1}, [r2, :64]! + subs r4, r4, #2 + vext.u8 d2, d2, d3, #4 + vld1.32 {d0[]}, [r0, :32] + vsub.i8 d6, d22, d2 + vld1.32 {d0[1]}, [r12, :32] + vmull.u8 q8, d1, d2 + vmlal.u8 q8, d0, d6 + vrshrn.i16 d20, q8, #6 + vst1.32 {d20[0]}, [r0, :32], r1 + vst1.32 {d20[1]}, [r12, :32], r1 + bgt 4b + pop {r4-r5,pc} +80: + vmov.i8 q8, #64 + add r12, r0, r1 + lsl r1, r1, #1 +8: + vld2.u8 {d2[], d3[]}, [r5, :16]! + vld1.u8 {d4, d5}, [r2, :128]! + vld1.u8 {d0}, [r0, :64] + vsub.i8 q9, q8, q1 + vld1.u8 {d1}, [r12, :64] + subs r4, r4, #2 + vmull.u8 q3, d2, d4 + vmlal.u8 q3, d0, d18 + vmull.u8 q10, d3, d5 + vmlal.u8 q10, d1, d19 + vrshrn.i16 d22, q3, #6 + vrshrn.i16 d23, q10, #6 + vst1.u8 {d22}, [r0, :64], r1 + vst1.u8 {d23}, [r12, :64], r1 + bgt 8b + pop {r4-r5,pc} +160: + vmov.i8 q12, #64 + add r12, r0, r1 + lsl r1, r1, #1 +16: + vld2.u8 {d28[], d29[]}, [r5, :16]! + vld1.u8 {d2, d3, d4, d5}, [r2, :128]! + vsub.i8 q15, q12, q14 + vld1.u8 {q0}, [r0, :128] + subs r4, r4, #2 + vld1.u8 {q13}, [r12, :128] + vmull.u8 q3, d2, d28 + vmlal.u8 q3, d0, d30 + vmull.u8 q8, d3, d28 + vmlal.u8 q8, d1, d30 + vrshrn.i16 d18, q3, #6 + vrshrn.i16 d19, q8, #6 + vmull.u8 q3, d4, d29 + vmlal.u8 q3, d26, d31 + vmull.u8 q8, d5, d29 + vmlal.u8 q8, d27, d31 + vrshrn.i16 d20, q3, #6 + vrshrn.i16 d21, q8, #6 + vst1.u8 {q9}, [r0, :128], r1 + vst1.u8 {q10}, [r12, :128], r1 + bgt 16b + pop {r4-r5,pc} +320: +640: +1280: + vmov.i8 d20, #64 + sub r1, r1, r3 +321: + vld1.u8 {d6[]}, [r5]! + vsub.i8 d7, d20, d6 + mov r12, r3 +32: + vld1.u8 {q8, q9}, [r2, :128]! + vld1.u8 {q0, q1}, [r0, :128] + vmull.u8 q15, d16, d6 + vmlal.u8 q15, d0, d7 + vmull.u8 q14, d17, d6 + vmlal.u8 q14, d1, d7 + vrshrn.i16 d0, q15, #6 + vrshrn.i16 d1, q14, #6 + vmull.u8 q15, d18, d6 + vmlal.u8 q15, d2, d7 + vmull.u8 q14, d19, d6 + vmlal.u8 q14, d3, d7 + vrshrn.i16 d2, q15, #6 + vrshrn.i16 d3, q14, #6 + subs r12, r12, #32 + vst1.u8 {q0, q1}, [r0, :128]! + bgt 32b + add r0, r0, r1 + subs r4, r4, #1 + bgt 321b + pop {r4-r5,pc} +endfunc + +function blend_v_8bpc_neon, export=1 + push {r4,lr} + ldr r4, [sp, #8] + movrel lr, X(obmc_masks) + add lr, lr, r3 + clz r12, r3 + adr r3, L(blend_v_tbl) + sub r12, r12, #26 + ldr r12, [r3, r12, lsl #2] + add r3, r3, r12 + bx r3 + + .align 2 +L(blend_v_tbl): + .word 320f - L(blend_v_tbl) + CONFIG_THUMB + .word 160f - L(blend_v_tbl) + CONFIG_THUMB + .word 80f - L(blend_v_tbl) + CONFIG_THUMB + .word 40f - L(blend_v_tbl) + CONFIG_THUMB + .word 20f - L(blend_v_tbl) + CONFIG_THUMB + +20: + vmov.i8 d22, #64 + vld1.8 {d2[]}, [lr] + add r12, r0, r1 + lsl r1, r1, #1 + vsub.i8 d3, d22, d2 +2: + vld1.16 {d1[0]}, [r2, :16]! + vld1.8 {d0[]}, [r0] + subs r4, r4, #2 + vld1.8 {d1[1]}, [r2] + vld1.8 {d0[1]}, [r12] + vmull.u8 q2, d1, d2 + vmlal.u8 q2, d0, d3 + vrshrn.i16 d6, q2, #6 + add r2, r2, #2 + vst1.8 {d6[0]}, [r0], r1 + vst1.8 {d6[1]}, [r12], r1 + bgt 2b + pop {r4,pc} +40: + vmov.i8 d22, #64 + vld1.32 {d4[]}, [lr, :32] + add r12, r0, r1 + lsl r1, r1, #1 + vsub.i8 d5, d22, d4 + sub r1, r1, #2 +4: + vld1.u8 {d2}, [r2, :64]! + vld1.32 {d0[]}, [r0, :32] + vld1.32 {d0[1]}, [r12, :32] + subs r4, r4, #2 + vmull.u8 q3, d2, d4 + vmlal.u8 q3, d0, d5 + vrshrn.i16 d20, q3, #6 + vst1.16 {d20[0]}, [r0, :16]! + vst1.16 {d20[2]}, [r12, :16]! + vst1.8 {d20[2]}, [r0], r1 + vst1.8 {d20[6]}, [r12], r1 + bgt 4b + pop {r4,pc} +80: + vmov.i8 d16, #64 + vld1.u8 {d2}, [lr, :64] + add r12, r0, r1 + lsl r1, r1, #1 + vsub.i8 d17, d16, d2 + sub r1, r1, #4 +8: + vld1.u8 {d4, d5}, [r2, :128]! + vld1.u8 {d0}, [r0, :64] + vld1.u8 {d1}, [r12, :64] + subs r4, r4, #2 + vmull.u8 q3, d2, d4 + vmlal.u8 q3, d0, d17 + vmull.u8 q10, d2, d5 + vmlal.u8 q10, d1, d17 + vrshrn.i16 d22, q3, #6 + vrshrn.i16 d23, q10, #6 + vst1.32 {d22[0]}, [r0, :32]! + vst1.32 {d23[0]}, [r12, :32]! + vst1.16 {d22[2]}, [r0, :16], r1 + vst1.16 {d23[2]}, [r12, :16], r1 + bgt 8b + pop {r4,pc} +160: + vmov.i8 q12, #64 + vld1.u8 {q14}, [lr, :128] + add r12, r0, r1 + lsl r1, r1, #1 + vsub.i8 q11, q12, q14 + sub r1, r1, #8 +16: + vld1.u8 {q1, q2}, [r2, :128]! + vld1.u8 {q0}, [r0, :128] + subs r4, r4, #2 + vld1.u8 {q13}, [r12, :128] + vmull.u8 q3, d2, d28 + vmlal.u8 q3, d0, d22 + vmull.u8 q8, d3, d29 + vmlal.u8 q8, d1, d23 + vrshrn.i16 d18, q3, #6 + vrshrn.i16 d19, q8, #6 + vmull.u8 q3, d4, d28 + vmlal.u8 q3, d26, d22 + vmull.u8 q8, d5, d29 + vmlal.u8 q8, d27, d23 + vrshrn.i16 d20, q3, #6 + vrshrn.i16 d21, q8, #6 + vst1.u8 {d18}, [r0, :64]! + vst1.u8 {d20}, [r12, :64]! + vst1.32 {d19[0]}, [r0, :32], r1 + vst1.32 {d21[0]}, [r12, :32], r1 + bgt 16b + pop {r4,pc} +320: + vmov.i8 q10, #64 + vld1.u8 {q2, q3}, [lr, :128] + vsub.i8 q11, q10, q2 + vsub.i8 d24, d20, d6 +32: + vld1.u8 {q8, q9}, [r2, :128]! + vld1.u8 {d0, d1, d2}, [r0, :64] + subs r4, r4, #1 + vmull.u8 q15, d16, d4 + vmlal.u8 q15, d0, d22 + vmull.u8 q14, d17, d5 + vmlal.u8 q14, d1, d23 + vrshrn.i16 d0, q15, #6 + vrshrn.i16 d1, q14, #6 + vmull.u8 q15, d18, d6 + vmlal.u8 q15, d2, d24 + vrshrn.i16 d2, q15, #6 + vst1.u8 {d0, d1, d2}, [r0, :64], r1 + bgt 32b + pop {r4,pc} +endfunc + + +// This has got the same signature as the put_8tap functions, +// assumes that the caller has loaded the h argument into r5, +// and assumes that r8 is set to (clz(w)-24). +function put_neon + adr r9, L(put_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(put_tbl): + .word 1280f - L(put_tbl) + CONFIG_THUMB + .word 640f - L(put_tbl) + CONFIG_THUMB + .word 32f - L(put_tbl) + CONFIG_THUMB + .word 160f - L(put_tbl) + CONFIG_THUMB + .word 8f - L(put_tbl) + CONFIG_THUMB + .word 4f - L(put_tbl) + CONFIG_THUMB + .word 2f - L(put_tbl) + CONFIG_THUMB + +2: + vld1.16 {d0[]}, [r2], r3 + vld1.16 {d1[]}, [r2], r3 + subs r5, r5, #2 + vst1.16 {d0[0]}, [r0, :16], r1 + vst1.16 {d1[0]}, [r0, :16], r1 + bgt 2b + pop {r4-r11,pc} +4: + vld1.32 {d0[]}, [r2], r3 + vld1.32 {d1[]}, [r2], r3 + subs r5, r5, #2 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d1[0]}, [r0, :32], r1 + bgt 4b + pop {r4-r11,pc} +8: + vld1.8 {d0}, [r2], r3 + vld1.8 {d1}, [r2], r3 + subs r5, r5, #2 + vst1.8 {d0}, [r0, :64], r1 + vst1.8 {d1}, [r0, :64], r1 + bgt 8b + pop {r4-r11,pc} +160: + add r8, r0, r1 + lsl r1, r1, #1 + add r9, r2, r3 + lsl r3, r3, #1 +16: + vld1.8 {q0}, [r2], r3 + vld1.8 {q1}, [r9], r3 + subs r5, r5, #2 + vst1.8 {q0}, [r0, :128], r1 + vst1.8 {q1}, [r8, :128], r1 + bgt 16b + pop {r4-r11,pc} +32: + vld1.8 {q0, q1}, [r2], r3 + subs r5, r5, #1 + vst1.8 {q0, q1}, [r0, :128], r1 + bgt 32b + pop {r4-r11,pc} +640: + sub r1, r1, #32 + sub r3, r3, #32 +64: + vld1.8 {q0, q1}, [r2]! + vst1.8 {q0, q1}, [r0, :128]! + vld1.8 {q2, q3}, [r2], r3 + subs r5, r5, #1 + vst1.8 {q2, q3}, [r0, :128], r1 + bgt 64b + pop {r4-r11,pc} +1280: + sub r1, r1, #96 + sub r3, r3, #96 +128: + vld1.8 {q8, q9}, [r2]! + vst1.8 {q8, q9}, [r0, :128]! + vld1.8 {q10, q11}, [r2]! + vst1.8 {q10, q11}, [r0, :128]! + vld1.8 {q12, q13}, [r2]! + vst1.8 {q12, q13}, [r0, :128]! + vld1.8 {q14, q15}, [r2], r3 + subs r5, r5, #1 + vst1.8 {q14, q15}, [r0, :128], r1 + bgt 128b + pop {r4-r11,pc} +endfunc + + +// This has got the same signature as the put_8tap functions, +// assumes that the caller has loaded the h argument into r4, +// and assumes that r8 is set to (clz(w)-24), and r7 to w*2. +function prep_neon + adr r9, L(prep_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(prep_tbl): + .word 1280f - L(prep_tbl) + CONFIG_THUMB + .word 640f - L(prep_tbl) + CONFIG_THUMB + .word 320f - L(prep_tbl) + CONFIG_THUMB + .word 160f - L(prep_tbl) + CONFIG_THUMB + .word 8f - L(prep_tbl) + CONFIG_THUMB + .word 4f - L(prep_tbl) + CONFIG_THUMB + +4: + vld1.32 {d0[]}, [r1], r2 + vld1.32 {d2[]}, [r1], r2 + subs r4, r4, #2 + vshll.u8 q0, d0, #4 + vshll.u8 q1, d2, #4 + vst1.16 {d1, d2}, [r0, :64]! + bgt 4b + pop {r4-r11,pc} +8: + vld1.8 {d0}, [r1], r2 + vld1.8 {d2}, [r1], r2 + subs r4, r4, #2 + vshll.u8 q0, d0, #4 + vshll.u8 q1, d2, #4 + vst1.16 {q0, q1}, [r0, :128]! + bgt 8b + pop {r4-r11,pc} +160: + add r9, r1, r2 + lsl r2, r2, #1 + add r8, r0, r7 + lsl r7, r7, #1 +16: + vld1.8 {q2}, [r1], r2 + vld1.8 {q3}, [r9], r2 + subs r4, r4, #2 + vshll.u8 q0, d4, #4 + vshll.u8 q1, d5, #4 + vshll.u8 q2, d6, #4 + vshll.u8 q3, d7, #4 + vst1.16 {q0, q1}, [r0, :128], r7 + vst1.16 {q2, q3}, [r8, :128], r7 + bgt 16b + pop {r4-r11,pc} +320: + add r8, r0, r3 +32: + vld1.8 {q0, q1}, [r1], r2 + subs r4, r4, #2 + vshll.u8 q8, d0, #4 + vshll.u8 q9, d1, #4 + vld1.8 {q2, q3}, [r1], r2 + vshll.u8 q10, d2, #4 + vshll.u8 q11, d3, #4 + vshll.u8 q12, d4, #4 + vst1.16 {q8, q9}, [r0, :128], r7 + vshll.u8 q13, d5, #4 + vst1.16 {q10, q11}, [r8, :128], r7 + vshll.u8 q14, d6, #4 + vst1.16 {q12, q13}, [r0, :128], r7 + vshll.u8 q15, d7, #4 + vst1.16 {q14, q15}, [r8, :128], r7 + bgt 32b + pop {r4-r11,pc} +640: + sub r2, r2, #32 + add r8, r0, #32 + mov r6, #64 +64: + vld1.8 {q0, q1}, [r1]! + subs r4, r4, #1 + vshll.u8 q8, d0, #4 + vshll.u8 q9, d1, #4 + vld1.8 {q2, q3}, [r1], r2 + vshll.u8 q10, d2, #4 + vshll.u8 q11, d3, #4 + vshll.u8 q12, d4, #4 + vst1.16 {q8, q9}, [r0, :128], r6 + vshll.u8 q13, d5, #4 + vshll.u8 q14, d6, #4 + vst1.16 {q10, q11}, [r8, :128], r6 + vshll.u8 q15, d7, #4 + vst1.16 {q12, q13}, [r0, :128], r6 + vst1.16 {q14, q15}, [r8, :128], r6 + bgt 64b + pop {r4-r11,pc} +1280: + sub r2, r2, #96 + add r8, r0, #32 + mov r6, #64 +128: + vld1.8 {q0, q1}, [r1]! + vld1.8 {q2, q3}, [r1]! + vshll.u8 q10, d0, #4 + vshll.u8 q11, d1, #4 + vshll.u8 q12, d2, #4 + vshll.u8 q13, d3, #4 + vshll.u8 q14, d4, #4 + vshll.u8 q15, d5, #4 + vld1.8 {q8, q9}, [r1]! + vst1.16 {q10, q11}, [r0, :128], r6 + vst1.16 {q12, q13}, [r8, :128], r6 + vshll.u8 q0, d6, #4 + vshll.u8 q1, d7, #4 + vshll.u8 q2, d16, #4 + vshll.u8 q3, d17, #4 + vshll.u8 q8, d18, #4 + vshll.u8 q9, d19, #4 + vld1.8 {q10, q11}, [r1], r2 + vst1.16 {q14, q15}, [r0, :128], r6 + vst1.16 {q0, q1}, [r8, :128], r6 + vshll.u8 q12, d20, #4 + vshll.u8 q13, d21, #4 + vshll.u8 q14, d22, #4 + vshll.u8 q15, d23, #4 + subs r4, r4, #1 + vst1.16 {q2, q3}, [r0, :128], r6 + vst1.16 {q8, q9}, [r8, :128], r6 + vst1.16 {q12, q13}, [r0, :128], r6 + vst1.16 {q14, q15}, [r8, :128], r6 + bgt 128b + pop {r4-r11,pc} +endfunc + + +.macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + vld1.\wd {\d0[]}, [\s0], \strd + vld1.\wd {\d1[]}, [\s1], \strd +.ifnb \d2 + vld1.\wd {\d2[]}, [\s0], \strd + vld1.\wd {\d3[]}, [\s1], \strd +.endif +.ifnb \d4 + vld1.\wd {\d4[]}, [\s0], \strd +.endif +.ifnb \d5 + vld1.\wd {\d5[]}, [\s1], \strd +.endif +.ifnb \d6 + vld1.\wd {\d6[]}, [\s0], \strd +.endif +.endm +.macro load_reg s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + vld1.8 {\d0}, [\s0], \strd + vld1.8 {\d1}, [\s1], \strd +.ifnb \d2 + vld1.8 {\d2}, [\s0], \strd + vld1.8 {\d3}, [\s1], \strd +.endif +.ifnb \d4 + vld1.8 {\d4}, [\s0], \strd +.endif +.ifnb \d5 + vld1.8 {\d5}, [\s1], \strd +.endif +.ifnb \d6 + vld1.8 {\d6}, [\s0], \strd +.endif +.endm +.macro load_16 s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, 16, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_32 s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, 32, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro interleave_1_16 r0, r1, r2, r3, r4 + vext.8 \r0, \r0, \r1, #6 + vext.8 \r1, \r1, \r2, #6 +.ifnb \r3 + vext.8 \r2, \r2, \r3, #6 + vext.8 \r3, \r3, \r4, #6 +.endif +.endm +.macro interleave_1_32 r0, r1, r2, r3, r4 + vext.8 \r0, \r0, \r1, #4 + vext.8 \r1, \r1, \r2, #4 +.ifnb \r3 + vext.8 \r2, \r2, \r3, #4 + vext.8 \r3, \r3, \r4, #4 +.endif +.endm +.macro vmovl_u8 q0, d0, q1, d1, q2, d2, q3, d3, q4, d4, q5, d5, q6, d6 + vmovl.u8 \q0, \d0 + vmovl.u8 \q1, \d1 +.ifnb \q2 + vmovl.u8 \q2, \d2 + vmovl.u8 \q3, \d3 +.endif +.ifnb \q4 + vmovl.u8 \q4, \d4 +.endif +.ifnb \q5 + vmovl.u8 \q5, \d5 +.endif +.ifnb \q6 + vmovl.u8 \q6, \d6 +.endif +.endm +.macro mul_mla_4 d, s0, s1, s2, s3 + vmul.s16 \d, \s0, d0[0] + vmla.s16 \d, \s1, d0[1] + vmla.s16 \d, \s2, d0[2] + vmla.s16 \d, \s3, d0[3] +.endm +.macro mul_mla_8_0 d0, s0, s1, s2, s3, s4, s5, s6, s7 + vmul.s16 \d0, \s0, d0[0] + vmla.s16 \d0, \s1, d0[1] + vmla.s16 \d0, \s2, d0[2] + vmla.s16 \d0, \s3, d0[3] + vmla.s16 \d0, \s4, d1[0] + vmla.s16 \d0, \s5, d1[1] + vmla.s16 \d0, \s6, d1[2] + vmla.s16 \d0, \s7, d1[3] +.endm +.macro mul_mla_8_1 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8 + vmul.s16 \d0, \s0, d0[0] + vmla.s16 \d0, \s1, d0[1] + vmla.s16 \d0, \s2, d0[2] + vmla.s16 \d0, \s3, d0[3] + vmla.s16 \d0, \s4, d1[0] + vmla.s16 \d0, \s5, d1[1] + vmla.s16 \d0, \s6, d1[2] + vmla.s16 \d0, \s7, d1[3] + vmul.s16 \d1, \s1, d0[0] + vmla.s16 \d1, \s2, d0[1] + vmla.s16 \d1, \s3, d0[2] + vmla.s16 \d1, \s4, d0[3] + vmla.s16 \d1, \s5, d1[0] + vmla.s16 \d1, \s6, d1[1] + vmla.s16 \d1, \s7, d1[2] + vmla.s16 \d1, \s8, d1[3] +.endm +.macro mul_mla_8_2 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9 + vmul.s16 \d0, \s0, d0[0] + vmla.s16 \d0, \s1, d0[1] + vmla.s16 \d0, \s2, d0[2] + vmla.s16 \d0, \s3, d0[3] + vmla.s16 \d0, \s4, d1[0] + vmla.s16 \d0, \s5, d1[1] + vmla.s16 \d0, \s6, d1[2] + vmla.s16 \d0, \s7, d1[3] + vmul.s16 \d1, \s2, d0[0] + vmla.s16 \d1, \s3, d0[1] + vmla.s16 \d1, \s4, d0[2] + vmla.s16 \d1, \s5, d0[3] + vmla.s16 \d1, \s6, d1[0] + vmla.s16 \d1, \s7, d1[1] + vmla.s16 \d1, \s8, d1[2] + vmla.s16 \d1, \s9, d1[3] +.endm +.macro vqrshrun_s16 shift, q0, d0, q1, d1, q2, d2, q3, d3 + vqrshrun.s16 \d0, \q0, #\shift +.ifnb \q1 + vqrshrun.s16 \d1, \q1, #\shift +.endif +.ifnb \q2 + vqrshrun.s16 \d2, \q2, #\shift + vqrshrun.s16 \d3, \q3, #\shift +.endif +.endm +.macro vrshr_s16 shift, r0, r1, r2, r3 + vrshr.s16 \r0, \r0, #\shift +.ifnb \r1 + vrshr.s16 \r1, \r1, #\shift +.endif +.ifnb \r2 + vrshr.s16 \r2, \r2, #\shift + vrshr.s16 \r3, \r3, #\shift +.endif +.endm +.macro st_16 strd, reg, lanes + vst1.16 {\reg[0]}, [r0, :16], \strd + vst1.16 {\reg[1]}, [r8, :16], \strd +.if \lanes > 2 + vst1.16 {\reg[2]}, [r0, :16], \strd + vst1.16 {\reg[3]}, [r8, :16], \strd +.endif +.endm +.macro st_32 strd, r0, r1 + vst1.32 {\r0[0]}, [r0, :32], \strd + vst1.32 {\r0[1]}, [r8, :32], \strd +.ifnb \r1 + vst1.32 {\r1[0]}, [r0, :32], \strd + vst1.32 {\r1[1]}, [r8, :32], \strd +.endif +.endm +.macro st_reg strd, align, r0, r1, r2, r3, r4, r5, r6, r7 + vst1.8 {\r0}, [r0, \align], \strd + vst1.8 {\r1}, [r8, \align], \strd +.ifnb \r2 + vst1.8 {\r2}, [r0, \align], \strd + vst1.8 {\r3}, [r8, \align], \strd +.endif +.ifnb \r4 + vst1.8 {\r4}, [r0, \align], \strd + vst1.8 {\r5}, [r8, \align], \strd + vst1.8 {\r6}, [r0, \align], \strd + vst1.8 {\r7}, [r8, \align], \strd +.endif +.endm +.macro shift_store_4 type, strd, q0, d0, d1, q1, d2, d3 +.ifc \type, put + vqrshrun_s16 6, \q0, \d0, \q1, \d2 + st_32 \strd, \d0, \d2 +.else + vrshr_s16 2, \q0, \q1 + st_reg \strd, :64, \d0, \d1, \d2, \d3 +.endif +.endm +.macro shift_store_8 type, strd, q0, d0, q1, d1, q2, d2, q3, d3 +.ifc \type, put + vqrshrun_s16 6, \q0, \d0, \q1, \d1, \q2, \d2, \q3, \d3 + st_reg \strd, :64, \d0, \d1, \d2, \d3 +.else + vrshr_s16 2, \q0, \q1, \q2, \q3 + st_reg \strd, :128,\q0, \q1, \q2, \q3 +.endif +.endm +.macro shift_store_16 type, strd, q0, d0, d1, q1, q2, d4, d5, q3 +.ifc \type, put + vqrshrun.s16 \d0, \q0, #6 + vqrshrun.s16 \d1, \q1, #6 + vqrshrun.s16 \d4, \q2, #6 + vqrshrun.s16 \d5, \q3, #6 + st_reg \strd, :128, \q0, \q2 +.else + vrshr_s16 2, \q0, \q1, \q2, \q3 + vst1.16 {\q0, \q1}, [r0, :128], \strd + vst1.16 {\q2, \q3}, [r8, :128], \strd +.endif +.endm + +.macro make_8tap_fn op, type, type_h, type_v +function \op\()_8tap_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + movw r8, \type_h + movw r9, \type_v + b \op\()_8tap_neon +endfunc +.endm + +// No spaces in these expressions, due to gas-preprocessor. +#define REGULAR ((0*15<<7)|3*15) +#define SMOOTH ((1*15<<7)|4*15) +#define SHARP ((2*15<<7)|3*15) + +.macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, my, ds2, sr2, shift_hv +make_8tap_fn \type, regular, REGULAR, REGULAR +make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH +make_8tap_fn \type, regular_sharp, REGULAR, SHARP +make_8tap_fn \type, smooth, SMOOTH, SMOOTH +make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR +make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP +make_8tap_fn \type, sharp, SHARP, SHARP +make_8tap_fn \type, sharp_regular, SHARP, REGULAR +make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH + +function \type\()_8tap_neon + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + movw r10, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + mul \mx, \mx, r10 + mul \my, \my, r10 + add \mx, \mx, r8 // mx, 8tap_h, 4tap_h + add \my, \my, r9 // my, 8tap_v, 4tap_v +.ifc \type, prep + lsl \d_strd, \w, #1 +.endif + + clz r8, \w + tst \mx, #(0x7f << 14) + sub r8, r8, #24 + movrel r10, X(mc_subpel_filters), -8 + bne L(\type\()_8tap_h) + tst \my, #(0x7f << 14) + bne L(\type\()_8tap_v) + b \type\()_neon + +L(\type\()_8tap_h): + cmp \w, #4 + ubfx r9, \mx, #7, #7 + and \mx, \mx, #0x7f + it gt + movgt \mx, r9 + tst \my, #(0x7f << 14) + add \mx, r10, \mx, lsl #3 + bne L(\type\()_8tap_hv) + + adr r9, L(\type\()_8tap_h_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_8tap_h_tbl): + .word 1280f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + +20: // 2xN h +.ifc \type, put + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + sub \src, \src, #1 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 +2: + vld1.8 {d4}, [\src], \s_strd + vld1.8 {d6}, [\sr2], \s_strd + vmovl.u8 q2, d4 + vmovl.u8 q3, d6 + vext.8 d5, d4, d5, #2 + vext.8 d7, d6, d7, #2 + subs \h, \h, #2 + vtrn.32 d4, d6 + vtrn.32 d5, d7 + vmul.s16 d2, d4, d0[0] + vmla.s16 d2, d5, d0[1] + vmla.s16 d2, d6, d0[2] + vmla.s16 d2, d7, d0[3] + vrshr.s16 d2, d2, #2 + vqrshrun.s16 d2, q1, #4 + vst1.16 {d2[0]}, [\dst, :16], \d_strd + vst1.16 {d2[1]}, [\ds2, :16], \d_strd + bgt 2b + pop {r4-r11,pc} +.endif + +40: // 4xN h + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + sub \src, \src, #1 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 +4: + vld1.8 {d16}, [\src], \s_strd + vld1.8 {d24}, [\sr2], \s_strd + vmovl.u8 q8, d16 + vmovl.u8 q12, d24 + vext.8 d18, d16, d17, #2 + vext.8 d20, d16, d17, #4 + vext.8 d22, d16, d17, #6 + vext.8 d26, d24, d25, #2 + vext.8 d28, d24, d25, #4 + vext.8 d30, d24, d25, #6 + subs \h, \h, #2 + vmul.s16 d4, d16, d0[0] + vmla.s16 d4, d18, d0[1] + vmla.s16 d4, d20, d0[2] + vmla.s16 d4, d22, d0[3] + vmul.s16 d5, d24, d0[0] + vmla.s16 d5, d26, d0[1] + vmla.s16 d5, d28, d0[2] + vmla.s16 d5, d30, d0[3] + vrshr.s16 q2, q2, #2 +.ifc \type, put + vqrshrun.s16 d4, q2, #4 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d4[1]}, [\ds2, :32], \d_strd +.else + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d5}, [\ds2, :64], \d_strd +.endif + bgt 4b + pop {r4-r11,pc} + +80: // 8xN h + vld1.8 {d0}, [\mx, :64] + sub \src, \src, #3 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 +8: + vld1.8 {q8}, [\src], \s_strd + vld1.8 {q12}, [\sr2], \s_strd + vmovl.u8 q9, d17 + vmovl.u8 q8, d16 + vmovl.u8 q13, d25 + vmovl.u8 q12, d24 + + vmul.s16 q10, q8, d0[0] + vmul.s16 q14, q12, d0[0] +.irpc i, 1234567 + vext.8 q11, q8, q9, #(2*\i) + vext.8 q15, q12, q13, #(2*\i) +.if \i < 4 + vmla.s16 q10, q11, d0[\i] + vmla.s16 q14, q15, d0[\i] +.else + vmla.s16 q10, q11, d1[\i-4] + vmla.s16 q14, q15, d1[\i-4] +.endif +.endr + subs \h, \h, #2 + vrshr.s16 q10, q10, #2 + vrshr.s16 q14, q14, #2 +.ifc \type, put + vqrshrun.s16 d20, q10, #4 + vqrshrun.s16 d28, q14, #4 + vst1.8 {d20}, [\dst, :64], \d_strd + vst1.8 {d28}, [\ds2, :64], \d_strd +.else + vst1.16 {q10}, [\dst, :128], \d_strd + vst1.16 {q14}, [\ds2, :128], \d_strd +.endif + bgt 8b + pop {r4-r11,pc} + +160: +320: +640: +1280: // 16xN, 32xN, ... h + // This could be done without touching q4-q6, by using only + // one temporary for vext in the loop. That's slower on A7 and A53, + // (but surprisingly, marginally faster on A8 and A73). + vpush {q4-q6} + vld1.8 {d0}, [\mx, :64] + sub \src, \src, #3 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 + + sub \s_strd, \s_strd, \w + sub \s_strd, \s_strd, #8 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w +.endif +161: + vld1.8 {d16, d17, d18}, [\src]! + vld1.8 {d24, d25, d26}, [\sr2]! + mov \mx, \w + vmovl.u8 q10, d18 + vmovl.u8 q9, d17 + vmovl.u8 q8, d16 + vmovl.u8 q14, d26 + vmovl.u8 q13, d25 + vmovl.u8 q12, d24 + +16: + vmul.s16 q1, q8, d0[0] + vmul.s16 q2, q9, d0[0] + vmul.s16 q3, q12, d0[0] + vmul.s16 q4, q13, d0[0] +.irpc i, 1234567 + vext.8 q5, q8, q9, #(2*\i) + vext.8 q6, q9, q10, #(2*\i) + vext.8 q11, q12, q13, #(2*\i) + vext.8 q15, q13, q14, #(2*\i) +.if \i < 4 + vmla.s16 q1, q5, d0[\i] + vmla.s16 q2, q6, d0[\i] + vmla.s16 q3, q11, d0[\i] + vmla.s16 q4, q15, d0[\i] +.else + vmla.s16 q1, q5, d1[\i-4] + vmla.s16 q2, q6, d1[\i-4] + vmla.s16 q3, q11, d1[\i-4] + vmla.s16 q4, q15, d1[\i-4] +.endif +.endr + vrshr.s16 q1, q1, #2 + vrshr.s16 q2, q2, #2 + vrshr.s16 q3, q3, #2 + vrshr.s16 q4, q4, #2 + subs \mx, \mx, #16 +.ifc \type, put + vqrshrun.s16 d2, q1, #4 + vqrshrun.s16 d3, q2, #4 + vqrshrun.s16 d4, q3, #4 + vqrshrun.s16 d5, q4, #4 + vst1.8 {q1}, [\dst, :128]! + vst1.8 {q2}, [\ds2, :128]! +.else + vst1.16 {q1, q2}, [\dst, :128]! + vst1.16 {q3, q4}, [\ds2, :128]! +.endif + ble 9f + + vmov q8, q10 + vmov q12, q14 + vld1.8 {d18, d19}, [\src]! + vld1.8 {d26, d27}, [\sr2]! + vmovl.u8 q10, d19 + vmovl.u8 q9, d18 + vmovl.u8 q14, d27 + vmovl.u8 q13, d26 + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + bgt 161b + vpop {q4-q6} + pop {r4-r11,pc} + +L(\type\()_8tap_v): + cmp \h, #4 + ubfx r9, \my, #7, #7 + and \my, \my, #0x7f + it gt + movgt \my, r9 + add \my, r10, \my, lsl #3 + + adr r9, L(\type\()_8tap_v_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_8tap_v_tbl): + .word 1280f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + +20: // 2xN v +.ifc \type, put + bgt 28f + + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + // 2x2 v + load_16 \src, \sr2, \s_strd, d1, d2, d3, d4, d5 + interleave_1_16 d1, d2, d3, d4, d5 + bgt 24f + vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4 + mul_mla_4 d6, d16, d18, d20, d22 + vqrshrun_s16 6, q3, d6 + st_16 \d_strd, d6, 2 + pop {r4-r11,pc} + +24: // 2x4 v + load_16 \sr2, \src, \s_strd, d6, d7 + interleave_1_16 d5, d6, d7 + vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4, q12, d5, q13, d6 + vmov d17, d20 + vmov d19, d22 + vmov d21, d24 + vmov d23, d26 + mul_mla_4 q3, q8, q9, q10, q11 + vqrshrun_s16 6, q3, d6 + st_16 \d_strd, d6, 4 + pop {r4-r11,pc} + +28: // 2x6, 2x8, 2x12, 2x16 v + vpush {q4-q7} + vld1.8 {d0}, [\my, :64] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 + + load_16 \src, \sr2, \s_strd, d2, d4, d6, d8, d10, d12, d14 + interleave_1_16 d2, d4, d6, d8, d10 + interleave_1_16 d10, d12, d14 + vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q5, d10, q6, d12 + vmov d3, d6 + vmov d5, d8 + vmov d7, d10 + vmov d9, d12 +216: + subs \h, \h, #4 + load_16 \sr2, \src, \s_strd, d16, d18, d20, d22 + interleave_1_16 d14, d16, d18, d20, d22 + vmovl_u8 q7, d14, q8, d16, q9, d18, q10, d20 + vmov d11, d14 + vmov d13, d16 + vmov d15, d18 + vmov d17, d20 + mul_mla_8_0 q1, q1, q2, q3, q4, q5, q6, q7, q8 + vqrshrun_s16 6, q1, d2 + st_16 \d_strd, d2, 4 + ble 0f + cmp \h, #2 + vmov q1, q5 + vmov q2, q6 + vmov q3, q7 + vmov q4, q8 + vmov q5, q9 + vmov q6, q10 + vmov d14, d22 + beq 26f + b 216b +26: + load_16 \sr2, \src, \s_strd, d16, d18 + interleave_1_16 d14, d16, d18 + vmovl_u8 q7, d14, q8, d16 + vmov d11, d14 + vmov d13, d16 + mul_mla_8_0 d2, d2, d4, d6, d8, d10, d12, d14, d16 + vqrshrun_s16 6, q1, d2 + st_16 \d_strd, d2, 2 +0: + vpop {q4-q7} + pop {r4-r11,pc} +.endif + +40: + bgt 480f + + // 4x2, 4x4 v + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_32 \src, \sr2, \s_strd, d1, d2, d3, d4, d5 + interleave_1_32 d1, d2, d3, d4, d5 + vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4 + mul_mla_4 q3, q8, q9, q10, q11 + shift_store_4 \type, \d_strd, q3, d6, d7 + ble 0f + load_32 \sr2, \src, \s_strd, d6, d7 + interleave_1_32 d5, d6, d7 + vmovl_u8 q12, d5, q13, d6 + mul_mla_4 q3, q10, q11, q12, q13 + shift_store_4 \type, \d_strd, q3, d6, d7 +0: + pop {r4-r11,pc} + +480: // 4x6, 4x8, 4x12, 4x16 v + vpush {q4} + vld1.8 {d0}, [\my, :64] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_32 \src, \sr2, \s_strd, d2, d4, d6, d8, d16, d18, d20 + interleave_1_32 d2, d4, d6 + interleave_1_32 d6, d8, d16, d18, d20 + vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q8, d16, q9, d18 + +48: + subs \h, \h, #4 + load_32 \sr2, \src, \s_strd, d22, d24, d26, d28 + interleave_1_32 d20, d22, d24, d26, d28 + vmovl_u8 q10, d20, q11, d22, q12, d24, q13, d26 + mul_mla_8_2 q1, q2, q1, q2, q3, q4, q8, q9, q10, q11, q12, q13 + shift_store_4 \type, \d_strd, q1, d2, d3, q2, d4, d5 + ble 0f + load_32 \sr2, \src, \s_strd, d30, d2 + subs \h, \h, #2 + interleave_1_32 d28, d30, d2 + vmovl_u8 q14, d28, q15, d30 + mul_mla_8_0 q8, q8, q9, q10, q11, q12, q13, q14, q15 + shift_store_4 \type, \d_strd, q8, d16, d17 + ble 0f + load_32 \sr2, \src, \s_strd, d4, d6 + subs \h, \h, #2 + interleave_1_32 d2, d4, d6 + vmovl_u8 q1, d2, q2, d4 + mul_mla_8_0 q9, q10, q11, q12, q13, q14, q15, q1, q2 + shift_store_4 \type, \d_strd, q9, d18, d19 + ble 0f + subs \h, \h, #4 + load_32 \sr2, \src, \s_strd, d8, d16, d18, d20 + interleave_1_32 d6, d8, d16, d18, d20 + vmovl_u8 q3, d6, q4, d8, q8, d16, q9, d18 + mul_mla_8_2 q12, q13, q12, q13, q14, q15, q1, q2, q3, q4, q8, q9 + shift_store_4 \type, \d_strd, q12, d24, d25, q13, d26, d27 + bgt 48b +0: + vpop {q4} + pop {r4-r11,pc} + +80: + bgt 880f + + // 8x2, 8x4 v + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_reg \src, \sr2, \s_strd, d1, d2, d3, d4, d5 + vmovl_u8 q8, d1, q9, d2, q10, d3, q11, d4, q12, d5 + mul_mla_4 q1, q8, q9, q10, q11 + mul_mla_4 q2, q9, q10, q11, q12 + shift_store_8 \type, \d_strd, q1, d2, q2, d4 + ble 0f + load_reg \sr2, \src, \s_strd, d6, d7 + vmovl_u8 q13, d6, q14, d7 + mul_mla_4 q1, q10, q11, q12, q13 + mul_mla_4 q2, q11, q12, q13, q14 + shift_store_8 \type, \d_strd, q1, d2, q2, d4 +0: + pop {r4-r11,pc} + +880: // 8x6, 8x8, 8x16, 8x32 v +1680: // 16x8, 16x16, ... +320: // 32x8, 32x16, ... +640: +1280: + vpush {q4} + vld1.8 {d0}, [\my, :64] + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + vmovl.s8 q0, d0 + mov \my, \h +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + load_reg \src, \sr2, \s_strd, d2, d4, d6, d8, d16, d18, d20 + vmovl_u8 q1, d2, q2, d4, q3, d6, q4, d8, q8, d16, q9, d18, q10, d20 + +88: + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, d22, d24 + vmovl_u8 q11, d22, q12, d24 + mul_mla_8_1 q1, q2, q1, q2, q3, q4, q8, q9, q10, q11, q12 + shift_store_8 \type, \d_strd, q1, d2, q2, d4 + ble 9f + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, d26, d28 + vmovl_u8 q13, d26, q14, d28 + mul_mla_8_1 q3, q4, q3, q4, q8, q9, q10, q11, q12, q13, q14 + shift_store_8 \type, \d_strd, q3, d6, q4, d8 + ble 9f + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, d30, d2 + vmovl_u8 q15, d30, q1, d2 + mul_mla_8_1 q8, q9, q8, q9, q10, q11, q12, q13, q14, q15, q1 + shift_store_8 \type, \d_strd, q8, d16, q9, d18 + ble 9f + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, d4, d6 + vmovl_u8 q2, d4, q3, d6 + mul_mla_8_1 q10, q11, q10, q11, q12, q13, q14, q15, q1, q2, q3 + shift_store_8 \type, \d_strd, q10, d20, q11, d22 + ble 9f + subs \h, \h, #4 + load_reg \sr2, \src, \s_strd, d8, d16, d18, d20 + vmovl_u8 q4, d8, q8, d16, q9, d18, q10, d20 + mul_mla_8_1 q12, q13, q12, q13, q14, q15, q1, q2, q3, q4, q8 + mul_mla_8_1 q14, q15, q14, q15, q1, q2, q3, q4, q8, q9, q10 + shift_store_8 \type, \d_strd, q12, d24, q13, d26, q14, d28, q15, d30 + bgt 88b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 168b +0: + vpop {q4} + pop {r4-r11,pc} + +160: + bgt 1680b + + // 16x2, 16x4 v + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + cmp \h, #2 + load_reg \src, \sr2, \s_strd, q11, q12, q13, q14, q15 + vmovl.u8 q1, d22 + vmovl.u8 q2, d24 + vmovl.u8 q3, d26 + vmovl.u8 q8, d28 + vmovl.u8 q9, d30 + vmovl.u8 q11, d23 + vmovl.u8 q12, d25 + vmovl.u8 q13, d27 + vmovl.u8 q14, d29 + vmovl.u8 q15, d31 + mul_mla_4 q1, q1, q2, q3, q8 + mul_mla_4 q10, q2, q3, q8, q9 + mul_mla_4 q2, q11, q12, q13, q14 + mul_mla_4 q11, q12, q13, q14, q15 + shift_store_16 \type, \d_strd, q1, d2, d3, q2, q10, d20, d21, q11 + ble 0f + load_reg \sr2, \src, \s_strd, q10, q11 + vmovl.u8 q1, d20 + vmovl.u8 q10, d21 + vmovl.u8 q12, d22 + vmovl.u8 q11, d23 + mul_mla_4 q2, q3, q8, q9, q1 + mul_mla_4 q3, q13, q14, q15, q10 + mul_mla_4 q13, q8, q9, q1, q12 + mul_mla_4 q14, q14, q15, q10, q11 + shift_store_16 \type, \d_strd, q2, d4, d5, q3, q13, d26, d27, q14 +0: + pop {r4-r11,pc} + +L(\type\()_8tap_hv): + cmp \h, #4 + ubfx r9, \my, #7, #7 + and \my, \my, #0x7f + it gt + movgt \my, r9 + add \my, r10, \my, lsl #3 + + adr r9, L(\type\()_8tap_hv_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_8tap_hv_tbl): + .word 1280f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + +20: +.ifc \type, put + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + bgt 280f + add \my, \my, #2 + vld1.32 {d2[]}, [\my] + + // 2x2, 2x4 hv + sub \sr2, \src, #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + + vld1.8 {d26}, [\src], \s_strd + vmovl.u8 q13, d26 + vext.8 q14, q13, q13, #2 + vmul.s16 d26, d26, d0 + vmul.s16 d28, d28, d0 + vpadd.s16 d26, d26, d28 + vpadd.s16 d26, d26, d26 + vrshr.s16 d16, d26, #2 + bl L(\type\()_8tap_filter_2) + + vext.8 d16, d16, d16, #4 + vmov d17, d26 + vext.8 d16, d16, d26, #4 + +2: + bl L(\type\()_8tap_filter_2) + + vext.8 d18, d17, d26, #4 + vmull.s16 q2, d16, d2[0] + vmlal.s16 q2, d17, d2[1] + vmlal.s16 q2, d18, d2[2] + vmlal.s16 q2, d26, d2[3] + + vqrshrn.s32 d4, q2, #\shift_hv + vqmovun.s16 d4, q2 + subs \h, \h, #2 + vst1.16 {d4[0]}, [\dst, :16], \d_strd + vst1.16 {d4[1]}, [\ds2, :16], \d_strd + ble 0f + vmov d16, d18 + vmov d17, d26 + b 2b + +280: // 2x8, 2x16, 2x32 hv + vld1.8 {d2}, [\my, :64] + sub \src, \src, #1 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + vld1.8 {d26}, [\src], \s_strd + vmovl.u8 q13, d26 + vext.8 q14, q13, q13, #2 + vmul.s16 d26, d26, d0 + vmul.s16 d28, d28, d0 + vpadd.s16 d26, d26, d28 + vpadd.s16 d26, d26, d26 + vrshr.s16 d16, d26, #2 + + bl L(\type\()_8tap_filter_2) + vext.8 d16, d16, d16, #4 + vmov d17, d26 + vext.8 d16, d16, d26, #4 + bl L(\type\()_8tap_filter_2) + vext.8 d18, d17, d26, #4 + vmov d19, d26 + bl L(\type\()_8tap_filter_2) + vext.8 d20, d19, d26, #4 + vmov d21, d26 + +28: + bl L(\type\()_8tap_filter_2) + vext.8 d22, d21, d26, #4 + vmull.s16 q2, d16, d2[0] + vmlal.s16 q2, d17, d2[1] + vmlal.s16 q2, d18, d2[2] + vmlal.s16 q2, d19, d2[3] + vmlal.s16 q2, d20, d3[0] + vmlal.s16 q2, d21, d3[1] + vmlal.s16 q2, d22, d3[2] + vmlal.s16 q2, d26, d3[3] + + vqrshrn.s32 d4, q2, #\shift_hv + vqmovun.s16 d4, q2 + subs \h, \h, #2 + vst1.16 {d4[0]}, [\dst, :16], \d_strd + vst1.16 {d4[1]}, [\ds2, :16], \d_strd + ble 0f + vmov d16, d18 + vmov d17, d19 + vmov d18, d20 + vmov d19, d21 + vmov d20, d22 + vmov d21, d26 + b 28b + +0: + pop {r4-r11,pc} + +L(\type\()_8tap_filter_2): + vld1.8 {d28}, [\sr2], \s_strd + vld1.8 {d30}, [\src], \s_strd + vext.8 d29, d28, d28, #1 + vext.8 d31, d30, d30, #1 + vmovl.u8 q13, d28 + vmovl.u8 q14, d29 + vmov d27, d28 + vmovl.u8 q14, d30 + vmovl.u8 q15, d31 + vtrn.32 d26, d28 + vtrn.32 d27, d30 + vmul.s16 d26, d26, d0[0] + vmla.s16 d26, d27, d0[1] + vmla.s16 d26, d28, d0[2] + vmla.s16 d26, d30, d0[3] + vrshr.s16 d26, d26, #2 + vext.8 d27, d26, d26, #4 + bx lr +.endif + +40: + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + bgt 480f + add \my, \my, #2 + vld1.32 {d2[]}, [\my] + sub \sr2, \src, #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + // 4x2, 4x4 hv + vld1.8 {d30}, [\src], \s_strd + vmovl.u8 q14, d30 + vext.8 d27, d28, d29, #2 + vext.8 d30, d28, d29, #4 + vext.8 d31, d28, d29, #6 + vmul.s16 d26, d28, d0[0] + vmla.s16 d26, d27, d0[1] + vmla.s16 d26, d30, d0[2] + vmla.s16 d26, d31, d0[3] + vrshr.s16 d16, d26, #2 + + bl L(\type\()_8tap_filter_4) + vmov d17, d26 + vmov d18, d27 + +4: + bl L(\type\()_8tap_filter_4) + vmull.s16 q2, d16, d2[0] + vmlal.s16 q2, d17, d2[1] + vmlal.s16 q2, d18, d2[2] + vmlal.s16 q2, d26, d2[3] + vmull.s16 q3, d17, d2[0] + vmlal.s16 q3, d18, d2[1] + vmlal.s16 q3, d26, d2[2] + vmlal.s16 q3, d27, d2[3] + vqrshrn.s32 d4, q2, #\shift_hv + vqrshrn.s32 d6, q3, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + vqmovun.s16 d4, q2 + vqmovun.s16 d6, q3 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d6[0]}, [\ds2, :32], \d_strd +.else + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d6}, [\ds2, :64], \d_strd +.endif + ble 0f + vmov d16, d18 + vmov d17, d26 + vmov d18, d27 + b 4b + +480: // 4x8, 4x16, 4x32 hv + vld1.8 {d2}, [\my, :64] + sub \src, \src, #1 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + vld1.8 {d30}, [\src], \s_strd + vmovl.u8 q14, d30 + vext.8 d27, d28, d29, #2 + vext.8 d30, d28, d29, #4 + vext.8 d31, d28, d29, #6 + vmul.s16 d26, d28, d0[0] + vmla.s16 d26, d27, d0[1] + vmla.s16 d26, d30, d0[2] + vmla.s16 d26, d31, d0[3] + vrshr.s16 d16, d26, #2 + + bl L(\type\()_8tap_filter_4) + vmov d17, d26 + vmov d18, d27 + bl L(\type\()_8tap_filter_4) + vmov d19, d26 + vmov d20, d27 + bl L(\type\()_8tap_filter_4) + vmov d21, d26 + vmov d22, d27 + +48: + bl L(\type\()_8tap_filter_4) + vmull.s16 q2, d16, d2[0] + vmlal.s16 q2, d17, d2[1] + vmlal.s16 q2, d18, d2[2] + vmlal.s16 q2, d19, d2[3] + vmlal.s16 q2, d20, d3[0] + vmlal.s16 q2, d21, d3[1] + vmlal.s16 q2, d22, d3[2] + vmlal.s16 q2, d26, d3[3] + vmull.s16 q3, d17, d2[0] + vmlal.s16 q3, d18, d2[1] + vmlal.s16 q3, d19, d2[2] + vmlal.s16 q3, d20, d2[3] + vmlal.s16 q3, d21, d3[0] + vmlal.s16 q3, d22, d3[1] + vmlal.s16 q3, d26, d3[2] + vmlal.s16 q3, d27, d3[3] + vqrshrn.s32 d4, q2, #\shift_hv + vqrshrn.s32 d6, q3, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + vqmovun.s16 d4, q2 + vqmovun.s16 d6, q3 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d6[0]}, [\ds2, :32], \d_strd +.else + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d6}, [\ds2, :64], \d_strd +.endif + ble 0f + vmov d16, d18 + vmov d17, d19 + vmov d18, d20 + vmov d19, d21 + vmov d20, d22 + vmov d21, d26 + vmov d22, d27 + b 48b +0: + pop {r4-r11,pc} + +L(\type\()_8tap_filter_4): + vld1.8 {d30}, [\sr2], \s_strd + vld1.8 {d31}, [\src], \s_strd + vmovl.u8 q14, d30 + vext.8 d27, d28, d29, #2 + vext.8 d30, d28, d29, #4 + vext.8 d1, d28, d29, #6 + vmul.s16 d26, d28, d0[0] + vmla.s16 d26, d27, d0[1] + vmla.s16 d26, d30, d0[2] + vmla.s16 d26, d1, d0[3] + + vmovl.u8 q14, d31 + vext.8 d30, d28, d29, #2 + vext.8 d31, d28, d29, #4 + vext.8 d1, d28, d29, #6 + vmul.s16 d27, d28, d0[0] + vmla.s16 d27, d30, d0[1] + vmla.s16 d27, d31, d0[2] + vmla.s16 d27, d1, d0[3] + vrshr.s16 d26, d26, #2 + vrshr.s16 d27, d27, #2 + bx lr + +80: +160: +320: + bgt 880f + vpush {q4-q7} + add \my, \my, #2 + vld1.8 {d0}, [\mx, :64] + vld1.32 {d2[]}, [\my] + sub \src, \src, #3 + sub \src, \src, \s_strd + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + mov \my, \h + +164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + vld1.8 {q14}, [\src], \s_strd + vmovl.u8 q12, d28 + vmovl.u8 q13, d29 + vmul.s16 q10, q12, d0[0] +.irpc i, 123 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d0[\i] +.endr +.irpc i, 4567 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d1[\i-4] +.endr + vrshr.s16 q3, q10, #2 + + bl L(\type\()_8tap_filter_8) + vmov q4, q10 + vmov q5, q11 + +8: + bl L(\type\()_8tap_filter_8) + vmull.s16 q12, d6, d2[0] + vmull.s16 q13, d7, d2[0] + vmull.s16 q14, d8, d2[0] + vmull.s16 q15, d9, d2[0] + vmlal.s16 q12, d8, d2[1] + vmlal.s16 q13, d9, d2[1] + vmlal.s16 q14, d10, d2[1] + vmlal.s16 q15, d11, d2[1] + vmlal.s16 q12, d10, d2[2] + vmlal.s16 q13, d11, d2[2] + vmlal.s16 q14, d20, d2[2] + vmlal.s16 q15, d21, d2[2] + vmlal.s16 q12, d20, d2[3] + vmlal.s16 q13, d21, d2[3] + vmlal.s16 q14, d22, d2[3] + vmlal.s16 q15, d23, d2[3] + vqrshrn.s32 d24, q12, #\shift_hv + vqrshrn.s32 d25, q13, #\shift_hv + vqrshrn.s32 d28, q14, #\shift_hv + vqrshrn.s32 d29, q15, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + vqmovun.s16 d24, q12 + vqmovun.s16 d28, q14 + vst1.8 {d24}, [\dst, :64], \d_strd + vst1.8 {d28}, [\ds2, :64], \d_strd +.else + vst1.16 {q12}, [\dst, :128], \d_strd + vst1.16 {q14}, [\ds2, :128], \d_strd +.endif + ble 9f + vmov q3, q5 + vmov q4, q10 + vmov q5, q11 + b 8b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #2 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 164b + +880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv +640: +1280: + vpush {q4-q7} + vld1.8 {d0}, [\mx, :64] + vld1.8 {d2}, [\my, :64] + sub \src, \src, #3 + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + mov \my, \h + +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + vld1.8 {q14}, [\src], \s_strd + vmovl.u8 q12, d28 + vmovl.u8 q13, d29 + vmul.s16 q10, q12, d0[0] +.irpc i, 123 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d0[\i] +.endr +.irpc i, 4567 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d1[\i-4] +.endr + vrshr.s16 q3, q10, #2 + + bl L(\type\()_8tap_filter_8) + vmov q4, q10 + vmov q5, q11 + bl L(\type\()_8tap_filter_8) + vmov q6, q10 + vmov q7, q11 + bl L(\type\()_8tap_filter_8) + vmov q8, q10 + vmov q9, q11 + +88: + bl L(\type\()_8tap_filter_8) + vmull.s16 q12, d6, d2[0] + vmull.s16 q13, d7, d2[0] + vmull.s16 q14, d8, d2[0] + vmull.s16 q15, d9, d2[0] + vmlal.s16 q12, d8, d2[1] + vmlal.s16 q13, d9, d2[1] + vmlal.s16 q14, d10, d2[1] + vmlal.s16 q15, d11, d2[1] + vmlal.s16 q12, d10, d2[2] + vmlal.s16 q13, d11, d2[2] + vmlal.s16 q14, d12, d2[2] + vmlal.s16 q15, d13, d2[2] + vmlal.s16 q12, d12, d2[3] + vmlal.s16 q13, d13, d2[3] + vmlal.s16 q14, d14, d2[3] + vmlal.s16 q15, d15, d2[3] + vmlal.s16 q12, d14, d3[0] + vmlal.s16 q13, d15, d3[0] + vmlal.s16 q14, d16, d3[0] + vmlal.s16 q15, d17, d3[0] + vmlal.s16 q12, d16, d3[1] + vmlal.s16 q13, d17, d3[1] + vmlal.s16 q14, d18, d3[1] + vmlal.s16 q15, d19, d3[1] + vmlal.s16 q12, d18, d3[2] + vmlal.s16 q13, d19, d3[2] + vmlal.s16 q14, d20, d3[2] + vmlal.s16 q15, d21, d3[2] + vmlal.s16 q12, d20, d3[3] + vmlal.s16 q13, d21, d3[3] + vmlal.s16 q14, d22, d3[3] + vmlal.s16 q15, d23, d3[3] + vqrshrn.s32 d24, q12, #\shift_hv + vqrshrn.s32 d25, q13, #\shift_hv + vqrshrn.s32 d28, q14, #\shift_hv + vqrshrn.s32 d29, q15, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + vqmovun.s16 d24, q12 + vqmovun.s16 d28, q14 + vst1.8 {d24}, [\dst, :64], \d_strd + vst1.8 {d28}, [\ds2, :64], \d_strd +.else + vst1.16 {q12}, [\dst, :128], \d_strd + vst1.16 {q14}, [\ds2, :128], \d_strd +.endif + ble 9f + vmov q3, q5 + vmov q4, q6 + vmov q5, q7 + vmov q6, q8 + vmov q7, q9 + vmov q8, q10 + vmov q9, q11 + b 88b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 168b +0: + vpop {q4-q7} + pop {r4-r11,pc} + +L(\type\()_8tap_filter_8): + vld1.8 {q14}, [\sr2], \s_strd + vld1.8 {q15}, [\src], \s_strd + vmovl.u8 q12, d28 + vmovl.u8 q13, d29 + vmul.s16 q10, q12, d0[0] +.irpc i, 123 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d0[\i] +.endr +.irpc i, 4567 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q10, q14, d1[\i-4] +.endr + vmovl.u8 q12, d30 + vmovl.u8 q13, d31 + vmul.s16 q11, q12, d0[0] +.irpc i, 123 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q11, q14, d0[\i] +.endr +.irpc i, 4567 + vext.8 q14, q12, q13, #(2*\i) + vmla.s16 q11, q14, d1[\i-4] +.endr + vrshr.s16 q10, q10, #2 + vrshr.s16 q11, q11, #2 + bx lr +endfunc + + +function \type\()_bilin_8bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + vdup.8 d1, \mx + vdup.8 d3, \my + rsb r8, \mx, #16 + rsb r9, \my, #16 + vdup.8 d0, r8 + vdup.8 d2, r9 +.ifc \type, prep + lsl \d_strd, \w, #1 +.endif + clz r8, \w + cmp \mx, #0 + sub r8, r8, #24 + bne L(\type\()_bilin_h) + cmp \my, #0 + bne L(\type\()_bilin_v) + b \type\()_neon + +L(\type\()_bilin_h): + cmp \my, #0 + bne L(\type\()_bilin_hv) + + adr r9, L(\type\()_bilin_h_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_bilin_h_tbl): + .word 1280f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + +20: // 2xN h +.ifc \type, put + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +2: + vld1.32 {d4[]}, [\src], \s_strd + vld1.32 {d6[]}, [\sr2], \s_strd + vext.8 d5, d4, d4, #1 + vext.8 d7, d6, d6, #1 + vtrn.16 q2, q3 + subs \h, \h, #2 + vmull.u8 q3, d4, d0 + vmlal.u8 q3, d5, d1 + vqrshrn.u16 d4, q3, #4 + vst1.16 {d4[0]}, [\dst, :16], \d_strd + vst1.16 {d4[1]}, [\ds2, :16], \d_strd + bgt 2b + pop {r4-r11,pc} +.endif + +40: // 4xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +4: + vld1.8 {d4}, [\src], \s_strd + vld1.8 {d6}, [\sr2], \s_strd + vext.8 d5, d4, d4, #1 + vext.8 d7, d6, d6, #1 + vtrn.32 q2, q3 + subs \h, \h, #2 + vmull.u8 q3, d4, d0 + vmlal.u8 q3, d5, d1 +.ifc \type, put + vqrshrn.u16 d4, q3, #4 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d4[1]}, [\ds2, :32], \d_strd +.else + vst1.16 {d6}, [\dst, :64], \d_strd + vst1.16 {d7}, [\ds2, :64], \d_strd +.endif + bgt 4b + pop {r4-r11,pc} + +80: // 8xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +8: + vld1.8 {q8}, [\src], \s_strd + vld1.8 {q10}, [\sr2], \s_strd + vext.8 q9, q8, q8, #1 + vext.8 q11, q10, q10, #1 + subs \h, \h, #2 + vmull.u8 q8, d16, d0 + vmull.u8 q10, d20, d0 + vmlal.u8 q8, d18, d1 + vmlal.u8 q10, d22, d1 +.ifc \type, put + vqrshrn.u16 d16, q8, #4 + vqrshrn.u16 d18, q10, #4 + vst1.8 {d16}, [\dst, :64], \d_strd + vst1.8 {d18}, [\ds2, :64], \d_strd +.else + vst1.16 {q8}, [\dst, :128], \d_strd + vst1.16 {q10}, [\ds2, :128], \d_strd +.endif + bgt 8b + pop {r4-r11,pc} +160: +320: +640: +1280: // 16xN, 32xN, ... h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + + sub \s_strd, \s_strd, \w + sub \s_strd, \s_strd, #8 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w +.endif +161: + vld1.8 {d16}, [\src]! + vld1.8 {d22}, [\sr2]! + mov \mx, \w + +16: + vld1.8 {d17,d18}, [\src]! + vld1.8 {d23,d24}, [\sr2]! + vext.8 q10, q8, q9, #1 + vext.8 q13, q11, q12, #1 + vmull.u8 q2, d16, d0 + vmull.u8 q3, d17, d0 + vmull.u8 q14, d22, d0 + vmull.u8 q15, d23, d0 + vmlal.u8 q2, d20, d1 + vmlal.u8 q3, d21, d1 + vmlal.u8 q14, d26, d1 + vmlal.u8 q15, d27, d1 + subs \mx, \mx, #16 +.ifc \type, put + vqrshrn.u16 d4, q2, #4 + vqrshrn.u16 d5, q3, #4 + vqrshrn.u16 d28, q14, #4 + vqrshrn.u16 d29, q15, #4 + vst1.8 {q2}, [\dst, :128]! + vst1.8 {q14}, [\ds2, :128]! +.else + vst1.16 {q2, q3}, [\dst, :128]! + vst1.16 {q14, q15}, [\ds2, :128]! +.endif + ble 9f + + vmov d16, d18 + vmov d22, d24 + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + bgt 161b + pop {r4-r11,pc} + +L(\type\()_bilin_v): + cmp \h, #4 + adr r9, L(\type\()_bilin_v_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_bilin_v_tbl): + .word 1280f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + +20: // 2xN v +.ifc \type, put + cmp \h, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + // 2x2 v + vld1.16 {d16[]}, [\src], \s_strd + bgt 24f +22: + vld1.16 {d17[]}, [\sr2], \s_strd + vld1.16 {d18[]}, [\src], \s_strd + vext.8 d16, d16, d17, #6 + vext.8 d17, d17, d18, #6 + vmull.u8 q2, d16, d2 + vmlal.u8 q2, d17, d3 + vqrshrn.u16 d4, q2, #4 + vst1.16 {d4[0]}, [\dst, :16] + vst1.16 {d4[1]}, [\ds2, :16] + pop {r4-r11,pc} +24: // 2x4, 2x6, 2x8, ... v + vld1.16 {d17[]}, [\sr2], \s_strd + vld1.16 {d18[]}, [\src], \s_strd + vld1.16 {d19[]}, [\sr2], \s_strd + vld1.16 {d20[]}, [\src], \s_strd + sub \h, \h, #4 + vext.8 d16, d16, d17, #6 + vext.8 d17, d17, d18, #6 + vext.8 d18, d18, d19, #6 + vext.8 d19, d19, d20, #6 + vtrn.32 d16, d18 + vtrn.32 d17, d19 + vmull.u8 q2, d16, d2 + vmlal.u8 q2, d17, d3 + cmp \h, #2 + vqrshrn.u16 d4, q2, #4 + vst1.16 {d4[0]}, [\dst, :16], \d_strd + vst1.16 {d4[1]}, [\ds2, :16], \d_strd + vst1.16 {d4[2]}, [\dst, :16], \d_strd + vst1.16 {d4[3]}, [\ds2, :16], \d_strd + blt 0f + vmov d16, d20 + beq 22b + b 24b +0: + pop {r4-r11,pc} +.endif + +40: // 4xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vld1.32 {d16[]}, [\src], \s_strd +4: + vld1.32 {d17[]}, [\sr2], \s_strd + vld1.32 {d18[]}, [\src], \s_strd + vext.8 d16, d16, d17, #4 + vext.8 d17, d17, d18, #4 + vmull.u8 q2, d16, d2 + vmlal.u8 q2, d17, d3 + subs \h, \h, #2 +.ifc \type, put + vqrshrn.u16 d4, q2, #4 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d4[1]}, [\ds2, :32], \d_strd +.else + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d5}, [\ds2, :64], \d_strd +.endif + ble 0f + vmov d16, d18 + b 4b +0: + pop {r4-r11,pc} + +80: // 8xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vld1.8 {d16}, [\src], \s_strd +8: + vld1.8 {d17}, [\sr2], \s_strd + vld1.8 {d18}, [\src], \s_strd + vmull.u8 q2, d16, d2 + vmull.u8 q3, d17, d2 + vmlal.u8 q2, d17, d3 + vmlal.u8 q3, d18, d3 + subs \h, \h, #2 +.ifc \type, put + vqrshrn.u16 d4, q2, #4 + vqrshrn.u16 d6, q3, #4 + vst1.8 {d4}, [\dst, :64], \d_strd + vst1.8 {d6}, [\ds2, :64], \d_strd +.else + vst1.16 {q2}, [\dst, :128], \d_strd + vst1.16 {q3}, [\ds2, :128], \d_strd +.endif + ble 0f + vmov d16, d18 + b 8b +0: + pop {r4-r11,pc} + +160: // 16xN, 32xN, ... +320: +640: +1280: + mov \my, \h +1: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.8 {q8}, [\src], \s_strd +2: + vld1.8 {q9}, [\sr2], \s_strd + vld1.8 {q10}, [\src], \s_strd + vmull.u8 q12, d16, d2 + vmull.u8 q13, d17, d2 + vmull.u8 q14, d18, d2 + vmull.u8 q15, d19, d2 + vmlal.u8 q12, d18, d3 + vmlal.u8 q13, d19, d3 + vmlal.u8 q14, d20, d3 + vmlal.u8 q15, d21, d3 + subs \h, \h, #2 +.ifc \type, put + vqrshrn.u16 d24, q12, #4 + vqrshrn.u16 d25, q13, #4 + vqrshrn.u16 d28, q14, #4 + vqrshrn.u16 d29, q15, #4 + vst1.8 {q12}, [\dst, :128], \d_strd + vst1.8 {q14}, [\ds2, :128], \d_strd +.else + vst1.16 {q12, q13}, [\dst, :128], \d_strd + vst1.16 {q14, q15}, [\ds2, :128], \d_strd +.endif + ble 9f + vmov q8, q10 + b 2b +9: + subs \w, \w, #16 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #16 +.ifc \type, put + add \dst, \dst, #16 +.else + add \dst, \dst, #32 +.endif + b 1b +0: + pop {r4-r11,pc} + +L(\type\()_bilin_hv): + vmovl.u8 q2, d2 + vmovl.u8 q3, d3 + adr r9, L(\type\()_bilin_hv_tbl) + ldr r8, [r9, r8, lsl #2] + add r9, r9, r8 + bx r9 + + .align 2 +L(\type\()_bilin_hv_tbl): + .word 1280f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + +20: // 2xN hv +.ifc \type, put + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.32 {d28[]}, [\src], \s_strd + vext.8 d29, d28, d28, #1 + vmull.u8 q8, d28, d0 + vmlal.u8 q8, d29, d1 + +2: + vld1.32 {d28[]}, [\sr2], \s_strd + vld1.32 {d30[]}, [\src], \s_strd + vext.8 d29, d28, d28, #1 + vext.8 d31, d30, d30, #1 + vtrn.16 d28, d30 + vtrn.16 d29, d31 + vmull.u8 q9, d28, d0 + vmlal.u8 q9, d29, d1 + + vtrn.32 d16, d18 + + vmul.u16 d20, d16, d4 + vmla.u16 d20, d19, d6 + vqrshrn.u16 d20, q10, #8 + subs \h, \h, #2 + vst1.16 {d20[0]}, [\dst, :16], \d_strd + vst1.16 {d20[1]}, [\ds2, :16], \d_strd + ble 0f + vtrn.32 d19, d16 + b 2b +0: + pop {r4-r11,pc} +.endif + +40: // 4xN hv + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.8 {d28}, [\src], \s_strd + vext.8 d29, d28, d28, #1 + vmull.u8 q8, d28, d0 + vmlal.u8 q8, d29, d1 + +4: + vld1.8 {d28}, [\sr2], \s_strd + vld1.8 {d30}, [\src], \s_strd + vext.8 d29, d28, d28, #1 + vext.8 d31, d30, d30, #1 + vtrn.32 d28, d30 + vtrn.32 d29, d31 + vmull.u8 q9, d28, d0 + vmlal.u8 q9, d29, d1 + + vmov d17, d18 + + vmul.u16 q10, q8, q2 + vmla.u16 q10, q9, q3 + subs \h, \h, #2 +.ifc \type, put + vqrshrn.u16 d20, q10, #8 + vst1.32 {d20[0]}, [\dst, :32], \d_strd + vst1.32 {d20[1]}, [\ds2, :32], \d_strd +.else + vrshr.u16 q10, q10, #4 + vst1.16 {d20}, [\dst, :64], \d_strd + vst1.16 {d21}, [\ds2, :64], \d_strd +.endif + ble 0f + vmov d16, d19 + b 4b +0: + pop {r4-r11,pc} + +80: // 8xN, 16xN, ... hv +160: +320: +640: +1280: + mov \my, \h + +1: + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.8 {q12}, [\src], \s_strd + vext.8 q13, q12, q12, #1 + vmull.u8 q8, d24, d0 + vmlal.u8 q8, d26, d1 + +2: + vld1.8 {q12}, [\sr2], \s_strd + vld1.8 {q14}, [\src], \s_strd + vext.8 q13, q12, q12, #1 + vext.8 q15, q14, q14, #1 + vmull.u8 q9, d24, d0 + vmlal.u8 q9, d26, d1 + vmull.u8 q10, d28, d0 + vmlal.u8 q10, d30, d1 + + vmul.u16 q8, q8, q2 + vmla.u16 q8, q9, q3 + vmul.u16 q9, q9, q2 + vmla.u16 q9, q10, q3 + subs \h, \h, #2 +.ifc \type, put + vqrshrn.u16 d16, q8, #8 + vqrshrn.u16 d18, q9, #8 + vst1.8 {d16}, [\dst, :64], \d_strd + vst1.8 {d18}, [\ds2, :64], \d_strd +.else + vrshr.u16 q8, q8, #4 + vrshr.u16 q9, q9, #4 + vst1.16 {q8}, [\dst, :128], \d_strd + vst1.16 {q9}, [\ds2, :128], \d_strd +.endif + ble 9f + vmov q8, q10 + b 2b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 1b +0: + pop {r4-r11,pc} +endfunc +.endm + +filter_fn put, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, 10 +filter_fn prep, r0, r7, r1, r2, r3, r4, r5, r6, r8, r9, 6 + +.macro load_filter_ptr src + asr r12, \src, #10 + add r12, r11, r12, lsl #3 +.endm + +.macro load_filter_coef dst, src, inc + add \src, \src, \inc + vld1.8 {\dst}, [r12, :64] +.endm + +.macro load_filter_row dst, src, inc + load_filter_ptr \src + load_filter_coef \dst, \src, \inc +.endm + +function warp_filter_horz_neon + load_filter_ptr r5 // filter 0 + vld1.16 {q7}, [r2], r3 + vmov.i8 q6, #128 + + load_filter_coef d0, r5, r7 // filter 0 + load_filter_row d1, r5, r7 // filter 1 + load_filter_row d2, r5, r7 // filter 2 + load_filter_ptr r5 // filter 3 + veor q7, q7, q6 // subtract by 128 to allow using vmull + load_filter_coef d3, r5, r7 // filter 3 + vext.8 d12, d14, d15, #1 // filter 1 pixels + vext.8 d13, d14, d15, #2 // filter 2 pixels + load_filter_ptr r5 // filter 4 + vmull.s8 q2, d14, d0 // filter 0 output + vmull.s8 q3, d12, d1 // filter 1 output + load_filter_coef d0, r5, r7 // filter 4 + load_filter_ptr r5 // filter 5 + vext.8 d12, d14, d15, #3 // filter 3 pixels + vmull.s8 q4, d13, d2 // filter 2 output + vext.8 d13, d14, d15, #4 // filter 4 pixels + vpadd.i16 d4, d4, d5 // pixel 0 (4x16) + vpadd.i16 d5, d6, d7 // pixel 1 (4x16) + load_filter_coef d1, r5, r7 // filter 5 + load_filter_ptr r5 // filter 6 + vmull.s8 q5, d12, d3 // filter 3 output + vext.8 d12, d14, d15, #5 // filter 5 pixels + vmull.s8 q3, d13, d0 // filter 4 output + load_filter_coef d0, r5, r7 // filter 6 + vext.8 d13, d14, d15, #6 // filter 6 pixels + load_filter_ptr r5 // filter 7 + vpadd.i16 d8, d8, d9 // pixel 2 (4x16) + vpadd.i16 d9, d10, d11 // pixel 3 (4x16) + vmull.s8 q5, d12, d1 // filter 5 output + load_filter_coef d1, r5, r7 // filter 7 + vext.8 d14, d14, d15, #7 // filter 7 pixels + vpadd.i16 d6, d6, d7 // pixel 4 (4x16) + vpadd.i16 d10, d10, d11 // pixel 5 (4x16) + vmull.s8 q6, d13, d0 // filter 6 output + vmull.s8 q7, d14, d1 // filter 7 output + + sub r5, r5, r7, lsl #3 + + vpadd.i16 d4, d4, d5 // pixel 0,1 (2x16) + vpadd.i16 d5, d8, d9 // pixel 2,3 (2x16) + vpadd.i16 d12, d12, d13 // pixel 6 (4x16) + vpadd.i16 d14, d14, d15 // pixel 7 (4x16) + vpadd.i16 d6, d6, d10 // pixel 4,5 (2x16) + vpadd.i16 d10, d12, d14 // pixel 6,7 (2x16) + vpadd.i16 d4, d4, d5 // pixel 0-3 + vpadd.i16 d5, d6, d10 // pixel 4-7 + + add r5, r5, r8 + + bx lr +endfunc + +// void dav1d_warp_affine_8x8_8bpc_neon( +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *const abcd, int mx, int my) +.macro warp t, shift +function warp_affine_8x8\t\()_8bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldr r6, [sp, #108] + ldrd r8, r9, [r4] + sxth r7, r8 + asr r8, r8, #16 + asr r4, r9, #16 + sxth r9, r9 + mov r10, #8 + sub r2, r2, r3, lsl #1 + sub r2, r2, r3 + sub r2, r2, #3 + movrel r11, X(mc_warp_filter), 64*8 +.ifnb \t + lsl r1, r1, #1 +.endif + add r5, r5, #512 + add r6, r6, #512 + + bl warp_filter_horz_neon + vrshr.s16 q8, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q9, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q10, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q11, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q12, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q13, q2, #3 + bl warp_filter_horz_neon + vrshr.s16 q14, q2, #3 + +1: + bl warp_filter_horz_neon + vrshr.s16 q15, q2, #3 + + load_filter_row d8, r6, r9 + load_filter_row d9, r6, r9 + load_filter_row d10, r6, r9 + load_filter_row d11, r6, r9 + load_filter_row d12, r6, r9 + load_filter_row d13, r6, r9 + load_filter_row d14, r6, r9 + load_filter_row d15, r6, r9 + transpose_8x8b q4, q5, q6, q7, d8, d9, d10, d11, d12, d13, d14, d15 + vmovl.s8 q1, d8 + vmovl.s8 q2, d9 + vmovl.s8 q3, d10 + vmovl.s8 q4, d11 + vmovl.s8 q5, d12 + vmovl.s8 q6, d13 + + sub r6, r6, r9, lsl #3 + + // This ordering of vmull/vmlal is highly beneficial for + // Cortex A8/A9/A53 here, but harmful for Cortex A7. + vmull.s16 q0, d16, d2 + vmlal.s16 q0, d18, d4 + vmlal.s16 q0, d20, d6 + vmlal.s16 q0, d22, d8 + vmlal.s16 q0, d24, d10 + vmlal.s16 q0, d26, d12 + vmull.s16 q1, d17, d3 + vmlal.s16 q1, d19, d5 + vmlal.s16 q1, d21, d7 + vmlal.s16 q1, d23, d9 + vmlal.s16 q1, d25, d11 + vmlal.s16 q1, d27, d13 + + vmovl.s8 q2, d14 + vmovl.s8 q3, d15 + + vmlal.s16 q0, d28, d4 + vmlal.s16 q0, d30, d6 + vmlal.s16 q1, d29, d5 + vmlal.s16 q1, d31, d7 + +.ifb \t + vmov.i16 q7, #128 +.else + vmov.i16 q7, #0x800 +.endif + + vmov q8, q9 + vmov q9, q10 + vqrshrn.s32 d0, q0, #\shift + vmov q10, q11 + vqrshrn.s32 d1, q1, #\shift + vmov q11, q12 + vadd.i16 q0, q0, q7 + vmov q12, q13 +.ifb \t + vqmovun.s16 d0, q0 +.endif + vmov q13, q14 + vmov q14, q15 + subs r10, r10, #1 +.ifnb \t + vst1.16 {q0}, [r0, :128], r1 +.else + vst1.8 {d0}, [r0, :64], r1 +.endif + + add r6, r6, r4 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +warp , 11 +warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + ldrd r8, r9, [sp, #52] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub r12, r3, #1 // ih - 1 + cmp r5, r3 + sub lr, r2, #1 // iw - 1 + it lt + movlt r12, r5 // min(y, ih - 1) + cmp r4, r2 + bic r12, r12, r12, asr #31 // max(min(y, ih - 1), 0) + it lt + movlt lr, r4 // min(x, iw - 1) + bic lr, lr, lr, asr #31 // max(min(x, iw - 1), 0) + mla r8, r12, r9, r8 // ref += iclip() * stride + add r8, r8, lr // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add r10, r5, r1 // y + bh + neg r5, r5 // -y + sub r10, r10, r3 // y + bh - ih + sub r12, r1, #1 // bh - 1 + cmp r10, r1 + bic r5, r5, r5, asr #31 // max(-y, 0) + it ge + movge r10, r12 // min(y + bh - ih, bh-1) + cmp r5, r1 + bic r10, r10, r10, asr #31 // max(min(y + bh - ih, bh-1), 0) + it ge + movge r5, r12 // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add r11, r4, r0 // x + bw + neg r4, r4 // -x + sub r11, r11, r2 // x + bw - iw + sub lr, r0, #1 // bw - 1 + cmp r11, r0 + bic r4, r4, r4, asr #31 // max(-x, 0) + it ge + movge r11, lr // min(x + bw - iw, bw-1) + cmp r4, r0 + bic r11, r11, r11, asr #31 // max(min(x + bw - iw, bw-1), 0) + it ge + movge r4, lr // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub r1, r1, r5 // bh - top_ext + mla r6, r5, r7, r6 + sub r2, r0, r4 // bw - left_ext + sub r1, r1, r10 // center_h = bh - top_ext - bottom_ext + sub r2, r2, r11 // center_w = bw - left_ext - right_ext + + mov r0, r6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + vld1.8 {d0[], d1[]}, [r8] + mov r12, r6 // out = dst + mov r3, r4 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12, :128]! + bgt 1b +.endif + mov lr, r8 + add r12, r6, r4 // out = dst + left_ext + mov r3, r2 +1: + vld1.8 {q0, q1}, [lr]! + subs r3, r3, #32 +.if \need_left + vst1.8 {q0, q1}, [r12]! +.else + vst1.8 {q0, q1}, [r12, :128]! +.endif + bgt 1b +.if \need_right + add r3, r8, r2 // in + center_w + sub r3, r3, #1 // in + center_w - 1 + add r12, r6, r4 // dst + left_ext + vld1.8 {d0[], d1[]}, [r3] + add r12, r12, r2 // out = dst + left_ext + center_w + mov r3, r11 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12]! + bgt 1b +.endif + + subs r1, r1, #1 // center_h-- + add r6, r6, r7 + add r8, r8, r9 + bgt 0b +.endm + + cmp r4, #0 + beq 2f + // need_left + cmp r11, #0 + beq 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cmp r11, #0 + beq 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + cmp r10, #0 + // Storing the original dst in r0 overwrote bw, recalculate it here + add r2, r2, r4 // center_w + left_ext + add r2, r2, r11 // bw = center_w + left_ext + right_ext + + beq 3f + // need_bottom + sub r8, r6, r7 // ref = dst - stride + mov r4, r2 +1: + vld1.8 {q0, q1}, [r8, :128]! + mov r3, r10 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r10, r6 // dst -= bottom_ext * stride + subs r4, r4, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + cmp r5, #0 + beq 3f + // need_top + mls r6, r7, r5, r0 // dst = stored_dst - top_ext * stride +1: + vld1.8 {q0, q1}, [r0, :128]! + mov r3, r5 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r5, r6 // dst -= top_ext * stride + subs r2, r2, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/mc16.S b/third_party/dav1d/src/arm/32/mc16.S new file mode 100644 index 0000000000..b7d845e219 --- /dev/null +++ b/third_party/dav1d/src/arm/32/mc16.S @@ -0,0 +1,3658 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define PREP_BIAS 8192 + +.macro avg d0, d00, d01, d1, d10, d11 + vld1.16 {q0, q1}, [r2, :128]! + vld1.16 {q2, q3}, [r3, :128]! + vqadd.s16 q0, q0, q2 + vqadd.s16 q1, q1, q3 + vmax.s16 q0, q0, q12 // -2*PREP_BIAS - 1 << intermediate_bits + vmax.s16 q1, q1, q12 // -2*PREP_BIAS - 1 << intermediate_bits + vqsub.s16 q0, q0, q12 // -2*PREP_BIAS - 1 << intermediate_bits + vqsub.s16 q1, q1, q12 // -2*PREP_BIAS - 1 << intermediate_bits + vshl.s16 \d0, q0, q13 // -(intermediate_bits+1) + vshl.s16 \d1, q1, q13 // -(intermediate_bits+1) +.endm + +.macro w_avg d0, d00, d01, d1, d10, d11 + vld1.16 {q0, q1}, [r2, :128]! + vld1.16 {q2, q3}, [r3, :128]! + // This difference requires a 17 bit range, and all bits are + // significant for the following multiplication. + vsubl.s16 \d0, d4, d0 + vsubl.s16 q0, d5, d1 + vsubl.s16 \d1, d6, d2 + vsubl.s16 q1, d7, d3 + vmul.s32 \d0, \d0, q4 + vmul.s32 q0, q0, q4 + vmul.s32 \d1, \d1, q4 + vmul.s32 q1, q1, q4 + vshr.s32 \d0, \d0, #4 + vshr.s32 q0, q0, #4 + vshr.s32 \d1, \d1, #4 + vshr.s32 q1, q1, #4 + vaddw.s16 \d0, \d0, d4 + vaddw.s16 q0, q0, d5 + vaddw.s16 \d1, \d1, d6 + vaddw.s16 q1, q1, d7 + vmovn.i32 \d00, \d0 + vmovn.i32 \d01, q0 + vmovn.i32 \d10, \d1 + vmovn.i32 \d11, q1 + vrshl.s16 \d0, \d0, q13 // -intermediate_bits + vrshl.s16 \d1, \d1, q13 // -intermediate_bits + vadd.s16 \d0, \d0, q12 // PREP_BIAS >> intermediate_bits + vadd.s16 \d1, \d1, q12 // PREP_BIAS >> intermediate_bits + vmin.s16 \d0, \d0, q15 // bitdepth_max + vmin.s16 \d1, \d1, q15 // bitdepth_max + vmax.s16 \d0, \d0, q14 // 0 + vmax.s16 \d1, \d1, q14 // 0 +.endm + +.macro mask d0, d00, d01, d1, d10, d11 + vld1.8 {q7}, [r6, :128]! + vld1.16 {q0, q1}, [r2, :128]! + vneg.s8 q7, q7 + vld1.16 {q2, q3}, [r3, :128]! + vmovl.s8 q6, d14 + vmovl.s8 q7, d15 + vmovl.s16 q4, d12 + vmovl.s16 q5, d13 + vmovl.s16 q6, d14 + vmovl.s16 q7, d15 + vsubl.s16 \d0, d4, d0 + vsubl.s16 q0, d5, d1 + vsubl.s16 \d1, d6, d2 + vsubl.s16 q1, d7, d3 + vmul.s32 \d0, \d0, q4 + vmul.s32 q0, q0, q5 + vmul.s32 \d1, \d1, q6 + vmul.s32 q1, q1, q7 + vshr.s32 \d0, \d0, #6 + vshr.s32 q0, q0, #6 + vshr.s32 \d1, \d1, #6 + vshr.s32 q1, q1, #6 + vaddw.s16 \d0, \d0, d4 + vaddw.s16 q0, q0, d5 + vaddw.s16 \d1, \d1, d6 + vaddw.s16 q1, q1, d7 + vmovn.i32 \d00, \d0 + vmovn.i32 \d01, q0 + vmovn.i32 \d10, \d1 + vmovn.i32 \d11, q1 + vrshl.s16 \d0, \d0, q13 // -intermediate_bits + vrshl.s16 \d1, \d1, q13 // -intermediate_bits + vadd.s16 \d0, \d0, q12 // PREP_BIAS >> intermediate_bits + vadd.s16 \d1, \d1, q12 // PREP_BIAS >> intermediate_bits + vmin.s16 \d0, \d0, q15 // bitdepth_max + vmin.s16 \d1, \d1, q15 // bitdepth_max + vmax.s16 \d0, \d0, q14 // 0 + vmax.s16 \d1, \d1, q14 // 0 +.endm + +.macro bidir_fn type, bdmax +function \type\()_16bpc_neon, export=1 + push {r4-r7,lr} + ldrd r4, r5, [sp, #20] + ldr r6, [sp, #28] + clz r4, r4 +.ifnc \type, avg + ldr r7, [sp, #32] + vmov.i16 q14, #0 + vdup.16 q15, r7 // bitdepth_max +.endif +.ifc \type, w_avg + vpush {q4} +.endif +.ifc \type, mask + vpush {q4-q7} +.endif + clz r7, \bdmax + sub r7, r7, #18 // intermediate_bits = clz(bitdepth_max) - 18 +.ifc \type, avg + mov lr, #1 + movw r12, #2*PREP_BIAS + lsl lr, lr, r7 // 1 << intermediate_bits + neg r12, r12 // -2*PREP_BIAS + add r7, r7, #1 + sub r12, r12, lr // -2*PREP_BIAS - 1 << intermediate_bits + neg r7, r7 // -(intermediate_bits+1) + vdup.16 q12, r12 // -2*PREP_BIAS - 1 << intermediate_bits + vdup.16 q13, r7 // -(intermediate_bits+1) +.else + mov r12, #PREP_BIAS + lsr r12, r12, r7 // PREP_BIAS >> intermediate_bits + neg r7, r7 // -intermediate_bits + vdup.16 q12, r12 // PREP_BIAS >> intermediate_bits + vdup.16 q13, r7 // -intermediate_bits +.endif +.ifc \type, w_avg + vdup.32 q4, r6 + vneg.s32 q4, q4 +.endif + adr r7, L(\type\()_tbl) + sub r4, r4, #24 + \type q8, d16, d17, q9, d18, d19 + ldr r4, [r7, r4, lsl #2] + add r7, r7, r4 + bx r7 + + .align 2 +L(\type\()_tbl): + .word 1280f - L(\type\()_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_tbl) + CONFIG_THUMB + +40: + add r7, r0, r1 + lsl r1, r1, #1 +4: + subs r5, r5, #4 + vst1.16 {d16}, [r0, :64], r1 + vst1.16 {d17}, [r7, :64], r1 + vst1.16 {d18}, [r0, :64], r1 + vst1.16 {d19}, [r7, :64], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 4b +80: + add r7, r0, r1 + lsl r1, r1, #1 +8: + vst1.16 {q8}, [r0, :128], r1 + subs r5, r5, #2 + vst1.16 {q9}, [r7, :128], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 8b +160: +16: + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r1 + subs r5, r5, #2 + vst1.16 {q10, q11}, [r0, :128], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 16b +320: + add r7, r0, #32 +32: + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r1 + subs r5, r5, #1 + vst1.16 {q10, q11}, [r7, :128], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 32b +640: + add r7, r0, #32 + mov r12, #64 + sub r1, r1, #64 +64: + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r12 + \type q8, d16, d17, q9, d18, d19 + vst1.16 {q10, q11}, [r7, :128], r12 + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r1 + subs r5, r5, #1 + vst1.16 {q10, q11}, [r7, :128], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 64b +1280: + add r7, r0, #32 + mov r12, #64 + sub r1, r1, #192 +128: + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r12 + \type q8, d16, d17, q9, d18, d19 + vst1.16 {q10, q11}, [r7, :128], r12 + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r12 + \type q8, d16, d17, q9, d18, d19 + vst1.16 {q10, q11}, [r7, :128], r12 + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r12 + \type q8, d16, d17, q9, d18, d19 + vst1.16 {q10, q11}, [r7, :128], r12 + \type q10, d20, d21, q11, d22, d23 + vst1.16 {q8, q9}, [r0, :128], r1 + subs r5, r5, #1 + vst1.16 {q10, q11}, [r7, :128], r1 + ble 0f + \type q8, d16, d17, q9, d18, d19 + b 128b +0: +.ifc \type, mask + vpop {q4-q7} +.endif +.ifc \type, w_avg + vpop {q4} +.endif + pop {r4-r7,pc} +endfunc +.endm + +bidir_fn avg, r6 +bidir_fn w_avg, r7 +bidir_fn mask, r7 + + +.macro w_mask_fn type +function w_mask_\type\()_16bpc_neon, export=1 + push {r4-r10,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #96] + ldrd r6, r7, [sp, #104] + ldr r8, [sp, #112] + clz r9, r4 + adr lr, L(w_mask_\type\()_tbl) + vdup.16 q15, r8 // bitdepth_max + sub r9, r9, #24 + clz r8, r8 // clz(bitdepth_max) + ldr r9, [lr, r9, lsl #2] + add r9, lr, r9 + sub r8, r8, #12 // sh = intermediate_bits + 6 = clz(bitdepth_max) - 12 + mov r10, #PREP_BIAS*64 + neg r8, r8 // -sh + movw r12, #27615 // (64 + 1 - 38)<> mask_sh + vshr.u16 q7, q7, #10 + vadd.i32 q2, q2, q13 // += PREP_BIAS*64 + vadd.i32 q3, q3, q13 + vadd.i32 q4, q4, q13 + vadd.i32 q5, q5, q13 + vmovl.u16 q12, d12 + vmovl.u16 q13, d13 + vmla.i32 q2, q8, q12 // (tmp2-tmp1)*(64-m) + vmovl.u16 q12, d14 + vmla.i32 q3, q9, q13 + vmovl.u16 q13, d15 + vmla.i32 q4, q10, q12 + vmla.i32 q5, q11, q13 + vrshl.s32 q2, q2, q14 // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + vrshl.s32 q3, q3, q14 + vrshl.s32 q4, q4, q14 + vrshl.s32 q5, q5, q14 + vqmovun.s32 d4, q2 // iclip_pixel + vqmovun.s32 d5, q3 + vqmovun.s32 d6, q4 + vqmovun.s32 d7, q5 + vmin.u16 q2, q2, q15 // iclip_pixel + vmin.u16 q3, q3, q15 // iclip_pixel +.if \type == 444 + vmovn.i16 d12, q6 // 64 - m + vmovn.i16 d13, q7 + vsub.i16 q6, q1, q6 // m + vst1.8 {q6}, [r6, :128]! +.elseif \type == 422 + vpadd.i16 d12, d12, d13 // (64 - m) + (64 - n) (column wise addition) + vpadd.i16 d13, d14, d15 + vmovn.i16 d12, q6 + vhsub.u8 d12, d2, d12 // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + vst1.8 {d12}, [r6, :64]! +.elseif \type == 420 + vadd.i16 d12, d12, d13 // (64 - my1) + (64 - my2) (row wise addition) + vadd.i16 d13, d14, d15 + vpadd.i16 d12, d12, d13 // (128 - m) + (128 - n) (column wise addition) + vsub.i16 d12, d2, d12 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.i16 d12, q6, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.32 {d12[0]}, [r6, :32]! +.endif + vst1.16 {d4}, [r0, :64], r1 + vst1.16 {d5}, [r12, :64], r1 + vst1.16 {d6}, [r0, :64], r1 + vst1.16 {d7}, [r12, :64], r1 + bgt 4b + vpop {q4-q7} + pop {r4-r10,pc} +8: + vld1.16 {q2, q3}, [r2, :128]! // tmp1 + vld1.16 {q4, q5}, [r3, :128]! // tmp2 + subs r5, r5, #2 + vdup.32 q13, r10 // PREP_BIAS*64 + vabd.s16 q6, q2, q4 // abs(tmp1 - tmp2) + vabd.s16 q7, q3, q5 + vsubl.s16 q8, d8, d4 // tmp2 - tmp1 (requires 17 bit) + vsubl.s16 q9, d9, d5 + vsubl.s16 q10, d10, d6 + vsubl.s16 q11, d11, d7 + vqsub.u16 q6, q0, q6 // 27615 - abs() + vqsub.u16 q7, q0, q7 + vshll.s16 q5, d7, #6 // tmp1 << 6 + vshll.s16 q4, d6, #6 + vshll.s16 q3, d5, #6 + vshll.s16 q2, d4, #6 + vshr.u16 q6, q6, #10 // 64-m = (27615 - abs()) >> mask_sh + vshr.u16 q7, q7, #10 + vadd.i32 q2, q2, q13 // += PREP_BIAS*64 + vadd.i32 q3, q3, q13 + vadd.i32 q4, q4, q13 + vadd.i32 q5, q5, q13 + vmovl.u16 q12, d12 + vmovl.u16 q13, d13 + vmla.i32 q2, q8, q12 // (tmp2-tmp1)*(64-m) + vmovl.u16 q12, d14 + vmla.i32 q3, q9, q13 + vmovl.u16 q13, d15 + vmla.i32 q4, q10, q12 + vmla.i32 q5, q11, q13 + vrshl.s32 q2, q2, q14 // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + vrshl.s32 q3, q3, q14 + vrshl.s32 q4, q4, q14 + vrshl.s32 q5, q5, q14 + vqmovun.s32 d4, q2 // iclip_pixel + vqmovun.s32 d5, q3 + vqmovun.s32 d6, q4 + vqmovun.s32 d7, q5 + vmin.u16 q2, q2, q15 // iclip_pixel + vmin.u16 q3, q3, q15 // iclip_pixel +.if \type == 444 + vmovn.i16 d12, q6 // 64 - m + vmovn.i16 d13, q7 + vsub.i16 q6, q1, q6 // m + vst1.8 {q6}, [r6, :128]! +.elseif \type == 422 + vpadd.i16 d12, d12, d13 // (64 - m) + (64 - n) (column wise addition) + vpadd.i16 d13, d14, d15 + vmovn.i16 d12, q6 + vhsub.u8 d12, d2, d12 // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + vst1.8 {d12}, [r6, :64]! +.elseif \type == 420 + vadd.i16 q6, q6, q7 // (64 - my1) + (64 - my2) (row wise addition) + vpadd.i16 d12, d12, d13 // (128 - m) + (128 - n) (column wise addition) + vsub.i16 d12, d2, d12 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.i16 d12, q6, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.32 {d12[0]}, [r6, :32]! +.endif + vst1.16 {q2}, [r0, :128], r1 + vst1.16 {q3}, [r12, :128], r1 + bgt 8b + vpop {q4-q7} + pop {r4-r10,pc} +1280: +640: +320: +160: + sub r1, r1, r4, lsl #1 +.if \type == 444 + add lr, r6, r4 +.elseif \type == 422 + add lr, r6, r4, lsr #1 +.endif + add r7, r2, r4, lsl #1 + add r9, r3, r4, lsl #1 +161: + mov r8, r4 +16: + vld1.16 {q2}, [r2, :128]! // tmp1 + vld1.16 {q4}, [r3, :128]! // tmp2 + vld1.16 {q3}, [r7, :128]! + vld1.16 {q5}, [r9, :128]! + subs r8, r8, #8 + vdup.32 q13, r10 // PREP_BIAS*64 + vabd.s16 q6, q2, q4 // abs(tmp1 - tmp2) + vabd.s16 q7, q3, q5 + vsubl.s16 q8, d8, d4 // tmp2 - tmp1 (requires 17 bit) + vsubl.s16 q9, d9, d5 + vsubl.s16 q10, d10, d6 + vsubl.s16 q11, d11, d7 + vqsub.u16 q6, q0, q6 // 27615 - abs() + vqsub.u16 q7, q0, q7 + vshll.s16 q5, d7, #6 // tmp1 << 6 + vshll.s16 q4, d6, #6 + vshll.s16 q3, d5, #6 + vshll.s16 q2, d4, #6 + vshr.u16 q6, q6, #10 // 64-m = (27615 - abs()) >> mask_sh + vshr.u16 q7, q7, #10 + vadd.i32 q2, q2, q13 // += PREP_BIAS*64 + vadd.i32 q3, q3, q13 + vadd.i32 q4, q4, q13 + vadd.i32 q5, q5, q13 + vmovl.u16 q12, d12 + vmovl.u16 q13, d13 + vmla.i32 q2, q8, q12 // (tmp2-tmp1)*(64-m) + vmovl.u16 q12, d14 + vmla.i32 q3, q9, q13 + vmovl.u16 q13, d15 + vmla.i32 q4, q10, q12 + vmla.i32 q5, q11, q13 + vrshl.s32 q2, q2, q14 // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + vrshl.s32 q3, q3, q14 + vrshl.s32 q4, q4, q14 + vrshl.s32 q5, q5, q14 + vqmovun.s32 d4, q2 // iclip_pixel + vqmovun.s32 d5, q3 + vqmovun.s32 d6, q4 + vqmovun.s32 d7, q5 + vmin.u16 q2, q2, q15 // iclip_pixel + vmin.u16 q3, q3, q15 // iclip_pixel +.if \type == 444 + vmovn.i16 d12, q6 // 64 - m + vmovn.i16 d13, q7 + vsub.i16 q6, q1, q6 // m + vst1.8 {d12}, [r6, :64]! + vst1.8 {d13}, [lr, :64]! +.elseif \type == 422 + vpadd.i16 d12, d12, d13 // (64 - m) + (64 - n) (column wise addition) + vpadd.i16 d13, d14, d15 + vmovn.i16 d12, q6 + vhsub.u8 d12, d2, d12 // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + vst1.32 {d12[0]}, [r6, :32]! + vst1.32 {d12[1]}, [lr, :32]! +.elseif \type == 420 + vadd.i16 q6, q6, q7 // (64 - my1) + (64 - my2) (row wise addition) + vpadd.i16 d12, d12, d13 // (128 - m) + (128 - n) (column wise addition) + vsub.i16 d12, d2, d12 // (256 - sign) - ((128 - m) + (128 - n)) + vrshrn.i16 d12, q6, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + vst1.32 {d12[0]}, [r6, :32]! +.endif + vst1.16 {q2}, [r0, :128]! + vst1.16 {q3}, [r12, :128]! + bgt 16b + subs r5, r5, #2 + add r2, r2, r4, lsl #1 + add r3, r3, r4, lsl #1 + add r7, r7, r4, lsl #1 + add r9, r9, r4, lsl #1 +.if \type == 444 + add r6, r6, r4 + add lr, lr, r4 +.elseif \type == 422 + add r6, r6, r4, lsr #1 + add lr, lr, r4, lsr #1 +.endif + add r0, r0, r1 + add r12, r12, r1 + bgt 161b + vpop {q4-q7} + pop {r4-r10,pc} +endfunc +.endm + +w_mask_fn 444 +w_mask_fn 422 +w_mask_fn 420 + +function blend_16bpc_neon, export=1 + push {r4-r5,lr} + ldrd r4, r5, [sp, #12] + clz lr, r3 + adr r3, L(blend_tbl) + sub lr, lr, #26 + ldr lr, [r3, lr, lsl #2] + add r3, r3, lr + bx r3 + + .align 2 +L(blend_tbl): + .word 320f - L(blend_tbl) + CONFIG_THUMB + .word 160f - L(blend_tbl) + CONFIG_THUMB + .word 80f - L(blend_tbl) + CONFIG_THUMB + .word 40f - L(blend_tbl) + CONFIG_THUMB + +40: + add r12, r0, r1 + lsl r1, r1, #1 +4: + vld1.8 {d4}, [r5, :64]! + vld1.16 {q1}, [r2, :128]! + vld1.16 {d0}, [r0, :64] + vneg.s8 d4, d4 // -m + subs r4, r4, #2 + vld1.16 {d1}, [r12, :64] + vmovl.s8 q2, d4 + vshl.i16 q2, q2, #9 // -m << 9 + vsub.i16 q1, q0, q1 // a - b + vqrdmulh.s16 q1, q1, q2 // ((a-b)*-m + 32) >> 6 + vadd.i16 q0, q0, q1 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d1}, [r12, :64], r1 + bgt 4b + pop {r4-r5,pc} +80: + add r12, r0, r1 + lsl r1, r1, #1 +8: + vld1.8 {q8}, [r5, :128]! + vld1.16 {q2, q3}, [r2, :128]! + vneg.s8 q9, q8 // -m + vld1.16 {q0}, [r0, :128] + vld1.16 {q1}, [r12, :128] + vmovl.s8 q8, d18 + vmovl.s8 q9, d19 + vshl.i16 q8, q8, #9 // -m << 9 + vshl.i16 q9, q9, #9 + vsub.i16 q2, q0, q2 // a - b + vsub.i16 q3, q1, q3 + subs r4, r4, #2 + vqrdmulh.s16 q2, q2, q8 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q3, q3, q9 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vst1.16 {q0}, [r0, :128], r1 + vst1.16 {q1}, [r12, :128], r1 + bgt 8b + pop {r4-r5,pc} +160: + add r12, r0, r1 + lsl r1, r1, #1 +16: + vld1.8 {q12, q13}, [r5, :128]! + vld1.16 {q8, q9}, [r2, :128]! + subs r4, r4, #2 + vneg.s8 q14, q12 // -m + vld1.16 {q0, q1}, [r0, :128] + vneg.s8 q15, q13 + vld1.16 {q10, q11}, [r2, :128]! + vmovl.s8 q12, d28 + vmovl.s8 q13, d29 + vmovl.s8 q14, d30 + vmovl.s8 q15, d31 + vld1.16 {q2, q3}, [r12, :128] + vshl.i16 q12, q12, #9 // -m << 9 + vshl.i16 q13, q13, #9 + vshl.i16 q14, q14, #9 + vshl.i16 q15, q15, #9 + vsub.i16 q8, q0, q8 // a - b + vsub.i16 q9, q1, q9 + vsub.i16 q10, q2, q10 + vsub.i16 q11, q3, q11 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q9, q9, q13 + vqrdmulh.s16 q10, q10, q14 + vqrdmulh.s16 q11, q11, q15 + vadd.i16 q0, q0, q8 + vadd.i16 q1, q1, q9 + vadd.i16 q2, q2, q10 + vst1.16 {q0, q1}, [r0, :128], r1 + vadd.i16 q3, q3, q11 + vst1.16 {q2, q3}, [r12, :128], r1 + bgt 16b + pop {r4-r5,pc} +320: + add r12, r0, #32 +32: + vld1.8 {q12, q13}, [r5, :128]! + vld1.16 {q8, q9}, [r2, :128]! + subs r4, r4, #1 + vneg.s8 q14, q12 // -m + vld1.16 {q0, q1}, [r0, :128] + vneg.s8 q15, q13 + vld1.16 {q10, q11}, [r2, :128]! + vmovl.s8 q12, d28 + vmovl.s8 q13, d29 + vmovl.s8 q14, d30 + vmovl.s8 q15, d31 + vld1.16 {q2, q3}, [r12, :128] + vshl.i16 q12, q12, #9 // -m << 9 + vshl.i16 q13, q13, #9 + vshl.i16 q14, q14, #9 + vshl.i16 q15, q15, #9 + vsub.i16 q8, q0, q8 // a - b + vsub.i16 q9, q1, q9 + vsub.i16 q10, q2, q10 + vsub.i16 q11, q3, q11 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q9, q9, q13 + vqrdmulh.s16 q10, q10, q14 + vqrdmulh.s16 q11, q11, q15 + vadd.i16 q0, q0, q8 + vadd.i16 q1, q1, q9 + vadd.i16 q2, q2, q10 + vst1.16 {q0, q1}, [r0, :128], r1 + vadd.i16 q3, q3, q11 + vst1.16 {q2, q3}, [r12, :128], r1 + bgt 32b + pop {r4-r5,pc} +endfunc + +function blend_h_16bpc_neon, export=1 + push {r4-r5,lr} + ldr r4, [sp, #12] + movrel r5, X(obmc_masks) + add r5, r5, r4 + sub r4, r4, r4, lsr #2 + clz lr, r3 + adr r12, L(blend_h_tbl) + sub lr, lr, #24 + ldr lr, [r12, lr, lsl #2] + add r12, r12, lr + bx r12 + + .align 2 +L(blend_h_tbl): + .word 1280f - L(blend_h_tbl) + CONFIG_THUMB + .word 640f - L(blend_h_tbl) + CONFIG_THUMB + .word 320f - L(blend_h_tbl) + CONFIG_THUMB + .word 160f - L(blend_h_tbl) + CONFIG_THUMB + .word 80f - L(blend_h_tbl) + CONFIG_THUMB + .word 40f - L(blend_h_tbl) + CONFIG_THUMB + .word 20f - L(blend_h_tbl) + CONFIG_THUMB + +20: + add r12, r0, r1 + lsl r1, r1, #1 +2: + vld2.8 {d4[], d5[]}, [r5, :16]! + vld1.16 {d2}, [r2, :64]! + vext.8 d4, d4, d5, #6 + subs r4, r4, #2 + vneg.s8 d4, d4 // -m + vld1.32 {d0[]}, [r0, :32] + vld1.32 {d0[1]}, [r12, :32] + vmovl.s8 q2, d4 + vshl.i16 d4, d4, #9 // -m << 9 + vsub.i16 d2, d0, d2 // a - b + vqrdmulh.s16 d2, d2, d4 // ((a-b)*-m + 32) >> 6 + vadd.i16 d0, d0, d2 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d0[1]}, [r12, :32], r1 + bgt 2b + pop {r4-r5,pc} +40: + add r12, r0, r1 + lsl r1, r1, #1 +4: + vld2.8 {d4[], d5[]}, [r5, :16]! + vld1.16 {q1}, [r2, :128]! + vext.8 d4, d4, d5, #4 + subs r4, r4, #2 + vneg.s8 d4, d4 // -m + vld1.16 {d0}, [r0, :64] + vld1.16 {d1}, [r12, :64] + vmovl.s8 q2, d4 + vshl.i16 q2, q2, #9 // -m << 9 + vsub.i16 q1, q0, q1 // a - b + vqrdmulh.s16 q1, q1, q2 // ((a-b)*-m + 32) >> 6 + vadd.i16 q0, q0, q1 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d1}, [r12, :64], r1 + bgt 4b + pop {r4-r5,pc} +80: + add r12, r0, r1 + lsl r1, r1, #1 +8: + vld2.8 {d16[], d17[]}, [r5, :16]! + vld1.16 {q2, q3}, [r2, :128]! + vneg.s8 q9, q8 // -m + vld1.16 {q0}, [r0, :128] + subs r4, r4, #2 + vmovl.s8 q8, d18 + vmovl.s8 q9, d19 + vld1.16 {q1}, [r12, :128] + vshl.i16 q8, q8, #9 // -m << 9 + vshl.i16 q9, q9, #9 + vsub.i16 q2, q0, q2 // a - b + vsub.i16 q3, q1, q3 + vqrdmulh.s16 q2, q2, q8 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q3, q3, q9 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vst1.16 {q0}, [r0, :128], r1 + vst1.16 {q1}, [r12, :128], r1 + bgt 8b + pop {r4-r5,pc} +160: + add r12, r0, r1 + lsl r1, r1, #1 +16: + vld2.8 {d24[], d25[]}, [r5, :16]! + vld1.16 {q8, q9}, [r2, :128]! + subs r4, r4, #2 + vneg.s8 q13, q12 // -m + vld1.16 {q0, q1}, [r0, :128] + vmovl.s8 q12, d26 + vld1.16 {q10, q11}, [r2, :128]! + vmovl.s8 q13, d27 + vld1.16 {q2, q3}, [r12, :128] + vshl.i16 q12, q12, #9 // -m << 9 + vshl.i16 q13, q13, #9 + vsub.i16 q8, q0, q8 // a - b + vsub.i16 q9, q1, q9 + vsub.i16 q10, q2, q10 + vsub.i16 q11, q3, q11 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q9, q9, q12 + vqrdmulh.s16 q10, q10, q13 + vqrdmulh.s16 q11, q11, q13 + vadd.i16 q0, q0, q8 + vadd.i16 q1, q1, q9 + vadd.i16 q2, q2, q10 + vadd.i16 q3, q3, q11 + vst1.16 {q0, q1}, [r0, :128], r1 + vst1.16 {q2, q3}, [r12, :128], r1 + bgt 16b + pop {r4-r5,pc} +1280: +640: +320: + sub r1, r1, r3, lsl #1 +321: + vld1.8 {d24[]}, [r5]! + mov r12, r3 + vneg.s8 d24, d24 // -m + vmovl.s8 q12, d24 + vshl.i16 q12, q12, #9 // -m << 9 +32: + vld1.16 {q8, q9}, [r2, :128]! + vld1.16 {q0, q1}, [r0, :128]! + subs r12, r12, #32 + vld1.16 {q10, q11}, [r2, :128]! + vld1.16 {q2, q3}, [r0, :128] + vsub.i16 q8, q0, q8 // a - b + vsub.i16 q9, q1, q9 + vsub.i16 q10, q2, q10 + vsub.i16 q11, q3, q11 + sub r0, r0, #32 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q9, q9, q12 + vqrdmulh.s16 q10, q10, q12 + vqrdmulh.s16 q11, q11, q12 + vadd.i16 q0, q0, q8 + vadd.i16 q1, q1, q9 + vadd.i16 q2, q2, q10 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q3, q3, q11 + vst1.16 {q2, q3}, [r0, :128]! + bgt 32b + subs r4, r4, #1 + add r0, r0, r1 + bgt 321b + pop {r4-r5,pc} +endfunc + +function blend_v_16bpc_neon, export=1 + push {r4,lr} + ldr r4, [sp, #8] + movrel lr, X(obmc_masks) + add lr, lr, r3 + clz r12, r3 + adr r3, L(blend_v_tbl) + sub r12, r12, #26 + ldr r12, [r3, r12, lsl #2] + add r3, r3, r12 + bx r3 + + .align 2 +L(blend_v_tbl): + .word 320f - L(blend_v_tbl) + CONFIG_THUMB + .word 160f - L(blend_v_tbl) + CONFIG_THUMB + .word 80f - L(blend_v_tbl) + CONFIG_THUMB + .word 40f - L(blend_v_tbl) + CONFIG_THUMB + .word 20f - L(blend_v_tbl) + CONFIG_THUMB + +20: + add r12, r0, r1 + lsl r1, r1, #1 + vld1.8 {d4[]}, [lr] + vneg.s8 d4, d4 // -m + vmovl.s8 q2, d4 + vshl.i16 d4, d4, #9 // -m << 9 +2: + vld1.32 {d2[]}, [r2, :32]! + vld1.16 {d0[]}, [r0, :16] + subs r4, r4, #2 + vld1.16 {d2[1]}, [r2, :16] + vld1.16 {d0[1]}, [r12, :16] + add r2, r2, #4 + vsub.i16 d2, d0, d2 // a - b + vqrdmulh.s16 d2, d2, d4 // ((a-b)*-m + 32) >> 6 + vadd.i16 d0, d0, d2 + vst1.16 {d0[0]}, [r0, :16], r1 + vst1.16 {d0[1]}, [r12, :16], r1 + bgt 2b + pop {r4,pc} +40: + vld1.32 {d4[]}, [lr, :32] + add r12, r0, r1 + vneg.s8 d4, d4 // -m + lsl r1, r1, #1 + vmovl.s8 q2, d4 + sub r1, r1, #4 + vshl.i16 q2, q2, #9 // -m << 9 +4: + vld1.16 {q1}, [r2, :128]! + vld1.16 {d0}, [r0, :64] + vld1.16 {d1}, [r12, :64] + subs r4, r4, #2 + vsub.i16 q1, q0, q1 // a - b + vqrdmulh.s16 q1, q1, q2 // ((a-b)*-m + 32) >> 6 + vadd.i16 q0, q0, q1 + vst1.32 {d0[0]}, [r0, :32]! + vst1.32 {d1[0]}, [r12, :32]! + vst1.16 {d0[2]}, [r0, :16], r1 + vst1.16 {d1[2]}, [r12, :16], r1 + bgt 4b + pop {r4,pc} +80: + vld1.8 {d16}, [lr, :64] + add r12, r0, r1 + vneg.s8 d16, d16 // -m + lsl r1, r1, #1 + vmovl.s8 q8, d16 + sub r1, r1, #8 + vshl.i16 q8, q8, #9 // -m << 9 +8: + vld1.16 {q2, q3}, [r2, :128]! + vld1.16 {q0}, [r0, :128] + vld1.16 {q1}, [r12, :128] + subs r4, r4, #2 + vsub.i16 q2, q0, q2 // a - b + vsub.i16 q3, q1, q3 + vqrdmulh.s16 q2, q2, q8 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q3, q3, q8 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vst1.16 {d0}, [r0, :64]! + vst1.16 {d2}, [r12, :64]! + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d3[0]}, [r12, :32], r1 + bgt 8b + pop {r4,pc} +160: + vld1.8 {q12}, [lr, :128] + add r12, r0, r1 + vneg.s8 q13, q12 // -m + lsl r1, r1, #1 + vmovl.s8 q12, d26 + vmovl.s8 q13, d27 + vshl.i16 q12, q12, #9 // -m << 9 + vshl.i16 d26, d26, #9 +16: + vld1.16 {q8, q9}, [r2, :128]! + vld1.16 {d0, d1, d2}, [r0, :64] + subs r4, r4, #2 + vld1.16 {q10, q11}, [r2, :128]! + vsub.i16 q8, q0, q8 // a - b + vld1.16 {d4, d5, d6}, [r12, :64] + vsub.i16 d18, d2, d18 + vsub.i16 q10, q2, q10 + vsub.i16 d22, d6, d22 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 d18, d18, d26 + vqrdmulh.s16 q10, q10, q12 + vqrdmulh.s16 d22, d22, d26 + vadd.i16 q0, q0, q8 + vadd.i16 d2, d2, d18 + vadd.i16 q2, q2, q10 + vst1.16 {d0, d1, d2}, [r0, :64], r1 + vadd.i16 d6, d6, d22 + vst1.16 {d4, d5, d6}, [r12, :64], r1 + bgt 16b + pop {r4,pc} +320: + vld1.8 {d24, d25, d26}, [lr, :64] + vneg.s8 q14, q12 // -m + vneg.s8 d30, d26 + vmovl.s8 q12, d28 + vmovl.s8 q13, d29 + vmovl.s8 q14, d30 + sub r1, r1, #32 + vshl.i16 q12, q12, #9 // -m << 9 + vshl.i16 q13, q13, #9 + vshl.i16 q14, q14, #9 +32: + vld1.16 {q8, q9}, [r2, :128]! + vld1.16 {q0, q1}, [r0, :128]! + subs r4, r4, #1 + vld1.16 {q10}, [r2, :128] + vsub.i16 q8, q0, q8 // a - b + vld1.16 {q2}, [r0, :128] + sub r0, r0, #32 + vsub.i16 q9, q1, q9 + vsub.i16 q10, q2, q10 + vqrdmulh.s16 q8, q8, q12 // ((a-b)*-m + 32) >> 6 + vqrdmulh.s16 q9, q9, q13 + vqrdmulh.s16 q10, q10, q14 + vadd.i16 q0, q0, q8 + vadd.i16 q1, q1, q9 + vadd.i16 q2, q2, q10 + vst1.16 {q0, q1}, [r0, :128]! + add r2, r2, #32 + vst1.16 {q2}, [r0, :128], r1 + bgt 32b + pop {r4,pc} +endfunc + +// This has got the same signature as the put_8tap functions, +// and assumes that r9 is set to (clz(w)-24). +function put_neon + adr r10, L(put_tbl) + ldr r9, [r10, r9, lsl #2] + add r10, r10, r9 + bx r10 + + .align 2 +L(put_tbl): + .word 1280f - L(put_tbl) + CONFIG_THUMB + .word 640f - L(put_tbl) + CONFIG_THUMB + .word 320f - L(put_tbl) + CONFIG_THUMB + .word 16f - L(put_tbl) + CONFIG_THUMB + .word 80f - L(put_tbl) + CONFIG_THUMB + .word 4f - L(put_tbl) + CONFIG_THUMB + .word 2f - L(put_tbl) + CONFIG_THUMB + +2: + vld1.32 {d0[]}, [r2], r3 + vld1.32 {d1[]}, [r2], r3 + subs r5, r5, #2 + vst1.32 {d0[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 + bgt 2b + pop {r4-r11,pc} +4: + vld1.16 {d0}, [r2], r3 + vld1.16 {d1}, [r2], r3 + subs r5, r5, #2 + vst1.16 {d0}, [r0, :64], r1 + vst1.16 {d1}, [r0, :64], r1 + bgt 4b + pop {r4-r11,pc} +80: + add r8, r0, r1 + lsl r1, r1, #1 + add r9, r2, r3 + lsl r3, r3, #1 +8: + vld1.16 {q0}, [r2], r3 + vld1.16 {q1}, [r9], r3 + subs r5, r5, #2 + vst1.16 {q0}, [r0, :128], r1 + vst1.16 {q1}, [r8, :128], r1 + bgt 8b + pop {r4-r11,pc} +16: + vld1.16 {q0, q1}, [r2], r3 + subs r5, r5, #1 + vst1.16 {q0, q1}, [r0, :128], r1 + bgt 16b + pop {r4-r11,pc} +320: + sub r1, r1, #32 + sub r3, r3, #32 +32: + vld1.16 {q0, q1}, [r2]! + vst1.16 {q0, q1}, [r0, :128]! + vld1.16 {q2, q3}, [r2], r3 + subs r5, r5, #1 + vst1.16 {q2, q3}, [r0, :128], r1 + bgt 32b + pop {r4-r11,pc} +640: + sub r1, r1, #96 + sub r3, r3, #96 +64: + vld1.16 {q8, q9}, [r2]! + vst1.16 {q8, q9}, [r0, :128]! + vld1.16 {q10, q11}, [r2]! + vst1.16 {q10, q11}, [r0, :128]! + vld1.16 {q12, q13}, [r2]! + vst1.16 {q12, q13}, [r0, :128]! + vld1.16 {q14, q15}, [r2], r3 + subs r5, r5, #1 + vst1.16 {q14, q15}, [r0, :128], r1 + bgt 64b + pop {r4-r11,pc} +1280: + sub r1, r1, #224 + sub r3, r3, #224 +128: + vld1.16 {q8, q9}, [r2]! + vst1.16 {q8, q9}, [r0, :128]! + vld1.16 {q10, q11}, [r2]! + vst1.16 {q10, q11}, [r0, :128]! + vld1.16 {q12, q13}, [r2]! + vst1.16 {q12, q13}, [r0, :128]! + vld1.16 {q14, q15}, [r2]! + vst1.16 {q14, q15}, [r0, :128]! + vld1.16 {q8, q9}, [r2]! + vst1.16 {q8, q9}, [r0, :128]! + vld1.16 {q10, q11}, [r2]! + vst1.16 {q10, q11}, [r0, :128]! + vld1.16 {q12, q13}, [r2]! + vst1.16 {q12, q13}, [r0, :128]! + vld1.16 {q14, q15}, [r2], r3 + subs r5, r5, #1 + vst1.16 {q14, q15}, [r0, :128], r1 + bgt 128b + pop {r4-r11,pc} +endfunc + +// This has got the same signature as the prep_8tap functions, +// and assumes that r9 is set to (clz(w)-24), r7 to intermediate_bits and +// r8 to w*2. +function prep_neon + adr r10, L(prep_tbl) + ldr r9, [r10, r9, lsl #2] + vdup.16 q15, r7 // intermediate_bits + vmov.i16 q14, #PREP_BIAS + add r10, r10, r9 + bx r10 + + .align 2 +L(prep_tbl): + .word 1280f - L(prep_tbl) + CONFIG_THUMB + .word 640f - L(prep_tbl) + CONFIG_THUMB + .word 320f - L(prep_tbl) + CONFIG_THUMB + .word 16f - L(prep_tbl) + CONFIG_THUMB + .word 80f - L(prep_tbl) + CONFIG_THUMB + .word 40f - L(prep_tbl) + CONFIG_THUMB + +40: + add r9, r1, r2 + lsl r2, r2, #1 +4: + vld1.16 {d0}, [r1], r2 + vld1.16 {d1}, [r9], r2 + subs r4, r4, #2 + vshl.s16 q0, q0, q15 + vsub.i16 q0, q0, q14 + vst1.16 {q0}, [r0, :128]! + bgt 4b + pop {r4-r11,pc} +80: + add r9, r1, r2 + lsl r2, r2, #1 +8: + vld1.16 {q0}, [r1], r2 + vld1.16 {q1}, [r9], r2 + subs r4, r4, #2 + vshl.s16 q0, q0, q15 + vshl.s16 q1, q1, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vst1.16 {q0, q1}, [r0, :128]! + bgt 8b + pop {r4-r11,pc} +16: + vld1.16 {q0, q1}, [r1], r2 + vshl.s16 q0, q0, q15 + vld1.16 {q2, q3}, [r1], r2 + subs r4, r4, #2 + vshl.s16 q1, q1, q15 + vshl.s16 q2, q2, q15 + vshl.s16 q3, q3, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vsub.i16 q2, q2, q14 + vst1.16 {q0, q1}, [r0, :128]! + vsub.i16 q3, q3, q14 + vst1.16 {q2, q3}, [r0, :128]! + bgt 16b + pop {r4-r11,pc} +320: + sub r2, r2, #32 +32: + vld1.16 {q0, q1}, [r1]! + subs r4, r4, #1 + vshl.s16 q0, q0, q15 + vld1.16 {q2, q3}, [r1], r2 + vshl.s16 q1, q1, q15 + vshl.s16 q2, q2, q15 + vshl.s16 q3, q3, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vsub.i16 q2, q2, q14 + vst1.16 {q0, q1}, [r0, :128]! + vsub.i16 q3, q3, q14 + vst1.16 {q2, q3}, [r0, :128]! + bgt 32b + pop {r4-r11,pc} +640: + sub r2, r2, #96 +64: + vld1.16 {q0, q1}, [r1]! + subs r4, r4, #1 + vshl.s16 q0, q0, q15 + vld1.16 {q2, q3}, [r1]! + vshl.s16 q1, q1, q15 + vld1.16 {q8, q9}, [r1]! + vshl.s16 q2, q2, q15 + vld1.16 {q10, q11}, [r1], r2 + vshl.s16 q3, q3, q15 + vshl.s16 q8, q8, q15 + vshl.s16 q9, q9, q15 + vshl.s16 q10, q10, q15 + vshl.s16 q11, q11, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vsub.i16 q2, q2, q14 + vsub.i16 q3, q3, q14 + vsub.i16 q8, q8, q14 + vst1.16 {q0, q1}, [r0, :128]! + vsub.i16 q9, q9, q14 + vst1.16 {q2, q3}, [r0, :128]! + vsub.i16 q10, q10, q14 + vst1.16 {q8, q9}, [r0, :128]! + vsub.i16 q11, q11, q14 + vst1.16 {q10, q11}, [r0, :128]! + bgt 64b + pop {r4-r11,pc} +1280: + sub r2, r2, #224 +128: + vld1.16 {q0, q1}, [r1]! + subs r4, r4, #1 + vshl.s16 q0, q0, q15 + vld1.16 {q2, q3}, [r1]! + vshl.s16 q1, q1, q15 + vld1.16 {q8, q9}, [r1]! + vshl.s16 q2, q2, q15 + vld1.16 {q10, q11}, [r1]! + vshl.s16 q3, q3, q15 + vshl.s16 q8, q8, q15 + vshl.s16 q9, q9, q15 + vshl.s16 q10, q10, q15 + vshl.s16 q11, q11, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vsub.i16 q2, q2, q14 + vsub.i16 q3, q3, q14 + vsub.i16 q8, q8, q14 + vst1.16 {q0, q1}, [r0, :128]! + vld1.16 {q0, q1}, [r1]! + vsub.i16 q9, q9, q14 + vsub.i16 q10, q10, q14 + vst1.16 {q2, q3}, [r0, :128]! + vld1.16 {q2, q3}, [r1]! + vsub.i16 q11, q11, q14 + vshl.s16 q0, q0, q15 + vst1.16 {q8, q9}, [r0, :128]! + vld1.16 {q8, q9}, [r1]! + vshl.s16 q1, q1, q15 + vshl.s16 q2, q2, q15 + vst1.16 {q10, q11}, [r0, :128]! + vld1.16 {q10, q11}, [r1], r2 + vshl.s16 q3, q3, q15 + vshl.s16 q8, q8, q15 + vshl.s16 q9, q9, q15 + vshl.s16 q10, q10, q15 + vshl.s16 q11, q11, q15 + vsub.i16 q0, q0, q14 + vsub.i16 q1, q1, q14 + vsub.i16 q2, q2, q14 + vsub.i16 q3, q3, q14 + vsub.i16 q8, q8, q14 + vst1.16 {q0, q1}, [r0, :128]! + vsub.i16 q9, q9, q14 + vst1.16 {q2, q3}, [r0, :128]! + vsub.i16 q10, q10, q14 + vst1.16 {q8, q9}, [r0, :128]! + vsub.i16 q11, q11, q14 + vst1.16 {q10, q11}, [r0, :128]! + bgt 128b + pop {r4-r11,pc} +endfunc + +.macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + vld1.\wd {\d0[]}, [\s0], \strd + vld1.\wd {\d1[]}, [\s1], \strd +.ifnb \d2 + vld1.\wd {\d2[]}, [\s0], \strd + vld1.\wd {\d3[]}, [\s1], \strd +.endif +.ifnb \d4 + vld1.\wd {\d4[]}, [\s0], \strd +.endif +.ifnb \d5 + vld1.\wd {\d5[]}, [\s1], \strd +.endif +.ifnb \d6 + vld1.\wd {\d6[]}, [\s0], \strd +.endif +.endm +.macro load_reg s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + vld1.16 {\d0}, [\s0], \strd + vld1.16 {\d1}, [\s1], \strd +.ifnb \d2 + vld1.16 {\d2}, [\s0], \strd + vld1.16 {\d3}, [\s1], \strd +.endif +.ifnb \d4 + vld1.16 {\d4}, [\s0], \strd +.endif +.ifnb \d5 + vld1.16 {\d5}, [\s1], \strd +.endif +.ifnb \d6 + vld1.16 {\d6}, [\s0], \strd +.endif +.endm +.macro load_regpair s0, s1, strd, d0, d1, d2, d3, d4, d5 + vld1.16 {\d0, \d1}, [\s0], \strd +.ifnb \d2 + vld1.16 {\d2, \d3}, [\s1], \strd +.endif +.ifnb \d4 + vld1.16 {\d4, \d5}, [\s0], \strd +.endif +.endm +.macro load_32 s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, 32, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_16s16 s0, s1, strd, d0, d1, d2, d3, d4, d5 + load_regpair \s0, \s1, \strd, \d0, \d1, \d2, \d3, \d4, \d5 +.endm +.macro interleave_1_32 r0, r1, r2, r3, r4 + vext.8 \r0, \r0, \r1, #4 + vext.8 \r1, \r1, \r2, #4 +.ifnb \r3 + vext.8 \r2, \r2, \r3, #4 + vext.8 \r3, \r3, \r4, #4 +.endif +.endm +.macro vmin_u16 c, r0, r1, r2, r3 + vmin.u16 \r0, \r0, \c +.ifnb \r1 + vmin.u16 \r1, \r1, \c +.endif +.ifnb \r2 + vmin.u16 \r2, \r2, \c + vmin.u16 \r3, \r3, \c +.endif +.endm +.macro vsub_i16 c, r0, r1, r2, r3 + vsub.i16 \r0, \r0, \c +.ifnb \r1 + vsub.i16 \r1, \r1, \c +.endif +.ifnb \r2 + vsub.i16 \r2, \r2, \c + vsub.i16 \r3, \r3, \c +.endif +.endm +.macro vmull_vmlal_4 d, s0, s1, s2, s3 + vmull.s16 \d, \s0, d0[0] + vmlal.s16 \d, \s1, d0[1] + vmlal.s16 \d, \s2, d0[2] + vmlal.s16 \d, \s3, d0[3] +.endm +.macro vmull_vmlal_8 d, s0, s1, s2, s3, s4, s5, s6, s7 + vmull.s16 \d, \s0, d0[0] + vmlal.s16 \d, \s1, d0[1] + vmlal.s16 \d, \s2, d0[2] + vmlal.s16 \d, \s3, d0[3] + vmlal.s16 \d, \s4, d1[0] + vmlal.s16 \d, \s5, d1[1] + vmlal.s16 \d, \s6, d1[2] + vmlal.s16 \d, \s7, d1[3] +.endm +.macro vqrshrun_s32 shift, q0, d0, q1, d1, q2, d2, q3, d3 + vqrshrun.s32 \d0, \q0, #\shift +.ifnb \q1 + vqrshrun.s32 \d1, \q1, #\shift +.endif +.ifnb \q2 + vqrshrun.s32 \d2, \q2, #\shift + vqrshrun.s32 \d3, \q3, #\shift +.endif +.endm +.macro vmovn_i32 q0, d0, q1, d1, q2, d2, q3, d3 + vmovn.i32 \d0, \q0 +.ifnb \q1 + vmovn.i32 \d1, \q1 +.endif +.ifnb \q2 + vmovn.i32 \d2, \q2 + vmovn.i32 \d3, \q3 +.endif +.endm +.macro vrshl_s32 shift, r0, r1, r2, r3 + vrshl.s32 \r0, \r0, \shift + vrshl.s32 \r1, \r1, \shift +.ifnb \r2 + vrshl.s32 \r2, \r2, \shift + vrshl.s32 \r3, \r3, \shift +.endif +.endm +.macro vst1_32 strd, r0, r1 + vst1.32 {\r0[0]}, [r0, :32], \strd + vst1.32 {\r0[1]}, [r9, :32], \strd +.ifnb \r1 + vst1.32 {\r1[0]}, [r0, :32], \strd + vst1.32 {\r1[1]}, [r9, :32], \strd +.endif +.endm +.macro vst1_reg strd, align, r0, r1, r2, r3, r4, r5, r6, r7 + vst1.16 {\r0}, [r0, \align], \strd + vst1.16 {\r1}, [r9, \align], \strd +.ifnb \r2 + vst1.16 {\r2}, [r0, \align], \strd + vst1.16 {\r3}, [r9, \align], \strd +.endif +.ifnb \r4 + vst1.16 {\r4}, [r0, \align], \strd + vst1.16 {\r5}, [r9, \align], \strd + vst1.16 {\r6}, [r0, \align], \strd + vst1.16 {\r7}, [r9, \align], \strd +.endif +.endm +.macro finalize type, q0, q1, d0, d1, q2, q3, d2, d3 +.ifc \type, put + vqrshrun_s32 6, \q0, \d0, \q1, \d1, \q2, \d2, \q3, \d3 + vmin_u16 q15, \q0, \q1 +.else + vrshl_s32 q14, \q0, \q1, \q2, \q3 // -(6-intermediate_bits) + vmovn_i32 \q0, \d0, \q1, \d1, \q2, \d2, \q3, \d3 + vsub_i16 q15, \q0, \q1 // PREP_BIAS +.endif +.endm +.macro shift_store_4 type, strd, q0, q1, d0, d1, q2, q3, d2, d3 + finalize \type, \q0, \q1, \d0, \d1, \q2, \q3, \d2, \d3 + vst1_reg \strd, :64, \d0, \d1, \d2, \d3 +.endm +.macro shift_store_8 type, strd, q0, q1, d0, d1, q2, q3, d2, d3 + finalize \type, \q0, \q1, \d0, \d1, \q2, \q3, \d2, \d3 + vst1_reg \strd, :128, \q0, \q1 +.endm +.macro shift_store_16 type, strd, q0, q1, d0, d1, q2, q3, d2, d3 + finalize \type, \q0, \q1, \d0, \d1, \q2, \q3, \d2, \d3 + vst1.16 {\q0, \q1}, [r0, :128], \strd +.endm + +.macro make_8tap_fn op, type, type_h, type_v +function \op\()_8tap_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + movw r9, \type_h + movw r10, \type_v + b \op\()_8tap_neon +endfunc +.endm + +// No spaces in these expressions, due to gas-preprocessor. +#define REGULAR ((0*15<<7)|3*15) +#define SMOOTH ((1*15<<7)|4*15) +#define SHARP ((2*15<<7)|3*15) + +.macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, my, bdmax, ds2, sr2 +make_8tap_fn \type, regular, REGULAR, REGULAR +make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH +make_8tap_fn \type, regular_sharp, REGULAR, SHARP +make_8tap_fn \type, smooth, SMOOTH, SMOOTH +make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR +make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP +make_8tap_fn \type, sharp, SHARP, SHARP +make_8tap_fn \type, sharp_regular, SHARP, REGULAR +make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH + +function \type\()_8tap_neon + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] +.ifc \bdmax, r8 + ldr r8, [sp, #52] +.endif + movw r11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + mul \mx, \mx, r11 + mul \my, \my, r11 + add \mx, \mx, r9 // mx, 8tap_h, 4tap_h + add \my, \my, r10 // my, 8tap_v, 4tap_v + +.ifc \type, prep + lsl \d_strd, \w, #1 +.endif + + vdup.16 q15, \bdmax // bitdepth_max + clz \bdmax, \bdmax + clz r9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + tst \mx, #(0x7f << 14) + sub r9, r9, #24 + add lr, \bdmax, #6 // 6 + intermediate_bits + rsb r12, \bdmax, #6 // 6 - intermediate_bits + movrel r11, X(mc_subpel_filters), -8 + bne L(\type\()_8tap_h) + tst \my, #(0x7f << 14) + bne L(\type\()_8tap_v) + b \type\()_neon + +L(\type\()_8tap_h): + cmp \w, #4 + ubfx r10, \mx, #7, #7 + and \mx, \mx, #0x7f + it gt + movgt \mx, r10 + tst \my, #(0x7f << 14) + add \mx, r11, \mx, lsl #3 + bne L(\type\()_8tap_hv) + + adr r10, L(\type\()_8tap_h_tbl) + vdup.32 q14, r12 // 6 - intermediate_bits + ldr r9, [r10, r9, lsl #2] + vneg.s32 q14, q14 // -(6-intermediate_bits) +.ifc \type, put + vdup.16 q13, \bdmax // intermediate_bits +.else + vmov.i16 q13, #PREP_BIAS +.endif + add r10, r10, r9 +.ifc \type, put + vneg.s16 q13, q13 // -intermediate_bits +.endif + bx r10 + + .align 2 +L(\type\()_8tap_h_tbl): + .word 1280f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_h_tbl) + CONFIG_THUMB + +20: // 2xN h +.ifc \type, put + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 +2: + vld1.16 {q2}, [\src], \s_strd + vld1.16 {q3}, [\sr2], \s_strd + vext.8 d5, d4, d5, #2 + vext.8 d7, d6, d7, #2 + subs \h, \h, #2 + vtrn.32 d4, d6 + vtrn.32 d5, d7 + vmull.s16 q1, d4, d0[0] + vmlal.s16 q1, d5, d0[1] + vmlal.s16 q1, d6, d0[2] + vmlal.s16 q1, d7, d0[3] + vrshl.s32 q1, q1, q14 // -(6-intermediate_bits) + vqmovun.s32 d2, q1 + vrshl.s16 d2, d2, d26 // -intermediate_bits + vmin.u16 d2, d2, d30 + vst1.32 {d2[0]}, [\dst, :32], \d_strd + vst1.32 {d2[1]}, [\ds2, :32], \d_strd + bgt 2b + pop {r4-r11,pc} +.endif + +40: // 4xN h + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 +4: + vld1.16 {q8}, [\src], \s_strd + vld1.16 {q11}, [\sr2], \s_strd + vext.8 d18, d16, d17, #2 + vext.8 d19, d16, d17, #4 + vext.8 d20, d16, d17, #6 + vext.8 d24, d22, d23, #2 + vext.8 d25, d22, d23, #4 + vext.8 d21, d22, d23, #6 + subs \h, \h, #2 + vmull.s16 q2, d16, d0[0] + vmlal.s16 q2, d18, d0[1] + vmlal.s16 q2, d19, d0[2] + vmlal.s16 q2, d20, d0[3] + vmull.s16 q3, d22, d0[0] + vmlal.s16 q3, d24, d0[1] + vmlal.s16 q3, d25, d0[2] + vmlal.s16 q3, d21, d0[3] + vrshl.s32 q2, q2, q14 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) +.ifc \type, put + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vrshl.s16 q2, q2, q13 // -intermediate_bits + vmin.u16 q2, q2, q15 +.else + vmovn.s32 d4, q2 + vmovn.s32 d5, q3 + vsub.i16 q2, q2, q13 // PREP_BIAS +.endif + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d5}, [\ds2, :64], \d_strd + bgt 4b + pop {r4-r11,pc} + +80: +160: +320: +640: +1280: // 8xN, 16xN, 32xN, ... h + vpush {q4-q5} + vld1.8 {d0}, [\mx, :64] + sub \src, \src, #6 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 + + sub \s_strd, \s_strd, \w, lsl #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, lsl #1 +.endif +81: + vld1.16 {q8, q9}, [\src]! + vld1.16 {q10, q11}, [\sr2]! + mov \mx, \w + +8: + vmull.s16 q1, d16, d0[0] + vmull.s16 q2, d17, d0[0] + vmull.s16 q3, d20, d0[0] + vmull.s16 q4, d21, d0[0] +.irpc i, 1234567 + vext.8 q12, q8, q9, #(2*\i) + vext.8 q5, q10, q11, #(2*\i) +.if \i < 4 + vmlal.s16 q1, d24, d0[\i] + vmlal.s16 q2, d25, d0[\i] + vmlal.s16 q3, d10, d0[\i] + vmlal.s16 q4, d11, d0[\i] +.else + vmlal.s16 q1, d24, d1[\i-4] + vmlal.s16 q2, d25, d1[\i-4] + vmlal.s16 q3, d10, d1[\i-4] + vmlal.s16 q4, d11, d1[\i-4] +.endif +.endr + subs \mx, \mx, #8 + vrshl.s32 q1, q1, q14 // -(6-intermediate_bits) + vrshl.s32 q2, q2, q14 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) + vrshl.s32 q4, q4, q14 // -(6-intermediate_bits) +.ifc \type, put + vqmovun.s32 d2, q1 + vqmovun.s32 d3, q2 + vqmovun.s32 d4, q3 + vqmovun.s32 d5, q4 + vrshl.s16 q1, q1, q13 // -intermediate_bits + vrshl.s16 q2, q2, q13 // -intermediate_bits + vmin.u16 q1, q1, q15 + vmin.u16 q2, q2, q15 +.else + vmovn.s32 d2, q1 + vmovn.s32 d3, q2 + vmovn.s32 d4, q3 + vmovn.s32 d5, q4 + vsub.i16 q1, q1, q13 // PREP_BIAS + vsub.i16 q2, q2, q13 // PREP_BIAS +.endif + vst1.16 {q1}, [\dst, :128]! + vst1.16 {q2}, [\ds2, :128]! + ble 9f + + vmov q8, q9 + vmov q10, q11 + vld1.16 {q9}, [\src]! + vld1.16 {q11}, [\sr2]! + b 8b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + bgt 81b + vpop {q4-q5} + pop {r4-r11,pc} + + +L(\type\()_8tap_v): + cmp \h, #4 + ubfx r10, \my, #7, #7 + and \my, \my, #0x7f + it gt + movgt \my, r10 + add \my, r11, \my, lsl #3 + +.ifc \type, prep + vdup.32 q14, r12 // 6 - intermediate_bits + vmov.i16 q15, #PREP_BIAS +.endif + adr r10, L(\type\()_8tap_v_tbl) + ldr r9, [r10, r9, lsl #2] +.ifc \type, prep + vneg.s32 q14, q14 // -(6-intermediate_bits) +.endif + add r10, r10, r9 + bx r10 + + .align 2 +L(\type\()_8tap_v_tbl): + .word 1280f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_v_tbl) + CONFIG_THUMB + +20: // 2xN v +.ifc \type, put + bgt 28f + + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + // 2x2 v + load_32 \src, \sr2, \s_strd, d1, d2, d3, d4, d5 + interleave_1_32 d1, d2, d3, d4, d5 + bgt 24f + vmull_vmlal_4 q8, d1, d2, d3, d4 + vqrshrun_s32 6, q8, d16 + vmin_u16 d30, d16 + vst1_32 \d_strd, d16 + pop {r4-r11,pc} + +24: // 2x4 v + load_32 \sr2, \src, \s_strd, d6, d7 + interleave_1_32 d5, d6, d7 + vmull_vmlal_4 q8, d1, d2, d3, d4 + vmull_vmlal_4 q9, d3, d4, d5, d6 + vqrshrun_s32 6, q8, d16, q9, d17 + vmin_u16 q15, q8 + vst1_32 \d_strd, d16, d17 + pop {r4-r11,pc} + +28: // 2x6, 2x8, 2x12, 2x16 v + vld1.8 {d0}, [\my, :64] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + vmovl.s8 q0, d0 + + load_32 \src, \sr2, \s_strd, d2, d3, d4, d5, d6, d7, d16 + interleave_1_32 d2, d3, d4, d5, d6 + interleave_1_32 d6, d7, d16 +216: + subs \h, \h, #4 + load_32 \sr2, \src, \s_strd, d17, d18, d19, d20 + interleave_1_32 d16, d17, d18, d19, d20 + vmull_vmlal_8 q13, d2, d3, d4, d5, d6, d7, d16, d17 + vmull_vmlal_8 q1, d4, d5, d6, d7, d16, d17, d18, d19 + vqrshrun_s32 6, q13, d26, q1, d27 + vmin_u16 q15, q13 + vst1_32 \d_strd, d26, d27 + ble 0f + cmp \h, #2 + vmov q1, q3 + vmov q2, q8 + vmov q3, q9 + vmov d16, d20 + beq 26f + b 216b +26: + load_32 \sr2, \src, \s_strd, d17, d18 + interleave_1_32 d16, d17, d18 + vmull_vmlal_8 q13, d2, d3, d4, d5, d6, d7, d16, d17 + vqrshrun_s32 6, q13, d26 + vmin_u16 d30, d26 + vst1_32 \d_strd, d26 +0: + pop {r4-r11,pc} +.endif + +40: + bgt 480f + + // 4x2, 4x4 v + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_reg \src, \sr2, \s_strd, d1, d2, d3, d4, d5 + vmull_vmlal_4 q8, d1, d2, d3, d4 + vmull_vmlal_4 q9, d2, d3, d4, d5 + shift_store_4 \type, \d_strd, q8, q9, d16, d17 + ble 0f + load_reg \sr2, \src, \s_strd, d6, d7 + vmull_vmlal_4 q8, d3, d4, d5, d6 + vmull_vmlal_4 q9, d4, d5, d6, d7 + shift_store_4 \type, \d_strd, q8, q9, d16, d17 +0: + pop {r4-r11,pc} + +480: // 4x6, 4x8, 4x12, 4x16 v + vld1.8 {d0}, [\my, :64] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_reg \src, \sr2, \s_strd, d16, d17, d18, d19, d20, d21, d22 + +48: + subs \h, \h, #4 + load_reg \sr2, \src, \s_strd, d23, d24, d25, d26 + vmull_vmlal_8 q1, d16, d17, d18, d19, d20, d21, d22, d23 + vmull_vmlal_8 q2, d17, d18, d19, d20, d21, d22, d23, d24 + vmull_vmlal_8 q3, d18, d19, d20, d21, d22, d23, d24, d25 + vmull_vmlal_8 q8, d19, d20, d21, d22, d23, d24, d25, d26 + shift_store_4 \type, \d_strd, q1, q2, d2, d3, q3, q8, d4, d5 + ble 0f + cmp \h, #2 + vmov q8, q10 + vmov q9, q11 + vmov q10, q12 + vmov d22, d26 + beq 46f + b 48b +46: + load_reg \sr2, \src, \s_strd, d23, d24 + vmull_vmlal_8 q1, d16, d17, d18, d19, d20, d21, d22, d23 + vmull_vmlal_8 q2, d17, d18, d19, d20, d21, d22, d23, d24 + shift_store_4 \type, \d_strd, q1, q2, d2, d3 +0: + pop {r4-r11,pc} + +80: + bgt 880f + + // 8x2, 8x4 v + cmp \h, #2 + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + + load_reg \src, \sr2, \s_strd, q1, q2, q3, q8, q9 + vmull_vmlal_4 q10, d2, d4, d6, d16 + vmull_vmlal_4 q11, d3, d5, d7, d17 + vmull_vmlal_4 q12, d4, d6, d16, d18 + vmull_vmlal_4 q13, d5, d7, d17, d19 + shift_store_8 \type, \d_strd, q10, q11, d20, d21, q12, q13, d22, d23 + ble 0f + load_reg \sr2, \src, \s_strd, q10, q11 + vmull_vmlal_4 q1, d6, d16, d18, d20 + vmull_vmlal_4 q2, d7, d17, d19, d21 + vmull_vmlal_4 q12, d16, d18, d20, d22 + vmull_vmlal_4 q13, d17, d19, d21, d23 + shift_store_8 \type, \d_strd, q1, q2, d2, d3, q12, q13, d4, d5 +0: + pop {r4-r11,pc} + +880: // 8x6, 8x8, 8x16, 8x32 v +1680: // 16x8, 16x16, ... +320: // 32x8, 32x16, ... +640: +1280: + vpush {q4-q7} + vld1.8 {d0}, [\my, :64] + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + vmovl.s8 q0, d0 + mov \my, \h +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + load_reg \src, \sr2, \s_strd, q5, q6, q7, q8, q9, q10, q11 + +88: + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, q12, q13 + vmull_vmlal_8 q1, d10, d12, d14, d16, d18, d20, d22, d24 + vmull_vmlal_8 q2, d11, d13, d15, d17, d19, d21, d23, d25 + vmull_vmlal_8 q3, d12, d14, d16, d18, d20, d22, d24, d26 + vmull_vmlal_8 q4, d13, d15, d17, d19, d21, d23, d25, d27 + shift_store_8 \type, \d_strd, q1, q2, d2, d3, q3, q4, d4, d5 + ble 9f + subs \h, \h, #2 + load_reg \sr2, \src, \s_strd, q1, q2 + vmull_vmlal_8 q3, d14, d16, d18, d20, d22, d24, d26, d2 + vmull_vmlal_8 q4, d15, d17, d19, d21, d23, d25, d27, d3 + vmull_vmlal_8 q5, d16, d18, d20, d22, d24, d26, d2, d4 + vmull_vmlal_8 q6, d17, d19, d21, d23, d25, d27, d3, d5 + shift_store_8 \type, \d_strd, q3, q4, d6, d7, q5, q6, d8, d9 + ble 9f + vmov q5, q9 + vmov q6, q10 + vmov q7, q11 + vmov q8, q12 + vmov q9, q13 + vmov q10, q1 + vmov q11, q2 + b 88b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + vpop {q4-q7} + pop {r4-r11,pc} + +160: + bgt 1680b + + // 16x2, 16x4 v + vpush {q6-q7} + add \my, \my, #2 + vld1.32 {d0[]}, [\my] + sub \src, \src, \s_strd + vmovl.s8 q0, d0 + + load_16s16 \src, \src, \s_strd, q6, q7, q8, q9, q10, q11 +16: + load_16s16 \src, \src, \s_strd, q12, q13 + subs \h, \h, #1 + vmull_vmlal_4 q1, d12, d16, d20, d24 + vmull_vmlal_4 q2, d13, d17, d21, d25 + vmull_vmlal_4 q3, d14, d18, d22, d26 + vmull_vmlal_4 q6, d15, d19, d23, d27 + shift_store_16 \type, \d_strd, q1, q2, d2, d3, q3, q6, d4, d5 + ble 0f + vmov q6, q8 + vmov q7, q9 + vmov q8, q10 + vmov q9, q11 + vmov q10, q12 + vmov q11, q13 + b 16b +0: + vpop {q6-q7} + pop {r4-r11,pc} + + +L(\type\()_8tap_hv): + cmp \h, #4 + ubfx r10, \my, #7, #7 + and \my, \my, #0x7f + it gt + movgt \my, r10 +4: + add \my, r11, \my, lsl #3 + + adr r10, L(\type\()_8tap_hv_tbl) + neg r12, r12 // -(6-intermediate_bits) + ldr r9, [r10, r9, lsl #2] + vdup.32 q14, r12 // -(6-intermediate_bits) +.ifc \type, put + neg r8, lr // -(6+intermeidate_bits) +.else + vmov.i16 q13, #PREP_BIAS +.endif + add r10, r10, r9 +.ifc \type, put + vdup.32 q13, r8 // -(6+intermediate_bits) +.endif + bx r10 + + .align 2 +L(\type\()_8tap_hv_tbl): + .word 1280f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_8tap_hv_tbl) + CONFIG_THUMB + +20: +.ifc \type, put + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + bgt 280f + add \my, \my, #2 + vld1.32 {d2[]}, [\my] + + // 2x2, 2x4 hv + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + vld1.16 {q11}, [\src], \s_strd + vext.8 d24, d22, d23, #2 + vmull.s16 q11, d22, d0 + vmull.s16 q12, d24, d0 + vpadd.s32 d22, d22, d23 + vpadd.s32 d23, d24, d25 + vpadd.s32 d22, d22, d23 + vrshl.s32 d16, d22, d28 // -(6-intermediate_bits) + vmovn.i32 d16, q8 + bl L(\type\()_8tap_filter_2) + + vext.8 d16, d16, d16, #4 + vext.8 d16, d16, d24, #4 + vmov d17, d24 + +2: + bl L(\type\()_8tap_filter_2) + + vext.8 d18, d17, d24, #4 + vmull.s16 q2, d16, d2[0] + vmlal.s16 q2, d17, d2[1] + vmlal.s16 q2, d18, d2[2] + vmlal.s16 q2, d24, d2[3] + + vrshl.s32 q2, q2, q13 // -(6+intermediate_bits) + vqmovun.s32 d4, q2 + vmin.u16 d4, d4, d30 + subs \h, \h, #2 + vst1.32 {d4[0]}, [\dst, :32], \d_strd + vst1.32 {d4[1]}, [\ds2, :32], \d_strd + ble 0f + vmov d16, d18 + vmov d17, d24 + b 2b + +280: // 2x8, 2x16, 2x32 hv + vld1.8 {d2}, [\my, :64] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + vld1.16 {q11}, [\src], \s_strd + vext.8 d24, d22, d23, #2 + vmull.s16 q11, d22, d0 + vmull.s16 q12, d24, d0 + vpadd.s32 d22, d22, d23 + vpadd.s32 d23, d24, d25 + vpadd.s32 d22, d22, d23 + vrshl.s32 d16, d22, d28 // -(6-intermediate_bits) + vmovn.i32 d16, q8 + + bl L(\type\()_8tap_filter_2) + + vext.8 d16, d16, d16, #4 + vext.8 d16, d16, d24, #4 + vmov d17, d24 + bl L(\type\()_8tap_filter_2) + vext.8 d18, d17, d24, #4 + vmov d19, d24 + bl L(\type\()_8tap_filter_2) + vext.8 d20, d19, d24, #4 + vmov d21, d24 + +28: + bl L(\type\()_8tap_filter_2) + vext.8 d22, d21, d24, #4 + vmull.s16 q3, d16, d2[0] + vmlal.s16 q3, d17, d2[1] + vmlal.s16 q3, d18, d2[2] + vmlal.s16 q3, d19, d2[3] + vmlal.s16 q3, d20, d3[0] + vmlal.s16 q3, d21, d3[1] + vmlal.s16 q3, d22, d3[2] + vmlal.s16 q3, d24, d3[3] + + vrshl.s32 q3, q3, q13 // -(6+intermediate_bits) + vqmovun.s32 d6, q3 + vmin.u16 d6, d6, d30 + subs \h, \h, #2 + vst1.32 {d6[0]}, [\dst, :32], \d_strd + vst1.32 {d6[1]}, [\ds2, :32], \d_strd + ble 0f + vmov q8, q9 + vmov q9, q10 + vmov d20, d22 + vmov d21, d24 + b 28b +0: + pop {r4-r11,pc} + +L(\type\()_8tap_filter_2): + vld1.16 {q11}, [\sr2], \s_strd + vld1.16 {q12}, [\src], \s_strd + vext.8 d23, d22, d23, #2 + vext.8 d25, d24, d25, #2 + vtrn.32 q11, q12 + vmull.s16 q3, d22, d0[0] + vmlal.s16 q3, d23, d0[1] + vmlal.s16 q3, d24, d0[2] + vmlal.s16 q3, d25, d0[3] + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) + vmovn.i32 d24, q3 + bx lr +.endif + +40: + add \mx, \mx, #2 + vld1.32 {d0[]}, [\mx] + bgt 480f + add \my, \my, #2 + vld1.32 {d2[]}, [\my] + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + // 4x2, 4x4 hv + vld1.16 {q11}, [\src], \s_strd + vext.8 d24, d22, d23, #2 + vext.8 d25, d22, d23, #4 + vext.8 d23, d22, d23, #6 + vmull.s16 q10, d22, d0[0] + vmlal.s16 q10, d24, d0[1] + vmlal.s16 q10, d25, d0[2] + vmlal.s16 q10, d23, d0[3] + vrshl.s32 q10, q10, q14 // -(6-intermediate_bits) + vmovn.i32 d17, q10 + + bl L(\type\()_8tap_filter_4) + vmov q9, q12 + +4: + bl L(\type\()_8tap_filter_4) + vmull.s16 q2, d17, d2[0] + vmlal.s16 q2, d18, d2[1] + vmlal.s16 q2, d19, d2[2] + vmlal.s16 q2, d24, d2[3] + vmull.s16 q3, d18, d2[0] + vmlal.s16 q3, d19, d2[1] + vmlal.s16 q3, d24, d2[2] + vmlal.s16 q3, d25, d2[3] +.ifc \type, put + vrshl.s32 q2, q2, q13 // -(6+intermediate_bits) + vrshl.s32 q3, q3, q13 // -(6+intermediate_bits) + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vmin.u16 q2, q2, q15 +.else + vrshrn.i32 d4, q2, #6 + vrshrn.i32 d5, q3, #6 + vsub.i16 q2, q2, q13 // PREP_BIAS +.endif + subs \h, \h, #2 + + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d5}, [\ds2, :64], \d_strd + ble 0f + vmov d17, d19 + vmov q9, q12 + b 4b +0: + pop {r4-r11,pc} + +480: // 4x8, 4x16, 4x32 hv + vpush {d13-d15} + vld1.8 {d2}, [\my, :64] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + + vld1.16 {q11}, [\src], \s_strd + vext.8 d24, d22, d23, #2 + vext.8 d25, d22, d23, #4 + vext.8 d23, d22, d23, #6 + vmull.s16 q10, d22, d0[0] + vmlal.s16 q10, d24, d0[1] + vmlal.s16 q10, d25, d0[2] + vmlal.s16 q10, d23, d0[3] + vrshl.s32 q10, q10, q14 // -(6-intermediate_bits) + vmovn.i32 d13, q10 + + bl L(\type\()_8tap_filter_4) + vmov q7, q12 + bl L(\type\()_8tap_filter_4) + vmov q8, q12 + bl L(\type\()_8tap_filter_4) + vmov q9, q12 + +48: + bl L(\type\()_8tap_filter_4) + vmull.s16 q2, d13, d2[0] + vmlal.s16 q2, d14, d2[1] + vmlal.s16 q2, d15, d2[2] + vmlal.s16 q2, d16, d2[3] + vmlal.s16 q2, d17, d3[0] + vmlal.s16 q2, d18, d3[1] + vmlal.s16 q2, d19, d3[2] + vmlal.s16 q2, d24, d3[3] + vmull.s16 q3, d14, d2[0] + vmlal.s16 q3, d15, d2[1] + vmlal.s16 q3, d16, d2[2] + vmlal.s16 q3, d17, d2[3] + vmlal.s16 q3, d18, d3[0] + vmlal.s16 q3, d19, d3[1] + vmlal.s16 q3, d24, d3[2] + vmlal.s16 q3, d25, d3[3] +.ifc \type, put + vrshl.s32 q2, q2, q13 // -(6+intermediate_bits) + vrshl.s32 q3, q3, q13 // -(6+intermediate_bits) + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vmin.u16 q2, q2, q15 +.else + vrshrn.i32 d4, q2, #6 + vrshrn.i32 d5, q3, #6 + vsub.i16 q2, q2, q13 // PREP_BIAS +.endif + subs \h, \h, #2 + vst1.16 {d4}, [\dst, :64], \d_strd + vst1.16 {d5}, [\ds2, :64], \d_strd + ble 0f + vmov d13, d15 + vmov q7, q8 + vmov q8, q9 + vmov q9, q12 + b 48b +0: + vpop {d13-d15} + pop {r4-r11,pc} + +L(\type\()_8tap_filter_4): + vld1.16 {q10}, [\sr2], \s_strd + vld1.16 {q11}, [\src], \s_strd + vext.8 d24, d20, d21, #2 + vext.8 d25, d20, d21, #4 + vext.8 d21, d20, d21, #6 + vmull.s16 q3, d20, d0[0] + vmlal.s16 q3, d24, d0[1] + vmlal.s16 q3, d25, d0[2] + vmlal.s16 q3, d21, d0[3] + vext.8 d24, d22, d23, #2 + vext.8 d25, d22, d23, #4 + vext.8 d23, d22, d23, #6 + vmull.s16 q10, d22, d0[0] + vmlal.s16 q10, d24, d0[1] + vmlal.s16 q10, d25, d0[2] + vmlal.s16 q10, d23, d0[3] + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) + vrshl.s32 q10, q10, q14 // -(6-intermediate_bits) + vmovn.i32 d24, q3 + vmovn.i32 d25, q10 + bx lr + +80: +160: +320: + bgt 880f + add \my, \my, #2 + vld1.8 {d0}, [\mx, :64] + vld1.32 {d2[]}, [\my] + sub \src, \src, #6 + sub \src, \src, \s_strd + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + mov \my, \h + +164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + vld1.16 {q11, q12}, [\src], \s_strd + vmull.s16 q2, d22, d0[0] + vmull.s16 q3, d23, d0[0] + vdup.32 q14, r12 // -(6-intermediate_bits) +.irpc i, 1234567 + vext.8 q10, q11, q12, #(2*\i) +.if \i < 4 + vmlal.s16 q2, d20, d0[\i] + vmlal.s16 q3, d21, d0[\i] +.else + vmlal.s16 q2, d20, d1[\i - 4] + vmlal.s16 q3, d21, d1[\i - 4] +.endif +.endr + vrshl.s32 q2, q2, q14 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) + vmovn.i32 d16, q2 + vmovn.i32 d17, q3 + + bl L(\type\()_8tap_filter_8) + vmov q9, q11 + vmov q10, q12 + +8: + bl L(\type\()_8tap_filter_8) + vmull.s16 q2, d16, d2[0] + vmull.s16 q3, d17, d2[0] + vmull.s16 q13, d18, d2[0] + vmull.s16 q14, d19, d2[0] +.ifc \type, put + vdup.32 q8, r8 // -(6+intermediate_bits) +.endif + vmlal.s16 q2, d18, d2[1] + vmlal.s16 q3, d19, d2[1] + vmlal.s16 q13, d20, d2[1] + vmlal.s16 q14, d21, d2[1] + vmlal.s16 q2, d20, d2[2] + vmlal.s16 q3, d21, d2[2] + vmlal.s16 q13, d22, d2[2] + vmlal.s16 q14, d23, d2[2] + vmlal.s16 q2, d22, d2[3] + vmlal.s16 q3, d23, d2[3] + vmlal.s16 q13, d24, d2[3] + vmlal.s16 q14, d25, d2[3] +.ifc \type, put + vdup.16 q9, \bdmax // bitdepth_max + vrshl.s32 q2, q2, q8 // -(6+intermediate_bits) + vrshl.s32 q3, q3, q8 // -(6+intermediate_bits) + vrshl.s32 q13, q13, q8 // -(6+intermediate_bits) + vrshl.s32 q14, q14, q8 // -(6+intermediate_bits) + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vqmovun.s32 d6, q13 + vqmovun.s32 d7, q14 + vmin.u16 q2, q2, q15 + vmin.u16 q3, q3, q15 +.else + vmov.i16 q9, #PREP_BIAS + vrshrn.i32 d4, q2, #6 + vrshrn.i32 d5, q3, #6 + vrshrn.i32 d6, q13, #6 + vrshrn.i32 d7, q14, #6 + vsub.i16 q2, q2, q9 // PREP_BIAS + vsub.i16 q3, q3, q9 // PREP_BIAS +.endif + subs \h, \h, #2 + vst1.16 {q2}, [\dst, :128], \d_strd + vst1.16 {q3}, [\ds2, :128], \d_strd + ble 9f + vmov q8, q10 + vmov q9, q11 + vmov q10, q12 + b 8b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #2 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 164b +0: + pop {r4-r11,pc} + +880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv +640: +1280: + vpush {q4-q7} + vld1.8 {d0}, [\mx, :64] + vld1.8 {d2}, [\my, :64] + sub \src, \src, #6 + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + vmovl.s8 q0, d0 + vmovl.s8 q1, d2 + mov \my, \h + +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + vld1.16 {q11, q12}, [\src], \s_strd + vmull.s16 q2, d22, d0[0] + vmull.s16 q3, d23, d0[0] + vdup.32 q14, r12 // -(6-intermediate_bits) +.irpc i, 1234567 + vext.8 q10, q11, q12, #(2*\i) +.if \i < 4 + vmlal.s16 q2, d20, d0[\i] + vmlal.s16 q3, d21, d0[\i] +.else + vmlal.s16 q2, d20, d1[\i - 4] + vmlal.s16 q3, d21, d1[\i - 4] +.endif +.endr + vrshl.s32 q2, q2, q14 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q14 // -(6-intermediate_bits) + vmovn.i32 d8, q2 + vmovn.i32 d9, q3 + + bl L(\type\()_8tap_filter_8) + vmov q5, q11 + vmov q6, q12 + bl L(\type\()_8tap_filter_8) + vmov q7, q11 + vmov q8, q12 + bl L(\type\()_8tap_filter_8) + vmov q9, q11 + vmov q10, q12 + +88: + bl L(\type\()_8tap_filter_8) + vmull.s16 q2, d8, d2[0] + vmull.s16 q3, d9, d2[0] + vmull.s16 q13, d10, d2[0] + vmull.s16 q14, d11, d2[0] +.ifc \type, put + vdup.32 q4, r8 // -(6+intermediate_bits) +.endif + vmlal.s16 q2, d10, d2[1] + vmlal.s16 q3, d11, d2[1] + vmlal.s16 q13, d12, d2[1] + vmlal.s16 q14, d13, d2[1] + vmlal.s16 q2, d12, d2[2] + vmlal.s16 q3, d13, d2[2] + vmlal.s16 q13, d14, d2[2] + vmlal.s16 q14, d15, d2[2] + vmlal.s16 q2, d14, d2[3] + vmlal.s16 q3, d15, d2[3] + vmlal.s16 q13, d16, d2[3] + vmlal.s16 q14, d17, d2[3] + vmlal.s16 q2, d16, d3[0] + vmlal.s16 q3, d17, d3[0] + vmlal.s16 q13, d18, d3[0] + vmlal.s16 q14, d19, d3[0] + vmlal.s16 q2, d18, d3[1] + vmlal.s16 q3, d19, d3[1] + vmlal.s16 q13, d20, d3[1] + vmlal.s16 q14, d21, d3[1] + vmlal.s16 q2, d20, d3[2] + vmlal.s16 q3, d21, d3[2] + vmlal.s16 q13, d22, d3[2] + vmlal.s16 q14, d23, d3[2] + vmlal.s16 q2, d22, d3[3] + vmlal.s16 q3, d23, d3[3] + vmlal.s16 q13, d24, d3[3] + vmlal.s16 q14, d25, d3[3] +.ifc \type, put + vrshl.s32 q2, q2, q4 // -(6+intermediate_bits) + vrshl.s32 q3, q3, q4 // -(6+intermediate_bits) + vrshl.s32 q13, q13, q4 // -(6+intermediate_bits) + vrshl.s32 q14, q14, q4 // -(6+intermediate_bits) + vqmovun.s32 d4, q2 + vqmovun.s32 d5, q3 + vqmovun.s32 d6, q13 + vqmovun.s32 d7, q14 + vmin.u16 q2, q2, q15 + vmin.u16 q3, q3, q15 +.else + vmov.i16 q5, #PREP_BIAS + vrshrn.i32 d4, q2, #6 + vrshrn.i32 d5, q3, #6 + vrshrn.i32 d6, q13, #6 + vrshrn.i32 d7, q14, #6 + vsub.i16 q2, q2, q5 // PREP_BIAS + vsub.i16 q3, q3, q5 // PREP_BIAS +.endif + subs \h, \h, #2 + vst1.16 {q2}, [\dst, :128], \d_strd + vst1.16 {q3}, [\ds2, :128], \d_strd + ble 9f + vmov q4, q6 + vmov q5, q7 + vmov q6, q8 + vmov q7, q9 + vmov q8, q10 + vmov q9, q11 + vmov q10, q12 + b 88b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + vpop {q4-q7} + pop {r4-r11,pc} + +L(\type\()_8tap_filter_8): + vld1.16 {q13, q14}, [\sr2], \s_strd + vmull.s16 q2, d26, d0[0] + vmull.s16 q3, d27, d0[0] +.irpc i, 1234567 + vext.8 q12, q13, q14, #(2*\i) +.if \i < 4 + vmlal.s16 q2, d24, d0[\i] + vmlal.s16 q3, d25, d0[\i] +.else + vmlal.s16 q2, d24, d1[\i - 4] + vmlal.s16 q3, d25, d1[\i - 4] +.endif +.endr + vdup.32 q12, r12 // -(6-intermediate_bits) + vld1.16 {q13, q14}, [\src], \s_strd + vrshl.s32 q2, q2, q12 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q12 // -(6-intermediate_bits) + vmovn.i32 d4, q2 + vmovn.i32 d5, q3 + + vmull.s16 q3, d26, d0[0] + vmull.s16 q11, d27, d0[0] +.irpc i, 1234567 + vext.8 q12, q13, q14, #(2*\i) +.if \i < 4 + vmlal.s16 q3, d24, d0[\i] + vmlal.s16 q11, d25, d0[\i] +.else + vmlal.s16 q3, d24, d1[\i - 4] + vmlal.s16 q11, d25, d1[\i - 4] +.endif +.endr + vdup.32 q13, r12 // -(6-intermediate_bits) + vrshl.s32 q3, q3, q13 // -(6-intermediate_bits) + vrshl.s32 q11, q11, q13 // -(6-intermediate_bits) + + vmovn.i32 d24, q3 + vmovn.i32 d25, q11 + vmov q11, q2 + bx lr +endfunc + +function \type\()_bilin_16bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] +.ifc \bdmax, r8 + ldr r8, [sp, #52] +.endif + vdup.16 q1, \mx + vdup.16 q3, \my + rsb r9, \mx, #16 + rsb r10, \my, #16 + vdup.16 q0, r9 + vdup.16 q2, r10 +.ifc \type, prep + lsl \d_strd, \w, #1 +.endif + clz \bdmax, \bdmax // bitdepth_max + clz r9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + cmp \mx, #0 + sub r9, r9, #24 + rsb r11, \bdmax, #4 // 4 - intermediate_bits + add r12, \bdmax, #4 // 4 + intermediate_bits + bne L(\type\()_bilin_h) + cmp \my, #0 + bne L(\type\()_bilin_v) + b \type\()_neon + +L(\type\()_bilin_h): + cmp \my, #0 + bne L(\type\()_bilin_hv) + + adr r10, L(\type\()_bilin_h_tbl) + vdup.16 q15, r11 // 4 - intermediate_bits + ldr r9, [r10, r9, lsl #2] + vneg.s16 q15, q15 // -(4-intermediate_bits) +.ifc \type, put + vdup.16 q14, \bdmax // intermediate_bits +.else + vmov.i16 q14, #PREP_BIAS +.endif + add r10, r10, r9 +.ifc \type, put + vneg.s16 q14, q14 // -intermediate_bits +.endif + bx r10 + + .align 2 +L(\type\()_bilin_h_tbl): + .word 1280f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_h_tbl) + CONFIG_THUMB + +20: // 2xN h +.ifc \type, put + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +2: + vld1.16 {d16}, [\src], \s_strd + vld1.16 {d18}, [\sr2], \s_strd + vext.8 d17, d16, d16, #2 + vext.8 d19, d18, d18, #2 + vtrn.32 d16, d18 + vtrn.32 d17, d19 + subs \h, \h, #2 + vmul.i16 d16, d16, d0 + vmla.i16 d16, d17, d2 + vrshl.u16 d16, d16, d30 + vrshl.u16 d16, d16, d28 + vst1.32 {d16[0]}, [\dst, :32], \d_strd + vst1.32 {d16[1]}, [\ds2, :32], \d_strd + bgt 2b + pop {r4-r11,pc} +.endif + +40: // 4xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +4: + vld1.16 {q8}, [\src], \s_strd + vld1.16 {q10}, [\sr2], \s_strd + vext.8 q9, q8, q8, #2 + vext.8 q11, q10, q10, #2 + vmov d17, d20 + vmov d19, d22 + subs \h, \h, #2 + vmul.i16 q8, q8, q0 + vmla.i16 q8, q9, q1 + vrshl.u16 q8, q8, q15 +.ifc \type, put + vrshl.u16 q8, q8, q14 +.else + vsub.i16 q8, q8, q14 +.endif + vst1.16 {d16}, [\dst, :64], \d_strd + vst1.16 {d17}, [\ds2, :64], \d_strd + bgt 4b + pop {r4-r11,pc} + +80: // 8xN h + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +8: + vld1.16 {d16, d17, d18}, [\src], \s_strd + vld1.16 {d20, d21, d22}, [\sr2], \s_strd + vext.8 q9, q8, q9, #2 + vext.8 q11, q10, q11, #2 + subs \h, \h, #2 + vmul.i16 q8, q8, q0 + vmla.i16 q8, q9, q1 + vmul.i16 q10, q10, q0 + vmla.i16 q10, q11, q1 + vrshl.u16 q8, q8, q15 + vrshl.u16 q10, q10, q15 +.ifc \type, put + vrshl.u16 q8, q8, q14 + vrshl.u16 q10, q10, q14 +.else + vsub.i16 q8, q8, q14 + vsub.i16 q10, q10, q14 +.endif + vst1.16 {q8}, [\dst, :128], \d_strd + vst1.16 {q10}, [\ds2, :128], \d_strd + bgt 8b + pop {r4-r11,pc} +160: +320: +640: +1280: // 16xN, 32xN, ... h + vpush {q4-q7} + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + + sub \s_strd, \s_strd, \w, lsl #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, lsl #1 +.endif +161: + vld1.16 {q4}, [\src]! + vld1.16 {q9}, [\sr2]! + mov \mx, \w + +16: + vld1.16 {q5, q6}, [\src]! + vld1.16 {q10, q11}, [\sr2]! + vext.8 q7, q4, q5, #2 + vext.8 q8, q5, q6, #2 + vext.8 q12, q9, q10, #2 + vext.8 q13, q10, q11, #2 + vmul.i16 q4, q4, q0 + vmla.i16 q4, q7, q1 + vmul.i16 q5, q5, q0 + vmla.i16 q5, q8, q1 + vmul.i16 q9, q9, q0 + vmla.i16 q9, q12, q1 + vmul.i16 q10, q10, q0 + vmla.i16 q10, q13, q1 + vrshl.u16 q4, q4, q15 + vrshl.u16 q5, q5, q15 + vrshl.u16 q9, q9, q15 + vrshl.u16 q10, q10, q15 + subs \mx, \mx, #16 +.ifc \type, put + vrshl.u16 q4, q4, q14 + vrshl.u16 q5, q5, q14 + vrshl.u16 q9, q9, q14 + vrshl.u16 q10, q10, q14 +.else + vsub.i16 q4, q4, q14 + vsub.i16 q5, q5, q14 + vsub.i16 q9, q9, q14 + vsub.i16 q10, q10, q14 +.endif + vst1.16 {q4, q5}, [\dst, :128]! + vst1.16 {q9, q10}, [\ds2, :128]! + ble 9f + + vmov q4, q6 + vmov q9, q11 + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + bgt 161b + vpop {q4-q7} + pop {r4-r11,pc} + + +L(\type\()_bilin_v): + cmp \h, #4 + adr r10, L(\type\()_bilin_v_tbl) +.ifc \type, prep + vdup.16 q15, r11 // 4 - intermediate_bits +.endif + ldr r9, [r10, r9, lsl #2] +.ifc \type, prep + vmov.i16 q14, #PREP_BIAS + vneg.s16 q15, q15 // -(4-intermediate_bits) +.endif + add r10, r10, r9 + bx r10 + + .align 2 +L(\type\()_bilin_v_tbl): + .word 1280f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_v_tbl) + CONFIG_THUMB + +20: // 2xN v +.ifc \type, put + cmp \h, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + // 2x2 v + vld1.32 {d16[]}, [\src], \s_strd + bgt 24f +22: + vld1.32 {d17[]}, [\sr2], \s_strd + vld1.32 {d18[]}, [\src], \s_strd + vext.8 d16, d16, d17, #4 + vext.8 d17, d17, d18, #4 + vmul.i16 d16, d16, d4 + vmla.i16 d16, d17, d6 + vrshr.u16 d16, d16, #4 + vst1.32 {d16[0]}, [\dst, :32] + vst1.32 {d16[1]}, [\ds2, :32] + pop {r4-r11,pc} +24: // 2x4, 2x6, 2x8, ... v + vld1.32 {d17[]}, [\sr2], \s_strd + vld1.32 {d18[]}, [\src], \s_strd + vld1.32 {d19[]}, [\sr2], \s_strd + vld1.32 {d20[]}, [\src], \s_strd + subs \h, \h, #4 + vext.8 d16, d16, d17, #4 + vext.8 d17, d17, d18, #4 + vext.8 d18, d18, d19, #4 + vext.8 d19, d19, d20, #4 + vswp d17, d18 + vmul.i16 q8, q8, q2 + vmla.i16 q8, q9, q3 + cmp \h, #2 + vrshr.u16 q8, q8, #4 + vst1.32 {d16[0]}, [\dst, :32], \d_strd + vst1.32 {d16[1]}, [\ds2, :32], \d_strd + vst1.32 {d17[0]}, [\dst, :32], \d_strd + vst1.32 {d17[1]}, [\ds2, :32], \d_strd + blt 0f + vmov d16, d20 + beq 22b + b 24b +0: + pop {r4-r11,pc} +.endif + +40: // 4xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vld1.16 {d16}, [\src], \s_strd +4: + vld1.16 {d17}, [\sr2], \s_strd + vld1.16 {d19}, [\src], \s_strd + vmov d18, d17 + vmul.i16 q8, q8, q2 + vmla.i16 q8, q9, q3 + subs \h, \h, #2 +.ifc \type, put + vrshr.u16 q8, q8, #4 +.else + vrshl.u16 q8, q8, q15 + vsub.i16 q8, q8, q14 +.endif + vst1.16 {d16}, [\dst, :64], \d_strd + vst1.16 {d17}, [\ds2, :64], \d_strd + ble 0f + vmov d16, d19 + b 4b +0: + pop {r4-r11,pc} + +80: // 8xN v + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + vld1.16 {q8}, [\src], \s_strd +8: + vld1.16 {q9}, [\sr2], \s_strd + vld1.16 {q10}, [\src], \s_strd + vmul.i16 q8, q8, q2 + vmla.i16 q8, q9, q3 + vmul.i16 q9, q9, q2 + vmla.i16 q9, q10, q3 + subs \h, \h, #2 +.ifc \type, put + vrshr.u16 q8, q8, #4 + vrshr.u16 q9, q9, #4 +.else + vrshl.u16 q8, q8, q15 + vrshl.u16 q9, q9, q15 + vsub.i16 q8, q8, q14 + vsub.i16 q9, q9, q14 +.endif + vst1.16 {q8}, [\dst, :128], \d_strd + vst1.16 {q9}, [\ds2, :128], \d_strd + ble 0f + vmov q8, q10 + b 8b +0: + pop {r4-r11,pc} + +160: // 16xN, 32xN, ... +320: +640: +1280: + mov \my, \h +1: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.16 {q8, q9}, [\src], \s_strd +2: + vld1.16 {q10, q11}, [\sr2], \s_strd + vld1.16 {q12, q13}, [\src], \s_strd + vmul.i16 q8, q8, q2 + vmla.i16 q8, q10, q3 + vmul.i16 q9, q9, q2 + vmla.i16 q9, q11, q3 + vmul.i16 q10, q10, q2 + vmla.i16 q10, q12, q3 + vmul.i16 q11, q11, q2 + vmla.i16 q11, q13, q3 + subs \h, \h, #2 +.ifc \type, put + vrshr.u16 q8, q8, #4 + vrshr.u16 q9, q9, #4 + vrshr.u16 q10, q10, #4 + vrshr.u16 q11, q11, #4 +.else + vrshl.u16 q8, q8, q15 + vrshl.u16 q9, q9, q15 + vrshl.u16 q10, q10, q15 + vrshl.u16 q11, q11, q15 + vsub.i16 q8, q8, q14 + vsub.i16 q9, q9, q14 + vsub.i16 q10, q10, q14 + vsub.i16 q11, q11, q14 +.endif + vst1.16 {q8, q9}, [\dst, :128], \d_strd + vst1.16 {q10, q11}, [\ds2, :128], \d_strd + ble 9f + vmov q8, q12 + vmov q9, q13 + b 2b +9: + subs \w, \w, #16 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #32 + add \dst, \dst, #32 + b 1b +0: + pop {r4-r11,pc} + +L(\type\()_bilin_hv): + adr r10, L(\type\()_bilin_hv_tbl) + vdup.16 q15, r11 // 4 - intermediate_bits + ldr r9, [r10, r9, lsl #2] + vneg.s16 q15, q15 // -(4-intermediate_bits) +.ifc \type, put + vdup.32 q14, r12 // 4 + intermediate_bits +.else + vmov.i16 q14, #PREP_BIAS +.endif + add r10, r10, r9 +.ifc \type, put + vneg.s32 q14, q14 // -(4+intermediate_bits) +.endif + bx r10 + + .align 2 +L(\type\()_bilin_hv_tbl): + .word 1280f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 640f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 320f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 160f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 80f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 40f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + .word 20f - L(\type\()_bilin_hv_tbl) + CONFIG_THUMB + +20: // 2xN hv +.ifc \type, put + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.16 {d20}, [\src], \s_strd + vext.8 d21, d20, d20, #2 + vmul.i16 d16, d20, d0 + vmla.i16 d16, d21, d2 + vrshl.u16 d16, d16, d30 + vext.8 d16, d16, d16, #4 + +2: + vld1.16 {d20}, [\sr2], \s_strd + vld1.16 {d22}, [\src], \s_strd + vext.8 d21, d20, d20, #2 + vext.8 d23, d22, d22, #2 + vtrn.32 d20, d22 + vtrn.32 d21, d23 + vmul.i16 d18, d20, d0 + vmla.i16 d18, d21, d2 + vrshl.u16 d18, d18, d30 + + vext.8 d16, d16, d18, #4 + + vmull.u16 q8, d16, d4 + vmlal.u16 q8, d18, d6 + vrshl.u32 q8, q8, q14 + vmovn.i32 d16, q8 + subs \h, \h, #2 + vst1.32 {d16[0]}, [\dst, :32], \d_strd + vst1.32 {d16[1]}, [\ds2, :32], \d_strd + ble 0f + vmov d16, d18 + b 2b +0: + pop {r4-r11,pc} +.endif + +40: // 4xN hv + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.16 {q10}, [\src], \s_strd + vext.8 d21, d20, d21, #2 + vmul.i16 d16, d20, d0 + vmla.i16 d16, d21, d2 + vrshl.u16 d16, d16, d30 + +4: + vld1.16 {q10}, [\sr2], \s_strd + vld1.16 {q11}, [\src], \s_strd + vext.8 d21, d20, d21, #2 + vext.8 d23, d22, d23, #2 + vswp d21, d22 + vmul.i16 q9, q10, q0 + vmla.i16 q9, q11, q1 + vrshl.u16 q9, q9, q15 + + vmull.u16 q10, d16, d4 + vmlal.u16 q10, d18, d6 + vmull.u16 q11, d18, d4 + vmlal.u16 q11, d19, d6 +.ifc \type, put + vrshl.u32 q10, q10, q14 + vrshl.u32 q11, q11, q14 + vmovn.i32 d20, q10 + vmovn.i32 d21, q11 +.else + vrshrn.i32 d20, q10, #4 + vrshrn.i32 d21, q11, #4 + vsub.i16 q10, q10, q14 +.endif + subs \h, \h, #2 + vst1.16 {d20}, [\dst, :64], \d_strd + vst1.16 {d21}, [\ds2, :64], \d_strd + ble 0f + vmov d16, d19 + b 4b +0: + pop {r4-r11,pc} + +80: // 8xN, 16xN, ... hv +160: +320: +640: +1280: + mov \my, \h + +1: + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + vld1.16 {d20, d21, d22}, [\src], \s_strd + vext.8 q11, q10, q11, #2 + vmul.i16 q8, q10, q0 + vmla.i16 q8, q11, q1 + vrshl.u16 q8, q8, q15 + +2: + vld1.16 {d20, d21, d22}, [\sr2], \s_strd + vld1.16 {d24, d25, d26}, [\src], \s_strd + vext.8 q11, q10, q11, #2 + vext.8 q13, q12, q13, #2 + vmul.i16 q9, q10, q0 + vmla.i16 q9, q11, q1 + vmul.i16 q10, q12, q0 + vmla.i16 q10, q13, q1 + vrshl.u16 q9, q9, q15 + vrshl.u16 q10, q10, q15 + + vmull.u16 q11, d16, d4 + vmlal.u16 q11, d18, d6 + vmull.u16 q12, d17, d4 + vmlal.u16 q12, d19, d6 + vmull.u16 q8, d18, d4 + vmlal.u16 q8, d20, d6 + vmull.u16 q9, d19, d4 + vmlal.u16 q9, d21, d6 +.ifc \type, put + vrshl.u32 q11, q11, q14 + vrshl.u32 q12, q12, q14 + vrshl.u32 q8, q8, q14 + vrshl.u32 q9, q9, q14 + vmovn.i32 d22, q11 + vmovn.i32 d23, q12 + vmovn.i32 d16, q8 + vmovn.i32 d17, q9 +.else + vrshrn.i32 d22, q11, #4 + vrshrn.i32 d23, q12, #4 + vrshrn.i32 d16, q8, #4 + vrshrn.i32 d17, q9, #4 + vsub.i16 q11, q11, q14 + vsub.i16 q8, q8, q14 +.endif + subs \h, \h, #2 + vst1.16 {q11}, [\dst, :128], \d_strd + vst1.16 {q8}, [\ds2, :128], \d_strd + ble 9f + vmov q8, q10 + b 2b +9: + subs \w, \w, #8 + ble 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + mls \src, \s_strd, \my, \src + mls \dst, \d_strd, \my, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 1b +0: + pop {r4-r11,pc} +endfunc +.endm + +filter_fn put, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 +filter_fn prep, r0, r8, r1, r2, r3, r4, r5, r6, r7, r9, r10 + +.macro load_filter_ptr src + asr r12, \src, #10 + add r12, r11, r12, lsl #3 +.endm + +.macro load_filter_coef dst, src, inc + add \src, \src, \inc + vld1.8 {\dst}, [r12, :64] +.endm + +.macro load_filter_row dst, src, inc + load_filter_ptr \src + load_filter_coef \dst, \src, \inc +.endm + +function warp_filter_horz_neon + load_filter_ptr r5 // filter 0 + vld1.16 {q6,q7}, [r2], r3 + + load_filter_coef d0, r5, r7 // filter 0 + load_filter_row d2, r5, r7 // filter 1 + vmovl.s8 q0, d0 // filter 0 + vext.8 q3, q6, q7, #2*1 // filter 1 pixels + vmovl.s8 q1, d2 // filter 1 + + vmull.s16 q4, d12, d0 // filter 0 output (0-3) + vmull.s16 q5, d13, d1 // filter 0 output (4-7) + + load_filter_ptr r5 // filter 2 + + vmull.s16 q2, d6, d2 // filter 1 output (0-3) + vmull.s16 q3, d7, d3 // filter 1 output (4-7) + + load_filter_coef d0, r5, r7 // filter 2 + + vpadd.i32 d8, d8, d9 // half pixel 0 (2x32) + vpadd.i32 d9, d10, d11 // half pixel 0 (2x32) + + load_filter_ptr r5 // filter 3 + + vpadd.i32 d4, d4, d5 // half pixel 1 (2x32) + vpadd.i32 d5, d6, d7 // half pixel 1 (2x32) + + vmovl.s8 q0, d0 // filter 2 + vext.8 q3, q6, q7, #2*2 // filter 2 pixels + + vpadd.i32 d8, d8, d9 // pixel 0 (2x32) + vpadd.i32 d9, d4, d5 // pixel 1 (2x32) + + load_filter_coef d2, r5, r7 // filter 3 + + vmull.s16 q2, d6, d0 // filter 2 output (0-3) + vmull.s16 q3, d7, d1 // filter 2 output (4-7) + + load_filter_ptr r5 // filter 4 + + vpadd.i32 d8, d8, d9 // pixel 0,1 + + vpadd.i32 d9, d4, d5 // half pixel 2 (2x32) + vpadd.i32 d10, d6, d7 // half pixel 2 (2x32) + + vmovl.s8 q1, d2 // filter 3 + vext.8 q3, q6, q7, #2*3 // filter 3 pixels + + load_filter_coef d0, r5, r7 // filter 4 + + vpadd.i32 d9, d9, d10 // pixel 2 (2x32) + + vmull.s16 q2, d6, d2 // filter 3 output (0-3) + vmull.s16 q3, d7, d3 // filter 3 output (4-7) + + vmovl.s8 q0, d0 // filter 4 + load_filter_ptr r5 // filter 5 + + vpadd.i32 d10, d4, d5 // half pixel 3 (2x32) + vpadd.i32 d11, d6, d7 // half pixel 3 (2x32) + + vext.8 q3, q6, q7, #2*4 // filter 4 pixels + load_filter_coef d2, r5, r7 // filter 5 + + vpadd.i32 d10, d10, d11 // pixel 3 (2x32) + + vpadd.i32 d9, d9, d10 // pixel 2,3 + + vmull.s16 q2, d6, d0 // filter 4 output (0-3) + vmull.s16 q3, d7, d1 // filter 4 output (4-7) + + vmovl.s8 q1, d2 // filter 5 + load_filter_ptr r5 // filter 6 + + vpadd.i32 d10, d4, d5 // half pixel 4 (2x32) + vpadd.i32 d11, d6, d7 // half pixel 4 (2x32) + + vext.8 q3, q6, q7, #2*5 // filter 5 pixels + load_filter_coef d0, r5, r7 // filter 6 + + vpadd.i32 d10, d10, d11 // pixel 4 (2x32) + + vmull.s16 q2, d6, d2 // filter 5 output (0-3) + vmull.s16 q3, d7, d3 // filter 5 output (4-7) + + vmovl.s8 q0, d0 // filter 6 + load_filter_ptr r5 // filter 7 + + vpadd.i32 d4, d4, d5 // half pixel 5 (2x32) + vpadd.i32 d5, d6, d7 // half pixel 5 (2x32) + + vext.8 q3, q6, q7, #2*6 // filter 6 pixels + load_filter_coef d2, r5, r7 // filter 7 + + vpadd.i32 d11, d4, d5 // pixel 5 (2x32) + + vmull.s16 q2, d6, d0 // filter 6 output (0-3) + vmull.s16 q3, d7, d1 // filter 6 output (4-7) + + vmovl.s8 q1, d2 // filter 7 + + vpadd.i32 d10, d10, d11 // pixel 4,5 + + vpadd.i32 d4, d4, d5 // half pixel 6 (2x32) + vpadd.i32 d5, d6, d7 // half pixel 6 (2x32) + + vext.8 q3, q6, q7, #2*7 // filter 7 pixels + + vpadd.i32 d11, d4, d5 // pixel 6 (2x32) + + vmull.s16 q2, d6, d2 // filter 7 output (0-3) + vmull.s16 q3, d7, d3 // filter 7 output (4-7) + + vld1.32 {d14[],d15[]}, [sp] // -(7 - intermediate_bits) + + vpadd.i32 d4, d4, d5 // half pixel 7 (2x32) + vpadd.i32 d5, d6, d7 // half pixel 7 (2x32) + + sub r5, r5, r7, lsl #3 + + vpadd.i32 d4, d4, d5 // pixel 7 (2x32) + + add r5, r5, r8 + + vpadd.i32 d11, d11, d4 // pixel 6,7 + + vrshl.s32 q4, q4, q7 // -(7 - intermediate_bits) + vrshl.s32 q5, q5, q7 // -(7 - intermediate_bits) + + bx lr +endfunc + +// void dav1d_warp_affine_8x8_16bpc_neon( +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *const abcd, int mx, int my, +// const int bitdepth_max) +.macro warp t +function warp_affine_8x8\t\()_16bpc_neon, export=1 + push {r4-r11,lr} + vpush {q4-q7} + ldrd r4, r5, [sp, #100] + ldrd r6, r7, [sp, #108] + sub sp, sp, #8 + + clz r7, r7 + // intermediate_bits = clz(bitdepth_max) - 18 +.ifb \t + sub r8, r7, #11 // 7 + intermediate_bits = clz(bitdepth_max) - 18 + 7 +.endif + sub r7, r7, #25 // -(7 - intermediate_bits) +.ifb \t + neg r8, r8 // -(7 + intermediate_bits) +.endif + str r7, [sp] // spill -(7 - intermediate_bits) on stack +.ifb \t + str r8, [sp, #4] // spill -(7 + intermediate_bits) on stack +.endif + + ldrd r8, r9, [r4] + sxth r7, r8 + asr r8, r8, #16 + asr r4, r9, #16 + sxth r9, r9 + mov r10, #8 + sub r2, r2, r3, lsl #1 + sub r2, r2, r3 + sub r2, r2, #6 + movrel r11, X(mc_warp_filter), 64*8 +.ifnb \t + lsl r1, r1, #1 +.endif + add r5, r5, #512 + add r6, r6, #512 + + bl warp_filter_horz_neon + vmovn.i32 d16, q4 + vmovn.i32 d17, q5 + bl warp_filter_horz_neon + vmovn.i32 d18, q4 + vmovn.i32 d19, q5 + bl warp_filter_horz_neon + vmovn.i32 d20, q4 + vmovn.i32 d21, q5 + bl warp_filter_horz_neon + vmovn.i32 d22, q4 + vmovn.i32 d23, q5 + bl warp_filter_horz_neon + vmovn.i32 d24, q4 + vmovn.i32 d25, q5 + bl warp_filter_horz_neon + vmovn.i32 d26, q4 + vmovn.i32 d27, q5 + bl warp_filter_horz_neon + vmovn.i32 d28, q4 + vmovn.i32 d29, q5 + +1: + bl warp_filter_horz_neon + vmovn.i32 d30, q4 + vmovn.i32 d31, q5 + + load_filter_row d8, r6, r9 + load_filter_row d9, r6, r9 + load_filter_row d10, r6, r9 + load_filter_row d11, r6, r9 + load_filter_row d12, r6, r9 + load_filter_row d13, r6, r9 + load_filter_row d14, r6, r9 + load_filter_row d15, r6, r9 + transpose_8x8b q4, q5, q6, q7, d8, d9, d10, d11, d12, d13, d14, d15 + vmovl.s8 q1, d8 + vmovl.s8 q2, d9 + vmovl.s8 q3, d10 + vmovl.s8 q4, d11 + vmovl.s8 q5, d12 + vmovl.s8 q6, d13 + + sub r6, r6, r9, lsl #3 + + // This ordering of vmull/vmlal is highly beneficial for + // Cortex A8/A9/A53 here, but harmful for Cortex A7. + vmull.s16 q0, d16, d2 + vmlal.s16 q0, d18, d4 + vmlal.s16 q0, d20, d6 + vmlal.s16 q0, d22, d8 + vmlal.s16 q0, d24, d10 + vmlal.s16 q0, d26, d12 + vmull.s16 q1, d17, d3 + vmlal.s16 q1, d19, d5 + vmlal.s16 q1, d21, d7 + vmlal.s16 q1, d23, d9 + vmlal.s16 q1, d25, d11 + vmlal.s16 q1, d27, d13 + + vmovl.s8 q2, d14 + vmovl.s8 q3, d15 + + vmlal.s16 q0, d28, d4 + vmlal.s16 q0, d30, d6 + vmlal.s16 q1, d29, d5 + vmlal.s16 q1, d31, d7 + +.ifb \t + ldr lr, [sp, #4] // -(7 + intermediate_bits) + ldr r12, [sp, #120] // bitdepth_max + vdup.32 q2, lr // -(7 + intermediate_bits) + vdup.16 q3, r12 // bitdepth_max +.endif + + vmov q8, q9 + vmov q9, q10 +.ifb \t + vrshl.s32 q0, q0, q2 // -(7 + intermediate_bits) + vrshl.s32 q1, q1, q2 // -(7 + intermediate_bits) +.else + vrshrn.s32 d0, q0, #7 + vrshrn.s32 d1, q1, #7 + vmov.i16 q3, #PREP_BIAS +.endif + vmov q10, q11 +.ifb \t + vqmovun.s32 d0, q0 + vqmovun.s32 d1, q1 +.else + vsub.i16 q0, q0, q3 // PREP_BIAS +.endif + vmov q11, q12 + vmov q12, q13 +.ifb \t + vmin.u16 q0, q0, q3 // bitdepth_max +.endif + vmov q13, q14 + vmov q14, q15 + subs r10, r10, #1 + vst1.16 {q0}, [r0, :128], r1 + + add r6, r6, r4 + bgt 1b + + add sp, sp, #8 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +warp +warp t + +// void dav1d_emu_edge_16bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_16bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + ldrd r8, r9, [sp, #52] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub r12, r3, #1 // ih - 1 + cmp r5, r3 + sub lr, r2, #1 // iw - 1 + it lt + movlt r12, r5 // min(y, ih - 1) + cmp r4, r2 + bic r12, r12, r12, asr #31 // max(min(y, ih - 1), 0) + it lt + movlt lr, r4 // min(x, iw - 1) + bic lr, lr, lr, asr #31 // max(min(x, iw - 1), 0) + mla r8, r12, r9, r8 // ref += iclip() * stride + add r8, r8, lr, lsl #1 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add r10, r5, r1 // y + bh + neg r5, r5 // -y + sub r10, r10, r3 // y + bh - ih + sub r12, r1, #1 // bh - 1 + cmp r10, r1 + bic r5, r5, r5, asr #31 // max(-y, 0) + it ge + movge r10, r12 // min(y + bh - ih, bh-1) + cmp r5, r1 + bic r10, r10, r10, asr #31 // max(min(y + bh - ih, bh-1), 0) + it ge + movge r5, r12 // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add r11, r4, r0 // x + bw + neg r4, r4 // -x + sub r11, r11, r2 // x + bw - iw + sub lr, r0, #1 // bw - 1 + cmp r11, r0 + bic r4, r4, r4, asr #31 // max(-x, 0) + it ge + movge r11, lr // min(x + bw - iw, bw-1) + cmp r4, r0 + bic r11, r11, r11, asr #31 // max(min(x + bw - iw, bw-1), 0) + it ge + movge r4, lr // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub r1, r1, r5 // bh - top_ext + mla r6, r5, r7, r6 + sub r2, r0, r4 // bw - left_ext + sub r1, r1, r10 // center_h = bh - top_ext - bottom_ext + sub r2, r2, r11 // center_w = bw - left_ext - right_ext + + mov r0, r6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + vld1.16 {d0[], d1[]}, [r8] + mov r12, r6 // out = dst + mov r3, r4 + vmov q1, q0 +1: + subs r3, r3, #16 + vst1.16 {q0, q1}, [r12, :128]! + bgt 1b +.endif + mov lr, r8 + add r12, r6, r4, lsl #1 // out = dst + left_ext + mov r3, r2 +1: + vld1.16 {q0, q1}, [lr]! + subs r3, r3, #32 + vld1.16 {q2, q3}, [lr]! +.if \need_left + vst1.16 {q0, q1}, [r12]! + vst1.16 {q2, q3}, [r12]! +.else + vst1.16 {q0, q1}, [r12, :128]! + vst1.16 {q2, q3}, [r12, :128]! +.endif + bgt 1b +.if \need_right + add r3, r8, r2, lsl #1 // in + center_w + sub r3, r3, #2 // in + center_w - 1 + add r12, r6, r4, lsl #1 // dst + left_ext + vld1.16 {d0[], d1[]}, [r3] + add r12, r12, r2, lsl #1 // out = dst + left_ext + center_w + mov r3, r11 + vmov q1, q0 +1: + subs r3, r3, #16 + vst1.16 {q0, q1}, [r12]! + bgt 1b +.endif + + subs r1, r1, #1 // center_h-- + add r6, r6, r7 + add r8, r8, r9 + bgt 0b +.endm + + cmp r4, #0 + beq 2f + // need_left + cmp r11, #0 + beq 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cmp r11, #0 + beq 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + cmp r10, #0 + // Storing the original dst in r0 overwrote bw, recalculate it here + add r2, r2, r4 // center_w + left_ext + add r2, r2, r11 // bw = center_w + left_ext + right_ext + + beq 3f + // need_bottom + sub r8, r6, r7 // ref = dst - stride + mov r4, r2 + sub r12, r7, #32 +1: + vld1.16 {q0, q1}, [r8, :128]! + mov r3, r10 + vld1.16 {q2, q3}, [r8, :128]! +2: + vst1.16 {q0, q1}, [r6, :128]! + subs r3, r3, #1 + vst1.16 {q2, q3}, [r6, :128], r12 + bgt 2b + mls r6, r7, r10, r6 // dst -= bottom_ext * stride + subs r4, r4, #32 // bw -= 32 + add r6, r6, #64 // dst += 32 + bgt 1b + +3: + cmp r5, #0 + beq 3f + // need_top + mls r6, r7, r5, r0 // dst = stored_dst - top_ext * stride + sub r12, r7, #32 +1: + vld1.16 {q0, q1}, [r0, :128]! + mov r3, r5 + vld1.16 {q2, q3}, [r0, :128]! +2: + vst1.16 {q0, q1}, [r6, :128]! + subs r3, r3, #1 + vst1.16 {q2, q3}, [r6, :128], r12 + bgt 2b + mls r6, r7, r5, r6 // dst -= top_ext * stride + subs r2, r2, #32 // bw -= 32 + add r6, r6, #64 // dst += 32 + bgt 1b + +3: + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/msac.S b/third_party/dav1d/src/arm/32/msac.S new file mode 100644 index 0000000000..b06e109dda --- /dev/null +++ b/third_party/dav1d/src/arm/32/msac.S @@ -0,0 +1,575 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define BUF_POS 0 +#define BUF_END 4 +#define DIF 8 +#define RNG 12 +#define CNT 16 +#define ALLOW_UPDATE_CDF 20 + +const coeffs + .short 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0 + .short 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +endconst + +const bits, align=4 + .short 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + .short 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000 +endconst + +.macro vld1_align_n d0, q0, q1, src, n +.if \n == 4 + vld1.16 {\d0}, [\src, :64] +.elseif \n == 8 + vld1.16 {\q0}, [\src, :128] +.else + vld1.16 {\q0, \q1}, [\src, :128] +.endif +.endm + +.macro vld1_n d0, q0, q1, src, n +.if \n == 4 + vld1.16 {\d0}, [\src] +.elseif \n == 8 + vld1.16 {\q0}, [\src] +.else + vld1.16 {\q0, \q1}, [\src] +.endif +.endm + +.macro vst1_align_n d0, q0, q1, src, n +.if \n == 4 + vst1.16 {\d0}, [\src, :64] +.elseif \n == 8 + vst1.16 {\q0}, [\src, :128] +.else + vst1.16 {\q0, \q1}, [\src, :128] +.endif +.endm + +.macro vst1_n d0, q0, q1, src, n +.if \n == 4 + vst1.16 {\d0}, [\src] +.elseif \n == 8 + vst1.16 {\q0}, [\src] +.else + vst1.16 {\q0, \q1}, [\src] +.endif +.endm + +.macro vshr_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vshr.u16 \d0, \s0, \s3 +.else + vshr.u16 \d1, \s1, \s4 +.if \n == 16 + vshr.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vadd.i16 \d0, \s0, \s3 +.else + vadd.i16 \d1, \s1, \s4 +.if \n == 16 + vadd.i16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vsub_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vsub.i16 \d0, \s0, \s3 +.else + vsub.i16 \d1, \s1, \s4 +.if \n == 16 + vsub.i16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vand_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vand \d0, \s0, \s3 +.else + vand \d1, \s1, \s4 +.if \n == 16 + vand \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vcge_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vcge.u16 \d0, \s0, \s3 +.else + vcge.u16 \d1, \s1, \s4 +.if \n == 16 + vcge.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vrhadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vrhadd.u16 \d0, \s0, \s3 +.else + vrhadd.u16 \d1, \s1, \s4 +.if \n == 16 + vrhadd.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vshl_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vshl.s16 \d0, \s0, \s3 +.else + vshl.s16 \d1, \s1, \s4 +.if \n == 16 + vshl.s16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vqdmulh_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vqdmulh.s16 \d0, \s0, \s3 +.else + vqdmulh.s16 \d1, \s1, \s4 +.if \n == 16 + vqdmulh.s16 \d2, \s2, \s5 +.endif +.endif +.endm + +// unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf, +// size_t n_symbols); + +function msac_decode_symbol_adapt4_neon, export=1 +.macro decode_update n + push {r4-r10,lr} + sub sp, sp, #48 + add r8, r0, #RNG + + vld1_align_n d0, q0, q1, r1, \n // cdf + vld1.16 {d16[]}, [r8, :16] // rng + movrel_local r9, coeffs, 30 + vmov.i16 d30, #0x7f00 // 0x7f00 + sub r9, r9, r2, lsl #1 + vmvn.i16 q14, #0x3f // 0xffc0 + add r8, sp, #14 + vand d22, d16, d30 // rng & 0x7f00 + vst1.16 {d16[0]}, [r8, :16] // store original u = s->rng + vand_n d4, q2, q3, d0, q0, q1, d28, q14, q14, \n // cdf & 0xffc0 +.if \n > 4 + vmov d23, d22 +.endif + + vld1_n d16, q8, q9, r9, \n // EC_MIN_PROB * (n_symbols - ret) + vqdmulh_n d20, q10, q11, d4, q2, q3, d22, q11, q11, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add r8, r0, #DIF + 2 + + vadd_n d16, q8, q9, d4, q2, q3, d16, q8, q9, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret) +.if \n == 4 + vmov.i16 d17, #0 +.endif + vadd_n d16, q8, q9, d20, q10, q11, d16, q8, q9, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + + add r9, sp, #16 + vld1.16 {d20[]}, [r8, :16] // dif >> (EC_WIN_SIZE - 16) + movrel_local r8, bits + vst1_n q8, q8, q9, r9, \n // store v values to allow indexed access + + vmov d21, d20 + vld1_align_n q12, q12, q13, r8, \n +.if \n == 16 + vmov q11, q10 +.endif + + vcge_n q2, q2, q3, q10, q10, q11, q8, q8, q9, \n // c >= v + + vand_n q10, q10, q11, q2, q2, q3, q12, q12, q13, \n // One bit per halfword set in the mask +.if \n == 16 + vadd.i16 q10, q10, q11 +.endif + vadd.i16 d20, d20, d21 // Aggregate mask bits + ldr r4, [r0, #ALLOW_UPDATE_CDF] + vpadd.i16 d20, d20, d20 + lsl r10, r2, #1 + vpadd.i16 d20, d20, d20 + vmov.u16 r3, d20[0] + cmp r4, #0 + rbit r3, r3 + clz lr, r3 // ret + + beq L(renorm) + // update_cdf + ldrh r3, [r1, r10] // count = cdf[n_symbols] + vmov.i8 q10, #0xff +.if \n == 16 + mov r4, #-5 +.else + mvn r12, r2 + mov r4, #-4 + cmn r12, #3 // set C if n_symbols <= 2 +.endif + vrhadd_n d16, q8, q9, d20, q10, q10, d4, q2, q3, \n // i >= val ? -1 : 32768 +.if \n == 16 + sub r4, r4, r3, lsr #4 // -((count >> 4) + 5) +.else + lsr r12, r3, #4 // count >> 4 + sbc r4, r4, r12 // -((count >> 4) + (n_symbols > 2) + 4) +.endif + vsub_n d16, q8, q9, d16, q8, q9, d0, q0, q1, \n // (32768 - cdf[i]) or (-1 - cdf[i]) +.if \n == 4 + vdup.16 d20, r4 // -rate +.else + vdup.16 q10, r4 // -rate +.endif + + sub r3, r3, r3, lsr #5 // count - (count == 32) + vsub_n d0, q0, q1, d0, q0, q1, d4, q2, q3, \n // cdf + (i >= val ? 1 : 0) + vshl_n d16, q8, q9, d16, q8, q9, d20, q10, q10, \n // ({32768,-1} - cdf[i]) >> rate + add r3, r3, #1 // count + (count < 32) + vadd_n d0, q0, q1, d0, q0, q1, d16, q8, q9, \n // cdf + (32768 - cdf[i]) >> rate + vst1_align_n d0, q0, q1, r1, \n + strh r3, [r1, r10] +.endm + + decode_update 4 + +L(renorm): + add r8, sp, #16 + add r8, r8, lr, lsl #1 + ldrh r3, [r8] // v + ldrh r4, [r8, #-2] // u + ldr r6, [r0, #CNT] + ldr r7, [r0, #DIF] + sub r4, r4, r3 // rng = u - v + clz r5, r4 // clz(rng) + eor r5, r5, #16 // d = clz(rng) ^ 16 + mvn r7, r7 // ~dif + add r7, r7, r3, lsl #16 // ~dif + (v << 16) +L(renorm2): + lsl r4, r4, r5 // rng << d + subs r6, r6, r5 // cnt -= d + lsl r7, r7, r5 // (~dif + (v << 16)) << d + str r4, [r0, #RNG] + mvn r7, r7 // ~dif + bhs 9f + + // refill + ldr r3, [r0, #BUF_POS] // BUF_POS + ldr r4, [r0, #BUF_END] // BUF_END + add r5, r3, #4 + cmp r5, r4 + bgt 2f + + ldr r3, [r3] // next_bits + add r8, r6, #23 // shift_bits = cnt + 23 + add r6, r6, #16 // cnt += 16 + rev r3, r3 // next_bits = bswap(next_bits) + sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3 + and r8, r8, #24 // shift_bits &= 24 + lsr r3, r3, r8 // next_bits >>= shift_bits + sub r8, r8, r6 // shift_bits -= 16 + cnt + str r5, [r0, #BUF_POS] + lsl r3, r3, r8 // next_bits <<= shift_bits + rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits + eor r7, r7, r3 // dif ^= next_bits + b 9f + +2: // refill_eob + rsb r5, r6, #8 // c = 8 - cnt +3: + cmp r3, r4 + bge 4f + ldrb r8, [r3], #1 + lsl r8, r8, r5 + eor r7, r7, r8 + subs r5, r5, #8 + bge 3b + +4: // refill_eob_end + str r3, [r0, #BUF_POS] + rsb r6, r5, #8 // cnt = 8 - c + +9: + str r6, [r0, #CNT] + str r7, [r0, #DIF] + + mov r0, lr + add sp, sp, #48 + + pop {r4-r10,pc} +endfunc + +function msac_decode_symbol_adapt8_neon, export=1 + decode_update 8 + b L(renorm) +endfunc + +function msac_decode_symbol_adapt16_neon, export=1 + decode_update 16 + b L(renorm) +endfunc + +function msac_decode_hi_tok_neon, export=1 + push {r4-r10,lr} + vld1.16 {d0}, [r1, :64] // cdf + add r4, r0, #RNG + vmov.i16 d31, #0x7f00 // 0x7f00 + movrel_local r5, coeffs, 30-2*3 + vmvn.i16 d30, #0x3f // 0xffc0 + ldrh r9, [r1, #6] // count = cdf[n_symbols] + vld1.16 {d1[]}, [r4, :16] // rng + movrel_local r4, bits + vld1.16 {d29}, [r5] // EC_MIN_PROB * (n_symbols - ret) + add r5, r0, #DIF + 2 + vld1.16 {q8}, [r4, :128] + mov r2, #-24 + vand d20, d0, d30 // cdf & 0xffc0 + ldr r10, [r0, #ALLOW_UPDATE_CDF] + vld1.16 {d2[]}, [r5, :16] // dif >> (EC_WIN_SIZE - 16) + sub sp, sp, #48 + ldr r6, [r0, #CNT] + ldr r7, [r0, #DIF] + vmov d3, d2 +1: + vand d23, d1, d31 // rng & 0x7f00 + vqdmulh.s16 d18, d20, d23 // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add r12, sp, #14 + vadd.i16 d6, d20, d29 // v = cdf + EC_MIN_PROB * (n_symbols - ret) + vadd.i16 d6, d18, d6 // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + vmov.i16 d7, #0 + vst1.16 {d1[0]}, [r12, :16] // store original u = s->rng + add r12, sp, #16 + vcge.u16 q2, q1, q3 // c >= v + vst1.16 {q3}, [r12] // store v values to allow indexed access + vand q9, q2, q8 // One bit per halfword set in the mask + + vadd.i16 d18, d18, d19 // Aggregate mask bits + vpadd.i16 d18, d18, d18 + vpadd.i16 d18, d18, d18 + vmov.u16 r3, d18[0] + cmp r10, #0 + add r2, r2, #5 + rbit r3, r3 + add r8, sp, #16 + clz lr, r3 // ret + + beq 2f + // update_cdf + vmov.i8 d22, #0xff + mov r4, #-5 + vrhadd.u16 d6, d22, d4 // i >= val ? -1 : 32768 + sub r4, r4, r9, lsr #4 // -((count >> 4) + 5) + vsub.i16 d6, d6, d0 // (32768 - cdf[i]) or (-1 - cdf[i]) + vdup.16 d18, r4 // -rate + + sub r9, r9, r9, lsr #5 // count - (count == 32) + vsub.i16 d0, d0, d4 // cdf + (i >= val ? 1 : 0) + vshl.s16 d6, d6, d18 // ({32768,-1} - cdf[i]) >> rate + add r9, r9, #1 // count + (count < 32) + vadd.i16 d0, d0, d6 // cdf + (32768 - cdf[i]) >> rate + vst1.16 {d0}, [r1, :64] + vand d20, d0, d30 // cdf & 0xffc0 + strh r9, [r1, #6] + +2: + add r8, r8, lr, lsl #1 + ldrh r3, [r8] // v + ldrh r4, [r8, #-2] // u + sub r4, r4, r3 // rng = u - v + clz r5, r4 // clz(rng) + eor r5, r5, #16 // d = clz(rng) ^ 16 + mvn r7, r7 // ~dif + add r7, r7, r3, lsl #16 // ~dif + (v << 16) + lsl r4, r4, r5 // rng << d + subs r6, r6, r5 // cnt -= d + lsl r7, r7, r5 // (~dif + (v << 16)) << d + str r4, [r0, #RNG] + vdup.16 d1, r4 + mvn r7, r7 // ~dif + bhs 9f + + // refill + ldr r3, [r0, #BUF_POS] // BUF_POS + ldr r4, [r0, #BUF_END] // BUF_END + add r5, r3, #4 + cmp r5, r4 + bgt 2f + + ldr r3, [r3] // next_bits + add r8, r6, #23 // shift_bits = cnt + 23 + add r6, r6, #16 // cnt += 16 + rev r3, r3 // next_bits = bswap(next_bits) + sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3 + and r8, r8, #24 // shift_bits &= 24 + lsr r3, r3, r8 // next_bits >>= shift_bits + sub r8, r8, r6 // shift_bits -= 16 + cnt + str r5, [r0, #BUF_POS] + lsl r3, r3, r8 // next_bits <<= shift_bits + rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits + eor r7, r7, r3 // dif ^= next_bits + b 9f + +2: // refill_eob + rsb r5, r6, #8 // c = 40 - cnt +3: + cmp r3, r4 + bge 4f + ldrb r8, [r3], #1 + lsl r8, r8, r5 + eor r7, r7, r8 + subs r5, r5, #8 + bge 3b + +4: // refill_eob_end + str r3, [r0, #BUF_POS] + rsb r6, r5, #8 // cnt = 40 - c + +9: + lsl lr, lr, #1 + sub lr, lr, #5 + lsr r12, r7, #16 + adds r2, r2, lr // carry = tok_br < 3 || tok == 15 + vdup.16 q1, r12 + bcc 1b // loop if !carry + add r2, r2, #30 + str r6, [r0, #CNT] + add sp, sp, #48 + str r7, [r0, #DIF] + lsr r0, r2, #1 + pop {r4-r10,pc} +endfunc + +function msac_decode_bool_equi_neon, export=1 + push {r4-r10,lr} + ldr r5, [r0, #RNG] + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + bic r4, r5, #0xff // r &= 0xff00 + add r4, r4, #8 + mov r2, #0 + subs r8, r7, r4, lsl #15 // dif - vw + lsr r4, r4, #1 // v + sub r5, r5, r4 // r - v + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + b L(renorm2) +endfunc + +function msac_decode_bool_neon, export=1 + push {r4-r10,lr} + ldr r5, [r0, #RNG] + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + lsr r4, r5, #8 // r >> 8 + bic r1, r1, #0x3f // f &= ~63 + mul r4, r4, r1 + mov r2, #0 + lsr r4, r4, #7 + add r4, r4, #4 // v + subs r8, r7, r4, lsl #16 // dif - vw + sub r5, r5, r4 // r - v + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + b L(renorm2) +endfunc + +function msac_decode_bool_adapt_neon, export=1 + push {r4-r10,lr} + ldr r9, [r1] // cdf[0-1] + ldr r5, [r0, #RNG] + movw lr, #0xffc0 + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + lsr r4, r5, #8 // r >> 8 + and r2, r9, lr // f &= ~63 + mul r4, r4, r2 + mov r2, #0 + lsr r4, r4, #7 + add r4, r4, #4 // v + subs r8, r7, r4, lsl #16 // dif - vw + sub r5, r5, r4 // r - v + ldr r10, [r0, #ALLOW_UPDATE_CDF] + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + cmp r10, #0 + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + + beq L(renorm2) + + lsr r2, r9, #16 // count = cdf[1] + uxth r9, r9 // cdf[0] + + sub r3, r2, r2, lsr #5 // count - (count >= 32) + lsr r2, r2, #4 // count >> 4 + add r10, r3, #1 // count + (count < 32) + add r2, r2, #4 // rate = (count >> 4) | 4 + + sub r9, r9, lr // cdf[0] -= bit + sub r3, r9, lr, lsl #15 // {cdf[0], cdf[0] - 32769} + asr r3, r3, r2 // {cdf[0], cdf[0] - 32769} >> rate + sub r9, r9, r3 // cdf[0] + + strh r9, [r1] + strh r10, [r1, #2] + + b L(renorm2) +endfunc diff --git a/third_party/dav1d/src/arm/32/refmvs.S b/third_party/dav1d/src/arm/32/refmvs.S new file mode 100644 index 0000000000..e16c5448d0 --- /dev/null +++ b/third_party/dav1d/src/arm/32/refmvs.S @@ -0,0 +1,97 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void dav1d_splat_mv_neon(refmvs_block **rr, const refmvs_block *rmv, +// int bx4, int bw4, int bh4) + +function splat_mv_neon, export=1 + push {r4, lr} + vld1.8 {q3}, [r1] + ldr r4, [sp, #8] + clz r3, r3 + adr lr, L(splat_tbl) + sub r3, r3, #26 + vext.8 q2, q3, q3, #12 + ldr r3, [lr, r3, lsl #2] + add r2, r2, r2, lsl #1 + vext.8 q0, q2, q3, #4 + add r3, lr, r3 + vext.8 q1, q2, q3, #8 + lsl r2, r2, #2 + vext.8 q2, q2, q3, #12 + vmov q3, q0 +1: + ldr r1, [r0], #4 + subs r4, r4, #1 + add r1, r1, r2 + bx r3 + + .align 2 +L(splat_tbl): + .word 320f - L(splat_tbl) + CONFIG_THUMB + .word 160f - L(splat_tbl) + CONFIG_THUMB + .word 80f - L(splat_tbl) + CONFIG_THUMB + .word 40f - L(splat_tbl) + CONFIG_THUMB + .word 20f - L(splat_tbl) + CONFIG_THUMB + .word 10f - L(splat_tbl) + CONFIG_THUMB + +10: + vst1.8 {d0}, [r1] + vstr s2, [r1, #8] + bgt 1b + pop {r4, pc} +20: + vst1.8 {q0}, [r1] + vstr d2, [r1, #16] + bgt 1b + pop {r4, pc} +40: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2}, [r1] + bgt 1b + pop {r4, pc} +320: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! +160: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! +80: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1] + bgt 1b + pop {r4, pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/util.S b/third_party/dav1d/src/arm/32/util.S new file mode 100644 index 0000000000..c3710d3767 --- /dev/null +++ b/third_party/dav1d/src/arm/32/util.S @@ -0,0 +1,184 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2015 Martin Storsjo + * Copyright © 2015 Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#ifndef DAV1D_SRC_ARM_32_UTIL_S +#define DAV1D_SRC_ARM_32_UTIL_S + +#include "config.h" +#include "src/arm/asm.S" + +.macro movrel_local rd, val, offset=0 +#if defined(PIC) + ldr \rd, 90001f + b 90002f +90001: + .word \val + \offset - (90002f + 8 - 4 * CONFIG_THUMB) +90002: + add \rd, \rd, pc +#else + movw \rd, #:lower16:\val+\offset + movt \rd, #:upper16:\val+\offset +#endif +.endm + +.macro movrel rd, val, offset=0 +#if defined(PIC) && defined(__APPLE__) + ldr \rd, 1f + b 2f +1: + .word 3f - (2f + 8 - 4 * CONFIG_THUMB) +2: + ldr \rd, [pc, \rd] +.if \offset < 0 + sub \rd, \rd, #-(\offset) +.elseif \offset > 0 + add \rd, \rd, #\offset +.endif + .non_lazy_symbol_pointer +3: + .indirect_symbol \val + .word 0 + .text +#else + movrel_local \rd, \val, \offset +#endif +.endm + +// This macro clobbers r7 (and r12 on windows) and stores data at the +// bottom of the stack; sp is the start of the space allocated that +// the caller can use. +.macro sub_sp_align space +#if CONFIG_THUMB + mov r7, sp + and r7, r7, #15 +#else + and r7, sp, #15 +#endif + sub sp, sp, r7 + // Now the stack is aligned, store the amount of adjustment back + // on the stack, as we don't want to waste a register as frame + // pointer. + str r7, [sp, #-16]! +#ifdef _WIN32 +.if \space > 8192 + // Here, we'd need to touch two (or more) pages while decrementing + // the stack pointer. + .error "sub_sp_align doesn't support values over 8K at the moment" +.elseif \space > 4096 + sub r7, sp, #4096 + ldr r12, [r7] + sub r7, r7, #(\space - 4096) + mov sp, r7 +.else + sub sp, sp, #\space +.endif +#else +.if \space >= 4096 + sub sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + sub sp, sp, #(\space)%4096 +.endif +#endif +.endm + +.macro add_sp_align space +.if \space >= 4096 + add sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + add sp, sp, #(\space)%4096 +.endif + ldr r7, [sp], #16 + // Add back the original stack adjustment + add sp, sp, r7 +.endm + +.macro transpose_8x8b q0, q1, q2, q3, r0, r1, r2, r3, r4, r5, r6, r7 + vtrn.32 \q0, \q2 + vtrn.32 \q1, \q3 + + vtrn.16 \r0, \r2 + vtrn.16 \r1, \r3 + vtrn.16 \r4, \r6 + vtrn.16 \r5, \r7 + + vtrn.8 \r0, \r1 + vtrn.8 \r2, \r3 + vtrn.8 \r4, \r5 + vtrn.8 \r6, \r7 +.endm + +.macro transpose_8x8h r0, r1, r2, r3, r4, r5, r6, r7, d0, d1, d2, d3, d4, d5, d6, d7 + vswp \d0, \d4 + vswp \d1, \d5 + vswp \d2, \d6 + vswp \d3, \d7 + + vtrn.32 \r0, \r2 + vtrn.32 \r1, \r3 + vtrn.32 \r4, \r6 + vtrn.32 \r5, \r7 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 + vtrn.16 \r4, \r5 + vtrn.16 \r6, \r7 +.endm + +.macro transpose_4x8b q0, q1, r0, r1, r2, r3 + vtrn.16 \q0, \q1 + + vtrn.8 \r0, \r1 + vtrn.8 \r2, \r3 +.endm + +.macro transpose_4x4s q0, q1, q2, q3, r0, r1, r2, r3, r4, r5, r6, r7 + vswp \r1, \r4 // vtrn.64 \q0, \q2 + vswp \r3, \r6 // vtrn.64 \q1, \q3 + + vtrn.32 \q0, \q1 + vtrn.32 \q2, \q3 +.endm + +.macro transpose_4x4h q0, q1, r0, r1, r2, r3 + vtrn.32 \q0, \q1 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 +.endm + +.macro transpose_4x8h r0, r1, r2, r3 + vtrn.32 \r0, \r2 + vtrn.32 \r1, \r3 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 +.endm + +#endif /* DAV1D_SRC_ARM_32_UTIL_S */ diff --git a/third_party/dav1d/src/arm/64/cdef.S b/third_party/dav1d/src/arm/64/cdef.S new file mode 100644 index 0000000000..32b258aba8 --- /dev/null +++ b/third_party/dav1d/src/arm/64/cdef.S @@ -0,0 +1,520 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "cdef_tmpl.S" + +.macro pad_top_bottom s1, s2, w, stride, rn, rw, ret + tst w7, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + sub \s1, \s1, #2 + sub \s2, \s2, #2 + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \rn\()0, [\s1] + ldr s1, [\s1, #\w] + ldr \rn\()2, [\s2] + ldr s3, [\s2, #\w] + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + str \rw\()0, [x0] + str d1, [x0, #2*\w] + add x0, x0, #2*\stride + str \rw\()2, [x0] + str d3, [x0, #2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \rn\()0, [\s1] + ldr h1, [\s1, #\w] + ldr \rn\()2, [\s2] + ldr h3, [\s2, #\w] + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + str \rw\()0, [x0] + str s1, [x0, #2*\w] + str s31, [x0, #2*\w+4] + add x0, x0, #2*\stride + str \rw\()2, [x0] + str s3, [x0, #2*\w] + str s31, [x0, #2*\w+4] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +2: + // !CDEF_HAVE_LEFT + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \rn\()0, [\s1] + ldr h1, [\s1, #\w] + ldr \rn\()2, [\s2] + ldr h3, [\s2, #\w] + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + str s31, [x0] + stur \rw\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \rw\()2, [x0, #4] + str s3, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \rn\()0, [\s1] + ldr \rn\()1, [\s2] + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + str s31, [x0] + stur \rw\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \rw\()1, [x0, #4] + str s31, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride +.endif +3: +.endm + +.macro load_n_incr dst, src, incr, w +.if \w == 4 + ld1 {\dst\().s}[0], [\src], \incr +.else + ld1 {\dst\().8b}, [\src], \incr +.endif +.endm + +// void dav1d_cdef_paddingX_8bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func w, stride, rn, rw +function cdef_padding\w\()_8bpc_neon, export=1 + cmp w7, #0xf // fully edged + b.eq cdef_padding\w\()_edged_8bpc_neon + movi v30.8h, #0x80, lsl #8 + mov v31.16b, v30.16b + sub x0, x0, #2*(2*\stride+2) + tst w7, #4 // CDEF_HAVE_TOP + b.ne 1f + // !CDEF_HAVE_TOP + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + b 3f +1: + // CDEF_HAVE_TOP + add x9, x4, x2 + pad_top_bottom x4, x9, \w, \stride, \rn, \rw, 0 + + // Middle section +3: + tst w7, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ld1 {v0.h}[0], [x3], #2 + ldr h2, [x1, #\w] + load_n_incr v1, x1, x2, \w + subs w6, w6, #1 + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + str s0, [x0] + stur \rw\()1, [x0, #4] + str s2, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ld1 {v0.h}[0], [x3], #2 + load_n_incr v1, x1, x2, \w + subs w6, w6, #1 + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + str s0, [x0] + stur \rw\()1, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + b 3f +2: + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ldr h1, [x1, #\w] + load_n_incr v0, x1, x2, \w + subs w6, w6, #1 + uxtl v0.8h, v0.8b + uxtl v1.8h, v1.8b + str s31, [x0] + stur \rw\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + load_n_incr v0, x1, x2, \w + subs w6, w6, #1 + uxtl v0.8h, v0.8b + str s31, [x0] + stur \rw\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + +3: + tst w7, #8 // CDEF_HAVE_BOTTOM + b.ne 1f + // !CDEF_HAVE_BOTTOM + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + ret +1: + // CDEF_HAVE_BOTTOM + add x9, x5, x2 + pad_top_bottom x5, x9, \w, \stride, \rn, \rw, 1 +endfunc +.endm + +padding_func 8, 16, d, q +padding_func 4, 8, s, d + +// void cdef_paddingX_edged_8bpc_neon(uint8_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_edged w, stride, reg +function cdef_padding\w\()_edged_8bpc_neon, export=1 + sub x4, x4, #2 + sub x5, x5, #2 + sub x0, x0, #(2*\stride+2) + +.if \w == 4 + ldr d0, [x4] + ldr d1, [x4, x2] + st1 {v0.8b, v1.8b}, [x0], #16 +.else + add x9, x4, x2 + ldr d0, [x4] + ldr s1, [x4, #8] + ldr d2, [x9] + ldr s3, [x9, #8] + str d0, [x0] + str s1, [x0, #8] + str d2, [x0, #\stride] + str s3, [x0, #\stride+8] + add x0, x0, #2*\stride +.endif + +0: + ld1 {v0.h}[0], [x3], #2 + ldr h2, [x1, #\w] + load_n_incr v1, x1, x2, \w + subs w6, w6, #1 + str h0, [x0] + stur \reg\()1, [x0, #2] + str h2, [x0, #2+\w] + add x0, x0, #\stride + b.gt 0b + +.if \w == 4 + ldr d0, [x5] + ldr d1, [x5, x2] + st1 {v0.8b, v1.8b}, [x0], #16 +.else + add x9, x5, x2 + ldr d0, [x5] + ldr s1, [x5, #8] + ldr d2, [x9] + ldr s3, [x9, #8] + str d0, [x0] + str s1, [x0, #8] + str d2, [x0, #\stride] + str s3, [x0, #\stride+8] +.endif + ret +endfunc +.endm + +padding_func_edged 8, 16, d +padding_func_edged 4, 8, s + +tables + +filter 8, 8 +filter 4, 8 + +find_dir 8 + +.macro load_px_8 d1, d2, w +.if \w == 8 + add x6, x2, w9, sxtb // x + off + sub x9, x2, w9, sxtb // x - off + ld1 {\d1\().d}[0], [x6] // p0 + add x6, x6, #16 // += stride + ld1 {\d2\().d}[0], [x9] // p1 + add x9, x9, #16 // += stride + ld1 {\d1\().d}[1], [x6] // p0 + ld1 {\d2\().d}[1], [x9] // p0 +.else + add x6, x2, w9, sxtb // x + off + sub x9, x2, w9, sxtb // x - off + ld1 {\d1\().s}[0], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[0], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[1], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[1], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[2], [x6] // p0 + add x6, x6, #8 // += stride + ld1 {\d2\().s}[2], [x9] // p1 + add x9, x9, #8 // += stride + ld1 {\d1\().s}[3], [x6] // p0 + ld1 {\d2\().s}[3], [x9] // p1 +.endif +.endm +.macro handle_pixel_8 s1, s2, thresh_vec, shift, tap, min +.if \min + umin v3.16b, v3.16b, \s1\().16b + umax v4.16b, v4.16b, \s1\().16b + umin v3.16b, v3.16b, \s2\().16b + umax v4.16b, v4.16b, \s2\().16b +.endif + uabd v16.16b, v0.16b, \s1\().16b // abs(diff) + uabd v20.16b, v0.16b, \s2\().16b // abs(diff) + ushl v17.16b, v16.16b, \shift // abs(diff) >> shift + ushl v21.16b, v20.16b, \shift // abs(diff) >> shift + uqsub v17.16b, \thresh_vec, v17.16b // clip = imax(0, threshold - (abs(diff) >> shift)) + uqsub v21.16b, \thresh_vec, v21.16b // clip = imax(0, threshold - (abs(diff) >> shift)) + cmhi v18.16b, v0.16b, \s1\().16b // px > p0 + cmhi v22.16b, v0.16b, \s2\().16b // px > p1 + umin v17.16b, v17.16b, v16.16b // imin(abs(diff), clip) + umin v21.16b, v21.16b, v20.16b // imin(abs(diff), clip) + dup v19.16b, \tap // taps[k] + neg v16.16b, v17.16b // -imin() + neg v20.16b, v21.16b // -imin() + bsl v18.16b, v16.16b, v17.16b // constrain() = apply_sign() + bsl v22.16b, v20.16b, v21.16b // constrain() = apply_sign() + mla v1.16b, v18.16b, v19.16b // sum += taps[k] * constrain() + mla v2.16b, v22.16b, v19.16b // sum += taps[k] * constrain() +.endm + +// void cdef_filterX_edged_8bpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint8_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h); +.macro filter_func_8 w, pri, sec, min, suffix +function cdef_filter\w\suffix\()_edged_8bpc_neon +.if \pri + movrel x8, pri_taps + and w9, w3, #1 + add x8, x8, w9, uxtw #1 +.endif + movrel x9, directions\w + add x5, x9, w5, uxtw #1 + movi v30.8b, #7 + dup v28.8b, w6 // damping + +.if \pri + dup v25.16b, w3 // threshold +.endif +.if \sec + dup v27.16b, w4 // threshold +.endif + trn1 v24.8b, v25.8b, v27.8b + clz v24.8b, v24.8b // clz(threshold) + sub v24.8b, v30.8b, v24.8b // ulog2(threshold) + uqsub v24.8b, v28.8b, v24.8b // shift = imax(0, damping - ulog2(threshold)) + neg v24.8b, v24.8b // -shift +.if \sec + dup v26.16b, v24.b[1] +.endif +.if \pri + dup v24.16b, v24.b[0] +.endif + +1: +.if \w == 8 + add x12, x2, #16 + ld1 {v0.d}[0], [x2] // px + ld1 {v0.d}[1], [x12] // px +.else + add x12, x2, #1*8 + add x13, x2, #2*8 + add x14, x2, #3*8 + ld1 {v0.s}[0], [x2] // px + ld1 {v0.s}[1], [x12] // px + ld1 {v0.s}[2], [x13] // px + ld1 {v0.s}[3], [x14] // px +.endif + + // We need 9-bits or two 8-bit accululators to fit the sum. + // Max of |sum| > 15*2*6(pri) + 4*4*3(sec) = 228. + // Start sum at -1 instead of 0 to help handle rounding later. + movi v1.16b, #255 // sum + movi v2.16b, #0 // sum +.if \min + mov v3.16b, v0.16b // min + mov v4.16b, v0.16b // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov w11, #2 // sec_taps[0] + +2: +.if \pri + ldrb w9, [x5] // off1 + + load_px_8 v5, v6, \w +.endif + +.if \sec + add x5, x5, #4 // +2*2 + ldrb w9, [x5] // off2 + load_px_8 v28, v29, \w +.endif + +.if \pri + ldrb w10, [x8] // *pri_taps + + handle_pixel_8 v5, v6, v25.16b, v24.16b, w10, \min +.endif + +.if \sec + add x5, x5, #8 // +2*4 + ldrb w9, [x5] // off3 + load_px_8 v5, v6, \w + + handle_pixel_8 v28, v29, v27.16b, v26.16b, w11, \min + + handle_pixel_8 v5, v6, v27.16b, v26.16b, w11, \min + + sub x5, x5, #11 // x5 -= 2*(2+4); x5 += 1; +.else + add x5, x5, #1 // x5 += 1 +.endif + subs w11, w11, #1 // sec_tap-- (value) +.if \pri + add x8, x8, #1 // pri_taps++ (pointer) +.endif + b.ne 2b + + // Perform halving adds since the value won't fit otherwise. + // To handle the offset for negative values, use both halving w/ and w/o rounding. + srhadd v5.16b, v1.16b, v2.16b // sum >> 1 + shadd v6.16b, v1.16b, v2.16b // (sum - 1) >> 1 + cmlt v1.16b, v5.16b, #0 // sum < 0 + bsl v1.16b, v6.16b, v5.16b // (sum - (sum < 0)) >> 1 + + srshr v1.16b, v1.16b, #3 // (8 + sum - (sum < 0)) >> 4 + + usqadd v0.16b, v1.16b // px + (8 + sum ...) >> 4 +.if \min + umin v0.16b, v0.16b, v4.16b + umax v0.16b, v0.16b, v3.16b // iclip(px + .., min, max) +.endif +.if \w == 8 + st1 {v0.d}[0], [x0], x1 + add x2, x2, #2*16 // tmp += 2*tmp_stride + subs w7, w7, #2 // h -= 2 + st1 {v0.d}[1], [x0], x1 +.else + st1 {v0.s}[0], [x0], x1 + add x2, x2, #4*8 // tmp += 4*tmp_stride + st1 {v0.s}[1], [x0], x1 + subs w7, w7, #4 // h -= 4 + st1 {v0.s}[2], [x0], x1 + st1 {v0.s}[3], [x0], x1 +.endif + + // Reset pri_taps and directions back to the original point + sub x5, x5, #2 +.if \pri + sub x8, x8, #2 +.endif + + b.gt 1b + ret +endfunc +.endm + +.macro filter_8 w +filter_func_8 \w, pri=1, sec=0, min=0, suffix=_pri +filter_func_8 \w, pri=0, sec=1, min=0, suffix=_sec +filter_func_8 \w, pri=1, sec=1, min=1, suffix=_pri_sec +.endm + +filter_8 8 +filter_8 4 diff --git a/third_party/dav1d/src/arm/64/cdef16.S b/third_party/dav1d/src/arm/64/cdef16.S new file mode 100644 index 0000000000..ecf864a26d --- /dev/null +++ b/third_party/dav1d/src/arm/64/cdef16.S @@ -0,0 +1,229 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "cdef_tmpl.S" + +.macro pad_top_bot_16 s1, s2, w, stride, reg, ret + tst w7, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + sub \s1, \s1, #4 + sub \s2, \s2, #4 + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr d1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr d3, [\s2, #2*\w] + str \reg\()0, [x0] + str d1, [x0, #2*\w] + add x0, x0, #2*\stride + str \reg\()2, [x0] + str d3, [x0, #2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr s1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr s3, [\s2, #2*\w] + str \reg\()0, [x0] + str s1, [x0, #2*\w] + str s31, [x0, #2*\w+4] + add x0, x0, #2*\stride + str \reg\()2, [x0] + str s3, [x0, #2*\w] + str s31, [x0, #2*\w+4] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +2: + // !CDEF_HAVE_LEFT + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr s1, [\s1, #2*\w] + ldr \reg\()2, [\s2] + ldr s3, [\s2, #2*\w] + str s31, [x0] + stur \reg\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \reg\()2, [x0, #4] + str s3, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride + b 3f +.endif + +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ldr \reg\()0, [\s1] + ldr \reg\()1, [\s2] + str s31, [x0] + stur \reg\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + str s31, [x0] + stur \reg\()1, [x0, #4] + str s31, [x0, #4+2*\w] +.if \ret + ret +.else + add x0, x0, #2*\stride +.endif +3: +.endm + +.macro load_n_incr_16 dst, src, incr, w +.if \w == 4 + ld1 {\dst\().4h}, [\src], \incr +.else + ld1 {\dst\().8h}, [\src], \incr +.endif +.endm + +// void dav1d_cdef_paddingX_16bpc_neon(uint16_t *tmp, const pixel *src, +// ptrdiff_t src_stride, const pixel (*left)[2], +// const pixel *const top, +// const pixel *const bottom, int h, +// enum CdefEdgeFlags edges); + +.macro padding_func_16 w, stride, reg +function cdef_padding\w\()_16bpc_neon, export=1 + movi v30.8h, #0x80, lsl #8 + mov v31.16b, v30.16b + sub x0, x0, #2*(2*\stride+2) + tst w7, #4 // CDEF_HAVE_TOP + b.ne 1f + // !CDEF_HAVE_TOP + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + b 3f +1: + // CDEF_HAVE_TOP + add x9, x4, x2 + pad_top_bot_16 x4, x9, \w, \stride, \reg, 0 + + // Middle section +3: + tst w7, #1 // CDEF_HAVE_LEFT + b.eq 2f + // CDEF_HAVE_LEFT + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ld1 {v0.s}[0], [x3], #4 + ldr s2, [x1, #2*\w] + load_n_incr_16 v1, x1, x2, \w + subs w6, w6, #1 + str s0, [x0] + stur \reg\()1, [x0, #4] + str s2, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + ld1 {v0.s}[0], [x3], #4 + load_n_incr_16 v1, x1, x2, \w + subs w6, w6, #1 + str s0, [x0] + stur \reg\()1, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + b 3f +2: + tst w7, #2 // CDEF_HAVE_RIGHT + b.eq 1f + // !CDEF_HAVE_LEFT+CDEF_HAVE_RIGHT +0: + ldr s1, [x1, #2*\w] + load_n_incr_16 v0, x1, x2, \w + subs w6, w6, #1 + str s31, [x0] + stur \reg\()0, [x0, #4] + str s1, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 0b + b 3f +1: + // !CDEF_HAVE_LEFT+!CDEF_HAVE_RIGHT + load_n_incr_16 v0, x1, x2, \w + subs w6, w6, #1 + str s31, [x0] + stur \reg\()0, [x0, #4] + str s31, [x0, #4+2*\w] + add x0, x0, #2*\stride + b.gt 1b + +3: + tst w7, #8 // CDEF_HAVE_BOTTOM + b.ne 1f + // !CDEF_HAVE_BOTTOM + st1 {v30.8h, v31.8h}, [x0], #32 +.if \w == 8 + st1 {v30.8h, v31.8h}, [x0], #32 +.endif + ret +1: + // CDEF_HAVE_BOTTOM + add x9, x5, x2 + pad_top_bot_16 x5, x9, \w, \stride, \reg, 1 +endfunc +.endm + +padding_func_16 8, 16, q +padding_func_16 4, 8, d + +tables + +filter 8, 16 +filter 4, 16 + +find_dir 16 diff --git a/third_party/dav1d/src/arm/64/cdef_tmpl.S b/third_party/dav1d/src/arm/64/cdef_tmpl.S new file mode 100644 index 0000000000..d35d7a09ba --- /dev/null +++ b/third_party/dav1d/src/arm/64/cdef_tmpl.S @@ -0,0 +1,511 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro dir_table w, stride +const directions\w + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 + .byte 1 * \stride + 0, 2 * \stride + 0 + .byte 1 * \stride + 0, 2 * \stride - 1 +// Repeated, to avoid & 7 + .byte -1 * \stride + 1, -2 * \stride + 2 + .byte 0 * \stride + 1, -1 * \stride + 2 + .byte 0 * \stride + 1, 0 * \stride + 2 + .byte 0 * \stride + 1, 1 * \stride + 2 + .byte 1 * \stride + 1, 2 * \stride + 2 + .byte 1 * \stride + 0, 2 * \stride + 1 +endconst +.endm + +.macro tables +dir_table 8, 16 +dir_table 4, 8 + +const pri_taps + .byte 4, 2, 3, 3 +endconst +.endm + +.macro load_px d1, d2, w +.if \w == 8 + add x6, x2, w9, sxtb #1 // x + off + sub x9, x2, w9, sxtb #1 // x - off + ld1 {\d1\().8h}, [x6] // p0 + ld1 {\d2\().8h}, [x9] // p1 +.else + add x6, x2, w9, sxtb #1 // x + off + sub x9, x2, w9, sxtb #1 // x - off + ld1 {\d1\().4h}, [x6] // p0 + add x6, x6, #2*8 // += stride + ld1 {\d2\().4h}, [x9] // p1 + add x9, x9, #2*8 // += stride + ld1 {\d1\().d}[1], [x6] // p0 + ld1 {\d2\().d}[1], [x9] // p1 +.endif +.endm +.macro handle_pixel s1, s2, thresh_vec, shift, tap, min +.if \min + umin v2.8h, v2.8h, \s1\().8h + smax v3.8h, v3.8h, \s1\().8h + umin v2.8h, v2.8h, \s2\().8h + smax v3.8h, v3.8h, \s2\().8h +.endif + uabd v16.8h, v0.8h, \s1\().8h // abs(diff) + uabd v20.8h, v0.8h, \s2\().8h // abs(diff) + ushl v17.8h, v16.8h, \shift // abs(diff) >> shift + ushl v21.8h, v20.8h, \shift // abs(diff) >> shift + uqsub v17.8h, \thresh_vec, v17.8h // clip = imax(0, threshold - (abs(diff) >> shift)) + uqsub v21.8h, \thresh_vec, v21.8h // clip = imax(0, threshold - (abs(diff) >> shift)) + sub v18.8h, \s1\().8h, v0.8h // diff = p0 - px + sub v22.8h, \s2\().8h, v0.8h // diff = p1 - px + neg v16.8h, v17.8h // -clip + neg v20.8h, v21.8h // -clip + smin v18.8h, v18.8h, v17.8h // imin(diff, clip) + smin v22.8h, v22.8h, v21.8h // imin(diff, clip) + dup v19.8h, \tap // taps[k] + smax v18.8h, v18.8h, v16.8h // constrain() = imax(imin(diff, clip), -clip) + smax v22.8h, v22.8h, v20.8h // constrain() = imax(imin(diff, clip), -clip) + mla v1.8h, v18.8h, v19.8h // sum += taps[k] * constrain() + mla v1.8h, v22.8h, v19.8h // sum += taps[k] * constrain() +.endm + +// void dav1d_cdef_filterX_Ybpc_neon(pixel *dst, ptrdiff_t dst_stride, +// const uint16_t *tmp, int pri_strength, +// int sec_strength, int dir, int damping, +// int h, size_t edges); +.macro filter_func w, bpc, pri, sec, min, suffix +function cdef_filter\w\suffix\()_\bpc\()bpc_neon +.if \bpc == 8 + ldr w8, [sp] // edges + cmp w8, #0xf + b.eq cdef_filter\w\suffix\()_edged_8bpc_neon +.endif +.if \pri +.if \bpc == 16 + ldr w9, [sp, #8] // bitdepth_max + clz w9, w9 + sub w9, w9, #24 // -bitdepth_min_8 + neg w9, w9 // bitdepth_min_8 +.endif + movrel x8, pri_taps +.if \bpc == 16 + lsr w9, w3, w9 // pri_strength >> bitdepth_min_8 + and w9, w9, #1 // (pri_strength >> bitdepth_min_8) & 1 +.else + and w9, w3, #1 +.endif + add x8, x8, w9, uxtw #1 +.endif + movrel x9, directions\w + add x5, x9, w5, uxtw #1 + movi v30.4h, #15 + dup v28.4h, w6 // damping + +.if \pri + dup v25.8h, w3 // threshold +.endif +.if \sec + dup v27.8h, w4 // threshold +.endif + trn1 v24.4h, v25.4h, v27.4h + clz v24.4h, v24.4h // clz(threshold) + sub v24.4h, v30.4h, v24.4h // ulog2(threshold) + uqsub v24.4h, v28.4h, v24.4h // shift = imax(0, damping - ulog2(threshold)) + neg v24.4h, v24.4h // -shift +.if \sec + dup v26.8h, v24.h[1] +.endif +.if \pri + dup v24.8h, v24.h[0] +.endif + +1: +.if \w == 8 + ld1 {v0.8h}, [x2] // px +.else + add x12, x2, #2*8 + ld1 {v0.4h}, [x2] // px + ld1 {v0.d}[1], [x12] // px +.endif + + movi v1.8h, #0 // sum +.if \min + mov v2.16b, v0.16b // min + mov v3.16b, v0.16b // max +.endif + + // Instead of loading sec_taps 2, 1 from memory, just set it + // to 2 initially and decrease for the second round. + // This is also used as loop counter. + mov w11, #2 // sec_taps[0] + +2: +.if \pri + ldrb w9, [x5] // off1 + + load_px v4, v5, \w +.endif + +.if \sec + add x5, x5, #4 // +2*2 + ldrb w9, [x5] // off2 + load_px v6, v7, \w +.endif + +.if \pri + ldrb w10, [x8] // *pri_taps + + handle_pixel v4, v5, v25.8h, v24.8h, w10, \min +.endif + +.if \sec + add x5, x5, #8 // +2*4 + ldrb w9, [x5] // off3 + load_px v4, v5, \w + + handle_pixel v6, v7, v27.8h, v26.8h, w11, \min + + handle_pixel v4, v5, v27.8h, v26.8h, w11, \min + + sub x5, x5, #11 // x5 -= 2*(2+4); x5 += 1; +.else + add x5, x5, #1 // x5 += 1 +.endif + subs w11, w11, #1 // sec_tap-- (value) +.if \pri + add x8, x8, #1 // pri_taps++ (pointer) +.endif + b.ne 2b + + cmlt v4.8h, v1.8h, #0 // -(sum < 0) + add v1.8h, v1.8h, v4.8h // sum - (sum < 0) + srshr v1.8h, v1.8h, #4 // (8 + sum - (sum < 0)) >> 4 + add v0.8h, v0.8h, v1.8h // px + (8 + sum ...) >> 4 +.if \min + smin v0.8h, v0.8h, v3.8h + smax v0.8h, v0.8h, v2.8h // iclip(px + .., min, max) +.endif +.if \bpc == 8 + xtn v0.8b, v0.8h +.endif +.if \w == 8 + add x2, x2, #2*16 // tmp += tmp_stride + subs w7, w7, #1 // h-- +.if \bpc == 8 + st1 {v0.8b}, [x0], x1 +.else + st1 {v0.8h}, [x0], x1 +.endif +.else +.if \bpc == 8 + st1 {v0.s}[0], [x0], x1 +.else + st1 {v0.d}[0], [x0], x1 +.endif + add x2, x2, #2*16 // tmp += 2*tmp_stride + subs w7, w7, #2 // h -= 2 +.if \bpc == 8 + st1 {v0.s}[1], [x0], x1 +.else + st1 {v0.d}[1], [x0], x1 +.endif +.endif + + // Reset pri_taps and directions back to the original point + sub x5, x5, #2 +.if \pri + sub x8, x8, #2 +.endif + + b.gt 1b + ret +endfunc +.endm + +.macro filter w, bpc +filter_func \w, \bpc, pri=1, sec=0, min=0, suffix=_pri +filter_func \w, \bpc, pri=0, sec=1, min=0, suffix=_sec +filter_func \w, \bpc, pri=1, sec=1, min=1, suffix=_pri_sec + +function cdef_filter\w\()_\bpc\()bpc_neon, export=1 + cbnz w3, 1f // pri_strength + b cdef_filter\w\()_sec_\bpc\()bpc_neon // only sec +1: + cbnz w4, 1f // sec_strength + b cdef_filter\w\()_pri_\bpc\()bpc_neon // only pri +1: + b cdef_filter\w\()_pri_sec_\bpc\()bpc_neon // both pri and sec +endfunc +.endm + +const div_table + .short 840, 420, 280, 210, 168, 140, 120, 105 +endconst + +const alt_fact + .short 420, 210, 140, 105, 105, 105, 105, 105, 140, 210, 420, 0 +endconst + +.macro cost_alt d1, d2, s1, s2, s3, s4 + smull v22.4s, \s1\().4h, \s1\().4h // sum_alt[n]*sum_alt[n] + smull2 v23.4s, \s1\().8h, \s1\().8h + smull v24.4s, \s2\().4h, \s2\().4h + smull v25.4s, \s3\().4h, \s3\().4h // sum_alt[n]*sum_alt[n] + smull2 v26.4s, \s3\().8h, \s3\().8h + smull v27.4s, \s4\().4h, \s4\().4h + mul v22.4s, v22.4s, v29.4s // sum_alt[n]^2*fact + mla v22.4s, v23.4s, v30.4s + mla v22.4s, v24.4s, v31.4s + mul v25.4s, v25.4s, v29.4s // sum_alt[n]^2*fact + mla v25.4s, v26.4s, v30.4s + mla v25.4s, v27.4s, v31.4s + addv \d1, v22.4s // *cost_ptr + addv \d2, v25.4s // *cost_ptr +.endm + +.macro find_best s1, s2, s3 +.ifnb \s2 + mov w5, \s2\().s[0] +.endif + cmp w4, w1 // cost[n] > best_cost + csel w0, w3, w0, gt // best_dir = n + csel w1, w4, w1, gt // best_cost = cost[n] +.ifnb \s2 + add w3, w3, #1 // n++ + cmp w5, w1 // cost[n] > best_cost + mov w4, \s3\().s[0] + csel w0, w3, w0, gt // best_dir = n + csel w1, w5, w1, gt // best_cost = cost[n] + add w3, w3, #1 // n++ +.endif +.endm + +// Steps for loading and preparing each row +.macro dir_load_step1 s1, bpc +.if \bpc == 8 + ld1 {\s1\().8b}, [x0], x1 +.else + ld1 {\s1\().8h}, [x0], x1 +.endif +.endm + +.macro dir_load_step2 s1, bpc +.if \bpc == 8 + usubl \s1\().8h, \s1\().8b, v31.8b +.else + ushl \s1\().8h, \s1\().8h, v8.8h +.endif +.endm + +.macro dir_load_step3 s1, bpc +// Nothing for \bpc == 8 +.if \bpc != 8 + sub \s1\().8h, \s1\().8h, v31.8h +.endif +.endm + +// int dav1d_cdef_find_dir_Xbpc_neon(const pixel *img, const ptrdiff_t stride, +// unsigned *const var) +.macro find_dir bpc +function cdef_find_dir_\bpc\()bpc_neon, export=1 +.if \bpc == 16 + str d8, [sp, #-0x10]! + clz w3, w3 // clz(bitdepth_max) + sub w3, w3, #24 // -bitdepth_min_8 + dup v8.8h, w3 +.endif + sub sp, sp, #32 // cost + mov w3, #8 +.if \bpc == 8 + movi v31.16b, #128 +.else + movi v31.8h, #128 +.endif + movi v30.16b, #0 + movi v1.8h, #0 // v0-v1 sum_diag[0] + movi v3.8h, #0 // v2-v3 sum_diag[1] + movi v5.8h, #0 // v4-v5 sum_hv[0-1] + movi v7.8h, #0 // v6-v7 sum_alt[0] + dir_load_step1 v26, \bpc // Setup first row early + movi v17.8h, #0 // v16-v17 sum_alt[1] + movi v18.8h, #0 // v18-v19 sum_alt[2] + dir_load_step2 v26, \bpc + movi v19.8h, #0 + dir_load_step3 v26, \bpc + movi v21.8h, #0 // v20-v21 sum_alt[3] + +.irpc i, 01234567 + addv h25, v26.8h // [y] + rev64 v27.8h, v26.8h + addp v28.8h, v26.8h, v30.8h // [(x >> 1)] + add v5.8h, v5.8h, v26.8h // sum_hv[1] + ext v27.16b, v27.16b, v27.16b, #8 // [-x] + rev64 v29.4h, v28.4h // [-(x >> 1)] + ins v4.h[\i], v25.h[0] // sum_hv[0] +.if \i < 6 + ext v22.16b, v30.16b, v26.16b, #(16-2*(3-(\i/2))) + ext v23.16b, v26.16b, v30.16b, #(16-2*(3-(\i/2))) + add v18.8h, v18.8h, v22.8h // sum_alt[2] + add v19.4h, v19.4h, v23.4h // sum_alt[2] +.else + add v18.8h, v18.8h, v26.8h // sum_alt[2] +.endif +.if \i == 0 + mov v20.16b, v26.16b // sum_alt[3] +.elseif \i == 1 + add v20.8h, v20.8h, v26.8h // sum_alt[3] +.else + ext v24.16b, v30.16b, v26.16b, #(16-2*(\i/2)) + ext v25.16b, v26.16b, v30.16b, #(16-2*(\i/2)) + add v20.8h, v20.8h, v24.8h // sum_alt[3] + add v21.4h, v21.4h, v25.4h // sum_alt[3] +.endif +.if \i == 0 + mov v0.16b, v26.16b // sum_diag[0] + dir_load_step1 v26, \bpc + mov v2.16b, v27.16b // sum_diag[1] + dir_load_step2 v26, \bpc + mov v6.16b, v28.16b // sum_alt[0] + dir_load_step3 v26, \bpc + mov v16.16b, v29.16b // sum_alt[1] +.else + ext v22.16b, v30.16b, v26.16b, #(16-2*\i) + ext v23.16b, v26.16b, v30.16b, #(16-2*\i) + ext v24.16b, v30.16b, v27.16b, #(16-2*\i) + ext v25.16b, v27.16b, v30.16b, #(16-2*\i) +.if \i != 7 // Nothing to load for the final row + dir_load_step1 v26, \bpc // Start setting up the next row early. +.endif + add v0.8h, v0.8h, v22.8h // sum_diag[0] + add v1.8h, v1.8h, v23.8h // sum_diag[0] + add v2.8h, v2.8h, v24.8h // sum_diag[1] + add v3.8h, v3.8h, v25.8h // sum_diag[1] +.if \i != 7 + dir_load_step2 v26, \bpc +.endif + ext v22.16b, v30.16b, v28.16b, #(16-2*\i) + ext v23.16b, v28.16b, v30.16b, #(16-2*\i) + ext v24.16b, v30.16b, v29.16b, #(16-2*\i) + ext v25.16b, v29.16b, v30.16b, #(16-2*\i) +.if \i != 7 + dir_load_step3 v26, \bpc +.endif + add v6.8h, v6.8h, v22.8h // sum_alt[0] + add v7.4h, v7.4h, v23.4h // sum_alt[0] + add v16.8h, v16.8h, v24.8h // sum_alt[1] + add v17.4h, v17.4h, v25.4h // sum_alt[1] +.endif +.endr + + movi v31.4s, #105 + + smull v26.4s, v4.4h, v4.4h // sum_hv[0]*sum_hv[0] + smlal2 v26.4s, v4.8h, v4.8h + smull v27.4s, v5.4h, v5.4h // sum_hv[1]*sum_hv[1] + smlal2 v27.4s, v5.8h, v5.8h + mul v26.4s, v26.4s, v31.4s // cost[2] *= 105 + mul v27.4s, v27.4s, v31.4s // cost[6] *= 105 + addv s4, v26.4s // cost[2] + addv s5, v27.4s // cost[6] + + rev64 v1.8h, v1.8h + rev64 v3.8h, v3.8h + ext v1.16b, v1.16b, v1.16b, #10 // sum_diag[0][14-n] + ext v3.16b, v3.16b, v3.16b, #10 // sum_diag[1][14-n] + + str s4, [sp, #2*4] // cost[2] + str s5, [sp, #6*4] // cost[6] + + movrel x4, div_table + ld1 {v31.8h}, [x4] + + smull v22.4s, v0.4h, v0.4h // sum_diag[0]*sum_diag[0] + smull2 v23.4s, v0.8h, v0.8h + smlal v22.4s, v1.4h, v1.4h + smlal2 v23.4s, v1.8h, v1.8h + smull v24.4s, v2.4h, v2.4h // sum_diag[1]*sum_diag[1] + smull2 v25.4s, v2.8h, v2.8h + smlal v24.4s, v3.4h, v3.4h + smlal2 v25.4s, v3.8h, v3.8h + uxtl v30.4s, v31.4h // div_table + uxtl2 v31.4s, v31.8h + mul v22.4s, v22.4s, v30.4s // cost[0] + mla v22.4s, v23.4s, v31.4s // cost[0] + mul v24.4s, v24.4s, v30.4s // cost[4] + mla v24.4s, v25.4s, v31.4s // cost[4] + addv s0, v22.4s // cost[0] + addv s2, v24.4s // cost[4] + + movrel x5, alt_fact + ld1 {v29.4h, v30.4h, v31.4h}, [x5]// div_table[2*m+1] + 105 + + str s0, [sp, #0*4] // cost[0] + str s2, [sp, #4*4] // cost[4] + + uxtl v29.4s, v29.4h // div_table[2*m+1] + 105 + uxtl v30.4s, v30.4h + uxtl v31.4s, v31.4h + + cost_alt s6, s16, v6, v7, v16, v17 // cost[1], cost[3] + cost_alt s18, s20, v18, v19, v20, v21 // cost[5], cost[7] + str s6, [sp, #1*4] // cost[1] + str s16, [sp, #3*4] // cost[3] + + mov w0, #0 // best_dir + mov w1, v0.s[0] // best_cost + mov w3, #1 // n + + str s18, [sp, #5*4] // cost[5] + str s20, [sp, #7*4] // cost[7] + + mov w4, v6.s[0] + + find_best v6, v4, v16 + find_best v16, v2, v18 + find_best v18, v5, v20 + find_best v20 + + eor w3, w0, #4 // best_dir ^4 + ldr w4, [sp, w3, uxtw #2] + sub w1, w1, w4 // best_cost - cost[best_dir ^ 4] + lsr w1, w1, #10 + str w1, [x2] // *var + + add sp, sp, #32 +.if \bpc == 16 + ldr d8, [sp], 0x10 +.endif + ret +endfunc +.endm diff --git a/third_party/dav1d/src/arm/64/filmgrain.S b/third_party/dav1d/src/arm/64/filmgrain.S new file mode 100644 index 0000000000..6cdd7ec5fa --- /dev/null +++ b/third_party/dav1d/src/arm/64/filmgrain.S @@ -0,0 +1,2010 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "src/arm/asm-offsets.h" + +#define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr w11, w2, #3 + lsr w12, w2, #12 + lsr w13, w2, #1 + eor w11, w2, w11 // (r >> 0) ^ (r >> 3) + eor w12, w12, w13 // (r >> 12) ^ (r >> 1) + eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr w2, w2, #\steps +.endif + and w11, w11, #((1 << \steps) - 1) // bit +.if \shift + orr w2, w2, w11, lsl #(16 - \steps) // *state +.else + orr w2, w2, w11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, x2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, x2, #17 - \bits, #\bits + lsr w2, w2, #1 +.endm + +// special calling convention: +// w2 holds seed +// x3 holds dav1d_gaussian_sequence +// clobbers x11-x15 +// returns in v0.8h +function get_gaussian_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + increment_seed 4 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + read_rand x14, 11, 3 + ld1 {v0.h}[3], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 2 + ld1 {v0.h}[4], [x14] + add x15, x3, x15, lsl #1 + read_rand x14, 11, 1 + ld1 {v0.h}[5], [x15] + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[6], [x14] + ld1 {v0.h}[7], [x15] + ret +endfunc + +.macro get_grain_row r0, r1, r2, r3, r4, r5 + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r0\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r0\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r1\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r1\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r2\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r2\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r3\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r3\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r4\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r4\().16b, \r5\().8h + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {\r5\().h}[0], [x14] + ld1 {\r5\().h}[1], [x15] + srshl v0.4h, \r5\().4h, v31.4h + xtn \r5\().8b, v0.8h +.endm + +.macro store_grain_row r0, r1, r2, r3, r4, r5 + st1 {\r0\().16b,\r1\().16b}, [x0], #32 + st1 {\r2\().16b,\r3\().16b}, [x0], #32 + st1 {\r4\().16b}, [x0], #16 + st1 {\r5\().h}[0], [x0], #2 +.endm + +.macro get_grain_row_44 r0, r1, r2 + bl get_gaussian_neon + srshl \r2\().8h, v0.8h, v31.8h + xtn \r0\().8b, \r2\().8h + bl get_gaussian_neon + srshl \r2\().8h, v0.8h, v31.8h + xtn2 \r0\().16b, \r2\().8h + bl get_gaussian_neon + srshl \r2\().8h, v0.8h, v31.8h + xtn \r1\().8b, \r2\().8h + bl get_gaussian_neon + srshl \r2\().8h, v0.8h, v31.8h + xtn2 \r1\().16b, \r2\().8h + bl get_gaussian_neon + srshl \r2\().8h, v0.8h, v31.8h + xtn \r2\().8b, \r2\().8h + + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + ld1 {v0.h}[3], [x15] + srshl v0.4h, v0.4h, v31.4h + xtn2 \r2\().16b, v0.8h +.endm + +.macro store_grain_row_44 r0, r1, r2 + st1 {\r0\().16b,\r1\().16b}, [x0], #32 + st1 {\r2\().16b}, [x0] + add x0, x0, #GRAIN_WIDTH-32 +.endm + +function get_grain_2_neon + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + ld1 {v0.h}[1], [x15] + srshl v0.4h, v0.4h, v31.4h + xtn v0.8b, v0.8h + ret +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif +.endm + +// w15 holds the number of entries to produce +// w14, w16 and w17 hold the previous output entries +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #1 +.if \n == 1 + madd w11, w14, w4, w11 // sum (above) + *coeff * prev output +.elseif \n == 2 + madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w14, w17, w11 // += *coeff * prev output 2 + mov w16, w14 +.else + madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 + madd w11, w14, w21, w11 // += *coeff * prev output 3 + mov w17, w16 + mov w16, w14 +.endif + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + add w12, w12, w10 // 1 << (4 + grain_scale_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + asr w12, w12, w9 // >> (4 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.b[15], w14 + b.gt 1b + ret +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + smull v2.8h, v3.8b, v28.8b + smull2 v3.8h, v3.16b, v28.16b + smull v4.8h, v0.8b, v27.8b + smull2 v5.8h, v0.16b, v27.16b + smull v6.8h, v1.8b, v29.8b + smull2 v7.8h, v1.16b, v29.16b + saddl v0.4s, v2.4h, v4.4h + saddl2 v1.4s, v2.8h, v4.8h + saddl v2.4s, v3.4h, v5.4h + saddl2 v3.4s, v3.8h, v5.8h + saddw v4.4s, v0.4s, v6.4h + saddw2 v5.4s, v1.4s, v6.8h + saddw v6.4s, v2.4s, v7.4h + saddw2 v7.4s, v3.4s, v7.8h + ret +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff + bl sum_\lag\()_above_neon +.ifc \type, uv_420 + add x12, x19, #GRAIN_WIDTH + ld1 {v22.16b, v23.16b}, [x19], #32 + ld1 {v24.16b, v25.16b}, [x12] + saddlp v22.8h, v22.16b + saddlp v23.8h, v23.16b + saddlp v24.8h, v24.16b + saddlp v25.8h, v25.16b + add v22.8h, v22.8h, v24.8h + add v23.8h, v23.8h, v25.8h + rshrn v0.8b, v22.8h, #2 + rshrn2 v0.16b, v23.8h, #2 +.endif +.ifc \type, uv_422 + ld1 {v22.16b, v23.16b}, [x19], #32 + saddlp v22.8h, v22.16b + saddlp v23.8h, v23.16b + rshrn v0.8b, v22.8h, #1 + rshrn2 v0.16b, v23.8h, #1 +.endif +.ifc \type, uv_444 + ld1 {v0.16b}, [x19], #16 +.endif +.if \uv_layout +.ifnb \uv_coeff + dup v1.16b, \uv_coeff + smull v2.8h, v0.8b, v1.8b + smull2 v3.8h, v0.16b, v1.16b +.else + smull v2.8h, v0.8b, v30.8b + smull2 v3.8h, v0.16b, v30.16b +.endif + saddw v4.4s, v4.4s, v2.4h + saddw2 v5.4s, v5.4s, v2.8h + saddw v6.4s, v6.4s, v3.4h + saddw2 v7.4s, v7.4s, v3.8h +.endif +.if \uv_layout && \elems == 16 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 15 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 9 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + xtn2 v0.16b, v0.8h + ext v4.16b, v4.16b, v4.16b, #12 +.ifc \lag, lag3 + smov w17, v0.b[13] +.endif +.ifnc \lag, lag1 + smov w16, v0.b[14] +.endif + smov w14, v0.b[15] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b + mov w15, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + mov v1.16b, v6.16b +.if \elems == 9 + mov w15, #1 + bl output_\lag\()_neon + lsr w2, w2, #3 + + read_rand x12, 11, 2 + read_rand x13, 11, 1 + read_rand x14, 11, 0 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v1.h}[0], [x12] + ld1 {v1.h}[1], [x13] + ld1 {v1.h}[2], [x14] + srshl v1.4h, v1.4h, v31.4h + xtn v1.8b, v1.8h + ext v0.16b, v0.16b, v1.16b, #7 +.else + mov w15, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + mov v1.16b, v7.16b + +.ifc \edge, right + mov w15, #3 + bl output_\lag\()_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #1 +.else + mov w15, #4 + bl output_\lag\()_neon +.endif +.endif +.if \store + st1 {v0.16b}, [x0], #16 +.endif + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag1_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 15 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 15 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 9 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 9 + +.macro sum_lag1 type, dst, left, mid, right, edge=mid + mov v3.16b, \mid\().16b + ext v0.16b, \left\().16b, \mid\().16b, #15 + ext v1.16b, \mid\().16b, \right\().16b, #1 + bl sum_\type\()_lag1_\edge\()_neon + mov \dst\().16b, v0.16b +.endm + +.macro sum_y_lag1 dst, left, mid, right, edge=mid + sum_lag1 y, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_444, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_422, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_420, \dst, \left, \mid, \right, \edge +.endm + + +function sum_lag2_above_neon + sub x12, x0, #2*GRAIN_WIDTH - 16 + sub x13, x0, #1*GRAIN_WIDTH - 16 + ld1 {v18.16b}, [x12] // load top right + ld1 {v21.16b}, [x13] + + ext v22.16b, v16.16b, v17.16b, #14 // top left, top mid + dup v26.16b, v30.b[0] + ext v23.16b, v16.16b, v17.16b, #15 + dup v27.16b, v30.b[1] + ext v0.16b, v17.16b, v18.16b, #1 // top mid, top right + dup v28.16b, v30.b[3] + ext v1.16b, v17.16b, v18.16b, #2 + dup v29.16b, v30.b[4] + + smull v2.8h, v22.8b, v26.8b + smull2 v3.8h, v22.16b, v26.16b + smull v4.8h, v23.8b, v27.8b + smull2 v5.8h, v23.16b, v27.16b + smull v6.8h, v0.8b, v28.8b + smull2 v7.8h, v0.16b, v28.16b + smull v0.8h, v1.8b, v29.8b + smull2 v1.8h, v1.16b, v29.16b + saddl v22.4s, v2.4h, v4.4h + saddl2 v23.4s, v2.8h, v4.8h + saddl v26.4s, v3.4h, v5.4h + saddl2 v27.4s, v3.8h, v5.8h + saddl v2.4s, v0.4h, v6.4h + saddl2 v3.4s, v0.8h, v6.8h + saddl v6.4s, v1.4h, v7.4h + saddl2 v7.4s, v1.8h, v7.8h + add v4.4s, v22.4s, v2.4s + add v5.4s, v23.4s, v3.4s + add v6.4s, v26.4s, v6.4s + add v7.4s, v27.4s, v7.4s + + ext v22.16b, v19.16b, v20.16b, #14 // top left, top mid + dup v26.16b, v30.b[5] + ext v23.16b, v19.16b, v20.16b, #15 + dup v27.16b, v30.b[6] + ext v0.16b, v20.16b, v21.16b, #1 // top mid, top right + dup v28.16b, v30.b[8] + ext v1.16b, v20.16b, v21.16b, #2 + dup v29.16b, v30.b[9] + + smull v2.8h, v22.8b, v26.8b + smull2 v3.8h, v22.16b, v26.16b + smull v22.8h, v23.8b, v27.8b + smull2 v23.8h, v23.16b, v27.16b + smull v26.8h, v0.8b, v28.8b + smull2 v27.8h, v0.16b, v28.16b + smull v28.8h, v1.8b, v29.8b + smull2 v29.8h, v1.16b, v29.16b + saddl v0.4s, v2.4h, v22.4h + saddl2 v1.4s, v2.8h, v22.8h + saddl v2.4s, v3.4h, v23.4h + saddl2 v3.4s, v3.8h, v23.8h + saddl v22.4s, v26.4h, v28.4h + saddl2 v23.4s, v26.8h, v28.8h + saddl v26.4s, v27.4h, v29.4h + saddl2 v27.4s, v27.8h, v29.8h + add v0.4s, v0.4s, v22.4s + add v1.4s, v1.4s, v23.4s + add v2.4s, v2.4s, v26.4s + add v3.4s, v3.4s, v27.4s + dup v26.16b, v30.b[2] + dup v27.16b, v30.b[7] + smull v22.8h, v17.8b, v26.8b + smull2 v23.8h, v17.16b, v26.16b + smull v24.8h, v20.8b, v27.8b + smull2 v25.8h, v20.16b, v27.16b + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + saddl v0.4s, v22.4h, v24.4h + saddl2 v1.4s, v22.8h, v24.8h + saddl v2.4s, v23.4h, v25.4h + saddl2 v3.4s, v23.8h, v25.8h + mov v19.16b, v20.16b + mov v20.16b, v21.16b + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + ret +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag2_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #2*GRAIN_WIDTH + sub x13, x0, #1*GRAIN_WIDTH + ld1 {v17.16b}, [x12] // load the previous block right above + ld1 {v20.16b}, [x13] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[12] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 15 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 15 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 9 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 9 + + +function sum_lag3_above_neon + sub x11, x0, #3*GRAIN_WIDTH - 16 + sub x12, x0, #2*GRAIN_WIDTH - 16 + sub x13, x0, #1*GRAIN_WIDTH - 16 + ld1 {v15.16b}, [x11] // load top right + ld1 {v18.16b}, [x12] + ld1 {v21.16b}, [x13] + + ext v8.16b, v13.16b, v14.16b, #13 // top left, top mid + dup v22.16b, v29.b[0] + ext v9.16b, v13.16b, v14.16b, #14 + dup v23.16b, v29.b[1] + ext v10.16b, v13.16b, v14.16b, #15 + dup v24.16b, v29.b[2] + dup v25.16b, v29.b[3] + ext v11.16b, v14.16b, v15.16b, #1 // top mid, top right + dup v26.16b, v29.b[4] + ext v12.16b, v14.16b, v15.16b, #2 + dup v27.16b, v29.b[5] + ext v13.16b, v14.16b, v15.16b, #3 + dup v28.16b, v29.b[6] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + saddl v22.4s, v0.4h, v2.4h + saddl2 v23.4s, v0.8h, v2.8h + saddl v24.4s, v1.4h, v3.4h + saddl2 v26.4s, v1.8h, v3.8h + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + smull v8.8h, v12.8b, v27.8b + smull2 v9.8h, v12.16b, v27.16b + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + smull v12.8h, v14.8b, v25.8b + smull2 v13.8h, v14.16b, v25.16b + add v4.4s, v22.4s, v0.4s + add v5.4s, v23.4s, v1.4s + add v6.4s, v24.4s, v2.4s + add v7.4s, v26.4s, v3.4s + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + saddw v4.4s, v4.4s, v12.4h + saddw2 v5.4s, v5.4s, v12.8h + saddw v6.4s, v6.4s, v13.4h + saddw2 v7.4s, v7.4s, v13.8h + + ext v8.16b, v16.16b, v17.16b, #13 // top left, top mid + dup v22.16b, v29.b[7] + ext v9.16b, v16.16b, v17.16b, #14 + dup v23.16b, v29.b[8] + ext v10.16b, v16.16b, v17.16b, #15 + dup v24.16b, v29.b[9] + dup v25.16b, v29.b[10] + ext v11.16b, v17.16b, v18.16b, #1 // top mid, top right + dup v26.16b, v29.b[11] + ext v12.16b, v17.16b, v18.16b, #2 + dup v27.16b, v29.b[12] + ext v13.16b, v17.16b, v18.16b, #3 + dup v28.16b, v29.b[13] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + saddl v22.4s, v0.4h, v2.4h + saddl2 v23.4s, v0.8h, v2.8h + saddl v24.4s, v1.4h, v3.4h + saddl2 v26.4s, v1.8h, v3.8h + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + smull v8.8h, v12.8b, v27.8b + smull2 v9.8h, v12.16b, v27.16b + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + smull v12.8h, v17.8b, v25.8b + smull2 v13.8h, v17.16b, v25.16b + add v22.4s, v22.4s, v0.4s + add v23.4s, v23.4s, v1.4s + add v24.4s, v24.4s, v2.4s + add v26.4s, v26.4s, v3.4s + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + add v4.4s, v4.4s, v22.4s + add v5.4s, v5.4s, v23.4s + add v6.4s, v6.4s, v24.4s + add v7.4s, v7.4s, v26.4s + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + saddw v4.4s, v4.4s, v12.4h + saddw2 v5.4s, v5.4s, v12.8h + saddw v6.4s, v6.4s, v13.4h + saddw2 v7.4s, v7.4s, v13.8h + + ext v8.16b, v19.16b, v20.16b, #13 // top left, top mid + dup v22.16b, v29.b[14] + ext v9.16b, v19.16b, v20.16b, #14 + dup v23.16b, v29.b[15] + ext v10.16b, v19.16b, v20.16b, #15 + dup v24.16b, v30.b[0] + dup v25.16b, v30.b[1] + ext v11.16b, v20.16b, v21.16b, #1 // top mid, top right + dup v26.16b, v30.b[2] + ext v12.16b, v20.16b, v21.16b, #2 + dup v27.16b, v30.b[3] + ext v13.16b, v20.16b, v21.16b, #3 + dup v28.16b, v30.b[4] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + saddl v22.4s, v0.4h, v2.4h + saddl2 v23.4s, v0.8h, v2.8h + saddl v24.4s, v1.4h, v3.4h + saddl2 v26.4s, v1.8h, v3.8h + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + smull v8.8h, v12.8b, v27.8b + smull2 v9.8h, v12.16b, v27.16b + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + smull v12.8h, v20.8b, v25.8b + smull2 v19.8h, v20.16b, v25.16b + add v22.4s, v22.4s, v0.4s + add v23.4s, v23.4s, v1.4s + add v24.4s, v24.4s, v2.4s + add v26.4s, v26.4s, v3.4s + saddl v0.4s, v8.4h, v10.4h + saddl2 v1.4s, v8.8h, v10.8h + saddl v2.4s, v9.4h, v11.4h + saddl2 v3.4s, v9.8h, v11.8h + add v4.4s, v4.4s, v22.4s + add v5.4s, v5.4s, v23.4s + add v6.4s, v6.4s, v24.4s + add v7.4s, v7.4s, v26.4s + mov v13.16b, v14.16b + mov v14.16b, v15.16b + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + mov v16.16b, v17.16b + mov v17.16b, v18.16b + saddw v4.4s, v4.4s, v12.4h + saddw2 v5.4s, v5.4s, v12.8h + saddw v6.4s, v6.4s, v19.4h + saddw2 v7.4s, v7.4s, v19.8h + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag3_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +.ifc \edge, left + sub x11, x0, #3*GRAIN_WIDTH + sub x12, x0, #2*GRAIN_WIDTH + sub x13, x0, #1*GRAIN_WIDTH + ld1 {v14.16b}, [x11] // load the previous block right above + ld1 {v17.16b}, [x12] + ld1 {v20.16b}, [x13] +.endif + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[8] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 15 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 15 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 9 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 9 + +function generate_grain_rows_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +1: + get_grain_row v16, v17, v18, v19, v20, v21 + subs w1, w1, #1 + store_grain_row v16, v17, v18, v19, v20, v21 + b.gt 1b + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function generate_grain_rows_44_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +1: + get_grain_row_44 v16, v17, v18 + subs w1, w1, #1 + store_grain_row_44 v16, v17, v18 + b.gt 1b + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function get_grain_row_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + get_grain_row v16, v17, v18, v19, v20, v21 + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function get_grain_row_44_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + get_grain_row_44 v16, v17, v18 + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function add_uv_444_coeff_lag0_neon +add_coeff_lag0_start: + smull v2.8h, v0.8b, v27.8b + smull2 v3.8h, v0.16b, v27.16b + srshl v2.8h, v2.8h, v28.8h + srshl v3.8h, v3.8h, v28.8h + saddw v2.8h, v2.8h, v1.8b + saddw2 v3.8h, v3.8h, v1.16b + sqxtn v2.8b, v2.8h + sqxtn2 v2.16b, v3.8h + ret +endfunc + +function add_uv_420_coeff_lag0_neon + ld1 {v4.16b, v5.16b}, [x19], #32 + ld1 {v6.16b, v7.16b}, [x12], #32 + saddlp v4.8h, v4.16b + saddlp v5.8h, v5.16b + saddlp v6.8h, v6.16b + saddlp v7.8h, v7.16b + add v4.8h, v4.8h, v6.8h + add v5.8h, v5.8h, v7.8h + rshrn v4.8b, v4.8h, #2 + rshrn2 v4.16b, v5.8h, #2 + and v0.16b, v4.16b, v0.16b + b add_coeff_lag0_start +endfunc + +function add_uv_422_coeff_lag0_neon + ld1 {v4.16b, v5.16b}, [x19], #32 + saddlp v4.8h, v4.16b + saddlp v5.8h, v5.16b + rshrn v4.8b, v4.8h, #1 + rshrn2 v4.16b, v5.8h, #1 + and v0.16b, v4.16b, v0.16b + b add_coeff_lag0_start +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x30, x19, [sp, #-96]! + +.ifc \type, uv_444 + mov w13, w3 + mov w14, #28 + add x19, x1, #3*GRAIN_WIDTH + mov x1, x2 + mul w13, w13, w14 +.endif + movrel x3, X(gaussian_sequence) + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add x4, x1, #FGD_AR_COEFFS_Y +.else + add x4, x1, #FGD_AR_COEFFS_UV +.endif + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + +.ifc \type, uv_444 + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne +.endif + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #127 + mov w6, #-128 + +.ifc \type, uv_444 + eor w2, w2, w11 +.endif + + br x16 + +L(generate_grain_\type\()_lag0): + AARCH64_VALID_JUMP_TARGET +.ifc \type, y + mov w1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + dup v28.8h, w7 + ld1r {v27.16b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + ext v29.16b, v0.16b, v1.16b, #13 + ext v30.16b, v1.16b, v0.16b, #1 + neg v28.8h, v28.8h + + mov w1, #3 + bl generate_grain_rows_neon + mov w1, #GRAIN_HEIGHT-3 +1: + ld1 {v22.16b, v23.16b, v24.16b, v25.16b}, [x19], #64 + bl get_grain_row_neon + and v0.16b, v22.16b, v29.16b + mov v1.16b, v16.16b + bl add_uv_444_coeff_lag0_neon + mov v0.16b, v23.16b + mov v1.16b, v17.16b + mov v16.16b, v2.16b + bl add_uv_444_coeff_lag0_neon + ld1 {v26.16b}, [x19], #16 + mov v0.16b, v24.16b + mov v1.16b, v18.16b + mov v17.16b, v2.16b + bl add_uv_444_coeff_lag0_neon + add x19, x19, #2 + mov v0.16b, v25.16b + mov v1.16b, v19.16b + mov v18.16b, v2.16b + bl add_uv_444_coeff_lag0_neon + and v0.16b, v26.16b, v30.16b + mov v1.16b, v20.16b + mov v19.16b, v2.16b + bl add_uv_444_coeff_lag0_neon + mov v20.16b, v2.16b + subs w1, w1, #1 + store_grain_row v16, v17, v18, v19, v20, v21 + b.gt 1b +.endif + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag1): + AARCH64_VALID_JUMP_TARGET + ld1r {v27.16b}, [x4], #1 // ar_coeffs_y[0] + ld1r {v28.16b}, [x4], #1 // ar_coeffs_y[1] + ld1r {v29.16b}, [x4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb w4, [x4, #1] // ar_coeffs_y[3] +.else + add x4, x4, #2 +.endif + + mov w1, #3 +.ifc \type, uv_444 + ld1r {v30.16b}, [x4] // ar_coeffs_uv[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + sum_\type\()_lag1 v22, v16, v16, v17, left + sum_\type\()_lag1 v23, v16, v17, v18 + sum_\type\()_lag1 v24, v17, v18, v19 + sum_\type\()_lag1 v25, v18, v19, v20 + sum_\type\()_lag1 v20, v19, v20, v21, right + get_grain_2 v21 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #2 +.endif + store_grain_row v22, v23, v24, v25, v20, v21 + mov v16.16b, v22.16b + mov v17.16b, v23.16b + mov v18.16b, v24.16b + mov v19.16b, v25.16b + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag2): + AARCH64_VALID_JUMP_TARGET + ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #2 +.endif + st1 {v16.h}[0], [x0], #2 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag3): + AARCH64_VALID_JUMP_TARGET + ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #2 +.endif + st1 {v16.h}[0], [x0], #2 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH-(3*32) +.else + sub \reg, \reg, #3*32-GRAIN_WIDTH +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x30, x19, [sp, #-96]! + + mov w13, w3 + mov w14, #28 + add x19, x1, #3*GRAIN_WIDTH-3 + mov x1, x2 + mul w13, w13, w14 + + movrel x3, X(gaussian_sequence) + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] + add x4, x1, #FGD_AR_COEFFS_UV + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #127 + mov w6, #-128 + + eor w2, w2, w11 + + br x16 + +L(generate_grain_\type\()_lag0): + AARCH64_VALID_JUMP_TARGET + dup v28.8h, w7 + ld1r {v27.16b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + ext v29.16b, v0.16b, v1.16b, #13 + ext v30.16b, v1.16b, v0.16b, #7 + neg v28.8h, v28.8h + + mov w1, #3 + bl generate_grain_rows_44_neon + set_height w1, \type +1: + bl get_grain_row_44_neon +.ifc \type, uv_420 + add x12, x19, #GRAIN_WIDTH +.endif + mov v0.16b, v29.16b + mov v1.16b, v16.16b + bl add_\type\()_coeff_lag0_neon + movi v0.16b, #255 + mov v1.16b, v17.16b + mov v16.16b, v2.16b + bl add_\type\()_coeff_lag0_neon + mov v0.16b, v30.16b + mov v1.16b, v18.16b + mov v17.16b, v2.16b + bl add_\type\()_coeff_lag0_neon + mov v18.16b, v2.16b + subs w1, w1, #1 + increment_y_ptr x19, \type + store_grain_row_44 v16, v17, v18 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag1): + AARCH64_VALID_JUMP_TARGET + ld1r {v27.16b}, [x4], #1 // ar_coeffs_uv[0] + ld1r {v28.16b}, [x4], #1 // ar_coeffs_uv[1] + ld1r {v29.16b}, [x4] // ar_coeffs_uv[2] + add x4, x4, #2 + + mov w1, #3 + ld1r {v30.16b}, [x4] // ar_coeffs_u4[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + sum_\type\()_lag1 v20, v16, v16, v17, left + sum_\type\()_lag1 v21, v16, v17, v18 + sum_\type\()_lag1 v18, v17, v18, v18, right + subs w1, w1, #1 + increment_y_ptr x19, \type + store_grain_row_44 v20, v21, v18 + mov v16.16b, v20.16b + mov v17.16b, v21.16b + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag2): + AARCH64_VALID_JUMP_TARGET + ld1 {v30.16b}, [x4] // ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH-48 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag3): + AARCH64_VALID_JUMP_TARGET + ldr q29, [x4] // ar_coeffs_uv[0-15] + ldr q30, [x4, #16] // ar_coeffs_uv[16-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH-48 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 + +.macro gather_interleaved dst1, dst2, src1, src2, off + umov w14, \src1[0+\off] + umov w15, \src2[8+\off] + umov w16, \src1[2+\off] + add x14, x14, x3 + umov w17, \src2[10+\off] + add x15, x15, x3 + ld1 {\dst1}[0+\off], [x14] + umov w14, \src1[4+\off] + add x16, x16, x3 + ld1 {\dst2}[8+\off], [x15] + umov w15, \src2[12+\off] + add x17, x17, x3 + ld1 {\dst1}[2+\off], [x16] + umov w16, \src1[6+\off] + add x14, x14, x3 + ld1 {\dst2}[10+\off], [x17] + umov w17, \src2[14+\off] + add x15, x15, x3 + ld1 {\dst1}[4+\off], [x14] + add x16, x16, x3 + ld1 {\dst2}[12+\off], [x15] + add x17, x17, x3 + ld1 {\dst1}[6+\off], [x16] + ld1 {\dst2}[14+\off], [x17] +.endm + +.macro gather dst1, dst2, src1, src2 + gather_interleaved \dst1, \dst2, \src1, \src2, 0 + gather_interleaved \dst2, \dst1, \src2, \src1, 0 + gather_interleaved \dst1, \dst2, \src1, \src2, 1 + gather_interleaved \dst2, \dst1, \src2, \src1, 1 +.endm + +function gather32_neon + gather v4.b, v5.b, v0.b, v1.b + ret +endfunc + +function gather16_neon + gather_interleaved v4.b, v5.b, v0.b, v0.b, 0 + gather_interleaved v4.b, v5.b, v0.b, v0.b, 1 + ins v4.d[1], v5.d[1] + ret +endfunc + +const overlap_coeffs_0, align=4 + .byte 27, 17, 0, 0, 0, 0, 0, 0 + .byte 17, 27, 32, 32, 32, 32, 32, 32 +endconst + +const overlap_coeffs_1, align=4 + .byte 23, 0, 0, 0, 0, 0, 0, 0 + .byte 22, 32, 32, 32, 32, 32, 32, 32 +endconst + +.macro calc_offset offx, offy, src, sx, sy + and \offy, \src, #0xF // randval & 0xF + lsr \offx, \src, #4 // randval >> 4 +.if \sy == 0 + add \offy, \offy, \offy // 2 * (randval & 0xF) +.endif +.if \sx == 0 + add \offx, \offx, \offx // 2 * (randval >> 4) +.endif +.endm + +.macro add_offset dst, offx, offy, src, stride + madd \dst, \stride, \offy, \src // grain_lut += grain_stride * offy + add \dst, \dst, \offx, uxtw // grain_lut += offx +.endm + +// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const int scaling_shift, +// const entry grain_lut[][GRAIN_WIDTH], +// const int offsets[][2], +// const int h, const ptrdiff_t clip, +// const ptrdiff_t type); +function fgy_32x32_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + ldr w11, [x6, #8] // offsets[1][0] + ldr w13, [x6, #4] // offsets[0][1] + ldr w15, [x6, #12] // offsets[1][1] + ldr w6, [x6] // offsets[0][0] + ldr w8, [sp, #16] // clip + mov x9, #GRAIN_WIDTH // grain_lut stride + + neg w4, w4 + dup v29.8h, w4 // -scaling_shift + + movrel x16, overlap_coeffs_0 + + cbz w8, 1f + // clip + movi v30.16b, #16 + movi v31.16b, #235 + b 2f +1: + // no clip + movi v30.16b, #0 + movi v31.16b, #255 +2: + + ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs + + add x5, x5, #9 // grain_lut += 9 + add x5, x5, x9, lsl #3 // grain_lut += 8 * grain_stride + add x5, x5, x9 // grain_lut += grain_stride + + calc_offset w11, w12, w11, 0, 0 + calc_offset w13, w14, w13, 0, 0 + calc_offset w15, w16, w15, 0, 0 + calc_offset w6, w10, w6, 0, 0 + + add_offset x12, w11, x12, x5, x9 + add_offset x14, w13, x14, x5, x9 + add_offset x16, w15, x16, x5, x9 + add_offset x5, w6, x10, x5, x9 + + ldr w11, [sp, #24] // type + adr x13, L(fgy_loop_tbl) + + add x4, x12, #32 // grain_lut += BLOCK_SIZE * bx + add x6, x14, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + + tst w11, #1 + ldrh w11, [x13, w11, uxtw #1] + + add x8, x16, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + add x8, x8, #32 // grain_lut += BLOCK_SIZE * bx + + sub x11, x13, w11, uxtw + + b.eq 1f + // y overlap + dup v6.16b, v27.b[0] + dup v7.16b, v27.b[1] + mov w10, w7 // backup actual h + mov w7, #2 +1: + br x11 +endfunc + +function fgy_loop_neon +.macro fgy ox, oy +L(loop_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: + ld1 {v0.16b, v1.16b}, [x1], x2 // src +.if \ox + ld1 {v20.8b}, [x4], x9 // grain_lut old +.endif +.if \oy + ld1 {v22.16b, v23.16b}, [x6], x9 // grain_lut top +.endif +.if \ox && \oy + ld1 {v21.8b}, [x8], x9 // grain_lut top old +.endif + ld1 {v18.16b, v19.16b}, [x5], x9 // grain_lut + + bl gather32_neon + +.if \ox + smull v20.8h, v20.8b, v27.8b + smlal v20.8h, v18.8b, v28.8b +.endif + +.if \oy +.if \ox + smull v21.8h, v21.8b, v27.8b + smlal v21.8h, v22.8b, v28.8b + sqrshrn v20.8b, v20.8h, #5 + sqrshrn v21.8b, v21.8h, #5 +.endif + +.if \ox + smull v16.8h, v20.8b, v7.8b +.else + smull v16.8h, v18.8b, v7.8b +.endif + smull2 v17.8h, v18.16b, v7.16b + smull v18.8h, v19.8b, v7.8b + smull2 v19.8h, v19.16b, v7.16b +.if \ox + smlal v16.8h, v21.8b, v6.8b +.else + smlal v16.8h, v22.8b, v6.8b +.endif + smlal2 v17.8h, v22.16b, v6.16b + smlal v18.8h, v23.8b, v6.8b + smlal2 v19.8h, v23.16b, v6.16b + sqrshrn v22.8b, v16.8h, #5 + sqrshrn2 v22.16b, v17.8h, #5 + sqrshrn v23.8b, v18.8h, #5 + sqrshrn2 v23.16b, v19.8h, #5 +.endif + + // sxtl of grain +.if \oy + sxtl v16.8h, v22.8b + sxtl2 v17.8h, v22.16b + sxtl v18.8h, v23.8b + sxtl2 v19.8h, v23.16b +.elseif \ox + sqrshrn v20.8b, v20.8h, #5 + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + sxtl v16.8h, v20.8b +.else + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b +.endif + + uxtl v2.8h, v4.8b // scaling + uxtl2 v3.8h, v4.16b + uxtl v4.8h, v5.8b + uxtl2 v5.8h, v5.16b + + mul v16.8h, v16.8h, v2.8h // scaling * grain + mul v17.8h, v17.8h, v3.8h + mul v18.8h, v18.8h, v4.8h + mul v19.8h, v19.8h, v5.8h + + srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) + srshl v17.8h, v17.8h, v29.8h + srshl v18.8h, v18.8h, v29.8h + srshl v19.8h, v19.8h, v29.8h + + uaddw v16.8h, v16.8h, v0.8b // *src + noise + uaddw2 v17.8h, v17.8h, v0.16b + uaddw v18.8h, v18.8h, v1.8b + uaddw2 v19.8h, v19.8h, v1.16b + + sqxtun v0.8b, v16.8h + sqxtun2 v0.16b, v17.8h + sqxtun v1.8b, v18.8h + sqxtun2 v1.16b, v19.8h + + umax v0.16b, v0.16b, v30.16b + umax v1.16b, v1.16b, v30.16b + umin v0.16b, v0.16b, v31.16b + umin v1.16b, v1.16b, v31.16b + + subs w7, w7, #1 +.if \oy + dup v6.16b, v28.b[0] + dup v7.16b, v28.b[1] +.endif + st1 {v0.16b, v1.16b}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w10, #2 + sub w7, w10, #2 // restore actual remaining h + b.gt L(loop_\ox\()0) +.endif + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.endm + + fgy 0, 0 + fgy 0, 1 + fgy 1, 0 + fgy 1, 1 + +L(fgy_loop_tbl): + .hword L(fgy_loop_tbl) - L(loop_00) + .hword L(fgy_loop_tbl) - L(loop_01) + .hword L(fgy_loop_tbl) - L(loop_10) + .hword L(fgy_loop_tbl) - L(loop_11) +endfunc + +// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst, +// const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const Dav1dFilmGrainData *const data, +// const entry grain_lut[][GRAIN_WIDTH], +// const pixel *const luma_row, +// const ptrdiff_t luma_stride, +// const int offsets[][2], +// const ptrdiff_t h, const ptrdiff_t uv, +// const ptrdiff_t is_id, +// const ptrdiff_t type); +.macro fguv layout, sx, sy +function fguv_32x32_\layout\()_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-32]! + str d8, [sp, #16] + ldp x8, x9, [sp, #32] // offsets, h + ldp x10, x11, [sp, #48] // uv, is_id + + ldr w13, [x4, #FGD_SCALING_SHIFT] + ldr w12, [x4, #FGD_CLIP_TO_RESTRICTED_RANGE] + neg w13, w13 // -scaling_shift + + // !csfl + add x10, x4, x10, lsl #2 // + 4*uv + add x14, x10, #FGD_UV_LUMA_MULT + add x15, x10, #FGD_UV_MULT + add x10, x10, #FGD_UV_OFFSET + ld1 {v8.h}[0], [x14] // uv_luma_mult + ld1r {v24.8h}, [x10] // uv_offset + ld1 {v8.h}[1], [x15] // uv_mult + + dup v29.8h, w13 // -scaling_shift + + cbz w12, 1f + // clip + movi v30.16b, #16 + movi v31.16b, #240 + cbz w11, 2f + // is_id + movi v31.16b, #235 + b 2f +1: + // no clip + movi v30.16b, #0 + movi v31.16b, #255 +2: + + ldr w12, [x8, #8] // offsets[1][0] + ldr w14, [x8, #4] // offsets[0][1] + ldr w16, [x8, #12] // offsets[1][1] + ldr w8, [x8] // offsets[0][0] + + mov x10, #GRAIN_WIDTH // grain_lut stride + + add x5, x5, #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6 +.if \sy + add x5, x5, x10, lsl #2 // grain_lut += 4 * grain_stride + add x5, x5, x10, lsl #1 // grain_lut += 2 * grain_stride +.else + add x5, x5, x10, lsl #3 // grain_lut += 8 * grain_stride + add x5, x5, x10 // grain_lut += grain_stride +.endif + + calc_offset w12, w13, w12, \sx, \sy + calc_offset w14, w15, w14, \sx, \sy + calc_offset w16, w17, w16, \sx, \sy + calc_offset w8, w11, w8, \sx, \sy + + add_offset x13, w12, x13, x5, x10 + add_offset x15, w14, x15, x5, x10 + add_offset x17, w16, x17, x5, x10 + add_offset x5, w8, x11, x5, x10 + + add x4, x13, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + add x8, x15, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x17, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x11, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + + ldr w13, [sp, #64] // type + + movrel x16, overlap_coeffs_\sx + adr x14, L(fguv_loop_sx\sx\()_tbl) + + ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs + tst w13, #1 + ldrh w13, [x14, w13, uxtw #1] + + b.eq 1f + // y overlap + sub w12, w9, #(2 >> \sy) // backup remaining h + mov w9, #(2 >> \sy) + +1: + sub x13, x14, w13, uxtw + +.if \sy + movi v25.16b, #23 + movi v26.16b, #22 +.else + movi v25.16b, #27 + movi v26.16b, #17 +.endif + +.if \sy + add x7, x7, x7 // luma_stride *= 2 +.endif + + br x13 +endfunc +.endm + +fguv 420, 1, 1 +fguv 422, 1, 0 +fguv 444, 0, 0 + +function fguv_loop_sx0_neon +.macro fguv_loop_sx0 csfl, ox, oy +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: + ld1 {v0.16b, v1.16b}, [x6], x7 // luma + ld1 {v6.16b, v7.16b}, [x1], x2 // src +.if \ox + ld1 {v20.8b}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v22.16b, v23.16b}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v21.8b}, [x11], x10 // grain_lut top old +.endif + ld1 {v18.16b, v19.16b}, [x5], x10 // grain_lut + +.if !\csfl + uxtl v2.8h, v0.8b + uxtl2 v3.8h, v0.16b + uxtl v4.8h, v1.8b + uxtl2 v5.8h, v1.16b + uxtl v0.8h, v6.8b + uxtl2 v1.8h, v6.16b + uxtl v16.8h, v7.8b + uxtl2 v17.8h, v7.16b + mul v2.8h, v2.8h, v8.h[0] + mul v3.8h, v3.8h, v8.h[0] + mul v4.8h, v4.8h, v8.h[0] + mul v5.8h, v5.8h, v8.h[0] + mul v0.8h, v0.8h, v8.h[1] + mul v1.8h, v1.8h, v8.h[1] + mul v16.8h, v16.8h, v8.h[1] + mul v17.8h, v17.8h, v8.h[1] + sqadd v2.8h, v2.8h, v0.8h + sqadd v3.8h, v3.8h, v1.8h + sqadd v4.8h, v4.8h, v16.8h + sqadd v5.8h, v5.8h, v17.8h + sshr v2.8h, v2.8h, #6 + sshr v3.8h, v3.8h, #6 + sshr v4.8h, v4.8h, #6 + sshr v5.8h, v5.8h, #6 + add v2.8h, v2.8h, v24.8h + add v3.8h, v3.8h, v24.8h + add v4.8h, v4.8h, v24.8h + add v5.8h, v5.8h, v24.8h + sqxtun v0.8b, v2.8h + sqxtun2 v0.16b, v3.8h + sqxtun v1.8b, v4.8h + sqxtun2 v1.16b, v5.8h +.endif + + bl gather32_neon + +.if \ox + smull v20.8h, v20.8b, v27.8b + smlal v20.8h, v18.8b, v28.8b +.endif + +.if \oy +.if \ox + smull v21.8h, v21.8b, v27.8b + smlal v21.8h, v22.8b, v28.8b + sqrshrn v20.8b, v20.8h, #5 + sqrshrn v21.8b, v21.8h, #5 +.endif + +.if \ox + smull v16.8h, v20.8b, v26.8b +.else + smull v16.8h, v18.8b, v26.8b +.endif + smull2 v17.8h, v18.16b, v26.16b + smull v18.8h, v19.8b, v26.8b + smull2 v19.8h, v19.16b, v26.16b +.if \ox + smlal v16.8h, v21.8b, v25.8b +.else + smlal v16.8h, v22.8b, v25.8b +.endif + smlal2 v17.8h, v22.16b, v25.16b + smlal v18.8h, v23.8b, v25.8b + smlal2 v19.8h, v23.16b, v25.16b + sqrshrn v22.8b, v16.8h, #5 + sqrshrn2 v22.16b, v17.8h, #5 + sqrshrn v23.8b, v18.8h, #5 + sqrshrn2 v23.16b, v19.8h, #5 +.endif + + // sxtl of grain +.if \oy + sxtl v16.8h, v22.8b + sxtl2 v17.8h, v22.16b + sxtl v18.8h, v23.8b + sxtl2 v19.8h, v23.16b +.elseif \ox + sqrshrn v20.8b, v20.8h, #5 + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + sxtl v16.8h, v20.8b +.else + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b +.endif + + uxtl v2.8h, v4.8b // scaling + uxtl2 v3.8h, v4.16b + uxtl v4.8h, v5.8b + uxtl2 v5.8h, v5.16b + + mul v16.8h, v16.8h, v2.8h // scaling * grain + mul v17.8h, v17.8h, v3.8h + mul v18.8h, v18.8h, v4.8h + mul v19.8h, v19.8h, v5.8h + + srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) + srshl v17.8h, v17.8h, v29.8h + srshl v18.8h, v18.8h, v29.8h + srshl v19.8h, v19.8h, v29.8h + + uaddw v16.8h, v16.8h, v6.8b // *src + noise + uaddw2 v17.8h, v17.8h, v6.16b + uaddw v18.8h, v18.8h, v7.8b + uaddw2 v19.8h, v19.8h, v7.16b + + sqxtun v0.8b, v16.8h + sqxtun2 v0.16b, v17.8h + sqxtun v1.8b, v18.8h + sqxtun2 v1.16b, v19.8h + + umax v0.16b, v0.16b, v30.16b + umax v1.16b, v1.16b, v30.16b + umin v0.16b, v0.16b, v31.16b + umin v1.16b, v1.16b, v31.16b + + subs w9, w9, #1 +.if \oy + dup v25.16b, v28.b[0] + dup v26.16b, v28.b[1] +.endif + st1 {v0.16b, v1.16b}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) +.endif + b 9f +.endm + fguv_loop_sx0 0, 0, 0 + fguv_loop_sx0 0, 0, 1 + fguv_loop_sx0 0, 1, 0 + fguv_loop_sx0 0, 1, 1 + fguv_loop_sx0 1, 0, 0 + fguv_loop_sx0 1, 0, 1 + fguv_loop_sx0 1, 1, 0 + fguv_loop_sx0 1, 1, 1 + +9: + ldr d8, [sp, #16] + ldr x30, [sp], #32 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(fguv_loop_sx0_tbl): + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_11) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_11) +endfunc + +function fguv_loop_sx1_neon +.macro fguv_loop_sx1 csfl, ox, oy +L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: + ld1 {v0.16b, v1.16b}, [x6], x7 // luma + ld1 {v6.16b}, [x1], x2 // src +.if \ox + ld1 {v20.8b}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v22.16b}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v21.8b}, [x11], x10 // grain_lut top old +.endif + ld1 {v18.16b}, [x5], x10 // grain_lut + + uaddlp v2.8h, v0.16b + uaddlp v3.8h, v1.16b +.if \csfl + rshrn v0.8b, v2.8h, #1 + rshrn2 v0.16b, v3.8h, #1 +.else + urshr v2.8h, v2.8h, #1 + urshr v3.8h, v3.8h, #1 + uxtl v0.8h, v6.8b + uxtl2 v1.8h, v6.16b + mul v2.8h, v2.8h, v8.h[0] + mul v3.8h, v3.8h, v8.h[0] + mul v0.8h, v0.8h, v8.h[1] + mul v1.8h, v1.8h, v8.h[1] + sqadd v2.8h, v2.8h, v0.8h + sqadd v3.8h, v3.8h, v1.8h + sshr v2.8h, v2.8h, #6 + sshr v3.8h, v3.8h, #6 + add v2.8h, v2.8h, v24.8h + add v3.8h, v3.8h, v24.8h + sqxtun v0.8b, v2.8h + sqxtun2 v0.16b, v3.8h +.endif + + bl gather16_neon + +.if \ox + smull v20.8h, v20.8b, v27.8b + smlal v20.8h, v18.8b, v28.8b +.endif + +.if \oy +.if \ox + smull v21.8h, v21.8b, v27.8b + smlal v21.8h, v22.8b, v28.8b + sqrshrn v20.8b, v20.8h, #5 + sqrshrn v21.8b, v21.8h, #5 +.endif + +.if \ox + smull v16.8h, v20.8b, v26.8b +.else + smull v16.8h, v18.8b, v26.8b +.endif + smull2 v17.8h, v18.16b, v26.16b +.if \ox + smlal v16.8h, v21.8b, v25.8b +.else + smlal v16.8h, v22.8b, v25.8b +.endif + smlal2 v17.8h, v22.16b, v25.16b + sqrshrn v22.8b, v16.8h, #5 + sqrshrn2 v22.16b, v17.8h, #5 +.endif + + // sxtl of grain +.if \oy + sxtl v16.8h, v22.8b + sxtl2 v17.8h, v22.16b +.elseif \ox + sqrshrn v20.8b, v20.8h, #5 + sxtl2 v17.8h, v18.16b + sxtl v16.8h, v20.8b +.else + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b +.endif + + uxtl v2.8h, v4.8b // scaling + uxtl2 v3.8h, v4.16b + + mul v16.8h, v16.8h, v2.8h // scaling * grain + mul v17.8h, v17.8h, v3.8h + + srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) + srshl v17.8h, v17.8h, v29.8h + + uaddw v16.8h, v16.8h, v6.8b // *src + noise + uaddw2 v17.8h, v17.8h, v6.16b + + sqxtun v0.8b, v16.8h + sqxtun2 v0.16b, v17.8h + + umax v0.16b, v0.16b, v30.16b + umin v0.16b, v0.16b, v31.16b + +.if \oy + mov v16.16b, v25.16b +.endif + subs w9, w9, #1 +.if \oy + mov v25.16b, v26.16b + mov v26.16b, v16.16b +.endif + st1 {v0.16b}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) +.endif + + b 9f +.endm + fguv_loop_sx1 0, 0, 0 + fguv_loop_sx1 0, 0, 1 + fguv_loop_sx1 0, 1, 0 + fguv_loop_sx1 0, 1, 1 + fguv_loop_sx1 1, 0, 0 + fguv_loop_sx1 1, 0, 1 + fguv_loop_sx1 1, 1, 0 + fguv_loop_sx1 1, 1, 1 + +9: + ldr d8, [sp, #16] + ldr x30, [sp], #32 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(fguv_loop_sx1_tbl): + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_11) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_11) +endfunc diff --git a/third_party/dav1d/src/arm/64/filmgrain16.S b/third_party/dav1d/src/arm/64/filmgrain16.S new file mode 100644 index 0000000000..7c4ff6dda9 --- /dev/null +++ b/third_party/dav1d/src/arm/64/filmgrain16.S @@ -0,0 +1,1997 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" +#include "src/arm/asm-offsets.h" + +#define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr w11, w2, #3 + lsr w12, w2, #12 + lsr w13, w2, #1 + eor w11, w2, w11 // (r >> 0) ^ (r >> 3) + eor w12, w12, w13 // (r >> 12) ^ (r >> 1) + eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr w2, w2, #\steps +.endif + and w11, w11, #((1 << \steps) - 1) // bit +.if \shift + orr w2, w2, w11, lsl #(16 - \steps) // *state +.else + orr w2, w2, w11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, x2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, x2, #17 - \bits, #\bits + lsr w2, w2, #1 +.endm + +// special calling convention: +// w2 holds seed +// x3 holds dav1d_gaussian_sequence +// clobbers x11-x15 +// returns in v0.8h +function get_gaussian_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + increment_seed 4 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + read_rand x14, 11, 3 + ld1 {v0.h}[3], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 2 + ld1 {v0.h}[4], [x14] + add x15, x3, x15, lsl #1 + read_rand x14, 11, 1 + ld1 {v0.h}[5], [x15] + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[6], [x14] + ld1 {v0.h}[7], [x15] + ret +endfunc + +.macro store_grain_row r0, r1, r2, r3, r4, r5 + st1 {\r0\().16b,\r1\().16b}, [x0], #32 + st1 {\r2\().16b,\r3\().16b}, [x0], #32 + st1 {\r4\().16b}, [x0], #16 + st1 {\r5\().h}[0], [x0], #2 +.endm + +function get_grain_2_neon + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + ld1 {v0.h}[1], [x15] + srshl v0.4h, v0.4h, v31.4h + ret +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif +.endm + +function get_grain_4_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + ld1 {v0.h}[3], [x15] + srshl v0.4h, v0.4h, v31.4h + ret +endfunc + +.macro get_grain_4 dst + bl get_grain_4_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif +.endm + +// w15 holds the number of entries to produce +// w14, w16 and w17 hold the previous output entries +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #2 +.if \n == 1 + madd w11, w14, w4, w11 // sum (above) + *coeff * prev output +.elseif \n == 2 + madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w14, w17, w11 // += *coeff * prev output 2 + mov w16, w14 +.else + madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 + madd w11, w14, w21, w11 // += *coeff * prev output 3 + mov w17, w16 + mov w16, w14 +.endif + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + add w12, w12, w10 // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + asr w12, w12, w9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.h[7], w14 + b.gt 1b + ret +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + sub x12, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v18.8h}, [x12] // load top right + + ext v0.16b, v16.16b, v17.16b, #14 // top left, top mid + ext v1.16b, v17.16b, v18.16b, #2 // top mid, top right + + smull v4.4s, v17.4h, v28.4h + smlal v4.4s, v0.4h, v27.4h + smlal v4.4s, v1.4h, v29.4h + smull2 v5.4s, v17.8h, v28.8h + smlal2 v5.4s, v0.8h, v27.8h + smlal2 v5.4s, v1.8h, v29.8h + + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + ret +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff + bl sum_\lag\()_above_neon +.ifc \type, uv_420 + add x12, x19, #GRAIN_WIDTH*2 + ld1 {v22.8h, v23.8h}, [x19], #32 + ld1 {v24.8h, v25.8h}, [x12] + addp v22.8h, v22.8h, v23.8h + addp v23.8h, v24.8h, v25.8h + add v22.8h, v22.8h, v23.8h + srshr v0.8h, v22.8h, #2 +.endif +.ifc \type, uv_422 + ld1 {v22.8h, v23.8h}, [x19], #32 + addp v22.8h, v22.8h, v23.8h + srshr v0.8h, v22.8h, #1 +.endif +.ifc \type, uv_444 + ld1 {v0.8h}, [x19], #16 +.endif +.if \uv_layout +.ifnb \uv_coeff + dup v1.8b, \uv_coeff + sxtl v1.8h, v1.8b + smlal v4.4s, v0.4h, v1.4h + smlal2 v5.4s, v0.8h, v1.8h +.else + smlal v4.4s, v0.4h, v30.4h + smlal2 v5.4s, v0.8h, v30.8h +.endif +.endif +.if \uv_layout && \elems == 8 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 7 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 1 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: +.if \elems > 4 +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + ext v4.16b, v4.16b, v4.16b, #12 +.ifc \lag, lag3 + smov w17, v0.h[5] +.endif +.ifnc \lag, lag1 + smov w16, v0.h[6] +.endif + smov w14, v0.h[7] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b +.ifc \edge, right + mov w15, #3 + bl output_\lag\()_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #2 +.else + mov w15, #4 + bl output_\lag\()_neon +.endif +.else + // elems == 1 + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #1 + bl output_\lag\()_neon + lsr w2, w2, #3 + + read_rand x12, 11, 2 + read_rand x13, 11, 1 + read_rand x14, 11, 0 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v1.h}[0], [x12] + ld1 {v1.h}[1], [x13] + ld1 {v1.h}[2], [x14] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #14 +.endif + st1 {v0.8h}, [x0], #16 + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag1_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #1*GRAIN_WIDTH*2 + ld1 {v17.8h}, [x12] // load the previous block right above +.endif + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 7 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 7 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 1 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 1 + + +function sum_lag2_above_neon + sub x12, x0, #2*GRAIN_WIDTH*2 - 16 + sub x13, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v18.8h}, [x12] // load top right + ld1 {v21.8h}, [x13] + + dup v26.8b, v30.b[0] + ext v22.16b, v16.16b, v17.16b, #12 // top left, top mid + dup v27.8b, v30.b[1] + ext v23.16b, v16.16b, v17.16b, #14 + sxtl v26.8h, v26.8b + dup v28.8b, v30.b[3] + ext v0.16b, v17.16b, v18.16b, #2 // top mid, top right + sxtl v27.8h, v27.8b + dup v29.8b, v30.b[4] + ext v1.16b, v17.16b, v18.16b, #4 + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + + smull v4.4s, v22.4h, v26.4h + smlal v4.4s, v23.4h, v27.4h + smlal v4.4s, v0.4h, v28.4h + smlal v4.4s, v1.4h, v29.4h + smull2 v5.4s, v22.8h, v26.8h + smlal2 v5.4s, v23.8h, v27.8h + smlal2 v5.4s, v0.8h, v28.8h + smlal2 v5.4s, v1.8h, v29.8h + + dup v26.16b, v30.b[5] + ext v22.16b, v19.16b, v20.16b, #12 // top left, top mid + dup v27.16b, v30.b[6] + ext v23.16b, v19.16b, v20.16b, #14 + sxtl v26.8h, v26.8b + dup v28.16b, v30.b[8] + ext v0.16b, v20.16b, v21.16b, #2 // top mid, top right + sxtl v27.8h, v27.8b + dup v29.16b, v30.b[9] + ext v1.16b, v20.16b, v21.16b, #4 + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + + smlal v4.4s, v22.4h, v26.4h + smlal v4.4s, v23.4h, v27.4h + smlal v4.4s, v0.4h, v28.4h + smlal v4.4s, v1.4h, v29.4h + smlal2 v5.4s, v22.8h, v26.8h + smlal2 v5.4s, v23.8h, v27.8h + smlal2 v5.4s, v0.8h, v28.8h + smlal2 v5.4s, v1.8h, v29.8h + + dup v26.16b, v30.b[2] + dup v27.16b, v30.b[7] + sxtl v26.8h, v26.8b + sxtl v27.8h, v27.8b + + smlal v4.4s, v17.4h, v26.4h + smlal v4.4s, v20.4h, v27.4h + smlal2 v5.4s, v17.8h, v26.8h + smlal2 v5.4s, v20.8h, v27.8h + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag2_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #2*GRAIN_WIDTH*2 + sub x13, x0, #1*GRAIN_WIDTH*2 + ld1 {v17.8h}, [x12] // load the previous block right above + ld1 {v20.8h}, [x13] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, v30.b[12] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 7 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 7 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 1 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 1 + + +function sum_lag3_above_neon + sub x11, x0, #3*GRAIN_WIDTH*2 - 16 + sub x12, x0, #2*GRAIN_WIDTH*2 - 16 + sub x13, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v15.8h}, [x11] // load top right + ld1 {v18.8h}, [x12] + ld1 {v21.8h}, [x13] + + dup v22.8b, v29.b[0] + ext v8.16b, v13.16b, v14.16b, #10 // top left, top mid + dup v23.8b, v29.b[1] + ext v9.16b, v13.16b, v14.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v29.b[2] + sxtl v23.8h, v23.8b + dup v25.8b, v29.b[3] + ext v10.16b, v13.16b, v14.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v29.b[4] + ext v11.16b, v14.16b, v15.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v29.b[5] + ext v12.16b, v14.16b, v15.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v29.b[6] + ext v13.16b, v14.16b, v15.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smull v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v14.4h, v25.4h + smull2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v14.8h, v25.8h + + dup v22.8b, v29.b[7] + ext v8.16b, v16.16b, v17.16b, #10 // top left, top mid + dup v23.8b, v29.b[8] + ext v9.16b, v16.16b, v17.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v29.b[9] + sxtl v23.8h, v23.8b + dup v25.8b, v29.b[10] + ext v10.16b, v16.16b, v17.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v29.b[11] + ext v11.16b, v17.16b, v18.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v29.b[12] + ext v12.16b, v17.16b, v18.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v29.b[13] + ext v13.16b, v17.16b, v18.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smlal v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v17.4h, v25.4h + smlal2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v17.8h, v25.8h + + dup v22.8b, v29.b[14] + ext v8.16b, v19.16b, v20.16b, #10 // top left, top mid + dup v23.8b, v29.b[15] + ext v9.16b, v19.16b, v20.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v30.b[0] + sxtl v23.8h, v23.8b + dup v25.8b, v30.b[1] + ext v10.16b, v19.16b, v20.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v30.b[2] + ext v11.16b, v20.16b, v21.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v30.b[3] + ext v12.16b, v20.16b, v21.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v30.b[4] + ext v13.16b, v20.16b, v21.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smlal v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v20.4h, v25.4h + mov v16.16b, v17.16b + mov v17.16b, v18.16b + smlal2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v20.8h, v25.8h + + mov v13.16b, v14.16b + mov v14.16b, v15.16b + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag3_\edge\()_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +.ifc \edge, left + sub x11, x0, #3*GRAIN_WIDTH*2 + sub x12, x0, #2*GRAIN_WIDTH*2 + sub x13, x0, #1*GRAIN_WIDTH*2 + ld1 {v14.8h}, [x11] // load the previous block right above + ld1 {v17.8h}, [x12] + ld1 {v20.8h}, [x13] +.endif + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, v30.b[8] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 7 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 7 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 1 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 1 + +function generate_grain_rows_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +1: + mov w16, #80 +2: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h + subs w16, w16, #8 + st1 {v0.8h}, [x0], #16 + b.gt 2b + get_grain_2 v0 + subs w1, w1, #1 + st1 {v0.s}[0], [x0], #4 + b.gt 1b + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function generate_grain_rows_44_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! +1: + mov w16, #40 +2: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h + subs w16, w16, #8 + st1 {v0.8h}, [x0], #16 + b.gt 2b + get_grain_4 v0 + subs w1, w1, #1 + st1 {v0.4h}, [x0] + add x0, x0, #GRAIN_WIDTH*2-80 + b.gt 1b + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function gen_grain_uv_444_lag0_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + ld1 {v4.8h}, [x19], #16 +gen_grain_uv_lag0_8_start: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h +gen_grain_uv_lag0_8_add: + and v4.16b, v4.16b, v1.16b + smull v2.4s, v4.4h, v27.4h + smull2 v3.4s, v4.8h, v27.8h + srshl v2.4s, v2.4s, v28.4s + srshl v3.4s, v3.4s, v28.4s + sqxtn v2.4h, v2.4s + sqxtn2 v2.8h, v3.4s + sqadd v2.8h, v2.8h, v0.8h + smin v2.8h, v2.8h, v25.8h + smax v2.8h, v2.8h, v26.8h + st1 {v2.8h}, [x0], #16 + ldr x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret +endfunc + +function gen_grain_uv_420_lag0_8_neon + AARCH64_SIGN_LINK_REGISTER + add x12, x19, #GRAIN_WIDTH*2 + str x30, [sp, #-16]! + ld1 {v16.8h, v17.8h}, [x19], #32 + ld1 {v18.8h, v19.8h}, [x12] + addp v16.8h, v16.8h, v17.8h + addp v17.8h, v18.8h, v19.8h + add v16.8h, v16.8h, v17.8h + srshr v4.8h, v16.8h, #2 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_422_lag0_8_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + ld1 {v16.8h, v17.8h}, [x19], #32 + addp v16.8h, v16.8h, v17.8h + srshr v4.8h, v16.8h, #1 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_420_lag0_4_neon + add x12, x19, #GRAIN_WIDTH*2 + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + ld1 {v16.4h, v17.4h}, [x19] + ld1 {v18.4h, v19.4h}, [x12] + add x19, x19, #32 + addp v16.4h, v16.4h, v17.4h + addp v17.4h, v18.4h, v19.4h + add v16.4h, v16.4h, v17.4h + srshr v4.4h, v16.4h, #2 + get_grain_4 v0 + b gen_grain_uv_lag0_8_add +endfunc + +function gen_grain_uv_422_lag0_4_neon + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-16]! + ld1 {v16.4h, v17.4h}, [x19] + add x19, x19, #32 + addp v16.4h, v16.4h, v17.4h + srshr v4.4h, v16.4h, #1 + get_grain_4 v0 + b gen_grain_uv_lag0_8_add +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_16bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x30, x19, [sp, #-96]! + +.ifc \type, uv_444 + mov w13, w3 + mov w14, #28 + add x19, x1, #3*GRAIN_WIDTH*2 + mov x1, x2 + mul w13, w13, w14 + clz w15, w4 +.else + clz w15, w2 +.endif + movrel x3, X(gaussian_sequence) + sub w15, w15, #24 // -bitdepth_min_8 + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add x4, x1, #FGD_AR_COEFFS_Y +.else + add x4, x1, #FGD_AR_COEFFS_UV +.endif + add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + +.ifc \type, uv_444 + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne +.endif + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + neg w15, w15 // bitdepth_min_8 + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #128 + lsl w5, w5, w15 // 128 << bitdepth_min_8 + neg w6, w5 // -(128 << bitpdeth_min_8) + sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 + +.ifc \type, uv_444 + eor w2, w2, w11 +.endif + + br x16 + +L(generate_grain_\type\()_lag0): + AARCH64_VALID_JUMP_TARGET +.ifc \type, y + mov w1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + dup v28.4s, w7 + ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + dup v25.8h, w5 + dup v26.8h, w6 + ext v29.16b, v0.16b, v1.16b, #10 + ext v30.16b, v1.16b, v0.16b, #2 + neg v28.4s, v28.4s + sxtl v27.8h, v27.8b + + mov w1, #3 + bl generate_grain_rows_neon + mov w1, #GRAIN_HEIGHT-3 +1: + mov v1.16b, v29.16b + bl gen_grain_uv_444_lag0_neon // 8 + movi v1.16b, #255 + bl gen_grain_uv_444_lag0_neon // 16 + bl gen_grain_uv_444_lag0_neon // 24 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 40 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 56 + bl gen_grain_uv_444_lag0_neon // 64 + bl gen_grain_uv_444_lag0_neon // 72 + mov v1.16b, v30.16b + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 + add x19, x19, #4 + st1 {v16.s}[0], [x0], #4 + b.gt 1b +.endif + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag1): + AARCH64_VALID_JUMP_TARGET + ld1r {v27.8b}, [x4], #1 // ar_coeffs_y[0] + ld1r {v28.8b}, [x4], #1 // ar_coeffs_y[1] + ld1r {v29.8b}, [x4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb w4, [x4, #1] // ar_coeffs_y[3] +.else + add x4, x4, #2 +.endif + + mov w1, #3 +.ifc \type, uv_444 + ld1r {v30.8b}, [x4] // ar_coeffs_uv[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b +.ifc \type, uv_444 + sxtl v30.8h, v30.8b +.endif + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_mid_neon // 48 + bl sum_\type\()_lag1_mid_neon // 56 + bl sum_\type\()_lag1_mid_neon // 64 + bl sum_\type\()_lag1_mid_neon // 72 + bl sum_\type\()_lag1_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag2): + AARCH64_VALID_JUMP_TARGET + ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_mid_neon // 48 + bl sum_\type\()_lag2_mid_neon // 56 + bl sum_\type\()_lag2_mid_neon // 64 + bl sum_\type\()_lag2_mid_neon // 72 + bl sum_\type\()_lag2_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag3): + AARCH64_VALID_JUMP_TARGET + ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_mid_neon // 48 + bl sum_\type\()_lag3_mid_neon // 56 + bl sum_\type\()_lag3_mid_neon // 64 + bl sum_\type\()_lag3_mid_neon // 72 + bl sum_\type\()_lag3_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) +.else + sub \reg, \reg, #6*32-GRAIN_WIDTH*2 +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_16bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x30, x19, [sp, #-96]! + + mov w13, w3 + mov w14, #28 + add x19, x1, #(3*GRAIN_WIDTH-3)*2 + mov x1, x2 + mul w13, w13, w14 + clz w15, w4 + + movrel x3, X(gaussian_sequence) + sub w15, w15, #24 // -bitdepth_min_8 + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] + add x4, x1, #FGD_AR_COEFFS_UV + add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + neg w15, w15 // bitdepth_min_8 + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #128 + lsl w5, w5, w15 // 128 << bitdepth_min_8 + neg w6, w5 // -(128 << bitpdeth_min_8) + sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 + + eor w2, w2, w11 + + br x16 + +L(generate_grain_\type\()_lag0): + AARCH64_VALID_JUMP_TARGET + dup v28.4s, w7 + ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + dup v25.8h, w5 + dup v26.8h, w6 + ext v29.16b, v0.16b, v1.16b, #10 + ext v30.16b, v1.16b, v0.16b, #14 + neg v28.4s, v28.4s + sxtl v27.8h, v27.8b + + mov w1, #3 + bl generate_grain_rows_44_neon + set_height w1, \type +1: + mov v1.16b, v29.16b + bl gen_grain_\type\()_lag0_8_neon // 8 + movi v1.16b, #255 + bl gen_grain_\type\()_lag0_8_neon // 16 + bl gen_grain_\type\()_lag0_8_neon // 24 + bl gen_grain_\type\()_lag0_8_neon // 32 + bl gen_grain_\type\()_lag0_8_neon // 40 + mov v1.16b, v30.16b + bl gen_grain_\type\()_lag0_4_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag1): + AARCH64_VALID_JUMP_TARGET + ld1r {v27.8b}, [x4], #1 // ar_coeffs_uv[0] + ld1r {v28.8b}, [x4], #1 // ar_coeffs_uv[1] + ld1r {v29.8b}, [x4] // ar_coeffs_uv[2] + add x4, x4, #2 + + mov w1, #3 + ld1r {v30.8b}, [x4] // ar_coeffs_u4[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + sxtl v30.8h, v30.8b + set_height w1, \type +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag2): + AARCH64_VALID_JUMP_TARGET + ld1 {v30.16b}, [x4] // ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(generate_grain_\type\()_lag3): + AARCH64_VALID_JUMP_TARGET + ldr q29, [x4] // ar_coeffs_uv[0-15] + ldr q30, [x4, #16] // ar_coeffs_uv[16-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 + +.macro gather_interleaved dst1, dst2, src1, src2, off + umov w14, \src1[0] + umov w15, \src2[1] + umov w16, \src1[2] + add x14, x14, x3 + umov w17, \src2[3] + add x15, x15, x3 + ld1 {\dst1}[0+\off], [x14] + umov w14, \src1[4] + add x16, x16, x3 + ld1 {\dst2}[1+\off], [x15] + umov w15, \src2[5] + add x17, x17, x3 + ld1 {\dst1}[2+\off], [x16] + umov w16, \src1[6] + add x14, x14, x3 + ld1 {\dst2}[3+\off], [x17] + umov w17, \src2[7] + add x15, x15, x3 + ld1 {\dst1}[4+\off], [x14] + add x16, x16, x3 + ld1 {\dst2}[5+\off], [x15] + add x17, x17, x3 + ld1 {\dst1}[6+\off], [x16] + ld1 {\dst2}[7+\off], [x17] +.endm + +.macro gather dst1, dst2, src1, src2, src3, src4 + gather_interleaved \dst1, \dst2, \src1, \src3, 0 + gather_interleaved \dst2, \dst1, \src3, \src1, 0 + gather_interleaved \dst1, \dst2, \src2, \src4, 8 + gather_interleaved \dst2, \dst1, \src4, \src2, 8 +.endm + +function gather32_neon + gather v6.b, v7.b, v0.h, v1.h, v2.h, v3.h + ret +endfunc + +function gather16_neon + gather_interleaved v6.b, v7.b, v0.h, v1.h, 0 + gather_interleaved v7.b, v6.b, v1.h, v0.h, 0 + ins v6.d[1], v7.d[0] + ret +endfunc + +const overlap_coeffs_0, align=4 + .short 27, 17, 0, 0 + .short 17, 27, 32, 32 +endconst + +const overlap_coeffs_1, align=4 + .short 23, 0, 0, 0 + .short 22, 32, 32, 32 +endconst + +.macro calc_offset offx, offy, src, sx, sy + and \offy, \src, #0xF // randval & 0xF + lsr \offx, \src, #4 // randval >> 4 +.if \sy == 0 + add \offy, \offy, \offy // 2 * (randval & 0xF) +.endif +.if \sx == 0 + add \offx, \offx, \offx // 2 * (randval >> 4) +.endif +.endm + +.macro add_offset dst, offx, offy, src, stride + madd \dst, \stride, \offy, \src // grain_lut += grain_stride * offy + add \dst, \dst, \offx, uxtw #1 // grain_lut += offx +.endm + +// void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const int scaling_shift, +// const entry grain_lut[][GRAIN_WIDTH], +// const int offsets[][2], +// const int h, const ptrdiff_t clip, +// const ptrdiff_t type, +// const int bitdepth_max); +function fgy_32x32_16bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-80]! + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + str d14, [sp, #64] + eor w4, w4, #15 // 15 - scaling_shift + ldr w11, [x6, #8] // offsets[1][0] + ldr w13, [x6, #4] // offsets[0][1] + ldr w15, [x6, #12] // offsets[1][1] + ldr w10, [sp, #96] // bitdepth_max + ldr w6, [x6] // offsets[0][0] + dup v26.8h, w10 // bitdepth_max + clz w10, w10 + ldr w8, [sp, #80] // clip + sub w10, w10, #24 // -bitdepth_min_8 + mov x9, #GRAIN_WIDTH*2 // grain_lut stride + neg w10, w10 // bitdepth_min_8 + + dup v29.8h, w4 // 15 - scaling_shift + dup v27.8h, w10 // bitdepth_min_8 + + movrel x16, overlap_coeffs_0 + + cbz w8, 1f + // clip + movi v30.8h, #16 + movi v31.8h, #235 + sshl v30.8h, v30.8h, v27.8h + sshl v31.8h, v31.8h, v27.8h + b 2f +1: + // no clip + movi v30.8h, #0 + mov v31.16b, v26.16b // bitdepth_max +2: + + ushr v26.8h, v26.8h, #1 // grain_max + not v25.16b, v26.16b // grain_min + + ld1 {v27.4h, v28.4h}, [x16] // overlap_coeffs + + add x5, x5, #18 // grain_lut += 9 + add x5, x5, x9, lsl #3 // grain_lut += 8 * grain_stride + add x5, x5, x9 // grain_lut += grain_stride + + calc_offset w11, w12, w11, 0, 0 + calc_offset w13, w14, w13, 0, 0 + calc_offset w15, w16, w15, 0, 0 + calc_offset w6, w10, w6, 0, 0 + + add_offset x12, w11, x12, x5, x9 + add_offset x14, w13, x14, x5, x9 + add_offset x16, w15, x16, x5, x9 + add_offset x5, w6, x10, x5, x9 + + ldr w11, [sp, #88] // type + adr x13, L(fgy_loop_tbl) + + add x4, x12, #32*2 // grain_lut += BLOCK_SIZE * bx + add x6, x14, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + + tst w11, #1 + ldrh w11, [x13, w11, uxtw #1] + + add x8, x16, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by + add x8, x8, #32*2 // grain_lut += BLOCK_SIZE * bx + + sub x11, x13, w11, uxtw + + b.eq 1f + // y overlap + dup v8.8h, v27.h[0] + dup v9.8h, v27.h[1] + mov w10, w7 // backup actual h + mov w7, #2 +1: + br x11 +endfunc + +function fgy_loop_neon +.macro fgy ox, oy +L(loop_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 // src +.if \ox + ld1 {v20.4h}, [x4], x9 // grain_lut old +.endif +.if \oy + ld1 {v21.8h, v22.8h, v23.8h, v24.8h}, [x6], x9 // grain_lut top +.endif +.if \ox && \oy + ld1 {v14.4h}, [x8], x9 // grain_lut top old +.endif + mvni v4.8h, #0xf0, lsl #8 // 0x0fff + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x9 // grain_lut + + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + and v0.16b, v0.16b, v4.16b + and v1.16b, v1.16b, v4.16b + and v2.16b, v2.16b, v4.16b + and v3.16b, v3.16b, v4.16b + bl gather32_neon + +.if \ox + smull v20.4s, v20.4h, v27.4h + smlal v20.4s, v16.4h, v28.4h +.endif + +.if \oy +.if \ox + smull v14.4s, v14.4h, v27.4h + smlal v14.4s, v21.4h, v28.4h + sqrshrn v20.4h, v20.4s, #5 + sqrshrn v14.4h, v14.4s, #5 + smin v20.4h, v20.4h, v26.4h + smin v14.4h, v14.4h, v26.4h + smax v20.4h, v20.4h, v25.4h + smax v14.4h, v14.4h, v25.4h +.endif + +.if \ox + smull v10.4s, v20.4h, v9.4h +.else + smull v10.4s, v16.4h, v9.4h +.endif + smull2 v11.4s, v16.8h, v9.8h + smull v12.4s, v17.4h, v9.4h + smull2 v13.4s, v17.8h, v9.8h + smull v16.4s, v18.4h, v9.4h + smull2 v17.4s, v18.8h, v9.8h + smull v18.4s, v19.4h, v9.4h + smull2 v19.4s, v19.8h, v9.8h +.if \ox + smlal v10.4s, v14.4h, v8.4h +.else + smlal v10.4s, v21.4h, v8.4h +.endif + smlal2 v11.4s, v21.8h, v8.8h + smlal v12.4s, v22.4h, v8.4h + smlal2 v13.4s, v22.8h, v8.8h + smlal v16.4s, v23.4h, v8.4h + smlal2 v17.4s, v23.8h, v8.8h + smlal v18.4s, v24.4h, v8.4h + smlal2 v19.4s, v24.8h, v8.8h + sqrshrn v10.4h, v10.4s, #5 + sqrshrn2 v10.8h, v11.4s, #5 + sqrshrn v11.4h, v12.4s, #5 + sqrshrn2 v11.8h, v13.4s, #5 + sqrshrn v12.4h, v16.4s, #5 + sqrshrn2 v12.8h, v17.4s, #5 + sqrshrn v13.4h, v18.4s, #5 + sqrshrn2 v13.8h, v19.4s, #5 + smin v16.8h, v10.8h, v26.8h + smin v17.8h, v11.8h, v26.8h + smin v18.8h, v12.8h, v26.8h + smin v19.8h, v13.8h, v26.8h + smax v16.8h, v16.8h, v25.8h + smax v17.8h, v17.8h, v25.8h + smax v18.8h, v18.8h, v25.8h + smax v19.8h, v19.8h, v25.8h +.endif + + uxtl v4.8h, v6.8b // scaling +.if \ox && !\oy + sqrshrn v20.4h, v20.4s, #5 +.endif + uxtl2 v5.8h, v6.16b +.if \ox && !\oy + smin v20.4h, v20.4h, v26.4h +.endif + uxtl v6.8h, v7.8b +.if \ox && !\oy + smax v20.4h, v20.4h, v25.4h +.endif + uxtl2 v7.8h, v7.16b +.if \ox && !\oy + ins v16.d[0], v20.d[0] +.endif + ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) + ushl v5.8h, v5.8h, v29.8h + ushl v6.8h, v6.8h, v29.8h + ushl v7.8h, v7.8h, v29.8h + + sqrdmulh v20.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) + sqrdmulh v21.8h, v17.8h, v5.8h + sqrdmulh v22.8h, v18.8h, v6.8h + sqrdmulh v23.8h, v19.8h, v7.8h + + usqadd v0.8h, v20.8h // *src + noise + usqadd v1.8h, v21.8h + usqadd v2.8h, v22.8h + usqadd v3.8h, v23.8h + + umax v0.8h, v0.8h, v30.8h + umax v1.8h, v1.8h, v30.8h + umax v2.8h, v2.8h, v30.8h + umax v3.8h, v3.8h, v30.8h + umin v0.8h, v0.8h, v31.8h + umin v1.8h, v1.8h, v31.8h + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h + + subs w7, w7, #1 +.if \oy + dup v8.8h, v28.h[0] + dup v9.8h, v28.h[1] +.endif + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w10, #2 + sub w7, w10, #2 // restore actual remaining h + b.gt L(loop_\ox\()0) +.endif + ldr d14, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldr x30, [sp], #80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.endm + + fgy 0, 0 + fgy 0, 1 + fgy 1, 0 + fgy 1, 1 + +L(fgy_loop_tbl): + .hword L(fgy_loop_tbl) - L(loop_00) + .hword L(fgy_loop_tbl) - L(loop_01) + .hword L(fgy_loop_tbl) - L(loop_10) + .hword L(fgy_loop_tbl) - L(loop_11) +endfunc + +// void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst, +// const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const Dav1dFilmGrainData *const data, +// const entry grain_lut[][GRAIN_WIDTH], +// const pixel *const luma_row, +// const ptrdiff_t luma_stride, +// const int offsets[][2], +// const ptrdiff_t h, const ptrdiff_t uv, +// const ptrdiff_t is_id, +// const ptrdiff_t type, +// const int bitdepth_max); +.macro fguv layout, sx, sy +function fguv_32x32_\layout\()_16bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + str x30, [sp, #-80]! + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + + ldp x8, x9, [sp, #80] // offsets, h + ldp x10, x11, [sp, #96] // uv, is_id + ldr w16, [sp, #120] // bitdepth_max + + ldr w13, [x4, #FGD_SCALING_SHIFT] + ldr w12, [x4, #FGD_CLIP_TO_RESTRICTED_RANGE] + dup v23.8h, w16 // bitdepth_max + clz w16, w16 + eor w13, w13, #15 // 15 - scaling_shift + sub w16, w16, #24 // -bitdepth_min_8 + + // !csfl + add x10, x4, x10, lsl #2 // + 4*uv + add x14, x10, #FGD_UV_LUMA_MULT + add x15, x10, #FGD_UV_MULT + add x10, x10, #FGD_UV_OFFSET + neg w16, w16 // bitdepth_min_8 + ld1r {v8.8h}, [x14] // uv_luma_mult + ld1r {v24.8h}, [x10] // uv_offset + ld1r {v9.8h}, [x15] // uv_mult + + dup v29.8h, w13 // 15 - scaling_shift + dup v27.8h, w16 // bitdepth_min_8 + + cbz w12, 1f + // clip + movi v30.8h, #16 + movi v31.8h, #240 + sshl v30.8h, v30.8h, v27.8h + sshl v31.8h, v31.8h, v27.8h + cbz w11, 2f + // is_id + movi v31.8h, #235 + sshl v31.8h, v31.8h, v27.8h + b 2f +1: + // no clip + movi v30.8h, #0 + mov v31.16b, v23.16b // bitdepth_max +2: + + ushr v15.8h, v23.8h, #1 // grain_max + sshl v24.8h, v24.8h, v27.8h // uv_offset << bitdepth_min_8 + not v14.16b, v15.16b // grain_min + + ldr w12, [x8, #8] // offsets[1][0] + ldr w14, [x8, #4] // offsets[0][1] + ldr w16, [x8, #12] // offsets[1][1] + ldr w8, [x8] // offsets[0][0] + + mov x10, #GRAIN_WIDTH*2 // grain_lut stride + + add x5, x5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6 +.if \sy + add x5, x5, x10, lsl #2 // grain_lut += 4 * grain_stride + add x5, x5, x10, lsl #1 // grain_lut += 2 * grain_stride +.else + add x5, x5, x10, lsl #3 // grain_lut += 8 * grain_stride + add x5, x5, x10 // grain_lut += grain_stride +.endif + + calc_offset w12, w13, w12, \sx, \sy + calc_offset w14, w15, w14, \sx, \sy + calc_offset w16, w17, w16, \sx, \sy + calc_offset w8, w11, w8, \sx, \sy + + add_offset x13, w12, x13, x5, x10 + add_offset x15, w14, x15, x5, x10 + add_offset x17, w16, x17, x5, x10 + add_offset x5, w8, x11, x5, x10 + + add x4, x13, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + add x8, x15, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x17, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x11, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + + ldr w13, [sp, #112] // type + + movrel x16, overlap_coeffs_\sx + adr x14, L(fguv_loop_sx\sx\()_tbl) + + ld1 {v27.4h, v28.4h}, [x16] // overlap_coeffs + tst w13, #1 + ldrh w13, [x14, w13, uxtw #1] + + b.eq 1f + // y overlap + sub w12, w9, #(2 >> \sy) // backup remaining h + mov w9, #(2 >> \sy) + +1: + sub x13, x14, w13, uxtw + +.if \sy + movi v25.8h, #23 + movi v26.8h, #22 +.else + movi v25.8h, #27 + movi v26.8h, #17 +.endif + +.if \sy + add x7, x7, x7 // luma_stride *= 2 +.endif + + br x13 +endfunc +.endm + +fguv 420, 1, 1 +fguv 422, 1, 0 +fguv 444, 0, 0 + +function fguv_loop_sx0_neon +.macro fguv_loop_sx0 csfl, ox, oy +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: +.if \ox + ld1 {v4.4h}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v5.4h}, [x11], x10 // grain_lut top old +.endif + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x10 // grain_lut + +.if \ox + smull v4.4s, v4.4h, v27.4h + smlal v4.4s, v16.4h, v28.4h +.endif + +.if \oy +.if \ox + smull v5.4s, v5.4h, v27.4h + smlal v5.4s, v0.4h, v28.4h + sqrshrn v4.4h, v4.4s, #5 + sqrshrn v5.4h, v5.4s, #5 + smin v4.4h, v4.4h, v15.4h + smin v5.4h, v5.4h, v15.4h + smax v4.4h, v4.4h, v14.4h + smax v5.4h, v5.4h, v14.4h + ins v16.d[0], v4.d[0] + ins v0.d[0], v5.d[0] +.endif + + smull v6.4s, v16.4h, v26.4h + smull2 v7.4s, v16.8h, v26.8h + smull v10.4s, v17.4h, v26.4h + smull2 v11.4s, v17.8h, v26.8h + smull v16.4s, v18.4h, v26.4h + smull2 v17.4s, v18.8h, v26.8h + smull v18.4s, v19.4h, v26.4h + smull2 v19.4s, v19.8h, v26.8h + smlal v6.4s, v0.4h, v25.4h + smlal2 v7.4s, v0.8h, v25.8h + smlal v10.4s, v1.4h, v25.4h + smlal2 v11.4s, v1.8h, v25.8h + smlal v16.4s, v2.4h, v25.4h + smlal2 v17.4s, v2.8h, v25.8h + smlal v18.4s, v3.4h, v25.4h + smlal2 v19.4s, v3.8h, v25.8h + sqrshrn v6.4h, v6.4s, #5 + sqrshrn2 v6.8h, v7.4s, #5 + sqrshrn v7.4h, v10.4s, #5 + sqrshrn2 v7.8h, v11.4s, #5 + sqrshrn v10.4h, v16.4s, #5 + sqrshrn2 v10.8h, v17.4s, #5 + sqrshrn v11.4h, v18.4s, #5 + sqrshrn2 v11.8h, v19.4s, #5 +.endif + +.if \ox && !\oy + sqrshrn v4.4h, v4.4s, #5 + smin v4.4h, v4.4h, v15.4h +.endif + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 // luma +.if \oy + smin v16.8h, v6.8h, v15.8h + smin v17.8h, v7.8h, v15.8h + smin v18.8h, v10.8h, v15.8h + smin v19.8h, v11.8h, v15.8h + smax v16.8h, v16.8h, v14.8h + smax v17.8h, v17.8h, v14.8h + smax v18.8h, v18.8h, v14.8h + smax v19.8h, v19.8h, v14.8h +.endif + +.if \ox && !\oy + smax v4.4h, v4.4h, v14.4h +.endif + ld1 {v10.8h, v11.8h, v12.8h, v13.8h}, [x1], x2 // src +.if \ox && !\oy + ins v16.d[0], v4.d[0] +.endif + +.if !\csfl + smull v4.4s, v0.4h, v8.4h + smull2 v5.4s, v0.8h, v8.8h + smull v6.4s, v1.4h, v8.4h + smull2 v7.4s, v1.8h, v8.8h + smull v0.4s, v2.4h, v8.4h + smull2 v1.4s, v2.8h, v8.8h + smull v2.4s, v3.4h, v8.4h + smull2 v3.4s, v3.8h, v8.8h + smlal v4.4s, v10.4h, v9.4h + smlal2 v5.4s, v10.8h, v9.8h + smlal v6.4s, v11.4h, v9.4h + smlal2 v7.4s, v11.8h, v9.8h + smlal v0.4s, v12.4h, v9.4h + smlal2 v1.4s, v12.8h, v9.8h + smlal v2.4s, v13.4h, v9.4h + smlal2 v3.4s, v13.8h, v9.8h + shrn v4.4h, v4.4s, #6 + shrn2 v4.8h, v5.4s, #6 + shrn v5.4h, v6.4s, #6 + shrn2 v5.8h, v7.4s, #6 + shrn v6.4h, v0.4s, #6 + shrn2 v6.8h, v1.4s, #6 + shrn v7.4h, v2.4s, #6 + shrn2 v7.8h, v3.4s, #6 + add v0.8h, v4.8h, v24.8h + add v1.8h, v5.8h, v24.8h + add v2.8h, v6.8h, v24.8h + add v3.8h, v7.8h, v24.8h + movi v20.8h, #0 + smin v0.8h, v0.8h, v23.8h + smin v1.8h, v1.8h, v23.8h + smin v2.8h, v2.8h, v23.8h + smin v3.8h, v3.8h, v23.8h + smax v0.8h, v0.8h, v20.8h + smax v1.8h, v1.8h, v20.8h + smax v2.8h, v2.8h, v20.8h + smax v3.8h, v3.8h, v20.8h +.else + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + and v0.16b, v0.16b, v23.16b + and v1.16b, v1.16b, v23.16b + and v2.16b, v2.16b, v23.16b + and v3.16b, v3.16b, v23.16b +.endif + + bl gather32_neon + + uxtl v4.8h, v6.8b // scaling + uxtl2 v5.8h, v6.16b + uxtl v6.8h, v7.8b + uxtl2 v7.8h, v7.16b + + ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) + ushl v5.8h, v5.8h, v29.8h + ushl v6.8h, v6.8h, v29.8h + ushl v7.8h, v7.8h, v29.8h + + sqrdmulh v16.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) + sqrdmulh v17.8h, v17.8h, v5.8h + sqrdmulh v18.8h, v18.8h, v6.8h + sqrdmulh v19.8h, v19.8h, v7.8h + + usqadd v10.8h, v16.8h // *src + noise + usqadd v11.8h, v17.8h + usqadd v12.8h, v18.8h + usqadd v13.8h, v19.8h + + umax v0.8h, v10.8h, v30.8h + umax v1.8h, v11.8h, v30.8h + umax v2.8h, v12.8h, v30.8h + umax v3.8h, v13.8h, v30.8h + umin v0.8h, v0.8h, v31.8h + umin v1.8h, v1.8h, v31.8h + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h + + subs w9, w9, #1 +.if \oy + dup v25.8h, v28.h[0] + dup v26.8h, v28.h[1] +.endif + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) +.endif + b 9f +.endm + fguv_loop_sx0 0, 0, 0 + fguv_loop_sx0 0, 0, 1 + fguv_loop_sx0 0, 1, 0 + fguv_loop_sx0 0, 1, 1 + fguv_loop_sx0 1, 0, 0 + fguv_loop_sx0 1, 0, 1 + fguv_loop_sx0 1, 1, 0 + fguv_loop_sx0 1, 1, 1 + +9: + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldr x30, [sp], #80 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(fguv_loop_sx0_tbl): + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_11) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_11) +endfunc + +function fguv_loop_sx1_neon +.macro fguv_loop_sx1 csfl, ox, oy +L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): + AARCH64_VALID_JUMP_TARGET +1: +.if \ox + ld1 {v18.4h}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v20.8h, v21.8h}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v19.4h}, [x11], x10 // grain_lut top old +.endif + ld1 {v16.8h, v17.8h}, [x5], x10 // grain_lut + +.if \ox + smull v18.4s, v18.4h, v27.4h + smlal v18.4s, v16.4h, v28.4h +.endif + +.if \oy +.if \ox + smull v19.4s, v19.4h, v27.4h + smlal v19.4s, v20.4h, v28.4h + sqrshrn v18.4h, v18.4s, #5 + sqrshrn v19.4h, v19.4s, #5 + smin v18.4h, v18.4h, v15.4h + smin v19.4h, v19.4h, v15.4h + smax v18.4h, v18.4h, v14.4h + smax v19.4h, v19.4h, v14.4h + ins v16.d[0], v18.d[0] + ins v20.d[0], v19.d[0] +.endif + + smull v0.4s, v16.4h, v26.4h + smull2 v1.4s, v16.8h, v26.8h + smull v2.4s, v17.4h, v26.4h + smull2 v3.4s, v17.8h, v26.8h + smlal v0.4s, v20.4h, v25.4h + smlal2 v1.4s, v20.8h, v25.8h + smlal v2.4s, v21.4h, v25.4h + smlal2 v3.4s, v21.8h, v25.8h + sqrshrn v16.4h, v0.4s, #5 + sqrshrn2 v16.8h, v1.4s, #5 + sqrshrn v17.4h, v2.4s, #5 + sqrshrn2 v17.8h, v3.4s, #5 +.endif + +.if \ox && !\oy + sqrshrn v18.4h, v18.4s, #5 + smin v18.4h, v18.4h, v15.4h +.endif + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 // luma +.if \oy + smin v16.8h, v16.8h, v15.8h + smin v17.8h, v17.8h, v15.8h + smax v16.8h, v16.8h, v14.8h + smax v17.8h, v17.8h, v14.8h +.endif + +.if \ox && !\oy + smax v18.4h, v18.4h, v14.4h +.endif + ld1 {v10.8h, v11.8h}, [x1], x2 // src +.if \ox && !\oy + ins v16.d[0], v18.d[0] +.endif + addp v0.8h, v0.8h, v1.8h + addp v1.8h, v2.8h, v3.8h + urshr v0.8h, v0.8h, #1 + urshr v1.8h, v1.8h, #1 +.if !\csfl + smull v2.4s, v0.4h, v8.4h + smull2 v3.4s, v0.8h, v8.8h + smull v0.4s, v1.4h, v8.4h + smull2 v1.4s, v1.8h, v8.8h + smlal v2.4s, v10.4h, v9.4h + smlal2 v3.4s, v10.8h, v9.8h + smlal v0.4s, v11.4h, v9.4h + smlal2 v1.4s, v11.8h, v9.8h + shrn v2.4h, v2.4s, #6 + shrn2 v2.8h, v3.4s, #6 + shrn v3.4h, v0.4s, #6 + shrn2 v3.8h, v1.4s, #6 + add v0.8h, v2.8h, v24.8h + add v1.8h, v3.8h, v24.8h + movi v2.8h, #0 + smin v0.8h, v0.8h, v23.8h + smin v1.8h, v1.8h, v23.8h + smax v0.8h, v0.8h, v2.8h + smax v1.8h, v1.8h, v2.8h +.else + // Make sure that uninitialized pixels out of range past the right + // edge are in range; their actual values shouldn't matter. + and v0.16b, v0.16b, v23.16b + and v1.16b, v1.16b, v23.16b +.endif + + bl gather16_neon + + uxtl v4.8h, v6.8b // scaling + uxtl2 v5.8h, v6.16b + + ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) + ushl v5.8h, v5.8h, v29.8h + + sqrdmulh v16.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) + sqrdmulh v17.8h, v17.8h, v5.8h + + usqadd v10.8h, v16.8h // *src + noise + usqadd v11.8h, v17.8h + + umax v0.8h, v10.8h, v30.8h + umax v1.8h, v11.8h, v30.8h + umin v0.8h, v0.8h, v31.8h + umin v1.8h, v1.8h, v31.8h + +.if \oy + mov v16.16b, v25.16b +.endif + subs w9, w9, #1 +.if \oy + mov v25.16b, v26.16b + mov v26.16b, v16.16b +.endif + st1 {v0.8h, v1.8h}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) +.endif + + b 9f +.endm + fguv_loop_sx1 0, 0, 0 + fguv_loop_sx1 0, 0, 1 + fguv_loop_sx1 0, 1, 0 + fguv_loop_sx1 0, 1, 1 + fguv_loop_sx1 1, 0, 0 + fguv_loop_sx1 1, 0, 1 + fguv_loop_sx1 1, 1, 0 + fguv_loop_sx1 1, 1, 1 + +9: + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldr x30, [sp], #80 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(fguv_loop_sx1_tbl): + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_11) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_11) +endfunc diff --git a/third_party/dav1d/src/arm/64/ipred.S b/third_party/dav1d/src/arm/64/ipred.S new file mode 100644 index 0000000000..dab67577e6 --- /dev/null +++ b/third_party/dav1d/src/arm/64/ipred.S @@ -0,0 +1,3985 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_dc_128_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + movi v0.16b, #128 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +4: + AARCH64_VALID_JUMP_TARGET + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + movi v1.16b, #128 +32: + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + movi v1.16b, #128 + movi v2.16b, #128 + movi v3.16b, #128 +64: + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_128_tbl): + .hword L(ipred_dc_128_tbl) - 640b + .hword L(ipred_dc_128_tbl) - 320b + .hword L(ipred_dc_128_tbl) - 16b + .hword L(ipred_dc_128_tbl) - 8b + .hword L(ipred_dc_128_tbl) - 4b +endfunc + +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_v_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #1 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x2] +4: + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2] +8: + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2] +16: + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b}, [x2] +32: + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] +64: + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + b.gt 64b + ret + +L(ipred_v_tbl): + .hword L(ipred_v_tbl) - 640b + .hword L(ipred_v_tbl) - 320b + .hword L(ipred_v_tbl) - 160b + .hword L(ipred_v_tbl) - 80b + .hword L(ipred_v_tbl) - 40b +endfunc + +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_h_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + sub x2, x2, #4 + sub x5, x5, w3, uxtw + mov x7, #-4 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +4: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 + st1 {v3.s}[0], [x0], x1 + st1 {v2.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v1.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 + st1 {v3.8b}, [x0], x1 + st1 {v2.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v1.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], x7 + st1 {v3.16b}, [x0], x1 + st1 {v2.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v1.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + st1 {v3.16b}, [x0], x1 + st1 {v2.16b}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + st1 {v1.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + st1 {v3.16b}, [x0], x1 + st1 {v2.16b}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + st1 {v1.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 64b + ret + +L(ipred_h_tbl): + .hword L(ipred_h_tbl) - 64b + .hword L(ipred_h_tbl) - 32b + .hword L(ipred_h_tbl) - 16b + .hword L(ipred_h_tbl) - 8b + .hword L(ipred_h_tbl) - 4b +endfunc + +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_dc_top_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #1 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2s}, [x2] + uaddlv h0, v0.8b + rshrn v0.8b, v0.8h, #3 + dup v0.8b, v0.b[0] +4: + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2] + uaddlv h0, v0.8b + rshrn v0.8b, v0.8h, #3 + dup v0.8b, v0.b[0] +8: + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2] + uaddlv h0, v0.16b + rshrn v0.8b, v0.8h, #4 + dup v0.16b, v0.b[0] +16: + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b}, [x2] + uaddlv h0, v0.16b + uaddlv h1, v1.16b + add v2.4h, v0.4h, v1.4h + rshrn v2.8b, v2.8h, #5 + dup v0.16b, v2.b[0] + dup v1.16b, v2.b[0] +32: + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] + uaddlv h0, v0.16b + uaddlv h1, v1.16b + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add v4.4h, v0.4h, v1.4h + add v5.4h, v2.4h, v3.4h + add v4.4h, v4.4h, v5.4h + rshrn v4.8b, v4.8h, #6 + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] + dup v2.16b, v4.b[0] + dup v3.16b, v4.b[0] +64: + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_top_tbl): + .hword L(ipred_dc_top_tbl) - 640b + .hword L(ipred_dc_top_tbl) - 320b + .hword L(ipred_dc_top_tbl) - 160b + .hword L(ipred_dc_top_tbl) - 80b + .hword L(ipred_dc_top_tbl) - 40b +endfunc + +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 + sub x2, x2, w4, uxtw + clz w3, w3 + clz w7, w4 + adr x5, L(ipred_dc_left_tbl) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w7, w7, #25 + ldrh w3, [x5, w3, uxtw #1] + ldrh w7, [x5, w7, uxtw #1] + sub x3, x5, w3, uxtw + sub x5, x5, w7, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_left_h4): + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2s}, [x2] + uaddlv h0, v0.8b + rshrn v0.8b, v0.8h, #3 + dup v0.16b, v0.b[0] + br x3 +L(ipred_dc_left_w4): + AARCH64_VALID_JUMP_TARGET + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt L(ipred_dc_left_w4) + ret + +L(ipred_dc_left_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2] + uaddlv h0, v0.8b + rshrn v0.8b, v0.8h, #3 + dup v0.16b, v0.b[0] + br x3 +L(ipred_dc_left_w8): + AARCH64_VALID_JUMP_TARGET + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt L(ipred_dc_left_w8) + ret + +L(ipred_dc_left_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2] + uaddlv h0, v0.16b + rshrn v0.8b, v0.8h, #4 + dup v0.16b, v0.b[0] + br x3 +L(ipred_dc_left_w16): + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt L(ipred_dc_left_w16) + ret + +L(ipred_dc_left_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b}, [x2] + uaddlv h0, v0.16b + uaddlv h1, v1.16b + add v0.4h, v0.4h, v1.4h + rshrn v0.8b, v0.8h, #5 + dup v0.16b, v0.b[0] + br x3 +L(ipred_dc_left_w32): + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b +1: + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h64): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] + uaddlv h0, v0.16b + uaddlv h1, v1.16b + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add v0.4h, v0.4h, v1.4h + add v2.4h, v2.4h, v3.4h + add v0.4h, v0.4h, v2.4h + rshrn v0.8b, v0.8h, #6 + dup v0.16b, v0.b[0] + br x3 +L(ipred_dc_left_w64): + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +1: + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_tbl): + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h4) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) +endfunc + +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 + sub x2, x2, w4, uxtw + add w7, w3, w4 // width + height + clz w3, w3 + clz w6, w4 + dup v16.8h, w7 // width + height + adr x5, L(ipred_dc_tbl) + rbit w7, w7 // rbit(width + height) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w6, w6, #25 + clz w7, w7 // ctz(width + height) + ldrh w3, [x5, w3, uxtw #1] + ldrh w6, [x5, w6, uxtw #1] + neg w7, w7 // -ctz(width + height) + sub x3, x5, w3, uxtw + sub x5, x5, w6, uxtw + ushr v16.8h, v16.8h, #1 // (width + height) >> 1 + dup v17.8h, w7 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x2], #4 + ins v0.s[1], wzr + uaddlv h0, v0.8b + add x2, x2, #1 + br x3 +L(ipred_dc_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.s}[0], [x2] + ins v1.s[1], wzr + add v0.4h, v0.4h, v16.4h + uaddlv h1, v1.8b + cmp w4, #4 + add v0.4h, v0.4h, v1.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 8/16 + mov w16, #(0x3334/2) + movk w16, #(0x5556/2), lsl #16 + add w17, w4, w4 // w17 = 2*h = 16 or 32 + lsr w16, w16, w17 + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8b, v0.b[0] +2: + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + subs w4, w4, #4 + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[0], [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2], #8 + uaddlv h0, v0.8b + add x2, x2, #1 + br x3 +L(ipred_dc_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.8b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h1, v1.8b + cmp w4, #8 + add v0.4h, v0.4h, v1.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #(0x3334/2) + mov w17, #(0x5556/2) + csel w16, w16, w17, eq + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8b, v0.b[0] +2: + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8b}, [x0], x1 + st1 {v0.8b}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2], #16 + uaddlv h0, v0.16b + add x2, x2, #1 + br x3 +L(ipred_dc_w16): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h1, v1.16b + cmp w4, #16 + add v0.4h, v0.4h, v1.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 4/8/32/64 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #(0x3334/2) + mov w17, #(0x5556/2) + csel w16, w16, w17, eq + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.16b, v0.b[0] +2: + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b}, [x0], x1 + st1 {v0.16b}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b}, [x2], #32 + uaddlv h0, v0.16b + uaddlv h1, v1.16b + add x2, x2, #1 + add v0.4h, v0.4h, v1.4h + br x3 +L(ipred_dc_w32): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b, v2.16b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h1, v1.16b + uaddlv h2, v2.16b + cmp w4, #32 + add v0.4h, v0.4h, v1.4h + add v0.4h, v0.4h, v2.4h + ushl v4.4h, v0.4h, v17.4h + b.eq 1f + // h = 8/16/64 + cmp w4, #8 + mov w16, #(0x3334/2) + mov w17, #(0x5556/2) + csel w16, w16, w17, eq + dup v16.4h, w16 + sqdmulh v4.4h, v4.4h, v16.4h +1: + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] +2: + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b}, [x0], x1 + st1 {v0.16b, v1.16b}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h64): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64 + uaddlv h0, v0.16b + uaddlv h1, v1.16b + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add v0.4h, v0.4h, v1.4h + add v2.4h, v2.4h, v3.4h + add x2, x2, #1 + add v0.4h, v0.4h, v2.4h + br x3 +L(ipred_dc_w64): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b, v2.16b, v3.16b, v4.16b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h1, v1.16b + uaddlv h2, v2.16b + uaddlv h3, v3.16b + uaddlv h4, v4.16b + add v1.4h, v1.4h, v2.4h + add v3.4h, v3.4h, v4.4h + cmp w4, #64 + add v0.4h, v0.4h, v1.4h + add v0.4h, v0.4h, v3.4h + ushl v4.4h, v0.4h, v17.4h + b.eq 1f + // h = 16/32 + mov w16, #(0x5556/2) + movk w16, #(0x3334/2), lsl #16 + lsr w16, w16, w4 + dup v16.4h, w16 + sqdmulh v4.4h, v4.4h, v16.4h +1: + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] + dup v2.16b, v4.b[0] + dup v3.16b, v4.b[0] +2: + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + subs w4, w4, #4 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_tbl): + .hword L(ipred_dc_tbl) - L(ipred_dc_h64) + .hword L(ipred_dc_tbl) - L(ipred_dc_h32) + .hword L(ipred_dc_tbl) - L(ipred_dc_h16) + .hword L(ipred_dc_tbl) - L(ipred_dc_h8) + .hword L(ipred_dc_tbl) - L(ipred_dc_h4) + .hword L(ipred_dc_tbl) - L(ipred_dc_w64) + .hword L(ipred_dc_tbl) - L(ipred_dc_w32) + .hword L(ipred_dc_tbl) - L(ipred_dc_w16) + .hword L(ipred_dc_tbl) - L(ipred_dc_w8) + .hword L(ipred_dc_tbl) - L(ipred_dc_w4) +endfunc + +// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_8bpc_neon, export=1 + clz w9, w3 + adr x5, L(ipred_paeth_tbl) + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.16b}, [x2] + add x8, x2, #1 + sub x2, x2, #4 + sub x5, x5, w9, uxtw + mov x7, #-4 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v5.4s}, [x8] + usubl v6.8h, v5.8b, v4.8b // top - topleft +4: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 + zip1 v0.2s, v0.2s, v1.2s + zip1 v2.2s, v2.2s, v3.2s + uaddw v16.8h, v6.8h, v0.8b + uaddw v17.8h, v6.8h, v2.8b + sqxtun v16.8b, v16.8h // base + sqxtun2 v16.16b, v17.8h + zip1 v0.2d, v0.2d, v2.2d + uabd v20.16b, v5.16b, v16.16b // tdiff + uabd v22.16b, v4.16b, v16.16b // tldiff + uabd v16.16b, v0.16b, v16.16b // ldiff + umin v18.16b, v20.16b, v22.16b // min(tdiff, tldiff) + cmhs v20.16b, v22.16b, v20.16b // tldiff >= tdiff + cmhs v16.16b, v18.16b, v16.16b // min(tdiff, tldiff) >= ldiff + bsl v20.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bit v20.16b, v0.16b, v16.16b // ldiff <= min ? left : ... + st1 {v20.s}[3], [x0], x1 + st1 {v20.s}[2], [x6], x1 + subs w4, w4, #4 + st1 {v20.s}[1], [x0], x1 + st1 {v20.s}[0], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1r {v5.2d}, [x8] + usubl v6.8h, v5.8b, v4.8b // top - topleft +8: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 + uaddw v16.8h, v6.8h, v0.8b + uaddw v17.8h, v6.8h, v1.8b + uaddw v18.8h, v6.8h, v2.8b + uaddw v19.8h, v6.8h, v3.8b + sqxtun v16.8b, v16.8h // base + sqxtun2 v16.16b, v17.8h + sqxtun v18.8b, v18.8h + sqxtun2 v18.16b, v19.8h + zip1 v2.2d, v2.2d, v3.2d + zip1 v0.2d, v0.2d, v1.2d + uabd v21.16b, v5.16b, v18.16b // tdiff + uabd v20.16b, v5.16b, v16.16b + uabd v23.16b, v4.16b, v18.16b // tldiff + uabd v22.16b, v4.16b, v16.16b + uabd v17.16b, v2.16b, v18.16b // ldiff + uabd v16.16b, v0.16b, v16.16b + umin v19.16b, v21.16b, v23.16b // min(tdiff, tldiff) + umin v18.16b, v20.16b, v22.16b + cmhs v21.16b, v23.16b, v21.16b // tldiff >= tdiff + cmhs v20.16b, v22.16b, v20.16b + cmhs v17.16b, v19.16b, v17.16b // min(tdiff, tldiff) >= ldiff + cmhs v16.16b, v18.16b, v16.16b + bsl v21.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v20.16b, v5.16b, v4.16b + bit v21.16b, v2.16b, v17.16b // ldiff <= min ? left : ... + bit v20.16b, v0.16b, v16.16b + st1 {v21.d}[1], [x0], x1 + st1 {v21.d}[0], [x6], x1 + subs w4, w4, #4 + st1 {v20.d}[1], [x0], x1 + st1 {v20.d}[0], [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v5.16b}, [x8], #16 + mov w9, w3 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw +1: + ld4r {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], x7 +2: + usubl v6.8h, v5.8b, v4.8b // top - topleft + usubl2 v7.8h, v5.16b, v4.16b + uaddw v24.8h, v6.8h, v0.8b + uaddw v25.8h, v7.8h, v0.8b + uaddw v26.8h, v6.8h, v1.8b + uaddw v27.8h, v7.8h, v1.8b + uaddw v28.8h, v6.8h, v2.8b + uaddw v29.8h, v7.8h, v2.8b + uaddw v30.8h, v6.8h, v3.8b + uaddw v31.8h, v7.8h, v3.8b + sqxtun v17.8b, v26.8h // base + sqxtun2 v17.16b, v27.8h + sqxtun v16.8b, v24.8h + sqxtun2 v16.16b, v25.8h + sqxtun v19.8b, v30.8h + sqxtun2 v19.16b, v31.8h + sqxtun v18.8b, v28.8h + sqxtun2 v18.16b, v29.8h + uabd v23.16b, v5.16b, v19.16b // tdiff + uabd v22.16b, v5.16b, v18.16b + uabd v21.16b, v5.16b, v17.16b + uabd v20.16b, v5.16b, v16.16b + uabd v27.16b, v4.16b, v19.16b // tldiff + uabd v26.16b, v4.16b, v18.16b + uabd v25.16b, v4.16b, v17.16b + uabd v24.16b, v4.16b, v16.16b + uabd v19.16b, v3.16b, v19.16b // ldiff + uabd v18.16b, v2.16b, v18.16b + uabd v17.16b, v1.16b, v17.16b + uabd v16.16b, v0.16b, v16.16b + umin v31.16b, v23.16b, v27.16b // min(tdiff, tldiff) + umin v30.16b, v22.16b, v26.16b + umin v29.16b, v21.16b, v25.16b + umin v28.16b, v20.16b, v24.16b + cmhs v23.16b, v27.16b, v23.16b // tldiff >= tdiff + cmhs v22.16b, v26.16b, v22.16b + cmhs v21.16b, v25.16b, v21.16b + cmhs v20.16b, v24.16b, v20.16b + cmhs v19.16b, v31.16b, v19.16b // min(tdiff, tldiff) >= ldiff + cmhs v18.16b, v30.16b, v18.16b + cmhs v17.16b, v29.16b, v17.16b + cmhs v16.16b, v28.16b, v16.16b + bsl v23.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v22.16b, v5.16b, v4.16b + bsl v21.16b, v5.16b, v4.16b + bsl v20.16b, v5.16b, v4.16b + bit v23.16b, v3.16b, v19.16b // ldiff <= min ? left : ... + bit v22.16b, v2.16b, v18.16b + bit v21.16b, v1.16b, v17.16b + bit v20.16b, v0.16b, v16.16b + subs w3, w3, #16 + st1 {v23.16b}, [x0], #16 + st1 {v22.16b}, [x6], #16 + st1 {v21.16b}, [x5], #16 + st1 {v20.16b}, [x10], #16 + b.le 8f + ld1 {v5.16b}, [x8], #16 + b 2b +8: + subs w4, w4, #4 + b.le 9f + // End of horizontal loop, move pointers to next four rows + sub x8, x8, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + // Load the top row as early as possible + ld1 {v5.16b}, [x8], #16 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_paeth_tbl): + .hword L(ipred_paeth_tbl) - 640b + .hword L(ipred_paeth_tbl) - 320b + .hword L(ipred_paeth_tbl) - 160b + .hword L(ipred_paeth_tbl) - 80b + .hword L(ipred_paeth_tbl) - 40b +endfunc + +// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_8bpc_neon, export=1 + movrel x10, X(sm_weights) + add x11, x10, w4, uxtw + add x10, x10, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_tbl) + sub x12, x2, w4, uxtw + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.16b}, [x12] // bottom + add x8, x2, #1 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v6.2s}, [x8] // top + ld1r {v7.2s}, [x10] // weights_hor + sub x2, x2, #4 + mov x7, #-4 + dup v5.16b, v6.b[3] // right + usubl v6.8h, v6.8b, v4.8b // top-bottom + uxtl v7.8h, v7.8b // weights_hor +4: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + zip1 v1.2s, v1.2s, v0.2s // left, flipped + zip1 v0.2s, v3.2s, v2.2s + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + shll v22.8h, v4.8b, #8 // bottom*256 + shll v23.8h, v4.8b, #8 + usubl v0.8h, v0.8b, v5.8b // left-right + usubl v1.8h, v1.8b, v5.8b + uxtl v16.8h, v16.8b // weights_ver + uxtl v18.8h, v18.8b + mla v20.8h, v0.8h, v7.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v1.8h, v7.8h + mla v22.8h, v6.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v23.8h, v6.8h, v18.8h + uhadd v20.8h, v20.8h, v22.8h + uhadd v21.8h, v21.8h, v23.8h + rshrn v20.8b, v20.8h, #8 + rshrn v21.8b, v21.8h, #8 + st1 {v20.s}[0], [x0], x1 + st1 {v20.s}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.s}[0], [x0], x1 + st1 {v21.s}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v6.8b}, [x8] // top + ld1 {v7.8b}, [x10] // weights_hor + sub x2, x2, #4 + mov x7, #-4 + dup v5.16b, v6.b[7] // right + usubl v6.8h, v6.8b, v4.8b // top-bottom + uxtl v7.8h, v7.8b // weights_hor +8: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + shll v22.8h, v5.8b, #8 + shll v23.8h, v5.8b, #8 + usubl v0.8h, v0.8b, v5.8b // left-right + usubl v1.8h, v1.8b, v5.8b + usubl v2.8h, v2.8b, v5.8b + usubl v3.8h, v3.8b, v5.8b + shll v24.8h, v4.8b, #8 // bottom*256 + shll v25.8h, v4.8b, #8 + shll v26.8h, v4.8b, #8 + shll v27.8h, v4.8b, #8 + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b + mla v20.8h, v3.8h, v7.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v2.8h, v7.8h // (left flipped) + mla v22.8h, v1.8h, v7.8h + mla v23.8h, v0.8h, v7.8h + mla v24.8h, v6.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v25.8h, v6.8h, v17.8h + mla v26.8h, v6.8h, v18.8h + mla v27.8h, v6.8h, v19.8h + uhadd v20.8h, v20.8h, v24.8h + uhadd v21.8h, v21.8h, v25.8h + uhadd v22.8h, v22.8h, v26.8h + uhadd v23.8h, v23.8h, v27.8h + rshrn v20.8b, v20.8h, #8 + rshrn v21.8b, v21.8h, #8 + rshrn v22.8b, v22.8h, #8 + rshrn v23.8b, v23.8h, #8 + st1 {v20.8b}, [x0], x1 + st1 {v21.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8b}, [x0], x1 + st1 {v23.8b}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + add x12, x2, w3, uxtw + sub x2, x2, #2 + mov x7, #-2 + ld1r {v5.16b}, [x12] // right + sub x1, x1, w3, uxtw + mov w9, w3 + +1: + ld2r {v0.8b, v1.8b}, [x2], x7 // left + ld2r {v16.8b, v17.8b}, [x11], #2 // weights_ver + usubl v0.8h, v0.8b, v5.8b // left-right + usubl v1.8h, v1.8b, v5.8b + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b +2: + ld1 {v7.16b}, [x10], #16 // weights_hor + ld1 {v3.16b}, [x8], #16 // top + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + shll v22.8h, v5.8b, #8 + shll v23.8h, v5.8b, #8 + uxtl v6.8h, v7.8b // weights_hor + uxtl2 v7.8h, v7.16b + usubl v2.8h, v3.8b, v4.8b // top-bottom + usubl2 v3.8h, v3.16b, v4.16b + mla v20.8h, v1.8h, v6.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v1.8h, v7.8h // (left flipped) + mla v22.8h, v0.8h, v6.8h + mla v23.8h, v0.8h, v7.8h + shll v24.8h, v4.8b, #8 // bottom*256 + shll v25.8h, v4.8b, #8 + shll v26.8h, v4.8b, #8 + shll v27.8h, v4.8b, #8 + mla v24.8h, v2.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v25.8h, v3.8h, v16.8h + mla v26.8h, v2.8h, v17.8h + mla v27.8h, v3.8h, v17.8h + uhadd v20.8h, v20.8h, v24.8h + uhadd v21.8h, v21.8h, v25.8h + uhadd v22.8h, v22.8h, v26.8h + uhadd v23.8h, v23.8h, v27.8h + rshrn v20.8b, v20.8h, #8 + rshrn2 v20.16b, v21.8h, #8 + rshrn v22.8b, v22.8h, #8 + rshrn2 v22.16b, v23.8h, #8 + subs w3, w3, #16 + st1 {v20.16b}, [x0], #16 + st1 {v22.16b}, [x6], #16 + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x8, w9, uxtw + sub x10, x10, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_tbl): + .hword L(ipred_smooth_tbl) - 640b + .hword L(ipred_smooth_tbl) - 320b + .hword L(ipred_smooth_tbl) - 160b + .hword L(ipred_smooth_tbl) - 80b + .hword L(ipred_smooth_tbl) - 40b +endfunc + +// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_8bpc_neon, export=1 + movrel x7, X(sm_weights) + add x7, x7, w4, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_v_tbl) + sub x8, x2, w4, uxtw + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.16b}, [x8] // bottom + add x2, x2, #1 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v6.2s}, [x2] // top + usubl v6.8h, v6.8b, v4.8b // top-bottom +4: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + shll v22.8h, v4.8b, #8 // bottom*256 + shll v23.8h, v4.8b, #8 + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + uxtl v16.8h, v16.8b // weights_ver + uxtl v18.8h, v18.8b + mla v22.8h, v6.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v23.8h, v6.8h, v18.8h + rshrn v22.8b, v22.8h, #8 + rshrn v23.8b, v23.8h, #8 + st1 {v22.s}[0], [x0], x1 + st1 {v22.s}[1], [x6], x1 + subs w4, w4, #4 + st1 {v23.s}[0], [x0], x1 + st1 {v23.s}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v6.8b}, [x2] // top + usubl v6.8h, v6.8b, v4.8b // top-bottom +8: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + shll v24.8h, v4.8b, #8 // bottom*256 + shll v25.8h, v4.8b, #8 + shll v26.8h, v4.8b, #8 + shll v27.8h, v4.8b, #8 + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b + mla v24.8h, v6.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v25.8h, v6.8h, v17.8h + mla v26.8h, v6.8h, v18.8h + mla v27.8h, v6.8h, v19.8h + rshrn v24.8b, v24.8h, #8 + rshrn v25.8b, v25.8h, #8 + rshrn v26.8b, v26.8h, #8 + rshrn v27.8b, v27.8h, #8 + st1 {v24.8b}, [x0], x1 + st1 {v25.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v26.8b}, [x0], x1 + st1 {v27.8b}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + // Set up pointers for four rows in parallel; x0, x6, x5, x8 + add x5, x0, x1 + add x8, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw + mov w9, w3 + +1: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b +2: + ld1 {v3.16b}, [x2], #16 // top + shll v20.8h, v4.8b, #8 // bottom*256 + shll v21.8h, v4.8b, #8 + shll v22.8h, v4.8b, #8 + shll v23.8h, v4.8b, #8 + shll v24.8h, v4.8b, #8 + shll v25.8h, v4.8b, #8 + shll v26.8h, v4.8b, #8 + shll v27.8h, v4.8b, #8 + usubl v2.8h, v3.8b, v4.8b // top-bottom + usubl2 v3.8h, v3.16b, v4.16b + mla v20.8h, v2.8h, v16.8h // bottom*256 + (top-bottom)*weights_ver + mla v21.8h, v3.8h, v16.8h + mla v22.8h, v2.8h, v17.8h + mla v23.8h, v3.8h, v17.8h + mla v24.8h, v2.8h, v18.8h + mla v25.8h, v3.8h, v18.8h + mla v26.8h, v2.8h, v19.8h + mla v27.8h, v3.8h, v19.8h + rshrn v20.8b, v20.8h, #8 + rshrn2 v20.16b, v21.8h, #8 + rshrn v22.8b, v22.8h, #8 + rshrn2 v22.16b, v23.8h, #8 + rshrn v24.8b, v24.8h, #8 + rshrn2 v24.16b, v25.8h, #8 + rshrn v26.8b, v26.8h, #8 + rshrn2 v26.16b, v27.8h, #8 + subs w3, w3, #16 + st1 {v20.16b}, [x0], #16 + st1 {v22.16b}, [x6], #16 + st1 {v24.16b}, [x5], #16 + st1 {v26.16b}, [x8], #16 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x2, x2, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x8, x8, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_v_tbl): + .hword L(ipred_smooth_v_tbl) - 640b + .hword L(ipred_smooth_v_tbl) - 320b + .hword L(ipred_smooth_v_tbl) - 160b + .hword L(ipred_smooth_v_tbl) - 80b + .hword L(ipred_smooth_v_tbl) - 40b +endfunc + +// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_8bpc_neon, export=1 + movrel x8, X(sm_weights) + add x8, x8, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_h_tbl) + add x12, x2, w3, uxtw + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v5.16b}, [x12] // right + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v7.2s}, [x8] // weights_hor + sub x2, x2, #4 + mov x7, #-4 + uxtl v7.8h, v7.8b // weights_hor +4: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + zip1 v1.2s, v1.2s, v0.2s // left, flipped + zip1 v0.2s, v3.2s, v2.2s + usubl v0.8h, v0.8b, v5.8b // left-right + usubl v1.8h, v1.8b, v5.8b + mla v20.8h, v0.8h, v7.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v1.8h, v7.8h + rshrn v20.8b, v20.8h, #8 + rshrn v21.8b, v21.8h, #8 + st1 {v20.s}[0], [x0], x1 + st1 {v20.s}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.s}[0], [x0], x1 + st1 {v21.s}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v7.8b}, [x8] // weights_hor + sub x2, x2, #4 + mov x7, #-4 + uxtl v7.8h, v7.8b // weights_hor +8: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + shll v22.8h, v5.8b, #8 + shll v23.8h, v5.8b, #8 + usubl v3.8h, v3.8b, v5.8b // left-right + usubl v2.8h, v2.8b, v5.8b + usubl v1.8h, v1.8b, v5.8b + usubl v0.8h, v0.8b, v5.8b + mla v20.8h, v3.8h, v7.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v2.8h, v7.8h // (left flipped) + mla v22.8h, v1.8h, v7.8h + mla v23.8h, v0.8h, v7.8h + rshrn v20.8b, v20.8h, #8 + rshrn v21.8b, v21.8h, #8 + rshrn v22.8b, v22.8h, #8 + rshrn v23.8b, v23.8h, #8 + st1 {v20.8b}, [x0], x1 + st1 {v21.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8b}, [x0], x1 + st1 {v23.8b}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + sub x2, x2, #4 + mov x7, #-4 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw + mov w9, w3 + +1: + ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left + usubl v0.8h, v0.8b, v5.8b // left-right + usubl v1.8h, v1.8b, v5.8b + usubl v2.8h, v2.8b, v5.8b + usubl v3.8h, v3.8b, v5.8b +2: + ld1 {v7.16b}, [x8], #16 // weights_hor + shll v20.8h, v5.8b, #8 // right*256 + shll v21.8h, v5.8b, #8 + shll v22.8h, v5.8b, #8 + shll v23.8h, v5.8b, #8 + shll v24.8h, v5.8b, #8 + shll v25.8h, v5.8b, #8 + shll v26.8h, v5.8b, #8 + shll v27.8h, v5.8b, #8 + uxtl v6.8h, v7.8b // weights_hor + uxtl2 v7.8h, v7.16b + mla v20.8h, v3.8h, v6.8h // right*256 + (left-right)*weights_hor + mla v21.8h, v3.8h, v7.8h // (left flipped) + mla v22.8h, v2.8h, v6.8h + mla v23.8h, v2.8h, v7.8h + mla v24.8h, v1.8h, v6.8h + mla v25.8h, v1.8h, v7.8h + mla v26.8h, v0.8h, v6.8h + mla v27.8h, v0.8h, v7.8h + rshrn v20.8b, v20.8h, #8 + rshrn2 v20.16b, v21.8h, #8 + rshrn v22.8b, v22.8h, #8 + rshrn2 v22.16b, v23.8h, #8 + rshrn v24.8b, v24.8h, #8 + rshrn2 v24.16b, v25.8h, #8 + rshrn v26.8b, v26.8h, #8 + rshrn2 v26.16b, v27.8h, #8 + subs w3, w3, #16 + st1 {v20.16b}, [x0], #16 + st1 {v22.16b}, [x6], #16 + st1 {v24.16b}, [x5], #16 + st1 {v26.16b}, [x10], #16 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x8, x8, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_h_tbl): + .hword L(ipred_smooth_h_tbl) - 640b + .hword L(ipred_smooth_h_tbl) - 320b + .hword L(ipred_smooth_h_tbl) - 160b + .hword L(ipred_smooth_h_tbl) - 80b + .hword L(ipred_smooth_h_tbl) - 40b +endfunc + +const padding_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +padding_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void ipred_z1_upsample_edge_8bpc_neon(pixel *out, const int hsz, +// const pixel *const in, const int end); +function ipred_z1_upsample_edge_8bpc_neon, export=1 + movrel x4, padding_mask + ld1 {v0.16b}, [x2] // in[] + add x5, x2, w3, uxtw // in[end] + sub x4, x4, w3, uxtw + + ld1r {v1.16b}, [x5] // padding + ld1 {v3.16b}, [x4] // padding_mask + + movi v31.8h, #9 + + bit v0.16b, v1.16b, v3.16b // padded in[] + + ext v4.16b, v0.16b, v1.16b, #1 + ext v5.16b, v0.16b, v1.16b, #2 + ext v6.16b, v0.16b, v1.16b, #3 + + uaddl v16.8h, v4.8b, v5.8b // in[i+1] + in[i+2] + uaddl2 v17.8h, v4.16b, v5.16b + uaddl v18.8h, v0.8b, v6.8b // in[i+0] + in[i+3] + uaddl2 v19.8h, v0.16b, v6.16b + mul v16.8h, v16.8h, v31.8h // 9*(in[i+1] + in[i+2]) + mul v17.8h, v17.8h, v31.8h + sub v16.8h, v16.8h, v18.8h + sub v17.8h, v17.8h, v19.8h + + sqrshrun v16.8b, v16.8h, #4 + sqrshrun2 v16.16b, v17.8h, #4 + + zip1 v0.16b, v4.16b, v16.16b + zip2 v1.16b, v4.16b, v16.16b + + st1 {v0.16b, v1.16b}, [x0] + + ret +endfunc + +const edge_filter + .byte 0, 4, 8, 0 + .byte 0, 5, 6, 0 +// Leaving out the coeffs for strength=3 +// .byte 2, 4, 4, 0 +endconst + +// void ipred_z1_filter_edge_8bpc_neon(pixel *out, const int sz, +// const pixel *const in, const int end, +// const int strength); +function ipred_z1_filter_edge_8bpc_neon, export=1 + cmp w4, #3 + b.eq L(fivetap) // if (strength == 3) goto fivetap + + movrel x5, edge_filter, -3 + add x5, x5, w4, uxtw #2 // edge_filter + (strength - 1)*4 + 1 + + ld1 {v31.h}[0], [x5] // kernel[1-2] + + ld1 {v0.16b}, [x2], #16 + + dup v30.16b, v31.b[0] + dup v31.16b, v31.b[1] +1: + // in[end], is the last valid pixel. We produce 16 pixels out by + // using 18 pixels in - the last pixel used is [17] of the ones + // read/buffered. + cmp w3, #17 + ld1 {v1.16b}, [x2], #16 + b.lt 2f + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + umull v4.8h, v0.8b, v30.8b + umlal v4.8h, v2.8b, v31.8b + umlal v4.8h, v3.8b, v30.8b + umull2 v5.8h, v0.16b, v30.16b + umlal2 v5.8h, v2.16b, v31.16b + umlal2 v5.8h, v3.16b, v30.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + rshrn v4.8b, v4.8h, #4 + rshrn2 v4.16b, v5.8h, #4 + sub w3, w3, #16 + st1 {v4.16b}, [x0], #16 + b.gt 1b + ret +2: + // Right padding + + // x2[w3-32] is the padding pixel (x2 points 32 bytes ahead) + movrel x5, padding_mask + sub w6, w3, #32 + sub x5, x5, w3, uxtw + add x6, x2, w6, sxtw + + ld1 {v2.16b}, [x5] // padding_mask + + ld1r {v1.16b}, [x6] + bit v0.16b, v1.16b, v2.16b // Pad v0-v1 + + // Filter one block + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + umull v4.8h, v0.8b, v30.8b + umlal v4.8h, v2.8b, v31.8b + umlal v4.8h, v3.8b, v30.8b + umull2 v5.8h, v0.16b, v30.16b + umlal2 v5.8h, v2.16b, v31.16b + umlal2 v5.8h, v3.16b, v30.16b + subs w1, w1, #16 + rshrn v4.8b, v4.8h, #4 + rshrn2 v4.16b, v5.8h, #4 + st1 {v4.16b}, [x0], #16 + b.le 9f +5: + // After one block, any remaining output would only be filtering + // padding - thus just store the padding. + subs w1, w1, #16 + st1 {v1.16b}, [x0], #16 + b.gt 5b +9: + ret + +L(fivetap): + sub x2, x2, #1 // topleft -= 1 + movi v29.16b, #2 + ld1 {v0.16b}, [x2], #16 + movi v30.16b, #4 + movi v31.16b, #4 + ins v0.b[0], v0.b[1] +1: + // in[end+1], is the last valid pixel. We produce 16 pixels out by + // using 20 pixels in - the last pixel used is [19] of the ones + // read/buffered. + cmp w3, #18 + ld1 {v1.16b}, [x2], #16 + b.lt 2f // if (end + 1 < 19) + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v0.16b, v1.16b, #3 + ext v5.16b, v0.16b, v1.16b, #4 + umull v6.8h, v0.8b, v29.8b + umlal v6.8h, v2.8b, v30.8b + umlal v6.8h, v3.8b, v31.8b + umlal v6.8h, v4.8b, v30.8b + umlal v6.8h, v5.8b, v29.8b + umull2 v7.8h, v0.16b, v29.16b + umlal2 v7.8h, v2.16b, v30.16b + umlal2 v7.8h, v3.16b, v31.16b + umlal2 v7.8h, v4.16b, v30.16b + umlal2 v7.8h, v5.16b, v29.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + rshrn v6.8b, v6.8h, #4 + rshrn2 v6.16b, v7.8h, #4 + sub w3, w3, #16 + st1 {v6.16b}, [x0], #16 + b.gt 1b + ret +2: + // Right padding + + // x2[w3+1-32] is the padding pixel (x2 points 32 bytes ahead) + movrel x5, padding_mask, -1 + sub w6, w3, #31 + sub x5, x5, w3, uxtw + add x6, x2, w6, sxtw + + ld1 {v2.16b, v3.16b}, [x5] // padding_mask + + ld1r {v28.16b}, [x6] + bit v0.16b, v28.16b, v2.16b // Pad v0-v1 + bit v1.16b, v28.16b, v3.16b +4: + // Filter one block + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v0.16b, v1.16b, #3 + ext v5.16b, v0.16b, v1.16b, #4 + umull v6.8h, v0.8b, v29.8b + umlal v6.8h, v2.8b, v30.8b + umlal v6.8h, v3.8b, v31.8b + umlal v6.8h, v4.8b, v30.8b + umlal v6.8h, v5.8b, v29.8b + umull2 v7.8h, v0.16b, v29.16b + umlal2 v7.8h, v2.16b, v30.16b + umlal2 v7.8h, v3.16b, v31.16b + umlal2 v7.8h, v4.16b, v30.16b + umlal2 v7.8h, v5.16b, v29.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + mov v1.16b, v28.16b + rshrn v6.8b, v6.8h, #4 + rshrn2 v6.16b, v7.8h, #4 + sub w3, w3, #16 + st1 {v6.16b}, [x0], #16 + b.le 9f + // v0-v1[w3+1] is the last valid pixel; if (w3 + 1 > 0) we need to + // filter properly once more - aka (w3 >= 0). + cmp w3, #0 + b.ge 4b +5: + // When w3 <= 0, all remaining pixels in v0-v1 are equal to the + // last valid pixel - thus just output that without filtering. + subs w1, w1, #16 + st1 {v1.16b}, [x0], #16 + b.gt 5b +9: + ret +endfunc + +// void ipred_pixel_set_8bpc_neon(pixel *out, const pixel px, +// const int n); +function ipred_pixel_set_8bpc_neon, export=1 + dup v0.16b, w1 +1: + subs w2, w2, #16 + st1 {v0.16b}, [x0], #16 + b.gt 1b + ret +endfunc + +// void ipred_z1_fill1_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const top, +// const int width, const int height, +// const int dx, const int max_base_x); +function ipred_z1_fill1_8bpc_neon, export=1 + clz w9, w3 + adr x8, L(ipred_z1_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw // top[max_base_x] + sub x8, x8, w9, uxtw + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + br x8 +40: + AARCH64_VALID_JUMP_TARGET +4: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + ldr d0, [x2, w8, uxtw] // top[base] + ldr d2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + ext v1.8b, v0.8b, v0.8b, #1 // top[base+1] + ext v3.8b, v2.8b, v2.8b, #1 + usubl v6.8h, v1.8b, v0.8b // top[base+1]-top[base] + usubl v7.8h, v3.8b, v2.8b + ushll v16.8h, v0.8b, #6 // top[base]*64 + ushll v17.8h, v2.8b, #6 + mla v16.4h, v6.4h, v4.4h // + top[base+1]*frac + mla v17.4h, v7.4h, v5.4h + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.s}[0], [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.s}[0], [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v31.s}[0], [x0], x1 + b.gt 49b + ret + +80: + AARCH64_VALID_JUMP_TARGET +8: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.8b, w9 // frac + dup v5.8b, w11 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8b, w9 // 64 - frac + dup v7.8b, w11 + ext v1.16b, v0.16b, v0.16b, #1 // top[base+1] + ext v3.16b, v2.16b, v2.16b, #1 + umull v16.8h, v0.8b, v6.8b // top[base]*(64-frac) + umlal v16.8h, v1.8b, v4.8b // + top[base+1]*frac + umull v17.8h, v2.8b, v7.8b + umlal v17.8h, v3.8b, v5.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8b}, [x0], x1 + b.gt 89b + ret + +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + + mov w12, w3 + + add x13, x0, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 169f + add x8, x2, w8, uxtw + add x10, x2, w10, uxtw + dup v4.16b, w9 // frac + dup v5.16b, w11 + ld1 {v0.16b, v1.16b}, [x8], #32 // top[base] + ld1 {v2.16b, v3.16b}, [x10], #32 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.16b, w9 // 64 - frac + dup v7.16b, w11 + add w7, w7, w5 // xpos += dx +2: + ext v16.16b, v0.16b, v1.16b, #1 // top[base+1] + ext v17.16b, v2.16b, v3.16b, #1 + subs w3, w3, #16 + umull v18.8h, v0.8b, v6.8b // top[base]*(64-frac) + umlal v18.8h, v16.8b, v4.8b // + top[base+1]*frac + umull2 v19.8h, v0.16b, v6.16b + umlal2 v19.8h, v16.16b, v4.16b + umull v20.8h, v2.8b, v7.8b + umlal v20.8h, v17.8b, v5.8b + umull2 v21.8h, v2.16b, v7.16b + umlal2 v21.8h, v17.16b, v5.16b + rshrn v16.8b, v18.8h, #6 + rshrn2 v16.16b, v19.8h, #6 + rshrn v17.8b, v20.8h, #6 + rshrn2 v17.16b, v21.8h, #6 + st1 {v16.16b}, [x0], #16 + st1 {v17.16b}, [x13], #16 + b.le 3f + mov v0.16b, v1.16b + ld1 {v1.16b}, [x8], #16 // top[base] + mov v2.16b, v3.16b + ld1 {v3.16b}, [x10], #16 + b 2b + +3: + subs w4, w4, #2 + b.le 9f + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 1b +9: + ret + +169: + st1 {v31.16b}, [x0], #16 + subs w3, w3, #16 + st1 {v31.16b}, [x13], #16 + b.gt 169b + subs w4, w4, #2 + b.le 9b + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 169b + +L(ipred_z1_fill1_tbl): + .hword L(ipred_z1_fill1_tbl) - 640b + .hword L(ipred_z1_fill1_tbl) - 320b + .hword L(ipred_z1_fill1_tbl) - 160b + .hword L(ipred_z1_fill1_tbl) - 80b + .hword L(ipred_z1_fill1_tbl) - 40b +endfunc + +function ipred_z1_fill2_8bpc_neon, export=1 + cmp w3, #8 + add x10, x2, w6, uxtw // top[max_base_x] + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + b.eq 8f + +4: // w == 4 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + ldr d0, [x2, w8, uxtw] // top[base] + ldr d2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + uzp2 v1.8b, v0.8b, v0.8b // top[base+1] + uzp1 v0.8b, v0.8b, v0.8b // top[base] + uzp2 v3.8b, v2.8b, v2.8b + uzp1 v2.8b, v2.8b, v2.8b + usubl v6.8h, v1.8b, v0.8b // top[base+1]-top[base] + usubl v7.8h, v3.8b, v2.8b + ushll v16.8h, v0.8b, #6 // top[base]*64 + ushll v17.8h, v2.8b, #6 + mla v16.4h, v6.4h, v4.4h // + top[base+1]*frac + mla v17.4h, v7.4h, v5.4h + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.s}[0], [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.s}[0], [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v31.s}[0], [x0], x1 + b.gt 49b + ret + +8: // w == 8 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.8b, w9 // frac + dup v5.8b, w11 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8b, w9 // 64 - frac + dup v7.8b, w11 + uzp2 v1.16b, v0.16b, v0.16b // top[base+1] + uzp1 v0.16b, v0.16b, v0.16b // top[base] + uzp2 v3.16b, v2.16b, v2.16b + uzp1 v2.16b, v2.16b, v2.16b + umull v16.8h, v1.8b, v4.8b // top[base+1]*frac + umlal v16.8h, v0.8b, v6.8b // + top[base]*(64-frac) + umull v17.8h, v3.8b, v5.8b + umlal v17.8h, v2.8b, v7.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8b}, [x0], x1 + b.gt 89b + ret +endfunc + +// void ipred_reverse_8bpc_neon(pixel *dst, const pixel *const src, +// const int n); +function ipred_reverse_8bpc_neon, export=1 + sub x1, x1, #16 + add x3, x0, #8 + mov x4, #16 +1: + ld1 {v0.16b}, [x1] + subs w2, w2, #16 + rev64 v0.16b, v0.16b + sub x1, x1, #16 + st1 {v0.d}[1], [x0], x4 + st1 {v0.d}[0], [x3], x4 + b.gt 1b + ret +endfunc + +const increments + .short 0, 1, 2, 3, 4, 5, 6, 7 +endconst + +// void ipred_z3_fill1_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const left, +// const int width, const int height, +// const int dy, const int max_base_y); +function ipred_z3_fill1_8bpc_neon, export=1 + cmp w6, #64 + clz w9, w3 + adr x8, L(ipred_z3_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw // left[max_base_y] + sub x8, x8, w9, uxtw + movrel x11, increments + ld1r {v31.16b}, [x10] // padding + ld1 {v30.8h}, [x11] // increments + mov w7, w5 + b.gt L(ipred_z3_fill1_large_h16) + br x8 + +40: + AARCH64_VALID_JUMP_TARGET + dup v29.4h, w5 // dy + + mul v30.4h, v30.4h, v29.4h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + // Worst case max_base_y is width+height-1, for w=4, h=16, <= 32 + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.4h, v29.4h, v30.4h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + mov v4.8b, v31.8b + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + + tbx v4.8b, {v0.16b, v1.16b}, v26.8b // left[base] + + trn1 v27.2s, v27.2s, v28.2s // base + 1, base + 2 + trn1 v24.2s, v24.2s, v24.2s // frac + trn1 v25.2s, v25.2s, v25.2s // 64 - frac +1: + mov v5.8b, v31.8b + tbx v5.8b, {v0.16b, v1.16b}, v27.8b // left[base+1], left[base+2] + + trn1 v4.2s, v4.2s, v5.2s // left[base], left[base+1] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + rshrn v16.8b, v16.8h, #6 + st1 {v16.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v16.s}[1], [x0], x1 + b.le 9f + + ext v4.8b, v5.8b, v5.8b, #4 + uqadd v27.8b, v27.8b, v21.8b // base += 2 + b 1b + +9: + ret + +80: + AARCH64_VALID_JUMP_TARGET + dup v29.8h, w5 // dy + + mul v30.8h, v30.8h, v29.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + // Worst case max_base_y is width+height-1, for w=8, h=32, <= 48 + ld1 {v0.16b, v1.16b, v2.16b}, [x2] // left[] + add v30.8h, v29.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + mov v4.8b, v31.8b + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + + tbx v4.8b, {v0.16b, v1.16b, v2.16b}, v26.8b // left[base] +1: + mov v5.8b, v31.8b + mov v6.8b, v31.8b + tbx v5.8b, {v0.16b, v1.16b, v2.16b}, v27.8b // left[base+1] + tbx v6.8b, {v0.16b, v1.16b, v2.16b}, v28.8b // left[base+2] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull v17.8h, v5.8b, v25.8b + umlal v17.8h, v6.8b, v24.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.le 9f + + mov v4.8b, v6.8b + uqadd v27.8b, v27.8b, v21.8b // base += 2 + uqadd v28.8b, v28.8b, v21.8b // base += 2 + b 1b + +9: + ret + +160: + AARCH64_VALID_JUMP_TARGET + dup v28.8h, w5 // dy + + shl v29.8h, v28.8h, #3 // 8*dy + mul v30.8h, v30.8h, v28.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + // This is only executed if we've checked that max_base_y <= 64. + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] // left[] + add v28.8h, v28.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + add v29.8h, v28.8h, v29.8h // ypos + 8*dy + + xtn v24.8b, v28.8h // (uint8_t)ypos + xtn2 v24.16b, v29.8h + uqshrn v26.8b, v28.8h, #6 // base + uqshrn2 v26.16b, v29.8h, #6 + and v24.16b, v24.16b, v23.16b // frac + + mov v4.16b, v31.16b + uqadd v27.16b, v26.16b, v20.16b // base + 1 + uqadd v28.16b, v26.16b, v21.16b // base + 2 + sub v25.16b, v22.16b, v24.16b // 64 - frac + + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v26.16b // left[base] +1: + mov v5.16b, v31.16b + mov v6.16b, v31.16b + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v27.16b // left[base+1] + tbx v6.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v28.16b // left[base+2] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + umull v18.8h, v5.8b, v25.8b + umlal v18.8h, v6.8b, v24.8b + umull2 v19.8h, v5.16b, v25.16b + umlal2 v19.8h, v6.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn2 v16.16b, v17.8h, #6 + rshrn v17.8b, v18.8h, #6 + rshrn2 v17.16b, v19.8h, #6 + st1 {v16.16b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.16b}, [x0], x1 + b.le 9f + + mov v4.16b, v6.16b + uqadd v27.16b, v27.16b, v21.16b // base += 2 + uqadd v28.16b, v28.16b, v21.16b // base += 2 + b 1b + +9: + ret +320: +640: + AARCH64_VALID_JUMP_TARGET + dup v28.8h, w5 // dy + mov w12, w3 + + add x13, x0, x1 + + shl v29.8h, v28.8h, #3 // 8*dy + mul v30.8h, v30.8h, v28.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + lsl x1, x1, #1 + sub x1, x1, w3, uxtw + add v30.8h, v28.8h, v30.8h // ypos + + // This is only executed if we've checked that max_base_y <= 64. + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] // left[] + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + +1: + mov v26.16b, v30.16b // reset ypos + +2: + add v27.8h, v26.8h, v29.8h // ypos + 8*dy + uqshrn v16.8b, v26.8h, #6 // base + uqshrn2 v16.16b, v27.8h, #6 + xtn v24.8b, v26.8h // (uint8_t)ypos + xtn2 v24.16b, v27.8h + umov w14, v16.b[0] + and v24.16b, v24.16b, v23.16b // frac + + uqadd v17.16b, v16.16b, v20.16b // base + 1 + cmp w14, w6 // base >= max_base_y + uqadd v18.16b, v16.16b, v21.16b // base + 2 + sub v25.16b, v22.16b, v24.16b // 64 - frac + + b.ge 4f + + mov v4.16b, v31.16b + mov v5.16b, v31.16b + mov v6.16b, v31.16b + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v16.16b // left[base] + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v17.16b // left[base+1] + tbx v6.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v18.16b // left[base+2] + + subs w3, w3, #16 + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + umull v18.8h, v5.8b, v25.8b + umlal v18.8h, v6.8b, v24.8b + umull2 v19.8h, v5.16b, v25.16b + umlal2 v19.8h, v6.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn2 v16.16b, v17.8h, #6 + rshrn v17.8b, v18.8h, #6 + rshrn2 v17.16b, v19.8h, #6 + st1 {v16.16b}, [x0], #16 + st1 {v17.16b}, [x13], #16 + b.le 3f + add v26.8h, v27.8h, v29.8h // ypos += 16*dy + b 2b + +3: + subs w4, w4, #2 + b.le 9f + movi v16.8h, #128 + add x0, x0, x1 + add x13, x13, x1 + add v30.8h, v30.8h, v16.8h // ypos = dy + y*(1<<6)*2 + mov w3, w12 + b 1b + +4: + subs w3, w3, #16 + st1 {v31.16b}, [x0], #16 + st1 {v31.16b}, [x13], #16 + b.gt 4b + b 3b + +9: + ret + +L(ipred_z3_fill1_large_h16): + // Fallback case for max_base_y > 64; similar to the z1 + // implementation. This does the filtering vertically, filling out + // a 2x pixel column at a time. + mov w15, #64 + add x13, x0, x1 + lsl x1, x1, #1 + + mov w12, w4 +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // ypos += dy + cmp w8, w6 // base >= max_base_y + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + add x8, x2, w8, uxtw + add x10, x2, w10, uxtw + dup v4.16b, w9 // frac + dup v5.16b, w11 + ld1 {v0.16b, v1.16b}, [x8], #32 // left[base] + ld1 {v2.16b, v3.16b}, [x10], #32 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.16b, w9 // 64 - frac + dup v7.16b, w11 + add w7, w7, w5 // ypos += dy +2: + ext v16.16b, v0.16b, v1.16b, #1 // left[base+1] + ext v17.16b, v2.16b, v3.16b, #1 + subs w4, w4, #16 + umull v18.8h, v16.8b, v4.8b // left[base+1]*frac + umlal v18.8h, v0.8b, v6.8b // + left[base]*(64-frac) + umull2 v19.8h, v16.16b, v4.16b + umlal2 v19.8h, v0.16b, v6.16b + umull v20.8h, v17.8b, v5.8b + umlal v20.8h, v2.8b, v7.8b + umull2 v21.8h, v17.16b, v5.16b + umlal2 v21.8h, v2.16b, v7.16b + rshrn v16.8b, v18.8h, #6 + rshrn2 v16.16b, v19.8h, #6 + rshrn v17.8b, v20.8h, #6 + rshrn2 v17.16b, v21.8h, #6 + zip1 v18.16b, v16.16b, v17.16b + zip2 v19.16b, v16.16b, v17.16b + st1 {v18.h}[0], [x0], x1 + st1 {v18.h}[1], [x13], x1 + st1 {v18.h}[2], [x0], x1 + st1 {v18.h}[3], [x13], x1 + st1 {v18.h}[4], [x0], x1 + st1 {v18.h}[5], [x13], x1 + st1 {v18.h}[6], [x0], x1 + st1 {v18.h}[7], [x13], x1 + st1 {v19.h}[0], [x0], x1 + st1 {v19.h}[1], [x13], x1 + st1 {v19.h}[2], [x0], x1 + st1 {v19.h}[3], [x13], x1 + st1 {v19.h}[4], [x0], x1 + st1 {v19.h}[5], [x13], x1 + st1 {v19.h}[6], [x0], x1 + st1 {v19.h}[7], [x13], x1 + b.le 3f + mov v0.16b, v1.16b + ld1 {v1.16b}, [x8], #16 // left[base] + mov v2.16b, v3.16b + ld1 {v3.16b}, [x10], #16 + b 2b + +3: + subs w3, w3, #2 + b.le 9f + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + lsl x1, x1, #1 + add x0, x0, #2 + add x13, x13, #2 + mov w4, w12 + b 1b +9: + ret + +L(ipred_z3_fill1_tbl): + .hword L(ipred_z3_fill1_tbl) - 640b + .hword L(ipred_z3_fill1_tbl) - 320b + .hword L(ipred_z3_fill1_tbl) - 160b + .hword L(ipred_z3_fill1_tbl) - 80b + .hword L(ipred_z3_fill1_tbl) - 40b +endfunc + +function ipred_z3_fill_padding_neon, export=0 + cmp w3, #16 + adr x8, L(ipred_z3_fill_padding_tbl) + b.gt L(ipred_z3_fill_padding_wide) + // w3 = remaining width, w4 = constant height + mov w12, w4 + +1: + // Fill a WxH rectangle with padding. W can be any number; + // this fills the exact width by filling in the largest + // power of two in the remaining width, and repeating. + clz w9, w3 + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + sub x9, x8, w9, uxtw + br x9 + +2: + st1 {v31.h}[0], [x0], x1 + subs w4, w4, #4 + st1 {v31.h}[0], [x13], x1 + st1 {v31.h}[0], [x0], x1 + st1 {v31.h}[0], [x13], x1 + b.gt 2b + subs w3, w3, #2 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #2 + add x13, x13, #2 + mov w4, w12 + b 1b + +4: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #4 + st1 {v31.s}[0], [x13], x1 + st1 {v31.s}[0], [x0], x1 + st1 {v31.s}[0], [x13], x1 + b.gt 4b + subs w3, w3, #4 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #4 + add x13, x13, #4 + mov w4, w12 + b 1b + +8: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #4 + st1 {v31.8b}, [x13], x1 + st1 {v31.8b}, [x0], x1 + st1 {v31.8b}, [x13], x1 + b.gt 4b + subs w3, w3, #8 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #8 + add x13, x13, #8 + mov w4, w12 + b 1b + +16: +32: +64: + st1 {v31.16b}, [x0], x1 + subs w4, w4, #4 + st1 {v31.16b}, [x13], x1 + st1 {v31.16b}, [x0], x1 + st1 {v31.16b}, [x13], x1 + b.gt 4b + subs w3, w3, #16 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #16 + add x13, x13, #16 + mov w4, w12 + b 1b + +9: + ret + +L(ipred_z3_fill_padding_tbl): + .hword L(ipred_z3_fill_padding_tbl) - 64b + .hword L(ipred_z3_fill_padding_tbl) - 32b + .hword L(ipred_z3_fill_padding_tbl) - 16b + .hword L(ipred_z3_fill_padding_tbl) - 8b + .hword L(ipred_z3_fill_padding_tbl) - 4b + .hword L(ipred_z3_fill_padding_tbl) - 2b + +L(ipred_z3_fill_padding_wide): + // Fill a WxH rectangle with padding, with W > 16. + lsr x1, x1, #1 + mov w12, w3 + sub x1, x1, w3, uxtw +1: + ands w5, w3, #15 + b.eq 2f + // If the width isn't aligned to 16, first do one 16 byte write + // and align the start pointer. + sub w3, w3, w5 + st1 {v31.16b}, [x0] + add x0, x0, w5, uxtw +2: + // Fill the rest of the line with aligned 16 byte writes. + subs w3, w3, #16 + st1 {v31.16b}, [x0], #16 + b.gt 2b + subs w4, w4, #1 + add x0, x0, x1 + b.le 9f + mov w3, w12 + b 1b +9: + ret +endfunc + +function ipred_z3_fill2_8bpc_neon, export=1 + cmp w3, #8 + add x10, x2, w6, uxtw // left[max_base_y] + movrel x11, increments + ld1r {v31.16b}, [x10] // padding + ld1 {v30.8h}, [x11] // increments + b.eq 80f + +40: // w == 4 + dup v29.4h, w5 // dy + + mul v30.4h, v30.4h, v29.4h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + // Worst case max_base_y is 2*(width+height)-2, but width+height <= 16, + // so max_base_y <= 32. + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.4h, v29.4h, v30.4h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + uqadd v29.8b, v27.8b, v21.8b // base + 3 + + trn1 v24.2s, v24.2s, v24.2s // frac + trn1 v26.2s, v26.2s, v28.2s // base + 0, base + 2 + trn1 v27.2s, v27.2s, v29.2s // base + 1, base + 3 + trn1 v25.2s, v25.2s, v25.2s // 64 - frac + + movi v21.16b, #4 +1: + mov v4.8b, v31.8b + mov v5.8b, v31.8b + tbx v4.8b, {v0.16b, v1.16b}, v26.8b // left[base], left[base+2] + tbx v5.8b, {v0.16b, v1.16b}, v27.8b // left[base+1], left[base+3] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + rshrn v16.8b, v16.8h, #6 + st1 {v16.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v16.s}[1], [x0], x1 + b.le 9f + + uqadd v26.8b, v26.8b, v21.8b // base += 4 + uqadd v27.8b, v27.8b, v21.8b // base += 4 + b 1b + +9: + ret + +80: // w == 8 + dup v29.8h, w5 // dy + + mul v30.8h, v30.8h, v29.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + // Worst case max_base_y is 2*(width+height)-2, but width+height <= 16, + // so max_base_y <= 32. + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.8h, v29.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + uqadd v29.8b, v27.8b, v21.8b // base + 3 + + trn1 v24.2d, v24.2d, v24.2d // frac + trn1 v26.2d, v26.2d, v28.2d // base + 0, base + 2 + trn1 v27.2d, v27.2d, v29.2d // base + 1, base + 3 + trn1 v25.2d, v25.2d, v25.2d // 64 - frac + + movi v21.16b, #4 +1: + mov v4.16b, v31.16b + mov v5.16b, v31.16b + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v26.16b // left[base], left[base+2] + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v27.16b // left[base+1], left[base+3] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.le 9f + + uqadd v26.16b, v26.16b, v21.16b // base += 4 + uqadd v27.16b, v27.16b, v21.16b // base += 4 + b 1b + +9: + ret +endfunc + + +// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height); +function ipred_filter_8bpc_neon, export=1 + and w5, w5, #511 + movrel x6, X(filter_intra_taps) + lsl w5, w5, #6 + add x6, x6, w5, uxtw + ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 + clz w9, w3 + adr x5, L(ipred_filter_tbl) + ld1 {v20.8b, v21.8b, v22.8b}, [x6] + sub w9, w9, #26 + ldrh w9, [x5, w9, uxtw #1] + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + sub x5, x5, w9, uxtw + sxtl v18.8h, v18.8b + sxtl v19.8h, v19.8b + add x6, x0, x1 + lsl x1, x1, #1 + sxtl v20.8h, v20.8b + sxtl v21.8h, v21.8b + sxtl v22.8h, v22.8b + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ldur s0, [x2, #1] // top (0-3) + sub x2, x2, #2 + mov x7, #-2 + uxtl v0.8h, v0.8b // top (0-3) +4: + ld1 {v1.s}[0], [x2], x7 // left (0-1) + topleft (2) + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + uxtl v1.8h, v1.8b // left (0-1) + topleft (2) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + sqrshrun v2.8b, v2.8h, #4 + subs w4, w4, #2 + st1 {v2.s}[0], [x0], x1 + uxtl v0.8h, v2.8b + st1 {v2.s}[1], [x6], x1 + ext v0.16b, v0.16b, v0.16b, #8 // move top from [4-7] to [0-3] + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ldur d0, [x2, #1] // top (0-7) + sub x2, x2, #2 + mov x7, #-2 + uxtl v0.8h, v0.8b // top (0-7) +8: + ld1 {v1.s}[0], [x2], x7 // left (0-1) + topleft (2) + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + uxtl v1.8h, v1.8b // left (0-1) + topleft (2) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + mul v3.8h, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + sqrshrun v2.8b, v2.8h, #4 + uxtl v1.8h, v2.8b // first block, in 16 bit + mla v3.8h, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + mla v3.8h, v16.8h, v0.h[3] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v1.h[3] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v1.h[7] // p6(left[1]) * filter(6) + sqrshrun v3.8b, v3.8h, #4 + subs w4, w4, #2 + st2 {v2.s, v3.s}[0], [x0], x1 + zip2 v0.2s, v2.2s, v3.2s + st2 {v2.s, v3.s}[1], [x6], x1 + uxtl v0.8h, v0.8b + b.gt 8b + ret +160: +320: + AARCH64_VALID_JUMP_TARGET + add x8, x2, #1 + sub x2, x2, #2 + mov x7, #-2 + sub x1, x1, w3, uxtw + mov w9, w3 + +1: + ld1 {v0.s}[0], [x2], x7 // left (0-1) + topleft (2) + uxtl v0.8h, v0.8b // left (0-1) + topleft (2) +2: + ld1 {v2.16b}, [x8], #16 // top(0-15) + mul v3.8h, v16.8h, v0.h[2] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + uxtl v1.8h, v2.8b // top(0-7) + uxtl2 v2.8h, v2.16b // top(8-15) + mla v3.8h, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + mla v3.8h, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + mla v3.8h, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + mul v4.8h, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + mla v4.8h, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + mla v4.8h, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + sqrshrun v3.8b, v3.8h, #4 + uxtl v0.8h, v3.8b // first block, in 16 bit + mla v4.8h, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + mla v4.8h, v16.8h, v1.h[3] // p0(topleft) * filter(0) + mla v4.8h, v21.8h, v0.h[3] // p5(left[0]) * filter(5) + mla v4.8h, v22.8h, v0.h[7] // p6(left[1]) * filter(6) + + mul v5.8h, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + mla v5.8h, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + mla v5.8h, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + sqrshrun v4.8b, v4.8h, #4 + uxtl v0.8h, v4.8b // second block, in 16 bit + mla v5.8h, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + mla v5.8h, v16.8h, v1.h[7] // p0(topleft) * filter(0) + mla v5.8h, v21.8h, v0.h[3] // p5(left[0]) * filter(5) + mla v5.8h, v22.8h, v0.h[7] // p6(left[1]) * filter(6) + + mul v6.8h, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + mla v6.8h, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + mla v6.8h, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + sqrshrun v5.8b, v5.8h, #4 + uxtl v0.8h, v5.8b // third block, in 16 bit + mla v6.8h, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + mla v6.8h, v16.8h, v2.h[3] // p0(topleft) * filter(0) + mla v6.8h, v21.8h, v0.h[3] // p5(left[0]) * filter(5) + mla v6.8h, v22.8h, v0.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + sqrshrun v6.8b, v6.8h, #4 + + st4 {v3.s, v4.s, v5.s, v6.s}[0], [x0], #16 + st4 {v3.s, v4.s, v5.s, v6.s}[1], [x6], #16 + b.le 8f + ins v0.h[2], v2.h[7] + ins v0.b[0], v6.b[7] + ins v0.b[2], v6.b[3] + b 2b +8: + subs w4, w4, #2 + b.le 9f + sub x8, x6, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_filter_tbl): + .hword L(ipred_filter_tbl) - 320b + .hword L(ipred_filter_tbl) - 160b + .hword L(ipred_filter_tbl) - 80b + .hword L(ipred_filter_tbl) - 40b +endfunc + +// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_8bpc_neon, export=1 + ld1 {v0.8h}, [x2] + clz w9, w4 + adr x6, L(pal_pred_tbl) + sub w9, w9, #25 + ldrh w9, [x6, w9, uxtw #1] + xtn v0.8b, v0.8h + sub x6, x6, w9, uxtw + add x2, x0, x1 + lsl x1, x1, #1 + br x6 +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b}, [x3], #16 + subs w5, w5, #4 + tbl v1.16b, {v0.16b}, v1.16b + st1 {v1.s}[0], [x0], x1 + st1 {v1.s}[1], [x2], x1 + st1 {v1.s}[2], [x0], x1 + st1 {v1.s}[3], [x2], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b, v2.16b}, [x3], #32 + subs w5, w5, #4 + tbl v1.16b, {v0.16b}, v1.16b + st1 {v1.d}[0], [x0], x1 + tbl v2.16b, {v0.16b}, v2.16b + st1 {v1.d}[1], [x2], x1 + st1 {v2.d}[0], [x0], x1 + st1 {v2.d}[1], [x2], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b, v2.16b, v3.16b, v4.16b}, [x3], #64 + subs w5, w5, #4 + tbl v1.16b, {v0.16b}, v1.16b + tbl v2.16b, {v0.16b}, v2.16b + st1 {v1.16b}, [x0], x1 + tbl v3.16b, {v0.16b}, v3.16b + st1 {v2.16b}, [x2], x1 + tbl v4.16b, {v0.16b}, v4.16b + st1 {v3.16b}, [x0], x1 + st1 {v4.16b}, [x2], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64 + ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x3], #64 + subs w5, w5, #4 + tbl v16.16b, {v0.16b}, v16.16b + tbl v17.16b, {v0.16b}, v17.16b + tbl v18.16b, {v0.16b}, v18.16b + tbl v19.16b, {v0.16b}, v19.16b + tbl v20.16b, {v0.16b}, v20.16b + st1 {v16.16b, v17.16b}, [x0], x1 + tbl v21.16b, {v0.16b}, v21.16b + st1 {v18.16b, v19.16b}, [x2], x1 + tbl v22.16b, {v0.16b}, v22.16b + st1 {v20.16b, v21.16b}, [x0], x1 + tbl v23.16b, {v0.16b}, v23.16b + st1 {v22.16b, v23.16b}, [x2], x1 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64 + ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x3], #64 + subs w5, w5, #2 + tbl v16.16b, {v0.16b}, v16.16b + tbl v17.16b, {v0.16b}, v17.16b + tbl v18.16b, {v0.16b}, v18.16b + tbl v19.16b, {v0.16b}, v19.16b + st1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x0], x1 + tbl v20.16b, {v0.16b}, v20.16b + tbl v21.16b, {v0.16b}, v21.16b + tbl v22.16b, {v0.16b}, v22.16b + tbl v23.16b, {v0.16b}, v23.16b + st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x2], x1 + b.gt 64b + ret + +L(pal_pred_tbl): + .hword L(pal_pred_tbl) - 64b + .hword L(pal_pred_tbl) - 32b + .hword L(pal_pred_tbl) - 16b + .hword L(pal_pred_tbl) - 8b + .hword L(pal_pred_tbl) - 4b +endfunc + +// void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_128_8bpc_neon, export=1 + clz w9, w3 + adr x7, L(ipred_cfl_128_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + movi v0.8h, #128 // dc + dup v1.8h, w6 // alpha + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x7 +L(ipred_cfl_splat_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h}, [x5], #32 + mul v2.8h, v2.8h, v1.8h // diff = ac * alpha + mul v3.8h, v3.8h, v1.8h + cmlt v4.8h, v2.8h, #0 // sign + cmlt v5.8h, v3.8h, #0 + add v2.8h, v2.8h, v4.8h // diff + sign + add v3.8h, v3.8h, v5.8h + srshr v2.8h, v2.8h, #6 // (diff + sign + 32) >> 6 = apply_sign() + srshr v3.8h, v3.8h, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + sqxtun v2.8b, v2.8h // iclip_pixel(dc + apply_sign()) + sqxtun v3.8b, v3.8h + st1 {v2.s}[0], [x0], x1 + st1 {v2.s}[1], [x6], x1 + subs w4, w4, #4 + st1 {v3.s}[0], [x0], x1 + st1 {v3.s}[1], [x6], x1 + b.gt L(ipred_cfl_splat_w4) + ret +L(ipred_cfl_splat_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x5], #64 + mul v2.8h, v2.8h, v1.8h // diff = ac * alpha + mul v3.8h, v3.8h, v1.8h + mul v4.8h, v4.8h, v1.8h + mul v5.8h, v5.8h, v1.8h + cmlt v16.8h, v2.8h, #0 // sign + cmlt v17.8h, v3.8h, #0 + cmlt v18.8h, v4.8h, #0 + cmlt v19.8h, v5.8h, #0 + add v2.8h, v2.8h, v16.8h // diff + sign + add v3.8h, v3.8h, v17.8h + add v4.8h, v4.8h, v18.8h + add v5.8h, v5.8h, v19.8h + srshr v2.8h, v2.8h, #6 // (diff + sign + 32) >> 6 = apply_sign() + srshr v3.8h, v3.8h, #6 + srshr v4.8h, v4.8h, #6 + srshr v5.8h, v5.8h, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + add v4.8h, v4.8h, v0.8h + add v5.8h, v5.8h, v0.8h + sqxtun v2.8b, v2.8h // iclip_pixel(dc + apply_sign()) + sqxtun v3.8b, v3.8h + sqxtun v4.8b, v4.8h + sqxtun v5.8b, v5.8h + st1 {v2.8b}, [x0], x1 + st1 {v3.8b}, [x6], x1 + subs w4, w4, #4 + st1 {v4.8b}, [x0], x1 + st1 {v5.8b}, [x6], x1 + b.gt L(ipred_cfl_splat_w8) + ret +L(ipred_cfl_splat_w16): + AARCH64_VALID_JUMP_TARGET + add x7, x5, w3, uxtw #1 + sub x1, x1, w3, uxtw + mov w9, w3 +1: + ld1 {v2.8h, v3.8h}, [x5], #32 + ld1 {v4.8h, v5.8h}, [x7], #32 + mul v2.8h, v2.8h, v1.8h // diff = ac * alpha + mul v3.8h, v3.8h, v1.8h + mul v4.8h, v4.8h, v1.8h + mul v5.8h, v5.8h, v1.8h + cmlt v16.8h, v2.8h, #0 // sign + cmlt v17.8h, v3.8h, #0 + cmlt v18.8h, v4.8h, #0 + cmlt v19.8h, v5.8h, #0 + add v2.8h, v2.8h, v16.8h // diff + sign + add v3.8h, v3.8h, v17.8h + add v4.8h, v4.8h, v18.8h + add v5.8h, v5.8h, v19.8h + srshr v2.8h, v2.8h, #6 // (diff + sign + 32) >> 6 = apply_sign() + srshr v3.8h, v3.8h, #6 + srshr v4.8h, v4.8h, #6 + srshr v5.8h, v5.8h, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + add v4.8h, v4.8h, v0.8h + add v5.8h, v5.8h, v0.8h + sqxtun v2.8b, v2.8h // iclip_pixel(dc + apply_sign()) + sqxtun v3.8b, v3.8h + sqxtun v4.8b, v4.8h + sqxtun v5.8b, v5.8h + subs w3, w3, #16 + st1 {v2.8b, v3.8b}, [x0], #16 + st1 {v4.8b, v5.8b}, [x6], #16 + b.gt 1b + subs w4, w4, #2 + add x5, x5, w9, uxtw #1 + add x7, x7, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b.gt 1b + ret + +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) +endfunc + +// void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_top_8bpc_neon, export=1 + clz w9, w3 + adr x7, L(ipred_cfl_top_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + dup v1.8h, w6 // alpha + add x2, x2, #1 + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x7 +4: + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2s}, [x2] + uaddlv h0, v0.8b + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2] + uaddlv h0, v0.8b + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) +16: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2] + uaddlv h0, v0.16b + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) +32: + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b, v3.16b}, [x2] + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add v2.4h, v2.4h, v3.4h + urshr v2.4h, v2.4h, #5 + dup v0.8h, v2.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_top_tbl): + .hword L(ipred_cfl_top_tbl) - 32b + .hword L(ipred_cfl_top_tbl) - 16b + .hword L(ipred_cfl_top_tbl) - 8b + .hword L(ipred_cfl_top_tbl) - 4b +endfunc + +// void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_left_8bpc_neon, export=1 + sub x2, x2, w4, uxtw + clz w9, w3 + clz w8, w4 + adr x10, L(ipred_cfl_splat_tbl) + adr x7, L(ipred_cfl_left_tbl) + sub w9, w9, #26 + sub w8, w8, #26 + ldrh w9, [x10, w9, uxtw #1] + ldrh w8, [x7, w8, uxtw #1] + dup v1.8h, w6 // alpha + sub x9, x10, w9, uxtw + sub x7, x7, w8, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x7 + +L(ipred_cfl_left_h4): + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2s}, [x2] + uaddlv h0, v0.8b + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2] + uaddlv h0, v0.8b + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2] + uaddlv h0, v0.16b + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b, v3.16b}, [x2] + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add v2.4h, v2.4h, v3.4h + urshr v2.4h, v2.4h, #5 + dup v0.8h, v2.h[0] + br x9 + +L(ipred_cfl_left_tbl): + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) +endfunc + +// void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_8bpc_neon, export=1 + sub x2, x2, w4, uxtw + add w8, w3, w4 // width + height + dup v1.8h, w6 // alpha + clz w9, w3 + clz w6, w4 + dup v16.8h, w8 // width + height + adr x7, L(ipred_cfl_tbl) + rbit w8, w8 // rbit(width + height) + sub w9, w9, #22 // 26 leading bits, minus table offset 4 + sub w6, w6, #26 + clz w8, w8 // ctz(width + height) + ldrh w9, [x7, w9, uxtw #1] + ldrh w6, [x7, w6, uxtw #1] + neg w8, w8 // -ctz(width + height) + sub x9, x7, w9, uxtw + sub x7, x7, w6, uxtw + ushr v16.8h, v16.8h, #1 // (width + height) >> 1 + dup v17.8h, w8 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + br x7 + +L(ipred_cfl_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x2], #4 + ins v0.s[1], wzr + add x2, x2, #1 + uaddlv h0, v0.8b + br x9 +L(ipred_cfl_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.s}[0], [x2] + ins v2.s[1], wzr + add v0.4h, v0.4h, v16.4h + uaddlv h2, v2.8b + cmp w4, #4 + add v0.4h, v0.4h, v2.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 8/16 + mov w16, #(0x3334/2) + movk w16, #(0x5556/2), lsl #16 + add w17, w4, w4 // w17 = 2*h = 16 or 32 + lsr w16, w16, w17 + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2], #8 + uaddlv h0, v0.8b + add x2, x2, #1 + br x9 +L(ipred_cfl_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h2, v2.8b + cmp w4, #8 + add v0.4h, v0.4h, v2.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #(0x3334/2) + mov w17, #(0x5556/2) + csel w16, w16, w17, eq + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x2], #16 + uaddlv h0, v0.16b + add x2, x2, #1 + br x9 +L(ipred_cfl_w16): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h2, v2.16b + cmp w4, #16 + add v0.4h, v0.4h, v2.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 4/8/32 + cmp w4, #4 + mov w16, #(0x3334/2) + mov w17, #(0x5556/2) + csel w16, w16, w17, eq + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b, v3.16b}, [x2], #32 + uaddlv h2, v2.16b + uaddlv h3, v3.16b + add x2, x2, #1 + add v0.4h, v2.4h, v3.4h + br x9 +L(ipred_cfl_w32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b, v3.16b}, [x2] + add v0.4h, v0.4h, v16.4h + uaddlv h2, v2.16b + uaddlv h3, v3.16b + cmp w4, #32 + add v0.4h, v0.4h, v2.4h + add v0.4h, v0.4h, v3.4h + ushl v0.4h, v0.4h, v17.4h + b.eq 1f + // h = 8/16 + mov w16, #(0x5556/2) + movk w16, #(0x3334/2), lsl #16 + add w17, w4, w4 // w17 = 2*h = 16 or 32 + lsr w16, w16, w17 + dup v16.4h, w16 + sqdmulh v0.4h, v0.4h, v16.4h +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_tbl): + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h4) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) +endfunc + +// void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_8bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_420_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_420_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input + ld1 {v0.8b}, [x1], x2 + ld1 {v1.8b}, [x10], x2 + ld1 {v0.d}[1], [x1], x2 + ld1 {v1.d}[1], [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v1.8h, v1.16b + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + subs w8, w8, #2 + st1 {v0.8h}, [x0], #16 + add v16.8h, v16.8h, v0.8h + b.gt 1b + trn2 v1.2d, v0.2d, v0.2d + trn2 v0.2d, v0.2d, v0.2d +L(ipred_cfl_ac_420_w4_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + b.gt 2b +3: + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + uaddlv s0, v0.8h // sum + sub x0, x0, w6, uxtw #3 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] +6: // Subtract dc from ac + ld1 {v0.8h, v1.8h}, [x0] + subs w6, w6, #4 + sub v0.8h, v0.8h, v4.8h + sub v1.8h, v1.8h, v4.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 6b + ret + +L(ipred_cfl_ac_420_w8): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.16b}, [x1], x2 + ld1 {v1.16b}, [x10], x2 + ld1 {v2.16b}, [x1], x2 + uaddlp v0.8h, v0.16b + ld1 {v3.16b}, [x10], x2 + uaddlp v1.8h, v1.16b + uaddlp v2.8h, v2.16b + uaddlp v3.8h, v3.16b + add v0.8h, v0.8h, v1.8h + add v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v2.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + b.gt 1b + mov v0.16b, v1.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8b}, [x1], x2 + ld1 {v1.8b}, [x10], x2 + ld1 {v0.d}[1], [x1], x2 + ld1 {v1.d}[1], [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v1.8h, v1.16b + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + dup v1.4h, v0.h[3] + dup v3.4h, v0.h[7] + trn2 v2.2d, v0.2d, v0.2d + subs w8, w8, #2 + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + add v16.4h, v16.4h, v0.4h + add v17.4h, v17.4h, v1.4h + add v18.4h, v18.4h, v2.4h + add v19.4h, v19.4h, v3.4h + b.gt 1b + trn1 v0.2d, v2.2d, v3.2d + trn1 v1.2d, v2.2d, v3.2d + +L(ipred_cfl_ac_420_w8_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + add v18.8h, v18.8h, v0.8h + add v19.8h, v19.8h, v1.8h + b.gt 2b +3: + +L(ipred_cfl_ac_420_w8_calc_subtract_dc): + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + add v2.8h, v18.8h, v19.8h + uaddlp v0.4s, v0.8h + uaddlp v2.4s, v2.8h + add v0.4s, v0.4s, v2.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #4 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] +L(ipred_cfl_ac_420_w8_subtract_dc): +6: // Subtract dc from ac + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + subs w6, w6, #4 + sub v0.8h, v0.8h, v4.8h + sub v1.8h, v1.8h, v4.8h + sub v2.8h, v2.8h, v4.8h + sub v3.8h, v3.8h, v4.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + b.gt 6b + ret + +L(ipred_cfl_ac_420_w16): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_420_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_420_w16_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, without padding + ld1 {v0.16b, v1.16b}, [x1], x2 + ld1 {v2.16b, v3.16b}, [x10], x2 + uaddlp v0.8h, v0.16b + ld1 {v4.16b, v5.16b}, [x1], x2 + uaddlp v1.8h, v1.16b + ld1 {v6.16b, v7.16b}, [x10], x2 + uaddlp v2.8h, v2.16b + uaddlp v3.8h, v3.16b + uaddlp v4.8h, v4.16b + uaddlp v5.8h, v5.16b + uaddlp v6.8h, v6.16b + uaddlp v7.8h, v7.16b + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + add v4.8h, v4.8h, v6.8h + add v5.8h, v5.8h, v7.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v1.8h, #1 + shl v2.8h, v4.8h, #1 + shl v3.8h, v5.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 4 + ldr d1, [x1, #16] + ld1 {v0.16b}, [x1], x2 + ldr d3, [x10, #16] + ld1 {v2.16b}, [x10], x2 + uaddlp v1.4h, v1.8b + ldr d5, [x1, #16] + uaddlp v0.8h, v0.16b + ld1 {v4.16b}, [x1], x2 + uaddlp v3.4h, v3.8b + ldr d7, [x10, #16] + uaddlp v2.8h, v2.16b + ld1 {v6.16b}, [x10], x2 + uaddlp v5.4h, v5.8b + uaddlp v4.8h, v4.16b + uaddlp v7.4h, v7.8b + uaddlp v6.8h, v6.16b + add v1.4h, v1.4h, v3.4h + add v0.8h, v0.8h, v2.8h + add v5.4h, v5.4h, v7.4h + add v4.8h, v4.8h, v6.8h + shl v1.4h, v1.4h, #1 + shl v0.8h, v0.8h, #1 + shl v3.4h, v5.4h, #1 + shl v2.8h, v4.8h, #1 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 8 + ld1 {v0.16b}, [x1], x2 + ld1 {v2.16b}, [x10], x2 + ld1 {v4.16b}, [x1], x2 + uaddlp v0.8h, v0.16b + ld1 {v6.16b}, [x10], x2 + uaddlp v2.8h, v2.16b + uaddlp v4.8h, v4.16b + uaddlp v6.8h, v6.16b + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v2.8h, v4.8h, #1 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 12 + ld1 {v0.8b}, [x1], x2 + ld1 {v2.8b}, [x10], x2 + ld1 {v4.8b}, [x1], x2 + uaddlp v0.4h, v0.8b + ld1 {v6.8b}, [x10], x2 + uaddlp v2.4h, v2.8b + uaddlp v4.4h, v4.8b + uaddlp v6.4h, v6.8b + add v0.4h, v0.4h, v2.4h + add v4.4h, v4.4h, v6.4h + shl v0.4h, v0.4h, #1 + shl v2.4h, v4.4h, #1 + dup v1.8h, v0.h[3] + dup v3.8h, v2.h[3] + trn1 v0.2d, v0.2d, v1.2d + trn1 v2.2d, v2.2d, v3.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + +L(ipred_cfl_ac_420_w16_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 2b +3: + + // Double the height and reuse the w8 summing/subtracting + lsl w6, w6, #1 + b L(ipred_cfl_ac_420_w8_calc_subtract_dc) + +L(ipred_cfl_ac_420_tbl): + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w4) + .hword 0 + +L(ipred_cfl_ac_420_w16_tbl): + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad0) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) +endfunc + +// void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_8bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_422_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_422_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input + ld1 {v0.8b}, [x1], x2 + ld1 {v0.d}[1], [x10], x2 + ld1 {v1.8b}, [x1], x2 + ld1 {v1.d}[1], [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v1.8h, v1.16b + shl v0.8h, v0.8h, #2 + shl v1.8h, v1.8h, #2 + subs w8, w8, #4 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.16b}, [x1], x2 + ld1 {v1.16b}, [x10], x2 + ld1 {v2.16b}, [x1], x2 + uaddlp v0.8h, v0.16b + ld1 {v3.16b}, [x10], x2 + uaddlp v1.8h, v1.16b + uaddlp v2.8h, v2.16b + uaddlp v3.8h, v3.16b + shl v0.8h, v0.8h, #2 + shl v1.8h, v1.8h, #2 + shl v2.8h, v2.8h, #2 + shl v3.8h, v3.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8b}, [x1], x2 + ld1 {v0.d}[1], [x10], x2 + ld1 {v2.8b}, [x1], x2 + ld1 {v2.d}[1], [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v2.8h, v2.16b + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v4.4h, v0.h[3] + dup v5.8h, v0.h[7] + dup v6.4h, v2.h[3] + dup v7.8h, v2.h[7] + trn2 v1.2d, v0.2d, v5.2d + trn1 v0.2d, v0.2d, v4.2d + trn2 v3.2d, v2.2d, v7.2d + trn1 v2.2d, v2.2d, v6.2d + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_422_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_422_w16_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, without padding + ld1 {v0.16b, v1.16b}, [x1], x2 + ld1 {v2.16b, v3.16b}, [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v1.8h, v1.16b + uaddlp v2.8h, v2.16b + uaddlp v3.8h, v3.16b + shl v0.8h, v0.8h, #2 + shl v1.8h, v1.8h, #2 + shl v2.8h, v2.8h, #2 + shl v3.8h, v3.8h, #2 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 4 + ldr d1, [x1, #16] + ld1 {v0.16b}, [x1], x2 + ldr d3, [x10, #16] + ld1 {v2.16b}, [x10], x2 + uaddlp v1.4h, v1.8b + uaddlp v0.8h, v0.16b + uaddlp v3.4h, v3.8b + uaddlp v2.8h, v2.16b + shl v1.4h, v1.4h, #2 + shl v0.8h, v0.8h, #2 + shl v3.4h, v3.4h, #2 + shl v2.8h, v2.8h, #2 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 8 + ld1 {v0.16b}, [x1], x2 + ld1 {v2.16b}, [x10], x2 + uaddlp v0.8h, v0.16b + uaddlp v2.8h, v2.16b + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 12 + ld1 {v0.8b}, [x1], x2 + ld1 {v2.8b}, [x10], x2 + uaddlp v0.4h, v0.8b + uaddlp v2.4h, v2.8b + shl v0.4h, v0.4h, #2 + shl v2.4h, v2.4h, #2 + dup v1.8h, v0.h[3] + dup v3.8h, v2.h[3] + trn1 v0.2d, v0.2d, v1.2d + trn1 v2.2d, v2.2d, v3.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_tbl): + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w16) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w8) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w4) + .hword 0 + +L(ipred_cfl_ac_422_w16_tbl): + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3) +endfunc + +// void cfl_ac_444_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_8bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_444_tbl) + sub w8, w8, #26 + ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_444_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input + ld1 {v0.s}[0], [x1], x2 + ld1 {v0.s}[1], [x10], x2 + ld1 {v1.s}[0], [x1], x2 + ld1 {v1.s}[1], [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v1.8h, v1.8b, #3 + subs w8, w8, #4 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input + ld1 {v0.8b}, [x1], x2 + ld1 {v1.8b}, [x10], x2 + ld1 {v2.8b}, [x1], x2 + ushll v0.8h, v0.8b, #3 + ld1 {v3.8b}, [x10], x2 + ushll v1.8h, v1.8b, #3 + ushll v2.8h, v2.8b, #3 + ushll v3.8h, v3.8b, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + ld1 {v0.16b}, [x1], x2 + ld1 {v2.16b}, [x10], x2 + ld1 {v4.16b}, [x1], x2 + ushll2 v1.8h, v0.16b, #3 + ushll v0.8h, v0.8b, #3 + ld1 {v6.16b}, [x10], x2 + ushll2 v3.8h, v2.16b, #3 + ushll v2.8h, v2.8b, #3 + ushll2 v5.8h, v4.16b, #3 + ushll v4.8h, v4.8b, #3 + ushll2 v7.8h, v6.16b, #3 + ushll v6.8h, v6.8b, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + mov v0.16b, v6.16b + mov v1.16b, v7.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + ld1 {v0.8b}, [x1], x2 + ld1 {v2.8b}, [x10], x2 + ld1 {v4.8b}, [x1], x2 + ld1 {v6.8b}, [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v2.8h, v2.8b, #3 + ushll v4.8h, v4.8b, #3 + ushll v6.8h, v6.8b, #3 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + dup v5.8h, v4.h[7] + dup v7.8h, v6.h[7] + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + mov v0.16b, v6.16b + mov v1.16b, v7.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_444_w32_tbl) + ldrh w3, [x7, w3, uxtw] // (w3>>1) << 1 + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_444_w32_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, without padding + ld1 {v2.16b, v3.16b}, [x1], x2 + ld1 {v6.16b, v7.16b}, [x10], x2 + ushll v0.8h, v2.8b, #3 + ushll2 v1.8h, v2.16b, #3 + ushll v2.8h, v3.8b, #3 + ushll2 v3.8h, v3.16b, #3 + ushll v4.8h, v6.8b, #3 + ushll2 v5.8h, v6.16b, #3 + ushll v6.8h, v7.8b, #3 + ushll2 v7.8h, v7.16b, #3 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 8 + ldr d2, [x1, #16] + ld1 {v1.16b}, [x1], x2 + ldr d6, [x10, #16] + ld1 {v5.16b}, [x10], x2 + ushll v2.8h, v2.8b, #3 + ushll v0.8h, v1.8b, #3 + ushll2 v1.8h, v1.16b, #3 + ushll v6.8h, v6.8b, #3 + ushll v4.8h, v5.8b, #3 + ushll2 v5.8h, v5.16b, #3 + dup v3.8h, v2.h[7] + dup v7.8h, v6.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 16 + ld1 {v1.16b}, [x1], x2 + ld1 {v5.16b}, [x10], x2 + ushll v0.8h, v1.8b, #3 + ushll2 v1.8h, v1.16b, #3 + ushll v4.8h, v5.8b, #3 + ushll2 v5.8h, v5.16b, #3 + dup v2.8h, v1.h[7] + dup v3.8h, v1.h[7] + dup v6.8h, v5.h[7] + dup v7.8h, v5.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 24 + ld1 {v0.8b}, [x1], x2 + ld1 {v4.8b}, [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v4.8h, v4.8b, #3 + dup v1.8h, v0.h[7] + dup v2.8h, v0.h[7] + dup v3.8h, v0.h[7] + dup v5.8h, v4.h[7] + dup v6.8h, v4.h[7] + dup v7.8h, v4.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + +L(ipred_cfl_ac_444_w32_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 2b +3: + + // Quadruple the height and reuse the w8 subtracting + lsl w6, w6, #2 + // Aggregate the sums, with wider intermediates earlier than in + // ipred_cfl_ac_420_w8_calc_subtract_dc. + uaddlp v0.4s, v16.8h + uaddlp v1.4s, v17.8h + uaddlp v2.4s, v18.8h + uaddlp v3.4s, v19.8h + add v0.4s, v0.4s, v1.4s + add v2.4s, v2.4s, v3.4s + add v0.4s, v0.4s, v2.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #4 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] + b L(ipred_cfl_ac_420_w8_subtract_dc) + +L(ipred_cfl_ac_444_tbl): + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w32) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w16) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w8) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w4) + +L(ipred_cfl_ac_444_w32_tbl): + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad0) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad2) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad4) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad6) +endfunc diff --git a/third_party/dav1d/src/arm/64/ipred16.S b/third_party/dav1d/src/arm/64/ipred16.S new file mode 100644 index 0000000000..c48c48583c --- /dev/null +++ b/third_party/dav1d/src/arm/64/ipred16.S @@ -0,0 +1,4204 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height, +// const int bitdepth_max); +function ipred_dc_128_16bpc_neon, export=1 + ldr w8, [sp] + clz w3, w3 + adr x5, L(ipred_dc_128_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + dup v0.8h, w8 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + urshr v0.8h, v0.8h, #1 + br x5 +4: + AARCH64_VALID_JUMP_TARGET + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_128_tbl): + .hword L(ipred_dc_128_tbl) - 640b + .hword L(ipred_dc_128_tbl) - 320b + .hword L(ipred_dc_128_tbl) - 160b + .hword L(ipred_dc_128_tbl) - 8b + .hword L(ipred_dc_128_tbl) - 4b +endfunc + +// void ipred_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_v_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h}, [x2] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + sub x1, x1, #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_v_tbl): + .hword L(ipred_v_tbl) - 640b + .hword L(ipred_v_tbl) - 320b + .hword L(ipred_v_tbl) - 160b + .hword L(ipred_v_tbl) - 80b + .hword L(ipred_v_tbl) - 40b +endfunc + +// void ipred_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_h_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + sub x2, x2, #8 + sub x5, x5, w3, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +4: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.4h}, [x0], x1 + st1 {v2.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + stp q3, q3, [x0, #64] + stp q2, q2, [x6, #64] + stp q3, q3, [x0, #96] + stp q2, q2, [x6, #96] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + stp q1, q1, [x0, #64] + stp q0, q0, [x6, #64] + stp q1, q1, [x0, #96] + stp q0, q0, [x6, #96] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_h_tbl): + .hword L(ipred_h_tbl) - 64b + .hword L(ipred_h_tbl) - 32b + .hword L(ipred_h_tbl) - 16b + .hword L(ipred_h_tbl) - 8b + .hword L(ipred_h_tbl) - 4b +endfunc + +// void ipred_dc_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_dc_top_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.4h, v0.h[0] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_top_tbl): + .hword L(ipred_dc_top_tbl) - 640b + .hword L(ipred_dc_top_tbl) - 320b + .hword L(ipred_dc_top_tbl) - 160b + .hword L(ipred_dc_top_tbl) - 80b + .hword L(ipred_dc_top_tbl) - 40b +endfunc + +// void ipred_dc_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + clz w3, w3 + clz w7, w4 + adr x5, L(ipred_dc_left_tbl) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w7, w7, #25 + ldrh w3, [x5, w3, uxtw #1] + ldrh w7, [x5, w7, uxtw #1] + sub x3, x5, w3, uxtw + sub x5, x5, w7, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_left_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w4): + AARCH64_VALID_JUMP_TARGET + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt L(ipred_dc_left_w4) + ret + +L(ipred_dc_left_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w8): + AARCH64_VALID_JUMP_TARGET + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt L(ipred_dc_left_w8) + ret + +L(ipred_dc_left_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] + br x3 +L(ipred_dc_left_w16): + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b +1: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlp v0.4s, v0.8h + addv s0, v0.4s + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w32): + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h64): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w64): + AARCH64_VALID_JUMP_TARGET + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_tbl): + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h4) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) +endfunc + +// void ipred_dc_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + add w7, w3, w4 // width + height + clz w3, w3 + clz w6, w4 + dup v16.4s, w7 // width + height + adr x5, L(ipred_dc_tbl) + rbit w7, w7 // rbit(width + height) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w6, w6, #25 + clz w7, w7 // ctz(width + height) + ldrh w3, [x5, w3, uxtw #1] + ldrh w6, [x5, w6, uxtw #1] + neg w7, w7 // -ctz(width + height) + sub x3, x5, w3, uxtw + sub x5, x5, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w7 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + add x2, x2, #2 + br x3 +L(ipred_dc_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.4h + cmp w4, #4 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.4h, v0.h[0] +2: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + add x2, x2, #2 + br x3 +L(ipred_dc_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.8h + cmp w4, #8 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] +2: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h}, [x2], #32 + addp v0.8h, v0.8h, v1.8h + add x2, x2, #2 + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w16): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.8h, v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + uaddlv s1, v1.8h + cmp w4, #16 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32/64 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + add x2, x2, #2 + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w32): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + addp v3.8h, v3.8h, v4.8h + addp v1.8h, v1.8h, v3.8h + uaddlv s1, v1.8h + cmp w4, #32 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16/64 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h64): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + add x2, x2, #2 + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w64): + AARCH64_VALID_JUMP_TARGET + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2], #64 + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2] + addp v3.8h, v3.8h, v4.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + addp v1.8h, v1.8h, v3.8h + addp v20.8h, v20.8h, v22.8h + addp v1.8h, v1.8h, v20.8h + uaddlv s1, v1.8h + cmp w4, #64 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 16/32 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_tbl): + .hword L(ipred_dc_tbl) - L(ipred_dc_h64) + .hword L(ipred_dc_tbl) - L(ipred_dc_h32) + .hword L(ipred_dc_tbl) - L(ipred_dc_h16) + .hword L(ipred_dc_tbl) - L(ipred_dc_h8) + .hword L(ipred_dc_tbl) - L(ipred_dc_h4) + .hword L(ipred_dc_tbl) - L(ipred_dc_w64) + .hword L(ipred_dc_tbl) - L(ipred_dc_w32) + .hword L(ipred_dc_tbl) - L(ipred_dc_w16) + .hword L(ipred_dc_tbl) - L(ipred_dc_w8) + .hword L(ipred_dc_tbl) - L(ipred_dc_w4) +endfunc + +// void ipred_paeth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_16bpc_neon, export=1 + clz w9, w3 + adr x5, L(ipred_paeth_tbl) + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x2] + add x8, x2, #2 + sub x2, x2, #8 + sub x5, x5, w9, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v5.2d}, [x8] + sub v6.8h, v5.8h, v4.8h // top - topleft +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 + zip1 v0.2d, v0.2d, v1.2d + zip1 v2.2d, v2.2d, v3.2d + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v2.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v4.8h, v16.8h // tldiff + sabd v23.8h, v4.8h, v17.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v2.8h, v17.8h + umin v18.8h, v20.8h, v22.8h // min(tdiff, tldiff) + umin v19.8h, v21.8h, v23.8h + cmge v20.8h, v22.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v23.8h, v21.8h + cmge v16.8h, v18.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v19.8h, v17.8h + bsl v21.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v20.16b, v5.16b, v4.16b + bit v21.16b, v2.16b, v17.16b // ldiff <= min ? left : ... + bit v20.16b, v0.16b, v16.16b + st1 {v21.d}[1], [x0], x1 + st1 {v21.d}[0], [x6], x1 + subs w4, w4, #4 + st1 {v20.d}[1], [x0], x1 + st1 {v20.d}[0], [x6], x1 + b.gt 4b + ret +80: +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + ld1 {v5.8h}, [x8], #16 + mov w9, w3 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 +2: + sub v6.8h, v5.8h, v4.8h // top - topleft + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v1.8h + add v18.8h, v6.8h, v2.8h + add v19.8h, v6.8h, v3.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v5.8h, v18.8h + sabd v23.8h, v5.8h, v19.8h + sabd v24.8h, v4.8h, v16.8h // tldiff + sabd v25.8h, v4.8h, v17.8h + sabd v26.8h, v4.8h, v18.8h + sabd v27.8h, v4.8h, v19.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v1.8h, v17.8h + sabd v18.8h, v2.8h, v18.8h + sabd v19.8h, v3.8h, v19.8h + umin v28.8h, v20.8h, v24.8h // min(tdiff, tldiff) + umin v29.8h, v21.8h, v25.8h + umin v30.8h, v22.8h, v26.8h + umin v31.8h, v23.8h, v27.8h + cmge v20.8h, v24.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v25.8h, v21.8h + cmge v22.8h, v26.8h, v22.8h + cmge v23.8h, v27.8h, v23.8h + cmge v16.8h, v28.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v29.8h, v17.8h + cmge v18.8h, v30.8h, v18.8h + cmge v19.8h, v31.8h, v19.8h + bsl v23.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v22.16b, v5.16b, v4.16b + bsl v21.16b, v5.16b, v4.16b + bsl v20.16b, v5.16b, v4.16b + bit v23.16b, v3.16b, v19.16b // ldiff <= min ? left : ... + bit v22.16b, v2.16b, v18.16b + bit v21.16b, v1.16b, v17.16b + bit v20.16b, v0.16b, v16.16b + st1 {v23.8h}, [x0], #16 + st1 {v22.8h}, [x6], #16 + subs w3, w3, #8 + st1 {v21.8h}, [x5], #16 + st1 {v20.8h}, [x10], #16 + b.le 8f + ld1 {v5.8h}, [x8], #16 + b 2b +8: + subs w4, w4, #4 + b.le 9f + // End of horizontal loop, move pointers to next four rows + sub x8, x8, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + // Load the top row as early as possible + ld1 {v5.8h}, [x8], #16 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_paeth_tbl): + .hword L(ipred_paeth_tbl) - 640b + .hword L(ipred_paeth_tbl) - 320b + .hword L(ipred_paeth_tbl) - 160b + .hword L(ipred_paeth_tbl) - 80b + .hword L(ipred_paeth_tbl) - 40b +endfunc + +// void ipred_smooth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_16bpc_neon, export=1 + movrel x10, X(sm_weights) + add x11, x10, w4, uxtw + add x10, x10, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_tbl) + sub x12, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x12] // bottom + add x8, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v6.2d}, [x8] // top + ld1r {v7.2s}, [x10] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + dup v5.8h, v6.h[3] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v18.8h, v18.8b + smlal v20.4s, v0.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v0.8h, v7.8h + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v18.4h + smlal2 v23.4s, v6.8h, v18.8h + rshrn v20.4h, v20.4s, #9 + rshrn v21.4h, v21.4s, #9 + rshrn v22.4h, v22.4s, #9 + rshrn v23.4h, v23.4s, #9 + st1 {v20.4h}, [x0], x1 + st1 {v21.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.4h}, [x0], x1 + st1 {v23.4h}, [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v6.8h}, [x8] // top + ld1 {v7.8b}, [x10] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + dup v5.8h, v6.h[7] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b + smlal v20.4s, v3.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v3.8h, v7.8h // (left flipped) + smlal v22.4s, v2.4h, v7.4h + smlal2 v23.4s, v2.8h, v7.8h + smlal v24.4s, v1.4h, v7.4h + smlal2 v25.4s, v1.8h, v7.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v17.4h + smlal2 v23.4s, v6.8h, v17.8h + smlal v24.4s, v6.4h, v18.4h + smlal2 v25.4s, v6.8h, v18.8h + smlal v26.4s, v6.4h, v19.4h + smlal2 v27.4s, v6.8h, v19.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + add x12, x2, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + ld1r {v5.8h}, [x12] // right + sub x2, x2, #4 + mov x7, #-4 + mov w9, w3 + add v31.4h, v4.4h, v5.4h // bottom+right + +1: + ld2r {v0.8h, v1.8h}, [x2], x7 // left + ld2r {v16.8b, v17.8b}, [x11], #2 // weights_ver + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b +2: + ld1 {v7.16b}, [x10], #16 // weights_hor + ld1 {v2.8h, v3.8h}, [x8], #32 // top + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + uxtl v6.8h, v7.8b // weights_hor + uxtl2 v7.8h, v7.16b + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + smlal v20.4s, v1.4h, v6.4h // += (left-right)*weights_hor + smlal2 v21.4s, v1.8h, v6.8h // (left flipped) + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v24.4s, v0.4h, v6.4h + smlal2 v25.4s, v0.8h, v6.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v2.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v2.8h, v16.8h + smlal v22.4s, v3.4h, v16.4h + smlal2 v23.4s, v3.8h, v16.8h + smlal v24.4s, v2.4h, v17.4h + smlal2 v25.4s, v2.8h, v17.8h + smlal v26.4s, v3.4h, v17.4h + smlal2 v27.4s, v3.8h, v17.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x8, w9, uxtw #1 + sub x10, x10, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_tbl): + .hword L(ipred_smooth_tbl) - 640b + .hword L(ipred_smooth_tbl) - 320b + .hword L(ipred_smooth_tbl) - 160b + .hword L(ipred_smooth_tbl) - 80b + .hword L(ipred_smooth_tbl) - 40b +endfunc + +// void ipred_smooth_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_16bpc_neon, export=1 + movrel x7, X(sm_weights) + add x7, x7, w4, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_v_tbl) + sub x8, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x8] // bottom + add x2, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v6.2d}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +4: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v18.8h, v18.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v18.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v6.8h}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +8: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v17.8h + sqrdmulh v22.8h, v6.8h, v18.8h + sqrdmulh v23.8h, v6.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + // Set up pointers for four rows in parallel; x0, x6, x5, x8 + add x5, x0, x1 + add x8, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 +2: + ld1 {v2.8h, v3.8h}, [x2], #32 // top + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + sqrdmulh v20.8h, v2.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v16.8h + sqrdmulh v22.8h, v2.8h, v17.8h + sqrdmulh v23.8h, v3.8h, v17.8h + sqrdmulh v24.8h, v2.8h, v18.8h + sqrdmulh v25.8h, v3.8h, v18.8h + sqrdmulh v26.8h, v2.8h, v19.8h + sqrdmulh v27.8h, v3.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + add v24.8h, v24.8h, v4.8h + add v25.8h, v25.8h, v4.8h + add v26.8h, v26.8h, v4.8h + add v27.8h, v27.8h, v4.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x8], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x2, x2, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x8, x8, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_v_tbl): + .hword L(ipred_smooth_v_tbl) - 640b + .hword L(ipred_smooth_v_tbl) - 320b + .hword L(ipred_smooth_v_tbl) - 160b + .hword L(ipred_smooth_v_tbl) - 80b + .hword L(ipred_smooth_v_tbl) - 40b +endfunc + +// void ipred_smooth_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_16bpc_neon, export=1 + movrel x8, X(sm_weights) + add x8, x8, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_h_tbl) + add x12, x2, w3, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v5.8h}, [x12] // right + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v7.2s}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sqrdmulh v20.8h, v0.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v1.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v7.8b}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v3.8h, v3.8h, v5.8h // left-right + sub v2.8h, v2.8h, v5.8h + sub v1.8h, v1.8h, v5.8h + sub v0.8h, v0.8h, v5.8h + sqrdmulh v20.8h, v3.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v2.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v1.8h, v7.8h + sqrdmulh v23.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + sub x2, x2, #8 + mov x7, #-8 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h +2: + ld1 {v7.16b}, [x8], #16 // weights_hor + ushll v6.8h, v7.8b, #7 // weights_hor << 7 + ushll2 v7.8h, v7.16b, #7 + sqrdmulh v20.8h, v3.8h, v6.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v2.8h, v6.8h + sqrdmulh v23.8h, v2.8h, v7.8h + sqrdmulh v24.8h, v1.8h, v6.8h + sqrdmulh v25.8h, v1.8h, v7.8h + sqrdmulh v26.8h, v0.8h, v6.8h + sqrdmulh v27.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + add v24.8h, v24.8h, v5.8h + add v25.8h, v25.8h, v5.8h + add v26.8h, v26.8h, v5.8h + add v27.8h, v27.8h, v5.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x10], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x8, x8, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_h_tbl): + .hword L(ipred_smooth_h_tbl) - 640b + .hword L(ipred_smooth_h_tbl) - 320b + .hword L(ipred_smooth_h_tbl) - 160b + .hword L(ipred_smooth_h_tbl) - 80b + .hword L(ipred_smooth_h_tbl) - 40b +endfunc + +const padding_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +padding_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void ipred_z1_upsample_edge_16bpc_neon(pixel *out, const int hsz, +// const pixel *const in, const int end, +// const int bitdepth_max); +function ipred_z1_upsample_edge_16bpc_neon, export=1 + dup v30.8h, w4 // bitdepth_max + movrel x4, padding_mask + ld1 {v0.8h, v1.8h}, [x2] // in[] + add x5, x2, w3, uxtw #1 // in[end] + sub x4, x4, w3, uxtw #1 + + ld1r {v2.8h}, [x5] // padding + ld1 {v3.8h, v4.8h}, [x4] // padding_mask + + movi v31.8h, #9 + + bit v0.16b, v2.16b, v3.16b // padded in[] + bit v1.16b, v2.16b, v4.16b + + ext v4.16b, v0.16b, v1.16b, #2 + ext v5.16b, v1.16b, v2.16b, #2 + ext v6.16b, v0.16b, v1.16b, #4 + ext v7.16b, v1.16b, v2.16b, #4 + ext v16.16b, v0.16b, v1.16b, #6 + ext v17.16b, v1.16b, v2.16b, #6 + + add v18.8h, v4.8h, v6.8h // in[i+1] + in[i+2] + add v19.8h, v5.8h, v7.8h + add v20.8h, v0.8h, v16.8h + add v21.8h, v1.8h, v17.8h + umull v22.4s, v18.4h, v31.4h // 9*(in[i+1] + in[i+2]) + umull2 v23.4s, v18.8h, v31.8h + umull v24.4s, v19.4h, v31.4h + umull2 v25.4s, v19.8h, v31.8h + usubw v22.4s, v22.4s, v20.4h + usubw2 v23.4s, v23.4s, v20.8h + usubw v24.4s, v24.4s, v21.4h + usubw2 v25.4s, v25.4s, v21.8h + + sqrshrun v16.4h, v22.4s, #4 + sqrshrun2 v16.8h, v23.4s, #4 + sqrshrun v17.4h, v24.4s, #4 + sqrshrun2 v17.8h, v25.4s, #4 + + smin v16.8h, v16.8h, v30.8h + smin v17.8h, v17.8h, v30.8h + + zip1 v0.8h, v4.8h, v16.8h + zip2 v1.8h, v4.8h, v16.8h + zip1 v2.8h, v5.8h, v17.8h + zip2 v3.8h, v5.8h, v17.8h + + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + + ret +endfunc + +const edge_filter + .short 0, 4, 8, 0 + .short 0, 5, 6, 0 +// Leaving out the coeffs for strength=3 +// .byte 2, 4, 4, 0 +endconst + +// void ipred_z1_filter_edge_16bpc_neon(pixel *out, const int sz, +// const pixel *const in, const int end, +// const int strength); +function ipred_z1_filter_edge_16bpc_neon, export=1 + cmp w4, #3 + b.eq L(fivetap) // if (strength == 3) goto fivetap + + movrel x5, edge_filter, -6 + add x5, x5, w4, uxtw #3 // edge_filter + 2*((strength - 1)*4 + 1) + + ld1 {v31.s}[0], [x5] // kernel[1-2] + + ld1 {v0.8h}, [x2], #16 + + dup v30.8h, v31.h[0] + dup v31.8h, v31.h[1] +1: + // in[end], is the last valid pixel. We produce 16 pixels out by + // using 18 pixels in - the last pixel used is [17] of the ones + // read/buffered. + cmp w3, #17 + ld1 {v1.8h, v2.8h}, [x2], #32 + b.lt 2f + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v1.16b, v2.16b, #2 + ext v5.16b, v0.16b, v1.16b, #4 + ext v6.16b, v1.16b, v2.16b, #4 + mul v16.8h, v0.8h, v30.8h + mla v16.8h, v3.8h, v31.8h + mla v16.8h, v5.8h, v30.8h + mul v17.8h, v1.8h, v30.8h + mla v17.8h, v4.8h, v31.8h + mla v17.8h, v6.8h, v30.8h + subs w1, w1, #16 + mov v0.16b, v2.16b + urshr v16.8h, v16.8h, #4 + urshr v17.8h, v17.8h, #4 + sub w3, w3, #16 + st1 {v16.8h, v17.8h}, [x0], #32 + b.gt 1b + ret +2: + // Right padding + + // x2[w3-24] is the padding pixel (x2 points 24 pixels ahead) + movrel x5, padding_mask + sub w6, w3, #24 + sub x5, x5, w3, uxtw #1 + add x6, x2, w6, sxtw #1 + + ld1 {v3.8h, v4.8h}, [x5] // padding_mask + + ld1r {v2.8h}, [x6] + bit v0.16b, v2.16b, v3.16b // Pad v0-v1 + bit v1.16b, v2.16b, v4.16b + + // Filter one block + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v1.16b, v2.16b, #2 + ext v5.16b, v0.16b, v1.16b, #4 + ext v6.16b, v1.16b, v2.16b, #4 + mul v16.8h, v0.8h, v30.8h + mla v16.8h, v3.8h, v31.8h + mla v16.8h, v5.8h, v30.8h + mul v17.8h, v1.8h, v30.8h + mla v17.8h, v4.8h, v31.8h + mla v17.8h, v6.8h, v30.8h + subs w1, w1, #16 + urshr v16.8h, v16.8h, #4 + urshr v17.8h, v17.8h, #4 + st1 {v16.8h, v17.8h}, [x0], #32 + b.le 9f +5: + // After one block, any remaining output would only be filtering + // padding - thus just store the padding. + subs w1, w1, #16 + st1 {v2.16b}, [x0], #16 + b.gt 5b +9: + ret + +L(fivetap): + sub x2, x2, #2 // topleft -= 1 pixel + movi v29.8h, #2 + ld1 {v0.8h}, [x2], #16 + movi v30.8h, #4 + movi v31.8h, #4 + ins v0.h[0], v0.h[1] +1: + // in[end+1], is the last valid pixel. We produce 16 pixels out by + // using 20 pixels in - the last pixel used is [19] of the ones + // read/buffered. + cmp w3, #18 + ld1 {v1.8h, v2.8h}, [x2], #32 + b.lt 2f // if (end + 1 < 19) + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v1.16b, v2.16b, #2 + ext v5.16b, v0.16b, v1.16b, #4 + ext v6.16b, v1.16b, v2.16b, #4 + ext v16.16b, v0.16b, v1.16b, #6 + ext v17.16b, v1.16b, v2.16b, #6 + ext v18.16b, v0.16b, v1.16b, #8 + ext v19.16b, v1.16b, v2.16b, #8 + mul v20.8h, v0.8h, v29.8h + mla v20.8h, v3.8h, v30.8h + mla v20.8h, v5.8h, v31.8h + mla v20.8h, v16.8h, v30.8h + mla v20.8h, v18.8h, v29.8h + mul v21.8h, v1.8h, v29.8h + mla v21.8h, v4.8h, v30.8h + mla v21.8h, v6.8h, v31.8h + mla v21.8h, v17.8h, v30.8h + mla v21.8h, v19.8h, v29.8h + subs w1, w1, #16 + mov v0.16b, v2.16b + urshr v20.8h, v20.8h, #4 + urshr v21.8h, v21.8h, #4 + sub w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + b.gt 1b + ret +2: + // Right padding + + // x2[w3+1-24] is the padding pixel (x2 points 24 pixels ahead) + movrel x5, padding_mask, -2 + sub w6, w3, #23 + sub x5, x5, w3, uxtw #1 + add x6, x2, w6, sxtw #1 + + ld1 {v3.8h, v4.8h, v5.8h}, [x5] // padding_mask + + ld1r {v28.8h}, [x6] + bit v0.16b, v28.16b, v3.16b // Pad v0-v2 + bit v1.16b, v28.16b, v4.16b + bit v2.16b, v28.16b, v5.16b +4: + // Filter one block + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v1.16b, v2.16b, #2 + ext v5.16b, v0.16b, v1.16b, #4 + ext v6.16b, v1.16b, v2.16b, #4 + ext v16.16b, v0.16b, v1.16b, #6 + ext v17.16b, v1.16b, v2.16b, #6 + ext v18.16b, v0.16b, v1.16b, #8 + ext v19.16b, v1.16b, v2.16b, #8 + mul v20.8h, v0.8h, v29.8h + mla v20.8h, v3.8h, v30.8h + mla v20.8h, v5.8h, v31.8h + mla v20.8h, v16.8h, v30.8h + mla v20.8h, v18.8h, v29.8h + mul v21.8h, v1.8h, v29.8h + mla v21.8h, v4.8h, v30.8h + mla v21.8h, v6.8h, v31.8h + mla v21.8h, v17.8h, v30.8h + mla v21.8h, v19.8h, v29.8h + subs w1, w1, #16 + mov v0.16b, v2.16b + mov v1.16b, v28.16b + mov v2.16b, v28.16b + urshr v20.8h, v20.8h, #4 + urshr v21.8h, v21.8h, #4 + sub w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + b.le 9f + // v0-v1[w3+1] is the last valid pixel; if (w3 + 1 > 0) we need to + // filter properly once more - aka (w3 >= 0). + cmp w3, #0 + b.ge 4b +5: + // When w3 <= 0, all remaining pixels in v0-v1 are equal to the + // last valid pixel - thus just output that without filtering. + subs w1, w1, #8 + st1 {v28.8h}, [x0], #16 + b.gt 5b +9: + ret +endfunc + +// void ipred_pixel_set_16bpc_neon(pixel *out, const pixel px, +// const int n); +function ipred_pixel_set_16bpc_neon, export=1 + dup v0.8h, w1 +1: + subs w2, w2, #8 + st1 {v0.8h}, [x0], #16 + b.gt 1b + ret +endfunc + +// void ipred_z1_fill1_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const top, +// const int width, const int height, +// const int dx, const int max_base_x); +function ipred_z1_fill1_16bpc_neon, export=1 + clz w9, w3 + adr x8, L(ipred_z1_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw #1 // top[max_base_x] + sub x8, x8, w9, uxtw + ld1r {v31.8h}, [x10] // padding + mov w7, w5 + mov w15, #64 + br x8 +40: + AARCH64_VALID_JUMP_TARGET +4: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + lsl w8, w8, #1 + lsl w10, w10, #1 + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + ext v1.16b, v0.16b, v0.16b, #2 // top[base+1] + ext v3.16b, v2.16b, v2.16b, #2 + sub v6.4h, v1.4h, v0.4h // top[base+1]-top[base] + sub v7.4h, v3.4h, v2.4h + ushll v16.4s, v0.4h, #6 // top[base]*64 + ushll v17.4s, v2.4h, #6 + smlal v16.4s, v6.4h, v4.4h // + top[base+1]*frac + smlal v17.4s, v7.4h, v5.4h + rshrn v16.4h, v16.4s, #6 + rshrn v17.4h, v17.4s, #6 + st1 {v16.4h}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.4h}, [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.4h}, [x0], x1 + subs w4, w4, #2 + st1 {v31.4h}, [x0], x1 + b.gt 49b + ret + +80: + AARCH64_VALID_JUMP_TARGET +8: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v4.8h, w9 // frac + dup v5.8h, w11 + ld1 {v0.8h}, [x8] // top[base] + ld1 {v2.8h}, [x10] + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + ldr h1, [x8, #16] + ldr h3, [x10, #16] + dup v6.8h, w9 // 64 - frac + dup v7.8h, w11 + ext v1.16b, v0.16b, v1.16b, #2 // top[base+1] + ext v3.16b, v2.16b, v3.16b, #2 + umull v16.4s, v0.4h, v6.4h // top[base]*(64-frac) + umlal v16.4s, v1.4h, v4.4h // + top[base+1]*frac + umull2 v17.4s, v0.8h, v6.8h + umlal2 v17.4s, v1.8h, v4.8h + umull v18.4s, v2.4h, v7.4h + umlal v18.4s, v3.4h, v5.4h + umull2 v19.4s, v2.8h, v7.8h + umlal2 v19.4s, v3.8h, v5.8h + rshrn v16.4h, v16.4s, #6 + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + st1 {v16.8h}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8h}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8h}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8h}, [x0], x1 + b.gt 89b + ret + +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + + mov w12, w3 + + add x13, x0, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 169f + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v6.8h, w9 // frac + dup v7.8h, w11 + ld1 {v0.8h, v1.8h, v2.8h}, [x8], #48 // top[base] + ld1 {v3.8h, v4.8h, v5.8h}, [x10], #48 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v16.8h, w9 // 64 - frac + dup v17.8h, w11 + add w7, w7, w5 // xpos += dx +2: + ext v18.16b, v0.16b, v1.16b, #2 // top[base+1] + ext v19.16b, v1.16b, v2.16b, #2 + ext v20.16b, v3.16b, v4.16b, #2 + ext v21.16b, v4.16b, v5.16b, #2 + subs w3, w3, #16 + umull v22.4s, v0.4h, v16.4h // top[base]*(64-frac) + umlal v22.4s, v18.4h, v6.4h // + top[base+1]*frac + umull2 v23.4s, v0.8h, v16.8h + umlal2 v23.4s, v18.8h, v6.8h + umull v24.4s, v1.4h, v16.4h + umlal v24.4s, v19.4h, v6.4h + umull2 v25.4s, v1.8h, v16.8h + umlal2 v25.4s, v19.8h, v6.8h + umull v26.4s, v3.4h, v17.4h + umlal v26.4s, v20.4h, v7.4h + umull2 v27.4s, v3.8h, v17.8h + umlal2 v27.4s, v20.8h, v7.8h + umull v28.4s, v4.4h, v17.4h + umlal v28.4s, v21.4h, v7.4h + umull2 v29.4s, v4.8h, v17.8h + umlal2 v29.4s, v21.8h, v7.8h + rshrn v22.4h, v22.4s, #6 + rshrn2 v22.8h, v23.4s, #6 + rshrn v23.4h, v24.4s, #6 + rshrn2 v23.8h, v25.4s, #6 + rshrn v24.4h, v26.4s, #6 + rshrn2 v24.8h, v27.4s, #6 + rshrn v25.4h, v28.4s, #6 + rshrn2 v25.8h, v29.4s, #6 + st1 {v22.8h, v23.8h}, [x0], #32 + st1 {v24.8h, v25.8h}, [x13], #32 + b.le 3f + mov v0.16b, v2.16b + ld1 {v1.8h, v2.8h}, [x8], #32 // top[base] + mov v3.16b, v5.16b + ld1 {v4.8h, v5.8h}, [x10], #32 + b 2b + +3: + subs w4, w4, #2 + b.le 9f + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 1b +9: + ret + +169: + st1 {v31.8h}, [x0], #16 + subs w3, w3, #8 + st1 {v31.8h}, [x13], #16 + b.gt 169b + subs w4, w4, #2 + b.le 9b + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 169b + +L(ipred_z1_fill1_tbl): + .hword L(ipred_z1_fill1_tbl) - 640b + .hword L(ipred_z1_fill1_tbl) - 320b + .hword L(ipred_z1_fill1_tbl) - 160b + .hword L(ipred_z1_fill1_tbl) - 80b + .hword L(ipred_z1_fill1_tbl) - 40b +endfunc + +function ipred_z1_fill2_16bpc_neon, export=1 + cmp w3, #8 + add x10, x2, w6, uxtw // top[max_base_x] + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + b.eq 8f + +4: // w == 4 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + lsl w8, w8, #1 + lsl w10, w10, #1 + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + uzp2 v1.8h, v0.8h, v0.8h // top[base+1] + uzp1 v0.8h, v0.8h, v0.8h // top[base] + uzp2 v3.8h, v2.8h, v2.8h + uzp1 v2.8h, v2.8h, v2.8h + sub v6.4h, v1.4h, v0.4h // top[base+1]-top[base] + sub v7.4h, v3.4h, v2.4h + ushll v16.4s, v0.4h, #6 // top[base]*64 + ushll v17.4s, v2.4h, #6 + smlal v16.4s, v6.4h, v4.4h // + top[base+1]*frac + smlal v17.4s, v7.4h, v5.4h + rshrn v16.4h, v16.4s, #6 + rshrn v17.4h, v17.4s, #6 + st1 {v16.4h}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.4h}, [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.4h}, [x0], x1 + subs w4, w4, #2 + st1 {v31.4h}, [x0], x1 + b.gt 49b + ret + +8: // w == 8 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v4.8h, w9 // frac + dup v5.8h, w11 + ld1 {v0.8h, v1.8h}, [x8] // top[base] + ld1 {v2.8h, v3.8h}, [x10] + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8h, w9 // 64 - frac + dup v7.8h, w11 + uzp2 v20.8h, v0.8h, v1.8h // top[base+1] + uzp1 v0.8h, v0.8h, v1.8h // top[base] + uzp2 v21.8h, v2.8h, v3.8h + uzp1 v2.8h, v2.8h, v3.8h + umull v16.4s, v0.4h, v6.4h // top[base]*(64-frac) + umlal v16.4s, v20.4h, v4.4h // + top[base+1]*frac + umull2 v17.4s, v0.8h, v6.8h + umlal2 v17.4s, v20.8h, v4.8h + umull v18.4s, v2.4h, v7.4h + umlal v18.4s, v21.4h, v5.4h + umull2 v19.4s, v2.8h, v7.8h + umlal2 v19.4s, v21.8h, v5.8h + rshrn v16.4h, v16.4s, #6 + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + st1 {v16.8h}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8h}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8h}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8h}, [x0], x1 + b.gt 89b + ret +endfunc + +// void ipred_reverse_16bpc_neon(pixel *dst, const pixel *const src, +// const int n); +function ipred_reverse_16bpc_neon, export=1 + sub x1, x1, #16 + add x3, x0, #8 + mov x4, #16 +1: + ld1 {v0.8h}, [x1] + subs w2, w2, #8 + rev64 v0.8h, v0.8h + sub x1, x1, #16 + st1 {v0.d}[1], [x0], x4 + st1 {v0.d}[0], [x3], x4 + b.gt 1b + ret +endfunc + +// void ipred_z3_fill1_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const left, +// const int width, const int height, +// const int dy, const int max_base_y); +function ipred_z3_fill1_16bpc_neon, export=1 + clz w9, w4 + adr x8, L(ipred_z3_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw #1 // left[max_base_y] + sub x8, x8, w9, uxtw + ld1r {v31.8h}, [x10] // padding + mov w7, w5 + mov w15, #64 + add x13, x0, x1 + lsl x1, x1, #1 + br x8 + +40: + AARCH64_VALID_JUMP_TARGET +4: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + lsl w8, w8, #1 + lsl w10, w10, #1 + ldr q0, [x2, w8, uxtw] // left[base] + ldr q2, [x2, w10, uxtw] + dup v4.8h, w9 // frac + dup v5.8h, w11 + ext v1.16b, v0.16b, v0.16b, #2 // left[base+1] + ext v3.16b, v2.16b, v2.16b, #2 + sub v6.4h, v1.4h, v0.4h // top[base+1]-top[base] + sub v7.4h, v3.4h, v2.4h + ushll v16.4s, v0.4h, #6 // top[base]*64 + ushll v17.4s, v2.4h, #6 + smlal v16.4s, v6.4h, v4.4h // + top[base+1]*frac + smlal v17.4s, v7.4h, v5.4h + rshrn v16.4h, v16.4s, #6 + rshrn v17.4h, v17.4s, #6 + subs w3, w3, #2 + zip1 v18.8h, v16.8h, v17.8h + st1 {v18.s}[0], [x0], x1 + st1 {v18.s}[1], [x13], x1 + add w7, w7, w5 // xpos += dx + st1 {v18.s}[2], [x0] + st1 {v18.s}[3], [x13] + b.le 9f + sub x0, x0, x1 // ptr -= 4 * (2*stride) + sub x13, x13, x1 + add x0, x0, #4 + add x13, x13, #4 + b 4b +9: + ret + +80: + AARCH64_VALID_JUMP_TARGET +8: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v4.8h, w9 // frac + dup v5.8h, w11 + ld1 {v0.8h}, [x8] // left[base] + ld1 {v2.8h}, [x10] + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + ldr h1, [x8, #16] + ldr h3, [x10, #16] + dup v6.8h, w9 // 64 - frac + dup v7.8h, w11 + ext v1.16b, v0.16b, v1.16b, #2 // left[base+1] + ext v3.16b, v2.16b, v3.16b, #2 + umull v16.4s, v0.4h, v6.4h // left[base]*(64-frac) + umlal v16.4s, v1.4h, v4.4h // + left[base+1]*frac + umull2 v17.4s, v0.8h, v6.8h + umlal2 v17.4s, v1.8h, v4.8h + umull v18.4s, v2.4h, v7.4h + umlal v18.4s, v3.4h, v5.4h + umull2 v19.4s, v2.8h, v7.8h + umlal2 v19.4s, v3.8h, v5.8h + rshrn v16.4h, v16.4s, #6 + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + subs w3, w3, #2 + zip1 v18.8h, v16.8h, v17.8h + zip2 v19.8h, v16.8h, v17.8h + add w7, w7, w5 // xpos += dx + st1 {v18.s}[0], [x0], x1 + st1 {v18.s}[1], [x13], x1 + st1 {v18.s}[2], [x0], x1 + st1 {v18.s}[3], [x13], x1 + st1 {v19.s}[0], [x0], x1 + st1 {v19.s}[1], [x13], x1 + st1 {v19.s}[2], [x0], x1 + st1 {v19.s}[3], [x13], x1 + b.le 9f + sub x0, x0, x1, lsl #2 // ptr -= 4 * (2*stride) + sub x13, x13, x1, lsl #2 + add x0, x0, #4 + add x13, x13, #4 + b 8b +9: + ret + +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + mov w12, w4 +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // ypos += dy + cmp w8, w6 // base >= max_base_y + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v6.8h, w9 // frac + dup v7.8h, w11 + ld1 {v0.8h, v1.8h, v2.8h}, [x8], #48 // left[base] + ld1 {v3.8h, v4.8h, v5.8h}, [x10], #48 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v16.8h, w9 // 64 - frac + dup v17.8h, w11 + add w7, w7, w5 // ypos += dy +2: + ext v18.16b, v0.16b, v1.16b, #2 // left[base+1] + ext v19.16b, v1.16b, v2.16b, #2 + ext v20.16b, v3.16b, v4.16b, #2 + ext v21.16b, v4.16b, v5.16b, #2 + subs w4, w4, #16 + umull v22.4s, v0.4h, v16.4h // left[base]*(64-frac) + umlal v22.4s, v18.4h, v6.4h // + left[base+1]*frac + umull2 v23.4s, v0.8h, v16.8h + umlal2 v23.4s, v18.8h, v6.8h + umull v24.4s, v1.4h, v16.4h + umlal v24.4s, v19.4h, v6.4h + umull2 v25.4s, v1.8h, v16.8h + umlal2 v25.4s, v19.8h, v6.8h + umull v26.4s, v3.4h, v17.4h + umlal v26.4s, v20.4h, v7.4h + umull2 v27.4s, v3.8h, v17.8h + umlal2 v27.4s, v20.8h, v7.8h + umull v28.4s, v4.4h, v17.4h + umlal v28.4s, v21.4h, v7.4h + umull2 v29.4s, v4.8h, v17.8h + umlal2 v29.4s, v21.8h, v7.8h + rshrn v22.4h, v22.4s, #6 + rshrn2 v22.8h, v23.4s, #6 + rshrn v23.4h, v24.4s, #6 + rshrn2 v23.8h, v25.4s, #6 + rshrn v24.4h, v26.4s, #6 + rshrn2 v24.8h, v27.4s, #6 + rshrn v25.4h, v28.4s, #6 + rshrn2 v25.8h, v29.4s, #6 + zip1 v18.8h, v22.8h, v24.8h + zip2 v19.8h, v22.8h, v24.8h + zip1 v20.8h, v23.8h, v25.8h + zip2 v21.8h, v23.8h, v25.8h + st1 {v18.s}[0], [x0], x1 + st1 {v18.s}[1], [x13], x1 + st1 {v18.s}[2], [x0], x1 + st1 {v18.s}[3], [x13], x1 + st1 {v19.s}[0], [x0], x1 + st1 {v19.s}[1], [x13], x1 + st1 {v19.s}[2], [x0], x1 + st1 {v19.s}[3], [x13], x1 + st1 {v20.s}[0], [x0], x1 + st1 {v20.s}[1], [x13], x1 + st1 {v20.s}[2], [x0], x1 + st1 {v20.s}[3], [x13], x1 + st1 {v21.s}[0], [x0], x1 + st1 {v21.s}[1], [x13], x1 + st1 {v21.s}[2], [x0], x1 + st1 {v21.s}[3], [x13], x1 + b.le 3f + mov v0.16b, v2.16b + ld1 {v1.8h, v2.8h}, [x8], #32 // left[base] + mov v3.16b, v5.16b + ld1 {v4.8h, v5.8h}, [x10], #32 + b 2b + +3: + subs w3, w3, #2 + b.le 9f + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + lsl x1, x1, #1 + add x0, x0, #4 + add x13, x13, #4 + mov w4, w12 + b 1b +9: + ret + +L(ipred_z3_fill1_tbl): + .hword L(ipred_z3_fill1_tbl) - 640b + .hword L(ipred_z3_fill1_tbl) - 320b + .hword L(ipred_z3_fill1_tbl) - 160b + .hword L(ipred_z3_fill1_tbl) - 80b + .hword L(ipred_z3_fill1_tbl) - 40b +endfunc + +function ipred_z3_fill_padding_neon, export=0 + cmp w3, #8 + adr x8, L(ipred_z3_fill_padding_tbl) + b.gt L(ipred_z3_fill_padding_wide) + // w3 = remaining width, w4 = constant height + mov w12, w4 + +1: + // Fill a WxH rectangle with padding. W can be any number; + // this fills the exact width by filling in the largest + // power of two in the remaining width, and repeating. + clz w9, w3 + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + sub x9, x8, w9, uxtw + br x9 + +2: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #4 + st1 {v31.s}[0], [x13], x1 + st1 {v31.s}[0], [x0], x1 + st1 {v31.s}[0], [x13], x1 + b.gt 2b + subs w3, w3, #2 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #4 + add x13, x13, #4 + mov w4, w12 + b 1b + +4: + st1 {v31.4h}, [x0], x1 + subs w4, w4, #4 + st1 {v31.4h}, [x13], x1 + st1 {v31.4h}, [x0], x1 + st1 {v31.4h}, [x13], x1 + b.gt 4b + subs w3, w3, #4 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #8 + add x13, x13, #8 + mov w4, w12 + b 1b + +8: +16: +32: +64: + st1 {v31.8h}, [x0], x1 + subs w4, w4, #4 + st1 {v31.8h}, [x13], x1 + st1 {v31.8h}, [x0], x1 + st1 {v31.8h}, [x13], x1 + b.gt 4b + subs w3, w3, #8 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #16 + add x13, x13, #16 + mov w4, w12 + b 1b + +9: + ret + +L(ipred_z3_fill_padding_tbl): + .hword L(ipred_z3_fill_padding_tbl) - 64b + .hword L(ipred_z3_fill_padding_tbl) - 32b + .hword L(ipred_z3_fill_padding_tbl) - 16b + .hword L(ipred_z3_fill_padding_tbl) - 8b + .hword L(ipred_z3_fill_padding_tbl) - 4b + .hword L(ipred_z3_fill_padding_tbl) - 2b + +L(ipred_z3_fill_padding_wide): + // Fill a WxH rectangle with padding, with W > 8. + lsr x1, x1, #1 + mov w12, w3 + sub x1, x1, w3, uxtw #1 +1: + ands w5, w3, #7 + b.eq 2f + // If the width isn't aligned to 8, first do one 8 pixel write + // and align the start pointer. + sub w3, w3, w5 + st1 {v31.8h}, [x0] + add x0, x0, w5, uxtw #1 +2: + // Fill the rest of the line with aligned 8 pixel writes. + subs w3, w3, #8 + st1 {v31.8h}, [x0], #16 + b.gt 2b + subs w4, w4, #1 + add x0, x0, x1 + b.le 9f + mov w3, w12 + b 1b +9: + ret +endfunc + +function ipred_z3_fill2_16bpc_neon, export=1 + cmp w4, #8 + add x10, x2, w6, uxtw // left[max_base_y] + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + add x13, x0, x1 + lsl x1, x1, #1 + b.eq 8f + +4: // h == 4 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + lsl w8, w8, #1 + lsl w10, w10, #1 + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + uzp2 v1.8h, v0.8h, v0.8h // top[base+1] + uzp1 v0.8h, v0.8h, v0.8h // top[base] + uzp2 v3.8h, v2.8h, v2.8h + uzp1 v2.8h, v2.8h, v2.8h + sub v6.4h, v1.4h, v0.4h // top[base+1]-top[base] + sub v7.4h, v3.4h, v2.4h + ushll v16.4s, v0.4h, #6 // top[base]*64 + ushll v17.4s, v2.4h, #6 + smlal v16.4s, v6.4h, v4.4h // + top[base+1]*frac + smlal v17.4s, v7.4h, v5.4h + rshrn v16.4h, v16.4s, #6 + rshrn v17.4h, v17.4s, #6 + subs w3, w3, #2 + zip1 v18.8h, v16.8h, v17.8h + st1 {v18.s}[0], [x0], x1 + st1 {v18.s}[1], [x13], x1 + add w7, w7, w5 // xpos += dx + st1 {v18.s}[2], [x0] + st1 {v18.s}[3], [x13] + b.le 9f + sub x0, x0, x1 // ptr -= 4 * (2*stride) + sub x13, x13, x1 + add x0, x0, #4 + add x13, x13, #4 + b 4b +9: + ret + +8: // h == 8 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + add x8, x2, w8, uxtw #1 + add x10, x2, w10, uxtw #1 + dup v4.8h, w9 // frac + dup v5.8h, w11 + ld1 {v0.8h, v1.8h}, [x8] // top[base] + ld1 {v2.8h, v3.8h}, [x10] + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8h, w9 // 64 - frac + dup v7.8h, w11 + uzp2 v20.8h, v0.8h, v1.8h // top[base+1] + uzp1 v0.8h, v0.8h, v1.8h // top[base] + uzp2 v21.8h, v2.8h, v3.8h + uzp1 v2.8h, v2.8h, v3.8h + umull v16.4s, v0.4h, v6.4h // top[base]*(64-frac) + umlal v16.4s, v20.4h, v4.4h // + top[base+1]*frac + umull2 v17.4s, v0.8h, v6.8h + umlal2 v17.4s, v20.8h, v4.8h + umull v18.4s, v2.4h, v7.4h + umlal v18.4s, v21.4h, v5.4h + umull2 v19.4s, v2.8h, v7.8h + umlal2 v19.4s, v21.8h, v5.8h + rshrn v16.4h, v16.4s, #6 + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + subs w3, w3, #2 + zip1 v18.8h, v16.8h, v17.8h + zip2 v19.8h, v16.8h, v17.8h + add w7, w7, w5 // xpos += dx + st1 {v18.s}[0], [x0], x1 + st1 {v18.s}[1], [x13], x1 + st1 {v18.s}[2], [x0], x1 + st1 {v18.s}[3], [x13], x1 + st1 {v19.s}[0], [x0], x1 + st1 {v19.s}[1], [x13], x1 + st1 {v19.s}[2], [x0], x1 + st1 {v19.s}[3], [x13], x1 + b.le 9f + sub x0, x0, x1, lsl #2 // ptr -= 4 * (2*stride) + sub x13, x13, x1, lsl #2 + add x0, x0, #4 + add x13, x13, #4 + b 8b +9: + ret +endfunc + + +// void ipred_filter_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height, +// const int bitdepth_max); +.macro filter_fn bpc +function ipred_filter_\bpc\()bpc_neon + and w5, w5, #511 + movrel x6, X(filter_intra_taps) + lsl w5, w5, #6 + add x6, x6, w5, uxtw + ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 + clz w9, w3 + adr x5, L(ipred_filter\bpc\()_tbl) + ld1 {v20.8b, v21.8b, v22.8b}, [x6] + sub w9, w9, #26 + ldrh w9, [x5, w9, uxtw #1] + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + sub x5, x5, w9, uxtw + sxtl v18.8h, v18.8b + sxtl v19.8h, v19.8b + add x6, x0, x1 + lsl x1, x1, #1 + sxtl v20.8h, v20.8b + sxtl v21.8h, v21.8b + sxtl v22.8h, v22.8b + dup v31.8h, w8 +.if \bpc == 10 + movi v30.8h, #0 +.endif + br x5 +40: + AARCH64_VALID_JUMP_TARGET + ldur d0, [x2, #2] // top (0-3) + sub x2, x2, #4 + mov x7, #-4 +4: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 +.endif + smin v2.8h, v2.8h, v31.8h + subs w4, w4, #2 + st1 {v2.d}[0], [x0], x1 + ext v0.16b, v2.16b, v2.16b, #8 // move top from [4-7] to [0-3] + st1 {v2.d}[1], [x6], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ldur q0, [x2, #2] // top (0-7) + sub x2, x2, #4 + mov x7, #-4 +8: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + mul v3.8h, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + mla v3.8h, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + mla v3.8h, v16.8h, v0.h[3] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + smull v4.4s, v17.4h, v0.h[4] // p1(top[0]) * filter(1) + smlal v4.4s, v18.4h, v0.h[5] // p2(top[1]) * filter(2) + smlal v4.4s, v19.4h, v0.h[6] // p3(top[2]) * filter(3) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 + smin v2.8h, v2.8h, v31.8h + smlal v4.4s, v20.4h, v0.h[7] // p4(top[3]) * filter(4) + smlal v4.4s, v16.4h, v0.h[3] // p0(topleft) * filter(0) + smlal v4.4s, v21.4h, v2.h[3] // p5(left[0]) * filter(5) + smlal v4.4s, v22.4h, v2.h[7] // p6(left[1]) * filter(6) + smull2 v5.4s, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + smlal2 v5.4s, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + smlal2 v5.4s, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + smlal2 v5.4s, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + smlal2 v5.4s, v16.8h, v0.h[3] // p0(topleft) * filter(0) + smlal2 v5.4s, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + smlal2 v5.4s, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + sqrshrun v3.4h, v4.4s, #4 + sqrshrun2 v3.8h, v5.4s, #4 +.endif + smin v3.8h, v3.8h, v31.8h + subs w4, w4, #2 + st2 {v2.d, v3.d}[0], [x0], x1 + zip2 v0.2d, v2.2d, v3.2d + st2 {v2.d, v3.d}[1], [x6], x1 + b.gt 8b + ret +160: +320: + AARCH64_VALID_JUMP_TARGET + add x8, x2, #2 + sub x2, x2, #4 + mov x7, #-4 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld1 {v0.4h}, [x2], x7 // left (0-1) + topleft (2) +2: + ld1 {v1.8h, v2.8h}, [x8], #32 // top(0-15) +.if \bpc == 10 + mul v3.8h, v16.8h, v0.h[2] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + mla v3.8h, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + mla v3.8h, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + mul v4.8h, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + mla v4.8h, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + mla v4.8h, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h + smin v3.8h, v3.8h, v31.8h + mla v4.8h, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + mla v4.8h, v16.8h, v1.h[3] // p0(topleft) * filter(0) + mla v4.8h, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + mla v4.8h, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + mul v5.8h, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + mla v5.8h, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + mla v5.8h, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + srshr v4.8h, v4.8h, #4 + smax v4.8h, v4.8h, v30.8h + smin v4.8h, v4.8h, v31.8h + mla v5.8h, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + mla v5.8h, v16.8h, v1.h[7] // p0(topleft) * filter(0) + mla v5.8h, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + mla v5.8h, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + mul v6.8h, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + mla v6.8h, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + mla v6.8h, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + srshr v5.8h, v5.8h, #4 + smax v5.8h, v5.8h, v30.8h + smin v5.8h, v5.8h, v31.8h + mla v6.8h, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + mla v6.8h, v16.8h, v2.h[3] // p0(topleft) * filter(0) + mla v6.8h, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + mla v6.8h, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + srshr v6.8h, v6.8h, #4 + smax v6.8h, v6.8h, v30.8h +.else + smull v3.4s, v16.4h, v0.h[2] // p0(topleft) * filter(0) + smlal v3.4s, v21.4h, v0.h[1] // p5(left[0]) * filter(5) + smlal v3.4s, v22.4h, v0.h[0] // p6(left[1]) * filter(6) + smlal v3.4s, v17.4h, v1.h[0] // p1(top[0]) * filter(1) + smlal v3.4s, v18.4h, v1.h[1] // p2(top[1]) * filter(2) + smlal v3.4s, v19.4h, v1.h[2] // p3(top[2]) * filter(3) + smlal v3.4s, v20.4h, v1.h[3] // p4(top[3]) * filter(4) + smull2 v4.4s, v16.8h, v0.h[2] // p0(topleft) * filter(0) + smlal2 v4.4s, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + smlal2 v4.4s, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + smlal2 v4.4s, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + smlal2 v4.4s, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + smlal2 v4.4s, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + smlal2 v4.4s, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + smull v5.4s, v17.4h, v1.h[4] // p1(top[0]) * filter(1) + smlal v5.4s, v18.4h, v1.h[5] // p2(top[1]) * filter(2) + smlal v5.4s, v19.4h, v1.h[6] // p3(top[2]) * filter(3) + sqrshrun v3.4h, v3.4s, #4 + sqrshrun2 v3.8h, v4.4s, #4 + smin v3.8h, v3.8h, v31.8h + smlal v5.4s, v20.4h, v1.h[7] // p4(top[3]) * filter(4) + smlal v5.4s, v16.4h, v1.h[3] // p0(topleft) * filter(0) + smlal v5.4s, v21.4h, v3.h[3] // p5(left[0]) * filter(5) + smlal v5.4s, v22.4h, v3.h[7] // p6(left[1]) * filter(6) + smull2 v6.4s, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + smlal2 v6.4s, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + smlal2 v6.4s, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + smlal2 v6.4s, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + smlal2 v6.4s, v16.8h, v1.h[3] // p0(topleft) * filter(0) + smlal2 v6.4s, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + smlal2 v6.4s, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + smull v24.4s, v17.4h, v2.h[0] // p1(top[0]) * filter(1) + smlal v24.4s, v18.4h, v2.h[1] // p2(top[1]) * filter(2) + smlal v24.4s, v19.4h, v2.h[2] // p3(top[2]) * filter(3) + sqrshrun v4.4h, v5.4s, #4 + sqrshrun2 v4.8h, v6.4s, #4 + smin v4.8h, v4.8h, v31.8h + smlal v24.4s, v20.4h, v2.h[3] // p4(top[3]) * filter(4) + smlal v24.4s, v16.4h, v1.h[7] // p0(topleft) * filter(0) + smlal v24.4s, v21.4h, v4.h[3] // p5(left[0]) * filter(5) + smlal v24.4s, v22.4h, v4.h[7] // p6(left[1]) * filter(6) + smull2 v25.4s, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + smlal2 v25.4s, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + smlal2 v25.4s, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + smlal2 v25.4s, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + smlal2 v25.4s, v16.8h, v1.h[7] // p0(topleft) * filter(0) + smlal2 v25.4s, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + smlal2 v25.4s, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + smull v26.4s, v17.4h, v2.h[4] // p1(top[0]) * filter(1) + smlal v26.4s, v18.4h, v2.h[5] // p2(top[1]) * filter(2) + smlal v26.4s, v19.4h, v2.h[6] // p3(top[2]) * filter(3) + sqrshrun v5.4h, v24.4s, #4 + sqrshrun2 v5.8h, v25.4s, #4 + smin v5.8h, v5.8h, v31.8h + smlal v26.4s, v20.4h, v2.h[7] // p4(top[3]) * filter(4) + smlal v26.4s, v16.4h, v2.h[3] // p0(topleft) * filter(0) + smlal v26.4s, v21.4h, v5.h[3] // p5(left[0]) * filter(5) + smlal v26.4s, v22.4h, v5.h[7] // p6(left[1]) * filter(6) + smull2 v27.4s, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + smlal2 v27.4s, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + smlal2 v27.4s, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + smlal2 v27.4s, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + smlal2 v27.4s, v16.8h, v2.h[3] // p0(topleft) * filter(0) + smlal2 v27.4s, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + smlal2 v27.4s, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + sqrshrun v6.4h, v26.4s, #4 + sqrshrun2 v6.8h, v27.4s, #4 +.endif + smin v6.8h, v6.8h, v31.8h + + ins v0.h[2], v2.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[0], [x0], #32 + ins v0.h[0], v6.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[1], [x6], #32 + ins v0.h[1], v6.h[3] + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x6, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_filter\bpc\()_tbl): + .hword L(ipred_filter\bpc\()_tbl) - 320b + .hword L(ipred_filter\bpc\()_tbl) - 160b + .hword L(ipred_filter\bpc\()_tbl) - 80b + .hword L(ipred_filter\bpc\()_tbl) - 40b +endfunc +.endm + +filter_fn 10 +filter_fn 12 + +function ipred_filter_16bpc_neon, export=1 + ldr w8, [sp] + cmp w8, 0x3ff + b.le ipred_filter_10bpc_neon + b ipred_filter_12bpc_neon +endfunc + +// void pal_pred_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_16bpc_neon, export=1 + ld1 {v30.8h}, [x2] + clz w9, w4 + adr x6, L(pal_pred_tbl) + sub w9, w9, #25 + ldrh w9, [x6, w9, uxtw #1] + movi v31.8h, #1, lsl #8 + sub x6, x6, w9, uxtw + br x6 +40: + AARCH64_VALID_JUMP_TARGET + add x2, x0, x1 + lsl x1, x1, #1 +4: + ld1 {v1.16b}, [x3], #16 + subs w5, w5, #4 + // Restructure v1 from a, b, c, ... into 2*a, 2*a+1, 2*b, 2*b+1, 2*c, 2*c+1, ... + add v1.16b, v1.16b, v1.16b + zip1 v0.16b, v1.16b, v1.16b + zip2 v1.16b, v1.16b, v1.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + st1 {v0.d}[0], [x0], x1 + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.d}[1], [x2], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x2], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + add x2, x0, x1 + lsl x1, x1, #1 +8: + ld1 {v2.16b, v3.16b}, [x3], #32 + subs w5, w5, #4 + add v2.16b, v2.16b, v2.16b + add v3.16b, v3.16b, v3.16b + zip1 v0.16b, v2.16b, v2.16b + zip2 v1.16b, v2.16b, v2.16b + zip1 v2.16b, v3.16b, v3.16b + zip2 v3.16b, v3.16b, v3.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.8h}, [x0], x1 + tbl v2.16b, {v30.16b}, v2.16b + st1 {v1.8h}, [x2], x1 + tbl v3.16b, {v30.16b}, v3.16b + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x2], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + add x2, x0, x1 + lsl x1, x1, #1 +16: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #4 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + st1 {v2.8h, v3.8h}, [x2], x1 + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h}, [x0], x1 + st1 {v6.8h, v7.8h}, [x2], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + add x2, x0, x1 + lsl x1, x1, #1 +32: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #2 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + add x2, x0, #64 +64: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #1 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 64b + ret + +L(pal_pred_tbl): + .hword L(pal_pred_tbl) - 640b + .hword L(pal_pred_tbl) - 320b + .hword L(pal_pred_tbl) - 160b + .hword L(pal_pred_tbl) - 80b + .hword L(pal_pred_tbl) - 40b +endfunc + +// void ipred_cfl_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_128_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_128_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + urshr v0.8h, v31.8h, #1 + dup v1.8h, w6 // alpha + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +L(ipred_cfl_splat_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #4 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + cmlt v16.4s, v2.4s, #0 // sign + cmlt v17.4s, v3.4s, #0 + cmlt v18.4s, v4.4s, #0 + cmlt v19.4s, v5.4s, #0 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.d}[0], [x0], x1 + st1 {v2.d}[1], [x6], x1 + st1 {v3.d}[0], [x0], x1 + st1 {v3.d}[1], [x6], x1 + b.gt L(ipred_cfl_splat_w4) + ret +L(ipred_cfl_splat_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #2 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + cmlt v16.4s, v2.4s, #0 // sign + cmlt v17.4s, v3.4s, #0 + cmlt v18.4s, v4.4s, #0 + cmlt v19.4s, v5.4s, #0 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x6], x1 + b.gt L(ipred_cfl_splat_w8) + ret +L(ipred_cfl_splat_w16): + AARCH64_VALID_JUMP_TARGET + add x7, x5, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 +1: + ld1 {v2.8h, v3.8h}, [x5], #32 + ld1 {v4.8h, v5.8h}, [x7], #32 + subs w3, w3, #16 + smull v16.4s, v2.4h, v1.4h // diff = ac * alpha + smull2 v17.4s, v2.8h, v1.8h + smull v18.4s, v3.4h, v1.4h + smull2 v19.4s, v3.8h, v1.8h + smull v2.4s, v4.4h, v1.4h + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + cmlt v20.4s, v16.4s, #0 // sign + cmlt v21.4s, v17.4s, #0 + cmlt v22.4s, v18.4s, #0 + cmlt v23.4s, v19.4s, #0 + cmlt v24.4s, v2.4s, #0 + cmlt v25.4s, v3.4s, #0 + cmlt v26.4s, v4.4s, #0 + cmlt v27.4s, v5.4s, #0 + add v16.4s, v16.4s, v20.4s // diff + sign + add v17.4s, v17.4s, v21.4s + add v18.4s, v18.4s, v22.4s + add v19.4s, v19.4s, v23.4s + add v2.4s, v2.4s, v24.4s + add v3.4s, v3.4s, v25.4s + add v4.4s, v4.4s, v26.4s + add v5.4s, v5.4s, v27.4s + rshrn v16.4h, v16.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + rshrn v6.4h, v2.4s, #6 + rshrn2 v6.8h, v3.4s, #6 + rshrn v7.4h, v4.4s, #6 + rshrn2 v7.8h, v5.4s, #6 + add v2.8h, v16.8h, v0.8h // dc + apply_sign() + add v3.8h, v17.8h, v0.8h + add v4.8h, v6.8h, v0.8h + add v5.8h, v7.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smax v4.8h, v4.8h, v30.8h + smax v5.8h, v5.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + smin v4.8h, v4.8h, v31.8h + smin v5.8h, v5.8h, v31.8h + st1 {v2.8h, v3.8h}, [x0], #32 + st1 {v4.8h, v5.8h}, [x6], #32 + b.gt 1b + subs w4, w4, #2 + add x5, x5, w9, uxtw #1 + add x7, x7, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b.gt 1b + ret + +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) +endfunc + +// void ipred_cfl_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_top_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_top_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + dup v1.8h, w6 // alpha + add x2, x2, #2 + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) +16: + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) +32: + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_top_tbl): + .hword L(ipred_cfl_top_tbl) - 32b + .hword L(ipred_cfl_top_tbl) - 16b + .hword L(ipred_cfl_top_tbl) - 8b + .hword L(ipred_cfl_top_tbl) - 4b +endfunc + +// void ipred_cfl_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_left_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + clz w9, w3 + clz w8, w4 + adr x10, L(ipred_cfl_splat_tbl) + adr x7, L(ipred_cfl_left_tbl) + sub w9, w9, #26 + sub w8, w8, #26 + ldrh w9, [x10, w9, uxtw #1] + ldrh w8, [x7, w8, uxtw #1] + dup v1.8h, w6 // alpha + sub x9, x10, w9, uxtw + sub x7, x7, w8, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_left_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_tbl): + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) +endfunc + +// void ipred_cfl_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + add w8, w3, w4 // width + height + dup v1.8h, w6 // alpha + clz w9, w3 + clz w6, w4 + dup v16.4s, w8 // width + height + adr x7, L(ipred_cfl_tbl) + rbit w8, w8 // rbit(width + height) + sub w9, w9, #22 // 26 leading bits, minus table offset 4 + sub w6, w6, #26 + clz w8, w8 // ctz(width + height) + ldrh w9, [x7, w9, uxtw #1] + ldrh w6, [x7, w6, uxtw #1] + neg w8, w8 // -ctz(width + height) + sub x9, x7, w9, uxtw + sub x7, x7, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w8 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_h4): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + add x2, x2, #2 + br x9 +L(ipred_cfl_w4): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.4h + cmp w4, #4 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + add x2, x2, #2 + br x9 +L(ipred_cfl_w8): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.8h + cmp w4, #8 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h}, [x2], #32 + addp v0.8h, v2.8h, v3.8h + add x2, x2, #2 + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w16): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v2.8h, v2.8h, v3.8h + uaddlv s2, v2.8h + cmp w4, #16 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + add x2, x2, #2 + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w32): + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + add v0.4s, v0.4s, v16.4s + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v2.8h, v2.8h, v4.8h + cmp w4, #32 + uaddlv s2, v2.8h + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_tbl): + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h4) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) +endfunc + +// void cfl_ac_420_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_420_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_420_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + subs w8, w8, #2 + st1 {v0.8h}, [x0], #16 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + b.gt 1b + trn2 v1.2d, v0.2d, v0.2d + trn2 v0.2d, v0.2d, v0.2d +L(ipred_cfl_ac_420_w4_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + add v24.4s, v24.4s, v25.4s + add v26.4s, v26.4s, v27.4s + add v0.4s, v24.4s, v26.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #3 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] +6: // Subtract dc from ac + ld1 {v0.8h, v1.8h}, [x0] + subs w6, w6, #4 + sub v0.8h, v0.8h, v4.8h + sub v1.8h, v1.8h, v4.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 6b + ret + +L(ipred_cfl_ac_420_w8): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v4.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + mov v0.16b, v1.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + dup v1.4h, v0.h[3] + dup v3.4h, v0.h[7] + trn2 v2.2d, v0.2d, v0.2d + subs w8, w8, #2 + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw v25.4s, v25.4s, v1.4h + uaddw v26.4s, v26.4s, v2.4h + uaddw v27.4s, v27.4s, v3.4h + b.gt 1b + trn1 v0.2d, v2.2d, v3.2d + trn1 v1.2d, v2.2d, v3.2d + +L(ipred_cfl_ac_420_w8_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl w6, w6, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_420_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_420_w16_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x1], x2 + add v0.8h, v0.8h, v4.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x10], x2 + add v2.8h, v2.8h, v6.8h + addp v16.8h, v16.8h, v17.8h + addp v18.8h, v18.8h, v19.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + add v16.8h, v16.8h, v20.8h + add v18.8h, v18.8h, v22.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v2.8h, #1 + shl v2.8h, v16.8h, #1 + shl v3.8h, v18.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q5, [x10, #32] + ld1 {v3.8h, v4.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v5.8h, v5.8h, v5.8h + addp v3.8h, v3.8h, v4.8h + ldr q18, [x1, #32] + add v2.4h, v2.4h, v5.4h + ld1 {v16.8h, v17.8h}, [x1], x2 + add v0.8h, v0.8h, v3.8h + ldr q21, [x10, #32] + ld1 {v19.8h, v20.8h}, [x10], x2 + addp v18.8h, v18.8h, v18.8h + addp v16.8h, v16.8h, v17.8h + addp v21.8h, v21.8h, v21.8h + addp v19.8h, v19.8h, v20.8h + add v18.4h, v18.4h, v21.4h + add v16.8h, v16.8h, v19.8h + shl v1.4h, v2.4h, #1 + shl v0.8h, v0.8h, #1 + shl v3.4h, v18.4h, #1 + shl v2.8h, v16.8h, #1 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v2.8h, v4.8h, #1 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + ld1 {v4.8h}, [x1], x2 + ld1 {v6.8h}, [x10], x2 + addp v0.8h, v0.8h, v4.8h + addp v2.8h, v2.8h, v6.8h + add v0.8h, v0.8h, v2.8h + shl v0.8h, v0.8h, #1 + dup v1.8h, v0.h[3] + dup v3.8h, v0.h[7] + trn2 v2.2d, v0.2d, v3.2d + trn1 v0.2d, v0.2d, v1.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + +L(ipred_cfl_ac_420_w16_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl w6, w6, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_tbl): + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w4) + .hword 0 + +L(ipred_cfl_ac_420_w16_tbl): + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad0) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) +endfunc + +// void cfl_ac_422_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_422_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_422_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v4.4h, v0.h[3] + dup v5.8h, v0.h[7] + dup v6.4h, v2.h[3] + dup v7.8h, v2.h[7] + trn2 v1.2d, v0.2d, v5.2d + trn1 v0.2d, v0.2d, v4.2d + trn2 v3.2d, v2.2d, v7.2d + trn1 v2.2d, v2.2d, v6.2d + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_422_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_422_w16_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q6, [x10, #32] + ld1 {v4.8h, v5.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v6.8h, v6.8h, v6.8h + addp v4.8h, v4.8h, v5.8h + shl v1.4h, v2.4h, #2 + shl v0.8h, v0.8h, #2 + shl v3.4h, v6.4h, #2 + shl v2.8h, v4.8h, #2 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): + AARCH64_VALID_JUMP_TARGET +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + addp v0.8h, v0.8h, v0.8h + addp v2.8h, v2.8h, v2.8h + shl v0.4h, v0.4h, #2 + shl v2.4h, v2.4h, #2 + dup v1.8h, v0.h[3] + dup v3.8h, v2.h[3] + trn1 v0.2d, v0.2d, v1.2d + trn1 v2.2d, v2.2d, v3.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_tbl): + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w16) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w8) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w4) + .hword 0 + +L(ipred_cfl_ac_422_w16_tbl): + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3) +endfunc + +// void cfl_ac_444_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_444_tbl) + sub w8, w8, #26 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_444_w4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input + ld1 {v0.4h}, [x1], x2 + ld1 {v0.d}[1], [x10], x2 + ld1 {v1.4h}, [x1], x2 + ld1 {v1.d}[1], [x10], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + ld1 {v3.8h}, [x10], x2 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + AARCH64_VALID_JUMP_TARGET + cbnz w3, L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + shl v0.8h, v0.8h, #3 + shl v2.8h, v2.8h, #3 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + AARCH64_VALID_JUMP_TARGET + adr x7, L(ipred_cfl_ac_444_w32_tbl) + ldrh w3, [x7, w3, uxtw] // (w3>>1) << 1 + lsr x2, x2, #1 // Restore the stride to one line increments + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_444_w32_wpad0): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 8 + ld1 {v0.8h, v1.8h, v2.8h}, [x1], x2 + shl v2.8h, v2.8h, #3 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + dup v3.8h, v2.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 16 + ld1 {v0.8h, v1.8h}, [x1], x2 + shl v1.8h, v1.8h, #3 + shl v0.8h, v0.8h, #3 + dup v2.8h, v1.h[7] + dup v3.8h, v1.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): + AARCH64_VALID_JUMP_TARGET +1: // Copy and expand input, padding 24 + ld1 {v0.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + dup v1.8h, v0.h[7] + dup v2.8h, v0.h[7] + dup v3.8h, v0.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + +L(ipred_cfl_ac_444_w32_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 2b +3: + + // Multiply the height by eight and reuse the w4 subtracting + lsl w6, w6, #3 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_444_tbl): + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w32) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w16) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w8) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w4) + +L(ipred_cfl_ac_444_w32_tbl): + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad0) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad2) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad4) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad6) +endfunc diff --git a/third_party/dav1d/src/arm/64/itx.S b/third_party/dav1d/src/arm/64/itx.S new file mode 100644 index 0000000000..b1b2f8fe65 --- /dev/null +++ b/third_party/dav1d/src/arm/64/itx.S @@ -0,0 +1,3270 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob); + +// Most of the functions use the following register layout: +// x0-x3 external parameters +// x4 function pointer to first transform +// x5 function pointer to second transform +// x6 output parameter for helper function +// x7 input parameter for helper function +// x8 input stride for helper function +// x9-x12 scratch variables for helper functions +// x13 pointer to list of eob thresholds +// x14 return pointer for helper function +// x15 return pointer for main function + +// The SIMD registers most often use the following layout: +// v0-v1 multiplication coefficients +// v2-v7 scratch registers +// v8-v15 unused +// v16-v31 inputs/outputs of transforms + +// Potential further optimizations, that are left unimplemented for now: +// - Trying to keep multiplication coefficients in registers across multiple +// transform functions. (The register layout is designed to potentially +// allow this.) +// - Use a simplified version of the transforms themselves for cases where +// we know a significant number of inputs are zero. E.g. if the eob value +// indicates only a quarter of input values are set, for idct16 and up, +// a significant amount of calculation can be skipped, at the cost of more +// code duplication and special casing. + +const idct_coeffs, align=4 + // idct4 + .short 2896, 2896*8, 1567, 3784 + // idct8 + .short 799, 4017, 3406, 2276 + // idct16 + .short 401, 4076, 3166, 2598 + .short 1931, 3612, 3920, 1189 + // idct32 + .short 201, 4091, 3035, 2751 + .short 1751, 3703, 3857, 1380 + .short 995, 3973, 3513, 2106 + .short 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .short 101*8, 4095*8, 2967*8, -2824*8 + .short 1660*8, 3745*8, 3822*8, -1474*8 + .short 4076, 401, 4017, 799 + .short 0, 0, 0, 0 + + .short 4036*8, -700*8, 2359*8, 3349*8 + .short 3461*8, -2191*8, 897*8, 3996*8 + .short -3166, -2598, -799, -4017 + .short 0, 0, 0, 0 + + .short 501*8, 4065*8, 3229*8, -2520*8 + .short 2019*8, 3564*8, 3948*8, -1092*8 + .short 3612, 1931, 2276, 3406 + .short 0, 0, 0, 0 + + .short 4085*8, -301*8, 2675*8, 3102*8 + .short 3659*8, -1842*8, 1285*8, 3889*8 + .short -3920, -1189, -3406, -2276 + .short 0, 0, 0, 0 +endconst + +const iadst4_coeffs, align=4 + // .h[4-5] can be interpreted as .s[2] + .short 1321, 3803, 2482, 3344, 3344, 0 +endconst + +const iadst8_coeffs, align=4 + .short 4076, 401, 3612, 1931 + .short 2598, 3166, 1189, 3920 + // idct_coeffs + .short 2896, 0, 1567, 3784, 0, 0, 0, 0 +endconst + +const iadst16_coeffs, align=4 + .short 4091, 201, 3973, 995 + .short 3703, 1751, 3290, 2440 + .short 2751, 3035, 2106, 3513 + .short 1380, 3857, 601, 4052 +endconst + +.macro smull_smlal d0, d1, s0, s1, c0, c1, sz + smull \d0\().4s, \s0\().4h, \c0 + smlal \d0\().4s, \s1\().4h, \c1 +.ifc \sz, .8h + smull2 \d1\().4s, \s0\().8h, \c0 + smlal2 \d1\().4s, \s1\().8h, \c1 +.endif +.endm + +.macro smull_smlsl d0, d1, s0, s1, c0, c1, sz + smull \d0\().4s, \s0\().4h, \c0 + smlsl \d0\().4s, \s1\().4h, \c1 +.ifc \sz, .8h + smull2 \d1\().4s, \s0\().8h, \c0 + smlsl2 \d1\().4s, \s1\().8h, \c1 +.endif +.endm + +.macro sqrshrn_sz d0, s0, s1, shift, sz + sqrshrn \d0\().4h, \s0\().4s, \shift +.ifc \sz, .8h + sqrshrn2 \d0\().8h, \s1\().4s, \shift +.endif +.endm + +.macro scale_input sz, c, r0, r1, r2 r3, r4, r5, r6, r7 + sqrdmulh \r0\sz, \r0\sz, \c + sqrdmulh \r1\sz, \r1\sz, \c + sqrdmulh \r2\sz, \r2\sz, \c + sqrdmulh \r3\sz, \r3\sz, \c +.ifnb \r4 + sqrdmulh \r4\sz, \r4\sz, \c + sqrdmulh \r5\sz, \r5\sz, \c + sqrdmulh \r6\sz, \r6\sz, \c + sqrdmulh \r7\sz, \r7\sz, \c +.endif +.endm + +.macro load_add_store load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src, shiftbits=4 +.ifnb \load + ld1 {\load}, [\src], x1 +.endif +.ifnb \shift + srshr \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + uaddw \adddst, \adddst, \addsrc +.endif +.ifnb \narrowsrc + sqxtun \narrowdst, \narrowsrc +.endif +.ifnb \store + st1 {\store}, [\dst], x1 +.endif +.endm +.macro load_add_store_8x16 dst, src + mov \src, \dst + load_add_store v2.8b, v16.8h, , , , , , \dst, \src + load_add_store v3.8b, v17.8h, , , , , , \dst, \src + load_add_store v4.8b, v18.8h, v2.8b, v16.8h, , , , \dst, \src + load_add_store v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b, , \dst, \src + load_add_store v6.8b, v20.8h, v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src + load_add_store v7.8b, v21.8h, v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src + load_add_store v2.8b, v22.8h, v6.8b, v20.8h, v19.8h, v5.8b, v4.8b, \dst, \src + load_add_store v3.8b, v23.8h, v7.8b, v21.8h, v20.8h, v6.8b, v5.8b, \dst, \src + load_add_store v4.8b, v24.8h, v2.8b, v22.8h, v21.8h, v7.8b, v6.8b, \dst, \src + load_add_store v5.8b, v25.8h, v3.8b, v23.8h, v22.8h, v2.8b, v7.8b, \dst, \src + load_add_store v6.8b, v26.8h, v4.8b, v24.8h, v23.8h, v3.8b, v2.8b, \dst, \src + load_add_store v7.8b, v27.8h, v5.8b, v25.8h, v24.8h, v4.8b, v3.8b, \dst, \src + load_add_store v2.8b, v28.8h, v6.8b, v26.8h, v25.8h, v5.8b, v4.8b, \dst, \src + load_add_store v3.8b, v29.8h, v7.8b, v27.8h, v26.8h, v6.8b, v5.8b, \dst, \src + load_add_store v4.8b, v30.8h, v2.8b, v28.8h, v27.8h, v7.8b, v6.8b, \dst, \src + load_add_store v5.8b, v31.8h, v3.8b, v29.8h, v28.8h, v2.8b, v7.8b, \dst, \src + load_add_store , , v4.8b, v30.8h, v29.8h, v3.8b, v2.8b, \dst, \src + load_add_store , , v5.8b, v31.8h, v30.8h, v4.8b, v3.8b, \dst, \src + load_add_store , , , , v31.8h, v5.8b, v4.8b, \dst, \src + load_add_store , , , , , , v5.8b, \dst, \src +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + load_add_store v2.8b, v16.8h, , , , , , \dst, \src, \shiftbits + load_add_store v3.8b, v17.8h, , , , , , \dst, \src, \shiftbits + load_add_store v4.8b, v18.8h, v2.8b, v16.8h, , , , \dst, \src, \shiftbits + load_add_store v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b, , \dst, \src, \shiftbits + load_add_store v6.8b, v20.8h, v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src, \shiftbits + load_add_store v7.8b, v21.8h, v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src, \shiftbits + load_add_store v2.8b, v22.8h, v6.8b, v20.8h, v19.8h, v5.8b, v4.8b, \dst, \src, \shiftbits + load_add_store v3.8b, v23.8h, v7.8b, v21.8h, v20.8h, v6.8b, v5.8b, \dst, \src, \shiftbits + load_add_store , , v2.8b, v22.8h, v21.8h, v7.8b, v6.8b, \dst, \src, \shiftbits + load_add_store , , v3.8b, v23.8h, v22.8h, v2.8b, v7.8b, \dst, \src, \shiftbits + load_add_store , , , , v23.8h, v3.8b, v2.8b, \dst, \src, \shiftbits + load_add_store , , , , , , v3.8b, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src + mov \src, \dst + load_add_store v2.8b, v16.8h, , , , , , \dst, \src + load_add_store v3.8b, v17.8h, , , , , , \dst, \src + load_add_store v4.8b, v18.8h, v2.8b, v16.8h, , , , \dst, \src + load_add_store v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b, , \dst, \src + load_add_store , , v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src + load_add_store , , v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src + load_add_store , , , , v19.8h, v5.8b, v4.8b, \dst, \src + load_add_store , , , , , , v5.8b, \dst, \src +.endm +.macro load_add_store4 load, inssrc, insdst, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src +.ifnb \load + ld1 {\load}[0], [\src], x1 +.endif +.ifnb \inssrc + ins \insdst\().d[1], \inssrc\().d[0] +.endif +.ifnb \shift + srshr \shift, \shift, #4 +.endif +.ifnb \load + ld1 {\load}[1], [\src], x1 +.endif +.ifnb \addsrc + uaddw \adddst, \adddst, \addsrc +.endif +.ifnb \store + st1 {\store}[0], [\dst], x1 +.endif +.ifnb \narrowsrc + sqxtun \narrowdst, \narrowsrc +.endif +.ifnb \store + st1 {\store}[1], [\dst], x1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + load_add_store4 v0.s, v17, v16, , , , , , , \dst, \src + load_add_store4 v1.s, v19, v18, , , , , , , \dst, \src + load_add_store4 v2.s, v21, v20, v16.8h, , , , , , \dst, \src + load_add_store4 v3.s, v23, v22, v18.8h, v0.8b, v16.8h, , , , \dst, \src + load_add_store4 v4.s, v25, v24, v20.8h, v1.8b, v18.8h, v16.8h, v0.8b, , \dst, \src + load_add_store4 v5.s, v27, v26, v22.8h, v2.8b, v20.8h, v18.8h, v1.8b, v0.s, \dst, \src + load_add_store4 v6.s, v29, v28, v24.8h, v3.8b, v22.8h, v20.8h, v2.8b, v1.s, \dst, \src + load_add_store4 v7.s, v31, v30, v26.8h, v4.8b, v24.8h, v22.8h, v3.8b, v2.s, \dst, \src + load_add_store4 , , , v28.8h, v5.8b, v26.8h, v24.8h, v4.8b, v3.s, \dst, \src + load_add_store4 , , , v30.8h, v6.8b, v28.8h, v26.8h, v5.8b, v4.s, \dst, \src + load_add_store4 , , , , v7.8b, v30.8h, v28.8h, v6.8b, v5.s, \dst, \src + load_add_store4 , , , , , , v30.8h, v7.8b, v6.s, \dst, \src + load_add_store4 , , , , , , , , v7.s, \dst, \src +.endm +.macro load_add_store_4x8 dst, src + mov \src, \dst + load_add_store4 v0.s, v17, v16, , , , , , , \dst, \src + load_add_store4 v1.s, v19, v18, , , , , , , \dst, \src + load_add_store4 v2.s, v21, v20, v16.8h, , , , , , \dst, \src + load_add_store4 v3.s, v23, v22, v18.8h, v0.8b, v16.8h, , , , \dst, \src + load_add_store4 , , , v20.8h, v1.8b, v18.8h, v16.8h, v0.8b, , \dst, \src + load_add_store4 , , , v22.8h, v2.8b, v20.8h, v18.8h, v1.8b, v0.s, \dst, \src + load_add_store4 , , , , v3.8b, v22.8h, v20.8h, v2.8b, v1.s, \dst, \src + load_add_store4 , , , , , , v22.8h, v3.8b, v2.s, \dst, \src + load_add_store4 , , , , , , , , v3.s, \dst, \src +.endm + +.macro idct_dc w, h, shift + cbnz w3, 1f + mov w16, #2896*8 + ld1r {v16.8h}, [x2] + dup v0.4h, w16 + sqrdmulh v16.8h, v16.8h, v0.h[0] + strh wzr, [x2] +.if (\w == 2*\h) || (2*\w == \h) + sqrdmulh v16.8h, v16.8h, v0.h[0] +.endif +.if \shift > 0 + srshr v16.8h, v16.8h, #\shift +.endif + sqrdmulh v16.8h, v16.8h, v0.h[0] + srshr v16.8h, v16.8h, #4 + mov w4, #\h + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon +1: + ld1 {v0.s}[0], [x0], x1 + ld1 {v0.s}[1], [x0], x1 + ld1 {v1.s}[0], [x0], x1 + ld1 {v1.s}[1], [x0], x1 + subs w4, w4, #4 + sub x0, x0, x1, lsl #2 + uaddw v0.8h, v16.8h, v0.8b + sqxtun v0.8b, v0.8h + uaddw v1.8h, v16.8h, v1.8b + st1 {v0.s}[0], [x0], x1 + sqxtun v1.8b, v1.8h + st1 {v0.s}[1], [x0], x1 + st1 {v1.s}[0], [x0], x1 + st1 {v1.s}[1], [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w8_neon +1: + ld1 {v0.8b}, [x0], x1 + ld1 {v1.8b}, [x0], x1 + ld1 {v2.8b}, [x0], x1 + uaddw v20.8h, v16.8h, v0.8b + ld1 {v3.8b}, [x0], x1 + sub x0, x0, x1, lsl #2 + subs w4, w4, #4 + uaddw v21.8h, v16.8h, v1.8b + sqxtun v0.8b, v20.8h + uaddw v22.8h, v16.8h, v2.8b + sqxtun v1.8b, v21.8h + uaddw v23.8h, v16.8h, v3.8b + st1 {v0.8b}, [x0], x1 + sqxtun v2.8b, v22.8h + st1 {v1.8b}, [x0], x1 + sqxtun v3.8b, v23.8h + st1 {v2.8b}, [x0], x1 + st1 {v3.8b}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w16_neon +1: + ld1 {v0.16b}, [x0], x1 + ld1 {v1.16b}, [x0], x1 + ld1 {v2.16b}, [x0], x1 + subs w4, w4, #4 + uaddw v20.8h, v16.8h, v0.8b + uaddw2 v21.8h, v16.8h, v0.16b + ld1 {v3.16b}, [x0], x1 + uaddw v22.8h, v16.8h, v1.8b + uaddw2 v23.8h, v16.8h, v1.16b + sub x0, x0, x1, lsl #2 + uaddw v24.8h, v16.8h, v2.8b + uaddw2 v25.8h, v16.8h, v2.16b + sqxtun v0.8b, v20.8h + sqxtun2 v0.16b, v21.8h + uaddw v26.8h, v16.8h, v3.8b + uaddw2 v27.8h, v16.8h, v3.16b + sqxtun v1.8b, v22.8h + sqxtun2 v1.16b, v23.8h + sqxtun v2.8b, v24.8h + sqxtun2 v2.16b, v25.8h + st1 {v0.16b}, [x0], x1 + sqxtun v3.8b, v26.8h + sqxtun2 v3.16b, v27.8h + st1 {v1.16b}, [x0], x1 + st1 {v2.16b}, [x0], x1 + st1 {v3.16b}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w32_neon +1: + ld1 {v0.16b, v1.16b}, [x0], x1 + subs w4, w4, #2 + uaddw v20.8h, v16.8h, v0.8b + uaddw2 v21.8h, v16.8h, v0.16b + ld1 {v2.16b, v3.16b}, [x0] + uaddw v22.8h, v16.8h, v1.8b + uaddw2 v23.8h, v16.8h, v1.16b + sub x0, x0, x1 + uaddw v24.8h, v16.8h, v2.8b + uaddw2 v25.8h, v16.8h, v2.16b + sqxtun v0.8b, v20.8h + sqxtun2 v0.16b, v21.8h + uaddw v26.8h, v16.8h, v3.8b + uaddw2 v27.8h, v16.8h, v3.16b + sqxtun v1.8b, v22.8h + sqxtun2 v1.16b, v23.8h + sqxtun v2.8b, v24.8h + sqxtun2 v2.16b, v25.8h + st1 {v0.16b, v1.16b}, [x0], x1 + sqxtun v3.8b, v26.8h + sqxtun2 v3.16b, v27.8h + st1 {v2.16b, v3.16b}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w64_neon +1: + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0] + subs w4, w4, #1 + uaddw v20.8h, v16.8h, v0.8b + uaddw2 v21.8h, v16.8h, v0.16b + uaddw v22.8h, v16.8h, v1.8b + uaddw2 v23.8h, v16.8h, v1.16b + uaddw v24.8h, v16.8h, v2.8b + uaddw2 v25.8h, v16.8h, v2.16b + sqxtun v0.8b, v20.8h + sqxtun2 v0.16b, v21.8h + uaddw v26.8h, v16.8h, v3.8b + uaddw2 v27.8h, v16.8h, v3.16b + sqxtun v1.8b, v22.8h + sqxtun2 v1.16b, v23.8h + sqxtun v2.8b, v24.8h + sqxtun2 v2.16b, v25.8h + sqxtun v3.8b, v26.8h + sqxtun2 v3.16b, v27.8h + st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 + b.gt 1b + ret +endfunc + +.macro iwht4 + add v16.4h, v16.4h, v17.4h + sub v21.4h, v18.4h, v19.4h + sub v20.4h, v16.4h, v21.4h + sshr v20.4h, v20.4h, #1 + sub v18.4h, v20.4h, v17.4h + sub v17.4h, v20.4h, v19.4h + add v19.4h, v21.4h, v18.4h + sub v16.4h, v16.4h, v17.4h +.endm + +.macro idct_4 r0, r1, r2, r3, sz + smull_smlal v6, v7, \r1, \r3, v0.h[3], v0.h[2], \sz + smull_smlsl v4, v5, \r1, \r3, v0.h[2], v0.h[3], \sz + smull_smlal v2, v3, \r0, \r2, v0.h[0], v0.h[0], \sz + sqrshrn_sz v6, v6, v7, #12, \sz + sqrshrn_sz v7, v4, v5, #12, \sz + smull_smlsl v4, v5, \r0, \r2, v0.h[0], v0.h[0], \sz + sqrshrn_sz v2, v2, v3, #12, \sz + sqrshrn_sz v3, v4, v5, #12, \sz + sqadd \r0\sz, v2\sz, v6\sz + sqsub \r3\sz, v2\sz, v6\sz + sqadd \r1\sz, v3\sz, v7\sz + sqsub \r2\sz, v3\sz, v7\sz +.endm + +function inv_dct_4h_x4_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.4h}, [x16] + idct_4 v16, v17, v18, v19, .4h + ret +endfunc + +function inv_dct_8h_x4_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.4h}, [x16] + idct_4 v16, v17, v18, v19, .8h + ret +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel x16, iadst4_coeffs + ld1 {v0.8h}, [x16] + + ssubl v3.4s, v16.4h, v18.4h + smull v4.4s, v16.4h, v0.h[0] + smlal v4.4s, v18.4h, v0.h[1] + smlal v4.4s, v19.4h, v0.h[2] + smull v7.4s, v17.4h, v0.h[3] + saddw v3.4s, v3.4s, v19.4h + smull v5.4s, v16.4h, v0.h[2] + smlsl v5.4s, v18.4h, v0.h[0] + smlsl v5.4s, v19.4h, v0.h[1] + + add \o3\().4s, v4.4s, v5.4s + mul \o2\().4s, v3.4s, v0.s[2] + add \o0\().4s, v4.4s, v7.4s + add \o1\().4s, v5.4s, v7.4s + sub \o3\().4s, \o3\().4s, v7.4s + + sqrshrn \o0\().4h, \o0\().4s, #12 + sqrshrn \o2\().4h, \o2\().4s, #12 + sqrshrn \o1\().4h, \o1\().4s, #12 + sqrshrn \o3\().4h, \o3\().4s, #12 +.endm + +function inv_adst_4h_x4_neon, export=1 + iadst_4x4 v16, v17, v18, v19 + ret +endfunc + +function inv_flipadst_4h_x4_neon, export=1 + iadst_4x4 v19, v18, v17, v16 + ret +endfunc + +.macro iadst_8x4 o0, o1, o2, o3 + movrel x16, iadst4_coeffs + ld1 {v0.8h}, [x16] + + ssubl v2.4s, v16.4h, v18.4h + ssubl2 v3.4s, v16.8h, v18.8h + smull v4.4s, v16.4h, v0.h[0] + smlal v4.4s, v18.4h, v0.h[1] + smlal v4.4s, v19.4h, v0.h[2] + smull2 v5.4s, v16.8h, v0.h[0] + smlal2 v5.4s, v18.8h, v0.h[1] + smlal2 v5.4s, v19.8h, v0.h[2] + saddw v2.4s, v2.4s, v19.4h + saddw2 v3.4s, v3.4s, v19.8h + smull v6.4s, v16.4h, v0.h[2] + smlsl v6.4s, v18.4h, v0.h[0] + smlsl v6.4s, v19.4h, v0.h[1] + smull2 v7.4s, v16.8h, v0.h[2] + smlsl2 v7.4s, v18.8h, v0.h[0] + smlsl2 v7.4s, v19.8h, v0.h[1] + + mul v18.4s, v2.4s, v0.s[2] + mul v19.4s, v3.4s, v0.s[2] + + smull v2.4s, v17.4h, v0.h[3] + smull2 v3.4s, v17.8h, v0.h[3] + + add v16.4s, v4.4s, v2.4s // out0 + add v17.4s, v5.4s, v3.4s + + add v4.4s, v4.4s, v6.4s // out3 + add v5.4s, v5.4s, v7.4s + + add v6.4s, v6.4s, v2.4s // out1 + add v7.4s, v7.4s, v3.4s + + sub v4.4s, v4.4s, v2.4s // out3 + sub v5.4s, v5.4s, v3.4s + + sqrshrn v18.4h, v18.4s, #12 + sqrshrn2 v18.8h, v19.4s, #12 + + sqrshrn \o0\().4h, v16.4s, #12 + sqrshrn2 \o0\().8h, v17.4s, #12 + +.ifc \o2, v17 + mov v17.16b, v18.16b +.endif + + sqrshrn \o1\().4h, v6.4s, #12 + sqrshrn2 \o1\().8h, v7.4s, #12 + + sqrshrn \o3\().4h, v4.4s, #12 + sqrshrn2 \o3\().8h, v5.4s, #12 +.endm + +function inv_adst_8h_x4_neon, export=1 + iadst_8x4 v16, v17, v18, v19 + ret +endfunc + +function inv_flipadst_8h_x4_neon, export=1 + iadst_8x4 v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4h_x4_neon, export=1 + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + sqrdmulh v4.4h, v16.4h, v0.h[0] + sqrdmulh v5.4h, v17.4h, v0.h[0] + sqrdmulh v6.4h, v18.4h, v0.h[0] + sqrdmulh v7.4h, v19.4h, v0.h[0] + sqadd v16.4h, v16.4h, v4.4h + sqadd v17.4h, v17.4h, v5.4h + sqadd v18.4h, v18.4h, v6.4h + sqadd v19.4h, v19.4h, v7.4h + ret +endfunc + +function inv_identity_8h_x4_neon, export=1 + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + sqrdmulh v4.8h, v16.8h, v0.h[0] + sqrdmulh v5.8h, v17.8h, v0.h[0] + sqrdmulh v6.8h, v18.8h, v0.h[0] + sqrdmulh v7.8h, v19.8h, v0.h[0] + sqadd v16.8h, v16.8h, v4.8h + sqadd v17.8h, v17.8h, v5.8h + sqadd v18.8h, v18.8h, v6.8h + sqadd v19.8h, v19.8h, v7.8h + ret +endfunc + +.macro identity_8x4_shift1 r0, r1, r2, r3, c +.irp i, \r0\().8h, \r1\().8h, \r2\().8h, \r3\().8h + sqrdmulh v2.8h, \i, \c + srhadd \i, \i, v2.8h +.endr +.endm + +function inv_txfm_add_wht_wht_4x4_8bpc_neon, export=1 + mov x15, x30 + movi v31.8h, #0 + ld1 {v16.4h,v17.4h,v18.4h,v19.4h}, [x2] + st1 {v31.8h}, [x2], #16 + + sshr v16.4h, v16.4h, #2 + sshr v17.4h, v17.4h, #2 + sshr v18.4h, v18.4h, #2 + sshr v19.4h, v19.4h, #2 + + iwht4 + + st1 {v31.8h}, [x2], #16 + transpose_4x4h v16, v17, v18, v19, v20, v21, v22, v23 + + iwht4 + + ld1 {v0.s}[0], [x0], x1 + ld1 {v0.s}[1], [x0], x1 + ins v16.d[1], v17.d[0] + ins v18.d[1], v19.d[0] + ld1 {v1.s}[0], [x0], x1 + ld1 {v1.s}[1], [x0], x1 + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + movi v31.8h, #0 + ld1 {v16.4h,v17.4h,v18.4h,v19.4h}, [x2] + st1 {v31.8h}, [x2], #16 + + blr x4 + + st1 {v31.8h}, [x2], #16 + transpose_4x4h v16, v17, v18, v19, v20, v21, v22, v23 + + blr x5 + + ld1 {v0.s}[0], [x0], x1 + ld1 {v0.s}[1], [x0], x1 + ins v16.d[1], v17.d[0] + ins v18.d[1], v19.d[0] + ld1 {v1.s}[0], [x0], x1 + ld1 {v1.s}[1], [x0], x1 + srshr v16.8h, v16.8h, #4 + srshr v18.8h, v18.8h, #4 + +L(itx_4x4_end): + sub x0, x0, x1, lsl #2 + uaddw v16.8h, v16.8h, v0.8b + sqxtun v0.8b, v16.8h + uaddw v18.8h, v18.8h, v1.8b + st1 {v0.s}[0], [x0], x1 + sqxtun v1.8b, v18.8h + st1 {v0.s}[1], [x0], x1 + st1 {v1.s}[0], [x0], x1 + st1 {v1.s}[1], [x0], x1 + + ret x15 +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_8bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + cbnz w3, 1f + mov w16, #2896*8 + ld1r {v16.8h}, [x2] + dup v4.8h, w16 + strh wzr, [x2] + sqrdmulh v16.8h, v16.8h, v4.h[0] + ld1 {v0.s}[0], [x0], x1 + sqrdmulh v20.8h, v16.8h, v4.h[0] + ld1 {v0.s}[1], [x0], x1 + srshr v16.8h, v20.8h, #4 + ld1 {v1.s}[0], [x0], x1 + srshr v18.8h, v20.8h, #4 + ld1 {v1.s}[1], [x0], x1 + b L(itx_4x4_end) +1: +.endif + adr x4, inv_\txfm1\()_4h_x4_neon + adr x5, inv_\txfm2\()_4h_x4_neon + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_8 r0, r1, r2, r3, r4, r5, r6, r7, sz, szb + idct_4 \r0, \r2, \r4, \r6, \sz + + smull_smlsl v2, v3, \r1, \r7, v0.h[4], v0.h[5], \sz // -> t4a + smull_smlal v4, v5, \r1, \r7, v0.h[5], v0.h[4], \sz // -> t7a + smull_smlsl v6, v7, \r5, \r3, v0.h[6], v0.h[7], \sz // -> t5a + sqrshrn_sz \r1, v2, v3, #12, \sz // t4a + sqrshrn_sz \r7, v4, v5, #12, \sz // t7a + smull_smlal v2, v3, \r5, \r3, v0.h[7], v0.h[6], \sz // -> t6a + sqrshrn_sz \r3, v6, v7, #12, \sz // t5a + sqrshrn_sz \r5, v2, v3, #12, \sz // t6a + + sqadd v2\sz, \r1\sz, \r3\sz // t4 + sqsub \r1\sz, \r1\sz, \r3\sz // t5a + sqadd v3\sz, \r7\sz, \r5\sz // t7 + sqsub \r3\sz, \r7\sz, \r5\sz // t6a + + smull_smlsl v4, v5, \r3, \r1, v0.h[0], v0.h[0], \sz // -> t5 + smull_smlal v6, v7, \r3, \r1, v0.h[0], v0.h[0], \sz // -> t6 + sqrshrn_sz v4, v4, v5, #12, \sz // t5 + sqrshrn_sz v5, v6, v7, #12, \sz // t6 + + sqsub \r7\sz, \r0\sz, v3\sz // out7 + sqadd \r0\sz, \r0\sz, v3\sz // out0 + sqadd \r1\sz, \r2\sz, v5\sz // out1 + sqsub v6\sz, \r2\sz, v5\sz // out6 + sqadd \r2\sz, \r4\sz, v4\sz // out2 + sqsub \r5\sz, \r4\sz, v4\sz // out5 + sqadd \r3\sz, \r6\sz, v2\sz // out3 + sqsub \r4\sz, \r6\sz, v2\sz // out4 + mov \r6\szb, v6\szb // out6 +.endm + +function inv_dct_8h_x8_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.8h}, [x16] + idct_8 v16, v17, v18, v19, v20, v21, v22, v23, .8h, .16b + ret +endfunc + +function inv_dct_4h_x8_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.8h}, [x16] + idct_8 v16, v17, v18, v19, v20, v21, v22, v23, .4h, .8b + ret +endfunc + +.macro iadst_8 o0, o1, o2, o3, o4, o5, o6, o7, sz + movrel x16, iadst8_coeffs + ld1 {v0.8h, v1.8h}, [x16] + + smull_smlal v2, v3, v23, v16, v0.h[0], v0.h[1], \sz + smull_smlsl v4, v5, v23, v16, v0.h[1], v0.h[0], \sz + smull_smlal v6, v7, v21, v18, v0.h[2], v0.h[3], \sz + sqrshrn_sz v16, v2, v3, #12, \sz // t0a + sqrshrn_sz v23, v4, v5, #12, \sz // t1a + smull_smlsl v2, v3, v21, v18, v0.h[3], v0.h[2], \sz + smull_smlal v4, v5, v19, v20, v0.h[4], v0.h[5], \sz + sqrshrn_sz v18, v6, v7, #12, \sz // t2a + sqrshrn_sz v21, v2, v3, #12, \sz // t3a + smull_smlsl v6, v7, v19, v20, v0.h[5], v0.h[4], \sz + smull_smlal v2, v3, v17, v22, v0.h[6], v0.h[7], \sz + sqrshrn_sz v20, v4, v5, #12, \sz // t4a + sqrshrn_sz v19, v6, v7, #12, \sz // t5a + smull_smlsl v4, v5, v17, v22, v0.h[7], v0.h[6], \sz + sqrshrn_sz v22, v2, v3, #12, \sz // t6a + sqrshrn_sz v17, v4, v5, #12, \sz // t7a + + sqadd v2\sz, v16\sz, v20\sz // t0 + sqsub v3\sz, v16\sz, v20\sz // t4 + sqadd v4\sz, v23\sz, v19\sz // t1 + sqsub v5\sz, v23\sz, v19\sz // t5 + sqadd v6\sz, v18\sz, v22\sz // t2 + sqsub v7\sz, v18\sz, v22\sz // t6 + sqadd v18\sz, v21\sz, v17\sz // t3 + sqsub v19\sz, v21\sz, v17\sz // t7 + + smull_smlal v16, v17, v3, v5, v1.h[3], v1.h[2], \sz + smull_smlsl v20, v21, v3, v5, v1.h[2], v1.h[3], \sz + smull_smlsl v22, v23, v19, v7, v1.h[3], v1.h[2], \sz + + sqrshrn_sz v3, v16, v17, #12, \sz // t4a + sqrshrn_sz v5, v20, v21, #12, \sz // t5a + + smull_smlal v16, v17, v19, v7, v1.h[2], v1.h[3], \sz + + sqrshrn_sz v7, v22, v23, #12, \sz // t6a + sqrshrn_sz v19, v16, v17, #12, \sz // t7a + + sqadd \o0\()\sz, v2\sz, v6\sz // out0 + sqsub v2\sz, v2\sz, v6\sz // t2 + sqadd \o7\()\sz, v4\sz, v18\sz // out7 + sqsub v4\sz, v4\sz, v18\sz // t3 + sqneg \o7\()\sz, \o7\()\sz // out7 + + sqadd \o1\()\sz, v3\sz, v7\sz // out1 + sqsub v3\sz, v3\sz, v7\sz // t6 + sqadd \o6\()\sz, v5\sz, v19\sz // out6 + sqsub v5\sz, v5\sz, v19\sz // t7 + sqneg \o1\()\sz, \o1\()\sz // out1 + + smull_smlal v18, v19, v2, v4, v1.h[0], v1.h[0], \sz // -> out3 (v19 or v20) + smull_smlsl v6, v7, v2, v4, v1.h[0], v1.h[0], \sz // -> out4 (v20 or v19) + smull_smlsl v20, v21, v3, v5, v1.h[0], v1.h[0], \sz // -> out5 (v21 or v18) + sqrshrn_sz v2, v18, v19, #12, \sz // out3 + smull_smlal v18, v19, v3, v5, v1.h[0], v1.h[0], \sz // -> out2 (v18 or v21) + sqrshrn_sz v3, v20, v21, #12, \sz // out5 + sqrshrn_sz \o2, v18, v19, #12, \sz // out2 (v18 or v21) + sqrshrn_sz \o4, v6, v7, #12, \sz // out4 (v20 or v19) + + sqneg \o3\()\sz, v2\sz // out3 + sqneg \o5\()\sz, v3\sz // out5 +.endm + +function inv_adst_8h_x8_neon, export=1 + iadst_8 v16, v17, v18, v19, v20, v21, v22, v23, .8h + ret +endfunc + +function inv_flipadst_8h_x8_neon, export=1 + iadst_8 v23, v22, v21, v20, v19, v18, v17, v16, .8h + ret +endfunc + +function inv_adst_4h_x8_neon, export=1 + iadst_8 v16, v17, v18, v19, v20, v21, v22, v23, .4h + ret +endfunc + +function inv_flipadst_4h_x8_neon, export=1 + iadst_8 v23, v22, v21, v20, v19, v18, v17, v16, .4h + ret +endfunc + +function inv_identity_8h_x8_neon, export=1 + sqshl v16.8h, v16.8h, #1 + sqshl v17.8h, v17.8h, #1 + sqshl v18.8h, v18.8h, #1 + sqshl v19.8h, v19.8h, #1 + sqshl v20.8h, v20.8h, #1 + sqshl v21.8h, v21.8h, #1 + sqshl v22.8h, v22.8h, #1 + sqshl v23.8h, v23.8h, #1 + ret +endfunc + +function inv_identity_4h_x8_neon, export=1 + sqshl v16.4h, v16.4h, #1 + sqshl v17.4h, v17.4h, #1 + sqshl v18.4h, v18.4h, #1 + sqshl v19.4h, v19.4h, #1 + sqshl v20.4h, v20.4h, #1 + sqshl v21.4h, v21.4h, #1 + sqshl v22.4h, v22.4h, #1 + sqshl v23.4h, v23.4h, #1 + ret +endfunc + +.macro def_fn_8x8_base variant +function inv_txfm_\variant\()add_8x8_neon + movi v28.8h, #0 + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 + ld1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x2] + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x2], #64 + ld1 {v20.8h,v21.8h,v22.8h,v23.8h}, [x2] + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x2] + +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else + blr x4 + + srshr v16.8h, v16.8h, #1 + srshr v17.8h, v17.8h, #1 + srshr v18.8h, v18.8h, #1 + srshr v19.8h, v19.8h, #1 + srshr v20.8h, v20.8h, #1 + srshr v21.8h, v21.8h, #1 + srshr v22.8h, v22.8h, #1 + srshr v23.8h, v23.8h, #1 +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v24, v25 + + blr x5 + + load_add_store_8x8 x0, x7 + ret x15 +endfunc +.endm + +def_fn_8x8_base +def_fn_8x8_base identity_ + +.macro def_fn_8x8 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_8bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + adr x5, inv_\txfm2\()_8h_x8_neon +.ifc \txfm1, identity + b inv_txfm_identity_add_8x8_neon +.else + adr x4, inv_\txfm1\()_8h_x8_neon + b inv_txfm_add_8x8_neon +.endif +endfunc +.endm + +def_fn_8x8 dct, dct +def_fn_8x8 identity, identity +def_fn_8x8 dct, adst +def_fn_8x8 dct, flipadst +def_fn_8x8 dct, identity +def_fn_8x8 adst, dct +def_fn_8x8 adst, adst +def_fn_8x8 adst, flipadst +def_fn_8x8 flipadst, dct +def_fn_8x8 flipadst, adst +def_fn_8x8 flipadst, flipadst +def_fn_8x8 identity, dct +def_fn_8x8 adst, identity +def_fn_8x8 flipadst, identity +def_fn_8x8 identity, adst +def_fn_8x8 identity, flipadst + +function inv_txfm_add_8x4_neon + movi v30.8h, #0 + movi v31.8h, #0 + mov w16, #2896*8 + dup v0.4h, w16 + ld1 {v16.4h,v17.4h,v18.4h,v19.4h}, [x2] + st1 {v30.8h,v31.8h}, [x2], #32 + ld1 {v20.4h,v21.4h,v22.4h,v23.4h}, [x2] + st1 {v30.8h,v31.8h}, [x2] + + scale_input .4h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + + blr x4 + + transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7 + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + ins v16.d[1], v20.d[0] + ins v17.d[1], v21.d[0] + ins v18.d[1], v22.d[0] + ins v19.d[1], v23.d[0] + + blr x5 + + load_add_store_8x4 x0, x7 + ret x15 +endfunc + +function inv_txfm_add_4x8_neon + movi v28.8h, #0 + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 + mov w16, #2896*8 + dup v0.4h, w16 + ld1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x2] + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x2] + + scale_input .8h, v0.h[0], v16, v17, v18, v19 + + blr x4 + + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + ins v20.d[0], v16.d[1] + ins v21.d[0], v17.d[1] + ins v22.d[0], v18.d[1] + ins v23.d[0], v19.d[1] + + blr x5 + + load_add_store_4x8 x0, x7 + ret x15 +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + adr x4, inv_\txfm1\()_\h\()h_x\w\()_neon + adr x5, inv_\txfm2\()_\w\()h_x\h\()_neon + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct +def_fn_48 \w, \h, identity, identity +def_fn_48 \w, \h, dct, adst +def_fn_48 \w, \h, dct, flipadst +def_fn_48 \w, \h, dct, identity +def_fn_48 \w, \h, adst, dct +def_fn_48 \w, \h, adst, adst +def_fn_48 \w, \h, adst, flipadst +def_fn_48 \w, \h, flipadst, dct +def_fn_48 \w, \h, flipadst, adst +def_fn_48 \w, \h, flipadst, flipadst +def_fn_48 \w, \h, identity, dct +def_fn_48 \w, \h, adst, identity +def_fn_48 \w, \h, flipadst, identity +def_fn_48 \w, \h, identity, adst +def_fn_48 \w, \h, identity, flipadst +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + + +.macro idct_16 sz, szb + idct_8 v16, v18, v20, v22, v24, v26, v28, v30, \sz, \szb + + smull_smlsl v2, v3, v17, v31, v1.h[0], v1.h[1], \sz // -> t8a + smull_smlal v4, v5, v17, v31, v1.h[1], v1.h[0], \sz // -> t15a + smull_smlsl v6, v7, v25, v23, v1.h[2], v1.h[3], \sz // -> t9a + sqrshrn_sz v17, v2, v3, #12, \sz // t8a + sqrshrn_sz v31, v4, v5, #12, \sz // t15a + smull_smlal v2, v3, v25, v23, v1.h[3], v1.h[2], \sz // -> t14a + smull_smlsl v4, v5, v21, v27, v1.h[4], v1.h[5], \sz // -> t10a + sqrshrn_sz v23, v6, v7, #12, \sz // t9a + sqrshrn_sz v25, v2, v3, #12, \sz // t14a + smull_smlal v6, v7, v21, v27, v1.h[5], v1.h[4], \sz // -> t13a + smull_smlsl v2, v3, v29, v19, v1.h[6], v1.h[7], \sz // -> t11a + sqrshrn_sz v21, v4, v5, #12, \sz // t10a + sqrshrn_sz v27, v6, v7, #12, \sz // t13a + smull_smlal v4, v5, v29, v19, v1.h[7], v1.h[6], \sz // -> t12a + sqrshrn_sz v19, v2, v3, #12, \sz // t11a + sqrshrn_sz v29, v4, v5, #12, \sz // t12a + + sqsub v2\sz, v17\sz, v23\sz // t9 + sqadd v17\sz, v17\sz, v23\sz // t8 + sqsub v3\sz, v31\sz, v25\sz // t14 + sqadd v31\sz, v31\sz, v25\sz // t15 + sqsub v23\sz, v19\sz, v21\sz // t10 + sqadd v19\sz, v19\sz, v21\sz // t11 + sqadd v25\sz, v29\sz, v27\sz // t12 + sqsub v29\sz, v29\sz, v27\sz // t13 + + smull_smlsl v4, v5, v3, v2, v0.h[2], v0.h[3], \sz // -> t9a + smull_smlal v6, v7, v3, v2, v0.h[3], v0.h[2], \sz // -> t14a + sqrshrn_sz v21, v4, v5, #12, \sz // t9a + sqrshrn_sz v27, v6, v7, #12, \sz // t14a + + smull_smlsl v4, v5, v29, v23, v0.h[2], v0.h[3], \sz // -> t13a + smull_smlal v6, v7, v29, v23, v0.h[3], v0.h[2], \sz // -> t10a + sqrshrn_sz v29, v4, v5, #12, \sz // t13a + neg v6.4s, v6.4s +.ifc \sz, .8h + neg v7.4s, v7.4s +.endif + sqrshrn_sz v23, v6, v7, #12, \sz // t10a + + sqsub v2\sz, v17\sz, v19\sz // t11a + sqadd v17\sz, v17\sz, v19\sz // t8a + sqsub v3\sz, v31\sz, v25\sz // t12a + sqadd v31\sz, v31\sz, v25\sz // t15a + sqadd v19\sz, v21\sz, v23\sz // t9 + sqsub v21\sz, v21\sz, v23\sz // t10 + sqsub v25\sz, v27\sz, v29\sz // t13 + sqadd v27\sz, v27\sz, v29\sz // t14 + + smull_smlsl v4, v5, v3, v2, v0.h[0], v0.h[0], \sz // -> t11 + smull_smlal v6, v7, v3, v2, v0.h[0], v0.h[0], \sz // -> t12 + smull_smlsl v2, v3, v25, v21, v0.h[0], v0.h[0], \sz // -> t10a + + sqrshrn_sz v4, v4, v5, #12, \sz // t11 + sqrshrn_sz v5, v6, v7, #12, \sz // t12 + smull_smlal v6, v7, v25, v21, v0.h[0], v0.h[0], \sz // -> t13a + sqrshrn_sz v2, v2, v3, #12, \sz // t10a + sqrshrn_sz v3, v6, v7, #12, \sz // t13a + + sqadd v6\sz, v16\sz, v31\sz // out0 + sqsub v31\sz, v16\sz, v31\sz // out15 + mov v16\szb, v6\szb + sqadd v23\sz, v30\sz, v17\sz // out7 + sqsub v7\sz, v30\sz, v17\sz // out8 + sqadd v17\sz, v18\sz, v27\sz // out1 + sqsub v30\sz, v18\sz, v27\sz // out14 + sqadd v18\sz, v20\sz, v3\sz // out2 + sqsub v29\sz, v20\sz, v3\sz // out13 + sqadd v3\sz, v28\sz, v19\sz // out6 + sqsub v25\sz, v28\sz, v19\sz // out9 + sqadd v19\sz, v22\sz, v5\sz // out3 + sqsub v28\sz, v22\sz, v5\sz // out12 + sqadd v20\sz, v24\sz, v4\sz // out4 + sqsub v27\sz, v24\sz, v4\sz // out11 + sqadd v21\sz, v26\sz, v2\sz // out5 + sqsub v26\sz, v26\sz, v2\sz // out10 + mov v24\szb, v7\szb + mov v22\szb, v3\szb +.endm + +function inv_dct_8h_x16_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.8h, v1.8h}, [x16] + idct_16 .8h, .16b + ret +endfunc + +function inv_dct_4h_x16_neon, export=1 + movrel x16, idct_coeffs + ld1 {v0.8h, v1.8h}, [x16] + idct_16 .4h, .8b + ret +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, sz, szb + movrel x16, iadst16_coeffs + ld1 {v0.8h, v1.8h}, [x16] + movrel x16, idct_coeffs + + smull_smlal v2, v3, v31, v16, v0.h[0], v0.h[1], \sz // -> t0 + smull_smlsl v4, v5, v31, v16, v0.h[1], v0.h[0], \sz // -> t1 + smull_smlal v6, v7, v29, v18, v0.h[2], v0.h[3], \sz // -> t2 + sqrshrn_sz v16, v2, v3, #12, \sz // t0 + sqrshrn_sz v31, v4, v5, #12, \sz // t1 + smull_smlsl v2, v3, v29, v18, v0.h[3], v0.h[2], \sz // -> t3 + smull_smlal v4, v5, v27, v20, v0.h[4], v0.h[5], \sz // -> t4 + sqrshrn_sz v18, v6, v7, #12, \sz // t2 + sqrshrn_sz v29, v2, v3, #12, \sz // t3 + smull_smlsl v6, v7, v27, v20, v0.h[5], v0.h[4], \sz // -> t5 + smull_smlal v2, v3, v25, v22, v0.h[6], v0.h[7], \sz // -> t6 + sqrshrn_sz v20, v4, v5, #12, \sz // t4 + sqrshrn_sz v27, v6, v7, #12, \sz // t5 + smull_smlsl v4, v5, v25, v22, v0.h[7], v0.h[6], \sz // -> t7 + smull_smlal v6, v7, v23, v24, v1.h[0], v1.h[1], \sz // -> t8 + sqrshrn_sz v22, v2, v3, #12, \sz // t6 + sqrshrn_sz v25, v4, v5, #12, \sz // t7 + smull_smlsl v2, v3, v23, v24, v1.h[1], v1.h[0], \sz // -> t9 + smull_smlal v4, v5, v21, v26, v1.h[2], v1.h[3], \sz // -> t10 + sqrshrn_sz v23, v6, v7, #12, \sz // t8 + sqrshrn_sz v24, v2, v3, #12, \sz // t9 + smull_smlsl v6, v7, v21, v26, v1.h[3], v1.h[2], \sz // -> t11 + smull_smlal v2, v3, v19, v28, v1.h[4], v1.h[5], \sz // -> t12 + sqrshrn_sz v21, v4, v5, #12, \sz // t10 + sqrshrn_sz v26, v6, v7, #12, \sz // t11 + smull_smlsl v4, v5, v19, v28, v1.h[5], v1.h[4], \sz // -> t13 + smull_smlal v6, v7, v17, v30, v1.h[6], v1.h[7], \sz // -> t14 + sqrshrn_sz v19, v2, v3, #12, \sz // t12 + sqrshrn_sz v28, v4, v5, #12, \sz // t13 + smull_smlsl v2, v3, v17, v30, v1.h[7], v1.h[6], \sz // -> t15 + sqrshrn_sz v17, v6, v7, #12, \sz // t14 + sqrshrn_sz v30, v2, v3, #12, \sz // t15 + + ld1 {v0.8h}, [x16] + + sqsub v2\sz, v16\sz, v23\sz // t8a + sqadd v16\sz, v16\sz, v23\sz // t0a + sqsub v3\sz, v31\sz, v24\sz // t9a + sqadd v31\sz, v31\sz, v24\sz // t1a + sqadd v23\sz, v18\sz, v21\sz // t2a + sqsub v18\sz, v18\sz, v21\sz // t10a + sqadd v24\sz, v29\sz, v26\sz // t3a + sqsub v29\sz, v29\sz, v26\sz // t11a + sqadd v21\sz, v20\sz, v19\sz // t4a + sqsub v20\sz, v20\sz, v19\sz // t12a + sqadd v26\sz, v27\sz, v28\sz // t5a + sqsub v27\sz, v27\sz, v28\sz // t13a + sqadd v19\sz, v22\sz, v17\sz // t6a + sqsub v22\sz, v22\sz, v17\sz // t14a + sqadd v28\sz, v25\sz, v30\sz // t7a + sqsub v25\sz, v25\sz, v30\sz // t15a + + smull_smlal v4, v5, v2, v3, v0.h[5], v0.h[4], \sz // -> t8 + smull_smlsl v6, v7, v2, v3, v0.h[4], v0.h[5], \sz // -> t9 + smull_smlal v2, v3, v18, v29, v0.h[7], v0.h[6], \sz // -> t10 + sqrshrn_sz v17, v4, v5, #12, \sz // t8 + sqrshrn_sz v30, v6, v7, #12, \sz // t9 + smull_smlsl v4, v5, v18, v29, v0.h[6], v0.h[7], \sz // -> t11 + smull_smlsl v6, v7, v27, v20, v0.h[5], v0.h[4], \sz // -> t12 + sqrshrn_sz v18, v2, v3, #12, \sz // t10 + sqrshrn_sz v29, v4, v5, #12, \sz // t11 + smull_smlal v2, v3, v27, v20, v0.h[4], v0.h[5], \sz // -> t13 + smull_smlsl v4, v5, v25, v22, v0.h[7], v0.h[6], \sz // -> t14 + sqrshrn_sz v27, v6, v7, #12, \sz // t12 + sqrshrn_sz v20, v2, v3, #12, \sz // t13 + smull_smlal v6, v7, v25, v22, v0.h[6], v0.h[7], \sz // -> t15 + sqrshrn_sz v25, v4, v5, #12, \sz // t14 + sqrshrn_sz v22, v6, v7, #12, \sz // t15 + + sqsub v2\sz, v16\sz, v21\sz // t4 + sqadd v16\sz, v16\sz, v21\sz // t0 + sqsub v3\sz, v31\sz, v26\sz // t5 + sqadd v31\sz, v31\sz, v26\sz // t1 + sqadd v21\sz, v23\sz, v19\sz // t2 + sqsub v23\sz, v23\sz, v19\sz // t6 + sqadd v26\sz, v24\sz, v28\sz // t3 + sqsub v24\sz, v24\sz, v28\sz // t7 + sqadd v19\sz, v17\sz, v27\sz // t8a + sqsub v17\sz, v17\sz, v27\sz // t12a + sqadd v28\sz, v30\sz, v20\sz // t9a + sqsub v30\sz, v30\sz, v20\sz // t13a + sqadd v27\sz, v18\sz, v25\sz // t10a + sqsub v18\sz, v18\sz, v25\sz // t14a + sqadd v20\sz, v29\sz, v22\sz // t11a + sqsub v29\sz, v29\sz, v22\sz // t15a + + smull_smlal v4, v5, v2, v3, v0.h[3], v0.h[2], \sz // -> t4a + smull_smlsl v6, v7, v2, v3, v0.h[2], v0.h[3], \sz // -> t5a + smull_smlsl v2, v3, v24, v23, v0.h[3], v0.h[2], \sz // -> t6a + sqrshrn_sz v22, v4, v5, #12, \sz // t4a + sqrshrn_sz v25, v6, v7, #12, \sz // t5a + smull_smlal v4, v5, v24, v23, v0.h[2], v0.h[3], \sz // -> t7a + smull_smlal v6, v7, v17, v30, v0.h[3], v0.h[2], \sz // -> t12 + sqrshrn_sz v24, v2, v3, #12, \sz // t6a + sqrshrn_sz v23, v4, v5, #12, \sz // t7a + smull_smlsl v2, v3, v17, v30, v0.h[2], v0.h[3], \sz // -> t13 + smull_smlsl v4, v5, v29, v18, v0.h[3], v0.h[2], \sz // -> t14 + sqrshrn_sz v17, v6, v7, #12, \sz // t12 + smull_smlal v6, v7, v29, v18, v0.h[2], v0.h[3], \sz // -> t15 + sqrshrn_sz v29, v2, v3, #12, \sz // t13 + sqrshrn_sz v30, v4, v5, #12, \sz // t14 + sqrshrn_sz v18, v6, v7, #12, \sz // t15 + + sqsub v2\sz, v16\sz, v21\sz // t2a +.ifc \o0, v16 + sqadd \o0\sz, v16\sz, v21\sz // out0 + sqsub v21\sz, v31\sz, v26\sz // t3a + sqadd \o15\sz, v31\sz, v26\sz // out15 +.else + sqadd v4\sz, v16\sz, v21\sz // out0 + sqsub v21\sz, v31\sz, v26\sz // t3a + sqadd \o15\sz, v31\sz, v26\sz // out15 + mov \o0\szb, v4\szb +.endif + sqneg \o15\sz, \o15\sz // out15 + + sqsub v3\sz, v29\sz, v18\sz // t15a + sqadd \o13\sz, v29\sz, v18\sz // out13 + sqadd \o2\sz, v17\sz, v30\sz // out2 + sqsub v26\sz, v17\sz, v30\sz // t14a + sqneg \o13\sz, \o13\sz // out13 + + sqadd \o1\sz, v19\sz, v27\sz // out1 + sqsub v27\sz, v19\sz, v27\sz // t10 + sqadd \o14\sz, v28\sz, v20\sz // out14 + sqsub v20\sz, v28\sz, v20\sz // t11 + sqneg \o1\sz, \o1\sz // out1 + + sqadd \o3\sz, v22\sz, v24\sz // out3 + sqsub v22\sz, v22\sz, v24\sz // t6 + sqadd \o12\sz, v25\sz, v23\sz // out12 + sqsub v23\sz, v25\sz, v23\sz // t7 + sqneg \o3\sz, \o3\sz // out3 + + smull_smlsl v24, v25, v2, v21, v0.h[0], v0.h[0], \sz // -> out8 (v24 or v23) + smull_smlal v4, v5, v2, v21, v0.h[0], v0.h[0], \sz // -> out7 (v23 or v24) + smull_smlal v6, v7, v26, v3, v0.h[0], v0.h[0], \sz // -> out5 (v21 or v26) + + sqrshrn_sz v24, v24, v25, #12, \sz // out8 + sqrshrn_sz v4, v4, v5, #12, \sz // out7 + sqrshrn_sz v5, v6, v7, #12, \sz // out5 + smull_smlsl v6, v7, v26, v3, v0.h[0], v0.h[0], \sz // -> out10 (v26 or v21) + smull_smlal v2, v3, v22, v23, v0.h[0], v0.h[0], \sz // -> out4 (v20 or v27) + sqrshrn_sz v26, v6, v7, #12, \sz // out10 + + smull_smlsl v6, v7, v22, v23, v0.h[0], v0.h[0], \sz // -> out11 (v27 or v20) + smull_smlal v22, v23, v27, v20, v0.h[0], v0.h[0], \sz // -> out6 (v22 or v25) + smull_smlsl v21, v25, v27, v20, v0.h[0], v0.h[0], \sz // -> out9 (v25 or v22) + + sqrshrn_sz \o4, v2, v3, #12, \sz // out4 + sqrshrn_sz v6, v6, v7, #12, \sz // out11 + sqrshrn_sz v7, v21, v25, #12, \sz // out9 + sqrshrn_sz \o6, v22, v23, #12, \sz // out6 + +.ifc \o8, v23 + mov \o8\szb, v24\szb + mov \o10\szb, v26\szb +.endif + + sqneg \o7\sz, v4\sz // out7 + sqneg \o5\sz, v5\sz // out5 + sqneg \o11\sz, v6\sz // out11 + sqneg \o9\sz, v7\sz // out9 +.endm + +function inv_adst_8h_x16_neon, export=1 + iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .8h, .16b + ret +endfunc + +function inv_flipadst_8h_x16_neon, export=1 + iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .8h, .16b + ret +endfunc + +function inv_adst_4h_x16_neon, export=1 + iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .4h, .8b + ret +endfunc + +function inv_flipadst_4h_x16_neon, export=1 + iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .4h, .8b + ret +endfunc + +function inv_identity_8h_x16_neon, export=1 + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + sqrdmulh v2.8h, v\i\().8h, v0.h[0] + sqadd v\i\().8h, v\i\().8h, v\i\().8h + sqadd v\i\().8h, v\i\().8h, v2.8h +.endr + ret +endfunc + +function inv_identity_4h_x16_neon, export=1 + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + sqrdmulh v2.4h, v\i\().4h, v0.h[0] + sqadd v\i\().4h, v\i\().4h, v\i\().4h + sqadd v\i\().4h, v\i\().4h, v2.4h +.endr + ret +endfunc + +.macro identity_8x16_shift2 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + sqrdmulh v2.8h, \i, \c + sshr v2.8h, v2.8h, #1 + srhadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x16_shift1 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + sqrdmulh v2.8h, \i, \c + srshr v2.8h, v2.8h, #1 + sqadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x8_shift1 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + sqrdmulh v2.8h, \i, \c + srshr v2.8h, v2.8h, #1 + sqadd \i, \i, v2.8h +.endr +.endm + +.macro identity_8x8 c +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + sqrdmulh v2.8h, \i, \c + sqadd \i, \i, \i + sqadd \i, \i, v2.8h +.endr +.endm + +.macro def_horz_16 scale=0, identity=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x8_neon + AARCH64_VALID_CALL_TARGET + mov x14, x30 + movi v7.8h, #0 +.if \identity + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.elseif \scale + mov w16, #2896*8 + dup v0.4h, w16 +.endif +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x7] + st1 {v7.8h}, [x7], x8 +.endr +.if \scale + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif +.if \identity + identity_8x16_shift2 v0.h[0] +.else + blr x4 +.endif +.if \shift > 0 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + srshr \i, \i, #\shift +.endr +.endif + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 + +.irp i, v16.8h, v24.8h, v17.8h, v25.8h, v18.8h, v26.8h, v19.8h, v27.8h, v20.8h, v28.8h, v21.8h, v29.8h, v22.8h, v30.8h, v23.8h, v31.8h + st1 {\i}, [x6], #16 +.endr + + ret x14 +endfunc +.endm + +def_horz_16 scale=0, identity=0, shift=2 +def_horz_16 scale=1, identity=0, shift=1, suffix=_scale +def_horz_16 scale=0, identity=1, shift=0, suffix=_identity + +function inv_txfm_add_vert_8x16_neon + mov x14, x30 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + blr x5 + load_add_store_8x16 x6, x7 + ret x14 +endfunc + +function inv_txfm_add_16x16_neon + mov x15, x30 + sub sp, sp, #512 +.irp i, 0, 8 + add x6, sp, #(\i*16*2) +.if \i == 8 + cmp w3, w13 + b.lt 1f +.endif + add x7, x2, #(\i*2) + mov x8, #16*2 + blr x9 +.endr + b 2f +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr +2: +.irp i, 0, 8 + add x6, x0, #(\i) + add x7, sp, #(\i*2) + mov x8, #32 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #512 + ret x15 +endfunc + +.macro def_fn_16x16 txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif +.ifc \txfm1, identity + adr x9, inv_txfm_horz_identity_16x8_neon +.else + adr x9, inv_txfm_horz_16x8_neon + adr x4, inv_\txfm1\()_8h_x16_neon +.endif + adr x5, inv_\txfm2\()_8h_x16_neon + mov x13, #\eob_half + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct, 36 +def_fn_16x16 identity, identity, 36 +def_fn_16x16 dct, adst, 36 +def_fn_16x16 dct, flipadst, 36 +def_fn_16x16 dct, identity, 8 +def_fn_16x16 adst, dct, 36 +def_fn_16x16 adst, adst, 36 +def_fn_16x16 adst, flipadst, 36 +def_fn_16x16 flipadst, dct, 36 +def_fn_16x16 flipadst, adst, 36 +def_fn_16x16 flipadst, flipadst, 36 +def_fn_16x16 identity, dct, 8 + +.macro def_fn_416_base variant +function inv_txfm_\variant\()add_16x4_neon + mov x15, x30 + movi v4.8h, #0 + +.ifc \variant, identity_ +.irp i, v16.4h, v17.4h, v18.4h, v19.4h + ld1 {\i}, [x2] + st1 {v4.4h}, [x2], #8 +.endr +.irp i, v16.d, v17.d, v18.d, v19.d + ld1 {\i}[1], [x2] + st1 {v4.4h}, [x2], #8 +.endr + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.irp i, v20.4h, v21.4h, v22.4h, v23.4h + ld1 {\i}, [x2] + st1 {v4.4h}, [x2], #8 +.endr +.irp i, v20.d, v21.d, v22.d, v23.d + ld1 {\i}[1], [x2] + st1 {v4.4h}, [x2], #8 +.endr + + identity_8x16_shift1 v0.h[0] +.else +.irp i, v16.4h, v17.4h, v18.4h, v19.4h, v20.4h, v21.4h, v22.4h, v23.4h, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h + ld1 {\i}, [x2] + st1 {v4.4h}, [x2], #8 +.endr + + blr x4 + + ins v16.d[1], v20.d[0] + ins v17.d[1], v21.d[0] + ins v18.d[1], v22.d[0] + ins v19.d[1], v23.d[0] +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + srshr \i, \i, #1 +.endr +.endif + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + mov x6, x0 + load_add_store_8x4 x6, x7 + +.ifc \variant, identity_ + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b +.else + ins v24.d[1], v28.d[0] + ins v25.d[1], v29.d[0] + ins v26.d[1], v30.d[0] + ins v27.d[1], v31.d[0] + srshr v16.8h, v24.8h, #1 + srshr v17.8h, v25.8h, #1 + srshr v18.8h, v26.8h, #1 + srshr v19.8h, v27.8h, #1 +.endif + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + add x6, x0, #8 + load_add_store_8x4 x6, x7 + + ret x15 +endfunc + +function inv_txfm_\variant\()add_4x16_neon + mov x15, x30 + movi v2.8h, #0 + + mov x11, #32 + cmp w3, w13 + b.lt 1f + + add x6, x2, #16 +.ifc \variant, identity_ +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + ld1 {\i}, [x6] + st1 {v2.8h}, [x6], x11 +.endr + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + identity_8x4_shift1 v24, v25, v26, v27, v0.h[0] +.else +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + ld1 {\i}, [x6] + st1 {v2.8h}, [x6], x11 +.endr + blr x4 + srshr v24.8h, v16.8h, #1 + srshr v25.8h, v17.8h, #1 + srshr v26.8h, v18.8h, #1 + srshr v27.8h, v19.8h, #1 +.endif + transpose_4x8h v24, v25, v26, v27, v4, v5, v6, v7 + ins v28.d[0], v24.d[1] + ins v29.d[0], v25.d[1] + ins v30.d[0], v26.d[1] + ins v31.d[0], v27.d[1] + + b 2f +1: +.irp i, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h + movi \i, #0 +.endr +2: + movi v2.8h, #0 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + ld1 {\i}, [x2] + st1 {v2.8h}, [x2], x11 +.endr +.ifc \variant, identity_ + mov w16, #(5793-4096)*8 + dup v0.4h, w16 + identity_8x4_shift1 v16, v17, v18, v19, v0.h[0] +.else + blr x4 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h + srshr \i, \i, #1 +.endr +.endif + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + ins v20.d[0], v16.d[1] + ins v21.d[0], v17.d[1] + ins v22.d[0], v18.d[1] + ins v23.d[0], v19.d[1] + + blr x5 + + load_add_store_4x16 x0, x6 + + ret x15 +endfunc +.endm + +def_fn_416_base +def_fn_416_base identity_ + +.macro def_fn_416 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif +.if \w == 4 + adr x4, inv_\txfm1\()_8h_x\w\()_neon + adr x5, inv_\txfm2\()_4h_x\h\()_neon + mov w13, #\eob_half +.else + adr x4, inv_\txfm1\()_4h_x\w\()_neon + adr x5, inv_\txfm2\()_8h_x\h\()_neon +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct, 29 +def_fn_416 \w, \h, identity, identity, 29 +def_fn_416 \w, \h, dct, adst, 29 +def_fn_416 \w, \h, dct, flipadst, 29 +def_fn_416 \w, \h, dct, identity, 8 +def_fn_416 \w, \h, adst, dct, 29 +def_fn_416 \w, \h, adst, adst, 29 +def_fn_416 \w, \h, adst, flipadst, 29 +def_fn_416 \w, \h, flipadst, dct, 29 +def_fn_416 \w, \h, flipadst, adst, 29 +def_fn_416 \w, \h, flipadst, flipadst, 29 +def_fn_416 \w, \h, identity, dct, 32 +def_fn_416 \w, \h, adst, identity, 8 +def_fn_416 \w, \h, flipadst, identity, 8 +def_fn_416 \w, \h, identity, adst, 32 +def_fn_416 \w, \h, identity, flipadst, 32 +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + + +.macro def_fn_816_base variant +function inv_txfm_\variant\()add_16x8_neon + mov x15, x30 + movi v4.8h, #0 + mov w16, #2896*8 + dup v0.4h, w16 + +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x2] + st1 {v4.8h}, [x2], #16 +.endr + + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 +.ifc \variant, identity_ + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 + identity_8x16_shift1 v0.h[0] +.else + blr x4 + +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + srshr \i, \i, #1 +.endr +.endif + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 + + blr x5 + + mov x6, x0 + load_add_store_8x8 x6, x7 + +.ifc \variant, identity_ + mov v16.16b, v24.16b + mov v17.16b, v25.16b + mov v18.16b, v26.16b + mov v19.16b, v27.16b + mov v20.16b, v28.16b + mov v21.16b, v29.16b + mov v22.16b, v30.16b + mov v23.16b, v31.16b +.else + srshr v16.8h, v24.8h, #1 + srshr v17.8h, v25.8h, #1 + srshr v18.8h, v26.8h, #1 + srshr v19.8h, v27.8h, #1 + srshr v20.8h, v28.8h, #1 + srshr v21.8h, v29.8h, #1 + srshr v22.8h, v30.8h, #1 + srshr v23.8h, v31.8h, #1 +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 + + blr x5 + + add x0, x0, #8 + load_add_store_8x8 x0, x7 + + ret x15 +endfunc + +function inv_txfm_\variant\()add_8x16_neon + mov x15, x30 + movi v4.8h, #0 + mov w16, #2896*8 + dup v0.4h, w16 + mov x11, #32 + + cmp w3, w13 + b.lt 1f + + add x6, x2, #16 +.ifc \variant, identity_ +.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x6] + st1 {v4.8h}, [x6], x11 +.endr + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 + // The identity shl #1 and downshift srshr #1 cancel out +.else +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x6] + st1 {v4.8h}, [x6], x11 +.endr + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + srshr v24.8h, v16.8h, #1 + srshr v25.8h, v17.8h, #1 + srshr v26.8h, v18.8h, #1 + srshr v27.8h, v19.8h, #1 + srshr v28.8h, v20.8h, #1 + srshr v29.8h, v21.8h, #1 + srshr v30.8h, v22.8h, #1 + srshr v31.8h, v23.8h, #1 +.endif + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v2, v3 + + b 2f + +1: +.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + movi \i, #0 +.endr + +2: + movi v4.8h, #0 + mov w16, #2896*8 + dup v0.4h, w16 + +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x2] + st1 {v4.8h}, [x2], x11 +.endr + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else + blr x4 + +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + srshr \i, \i, #1 +.endr +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v2, v3 + + blr x5 + + load_add_store_8x16 x0, x6 + + ret x15 +endfunc +.endm + +def_fn_816_base +def_fn_816_base identity_ + +.macro def_fn_816 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + adr x4, inv_\txfm1\()_8h_x\w\()_neon + adr x5, inv_\txfm2\()_8h_x\h\()_neon +.if \w == 8 + mov x13, #\eob_half +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct, 43 +def_fn_816 \w, \h, identity, identity, 43 +def_fn_816 \w, \h, dct, adst, 43 +def_fn_816 \w, \h, dct, flipadst, 43 +def_fn_816 \w, \h, dct, identity, 8 +def_fn_816 \w, \h, adst, dct, 43 +def_fn_816 \w, \h, adst, adst, 43 +def_fn_816 \w, \h, adst, flipadst, 43 +def_fn_816 \w, \h, flipadst, dct, 43 +def_fn_816 \w, \h, flipadst, adst, 43 +def_fn_816 \w, \h, flipadst, flipadst, 43 +def_fn_816 \w, \h, identity, dct, 64 +def_fn_816 \w, \h, adst, identity, 8 +def_fn_816 \w, \h, flipadst, identity, 8 +def_fn_816 \w, \h, identity, adst, 64 +def_fn_816 \w, \h, identity, flipadst, 64 +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_8h_x16_neon, export=1 + movrel x16, idct_coeffs, 2*16 + ld1 {v0.8h, v1.8h}, [x16] + sub x16, x16, #2*16 + + smull_smlsl v2, v3, v16, v31, v0.h[0], v0.h[1], .8h // -> t16a + smull_smlal v4, v5, v16, v31, v0.h[1], v0.h[0], .8h // -> t31a + smull_smlsl v6, v7, v24, v23, v0.h[2], v0.h[3], .8h // -> t17a + sqrshrn_sz v16, v2, v3, #12, .8h // t16a + sqrshrn_sz v31, v4, v5, #12, .8h // t31a + smull_smlal v2, v3, v24, v23, v0.h[3], v0.h[2], .8h // -> t30a + smull_smlsl v4, v5, v20, v27, v0.h[4], v0.h[5], .8h // -> t18a + sqrshrn_sz v24, v6, v7, #12, .8h // t17a + sqrshrn_sz v23, v2, v3, #12, .8h // t30a + smull_smlal v6, v7, v20, v27, v0.h[5], v0.h[4], .8h // -> t29a + smull_smlsl v2, v3, v28, v19, v0.h[6], v0.h[7], .8h // -> t19a + sqrshrn_sz v20, v4, v5, #12, .8h // t18a + sqrshrn_sz v27, v6, v7, #12, .8h // t29a + smull_smlal v4, v5, v28, v19, v0.h[7], v0.h[6], .8h // -> t28a + smull_smlsl v6, v7, v18, v29, v1.h[0], v1.h[1], .8h // -> t20a + sqrshrn_sz v28, v2, v3, #12, .8h // t19a + sqrshrn_sz v19, v4, v5, #12, .8h // t28a + smull_smlal v2, v3, v18, v29, v1.h[1], v1.h[0], .8h // -> t27a + smull_smlsl v4, v5, v26, v21, v1.h[2], v1.h[3], .8h // -> t21a + sqrshrn_sz v18, v6, v7, #12, .8h // t20a + sqrshrn_sz v29, v2, v3, #12, .8h // t27a + smull_smlal v6, v7, v26, v21, v1.h[3], v1.h[2], .8h // -> t26a + smull_smlsl v2, v3, v22, v25, v1.h[4], v1.h[5], .8h // -> t22a + sqrshrn_sz v26, v4, v5, #12, .8h // t21a + sqrshrn_sz v21, v6, v7, #12, .8h // t26a + smull_smlal v4, v5, v22, v25, v1.h[5], v1.h[4], .8h // -> t25a + smull_smlsl v6, v7, v30, v17, v1.h[6], v1.h[7], .8h // -> t23a + sqrshrn_sz v22, v2, v3, #12, .8h // t22a + sqrshrn_sz v25, v4, v5, #12, .8h // t25a + smull_smlal v2, v3, v30, v17, v1.h[7], v1.h[6], .8h // -> t24a + sqrshrn_sz v30, v6, v7, #12, .8h // t23a + sqrshrn_sz v17, v2, v3, #12, .8h // t24a + + ld1 {v0.8h}, [x16] + + sqsub v2.8h, v16.8h, v24.8h // t17 + sqadd v16.8h, v16.8h, v24.8h // t16 + sqsub v3.8h, v31.8h, v23.8h // t30 + sqadd v31.8h, v31.8h, v23.8h // t31 + sqsub v24.8h, v28.8h, v20.8h // t18 + sqadd v28.8h, v28.8h, v20.8h // t19 + sqadd v23.8h, v18.8h, v26.8h // t20 + sqsub v18.8h, v18.8h, v26.8h // t21 + sqsub v20.8h, v30.8h, v22.8h // t22 + sqadd v30.8h, v30.8h, v22.8h // t23 + sqadd v26.8h, v17.8h, v25.8h // t24 + sqsub v17.8h, v17.8h, v25.8h // t25 + sqsub v22.8h, v29.8h, v21.8h // t26 + sqadd v29.8h, v29.8h, v21.8h // t27 + sqadd v25.8h, v19.8h, v27.8h // t28 + sqsub v19.8h, v19.8h, v27.8h // t29 + + smull_smlsl v4, v5, v3, v2, v0.h[4], v0.h[5], .8h // -> t17a + smull_smlal v6, v7, v3, v2, v0.h[5], v0.h[4], .8h // -> t30a + smull_smlal v2, v3, v19, v24, v0.h[5], v0.h[4], .8h // -> t18a + sqrshrn_sz v21, v4, v5, #12, .8h // t17a + sqrshrn_sz v27, v6, v7, #12, .8h // t30a + neg v2.4s, v2.4s // -> t18a + neg v3.4s, v3.4s // -> t18a + smull_smlsl v4, v5, v19, v24, v0.h[4], v0.h[5], .8h // -> t29a + smull_smlsl v6, v7, v22, v18, v0.h[6], v0.h[7], .8h // -> t21a + sqrshrn_sz v19, v2, v3, #12, .8h // t18a + sqrshrn_sz v24, v4, v5, #12, .8h // t29a + smull_smlal v2, v3, v22, v18, v0.h[7], v0.h[6], .8h // -> t26a + smull_smlal v4, v5, v17, v20, v0.h[7], v0.h[6], .8h // -> t22a + sqrshrn_sz v22, v6, v7, #12, .8h // t21a + sqrshrn_sz v18, v2, v3, #12, .8h // t26a + neg v4.4s, v4.4s // -> t22a + neg v5.4s, v5.4s // -> t22a + smull_smlsl v6, v7, v17, v20, v0.h[6], v0.h[7], .8h // -> t25a + sqrshrn_sz v17, v4, v5, #12, .8h // t22a + sqrshrn_sz v20, v6, v7, #12, .8h // t25a + + sqsub v2.8h, v27.8h, v24.8h // t29 + sqadd v27.8h, v27.8h, v24.8h // t30 + sqsub v3.8h, v21.8h, v19.8h // t18 + sqadd v21.8h, v21.8h, v19.8h // t17 + sqsub v24.8h, v16.8h, v28.8h // t19a + sqadd v16.8h, v16.8h, v28.8h // t16a + sqsub v19.8h, v30.8h, v23.8h // t20a + sqadd v30.8h, v30.8h, v23.8h // t23a + sqsub v28.8h, v17.8h, v22.8h // t21 + sqadd v17.8h, v17.8h, v22.8h // t22 + sqadd v23.8h, v26.8h, v29.8h // t24a + sqsub v26.8h, v26.8h, v29.8h // t27a + sqadd v22.8h, v20.8h, v18.8h // t25 + sqsub v20.8h, v20.8h, v18.8h // t26 + sqsub v29.8h, v31.8h, v25.8h // t28a + sqadd v31.8h, v31.8h, v25.8h // t31a + + smull_smlsl v4, v5, v2, v3, v0.h[2], v0.h[3], .8h // -> t18a + smull_smlal v6, v7, v2, v3, v0.h[3], v0.h[2], .8h // -> t29a + smull_smlsl v2, v3, v29, v24, v0.h[2], v0.h[3], .8h // -> t19 + sqrshrn_sz v18, v4, v5, #12, .8h // t18a + sqrshrn_sz v25, v6, v7, #12, .8h // t29a + smull_smlal v4, v5, v29, v24, v0.h[3], v0.h[2], .8h // -> t28 + smull_smlal v6, v7, v26, v19, v0.h[3], v0.h[2], .8h // -> t20 + sqrshrn_sz v29, v2, v3, #12, .8h // t19 + sqrshrn_sz v24, v4, v5, #12, .8h // t28 + neg v6.4s, v6.4s // -> t20 + neg v7.4s, v7.4s // -> t20 + smull_smlsl v2, v3, v26, v19, v0.h[2], v0.h[3], .8h // -> t27 + smull_smlal v4, v5, v20, v28, v0.h[3], v0.h[2], .8h // -> t21a + sqrshrn_sz v26, v6, v7, #12, .8h // t20 + sqrshrn_sz v19, v2, v3, #12, .8h // t27 + neg v4.4s, v4.4s // -> t21a + neg v5.4s, v5.4s // -> t21a + smull_smlsl v6, v7, v20, v28, v0.h[2], v0.h[3], .8h // -> t26a + sqrshrn_sz v20, v4, v5, #12, .8h // t21a + sqrshrn_sz v28, v6, v7, #12, .8h // t26a + + sqsub v2.8h, v16.8h, v30.8h // t23 + sqadd v16.8h, v16.8h, v30.8h // t16 = out16 + sqsub v3.8h, v31.8h, v23.8h // t24 + sqadd v31.8h, v31.8h, v23.8h // t31 = out31 + sqsub v23.8h, v21.8h, v17.8h // t22a + sqadd v17.8h, v21.8h, v17.8h // t17a = out17 + sqadd v30.8h, v27.8h, v22.8h // t30a = out30 + sqsub v21.8h, v27.8h, v22.8h // t25a + sqsub v27.8h, v18.8h, v20.8h // t21 + sqadd v18.8h, v18.8h, v20.8h // t18 = out18 + sqadd v4.8h, v29.8h, v26.8h // t19a = out19 + sqsub v26.8h, v29.8h, v26.8h // t20a + sqadd v29.8h, v25.8h, v28.8h // t29 = out29 + sqsub v25.8h, v25.8h, v28.8h // t26 + sqadd v28.8h, v24.8h, v19.8h // t28a = out28 + sqsub v24.8h, v24.8h, v19.8h // t27a + mov v19.16b, v4.16b // out19 + + smull_smlsl v4, v5, v24, v26, v0.h[0], v0.h[0], .8h // -> t20 + smull_smlal v6, v7, v24, v26, v0.h[0], v0.h[0], .8h // -> t27 + sqrshrn_sz v20, v4, v5, #12, .8h // t20 + sqrshrn_sz v22, v6, v7, #12, .8h // t27 + + smull_smlal v4, v5, v25, v27, v0.h[0], v0.h[0], .8h // -> t26a + smull_smlsl v6, v7, v25, v27, v0.h[0], v0.h[0], .8h // -> t21a + mov v27.16b, v22.16b // t27 + sqrshrn_sz v26, v4, v5, #12, .8h // t26a + + smull_smlsl v24, v25, v21, v23, v0.h[0], v0.h[0], .8h // -> t22 + smull_smlal v4, v5, v21, v23, v0.h[0], v0.h[0], .8h // -> t25 + sqrshrn_sz v21, v6, v7, #12, .8h // t21a + sqrshrn_sz v22, v24, v25, #12, .8h // t22 + sqrshrn_sz v25, v4, v5, #12, .8h // t25 + + smull_smlsl v4, v5, v3, v2, v0.h[0], v0.h[0], .8h // -> t23a + smull_smlal v6, v7, v3, v2, v0.h[0], v0.h[0], .8h // -> t24a + sqrshrn_sz v23, v4, v5, #12, .8h // t23a + sqrshrn_sz v24, v6, v7, #12, .8h // t24a + + ret +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x8_neon + mov x14, x30 + movi v7.8h, #0 + lsl x8, x8, #1 +.if \scale + mov w16, #2896*8 + dup v0.4h, w16 +.endif + +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x7] + st1 {v7.8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 +.if \scale + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct_8h_x16_neon + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 + +.macro store1 r0, r1 + st1 {\r0}, [x6], #16 + st1 {\r1}, [x6], #16 + add x6, x6, #32 +.endm + store1 v16.8h, v24.8h + store1 v17.8h, v25.8h + store1 v18.8h, v26.8h + store1 v19.8h, v27.8h + store1 v20.8h, v28.8h + store1 v21.8h, v29.8h + store1 v22.8h, v30.8h + store1 v23.8h, v31.8h +.purgem store1 + sub x6, x6, #64*8 + + movi v7.8h, #0 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x7] + st1 {v7.8h}, [x7], x8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in v0.h[1] + scale_input .8h, v0.h[1], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .8h, v0.h[1], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct32_odd_8h_x16_neon + transpose_8x8h v31, v30, v29, v28, v27, v26, v25, v24, v4, v5 + transpose_8x8h v23, v22, v21, v20, v19, v18, v17, v16, v4, v5 +.macro store2 r0, r1, shift + ld1 {v4.8h, v5.8h}, [x6] + sqsub v7.8h, v4.8h, \r0 + sqsub v6.8h, v5.8h, \r1 + sqadd v4.8h, v4.8h, \r0 + sqadd v5.8h, v5.8h, \r1 + rev64 v6.8h, v6.8h + rev64 v7.8h, v7.8h + srshr v4.8h, v4.8h, #\shift + srshr v5.8h, v5.8h, #\shift + srshr v6.8h, v6.8h, #\shift + srshr v7.8h, v7.8h, #\shift + ext v6.16b, v6.16b, v6.16b, #8 + st1 {v4.8h, v5.8h}, [x6], #32 + ext v7.16b, v7.16b, v7.16b, #8 + st1 {v6.8h, v7.8h}, [x6], #32 +.endm + + store2 v31.8h, v23.8h, \shift + store2 v30.8h, v22.8h, \shift + store2 v29.8h, v21.8h, \shift + store2 v28.8h, v20.8h, \shift + store2 v27.8h, v19.8h, \shift + store2 v26.8h, v18.8h, \shift + store2 v25.8h, v17.8h, \shift + store2 v24.8h, v16.8h, \shift +.purgem store2 + ret x14 +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_8x32_neon + mov x14, x30 + lsl x8, x8, #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + + bl inv_dct_8h_x16_neon + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + st1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + sub x7, x7, x8, lsr #1 + bl inv_dct32_odd_8h_x16_neon + + neg x9, x8 + mov x10, x6 +.macro combine r0, r1, r2, r3, op, stride + ld1 {v5.8h}, [x7], \stride + ld1 {v2.8b}, [x10], x1 + ld1 {v6.8h}, [x7], \stride + ld1 {v3.8b}, [x10], x1 + \op v5.8h, v5.8h, \r0 + ld1 {v7.8h}, [x7], \stride + ld1 {v4.8b}, [x10], x1 + srshr v5.8h, v5.8h, #4 + \op v6.8h, v6.8h, \r1 + uaddw v5.8h, v5.8h, v2.8b + srshr v6.8h, v6.8h, #4 + \op v7.8h, v7.8h, \r2 + sqxtun v2.8b, v5.8h + ld1 {v5.8h}, [x7], \stride + uaddw v6.8h, v6.8h, v3.8b + srshr v7.8h, v7.8h, #4 + \op v5.8h, v5.8h, \r3 + st1 {v2.8b}, [x6], x1 + ld1 {v2.8b}, [x10], x1 + sqxtun v3.8b, v6.8h + uaddw v7.8h, v7.8h, v4.8b + srshr v5.8h, v5.8h, #4 + st1 {v3.8b}, [x6], x1 + sqxtun v4.8b, v7.8h + uaddw v5.8h, v5.8h, v2.8b + st1 {v4.8b}, [x6], x1 + sqxtun v2.8b, v5.8h + st1 {v2.8b}, [x6], x1 +.endm + combine v31.8h, v30.8h, v29.8h, v28.8h, sqadd, x8 + combine v27.8h, v26.8h, v25.8h, v24.8h, sqadd, x8 + combine v23.8h, v22.8h, v21.8h, v20.8h, sqadd, x8 + combine v19.8h, v18.8h, v17.8h, v16.8h, sqadd, x8 + sub x7, x7, x8 + combine v16.8h, v17.8h, v18.8h, v19.8h, sqsub, x9 + combine v20.8h, v21.8h, v22.8h, v23.8h, sqsub, x9 + combine v24.8h, v25.8h, v26.8h, v27.8h, sqsub, x9 + combine v28.8h, v29.8h, v30.8h, v31.8h, sqsub, x9 +.purgem combine + + ret x14 +endfunc + +const eob_32x32 + .short 36, 136, 300, 1024 +endconst + +const eob_16x32 + .short 36, 151, 279, 512 +endconst + +const eob_16x32_shortside + .short 36, 512 +endconst + +const eob_8x32 + .short 43, 107, 171, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_8bpc_neon, export=1 + movi v0.8h, #0 + movrel x13, eob_32x32 + + mov x8, #2*32 +1: + mov w9, #0 + movrel x12, eob_32x32 +2: + add w9, w9, #8 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().8h}, [x2] + st1 {v0.8h}, [x2], x8 +.endr + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + load_add_store_8x8 x0, x7, shiftbits=2 + ldrh w11, [x12], #2 + sub x0, x0, x1, lsl #3 + add x0, x0, #8 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #2 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #2*8 + b 1b +9: + ret +endfunc + +.macro shift_8_regs op, shift +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + mov w16, #2896*8 + mov w17, #2*(5793-4096)*8 + dup v1.4h, w16 + movi v0.8h, #0 + mov v1.h[1], w17 + movrel x13, eob_16x32\hshort + + mov x8, #2*\h +1: + mov w9, #0 + movrel x12, eob_16x32\wshort +2: + add w9, w9, #8 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x2] + st1 {v0.8h}, [x2], x8 +.endr + scale_input .8h, v1.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + +.if \w == 16 + // 16x32 + identity_8x8_shift1 v1.h[1] +.else + // 32x16 + shift_8_regs sqshl, 1 + identity_8x8 v1.h[1] +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + +.if \w == 16 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=4 +.endif + ldrh w11, [x12], #2 + sub x0, x0, x1, lsl #3 + add x0, x0, #8 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #2 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #2*8 + b 1b +9: + ret +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + movi v0.8h, #0 + movrel x13, eob_8x32 + + mov w8, #2*\h +1: + ldrh w12, [x13], #2 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h + ld1 {\i}, [x2] + st1 {v0.8h}, [x2], x8 +.endr + +.if \w == 8 + // 8x32 + shift_8_regs srshr, 1 +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + cmp w3, w12 +.if \w == 8 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=3 +.endif + + b.lt 9f +.if \w == 8 + sub x2, x2, x8, lsl #3 + add x2, x2, #2*8 +.else + sub x0, x0, x1, lsl #3 + add x0, x0, #8 +.endif + b 1b + +9: + ret +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_8bpc_neon, export=1 + idct_dc 32, 32, 2 + + mov x15, x30 + sub sp, sp, #2048 + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 8, 16, 24 + add x6, sp, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*2) + mov x8, #32*2 + bl inv_txfm_horz_dct_32x8_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #2048 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_16x32_8bpc_neon, export=1 + idct_dc 16, 32, 1 + + mov x15, x30 + sub sp, sp, #1024 + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + adr x4, inv_dct_8h_x16_neon + +.irp i, 0, 8, 16, 24 + add x6, sp, #(\i*16*2) + add x7, x2, #(\i*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endif + mov x8, #2*32 + bl inv_txfm_horz_scale_16x8_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #8 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x6, x0, #(\i) + add x7, sp, #(\i*2) + mov x8, #16*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #1024 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x16_8bpc_neon, export=1 + idct_dc 32, 16, 1 + + mov x15, x30 + sub sp, sp, #1024 + + adr x5, inv_dct_8h_x16_neon + +.irp i, 0, 8 + add x6, sp, #(\i*32*2) + add x7, x2, #(\i*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, #36 + b.lt 1f +.endif + mov x8, #2*16 + bl inv_txfm_horz_scale_dct_32x8_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #1024 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_8x32_8bpc_neon, export=1 + idct_dc 8, 32, 2 + + mov x15, x30 + sub sp, sp, #512 + + movrel x13, eob_8x32 + + movi v28.8h, #0 + mov x8, #2*32 + mov w9, #32 + mov x6, sp +1: +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().8h}, [x2] + st1 {v28.8h}, [x2], x8 +.endr + ldrh w12, [x13], #2 + sub x2, x2, x8, lsl #3 + sub w9, w9, #8 + add x2, x2, #2*8 + + bl inv_dct_8h_x8_neon + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + srshr v\i\().8h, v\i\().8h, #2 +.endr + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v24, v25 + + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64 + cmp w3, w12 + st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x6], #64 + + b.ge 1b + cbz w9, 3f + + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 +2: + subs w9, w9, #8 +.rept 2 + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x6], #64 +.endr + b.gt 2b + +3: + mov x6, x0 + mov x7, sp + mov x8, #8*2 + bl inv_txfm_add_vert_dct_8x32_neon + + add sp, sp, #512 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x8_8bpc_neon, export=1 + idct_dc 32, 8, 2 + + mov x15, x30 + sub sp, sp, #512 + + mov x6, sp + mov x7, x2 + mov x8, #8*2 + bl inv_txfm_horz_dct_32x8_neon + + mov x8, #2*32 + mov w9, #0 +1: + add x6, x0, x9 + add x7, sp, x9, lsl #1 // #(\i*2) + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().8h}, [x7], x8 +.endr + add w9, w9, #8 + + bl inv_dct_8h_x8_neon + + cmp w9, #32 + + load_add_store_8x8 x6, x7 + + b.lt 1b + + add sp, sp, #512 + ret x15 +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + ld1 {v0.8h, v1.8h}, [x17], #32 + + sqrdmulh v23.8h, v16.8h, v0.h[1] // t63a + sqrdmulh v16.8h, v16.8h, v0.h[0] // t32a + sqrdmulh v22.8h, v17.8h, v0.h[2] // t62a + sqrdmulh v17.8h, v17.8h, v0.h[3] // t33a + sqrdmulh v21.8h, v18.8h, v0.h[5] // t61a + sqrdmulh v18.8h, v18.8h, v0.h[4] // t34a + sqrdmulh v20.8h, v19.8h, v0.h[6] // t60a + sqrdmulh v19.8h, v19.8h, v0.h[7] // t35a + + sqadd v24.8h, v16.8h, v17.8h // t32 + sqsub v25.8h, v16.8h, v17.8h // t33 + sqsub v26.8h, v19.8h, v18.8h // t34 + sqadd v27.8h, v19.8h, v18.8h // t35 + sqadd v28.8h, v20.8h, v21.8h // t60 + sqsub v29.8h, v20.8h, v21.8h // t61 + sqsub v30.8h, v23.8h, v22.8h // t62 + sqadd v31.8h, v23.8h, v22.8h // t63 + + smull_smlal v2, v3, v29, v26, v1.h[0], v1.h[1], .8h // -> t34a + smull_smlsl v4, v5, v29, v26, v1.h[1], v1.h[0], .8h // -> t61a + neg v2.4s, v2.4s // t34a + neg v3.4s, v3.4s // t34a + smull_smlsl v6, v7, v30, v25, v1.h[1], v1.h[0], .8h // -> t33a + sqrshrn_sz v26, v2, v3, #12, .8h // t34a + smull_smlal v2, v3, v30, v25, v1.h[0], v1.h[1], .8h // -> t62a + sqrshrn_sz v29, v4, v5, #12, .8h // t61a + sqrshrn_sz v25, v6, v7, #12, .8h // t33a + sqrshrn_sz v30, v2, v3, #12, .8h // t62a + + sqadd v16.8h, v24.8h, v27.8h // t32a + sqsub v19.8h, v24.8h, v27.8h // t35a + sqadd v17.8h, v25.8h, v26.8h // t33 + sqsub v18.8h, v25.8h, v26.8h // t34 + sqsub v20.8h, v31.8h, v28.8h // t60a + sqadd v23.8h, v31.8h, v28.8h // t63a + sqsub v21.8h, v30.8h, v29.8h // t61 + sqadd v22.8h, v30.8h, v29.8h // t62 + + smull_smlal v2, v3, v21, v18, v1.h[2], v1.h[3], .8h // -> t61a + smull_smlsl v4, v5, v21, v18, v1.h[3], v1.h[2], .8h // -> t34a + smull_smlal v6, v7, v20, v19, v1.h[2], v1.h[3], .8h // -> t60 + sqrshrn_sz v21, v2, v3, #12, .8h // t61a + sqrshrn_sz v18, v4, v5, #12, .8h // t34a + smull_smlsl v2, v3, v20, v19, v1.h[3], v1.h[2], .8h // -> t35 + sqrshrn_sz v20, v6, v7, #12, .8h // t60 + sqrshrn_sz v19, v2, v3, #12, .8h // t35 + + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64 + st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x6], #64 + + ret +endfunc + +function inv_dct64_step2_neon + movrel x16, idct_coeffs + ld1 {v0.4h}, [x16] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + ldr q16, [x6, #2*8*0] // t32a + ldr q17, [x9, #2*8*8] // t39a + ldr q18, [x9, #2*8*0] // t63a + ldr q19, [x6, #2*8*8] // t56a + ldr q20, [x6, #2*8*16] // t40a + ldr q21, [x9, #2*8*24] // t47a + ldr q22, [x9, #2*8*16] // t55a + ldr q23, [x6, #2*8*24] // t48a + + sqadd v24.8h, v16.8h, v17.8h // t32 + sqsub v25.8h, v16.8h, v17.8h // t39 + sqadd v26.8h, v18.8h, v19.8h // t63 + sqsub v27.8h, v18.8h, v19.8h // t56 + sqsub v28.8h, v21.8h, v20.8h // t40 + sqadd v29.8h, v21.8h, v20.8h // t47 + sqadd v30.8h, v23.8h, v22.8h // t48 + sqsub v31.8h, v23.8h, v22.8h // t55 + + smull_smlal v2, v3, v27, v25, v0.h[3], v0.h[2], .8h // -> t56a + smull_smlsl v4, v5, v27, v25, v0.h[2], v0.h[3], .8h // -> t39a + smull_smlal v6, v7, v31, v28, v0.h[3], v0.h[2], .8h // -> t40a + sqrshrn_sz v25, v2, v3, #12, .8h // t56a + sqrshrn_sz v27, v4, v5, #12, .8h // t39a + neg v6.4s, v6.4s // t40a + neg v7.4s, v7.4s // t40a + smull_smlsl v2, v3, v31, v28, v0.h[2], v0.h[3], .8h // -> t55a + sqrshrn_sz v31, v6, v7, #12, .8h // t40a + sqrshrn_sz v28, v2, v3, #12, .8h // t55a + + sqadd v16.8h, v24.8h, v29.8h // t32a + sqsub v19.8h, v24.8h, v29.8h // t47a + sqadd v17.8h, v27.8h, v31.8h // t39 + sqsub v18.8h, v27.8h, v31.8h // t40 + sqsub v20.8h, v26.8h, v30.8h // t48a + sqadd v23.8h, v26.8h, v30.8h // t63a + sqsub v21.8h, v25.8h, v28.8h // t55 + sqadd v22.8h, v25.8h, v28.8h // t56 + + smull_smlsl v2, v3, v21, v18, v0.h[0], v0.h[0], .8h // -> t40a + smull_smlal v4, v5, v21, v18, v0.h[0], v0.h[0], .8h // -> t55a + smull_smlsl v6, v7, v20, v19, v0.h[0], v0.h[0], .8h // -> t47 + sqrshrn_sz v18, v2, v3, #12, .8h // t40a + sqrshrn_sz v21, v4, v5, #12, .8h // t55a + smull_smlal v2, v3, v20, v19, v0.h[0], v0.h[0], .8h // -> t48 + sqrshrn_sz v19, v6, v7, #12, .8h // t47 + sqrshrn_sz v20, v2, v3, #12, .8h // t48 + + str q16, [x6, #2*8*0] // t32a + str q17, [x9, #2*8*0] // t39 + str q18, [x6, #2*8*8] // t40a + str q19, [x9, #2*8*8] // t47 + str q20, [x6, #2*8*16] // t48 + str q21, [x9, #2*8*16] // t55a + str q22, [x6, #2*8*24] // t56 + str q23, [x9, #2*8*24] // t63a + + add x6, x6, #2*8 + sub x9, x9, #2*8 + cmp x6, x9 + b.lt 1b + ret +endfunc + +.macro load8 src, strd, zero, clear +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h +.if \clear + ld1 {\i}, [\src] + st1 {\zero}, [\src], \strd +.else + ld1 {\i}, [\src], \strd +.endif +.endr +.endm + +.macro store16 dst +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + st1 {\i}, [\dst], #16 +.endr +.endm + +.macro clear_upper8 +.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + movi \i, #0 +.endr +.endm + +.macro movi_if reg, val, cond +.if \cond + movi \reg, \val +.endif +.endm + +.macro movdup_if reg, gpr, val, cond +.if \cond + mov \gpr, \val + dup \reg, \gpr +.endif +.endm + +.macro st1_if regs, dst, cond +.if \cond + st1 \regs, \dst +.endif +.endm + +.macro str_if reg, dst, cond +.if \cond + str \reg, \dst +.endif +.endm + +.macro stroff_if reg, dst, dstoff, cond +.if \cond + str \reg, \dst, \dstoff +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input .8h, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_8h_x64_neon, export=1 + mov x14, x30 + mov x6, sp + lsl x8, x8, #2 + + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + load8 x7, x8, v7.8h, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + add x7, x7, x8, lsr #1 + scale_if \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct_8h_x16_neon + + store16 x6 + + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + load8 x7, x8, v7.8h, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + lsr x8, x8, #1 + sub x7, x7, x8, lsr #1 + scale_if \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct32_odd_8h_x16_neon + + add x10, x6, #16*15 + sub x6, x6, #16*16 + + mov x9, #-16 + +.macro store_addsub r0, r1, r2, r3 + ld1 {v2.8h}, [x6], #16 + ld1 {v3.8h}, [x6], #16 + sqadd v6.8h, v2.8h, \r0 + sqsub \r0, v2.8h, \r0 + ld1 {v4.8h}, [x6], #16 + sqadd v7.8h, v3.8h, \r1 + sqsub \r1, v3.8h, \r1 + ld1 {v5.8h}, [x6], #16 + sqadd v2.8h, v4.8h, \r2 + sub x6, x6, #16*4 + sqsub \r2, v4.8h, \r2 + st1 {v6.8h}, [x6], #16 + st1 {\r0}, [x10], x9 + sqadd v3.8h, v5.8h, \r3 + sqsub \r3, v5.8h, \r3 + st1 {v7.8h}, [x6], #16 + st1 {\r1}, [x10], x9 + st1 {v2.8h}, [x6], #16 + st1 {\r2}, [x10], x9 + st1 {v3.8h}, [x6], #16 + st1 {\r3}, [x10], x9 +.endm + store_addsub v31.8h, v30.8h, v29.8h, v28.8h + store_addsub v27.8h, v26.8h, v25.8h, v24.8h + store_addsub v23.8h, v22.8h, v21.8h, v20.8h + store_addsub v19.8h, v18.8h, v17.8h, v16.8h +.purgem store_addsub + + add x6, x6, #2*8*16 + + movrel x17, idct64_coeffs + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + add x9, x7, x8, lsl #4 // offset 16 + add x10, x7, x8, lsl #3 // offset 8 + sub x9, x9, x8 // offset 15 + sub x11, x10, x8 // offset 7 + ld1 {v16.8h}, [x7] // in1 (offset 0) + ld1 {v17.8h}, [x9] // in31 (offset 15) + ld1 {v18.8h}, [x10] // in17 (offset 8) + ld1 {v19.8h}, [x11] // in15 (offset 7) + st1_if {v7.8h}, [x7], \clear + st1_if {v7.8h}, [x9], \clear + st1_if {v7.8h}, [x10], \clear + st1_if {v7.8h}, [x11], \clear + scale_if \scale, v0.h[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + add x7, x7, x8, lsl #2 // offset 4 + sub x9, x9, x8, lsl #2 // offset 11 + sub x10, x7, x8 // offset 3 + add x11, x9, x8 // offset 12 + ld1 {v16.8h}, [x10] // in7 (offset 3) + ld1 {v17.8h}, [x11] // in25 (offset 12) + ld1 {v18.8h}, [x9] // in23 (offset 11) + ld1 {v19.8h}, [x7] // in9 (offset 4) + st1_if {v7.8h}, [x7], \clear + st1_if {v7.8h}, [x9], \clear + st1_if {v7.8h}, [x10], \clear + st1_if {v7.8h}, [x11], \clear + scale_if \scale, v0.h[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + sub x10, x10, x8, lsl #1 // offset 1 + sub x9, x9, x8, lsl #1 // offset 9 + add x7, x7, x8 // offset 5 + add x11, x11, x8 // offset 13 + ldr q16, [x10, x8] // in5 (offset 2) + ldr q17, [x11] // in27 (offset 13) + ldr q18, [x9, x8] // in21 (offset 10) + ldr q19, [x7] // in11 (offset 5) + stroff_if q7, [x10, x8], \clear + str_if q7, [x11], \clear + stroff_if q7, [x9, x8], \clear + str_if q7, [x7], \clear + scale_if \scale, v0.h[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movdup_if v0.4h, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + ldr q16, [x10] // in3 (offset 1) + ldr q17, [x11, x8] // in29 (offset 14) + ldr q18, [x9] // in19 (offset 9) + ldr q19, [x7, x8] // in13 (offset 6) + str_if q7, [x10], \clear + stroff_if q7, [x11, x8], \clear + str_if q7, [x9], \clear + stroff_if q7, [x7, x8], \clear + scale_if \scale, v0.h[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + + sub x6, x6, #2*8*32 + add x9, x6, #2*8*7 + + bl inv_dct64_step2_neon + + ret x14 +endfunc +.endm + +def_dct64_func +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + + +function inv_txfm_horz_dct_64x8_neon + mov x14, x30 + + mov x7, sp + add x8, sp, #2*8*(64 - 4) + add x9, x6, #2*56 + mov x10, #2*64 + mov x11, #-2*8*4 + + dup v7.8h, w12 +1: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64 + ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11 + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11 + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + transpose_8x8h v31, v30, v29, v28, v27, v26, v25, v24, v4, v5 + +.macro store_addsub src0, src1, src2, src3 + sqsub v1.8h, \src0, \src1 + sqadd v0.8h, \src0, \src1 + sqsub v3.8h, \src2, \src3 + srshl v1.8h, v1.8h, v7.8h + sqadd v2.8h, \src2, \src3 + srshl v0.8h, v0.8h, v7.8h + srshl v3.8h, v3.8h, v7.8h + rev64 v1.8h, v1.8h + srshl v2.8h, v2.8h, v7.8h + rev64 v3.8h, v3.8h + ext v1.16b, v1.16b, v1.16b, #8 + st1 {v0.8h}, [x6], x10 + ext v3.16b, v3.16b, v3.16b, #8 + st1 {v1.8h}, [x9], x10 + st1 {v2.8h}, [x6], x10 + st1 {v3.8h}, [x9], x10 +.endm + store_addsub v16.8h, v31.8h, v17.8h, v30.8h + store_addsub v18.8h, v29.8h, v19.8h, v28.8h + store_addsub v20.8h, v27.8h, v21.8h, v26.8h + store_addsub v22.8h, v25.8h, v23.8h, v24.8h +.purgem store_addsub + sub x6, x6, x10, lsl #3 + sub x9, x9, x10, lsl #3 + add x6, x6, #16 + sub x9, x9, #16 + + cmp x7, x8 + b.lt 1b + ret x14 +endfunc + +function inv_txfm_add_vert_dct_8x64_neon + mov x14, x30 + lsl x8, x8, #1 + + mov x7, sp + add x8, sp, #2*8*(64 - 4) + add x9, x6, x1, lsl #6 + sub x9, x9, x1 + neg x10, x1 + mov x11, #-2*8*4 + +1: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64 + ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11 + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11 + +.macro add_dest_addsub src0, src1, src2, src3 + ld1 {v0.8b}, [x6], x1 + ld1 {v1.8b}, [x9], x10 + sqadd v4.8h, \src0, \src1 + ld1 {v2.8b}, [x6] + sqsub v5.8h, \src0, \src1 + ld1 {v3.8b}, [x9] + sqadd v6.8h, \src2, \src3 + sqsub v7.8h, \src2, \src3 + sub x6, x6, x1 + sub x9, x9, x10 + srshr v4.8h, v4.8h, #4 + srshr v5.8h, v5.8h, #4 + srshr v6.8h, v6.8h, #4 + uaddw v4.8h, v4.8h, v0.8b + srshr v7.8h, v7.8h, #4 + uaddw v5.8h, v5.8h, v1.8b + uaddw v6.8h, v6.8h, v2.8b + sqxtun v0.8b, v4.8h + uaddw v7.8h, v7.8h, v3.8b + sqxtun v1.8b, v5.8h + st1 {v0.8b}, [x6], x1 + sqxtun v2.8b, v6.8h + st1 {v1.8b}, [x9], x10 + sqxtun v3.8b, v7.8h + st1 {v2.8b}, [x6], x1 + st1 {v3.8b}, [x9], x10 +.endm + add_dest_addsub v16.8h, v31.8h, v17.8h, v30.8h + add_dest_addsub v18.8h, v29.8h, v19.8h, v28.8h + add_dest_addsub v20.8h, v27.8h, v21.8h, v26.8h + add_dest_addsub v22.8h, v25.8h, v23.8h, v24.8h +.purgem add_dest_addsub + cmp x7, x8 + b.lt 1b + + ret x14 +endfunc + +function inv_txfm_add_dct_dct_64x64_8bpc_neon, export=1 + idct_dc 64, 64, 2 + + mov x15, x30 + + sub_sp 64*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_32x32 + +.irp i, 0, 8, 16, 24 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*2) + mov x8, #32*2 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_8h_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x8_neon +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x7, x5, #(\i*2) + mov x8, #64*2 + bl inv_txfm_dct_8h_x64_neon + add x6, x0, #(\i) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #64*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_64x32_8bpc_neon, export=1 + idct_dc 64, 32, 1 + + mov x15, x30 + + sub_sp 64*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_32x32 + +.irp i, 0, 8, 16, 24 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*2) + mov x8, #32*2 + mov x12, #-1 // shift + bl inv_txfm_dct_clear_scale_8h_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x8_neon +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i) + add x7, x5, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, x5, #64*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x64_8bpc_neon, export=1 + idct_dc 32, 64, 1 + + mov x15, x30 + + sub_sp 32*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 8, 16, 24 + add x6, x5, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*2) + mov x8, #32*2 + bl inv_txfm_horz_scale_dct_32x8_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x7, x5, #(\i*2) + mov x8, #32*2 + bl inv_txfm_dct_8h_x64_neon + add x6, x0, #(\i) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #32*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_64x16_8bpc_neon, export=1 + idct_dc 64, 16, 2 + + mov x15, x30 + + sub_sp 64*16*2+64*8*2 + add x4, sp, #64*8*2 + + movrel x13, eob_16x32 + +.irp i, 0, 8 + add x6, x4, #(\i*64*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*2) + mov x8, #16*2 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_8h_x64_neon + add x6, x4, #(\i*64*2) + bl inv_txfm_horz_dct_64x8_neon +.if \i < 8 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: + adr x5, inv_dct_8h_x16_neon +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i) + add x7, x4, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, x4, #64*16*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_16x64_8bpc_neon, export=1 + idct_dc 16, 64, 2 + + mov x15, x30 + + sub_sp 16*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + + adr x4, inv_dct_8h_x16_neon +.irp i, 0, 8, 16, 24 + add x6, x5, #(\i*16*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 24 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*2) + mov x8, #32*2 + bl inv_txfm_horz_16x8_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #8 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x7, x5, #(\i*2) + mov x8, #16*2 + bl inv_txfm_dct_8h_x64_neon + add x6, x0, #(\i) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #16*32*2 + ret x15 +endfunc diff --git a/third_party/dav1d/src/arm/64/itx16.S b/third_party/dav1d/src/arm/64/itx16.S new file mode 100644 index 0000000000..eee3a9636d --- /dev/null +++ b/third_party/dav1d/src/arm/64/itx16.S @@ -0,0 +1,3648 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob, +// int bitdepth_max); + +// Most of the functions use the following register layout: +// x0-x3 external parameters +// x4 function pointer to first transform +// x5 function pointer to second transform +// x6 output parameter for helper function +// x7 input parameter for helper function +// x8 input stride for helper function +// x9-x12 scratch variables for helper functions +// x13 pointer to list of eob thresholds +// x14 return pointer for helper function +// x15 return pointer for main function + +// The SIMD registers most often use the following layout: +// v0-v1 multiplication coefficients +// v2-v7 scratch registers +// v8-v15 unused +// v16-v31 inputs/outputs of transforms + +const idct_coeffs, align=4 + // idct4 + .int 2896, 2896*8*(1<<16), 1567, 3784 + // idct8 + .int 799, 4017, 3406, 2276 + // idct16 + .int 401, 4076, 3166, 2598 + .int 1931, 3612, 3920, 1189 + // idct32 + .int 201, 4091, 3035, 2751 + .int 1751, 3703, 3857, 1380 + .int 995, 3973, 3513, 2106 + .int 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .int 101*8*(1<<16), 4095*8*(1<<16), 2967*8*(1<<16), -2824*8*(1<<16) + .int 1660*8*(1<<16), 3745*8*(1<<16), 3822*8*(1<<16), -1474*8*(1<<16) + .int 4076, 401, 4017, 799 + + .int 4036*8*(1<<16), -700*8*(1<<16), 2359*8*(1<<16), 3349*8*(1<<16) + .int 3461*8*(1<<16), -2191*8*(1<<16), 897*8*(1<<16), 3996*8*(1<<16) + .int -3166, -2598, -799, -4017 + + .int 501*8*(1<<16), 4065*8*(1<<16), 3229*8*(1<<16), -2520*8*(1<<16) + .int 2019*8*(1<<16), 3564*8*(1<<16), 3948*8*(1<<16), -1092*8*(1<<16) + .int 3612, 1931, 2276, 3406 + + .int 4085*8*(1<<16), -301*8*(1<<16), 2675*8*(1<<16), 3102*8*(1<<16) + .int 3659*8*(1<<16), -1842*8*(1<<16), 1285*8*(1<<16), 3889*8*(1<<16) + .int -3920, -1189, -3406, -2276 +endconst + +const iadst4_coeffs, align=4 + .int 1321, 3803, 2482, 3344 +endconst + +const iadst8_coeffs, align=4 + .int 4076, 401, 3612, 1931 + .int 2598, 3166, 1189, 3920 + // idct_coeffs + .int 2896, 0, 1567, 3784 +endconst + +const iadst16_coeffs, align=4 + .int 4091, 201, 3973, 995 + .int 3703, 1751, 3290, 2440 + .int 2751, 3035, 2106, 3513 + .int 1380, 3857, 601, 4052 +endconst + +.macro mul_mla d, s0, s1, c0, c1 + mul \d\().4s, \s0\().4s, \c0 + mla \d\().4s, \s1\().4s, \c1 +.endm + +.macro mul_mls d, s0, s1, c0, c1 + mul \d\().4s, \s0\().4s, \c0 + mls \d\().4s, \s1\().4s, \c1 +.endm + +.macro scale_input sz, c, r0, r1, r2 r3, r4, r5, r6, r7 + sqrdmulh \r0\sz, \r0\sz, \c + sqrdmulh \r1\sz, \r1\sz, \c + sqrdmulh \r2\sz, \r2\sz, \c + sqrdmulh \r3\sz, \r3\sz, \c +.ifnb \r4 + sqrdmulh \r4\sz, \r4\sz, \c + sqrdmulh \r5\sz, \r5\sz, \c + sqrdmulh \r6\sz, \r6\sz, \c + sqrdmulh \r7\sz, \r7\sz, \c +.endif +.endm + +.macro smin_4s r0, r1, r2 + smin \r0\().4s, \r1\().4s, \r2\().4s +.endm +.macro smax_4s r0, r1, r2 + smax \r0\().4s, \r1\().4s, \r2\().4s +.endm + +.macro load_add_store load, shift, addsrc, adddst, min, store, dst, src, shiftbits=4 +.ifnb \load + ld1 {\load}, [\src], x1 +.endif +.ifnb \shift + srshr \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + usqadd \adddst, \addsrc +.endif +.ifnb \min + smin \min, \min, v7.8h +.endif +.ifnb \store + st1 {\store}, [\dst], x1 +.endif +.endm +.macro load_add_store_8x16 dst, src + mov \src, \dst + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , \dst, \src + load_add_store v3.8h, v17.8h, , , , , \dst, \src + load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src + load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src + load_add_store v16.8h, v20.8h, v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src + load_add_store v17.8h, v21.8h, v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src + load_add_store v18.8h, v22.8h, v20.8h, v16.8h, v5.8h, v4.8h, \dst, \src + load_add_store v19.8h, v23.8h, v21.8h, v17.8h, v16.8h, v5.8h, \dst, \src + load_add_store v20.8h, v24.8h, v22.8h, v18.8h, v17.8h, v16.8h, \dst, \src + load_add_store v21.8h, v25.8h, v23.8h, v19.8h, v18.8h, v17.8h, \dst, \src + load_add_store v22.8h, v26.8h, v24.8h, v20.8h, v19.8h, v18.8h, \dst, \src + load_add_store v23.8h, v27.8h, v25.8h, v21.8h, v20.8h, v19.8h, \dst, \src + load_add_store v24.8h, v28.8h, v26.8h, v22.8h, v21.8h, v20.8h, \dst, \src + load_add_store v25.8h, v29.8h, v27.8h, v23.8h, v22.8h, v21.8h, \dst, \src + load_add_store v26.8h, v30.8h, v28.8h, v24.8h, v23.8h, v22.8h, \dst, \src + load_add_store v27.8h, v31.8h, v29.8h, v25.8h, v24.8h, v23.8h, \dst, \src + load_add_store , , v30.8h, v26.8h, v25.8h, v24.8h, \dst, \src + load_add_store , , v31.8h, v27.8h, v26.8h, v25.8h, \dst, \src + load_add_store , , , , v27.8h, v26.8h, \dst, \src + load_add_store , , , , , v27.8h, \dst, \src +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , \dst, \src, \shiftbits + load_add_store v3.8h, v17.8h, , , , , \dst, \src, \shiftbits + load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src, \shiftbits + load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src, \shiftbits + load_add_store v16.8h, v20.8h, v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src, \shiftbits + load_add_store v17.8h, v21.8h, v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src, \shiftbits + load_add_store v18.8h, v22.8h, v20.8h, v16.8h, v5.8h, v4.8h, \dst, \src, \shiftbits + load_add_store v19.8h, v23.8h, v21.8h, v17.8h, v16.8h, v5.8h, \dst, \src, \shiftbits + load_add_store , , v22.8h, v18.8h, v17.8h, v16.8h, \dst, \src, \shiftbits + load_add_store , , v23.8h, v19.8h, v18.8h, v17.8h, \dst, \src, \shiftbits + load_add_store , , , , v19.8h, v18.8h, \dst, \src, \shiftbits + load_add_store , , , , , v19.8h, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src, shiftbits=4 + mov \src, \dst + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , \dst, \src, \shiftbits + load_add_store v3.8h, v17.8h, , , , , \dst, \src, \shiftbits + load_add_store v4.8h, v18.8h, v16.8h, v2.8h, , , \dst, \src, \shiftbits + load_add_store v5.8h, v19.8h, v17.8h, v3.8h, v2.8h, , \dst, \src, \shiftbits + load_add_store , , v18.8h, v4.8h, v3.8h, v2.8h, \dst, \src, \shiftbits + load_add_store , , v19.8h, v5.8h, v4.8h, v3.8h, \dst, \src, \shiftbits + load_add_store , , , , v5.8h, v4.8h, \dst, \src, \shiftbits + load_add_store , , , , , v5.8h, \dst, \src, \shiftbits +.endm +.macro load_add_store4 load, inssrc, insdst, shift, addsrc, adddst, min, store, dst, src +.ifnb \load + ld1 {\load}[0], [\src], x1 +.endif +.ifnb \inssrc + ins \insdst\().d[1], \inssrc\().d[0] +.endif +.ifnb \shift + srshr \shift, \shift, #4 +.endif +.ifnb \load + ld1 {\load}[1], [\src], x1 +.endif +.ifnb \addsrc + usqadd \adddst, \addsrc +.endif +.ifnb \store + st1 {\store}[0], [\dst], x1 +.endif +.ifnb \min + smin \min, \min, v7.8h +.endif +.ifnb \store + st1 {\store}[1], [\dst], x1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store4 v0.d, v17, v16, , , , , , \dst, \src + load_add_store4 v1.d, v19, v18, , , , , , \dst, \src + load_add_store4 v2.d, v21, v20, v16.8h, , , , , \dst, \src + load_add_store4 v3.d, v23, v22, v18.8h, v16.8h, v0.8h, , , \dst, \src + load_add_store4 v17.d, v25, v24, v20.8h, v18.8h, v1.8h, v0.8h, , \dst, \src + load_add_store4 v19.d, v27, v26, v22.8h, v20.8h, v2.8h, v1.8h, v0.d, \dst, \src + load_add_store4 v21.d, v29, v28, v24.8h, v22.8h, v3.8h, v2.8h, v1.d, \dst, \src + load_add_store4 v23.d, v31, v30, v26.8h, v24.8h, v17.8h, v3.8h, v2.d, \dst, \src + load_add_store4 , , , v28.8h, v26.8h, v19.8h, v17.8h, v3.d, \dst, \src + load_add_store4 , , , v30.8h, v28.8h, v21.8h, v19.8h, v17.d, \dst, \src + load_add_store4 , , , , v30.8h, v23.8h, v21.8h, v19.d, \dst, \src + load_add_store4 , , , , , , v23.8h, v21.d, \dst, \src + load_add_store4 , , , , , , , v23.d, \dst, \src +.endm +.macro load_add_store_4x8 dst, src + mov \src, \dst + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store4 v0.d, v17, v16, , , , , , \dst, \src + load_add_store4 v1.d, v19, v18, , , , , , \dst, \src + load_add_store4 v2.d, v21, v20, v16.8h, , , , , \dst, \src + load_add_store4 v3.d, v23, v22, v18.8h, v16.8h, v0.8h, , , \dst, \src + load_add_store4 , , , v20.8h, v18.8h, v1.8h, v0.8h, , \dst, \src + load_add_store4 , , , v22.8h, v20.8h, v2.8h, v1.8h, v0.d, \dst, \src + load_add_store4 , , , , v22.8h, v3.8h, v2.8h, v1.d, \dst, \src + load_add_store4 , , , , , , v3.8h, v2.d, \dst, \src + load_add_store4 , , , , , , , v3.d, \dst, \src +.endm + +.macro idct_dc w, h, shift + cbnz w3, 1f + movz w16, #2896*8, lsl #16 + ld1r {v16.4s}, [x2] + dup v0.2s, w16 + sqrdmulh v20.4s, v16.4s, v0.s[0] + str wzr, [x2] +.if (\w == 2*\h) || (2*\w == \h) + sqrdmulh v20.4s, v20.4s, v0.s[0] +.endif +.if \shift > 0 + sqrshrn v16.4h, v20.4s, #\shift + sqrshrn2 v16.8h, v20.4s, #\shift +.else + sqxtn v16.4h, v20.4s + sqxtn2 v16.8h, v20.4s +.endif + sqrdmulh v16.8h, v16.8h, v0.h[1] + srshr v16.8h, v16.8h, #4 + mov w4, #\h + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.d}[0], [x0], x1 + ld1 {v0.d}[1], [x0], x1 + ld1 {v1.d}[0], [x0], x1 + subs w4, w4, #4 + ld1 {v1.d}[1], [x0], x1 + usqadd v0.8h, v16.8h + sub x0, x0, x1, lsl #2 + usqadd v1.8h, v16.8h + smin v0.8h, v0.8h, v31.8h + st1 {v0.d}[0], [x0], x1 + smin v1.8h, v1.8h, v31.8h + st1 {v0.d}[1], [x0], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w8_neon + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h}, [x0], x1 + subs w4, w4, #4 + ld1 {v1.8h}, [x0], x1 + usqadd v0.8h, v16.8h + ld1 {v2.8h}, [x0], x1 + usqadd v1.8h, v16.8h + ld1 {v3.8h}, [x0], x1 + usqadd v2.8h, v16.8h + usqadd v3.8h, v16.8h + sub x0, x0, x1, lsl #2 + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + st1 {v0.8h}, [x0], x1 + smin v2.8h, v2.8h, v31.8h + st1 {v1.8h}, [x0], x1 + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w16_neon + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h, v1.8h}, [x0], x1 + subs w4, w4, #2 + ld1 {v2.8h, v3.8h}, [x0], x1 + usqadd v0.8h, v16.8h + usqadd v1.8h, v16.8h + sub x0, x0, x1, lsl #1 + usqadd v2.8h, v16.8h + usqadd v3.8h, v16.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + st1 {v0.8h, v1.8h}, [x0], x1 + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h, v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w32_neon + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + subs w4, w4, #1 + usqadd v0.8h, v16.8h + usqadd v1.8h, v16.8h + usqadd v2.8h, v16.8h + usqadd v3.8h, v16.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w64_neon + mvni v31.8h, #0xfc, lsl #8 // 0x3ff + sub x1, x1, #64 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + subs w4, w4, #1 + usqadd v0.8h, v16.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0] + usqadd v1.8h, v16.8h + sub x0, x0, #64 + usqadd v2.8h, v16.8h + usqadd v3.8h, v16.8h + usqadd v4.8h, v16.8h + usqadd v5.8h, v16.8h + usqadd v6.8h, v16.8h + usqadd v7.8h, v16.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + smin v4.8h, v4.8h, v31.8h + smin v5.8h, v5.8h, v31.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + smin v6.8h, v6.8h, v31.8h + smin v7.8h, v7.8h, v31.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +.macro iwht4 + add v16.4s, v16.4s, v17.4s + sub v21.4s, v18.4s, v19.4s + sub v20.4s, v16.4s, v21.4s + sshr v20.4s, v20.4s, #1 + sub v18.4s, v20.4s, v17.4s + sub v17.4s, v20.4s, v19.4s + add v19.4s, v21.4s, v18.4s + sub v16.4s, v16.4s, v17.4s +.endm + +.macro idct_4 r0, r1, r2, r3 + mul_mla v6, \r1, \r3, v0.s[3], v0.s[2] + mul_mla v2, \r0, \r2, v0.s[0], v0.s[0] + mul_mls v4, \r1, \r3, v0.s[2], v0.s[3] + mul_mls v3, \r0, \r2, v0.s[0], v0.s[0] + srshr v6.4s, v6.4s, #12 + srshr v2.4s, v2.4s, #12 + srshr v7.4s, v4.4s, #12 + srshr v3.4s, v3.4s, #12 + sqadd \r0\().4s, v2.4s, v6.4s + sqsub \r3\().4s, v2.4s, v6.4s + sqadd \r1\().4s, v3.4s, v7.4s + sqsub \r2\().4s, v3.4s, v7.4s +.endm + +function inv_dct_4s_x4_neon + AARCH64_VALID_CALL_TARGET + movrel x16, idct_coeffs + ld1 {v0.4s}, [x16] + idct_4 v16, v17, v18, v19 + ret +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel x16, iadst4_coeffs + ld1 {v0.4s}, [x16] + + sub v3.4s, v16.4s, v18.4s + mul v4.4s, v16.4s, v0.s[0] + mla v4.4s, v18.4s, v0.s[1] + mla v4.4s, v19.4s, v0.s[2] + mul v7.4s, v17.4s, v0.s[3] + add v3.4s, v3.4s, v19.4s + mul v5.4s, v16.4s, v0.s[2] + mls v5.4s, v18.4s, v0.s[0] + mls v5.4s, v19.4s, v0.s[1] + + add \o3\().4s, v4.4s, v5.4s + mul \o2\().4s, v3.4s, v0.s[3] + add \o0\().4s, v4.4s, v7.4s + add \o1\().4s, v5.4s, v7.4s + sub \o3\().4s, \o3\().4s, v7.4s + + srshr \o0\().4s, \o0\().4s, #12 + srshr \o2\().4s, \o2\().4s, #12 + srshr \o1\().4s, \o1\().4s, #12 + srshr \o3\().4s, \o3\().4s, #12 +.endm + +function inv_adst_4s_x4_neon + AARCH64_VALID_CALL_TARGET + iadst_4x4 v16, v17, v18, v19 + ret +endfunc + +function inv_flipadst_4s_x4_neon + AARCH64_VALID_CALL_TARGET + iadst_4x4 v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x4_neon + AARCH64_VALID_CALL_TARGET + movz w16, #(5793-4096)*8, lsl #16 + dup v0.2s, w16 + sqrdmulh v4.4s, v16.4s, v0.s[0] + sqrdmulh v5.4s, v17.4s, v0.s[0] + sqrdmulh v6.4s, v18.4s, v0.s[0] + sqrdmulh v7.4s, v19.4s, v0.s[0] + sqadd v16.4s, v16.4s, v4.4s + sqadd v17.4s, v17.4s, v5.4s + sqadd v18.4s, v18.4s, v6.4s + sqadd v19.4s, v19.4s, v7.4s + ret +endfunc + +function inv_txfm_add_wht_wht_4x4_16bpc_neon, export=1 + mov x15, x30 + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v30.4s, v31.4s}, [x2], #32 + + sshr v16.4s, v16.4s, #2 + sshr v17.4s, v17.4s, #2 + sshr v18.4s, v18.4s, #2 + sshr v19.4s, v19.4s, #2 + + iwht4 + + st1 {v30.4s, v31.4s}, [x2], #32 + transpose_4x4s v16, v17, v18, v19, v20, v21, v22, v23 + + iwht4 + + ld1 {v0.d}[0], [x0], x1 + sqxtn v16.4h, v16.4s + ld1 {v0.d}[1], [x0], x1 + sqxtn2 v16.8h, v17.4s + ld1 {v1.d}[0], [x0], x1 + sqxtn v18.4h, v18.4s + ld1 {v1.d}[1], [x0], x1 + sqxtn2 v18.8h, v19.4s + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v30.4s, v31.4s}, [x2], #32 + + blr x4 + + st1 {v30.4s, v31.4s}, [x2], #32 + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + transpose_4x4h v16, v17, v18, v19, v20, v21, v22, v23 + + blr x5 + + ld1 {v0.d}[0], [x0], x1 + ld1 {v0.d}[1], [x0], x1 + ins v16.d[1], v17.d[0] + ins v18.d[1], v19.d[0] + ld1 {v1.d}[0], [x0], x1 + ld1 {v1.d}[1], [x0], x1 + srshr v16.8h, v16.8h, #4 + srshr v18.8h, v18.8h, #4 + +L(itx_4x4_end): + mvni v31.8h, #0xfc, lsl #8 // 0x3ff + sub x0, x0, x1, lsl #2 + usqadd v0.8h, v16.8h + usqadd v1.8h, v18.8h + smin v0.8h, v0.8h, v31.8h + st1 {v0.d}[0], [x0], x1 + smin v1.8h, v1.8h, v31.8h + st1 {v0.d}[1], [x0], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x0], x1 + + ret x15 +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + cbnz w3, 1f + movz w16, #2896*8, lsl #16 + ld1r {v16.4s}, [x2] + dup v4.2s, w16 + str wzr, [x2] + sqrdmulh v16.4s, v16.4s, v4.s[0] + ld1 {v0.d}[0], [x0], x1 + sqxtn v20.4h, v16.4s + sqxtn2 v20.8h, v16.4s + ld1 {v0.d}[1], [x0], x1 + sqrdmulh v20.8h, v20.8h, v4.h[1] + ld1 {v1.d}[0], [x0], x1 + srshr v16.8h, v20.8h, #4 + ld1 {v1.d}[1], [x0], x1 + srshr v18.8h, v20.8h, #4 + movi v30.8h, #0 + b L(itx_4x4_end) +1: +.endif + adr x4, inv_\txfm1\()_4s_x4_neon + movrel x5, X(inv_\txfm2\()_4h_x4_neon) + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_4 \r0, \r2, \r4, \r6 + + movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 +.irp r, \r0, \r2, \r4, \r6 + smin_4s \r, \r, v5 +.endr +.irp r, \r0, \r2, \r4, \r6 + smax_4s \r, \r, v4 +.endr + + mul_mls v2, \r1, \r7, v1.s[0], v1.s[1] // -> t4a + mul_mla v3, \r1, \r7, v1.s[1], v1.s[0] // -> t7a + mul_mls v6, \r5, \r3, v1.s[2], v1.s[3] // -> t5a + mul_mla v7, \r5, \r3, v1.s[3], v1.s[2] // -> t6a + srshr \r1\().4s, v2.4s, #12 // t4a + srshr \r7\().4s, v3.4s, #12 // t7a + srshr \r3\().4s, v6.4s, #12 // t5a + srshr \r5\().4s, v7.4s, #12 // t6a + + sqadd v2.4s, \r1\().4s, \r3\().4s // t4 + sqsub \r1\().4s, \r1\().4s, \r3\().4s // t5a + sqadd v3.4s, \r7\().4s, \r5\().4s // t7 + sqsub \r3\().4s, \r7\().4s, \r5\().4s // t6a + +.irp r, v2, \r1, v3, \r3 + smin_4s \r, \r, v5 +.endr +.irp r, v2, \r1, v3, \r3 + smax_4s \r, \r, v4 +.endr + + mul_mls v7, \r3, \r1, v0.s[0], v0.s[0] // -> t5 + mul_mla v6, \r3, \r1, v0.s[0], v0.s[0] // -> t6 + srshr v7.4s, v7.4s, #12 // t5 + srshr v6.4s, v6.4s, #12 // t6 + + sqsub \r7\().4s, \r0\().4s, v3.4s // out7 + sqadd \r0\().4s, \r0\().4s, v3.4s // out0 + sqadd \r1\().4s, \r2\().4s, v6.4s // out1 + sqsub v6.4s, \r2\().4s, v6.4s // out6 + sqadd \r2\().4s, \r4\().4s, v7.4s // out2 + sqsub \r5\().4s, \r4\().4s, v7.4s // out5 + sqadd \r3\().4s, \r6\().4s, v2.4s // out3 + sqsub \r4\().4s, \r6\().4s, v2.4s // out4 + mov \r6\().16b, v6.16b // out6 +.endm + +function inv_dct_4s_x8_neon + AARCH64_VALID_CALL_TARGET + movrel x16, idct_coeffs + ld1 {v0.4s, v1.4s}, [x16] + idct_8 v16, v17, v18, v19, v20, v21, v22, v23 + ret +endfunc + +.macro iadst_8 o0, o1, o2, o3, o4, o5, o6, o7 + movrel x16, iadst8_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mla v2, v23, v16, v0.s[0], v0.s[1] + mul_mls v4, v23, v16, v0.s[1], v0.s[0] + mul_mla v6, v21, v18, v0.s[2], v0.s[3] + srshr v16.4s, v2.4s, #12 // t0a + srshr v23.4s, v4.4s, #12 // t1a + mul_mls v2, v21, v18, v0.s[3], v0.s[2] + mul_mla v4, v19, v20, v1.s[0], v1.s[1] + srshr v18.4s, v6.4s, #12 // t2a + srshr v21.4s, v2.4s, #12 // t3a + mul_mls v6, v19, v20, v1.s[1], v1.s[0] + mul_mla v2, v17, v22, v1.s[2], v1.s[3] + srshr v20.4s, v4.4s, #12 // t4a + srshr v19.4s, v6.4s, #12 // t5a + mul_mls v4, v17, v22, v1.s[3], v1.s[2] + srshr v22.4s, v2.4s, #12 // t6a + srshr v17.4s, v4.4s, #12 // t7a + + ld1 {v0.4s}, [x16] + + movi v1.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + + sqadd v2.4s, v16.4s, v20.4s // t0 + sqsub v3.4s, v16.4s, v20.4s // t4 + mvni v20.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + sqadd v4.4s, v23.4s, v19.4s // t1 + sqsub v5.4s, v23.4s, v19.4s // t5 + sqadd v6.4s, v18.4s, v22.4s // t2 + sqsub v7.4s, v18.4s, v22.4s // t6 + sqadd v18.4s, v21.4s, v17.4s // t3 + sqsub v19.4s, v21.4s, v17.4s // t7 + +.irp r, v2, v3, v4, v5, v6, v7, v18, v19 + smin_4s \r, \r, v1 +.endr +.irp r, v2, v3, v4, v5, v6, v7, v18, v19 + smax_4s \r, \r, v20 +.endr + + mul_mla v16, v3, v5, v0.s[3], v0.s[2] + mul_mls v20, v3, v5, v0.s[2], v0.s[3] + mul_mls v22, v19, v7, v0.s[3], v0.s[2] + + srshr v3.4s, v16.4s, #12 // t4a + srshr v5.4s, v20.4s, #12 // t5a + + mul_mla v16, v19, v7, v0.s[2], v0.s[3] + + srshr v7.4s, v22.4s, #12 // t6a + srshr v19.4s, v16.4s, #12 // t7a + + sqadd \o0\().4s, v2.4s, v6.4s // out0 + sqsub v2.4s, v2.4s, v6.4s // t2 + sqadd \o7\().4s, v4.4s, v18.4s // out7 + sqsub v4.4s, v4.4s, v18.4s // t3 + + mvni v18.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + + sqadd \o1\().4s, v3.4s, v7.4s // out1 + sqsub v3.4s, v3.4s, v7.4s // t6 + sqadd \o6\().4s, v5.4s, v19.4s // out6 + sqsub v5.4s, v5.4s, v19.4s // t7 + + // Not clipping the output registers, as they will be downshifted and + // narrowed afterwards anyway. +.irp r, v2, v4, v3, v5 + smin_4s \r, \r, v1 +.endr +.irp r, v2, v4, v3, v5 + smax_4s \r, \r, v18 +.endr + + sqneg \o7\().4s, \o7\().4s // out7 + sqneg \o1\().4s, \o1\().4s // out1 + + mul_mla v18, v2, v4, v0.s[0], v0.s[0] // -> out3 (v19 or v20) + mul_mls v6, v2, v4, v0.s[0], v0.s[0] // -> out4 (v20 or v19) + mul_mls v20, v3, v5, v0.s[0], v0.s[0] // -> out5 (v21 or v18) + srshr v2.4s, v18.4s, #12 // out3 + mul_mla v18, v3, v5, v0.s[0], v0.s[0] // -> out2 (v18 or v21) + srshr v3.4s, v20.4s, #12 // out5 + srshr \o2\().4s, v18.4s, #12 // out2 (v18 or v21) + srshr \o4\().4s, v6.4s, #12 // out4 (v20 or v19) + + sqneg \o3\().4s, v2.4s // out3 + sqneg \o5\().4s, v3.4s // out5 +.endm + +function inv_adst_4s_x8_neon + AARCH64_VALID_CALL_TARGET + iadst_8 v16, v17, v18, v19, v20, v21, v22, v23 + ret +endfunc + +function inv_flipadst_4s_x8_neon + AARCH64_VALID_CALL_TARGET + iadst_8 v23, v22, v21, v20, v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x8_neon + AARCH64_VALID_CALL_TARGET + sqshl v16.4s, v16.4s, #1 + sqshl v17.4s, v17.4s, #1 + sqshl v18.4s, v18.4s, #1 + sqshl v19.4s, v19.4s, #1 + sqshl v20.4s, v20.4s, #1 + sqshl v21.4s, v21.4s, #1 + sqshl v22.4s, v22.4s, #1 + sqshl v23.4s, v23.4s, #1 + ret +endfunc + +function inv_txfm_add_8x8_neon + movi v31.4s, #0 + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v31.4s}, [x6], x11 +.endr + + blr x4 + + sqrshrn v24.4h, v16.4s, #1 + sqrshrn v25.4h, v17.4s, #1 + sqrshrn v26.4h, v18.4s, #1 + sqrshrn v27.4h, v19.4s, #1 + sqrshrn2 v24.8h, v20.4s, #1 + sqrshrn2 v25.8h, v21.4s, #1 + sqrshrn2 v26.8h, v22.4s, #1 + sqrshrn2 v27.8h, v23.4s, #1 + + transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + movi \i, #0 +.endr + +2: + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x2] + st1 {v31.4s}, [x2], x11 +.endr + + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + + transpose_4x8h v16, v17, v18, v19, v20, v21, v22, v23 + + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v22.16b, v26.16b + mov v23.16b, v27.16b + + blr x5 + + load_add_store_8x8 x0, x7 + ret x15 +endfunc + +.macro def_fn_8x8 txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + movrel x5, X(inv_\txfm2\()_8h_x8_neon) + mov w13, #\eob_half + adr x4, inv_\txfm1\()_4s_x8_neon + b inv_txfm_add_8x8_neon +endfunc +.endm + +def_fn_8x8 dct, dct, 10 +def_fn_8x8 identity, identity, 10 +def_fn_8x8 dct, adst, 10 +def_fn_8x8 dct, flipadst, 10 +def_fn_8x8 dct, identity, 4 +def_fn_8x8 adst, dct, 10 +def_fn_8x8 adst, adst, 10 +def_fn_8x8 adst, flipadst, 10 +def_fn_8x8 flipadst, dct, 10 +def_fn_8x8 flipadst, adst, 10 +def_fn_8x8 flipadst, flipadst, 10 +def_fn_8x8 identity, dct, 4 +def_fn_8x8 adst, identity, 4 +def_fn_8x8 flipadst, identity, 4 +def_fn_8x8 identity, adst, 4 +def_fn_8x8 identity, flipadst, 4 + +function inv_txfm_add_8x4_neon + movi v28.4s, #0 + movi v29.4s, #0 + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2], #64 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + ld1 {v20.4s,v21.4s,v22.4s,v23.4s}, [x2] + st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2] + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + blr x4 + + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + sqxtn v20.4h, v20.4s + sqxtn v21.4h, v21.4s + sqxtn v22.4h, v22.4s + sqxtn v23.4h, v23.4s + + transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7 + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + ins v16.d[1], v20.d[0] + ins v17.d[1], v21.d[0] + ins v18.d[1], v22.d[0] + ins v19.d[1], v23.d[0] + + blr x5 + + load_add_store_8x4 x0, x7 + ret x15 +endfunc + +function inv_txfm_add_4x8_neon + movz w16, #2896*8, lsl #16 + movi v31.4s, #0 + dup v30.2s, w16 + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v31.4s}, [x6], x11 +.endr + scale_input .4s, v30.s[0], v16, v17, v18, v19 + blr x4 + sqxtn v20.4h, v16.4s + sqxtn v21.4h, v17.4s + sqxtn v22.4h, v18.4s + sqxtn v23.4h, v19.4s + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + + b 2f + +1: +.irp i, v20, v21, v22, v23 + movi \i\().4h, #0 +.endr + +2: + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x2] + st1 {v31.4s}, [x2], x11 +.endr + scale_input .4s, v30.s[0], v16, v17, v18, v19 + blr x4 + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7 + + blr x5 + + load_add_store_4x8 x0, x7 + ret x15 +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + adr x4, inv_\txfm1\()_4s_x\w\()_neon +.if \w == 4 + mov w13, #\eob_half +.endif + movrel x5, X(inv_\txfm2\()_\w\()h_x\h\()_neon) + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct, 13 +def_fn_48 \w, \h, identity, identity, 13 +def_fn_48 \w, \h, dct, adst, 13 +def_fn_48 \w, \h, dct, flipadst, 13 +def_fn_48 \w, \h, dct, identity, 4 +def_fn_48 \w, \h, adst, dct, 13 +def_fn_48 \w, \h, adst, adst, 13 +def_fn_48 \w, \h, adst, flipadst, 13 +def_fn_48 \w, \h, flipadst, dct, 13 +def_fn_48 \w, \h, flipadst, adst, 13 +def_fn_48 \w, \h, flipadst, flipadst, 13 +def_fn_48 \w, \h, identity, dct, 16 +def_fn_48 \w, \h, adst, identity, 4 +def_fn_48 \w, \h, flipadst, identity, 4 +def_fn_48 \w, \h, identity, adst, 16 +def_fn_48 \w, \h, identity, flipadst, 16 +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + + +function inv_dct_4s_x16_neon + AARCH64_VALID_CALL_TARGET + movrel x16, idct_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + idct_8 v16, v18, v20, v22, v24, v26, v28, v30 + + // idct_8 leaves the row_clip_max/min constants in v5 and v4 +.irp r, v16, v18, v20, v22, v24, v26, v28, v30 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v16, v18, v20, v22, v24, v26, v28, v30 + smax \r\().4s, \r\().4s, v4.4s +.endr + + ld1 {v0.4s, v1.4s}, [x16] + sub x16, x16, #32 + + mul_mls v2, v17, v31, v0.s[0], v0.s[1] // -> t8a + mul_mla v3, v17, v31, v0.s[1], v0.s[0] // -> t15a + mul_mls v6, v25, v23, v0.s[2], v0.s[3] // -> t9a + srshr v17.4s, v2.4s, #12 // t8a + srshr v31.4s, v3.4s, #12 // t15a + mul_mla v2, v25, v23, v0.s[3], v0.s[2] // -> t14a + mul_mls v3, v21, v27, v1.s[0], v1.s[1] // -> t10a + srshr v23.4s, v6.4s, #12 // t9a + srshr v25.4s, v2.4s, #12 // t14a + mul_mla v6, v21, v27, v1.s[1], v1.s[0] // -> t13a + mul_mls v2, v29, v19, v1.s[2], v1.s[3] // -> t11a + srshr v21.4s, v3.4s, #12 // t10a + srshr v27.4s, v6.4s, #12 // t13a + mul_mla v3, v29, v19, v1.s[3], v1.s[2] // -> t12a + srshr v19.4s, v2.4s, #12 // t11a + srshr v29.4s, v3.4s, #12 // t12a + + ld1 {v0.4s}, [x16] + + sqsub v2.4s, v17.4s, v23.4s // t9 + sqadd v17.4s, v17.4s, v23.4s // t8 + sqsub v3.4s, v31.4s, v25.4s // t14 + sqadd v31.4s, v31.4s, v25.4s // t15 + sqsub v23.4s, v19.4s, v21.4s // t10 + sqadd v19.4s, v19.4s, v21.4s // t11 + sqadd v25.4s, v29.4s, v27.4s // t12 + sqsub v29.4s, v29.4s, v27.4s // t13 + +.irp r, v2, v17, v3, v31, v23, v19, v25, v29 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v2, v17, v3, v31, v23, v19, v25, v29 + smax \r\().4s, \r\().4s, v4.4s +.endr + + mul_mls v7, v3, v2, v0.s[2], v0.s[3] // -> t9a + mul_mla v6, v3, v2, v0.s[3], v0.s[2] // -> t14a + srshr v21.4s, v7.4s, #12 // t9a + srshr v27.4s, v6.4s, #12 // t14a + + mul_mls v7, v29, v23, v0.s[2], v0.s[3] // -> t13a + mul_mla v6, v29, v23, v0.s[3], v0.s[2] // -> t10a + srshr v29.4s, v7.4s, #12 // t13a + neg v6.4s, v6.4s + srshr v23.4s, v6.4s, #12 // t10a + + sqsub v2.4s, v17.4s, v19.4s // t11a + sqadd v17.4s, v17.4s, v19.4s // t8a + sqsub v3.4s, v31.4s, v25.4s // t12a + sqadd v31.4s, v31.4s, v25.4s // t15a + sqadd v19.4s, v21.4s, v23.4s // t9 + sqsub v21.4s, v21.4s, v23.4s // t10 + sqsub v25.4s, v27.4s, v29.4s // t13 + sqadd v27.4s, v27.4s, v29.4s // t14 + +.irp r, v2, v17, v3, v31, v19, v21, v25, v27 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v2, v17, v3, v31, v19, v21, v25, v27 + smax \r\().4s, \r\().4s, v4.4s +.endr + + mul_mls v7, v3, v2, v0.s[0], v0.s[0] // -> t11 + mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t12 + mul_mls v2, v25, v21, v0.s[0], v0.s[0] // -> t10a + + srshr v7.4s, v7.4s, #12 // t11 + srshr v6.4s, v6.4s, #12 // t12 + mul_mla v3, v25, v21, v0.s[0], v0.s[0] // -> t13a + srshr v2.4s, v2.4s, #12 // t10a + srshr v3.4s, v3.4s, #12 // t13a + + sqadd v1.4s, v16.4s, v31.4s // out0 + sqsub v31.4s, v16.4s, v31.4s // out15 + mov v16.16b, v1.16b + sqadd v23.4s, v30.4s, v17.4s // out7 + sqsub v1.4s, v30.4s, v17.4s // out8 + sqadd v17.4s, v18.4s, v27.4s // out1 + sqsub v30.4s, v18.4s, v27.4s // out14 + sqadd v18.4s, v20.4s, v3.4s // out2 + sqsub v29.4s, v20.4s, v3.4s // out13 + sqadd v3.4s, v28.4s, v19.4s // out6 + sqsub v25.4s, v28.4s, v19.4s // out9 + sqadd v19.4s, v22.4s, v6.4s // out3 + sqsub v28.4s, v22.4s, v6.4s // out12 + sqadd v20.4s, v24.4s, v7.4s // out4 + sqsub v27.4s, v24.4s, v7.4s // out11 + sqadd v21.4s, v26.4s, v2.4s // out5 + sqsub v26.4s, v26.4s, v2.4s // out10 + mov v24.16b, v1.16b + mov v22.16b, v3.16b + + ret +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15 + movrel x16, iadst16_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mla v2, v31, v16, v0.s[0], v0.s[1] // -> t0 + mul_mls v4, v31, v16, v0.s[1], v0.s[0] // -> t1 + mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t2 + srshr v16.4s, v2.4s, #12 // t0 + srshr v31.4s, v4.4s, #12 // t1 + mul_mls v2, v29, v18, v0.s[3], v0.s[2] // -> t3 + mul_mla v4, v27, v20, v1.s[0], v1.s[1] // -> t4 + srshr v18.4s, v6.4s, #12 // t2 + srshr v29.4s, v2.4s, #12 // t3 + mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t5 + mul_mla v2, v25, v22, v1.s[2], v1.s[3] // -> t6 + srshr v20.4s, v4.4s, #12 // t4 + srshr v27.4s, v6.4s, #12 // t5 + mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t7 + ld1 {v0.4s, v1.4s}, [x16] + movrel x16, idct_coeffs + mul_mla v6, v23, v24, v0.s[0], v0.s[1] // -> t8 + srshr v22.4s, v2.4s, #12 // t6 + srshr v25.4s, v4.4s, #12 // t7 + mul_mls v2, v23, v24, v0.s[1], v0.s[0] // -> t9 + mul_mla v4, v21, v26, v0.s[2], v0.s[3] // -> t10 + srshr v23.4s, v6.4s, #12 // t8 + srshr v24.4s, v2.4s, #12 // t9 + mul_mls v6, v21, v26, v0.s[3], v0.s[2] // -> t11 + mul_mla v2, v19, v28, v1.s[0], v1.s[1] // -> t12 + srshr v21.4s, v4.4s, #12 // t10 + srshr v26.4s, v6.4s, #12 // t11 + mul_mls v4, v19, v28, v1.s[1], v1.s[0] // -> t13 + mul_mla v6, v17, v30, v1.s[2], v1.s[3] // -> t14 + srshr v19.4s, v2.4s, #12 // t12 + srshr v28.4s, v4.4s, #12 // t13 + mul_mls v2, v17, v30, v1.s[3], v1.s[2] // -> t15 + srshr v17.4s, v6.4s, #12 // t14 + srshr v30.4s, v2.4s, #12 // t15 + + ld1 {v0.4s, v1.4s}, [x16] + + movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + mvni v7.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + + sqsub v2.4s, v16.4s, v23.4s // t8a + sqadd v16.4s, v16.4s, v23.4s // t0a + sqsub v3.4s, v31.4s, v24.4s // t9a + sqadd v31.4s, v31.4s, v24.4s // t1a + sqadd v23.4s, v18.4s, v21.4s // t2a + sqsub v18.4s, v18.4s, v21.4s // t10a + sqadd v24.4s, v29.4s, v26.4s // t3a + sqsub v29.4s, v29.4s, v26.4s // t11a + sqadd v21.4s, v20.4s, v19.4s // t4a + sqsub v20.4s, v20.4s, v19.4s // t12a + sqadd v26.4s, v27.4s, v28.4s // t5a + sqsub v27.4s, v27.4s, v28.4s // t13a + sqadd v19.4s, v22.4s, v17.4s // t6a + sqsub v22.4s, v22.4s, v17.4s // t14a + sqadd v28.4s, v25.4s, v30.4s // t7a + sqsub v25.4s, v25.4s, v30.4s // t15a + +.irp r, v2, v16, v3, v31, v23, v18, v24, v29, v21, v20, v26, v27, v19, v22, v28, v25 + smin_4s \r, \r, v5 +.endr +.irp r, v2, v16, v3, v31, v23, v18, v24, v29, v21, v20, v26, v27, v19, v22, v28, v25 + smax_4s \r, \r, v7 +.endr + + mul_mla v4, v2, v3, v1.s[1], v1.s[0] // -> t8 + mul_mls v6, v2, v3, v1.s[0], v1.s[1] // -> t9 + mul_mla v2, v18, v29, v1.s[3], v1.s[2] // -> t10 + srshr v17.4s, v4.4s, #12 // t8 + srshr v30.4s, v6.4s, #12 // t9 + mul_mls v4, v18, v29, v1.s[2], v1.s[3] // -> t11 + mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t12 + srshr v18.4s, v2.4s, #12 // t10 + srshr v29.4s, v4.4s, #12 // t11 + mul_mla v2, v27, v20, v1.s[0], v1.s[1] // -> t13 + mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t14 + srshr v27.4s, v6.4s, #12 // t12 + srshr v20.4s, v2.4s, #12 // t13 + mul_mla v6, v25, v22, v1.s[2], v1.s[3] // -> t15 + srshr v25.4s, v4.4s, #12 // t14 + srshr v22.4s, v6.4s, #12 // t15 + + sqsub v2.4s, v16.4s, v21.4s // t4 + sqadd v16.4s, v16.4s, v21.4s // t0 + sqsub v3.4s, v31.4s, v26.4s // t5 + sqadd v31.4s, v31.4s, v26.4s // t1 + sqadd v21.4s, v23.4s, v19.4s // t2 + sqsub v23.4s, v23.4s, v19.4s // t6 + sqadd v26.4s, v24.4s, v28.4s // t3 + sqsub v24.4s, v24.4s, v28.4s // t7 + sqadd v19.4s, v17.4s, v27.4s // t8a + sqsub v17.4s, v17.4s, v27.4s // t12a + sqadd v28.4s, v30.4s, v20.4s // t9a + sqsub v30.4s, v30.4s, v20.4s // t13a + sqadd v27.4s, v18.4s, v25.4s // t10a + sqsub v18.4s, v18.4s, v25.4s // t14a + sqadd v20.4s, v29.4s, v22.4s // t11a + sqsub v29.4s, v29.4s, v22.4s // t15a + +.irp r, v2, v16, v3, v31, v21, v23, v26, v24, v19, v17, v28, v30, v27, v18, v20, v29 + smin_4s \r, \r, v5 +.endr +.irp r, v2, v16, v3, v31, v21, v23, v26, v24, v19, v17, v28, v30, v27, v18, v20, v29 + smax_4s \r, \r, v7 +.endr + + mul_mla v4, v2, v3, v0.s[3], v0.s[2] // -> t4a + mul_mls v6, v2, v3, v0.s[2], v0.s[3] // -> t5a + mul_mls v2, v24, v23, v0.s[3], v0.s[2] // -> t6a + srshr v22.4s, v4.4s, #12 // t4a + srshr v25.4s, v6.4s, #12 // t5a + mul_mla v4, v24, v23, v0.s[2], v0.s[3] // -> t7a + mul_mla v6, v17, v30, v0.s[3], v0.s[2] // -> t12 + srshr v24.4s, v2.4s, #12 // t6a + srshr v23.4s, v4.4s, #12 // t7a + mul_mls v2, v17, v30, v0.s[2], v0.s[3] // -> t13 + mul_mls v4, v29, v18, v0.s[3], v0.s[2] // -> t14 + srshr v17.4s, v6.4s, #12 // t12 + mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t15 + srshr v29.4s, v2.4s, #12 // t13 + srshr v30.4s, v4.4s, #12 // t14 + srshr v18.4s, v6.4s, #12 // t15 + + sqsub v2.4s, v16.4s, v21.4s // t2a +.ifc \o0, v16 + sqadd \o0\().4s, v16.4s, v21.4s // out0 + sqsub v21.4s, v31.4s, v26.4s // t3a + sqadd \o15\().4s, v31.4s, v26.4s // out15 +.else + sqadd v4.4s, v16.4s, v21.4s // out0 + sqsub v21.4s, v31.4s, v26.4s // t3a + sqadd \o15\().4s, v31.4s, v26.4s // out15 + mov \o0\().16b, v4.16b +.endif + + sqsub v3.4s, v29.4s, v18.4s // t15a + sqadd \o13\().4s, v29.4s, v18.4s // out13 + sqadd \o2\().4s, v17.4s, v30.4s // out2 + sqsub v26.4s, v17.4s, v30.4s // t14a + + sqadd \o1\().4s, v19.4s, v27.4s // out1 + sqsub v27.4s, v19.4s, v27.4s // t10 + sqadd \o14\().4s, v28.4s, v20.4s // out14 + sqsub v20.4s, v28.4s, v20.4s // t11 + + sqadd \o3\().4s, v22.4s, v24.4s // out3 + sqsub v22.4s, v22.4s, v24.4s // t6 + sqadd \o12\().4s, v25.4s, v23.4s // out12 + sqsub v23.4s, v25.4s, v23.4s // t7 + + // Not clipping the output registers, as they will be downshifted and + // narrowed afterwards anyway. +.irp r, v2, v21, v3, v26, v27, v20, v22, v23 + smin_4s \r, \r, v5 +.endr +.irp r, v2, v21, v3, v26, v27, v20, v22, v23 + smax_4s \r, \r, v7 +.endr + + sqneg \o15\().4s, \o15\().4s // out15 + sqneg \o13\().4s, \o13\().4s // out13 + sqneg \o1\().4s, \o1\().4s // out1 + sqneg \o3\().4s, \o3\().4s // out3 + + mul_mls v24, v2, v21, v0.s[0], v0.s[0] // -> out8 (v24 or v23) + mul_mla v4, v2, v21, v0.s[0], v0.s[0] // -> out7 (v23 or v24) + mul_mla v6, v26, v3, v0.s[0], v0.s[0] // -> out5 (v21 or v26) + + srshr v24.4s, v24.4s, #12 // out8 + srshr v4.4s, v4.4s, #12 // out7 + srshr v5.4s, v6.4s, #12 // out5 + mul_mls v6, v26, v3, v0.s[0], v0.s[0] // -> out10 (v26 or v21) + mul_mla v2, v22, v23, v0.s[0], v0.s[0] // -> out4 (v20 or v27) + srshr v26.4s, v6.4s, #12 // out10 + + mul_mls v6, v22, v23, v0.s[0], v0.s[0] // -> out11 (v27 or v20) + mul_mla v22, v27, v20, v0.s[0], v0.s[0] // -> out6 (v22 or v25) + mul_mls v21, v27, v20, v0.s[0], v0.s[0] // -> out9 (v25 or v22) + + srshr \o4\().4s, v2.4s, #12 // out4 + srshr v6.4s, v6.4s, #12 // out11 + srshr v7.4s, v21.4s, #12 // out9 + srshr \o6\().4s, v22.4s, #12 // out6 + +.ifc \o8, v23 + mov \o8\().16b, v24.16b + mov \o10\().16b, v26.16b +.endif + + sqneg \o7\().4s, v4.4s // out7 + sqneg \o5\().4s, v5.4s // out5 + sqneg \o11\().4s, v6.4s // out11 + sqneg \o9\().4s, v7.4s // out9 +.endm + +function inv_adst_4s_x16_neon + AARCH64_VALID_CALL_TARGET + iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + ret +endfunc + +function inv_flipadst_4s_x16_neon + AARCH64_VALID_CALL_TARGET + iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x16_neon + AARCH64_VALID_CALL_TARGET + movz w16, #2*(5793-4096)*8, lsl #16 + dup v0.2s, w16 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + sqrdmulh v2.4s, v\i\().4s, v0.s[0] + sqadd v\i\().4s, v\i\().4s, v\i\().4s + sqadd v\i\().4s, v\i\().4s, v2.4s +.endr + ret +endfunc + +.macro identity_4x16_shift1 c +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + sqrdmulh v3.4s, \i, \c + srshr v3.4s, v3.4s, #1 + sqadd \i, \i, v3.4s +.endr +.endm + +.macro identity_4x16 c +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + sqrdmulh v3.4s, \i, \c + sqadd \i, \i, \i + sqadd \i, \i, v3.4s +.endr +.endm + +.macro def_horz_16 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x4_neon + mov x14, x30 + movi v7.4s, #0 +.if \scale + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.endif +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr +.if \scale + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + blr x4 + sqrshrn v16.4h, v16.4s, #\shift + sqrshrn v17.4h, v17.4s, #\shift + sqrshrn v18.4h, v18.4s, #\shift + sqrshrn v19.4h, v19.4s, #\shift + sqrshrn2 v16.8h, v20.4s, #\shift + sqrshrn2 v17.8h, v21.4s, #\shift + sqrshrn2 v18.8h, v22.4s, #\shift + sqrshrn2 v19.8h, v23.4s, #\shift + sqrshrn v20.4h, v24.4s, #\shift + sqrshrn v21.4h, v25.4s, #\shift + sqrshrn v22.4h, v26.4s, #\shift + sqrshrn v23.4h, v27.4s, #\shift + sqrshrn2 v20.8h, v28.4s, #\shift + sqrshrn2 v21.8h, v29.4s, #\shift + sqrshrn2 v22.8h, v30.4s, #\shift + sqrshrn2 v23.8h, v31.4s, #\shift + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + transpose_4x8h v20, v21, v22, v23, v4, v5, v6, v7 + +.irp i, v16.8h, v20.8h, v17.8h, v21.8h, v18.8h, v22.8h, v19.8h, v23.8h + st1 {\i}, [x6], #16 +.endr + + ret x14 +endfunc +.endm + +def_horz_16 scale=0, shift=2 +def_horz_16 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_8x16_neon + mov x14, x30 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + blr x5 + load_add_store_8x16 x6, x7 + ret x14 +endfunc + +function inv_txfm_add_16x16_neon + mov x15, x30 + sub sp, sp, #512 + ldrh w12, [x13], #2 +.irp i, 0, 4, 8, 12 + add x6, sp, #(\i*16*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 12 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*4) + mov x8, #16*4 + bl inv_txfm_horz_16x4_neon +.endr + b 3f +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b +3: +.irp i, 0, 8 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #512 + ret x15 +endfunc + +const eob_16x16 + .short 10, 36, 78, 256 +endconst + +const eob_16x16_identity + .short 4, 8, 12, 256 +endconst + +.macro def_fn_16x16 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif + adr x4, inv_\txfm1\()_4s_x16_neon + movrel x5, X(inv_\txfm2\()_8h_x16_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_16x16 +.else + movrel x13, eob_16x16_identity +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_16x16_identity +.else + movrel x13, eob_16x16 +.endif +.endif + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct +def_fn_16x16 identity, identity +def_fn_16x16 dct, adst +def_fn_16x16 dct, flipadst +def_fn_16x16 dct, identity +def_fn_16x16 adst, dct +def_fn_16x16 adst, adst +def_fn_16x16 adst, flipadst +def_fn_16x16 flipadst, dct +def_fn_16x16 flipadst, adst +def_fn_16x16 flipadst, flipadst +def_fn_16x16 identity, dct + +function inv_txfm_add_16x4_neon + mov x15, x30 + movi v4.4s, #0 + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], #16 +.endr + + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + mov x6, x0 + load_add_store_8x4 x6, x7 + + sqrshrn v16.4h, v24.4s, #1 + sqrshrn v17.4h, v25.4s, #1 + sqrshrn v18.4h, v26.4s, #1 + sqrshrn v19.4h, v27.4s, #1 + sqrshrn2 v16.8h, v28.4s, #1 + sqrshrn2 v17.8h, v29.4s, #1 + sqrshrn2 v18.8h, v30.4s, #1 + sqrshrn2 v19.8h, v31.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + add x6, x0, #16 + load_add_store_8x4 x6, x7 + + ret x15 +endfunc + +function inv_txfm_add_4x16_neon + ldrh w12, [x13, #4] + mov x15, x30 + + mov x11, #64 + + cmp w3, w12 + ldrh w12, [x13, #2] + b.lt 1f + + add x6, x2, #48 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + sqrshrn v28.4h, v16.4s, #1 + sqrshrn v29.4h, v17.4s, #1 + sqrshrn v30.4h, v18.4s, #1 + sqrshrn v31.4h, v19.4s, #1 + transpose_4x4h v28, v29, v30, v31, v4, v5, v6, v7 + + b 2f +1: +.irp i, v28.4h, v29.4h, v30.4h, v31.4h + movi \i, #0 +.endr +2: + cmp w3, w12 + ldrh w12, [x13, #0] + b.lt 1f + + add x6, x2, #32 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + sqrshrn v24.4h, v16.4s, #1 + sqrshrn v25.4h, v17.4s, #1 + sqrshrn v26.4h, v18.4s, #1 + sqrshrn v27.4h, v19.4s, #1 + transpose_4x4h v24, v25, v26, v27, v4, v5, v6, v7 + + b 2f +1: +.irp i, v24.4h, v25.4h, v26.4h, v27.4h + movi \i, #0 +.endr +2: + cmp w3, w12 + b.lt 1f + + add x6, x2, #16 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + sqrshrn v20.4h, v16.4s, #1 + sqrshrn v21.4h, v17.4s, #1 + sqrshrn v22.4h, v18.4s, #1 + sqrshrn v23.4h, v19.4s, #1 + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + + b 2f +1: +.irp i, v20.4h, v21.4h, v22.4h, v23.4h + movi \i, #0 +.endr +2: + + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x2] + st1 {v2.4s}, [x2], x11 +.endr + blr x4 + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + + blr x5 + + load_add_store_4x16 x0, x6 + + ret x15 +endfunc + +const eob_4x16 + .short 13, 29, 45, 64 +endconst + +const eob_4x16_identity1 + .short 16, 32, 48, 64 +endconst + +const eob_4x16_identity2 + .short 4, 8, 12, 64 +endconst + +.macro def_fn_416 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif +.if \w == 4 + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_4h_x\h\()_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_4x16 +.else + movrel x13, eob_4x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_4x16_identity2 +.else + movrel x13, eob_4x16 +.endif +.endif +.else + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon) +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct +def_fn_416 \w, \h, identity, identity +def_fn_416 \w, \h, dct, adst +def_fn_416 \w, \h, dct, flipadst +def_fn_416 \w, \h, dct, identity +def_fn_416 \w, \h, adst, dct +def_fn_416 \w, \h, adst, adst +def_fn_416 \w, \h, adst, flipadst +def_fn_416 \w, \h, flipadst, dct +def_fn_416 \w, \h, flipadst, adst +def_fn_416 \w, \h, flipadst, flipadst +def_fn_416 \w, \h, identity, dct +def_fn_416 \w, \h, adst, identity +def_fn_416 \w, \h, flipadst, identity +def_fn_416 \w, \h, identity, adst +def_fn_416 \w, \h, identity, flipadst +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + + +function inv_txfm_add_16x8_neon + mov x15, x30 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + blr x4 + + sqrshrn v8.4h, v16.4s, #1 + sqrshrn v9.4h, v17.4s, #1 + sqrshrn v10.4h, v18.4s, #1 + sqrshrn v11.4h, v19.4s, #1 + sqrshrn2 v8.8h, v20.4s, #1 + sqrshrn2 v9.8h, v21.4s, #1 + sqrshrn2 v10.8h, v22.4s, #1 + sqrshrn2 v11.8h, v23.4s, #1 + sqrshrn v12.4h, v24.4s, #1 + sqrshrn v13.4h, v25.4s, #1 + sqrshrn v14.4h, v26.4s, #1 + sqrshrn v15.4h, v27.4s, #1 + sqrshrn2 v12.8h, v28.4s, #1 + sqrshrn2 v13.8h, v29.4s, #1 + sqrshrn2 v14.8h, v30.4s, #1 + sqrshrn2 v15.8h, v31.4s, #1 + + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + transpose_4x8h v12, v13, v14, v15, v2, v3, v4, v5 + + b 2f +1: +.irp i, v8.8h, v9.8h, v10.8h, v11.8h, v12.8h, v13.8h, v14.8h, v15.8h + movi \i, #0 +.endr +2: + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + + movi v4.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], x11 +.endr + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + + mov v20.16b, v8.16b + mov v21.16b, v9.16b + mov v22.16b, v10.16b + mov v23.16b, v11.16b + + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + sqrshrn v8.4h, v24.4s, #1 + sqrshrn v9.4h, v25.4s, #1 + sqrshrn v10.4h, v26.4s, #1 + sqrshrn v11.4h, v27.4s, #1 + sqrshrn2 v8.8h, v28.4s, #1 + sqrshrn2 v9.8h, v29.4s, #1 + sqrshrn2 v10.8h, v30.4s, #1 + sqrshrn2 v11.8h, v31.4s, #1 + + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + + blr x5 + + mov x6, x0 + load_add_store_8x8 x6, x7 + + mov v16.16b, v8.16b + mov v17.16b, v9.16b + mov v18.16b, v10.16b + mov v19.16b, v11.16b + mov v20.16b, v12.16b + mov v21.16b, v13.16b + mov v22.16b, v14.16b + mov v23.16b, v15.16b + + blr x5 + + add x0, x0, #16 + load_add_store_8x8 x0, x7 + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + ret x15 +endfunc + +function inv_txfm_add_8x16_neon + mov x15, x30 + stp d8, d9, [sp, #-0x20]! + stp d10, d11, [sp, #0x10] + ldrh w12, [x13, #4] + + mov x11, #64 + + cmp w3, w12 + ldrh w12, [x13, #2] + b.lt 1f + + add x6, x2, #48 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v28.4h, v16.4s, #1 + sqrshrn v29.4h, v17.4s, #1 + sqrshrn v30.4h, v18.4s, #1 + sqrshrn v31.4h, v19.4s, #1 + sqrshrn2 v28.8h, v20.4s, #1 + sqrshrn2 v29.8h, v21.4s, #1 + sqrshrn2 v30.8h, v22.4s, #1 + sqrshrn2 v31.8h, v23.4s, #1 + transpose_4x8h v28, v29, v30, v31, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v28.8h, v29.8h, v30.8h, v31.8h + movi \i, #0 +.endr + +2: + cmp w3, w12 + ldrh w12, [x13, #0] + b.lt 1f + + add x6, x2, #32 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v24.4h, v16.4s, #1 + sqrshrn v25.4h, v17.4s, #1 + sqrshrn v26.4h, v18.4s, #1 + sqrshrn v27.4h, v19.4s, #1 + sqrshrn2 v24.8h, v20.4s, #1 + sqrshrn2 v25.8h, v21.4s, #1 + sqrshrn2 v26.8h, v22.4s, #1 + sqrshrn2 v27.8h, v23.4s, #1 + transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + movi \i, #0 +.endr + +2: + cmp w3, w12 + b.lt 1f + + add x6, x2, #16 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v8.4h, v16.4s, #1 + sqrshrn v9.4h, v17.4s, #1 + sqrshrn v10.4h, v18.4s, #1 + sqrshrn v11.4h, v19.4s, #1 + sqrshrn2 v8.8h, v20.4s, #1 + sqrshrn2 v9.8h, v21.4s, #1 + sqrshrn2 v10.8h, v22.4s, #1 + sqrshrn2 v11.8h, v23.4s, #1 + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v8.8h, v9.8h, v10.8h, v11.8h + movi \i, #0 +.endr + +2: + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + mov v20.16b, v8.16b + mov v21.16b, v9.16b + mov v22.16b, v10.16b + mov v23.16b, v11.16b + + blr x5 + + load_add_store_8x16 x0, x6 + + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x20 + + ret x15 +endfunc + +const eob_8x16 + .short 10, 43, 75, 128 +endconst + +const eob_8x16_identity1 + .short 4, 64, 96, 128 +endconst + +const eob_8x16_identity2 + .short 4, 8, 12, 128 +endconst + +.macro def_fn_816 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_8x16 +.else + movrel x13, eob_8x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_8x16_identity2 +.else + movrel x13, eob_8x16 +.endif +.endif +.if \h == 8 + ldrh w13, [x13] +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct +def_fn_816 \w, \h, identity, identity +def_fn_816 \w, \h, dct, adst +def_fn_816 \w, \h, dct, flipadst +def_fn_816 \w, \h, dct, identity +def_fn_816 \w, \h, adst, dct +def_fn_816 \w, \h, adst, adst +def_fn_816 \w, \h, adst, flipadst +def_fn_816 \w, \h, flipadst, dct +def_fn_816 \w, \h, flipadst, adst +def_fn_816 \w, \h, flipadst, flipadst +def_fn_816 \w, \h, identity, dct +def_fn_816 \w, \h, adst, identity +def_fn_816 \w, \h, flipadst, identity +def_fn_816 \w, \h, identity, adst +def_fn_816 \w, \h, identity, flipadst +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_4s_x16_neon + movrel x16, idct_coeffs, 4*16 + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mls v2, v16, v31, v0.s[0], v0.s[1] // -> t16a + mul_mla v4, v16, v31, v0.s[1], v0.s[0] // -> t31a + mul_mls v6, v24, v23, v0.s[2], v0.s[3] // -> t17a + srshr v16.4s, v2.4s, #12 // t16a + srshr v31.4s, v4.4s, #12 // t31a + mul_mla v2, v24, v23, v0.s[3], v0.s[2] // -> t30a + mul_mls v4, v20, v27, v1.s[0], v1.s[1] // -> t18a + srshr v24.4s, v6.4s, #12 // t17a + srshr v23.4s, v2.4s, #12 // t30a + mul_mla v6, v20, v27, v1.s[1], v1.s[0] // -> t29a + mul_mls v2, v28, v19, v1.s[2], v1.s[3] // -> t19a + srshr v20.4s, v4.4s, #12 // t18a + srshr v27.4s, v6.4s, #12 // t29a + mul_mla v4, v28, v19, v1.s[3], v1.s[2] // -> t28a + ld1 {v0.4s, v1.4s}, [x16] + sub x16, x16, #4*24 + mul_mls v6, v18, v29, v0.s[0], v0.s[1] // -> t20a + srshr v28.4s, v2.4s, #12 // t19a + srshr v19.4s, v4.4s, #12 // t28a + mul_mla v2, v18, v29, v0.s[1], v0.s[0] // -> t27a + mul_mls v4, v26, v21, v0.s[2], v0.s[3] // -> t21a + srshr v18.4s, v6.4s, #12 // t20a + srshr v29.4s, v2.4s, #12 // t27a + mul_mla v6, v26, v21, v0.s[3], v0.s[2] // -> t26a + mul_mls v2, v22, v25, v1.s[0], v1.s[1] // -> t22a + srshr v26.4s, v4.4s, #12 // t21a + srshr v21.4s, v6.4s, #12 // t26a + mul_mla v4, v22, v25, v1.s[1], v1.s[0] // -> t25a + mul_mls v6, v30, v17, v1.s[2], v1.s[3] // -> t23a + srshr v22.4s, v2.4s, #12 // t22a + srshr v25.4s, v4.4s, #12 // t25a + mul_mla v2, v30, v17, v1.s[3], v1.s[2] // -> t24a + srshr v30.4s, v6.4s, #12 // t23a + srshr v17.4s, v2.4s, #12 // t24a + + ld1 {v0.4s, v1.4s}, [x16] + + movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + + sqsub v2.4s, v16.4s, v24.4s // t17 + sqadd v16.4s, v16.4s, v24.4s // t16 + sqsub v3.4s, v31.4s, v23.4s // t30 + sqadd v31.4s, v31.4s, v23.4s // t31 + sqsub v24.4s, v28.4s, v20.4s // t18 + sqadd v28.4s, v28.4s, v20.4s // t19 + sqadd v23.4s, v18.4s, v26.4s // t20 + sqsub v18.4s, v18.4s, v26.4s // t21 + sqsub v20.4s, v30.4s, v22.4s // t22 + sqadd v30.4s, v30.4s, v22.4s // t23 + sqadd v26.4s, v17.4s, v25.4s // t24 + sqsub v17.4s, v17.4s, v25.4s // t25 + sqsub v22.4s, v29.4s, v21.4s // t26 + sqadd v29.4s, v29.4s, v21.4s // t27 + sqadd v25.4s, v19.4s, v27.4s // t28 + sqsub v19.4s, v19.4s, v27.4s // t29 + +.irp r, v2, v16, v3, v31, v24, v28, v23, v18, v20, v30, v26, v17, v22, v29, v25, v19 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v2, v16, v3, v31, v24, v28, v23, v18, v20, v30, v26, v17, v22, v29, v25, v19 + smax \r\().4s, \r\().4s, v4.4s +.endr + + mul_mls v7, v3, v2, v1.s[0], v1.s[1] // -> t17a + mul_mla v6, v3, v2, v1.s[1], v1.s[0] // -> t30a + mul_mla v2, v19, v24, v1.s[1], v1.s[0] // -> t18a + srshr v21.4s, v7.4s, #12 // t17a + srshr v27.4s, v6.4s, #12 // t30a + neg v2.4s, v2.4s // -> t18a + mul_mls v7, v19, v24, v1.s[0], v1.s[1] // -> t29a + mul_mls v6, v22, v18, v1.s[2], v1.s[3] // -> t21a + srshr v19.4s, v2.4s, #12 // t18a + srshr v24.4s, v7.4s, #12 // t29a + mul_mla v2, v22, v18, v1.s[3], v1.s[2] // -> t26a + mul_mla v7, v17, v20, v1.s[3], v1.s[2] // -> t22a + srshr v22.4s, v6.4s, #12 // t21a + srshr v18.4s, v2.4s, #12 // t26a + neg v7.4s, v7.4s // -> t22a + mul_mls v6, v17, v20, v1.s[2], v1.s[3] // -> t25a + srshr v17.4s, v7.4s, #12 // t22a + srshr v20.4s, v6.4s, #12 // t25a + + sqsub v2.4s, v27.4s, v24.4s // t29 + sqadd v27.4s, v27.4s, v24.4s // t30 + sqsub v3.4s, v21.4s, v19.4s // t18 + sqadd v21.4s, v21.4s, v19.4s // t17 + sqsub v24.4s, v16.4s, v28.4s // t19a + sqadd v16.4s, v16.4s, v28.4s // t16a + sqsub v19.4s, v30.4s, v23.4s // t20a + sqadd v30.4s, v30.4s, v23.4s // t23a + sqsub v28.4s, v17.4s, v22.4s // t21 + sqadd v17.4s, v17.4s, v22.4s // t22 + sqadd v23.4s, v26.4s, v29.4s // t24a + sqsub v26.4s, v26.4s, v29.4s // t27a + sqadd v22.4s, v20.4s, v18.4s // t25 + sqsub v20.4s, v20.4s, v18.4s // t26 + sqsub v29.4s, v31.4s, v25.4s // t28a + sqadd v31.4s, v31.4s, v25.4s // t31a + +.irp r, v2, v27, v3, v21, v24, v16, v19, v30, v28, v17, v23, v26, v22, v20, v29, v31 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v2, v27, v3, v21, v24, v16, v19, v30, v28, v17, v23, v26, v22, v20, v29, v31 + smax \r\().4s, \r\().4s, v4.4s +.endr + + mul_mls v7, v2, v3, v0.s[2], v0.s[3] // -> t18a + mul_mla v6, v2, v3, v0.s[3], v0.s[2] // -> t29a + mul_mls v2, v29, v24, v0.s[2], v0.s[3] // -> t19 + srshr v18.4s, v7.4s, #12 // t18a + srshr v25.4s, v6.4s, #12 // t29a + mul_mla v7, v29, v24, v0.s[3], v0.s[2] // -> t28 + mul_mla v6, v26, v19, v0.s[3], v0.s[2] // -> t20 + srshr v29.4s, v2.4s, #12 // t19 + srshr v24.4s, v7.4s, #12 // t28 + neg v6.4s, v6.4s // -> t20 + mul_mls v2, v26, v19, v0.s[2], v0.s[3] // -> t27 + mul_mla v7, v20, v28, v0.s[3], v0.s[2] // -> t21a + srshr v26.4s, v6.4s, #12 // t20 + srshr v19.4s, v2.4s, #12 // t27 + neg v7.4s, v7.4s // -> t21a + mul_mls v6, v20, v28, v0.s[2], v0.s[3] // -> t26a + srshr v20.4s, v7.4s, #12 // t21a + srshr v28.4s, v6.4s, #12 // t26a + + sqsub v2.4s, v16.4s, v30.4s // t23 + sqadd v16.4s, v16.4s, v30.4s // t16 = out16 + sqsub v3.4s, v31.4s, v23.4s // t24 + sqadd v31.4s, v31.4s, v23.4s // t31 = out31 + sqsub v23.4s, v21.4s, v17.4s // t22a + sqadd v17.4s, v21.4s, v17.4s // t17a = out17 + sqadd v30.4s, v27.4s, v22.4s // t30a = out30 + sqsub v21.4s, v27.4s, v22.4s // t25a + sqsub v27.4s, v18.4s, v20.4s // t21 + sqadd v18.4s, v18.4s, v20.4s // t18 = out18 + sqadd v7.4s, v29.4s, v26.4s // t19a = out19 + sqsub v26.4s, v29.4s, v26.4s // t20a + sqadd v29.4s, v25.4s, v28.4s // t29 = out29 + sqsub v25.4s, v25.4s, v28.4s // t26 + sqadd v28.4s, v24.4s, v19.4s // t28a = out28 + sqsub v24.4s, v24.4s, v19.4s // t27a + mov v19.16b, v7.16b // out19 + +.irp r, v2, v16, v3, v31, v23, v17, v30, v21, v27, v18, v19, v26, v29, v25, v28, v24 + smin \r\().4s, \r\().4s, v5.4s +.endr +.irp r, v2, v16, v3, v31, v23, v17, v30, v21, v27, v18, v19, v26, v29, v25, v28, v24 + smax \r\().4s, \r\().4s, v4.4s +.endr + + mul_mls v7, v24, v26, v0.s[0], v0.s[0] // -> t20 + mul_mla v6, v24, v26, v0.s[0], v0.s[0] // -> t27 + srshr v20.4s, v7.4s, #12 // t20 + srshr v22.4s, v6.4s, #12 // t27 + + mul_mla v7, v25, v27, v0.s[0], v0.s[0] // -> t26a + mul_mls v6, v25, v27, v0.s[0], v0.s[0] // -> t21a + mov v27.16b, v22.16b // t27 + srshr v26.4s, v7.4s, #12 // t26a + + mul_mls v24, v21, v23, v0.s[0], v0.s[0] // -> t22 + mul_mla v7, v21, v23, v0.s[0], v0.s[0] // -> t25 + srshr v21.4s, v6.4s, #12 // t21a + srshr v22.4s, v24.4s, #12 // t22 + srshr v25.4s, v7.4s, #12 // t25 + + mul_mls v7, v3, v2, v0.s[0], v0.s[0] // -> t23a + mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t24a + srshr v23.4s, v7.4s, #12 // t23a + srshr v24.4s, v6.4s, #12 // t24a + + ret +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x4_neon + mov x14, x30 + movi v7.4s, #0 + lsl x8, x8, #1 +.if \scale + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.endif + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 +.if \scale + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct_4s_x16_neon + + // idct_16 leaves the row_clip_max/min constants in v5 and v4 +.irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + smin_4s \r, \r, v5 +.endr +.irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + smax_4s \r, \r, v4 +.endr + + transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5 + transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5 + transpose_4x4s v24, v25, v26, v27, v2, v3, v4, v5 + transpose_4x4s v28, v29, v30, v31, v2, v3, v4, v5 + +.macro store1 r0, r1, r2, r3 + st1 {\r0}, [x6], #16 + st1 {\r1}, [x6], #16 + st1 {\r2}, [x6], #16 + st1 {\r3}, [x6], #16 +.endm + store1 v16.4s, v20.4s, v24.4s, v28.4s + store1 v17.4s, v21.4s, v25.4s, v29.4s + store1 v18.4s, v22.4s, v26.4s, v30.4s + store1 v19.4s, v23.4s, v27.4s, v31.4s +.purgem store1 + sub x6, x6, #64*4 + + movi v7.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in v0.s[1] + scale_input .4s, v0.s[1], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[1], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct32_odd_4s_x16_neon + transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5 + transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5 + transpose_4x4s v23, v22, v21, v20, v2, v3, v4, v5 + transpose_4x4s v19, v18, v17, v16, v2, v3, v4, v5 +.macro store2 r0, r1, r2, r3, shift + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x6] + sqsub v4.4s, v0.4s, \r0 + sqadd v0.4s, v0.4s, \r0 + sqsub v5.4s, v1.4s, \r1 + sqadd v1.4s, v1.4s, \r1 + sqsub v6.4s, v2.4s, \r2 + sqadd v2.4s, v2.4s, \r2 + sqsub v7.4s, v3.4s, \r3 + sqadd v3.4s, v3.4s, \r3 + sqrshrn v0.4h, v0.4s, #\shift + sqrshrn2 v0.8h, v1.4s, #\shift + sqrshrn v1.4h, v2.4s, #\shift + sqrshrn2 v1.8h, v3.4s, #\shift + sqrshrn v2.4h, v7.4s, #\shift + sqrshrn2 v2.8h, v6.4s, #\shift + sqrshrn v3.4h, v5.4s, #\shift + sqrshrn2 v3.8h, v4.4s, #\shift + st1 {v0.8h, v1.8h}, [x6], #32 + rev64 v2.8h, v2.8h + rev64 v3.8h, v3.8h + st1 {v2.8h, v3.8h}, [x6], #32 +.endm + + store2 v31.4s, v27.4s, v23.4s, v19.4s, \shift + store2 v30.4s, v26.4s, v22.4s, v18.4s, \shift + store2 v29.4s, v25.4s, v21.4s, v17.4s, \shift + store2 v28.4s, v24.4s, v20.4s, v16.4s, \shift +.purgem store2 + ret x14 +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_8x32_neon + mov x14, x30 + lsl x8, x8, #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + + bl X(inv_dct_8h_x16_neon) + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + st1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + sub x7, x7, x8, lsr #1 + bl X(inv_dct32_odd_8h_x16_neon) + + neg x9, x8 + mov x10, x6 + mvni v1.8h, #0xfc, lsl #8 // 0x3ff +.macro combine r0, r1, r2, r3, op, stride + ld1 {v5.8h}, [x7], \stride + ld1 {v2.8h}, [x10], x1 + ld1 {v6.8h}, [x7], \stride + ld1 {v3.8h}, [x10], x1 + \op v5.8h, v5.8h, \r0 + ld1 {v7.8h}, [x7], \stride + ld1 {v4.8h}, [x10], x1 + srshr v5.8h, v5.8h, #4 + \op v6.8h, v6.8h, \r1 + usqadd v2.8h, v5.8h + srshr v6.8h, v6.8h, #4 + \op v7.8h, v7.8h, \r2 + ld1 {v5.8h}, [x7], \stride + usqadd v3.8h, v6.8h + smin v2.8h, v2.8h, v1.8h + srshr v7.8h, v7.8h, #4 + \op v5.8h, v5.8h, \r3 + st1 {v2.8h}, [x6], x1 + ld1 {v2.8h}, [x10], x1 + usqadd v4.8h, v7.8h + smin v3.8h, v3.8h, v1.8h + srshr v5.8h, v5.8h, #4 + st1 {v3.8h}, [x6], x1 + usqadd v2.8h, v5.8h + smin v4.8h, v4.8h, v1.8h + st1 {v4.8h}, [x6], x1 + smin v2.8h, v2.8h, v1.8h + st1 {v2.8h}, [x6], x1 +.endm + combine v31.8h, v30.8h, v29.8h, v28.8h, sqadd, x8 + combine v27.8h, v26.8h, v25.8h, v24.8h, sqadd, x8 + combine v23.8h, v22.8h, v21.8h, v20.8h, sqadd, x8 + combine v19.8h, v18.8h, v17.8h, v16.8h, sqadd, x8 + sub x7, x7, x8 + combine v16.8h, v17.8h, v18.8h, v19.8h, sqsub, x9 + combine v20.8h, v21.8h, v22.8h, v23.8h, sqsub, x9 + combine v24.8h, v25.8h, v26.8h, v27.8h, sqsub, x9 + combine v28.8h, v29.8h, v30.8h, v31.8h, sqsub, x9 +.purgem combine + + ret x14 +endfunc + +const eob_32x32 + .short 10, 36, 78, 136, 210, 300, 406, 1024 +endconst + +const eob_16x32 + .short 10, 36, 78, 151, 215, 279, 343, 512 +endconst + +const eob_16x32_shortside + .short 10, 36, 78, 512 +endconst + +const eob_8x32 + .short 10, 43, 75, 107, 139, 171, 203, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_16bpc_neon, export=1 + movi v0.8h, #0 + movi v1.8h, #0 + movrel x13, eob_32x32, 2 + + mov x8, #4*32 +1: + mov w9, #0 + movrel x12, eob_32x32, 2 +2: + add w9, w9, #8 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + load_add_store_8x8 x0, x7, shiftbits=2 + ldrh w11, [x12], #4 + sub x0, x0, x1, lsl #3 + add x0, x0, #2*8 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #4 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw #1 + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #4*8 + b 1b +9: + ret +endfunc + +.macro shift_16_regs op, shift +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + movz w16, #2896*8, lsl #16 + movz w17, #2*(5793-4096)*8, lsl #16 + movi v0.4s, #0 + movi v1.4s, #0 + movrel x13, eob_16x32\hshort, 2 + + mov x8, #4*\h +1: + mov w9, #0 + movrel x12, eob_16x32\wshort, 2 +2: + add w9, w9, #8 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + dup v2.2s, w16 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + mov v2.s[1], w17 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + scale_input .4s, v2.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v2.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + +.if \w == 16 + // 16x32 + identity_4x16_shift1 v2.s[1] +.else + // 32x16 + shift_16_regs sqshl, 1 + identity_4x16 v2.s[1] +.endif + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + +.if \w == 16 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=4 +.endif + ldrh w11, [x12], #4 + sub x0, x0, x1, lsl #3 + add x0, x0, #16 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #4 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw #1 + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #4*8 + b 1b +9: + ret +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + movi v0.4s, #0 + movi v1.4s, #0 + // Working on 8x8 blocks, read every other entry from eob_8x32 + movrel x13, eob_8x32, 2 + + mov w8, #4*\h +1: + // Working on 8x8 blocks, read every other entry from eob_8x32 + ldrh w12, [x13], #4 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + +.if \w == 8 + sqrshrn v16.4h, v16.4s, #1 + sqrshrn2 v16.8h, v17.4s, #1 + sqrshrn v17.4h, v18.4s, #1 + sqrshrn2 v17.8h, v19.4s, #1 + sqrshrn v18.4h, v20.4s, #1 + sqrshrn2 v18.8h, v21.4s, #1 + sqrshrn v19.4h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + sqrshrn v20.4h, v24.4s, #1 + sqrshrn2 v20.8h, v25.4s, #1 + sqrshrn v21.4h, v26.4s, #1 + sqrshrn2 v21.8h, v27.4s, #1 + sqrshrn v22.4h, v28.4s, #1 + sqrshrn2 v22.8h, v29.4s, #1 + sqrshrn v23.4h, v30.4s, #1 + sqrshrn2 v23.8h, v31.4s, #1 +.else + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + + cmp w3, w12 +.if \w == 8 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=3 +.endif + + b.lt 9f +.if \w == 8 + sub x2, x2, x8, lsl #3 + add x2, x2, #4*8 +.else + sub x0, x0, x1, lsl #3 + add x0, x0, #2*8 +.endif + b 1b + +9: + ret +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_16bpc_neon, export=1 + idct_dc 32, 32, 2 + + mov x15, x30 + sub sp, sp, #2048 + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, sp, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #2048 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_16x32_16bpc_neon, export=1 + idct_dc 16, 32, 1 + + mov x15, x30 + sub sp, sp, #1024 + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + adr x4, inv_dct_4s_x16_neon + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, sp, #(\i*16*2) + add x7, x2, #(\i*4) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endif + mov x8, #4*32 + bl inv_txfm_horz_scale_16x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #16*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #1024 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x16_16bpc_neon, export=1 + idct_dc 32, 16, 1 + + mov x15, x30 + sub sp, sp, #1024 + + movrel x13, eob_16x32 + movrel x5, X(inv_dct_8h_x16_neon) + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12 + add x6, sp, #(\i*32*2) + add x7, x2, #(\i*4) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 12 + ldrh w12, [x13], #2 +.endif +.endif + mov x8, #4*16 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #1024 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_8x32_16bpc_neon, export=1 + idct_dc 8, 32, 2 + + mov x15, x30 + sub sp, sp, #512 + + movrel x13, eob_8x32 + + movi v28.4s, #0 + mov x8, #4*32 + mov w9, #32 + mov x6, sp + mov x7, x2 +1: +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().4s}, [x7] + st1 {v28.4s}, [x7], x8 +.endr + ldrh w12, [x13], #2 + sub w9, w9, #4 + sub x7, x7, x8, lsl #3 + add x7, x7, #4*4 + + bl inv_dct_4s_x8_neon + + sqrshrn v16.4h, v16.4s, #2 + sqrshrn v17.4h, v17.4s, #2 + sqrshrn v18.4h, v18.4s, #2 + sqrshrn v19.4h, v19.4s, #2 + sqrshrn2 v16.8h, v20.4s, #2 + sqrshrn2 v17.8h, v21.4s, #2 + sqrshrn2 v18.8h, v22.4s, #2 + sqrshrn2 v19.8h, v23.4s, #2 + + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + cmp w3, w12 + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64 + + b.ge 1b + cbz w9, 3f + + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 +2: + subs w9, w9, #4 + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x6], #64 + b.gt 2b + +3: + mov x6, x0 + mov x7, sp + mov x8, #8*2 + bl inv_txfm_add_vert_dct_8x32_neon + + add sp, sp, #512 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x8_16bpc_neon, export=1 + idct_dc 32, 8, 2 + + mov x15, x30 + sub sp, sp, #512 + +.irp i, 0, 4 + add x6, sp, #(\i*32*2) + add x7, x2, #(\i*4) +.if \i > 0 + cmp w3, #10 + b.lt 1f +.endif + mov x8, #8*4 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 2f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + +2: + mov x8, #2*32 + mov w9, #0 +1: + add x6, x0, x9, lsl #1 + add x7, sp, x9, lsl #1 // #(\i*2) + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().8h}, [x7], x8 +.endr + add w9, w9, #8 + + bl X(inv_dct_8h_x8_neon) + + cmp w9, #32 + + load_add_store_8x8 x6, x7 + + b.lt 1b + + add sp, sp, #512 + ret x15 +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + ld1 {v0.4s, v1.4s}, [x17], #32 + + sqrdmulh v23.4s, v16.4s, v0.s[1] // t63a + sqrdmulh v16.4s, v16.4s, v0.s[0] // t32a + sqrdmulh v22.4s, v17.4s, v0.s[2] // t62a + sqrdmulh v17.4s, v17.4s, v0.s[3] // t33a + sqrdmulh v21.4s, v18.4s, v1.s[1] // t61a + sqrdmulh v18.4s, v18.4s, v1.s[0] // t34a + sqrdmulh v20.4s, v19.4s, v1.s[2] // t60a + sqrdmulh v19.4s, v19.4s, v1.s[3] // t35a + + ld1 {v0.4s}, [x17], #16 + + sqadd v24.4s, v16.4s, v17.4s // t32 + sqsub v25.4s, v16.4s, v17.4s // t33 + sqsub v26.4s, v19.4s, v18.4s // t34 + sqadd v27.4s, v19.4s, v18.4s // t35 + sqadd v28.4s, v20.4s, v21.4s // t60 + sqsub v29.4s, v20.4s, v21.4s // t61 + sqsub v30.4s, v23.4s, v22.4s // t62 + sqadd v31.4s, v23.4s, v22.4s // t63 + +.irp r, v24, v25, v26, v27, v28, v29, v30, v31 + smin_4s \r, \r, v5 +.endr +.irp r, v24, v25, v26, v27, v28, v29, v30, v31 + smax_4s \r, \r, v4 +.endr + + mul_mla v2, v29, v26, v0.s[0], v0.s[1] // -> t34a + mul_mls v7, v29, v26, v0.s[1], v0.s[0] // -> t61a + neg v2.4s, v2.4s // t34a + mul_mls v6, v30, v25, v0.s[1], v0.s[0] // -> t33a + srshr v26.4s, v2.4s, #12 // t34a + mul_mla v2, v30, v25, v0.s[0], v0.s[1] // -> t62a + srshr v29.4s, v7.4s, #12 // t61a + srshr v25.4s, v6.4s, #12 // t33a + srshr v30.4s, v2.4s, #12 // t62a + + sqadd v16.4s, v24.4s, v27.4s // t32a + sqsub v19.4s, v24.4s, v27.4s // t35a + sqadd v17.4s, v25.4s, v26.4s // t33 + sqsub v18.4s, v25.4s, v26.4s // t34 + sqsub v20.4s, v31.4s, v28.4s // t60a + sqadd v23.4s, v31.4s, v28.4s // t63a + sqsub v21.4s, v30.4s, v29.4s // t61 + sqadd v22.4s, v30.4s, v29.4s // t62 + +.irp r, v16, v19, v17, v18, v20, v23, v21, v22 + smin_4s \r, \r, v5 +.endr +.irp r, v16, v19, v17, v18, v20, v23, v21, v22 + smax_4s \r, \r, v4 +.endr + + mul_mla v2, v21, v18, v0.s[2], v0.s[3] // -> t61a + mul_mls v7, v21, v18, v0.s[3], v0.s[2] // -> t34a + mul_mla v6, v20, v19, v0.s[2], v0.s[3] // -> t60 + srshr v21.4s, v2.4s, #12 // t61a + srshr v18.4s, v7.4s, #12 // t34a + mul_mls v2, v20, v19, v0.s[3], v0.s[2] // -> t35 + srshr v20.4s, v6.4s, #12 // t60 + srshr v19.4s, v2.4s, #12 // t35 + + st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], #64 + st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x6], #64 + + ret +endfunc + +function inv_dct64_step2_neon + movrel x16, idct_coeffs + ld1 {v0.4s}, [x16] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + ldr q16, [x6, #4*4*0] // t32a + ldr q17, [x9, #4*4*8] // t39a + ldr q18, [x9, #4*4*0] // t63a + ldr q19, [x6, #4*4*8] // t56a + ldr q20, [x6, #4*4*16] // t40a + ldr q21, [x9, #4*4*24] // t47a + ldr q22, [x9, #4*4*16] // t55a + ldr q23, [x6, #4*4*24] // t48a + + sqadd v24.4s, v16.4s, v17.4s // t32 + sqsub v25.4s, v16.4s, v17.4s // t39 + sqadd v26.4s, v18.4s, v19.4s // t63 + sqsub v27.4s, v18.4s, v19.4s // t56 + sqsub v28.4s, v21.4s, v20.4s // t40 + sqadd v29.4s, v21.4s, v20.4s // t47 + sqadd v30.4s, v23.4s, v22.4s // t48 + sqsub v31.4s, v23.4s, v22.4s // t55 + +.irp r, v24, v25, v26, v27, v28, v29, v30, v31 + smin_4s \r, \r, v5 +.endr +.irp r, v24, v25, v26, v27, v28, v29, v30, v31 + smax_4s \r, \r, v4 +.endr + + mul_mla v2, v27, v25, v0.s[3], v0.s[2] // -> t56a + mul_mls v7, v27, v25, v0.s[2], v0.s[3] // -> t39a + mul_mla v6, v31, v28, v0.s[3], v0.s[2] // -> t40a + srshr v25.4s, v2.4s, #12 // t56a + srshr v27.4s, v7.4s, #12 // t39a + neg v6.4s, v6.4s // t40a + mul_mls v2, v31, v28, v0.s[2], v0.s[3] // -> t55a + srshr v31.4s, v6.4s, #12 // t40a + srshr v28.4s, v2.4s, #12 // t55a + + sqadd v16.4s, v24.4s, v29.4s // t32a + sqsub v19.4s, v24.4s, v29.4s // t47a + sqadd v17.4s, v27.4s, v31.4s // t39 + sqsub v18.4s, v27.4s, v31.4s // t40 + sqsub v20.4s, v26.4s, v30.4s // t48a + sqadd v23.4s, v26.4s, v30.4s // t63a + sqsub v21.4s, v25.4s, v28.4s // t55 + sqadd v22.4s, v25.4s, v28.4s // t56 + +.irp r, v16, v19, v17, v18, v20, v23, v21, v22 + smin_4s \r, \r, v5 +.endr +.irp r, v16, v19, v17, v18, v20, v23, v21, v22 + smax_4s \r, \r, v4 +.endr + + mul_mls v2, v21, v18, v0.s[0], v0.s[0] // -> t40a + mul_mla v7, v21, v18, v0.s[0], v0.s[0] // -> t55a + mul_mls v6, v20, v19, v0.s[0], v0.s[0] // -> t47 + srshr v18.4s, v2.4s, #12 // t40a + srshr v21.4s, v7.4s, #12 // t55a + mul_mla v2, v20, v19, v0.s[0], v0.s[0] // -> t48 + srshr v19.4s, v6.4s, #12 // t47 + srshr v20.4s, v2.4s, #12 // t48 + + str q16, [x6, #4*4*0] // t32a + str q17, [x9, #4*4*0] // t39 + str q18, [x6, #4*4*8] // t40a + str q19, [x9, #4*4*8] // t47 + str q20, [x6, #4*4*16] // t48 + str q21, [x9, #4*4*16] // t55a + str q22, [x6, #4*4*24] // t56 + str q23, [x9, #4*4*24] // t63a + + add x6, x6, #4*4 + sub x9, x9, #4*4 + cmp x6, x9 + b.lt 1b + ret +endfunc + +.macro load8 src, strd, zero, clear +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s +.if \clear + ld1 {\i}, [\src] + st1 {\zero}, [\src], \strd +.else + ld1 {\i}, [\src], \strd +.endif +.endr +.endm + +.macro store16 dst +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + st1 {\i}, [\dst], #16 +.endr +.endm + +.macro clear_upper8 +.irp i, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + movi \i, #0 +.endr +.endm + +.macro movi_if reg, val, cond +.if \cond + movi \reg, \val +.endif +.endm + +.macro movz16dup_if reg, gpr, val, cond +.if \cond + movz \gpr, \val, lsl #16 + dup \reg, \gpr +.endif +.endm + +.macro st1_if regs, dst, cond +.if \cond + st1 \regs, \dst +.endif +.endm + +.macro str_if reg, dst, cond +.if \cond + str \reg, \dst +.endif +.endm + +.macro stroff_if reg, dst, dstoff, cond +.if \cond + str \reg, \dst, \dstoff +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input .4s, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_4s_x64_neon + mov x14, x30 + mov x6, sp + lsl x8, x8, #2 + + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + load8 x7, x8, v7.4s, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + add x7, x7, x8, lsr #1 + scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct_4s_x16_neon + + // idct_16 leaves the row_clip_max/min constants in v5 and v4 +.irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + smin_4s \r, \r, v5 +.endr +.irp r, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + smax_4s \r, \r, v4 +.endr + + store16 x6 + + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + load8 x7, x8, v7.4s, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + lsr x8, x8, #1 + sub x7, x7, x8, lsr #1 + scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct32_odd_4s_x16_neon + + add x10, x6, #16*15 + sub x6, x6, #16*16 + + mov x9, #-16 + + movi v1.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + mvni v0.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + +.macro store_addsub r0, r1, r2, r3 + ld1 {v2.4s}, [x6], #16 + ld1 {v3.4s}, [x6], #16 + sqadd v6.4s, v2.4s, \r0 + sqsub \r0, v2.4s, \r0 + ld1 {v4.4s}, [x6], #16 + sqadd v7.4s, v3.4s, \r1 + sqsub \r1, v3.4s, \r1 + smin v6.4s, v6.4s, v1.4s + smin \r0, \r0, v1.4s + ld1 {v5.4s}, [x6], #16 + sqadd v2.4s, v4.4s, \r2 + sub x6, x6, #16*4 + smax v6.4s, v6.4s, v0.4s + smax \r0, \r0, v0.4s + sqsub \r2, v4.4s, \r2 + smin v7.4s, v7.4s, v1.4s + smin \r1, \r1, v1.4s + st1 {v6.4s}, [x6], #16 + st1 {\r0}, [x10], x9 + smin v2.4s, v2.4s, v1.4s + smin \r2, \r2, v1.4s + smax v7.4s, v7.4s, v0.4s + smax \r1, \r1, v0.4s + sqadd v3.4s, v5.4s, \r3 + sqsub \r3, v5.4s, \r3 + smax v2.4s, v2.4s, v0.4s + smax \r2, \r2, v0.4s + smin v3.4s, v3.4s, v1.4s + smin \r3, \r3, v1.4s + st1 {v7.4s}, [x6], #16 + st1 {\r1}, [x10], x9 + smax v3.4s, v3.4s, v0.4s + smax \r3, \r3, v0.4s + st1 {v2.4s}, [x6], #16 + st1 {\r2}, [x10], x9 + st1 {v3.4s}, [x6], #16 + st1 {\r3}, [x10], x9 +.endm + store_addsub v31.4s, v30.4s, v29.4s, v28.4s + store_addsub v27.4s, v26.4s, v25.4s, v24.4s + store_addsub v23.4s, v22.4s, v21.4s, v20.4s + store_addsub v19.4s, v18.4s, v17.4s, v16.4s +.purgem store_addsub + + add x6, x6, #4*4*16 + + movrel x17, idct64_coeffs + movi v5.4s, #1, msl #16 // row_clip_max = ~(~bdmax << 7), 0x1ffff + mvni v4.4s, #1, msl #16 // row_clip_min = (~bdmax << 7), 0xfffe0000 + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + add x9, x7, x8, lsl #4 // offset 16 + add x10, x7, x8, lsl #3 // offset 8 + sub x9, x9, x8 // offset 15 + sub x11, x10, x8 // offset 7 + ld1 {v16.4s}, [x7] // in1 (offset 0) + ld1 {v17.4s}, [x9] // in31 (offset 15) + ld1 {v18.4s}, [x10] // in17 (offset 8) + ld1 {v19.4s}, [x11] // in15 (offset 7) + st1_if {v7.4s}, [x7], \clear + st1_if {v7.4s}, [x9], \clear + st1_if {v7.4s}, [x10], \clear + st1_if {v7.4s}, [x11], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + add x7, x7, x8, lsl #2 // offset 4 + sub x9, x9, x8, lsl #2 // offset 11 + sub x10, x7, x8 // offset 3 + add x11, x9, x8 // offset 12 + ld1 {v16.4s}, [x10] // in7 (offset 3) + ld1 {v17.4s}, [x11] // in25 (offset 12) + ld1 {v18.4s}, [x9] // in23 (offset 11) + ld1 {v19.4s}, [x7] // in9 (offset 4) + st1_if {v7.4s}, [x7], \clear + st1_if {v7.4s}, [x9], \clear + st1_if {v7.4s}, [x10], \clear + st1_if {v7.4s}, [x11], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + sub x10, x10, x8, lsl #1 // offset 1 + sub x9, x9, x8, lsl #1 // offset 9 + add x7, x7, x8 // offset 5 + add x11, x11, x8 // offset 13 + ldr q16, [x10, x8] // in5 (offset 2) + ldr q17, [x11] // in27 (offset 13) + ldr q18, [x9, x8] // in21 (offset 10) + ldr q19, [x7] // in11 (offset 5) + stroff_if q7, [x10, x8], \clear + str_if q7, [x11], \clear + stroff_if q7, [x9, x8], \clear + str_if q7, [x7], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + ldr q16, [x10] // in3 (offset 1) + ldr q17, [x11, x8] // in29 (offset 14) + ldr q18, [x9] // in19 (offset 9) + ldr q19, [x7, x8] // in13 (offset 6) + str_if q7, [x10], \clear + stroff_if q7, [x11, x8], \clear + str_if q7, [x9], \clear + stroff_if q7, [x7, x8], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + + sub x6, x6, #4*4*32 + add x9, x6, #4*4*7 + + bl inv_dct64_step2_neon + + ret x14 +endfunc +.endm + +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + + +function inv_txfm_horz_dct_64x4_neon + mov x14, x30 + + mov x7, sp + add x8, sp, #4*4*(64 - 4) + add x9, x6, #2*56 + mov x10, #2*64 + mov x11, #-4*4*4 + + dup v7.4s, w12 +1: + ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x7], #64 + ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x8], x11 + ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x7], #64 + ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x8], x11 + transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5 + transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5 + transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5 + transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5 + +.macro store_addsub src0, src1, src2, src3 + sqsub v1.4s, \src0, \src1 + sqadd v0.4s, \src0, \src1 + sqsub v3.4s, \src2, \src3 + srshl v1.4s, v1.4s, v7.4s + sqadd v2.4s, \src2, \src3 + srshl v3.4s, v3.4s, v7.4s + srshl v0.4s, v0.4s, v7.4s + srshl v2.4s, v2.4s, v7.4s + sqxtn v3.4h, v3.4s + sqxtn2 v3.8h, v1.4s + sqxtn v0.4h, v0.4s + sqxtn2 v0.8h, v2.4s + rev64 v3.8h, v3.8h + st1 {v0.8h}, [x6], x10 + st1 {v3.8h}, [x9], x10 +.endm + store_addsub v16.4s, v31.4s, v20.4s, v27.4s + store_addsub v17.4s, v30.4s, v21.4s, v26.4s + store_addsub v18.4s, v29.4s, v22.4s, v25.4s + store_addsub v19.4s, v28.4s, v23.4s, v24.4s +.purgem store_addsub + sub x6, x6, x10, lsl #2 + sub x9, x9, x10, lsl #2 + add x6, x6, #16 + sub x9, x9, #16 + + cmp x7, x8 + b.lt 1b + ret x14 +endfunc + +function inv_txfm_add_vert_dct_8x64_neon + mov x14, x30 + lsl x8, x8, #1 + + mov x7, sp + add x8, sp, #2*8*(64 - 4) + add x9, x6, x1, lsl #6 + sub x9, x9, x1 + neg x10, x1 + mov x11, #-2*8*4 + +1: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64 + ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11 + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11 + + mvni v7.8h, #0xfc, lsl #8 // 0x3ff +.macro add_dest_addsub src0, src1, src2, src3 + ld1 {v0.8h}, [x6], x1 + ld1 {v1.8h}, [x9], x10 + sqadd v4.8h, \src0, \src1 + ld1 {v2.8h}, [x6] + sqsub \src0, \src0, \src1 + ld1 {v3.8h}, [x9] + sqadd v5.8h, \src2, \src3 + sqsub \src2, \src2, \src3 + sub x6, x6, x1 + sub x9, x9, x10 + srshr v4.8h, v4.8h, #4 + srshr v5.8h, v5.8h, #4 + srshr \src0, \src0, #4 + usqadd v0.8h, v4.8h + srshr \src2, \src2, #4 + usqadd v1.8h, \src0 + usqadd v2.8h, v5.8h + smin v0.8h, v0.8h, v7.8h + usqadd v3.8h, \src2 + smin v1.8h, v1.8h, v7.8h + st1 {v0.8h}, [x6], x1 + smin v2.8h, v2.8h, v7.8h + st1 {v1.8h}, [x9], x10 + smin v3.8h, v3.8h, v7.8h + st1 {v2.8h}, [x6], x1 + st1 {v3.8h}, [x9], x10 +.endm + add_dest_addsub v16.8h, v31.8h, v17.8h, v30.8h + add_dest_addsub v18.8h, v29.8h, v19.8h, v28.8h + add_dest_addsub v20.8h, v27.8h, v21.8h, v26.8h + add_dest_addsub v22.8h, v25.8h, v23.8h, v24.8h +.purgem add_dest_addsub + cmp x7, x8 + b.lt 1b + + ret x14 +endfunc + +function inv_txfm_add_dct_dct_64x64_16bpc_neon, export=1 + idct_dc 64, 64, 2 + + mov x15, x30 + + sub_sp 64*32*2+64*4*4 + add x5, sp, #64*4*4 + + movrel x13, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_4s_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x7, x5, #(\i*2) + mov x8, #64*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #64*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_64x32_16bpc_neon, export=1 + idct_dc 64, 32, 1 + + mov x15, x30 + + sub_sp 64*32*2+64*4*4 + add x5, sp, #64*4*4 + + movrel x13, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + mov x12, #-1 // shift + bl inv_txfm_dct_clear_scale_4s_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i*2) + add x7, x5, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, x5, #64*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_32x64_16bpc_neon, export=1 + idct_dc 32, 64, 1 + + mov x15, x30 + + sub_sp 32*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f + ldrh w12, [x13], #2 +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x7, x5, #(\i*2) + mov x8, #32*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #32*32*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_64x16_16bpc_neon, export=1 + idct_dc 64, 16, 2 + + mov x15, x30 + + sub_sp 64*16*2+64*4*4 + add x4, sp, #64*4*4 + + movrel x13, eob_16x32 + +.irp i, 0, 4, 8, 12 + add x6, x4, #(\i*64*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #16*4 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_4s_x64_neon + add x6, x4, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 12 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: + movrel x5, X(inv_dct_8h_x16_neon) +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i*2) + add x7, x4, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, x4, #64*16*2 + ret x15 +endfunc + +function inv_txfm_add_dct_dct_16x64_16bpc_neon, export=1 + idct_dc 16, 64, 2 + + mov x15, x30 + + sub_sp 16*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + + adr x4, inv_dct_4s_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*16*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_16x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x7, x5, #(\i*2) + mov x8, #16*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #16*32*2 + ret x15 +endfunc diff --git a/third_party/dav1d/src/arm/64/loopfilter.S b/third_party/dav1d/src/arm/64/loopfilter.S new file mode 100644 index 0000000000..63d5de10ad --- /dev/null +++ b/third_party/dav1d/src/arm/64/loopfilter.S @@ -0,0 +1,1129 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// depending on how many pixels need to be stored, returns: +// x14 = (1 << 0) : 0 pixels +// x14 = (1 << 4) : inner 4 pixels +// x14 = (1 << 6) : inner 6 pixels +// x14 = 0 : all pixels +.macro loop_filter wd +function lpf_16_wd\wd\()_neon + uabd v0.16b, v22.16b, v23.16b // abs(p1 - p0) + uabd v1.16b, v25.16b, v24.16b // abs(q1 - q0) + uabd v2.16b, v23.16b, v24.16b // abs(p0 - q0) + uabd v3.16b, v22.16b, v25.16b // abs(p1 - q1) +.if \wd >= 6 + uabd v4.16b, v21.16b, v22.16b // abs(p2 - p1) + uabd v5.16b, v26.16b, v25.16b // abs(q2 - q1) +.endif +.if \wd >= 8 + uabd v6.16b, v20.16b, v21.16b // abs(p3 - p2) + uabd v7.16b, v27.16b, v26.16b // abs(q3 - q3) +.endif +.if \wd >= 6 + umax v4.16b, v4.16b, v5.16b +.endif + uqadd v2.16b, v2.16b, v2.16b // abs(p0 - q0) * 2 +.if \wd >= 8 + umax v6.16b, v6.16b, v7.16b +.endif + ushr v3.16b, v3.16b, #1 +.if \wd >= 8 + umax v4.16b, v4.16b, v6.16b +.endif +.if \wd >= 6 + and v4.16b, v4.16b, v14.16b +.endif + umax v0.16b, v0.16b, v1.16b // max(abs(p1 - p0), abs(q1 - q0)) + uqadd v2.16b, v2.16b, v3.16b // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 +.if \wd >= 6 + umax v4.16b, v0.16b, v4.16b + cmhs v1.16b, v11.16b, v4.16b // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I +.else + cmhs v1.16b, v11.16b, v0.16b // max(abs(p1 - p0), abs(q1 - q0)) <= I +.endif + cmhs v2.16b, v10.16b, v2.16b // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E + and v1.16b, v1.16b, v2.16b // fm + and v1.16b, v1.16b, v13.16b // fm && wd >= 4 +.if \wd >= 6 + and v14.16b, v14.16b, v1.16b // fm && wd > 4 +.endif +.if \wd >= 16 + and v15.16b, v15.16b, v1.16b // fm && wd == 16 +.endif + + mov x16, v1.d[0] + mov x17, v1.d[1] + adds x16, x16, x17 + b.ne 9f // if (!fm || wd < 4) return; + mov x14, #(1 << 0) + ret +9: +.if \wd >= 6 + movi v10.16b, #1 + uabd v2.16b, v21.16b, v23.16b // abs(p2 - p0) + uabd v3.16b, v22.16b, v23.16b // abs(p1 - p0) + uabd v4.16b, v25.16b, v24.16b // abs(q1 - q0) + uabd v5.16b, v26.16b, v24.16b // abs(q2 - q0) +.if \wd >= 8 + uabd v6.16b, v20.16b, v23.16b // abs(p3 - p0) + uabd v7.16b, v27.16b, v24.16b // abs(q3 - q0) +.endif + umax v2.16b, v2.16b, v3.16b + umax v4.16b, v4.16b, v5.16b +.if \wd >= 8 + umax v6.16b, v6.16b, v7.16b +.endif + umax v2.16b, v2.16b, v4.16b +.if \wd >= 8 + umax v2.16b, v2.16b, v6.16b +.endif + +.if \wd == 16 + uabd v3.16b, v17.16b, v23.16b // abs(p6 - p0) + uabd v4.16b, v18.16b, v23.16b // abs(p5 - p0) + uabd v5.16b, v19.16b, v23.16b // abs(p4 - p0) +.endif + cmhs v2.16b, v10.16b, v2.16b // flat8in +.if \wd == 16 + uabd v6.16b, v28.16b, v24.16b // abs(q4 - q0) + uabd v7.16b, v29.16b, v24.16b // abs(q5 - q0) + uabd v8.16b, v30.16b, v24.16b // abs(q6 - q0) +.endif + and v14.16b, v2.16b, v14.16b // flat8in && fm && wd > 4 + bic v1.16b, v1.16b, v14.16b // fm && wd >= 4 && !flat8in +.if \wd == 16 + umax v3.16b, v3.16b, v4.16b + umax v5.16b, v5.16b, v6.16b +.endif + mov x16, v1.d[0] + mov x17, v1.d[1] +.if \wd == 16 + umax v7.16b, v7.16b, v8.16b + umax v3.16b, v3.16b, v5.16b + umax v3.16b, v3.16b, v7.16b + cmhs v3.16b, v10.16b, v3.16b // flat8out +.endif + adds x16, x16, x17 +.if \wd == 16 + and v15.16b, v15.16b, v3.16b // flat8out && fm && wd == 16 + and v15.16b, v15.16b, v14.16b // flat8out && flat8in && fm && wd == 16 + bic v14.16b, v14.16b, v15.16b // flat8in && fm && wd >= 4 && !flat8out +.endif + b.eq 1f // skip wd == 4 case +.endif + movi v3.16b, #128 + eor v2.16b, v22.16b, v3.16b // p1 - 128 + eor v3.16b, v25.16b, v3.16b // q1 - 128 + cmhi v0.16b, v0.16b, v12.16b // hev + sqsub v2.16b, v2.16b, v3.16b // iclip_diff(p1 - q1) + and v4.16b, v2.16b, v0.16b // if (hev) iclip_diff(p1 - q1) + bic v0.16b, v1.16b, v0.16b // (fm && wd >= 4 && !hev) + usubl v2.8h, v24.8b, v23.8b + movi v5.8h, #3 + usubl2 v3.8h, v24.16b, v23.16b + mul v2.8h, v2.8h, v5.8h + mul v3.8h, v3.8h, v5.8h + movi v6.16b, #4 + saddw v2.8h, v2.8h, v4.8b + saddw2 v3.8h, v3.8h, v4.16b + movi v7.16b, #3 + sqxtn v2.8b, v2.8h // f + sqxtn2 v2.16b, v3.8h + sqadd v4.16b, v6.16b, v2.16b // imin(f + 4, 127) + sqadd v5.16b, v7.16b, v2.16b // imin(f + 3, 127) + sshr v4.16b, v4.16b, #3 // f1 + sshr v5.16b, v5.16b, #3 // f2 + mov v2.16b, v23.16b // p0 + mov v3.16b, v24.16b // q0 + neg v6.16b, v4.16b // -f1 + srshr v4.16b, v4.16b, #1 // (f1 + 1) >> 1 + // p0 + f2, q0 - f1 + usqadd v2.16b, v5.16b // out p0 + usqadd v3.16b, v6.16b // out q0 + neg v6.16b, v4.16b // -((f1 + 1) >> 1) + bit v23.16b, v2.16b, v1.16b // if (fm && wd >= 4) + bit v24.16b, v3.16b, v1.16b // if (fm && wd >= 4) + mov v2.16b, v22.16b // p1 + mov v3.16b, v25.16b // q1 + // p1 + ((f1 + 1) >> 1), q1 - ((f1 + 1) >> 1) + usqadd v2.16b, v4.16b // out p1 + usqadd v3.16b, v6.16b // out q1 + bit v22.16b, v2.16b, v0.16b // if (fm && wd >= 4 && !hev) + bit v25.16b, v3.16b, v0.16b // if (fm && wd >= 4 && !hev) +1: + +.if \wd == 6 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 2f // skip if there's no flat8in + + uaddl v0.8h, v21.8b, v21.8b // p2 * 2 + uaddl2 v1.8h, v21.16b, v21.16b + uaddl v2.8h, v21.8b, v22.8b // p2 + p1 + uaddl2 v3.8h, v21.16b, v22.16b + uaddl v4.8h, v22.8b, v23.8b // p1 + p0 + uaddl2 v5.8h, v22.16b, v23.16b + uaddl v6.8h, v23.8b, v24.8b // p0 + q0 + uaddl2 v7.8h, v23.16b, v24.16b + add v8.8h, v0.8h, v2.8h + add v9.8h, v1.8h, v3.8h + add v10.8h, v4.8h, v6.8h + add v11.8h, v5.8h, v7.8h + uaddl v12.8h, v24.8b, v25.8b // q0 + q1 + uaddl2 v13.8h, v24.16b, v25.16b + add v8.8h, v8.8h, v10.8h + add v9.8h, v9.8h, v11.8h + sub v12.8h, v12.8h, v0.8h + sub v13.8h, v13.8h, v1.8h + uaddl v10.8h, v25.8b, v26.8b // q1 + q2 + uaddl2 v11.8h, v25.16b, v26.16b + rshrn v0.8b, v8.8h, #3 // out p1 + rshrn2 v0.16b, v9.8h, #3 + + add v8.8h, v8.8h, v12.8h + add v9.8h, v9.8h, v13.8h + sub v10.8h, v10.8h, v2.8h + sub v11.8h, v11.8h, v3.8h + uaddl v12.8h, v26.8b, v26.8b // q2 + q2 + uaddl2 v13.8h, v26.16b, v26.16b + rshrn v1.8b, v8.8h, #3 // out p0 + rshrn2 v1.16b, v9.8h, #3 + + add v8.8h, v8.8h, v10.8h + add v9.8h, v9.8h, v11.8h + sub v12.8h, v12.8h, v4.8h + sub v13.8h, v13.8h, v5.8h + rshrn v2.8b, v8.8h, #3 // out q0 + rshrn2 v2.16b, v9.8h, #3 + + bit v22.16b, v0.16b, v14.16b // p1 if (flat8in) + add v8.8h, v8.8h, v12.8h + add v9.8h, v9.8h, v13.8h + bit v23.16b, v1.16b, v14.16b // p0 if (flat8in) + rshrn v3.8b, v8.8h, #3 // out q1 + rshrn2 v3.16b, v9.8h, #3 + bit v24.16b, v2.16b, v14.16b // q0 if (flat8in) + bit v25.16b, v3.16b, v14.16b // q1 if (flat8in) +.elseif \wd >= 8 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 +.if \wd == 8 + b.eq 8f // skip if there's no flat8in +.else + b.eq 2f // skip if there's no flat8in +.endif + + uaddl v0.8h, v20.8b, v21.8b // p3 + p2 + uaddl2 v1.8h, v20.16b, v21.16b + uaddl v2.8h, v22.8b, v25.8b // p1 + q1 + uaddl2 v3.8h, v22.16b, v25.16b + uaddl v4.8h, v20.8b, v22.8b // p3 + p1 + uaddl2 v5.8h, v20.16b, v22.16b + uaddl v6.8h, v23.8b, v26.8b // p0 + q2 + uaddl2 v7.8h, v23.16b, v26.16b + add v8.8h, v0.8h, v0.8h // 2 * (p3 + p2) + add v9.8h, v1.8h, v1.8h + uaddw v8.8h, v8.8h, v23.8b // + p0 + uaddw2 v9.8h, v9.8h, v23.16b + uaddw v8.8h, v8.8h, v24.8b // + q0 + uaddw2 v9.8h, v9.8h, v24.16b + add v8.8h, v8.8h, v4.8h + add v9.8h, v9.8h, v5.8h // + p3 + p1 + sub v2.8h, v2.8h, v0.8h // p1 + q1 - p3 - p2 + sub v3.8h, v3.8h, v1.8h + sub v6.8h, v6.8h, v4.8h // p0 + q2 - p3 - p1 + sub v7.8h, v7.8h, v5.8h + rshrn v10.8b, v8.8h, #3 // out p2 + rshrn2 v10.16b, v9.8h, #3 + + add v8.8h, v8.8h, v2.8h + add v9.8h, v9.8h, v3.8h + uaddl v0.8h, v20.8b, v23.8b // p3 + p0 + uaddl2 v1.8h, v20.16b, v23.16b + uaddl v2.8h, v24.8b, v27.8b // q0 + q3 + uaddl2 v3.8h, v24.16b, v27.16b + rshrn v11.8b, v8.8h, #3 // out p1 + rshrn2 v11.16b, v9.8h, #3 + + add v8.8h, v8.8h, v6.8h + add v9.8h, v9.8h, v7.8h + sub v2.8h, v2.8h, v0.8h // q0 + q3 - p3 - p0 + sub v3.8h, v3.8h, v1.8h + uaddl v4.8h, v21.8b, v24.8b // p2 + q0 + uaddl2 v5.8h, v21.16b, v24.16b + uaddl v6.8h, v25.8b, v27.8b // q1 + q3 + uaddl2 v7.8h, v25.16b, v27.16b + rshrn v12.8b, v8.8h, #3 // out p0 + rshrn2 v12.16b, v9.8h, #3 + + add v8.8h, v8.8h, v2.8h + add v9.8h, v9.8h, v3.8h + sub v6.8h, v6.8h, v4.8h // q1 + q3 - p2 - q0 + sub v7.8h, v7.8h, v5.8h + uaddl v0.8h, v22.8b, v25.8b // p1 + q1 + uaddl2 v1.8h, v22.16b, v25.16b + uaddl v2.8h, v26.8b, v27.8b // q2 + q3 + uaddl2 v3.8h, v26.16b, v27.16b + rshrn v13.8b, v8.8h, #3 // out q0 + rshrn2 v13.16b, v9.8h, #3 + + add v8.8h, v8.8h, v6.8h + add v9.8h, v9.8h, v7.8h + sub v2.8h, v2.8h, v0.8h // q2 + q3 - p1 - q1 + sub v3.8h, v3.8h, v1.8h + rshrn v0.8b, v8.8h, #3 // out q1 + rshrn2 v0.16b, v9.8h, #3 + + add v8.8h, v8.8h, v2.8h + add v9.8h , v9.8h, v3.8h + + bit v21.16b, v10.16b, v14.16b + bit v22.16b, v11.16b, v14.16b + bit v23.16b, v12.16b, v14.16b + rshrn v1.8b, v8.8h, #3 // out q2 + rshrn2 v1.16b, v9.8h, #3 + bit v24.16b, v13.16b, v14.16b + bit v25.16b, v0.16b, v14.16b + bit v26.16b, v1.16b, v14.16b +.endif +2: +.if \wd == 16 + mov x16, v15.d[0] + mov x17, v15.d[1] + adds x16, x16, x17 + b.ne 1f // check if flat8out is needed + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 8f // if there was no flat8in, just write the inner 4 pixels + b 7f // if flat8in was used, write the inner 6 pixels +1: + + uaddl v2.8h, v17.8b, v17.8b // p6 + p6 + uaddl2 v3.8h, v17.16b, v17.16b + uaddl v4.8h, v17.8b, v18.8b // p6 + p5 + uaddl2 v5.8h, v17.16b, v18.16b + uaddl v6.8h, v17.8b, v19.8b // p6 + p4 + uaddl2 v7.8h, v17.16b, v19.16b + uaddl v8.8h, v17.8b, v20.8b // p6 + p3 + uaddl2 v9.8h, v17.16b, v20.16b + add v12.8h, v2.8h, v4.8h + add v13.8h, v3.8h, v5.8h + add v10.8h, v6.8h, v8.8h + add v11.8h, v7.8h, v9.8h + uaddl v6.8h, v17.8b, v21.8b // p6 + p2 + uaddl2 v7.8h, v17.16b, v21.16b + add v12.8h, v12.8h, v10.8h + add v13.8h, v13.8h, v11.8h + uaddl v8.8h, v17.8b, v22.8b // p6 + p1 + uaddl2 v9.8h, v17.16b, v22.16b + uaddl v10.8h, v18.8b, v23.8b // p5 + p0 + uaddl2 v11.8h, v18.16b, v23.16b + add v6.8h, v6.8h, v8.8h + add v7.8h, v7.8h, v9.8h + uaddl v8.8h, v19.8b, v24.8b // p4 + q0 + uaddl2 v9.8h, v19.16b, v24.16b + add v12.8h, v12.8h, v6.8h + add v13.8h, v13.8h, v7.8h + add v10.8h, v10.8h, v8.8h + add v11.8h, v11.8h, v9.8h + uaddl v6.8h, v20.8b, v25.8b // p3 + q1 + uaddl2 v7.8h, v20.16b, v25.16b + add v12.8h, v12.8h, v10.8h + add v13.8h, v13.8h, v11.8h + sub v6.8h, v6.8h, v2.8h + sub v7.8h, v7.8h, v3.8h + uaddl v2.8h, v21.8b, v26.8b // p2 + q2 + uaddl2 v3.8h, v21.16b, v26.16b + rshrn v0.8b, v12.8h, #4 // out p5 + rshrn2 v0.16b, v13.8h, #4 + add v12.8h, v12.8h, v6.8h // - (p6 + p6) + (p3 + q1) + add v13.8h, v13.8h, v7.8h + sub v2.8h, v2.8h, v4.8h + sub v3.8h, v3.8h, v5.8h + uaddl v4.8h, v22.8b, v27.8b // p1 + q3 + uaddl2 v5.8h, v22.16b, v27.16b + uaddl v6.8h, v17.8b, v19.8b // p6 + p4 + uaddl2 v7.8h, v17.16b, v19.16b + rshrn v1.8b, v12.8h, #4 // out p4 + rshrn2 v1.16b, v13.8h, #4 + add v12.8h, v12.8h, v2.8h // - (p6 + p5) + (p2 + q2) + add v13.8h, v13.8h, v3.8h + sub v4.8h, v4.8h, v6.8h + sub v5.8h, v5.8h, v7.8h + uaddl v6.8h, v23.8b, v28.8b // p0 + q4 + uaddl2 v7.8h, v23.16b, v28.16b + uaddl v8.8h, v17.8b, v20.8b // p6 + p3 + uaddl2 v9.8h, v17.16b, v20.16b + rshrn v2.8b, v12.8h, #4 // out p3 + rshrn2 v2.16b, v13.8h, #4 + add v12.8h, v12.8h, v4.8h // - (p6 + p4) + (p1 + q3) + add v13.8h, v13.8h, v5.8h + sub v6.8h, v6.8h, v8.8h + sub v7.8h, v7.8h, v9.8h + uaddl v8.8h, v24.8b, v29.8b // q0 + q5 + uaddl2 v9.8h, v24.16b, v29.16b + uaddl v4.8h, v17.8b, v21.8b // p6 + p2 + uaddl2 v5.8h, v17.16b, v21.16b + rshrn v3.8b, v12.8h, #4 // out p2 + rshrn2 v3.16b, v13.8h, #4 + add v12.8h, v12.8h, v6.8h // - (p6 + p3) + (p0 + q4) + add v13.8h, v13.8h, v7.8h + sub v8.8h, v8.8h, v4.8h + sub v9.8h, v9.8h, v5.8h + uaddl v6.8h, v25.8b, v30.8b // q1 + q6 + uaddl2 v7.8h, v25.16b, v30.16b + uaddl v10.8h, v17.8b, v22.8b // p6 + p1 + uaddl2 v11.8h, v17.16b, v22.16b + rshrn v4.8b, v12.8h, #4 // out p1 + rshrn2 v4.16b, v13.8h, #4 + add v12.8h, v12.8h, v8.8h // - (p6 + p2) + (q0 + q5) + add v13.8h, v13.8h, v9.8h + sub v6.8h, v6.8h, v10.8h + sub v7.8h, v7.8h, v11.8h + uaddl v8.8h, v26.8b, v30.8b // q2 + q6 + uaddl2 v9.8h, v26.16b, v30.16b + bif v0.16b, v18.16b, v15.16b // out p5 + uaddl v10.8h, v18.8b, v23.8b // p5 + p0 + uaddl2 v11.8h, v18.16b, v23.16b + rshrn v5.8b, v12.8h, #4 // out p0 + rshrn2 v5.16b, v13.8h, #4 + add v12.8h, v12.8h, v6.8h // - (p6 + p1) + (q1 + q6) + add v13.8h, v13.8h, v7.8h + sub v8.8h, v8.8h, v10.8h + sub v9.8h, v9.8h, v11.8h + uaddl v10.8h, v27.8b, v30.8b // q3 + q6 + uaddl2 v11.8h, v27.16b, v30.16b + bif v1.16b, v19.16b, v15.16b // out p4 + uaddl v18.8h, v19.8b, v24.8b // p4 + q0 + uaddl2 v19.8h, v19.16b, v24.16b + rshrn v6.8b, v12.8h, #4 // out q0 + rshrn2 v6.16b, v13.8h, #4 + add v12.8h, v12.8h, v8.8h // - (p5 + p0) + (q2 + q6) + add v13.8h, v13.8h, v9.8h + sub v10.8h, v10.8h, v18.8h + sub v11.8h, v11.8h, v19.8h + uaddl v8.8h, v28.8b, v30.8b // q4 + q6 + uaddl2 v9.8h, v28.16b, v30.16b + bif v2.16b, v20.16b, v15.16b // out p3 + uaddl v18.8h, v20.8b, v25.8b // p3 + q1 + uaddl2 v19.8h, v20.16b, v25.16b + rshrn v7.8b, v12.8h, #4 // out q1 + rshrn2 v7.16b, v13.8h, #4 + add v12.8h, v12.8h, v10.8h // - (p4 + q0) + (q3 + q6) + add v13.8h, v13.8h, v11.8h + sub v18.8h, v8.8h, v18.8h + sub v19.8h, v9.8h, v19.8h + uaddl v10.8h, v29.8b, v30.8b // q5 + q6 + uaddl2 v11.8h, v29.16b, v30.16b + bif v3.16b, v21.16b, v15.16b // out p2 + uaddl v20.8h, v21.8b, v26.8b // p2 + q2 + uaddl2 v21.8h, v21.16b, v26.16b + rshrn v8.8b, v12.8h, #4 // out q2 + rshrn2 v8.16b, v13.8h, #4 + add v12.8h, v12.8h, v18.8h // - (p3 + q1) + (q4 + q6) + add v13.8h, v13.8h, v19.8h + sub v10.8h, v10.8h, v20.8h + sub v11.8h, v11.8h, v21.8h + uaddl v18.8h, v30.8b, v30.8b // q6 + q6 + uaddl2 v19.8h, v30.16b, v30.16b + bif v4.16b, v22.16b, v15.16b // out p1 + uaddl v20.8h, v22.8b, v27.8b // p1 + q3 + uaddl2 v21.8h, v22.16b, v27.16b + rshrn v9.8b, v12.8h, #4 // out q3 + rshrn2 v9.16b, v13.8h, #4 + add v12.8h, v12.8h, v10.8h // - (p2 + q2) + (q5 + q6) + add v13.8h, v13.8h, v11.8h + sub v18.8h, v18.8h, v20.8h + sub v19.8h, v19.8h, v21.8h + bif v5.16b, v23.16b, v15.16b // out p0 + rshrn v10.8b, v12.8h, #4 // out q4 + rshrn2 v10.16b, v13.8h, #4 + add v12.8h, v12.8h, v18.8h // - (p1 + q3) + (q6 + q6) + add v13.8h, v13.8h, v19.8h + rshrn v11.8b, v12.8h, #4 // out q5 + rshrn2 v11.16b, v13.8h, #4 + bif v6.16b, v24.16b, v15.16b // out q0 + bif v7.16b, v25.16b, v15.16b // out q1 + bif v8.16b, v26.16b, v15.16b // out q2 + bif v9.16b, v27.16b, v15.16b // out q3 + bif v10.16b, v28.16b, v15.16b // out q4 + bif v11.16b, v29.16b, v15.16b // out q5 +.endif + + mov x14, #0 + ret +.if \wd == 16 +7: + // Return to a shorter epilogue, writing only the inner 6 pixels + mov x14, #(1 << 6) + ret +.endif +.if \wd >= 8 +8: + // Return to a shorter epilogue, writing only the inner 4 pixels + mov x14, #(1 << 4) + ret +.endif +endfunc +.endm + +loop_filter 16 +loop_filter 8 +loop_filter 6 +loop_filter 4 + +.macro lpf_16_wd16 + bl lpf_16_wd16_neon + cbz x14, 1f + tbnz x14, #6, 7f + tbnz x14, #4, 8f + ret x15 +1: +.endm + +.macro lpf_16_wd8 + bl lpf_16_wd8_neon + cbz x14, 1f + tbnz x14, #4, 8f + ret x15 +1: +.endm + +.macro lpf_16_wd6 + bl lpf_16_wd6_neon + cbz x14, 1f + ret x15 +1: +.endm + +.macro lpf_16_wd4 + bl lpf_16_wd4_neon + cbz x14, 1f + ret x15 +1: +.endm + +function lpf_v_4_16_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + ld1 {v22.16b}, [x16], x1 // p1 + ld1 {v24.16b}, [x0], x1 // q0 + ld1 {v23.16b}, [x16], x1 // p0 + ld1 {v25.16b}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + + lpf_16_wd4 + + sub x16, x0, x1, lsl #1 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v25.16b}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_4_16_neon + mov x15, x30 + sub x16, x0, #2 + add x0, x16, x1, lsl #3 + ld1 {v22.s}[0], [x16], x1 + ld1 {v22.s}[2], [x0], x1 + ld1 {v23.s}[0], [x16], x1 + ld1 {v23.s}[2], [x0], x1 + ld1 {v24.s}[0], [x16], x1 + ld1 {v24.s}[2], [x0], x1 + ld1 {v25.s}[0], [x16], x1 + ld1 {v25.s}[2], [x0], x1 + ld1 {v22.s}[1], [x16], x1 + ld1 {v22.s}[3], [x0], x1 + ld1 {v23.s}[1], [x16], x1 + ld1 {v23.s}[3], [x0], x1 + ld1 {v24.s}[1], [x16], x1 + ld1 {v24.s}[3], [x0], x1 + ld1 {v25.s}[1], [x16], x1 + ld1 {v25.s}[3], [x0], x1 + add x0, x0, #2 + + transpose_4x16b v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_16_wd4 + + sub x16, x0, x1, lsl #4 + sub x16, x16, #2 + transpose_4x16b v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v22.s}[0], [x16], x1 + st1 {v22.s}[2], [x0], x1 + st1 {v23.s}[0], [x16], x1 + st1 {v23.s}[2], [x0], x1 + st1 {v24.s}[0], [x16], x1 + st1 {v24.s}[2], [x0], x1 + st1 {v25.s}[0], [x16], x1 + st1 {v25.s}[2], [x0], x1 + st1 {v22.s}[1], [x16], x1 + st1 {v22.s}[3], [x0], x1 + st1 {v23.s}[1], [x16], x1 + st1 {v23.s}[3], [x0], x1 + st1 {v24.s}[1], [x16], x1 + st1 {v24.s}[3], [x0], x1 + st1 {v25.s}[1], [x16], x1 + st1 {v25.s}[3], [x0], x1 + add x0, x0, #2 + ret x15 +endfunc + +function lpf_v_6_16_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + ld1 {v21.16b}, [x16], x1 // p2 + ld1 {v24.16b}, [x0], x1 // q0 + ld1 {v22.16b}, [x16], x1 // p1 + ld1 {v25.16b}, [x0], x1 // q1 + ld1 {v23.16b}, [x16], x1 // p0 + ld1 {v26.16b}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + + lpf_16_wd6 + + sub x16, x0, x1, lsl #1 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v25.16b}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_6_16_neon + mov x15, x30 + sub x16, x0, #4 + add x0, x16, x1, lsl #3 + ld1 {v20.d}[0], [x16], x1 + ld1 {v20.d}[1], [x0], x1 + ld1 {v21.d}[0], [x16], x1 + ld1 {v21.d}[1], [x0], x1 + ld1 {v22.d}[0], [x16], x1 + ld1 {v22.d}[1], [x0], x1 + ld1 {v23.d}[0], [x16], x1 + ld1 {v23.d}[1], [x0], x1 + ld1 {v24.d}[0], [x16], x1 + ld1 {v24.d}[1], [x0], x1 + ld1 {v25.d}[0], [x16], x1 + ld1 {v25.d}[1], [x0], x1 + ld1 {v26.d}[0], [x16], x1 + ld1 {v26.d}[1], [x0], x1 + ld1 {v27.d}[0], [x16], x1 + ld1 {v27.d}[1], [x0], x1 + add x0, x0, #4 + + transpose_8x16b v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_16_wd6 + + sub x16, x0, x1, lsl #4 + sub x16, x16, #2 + transpose_4x16b v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v22.s}[0], [x16], x1 + st1 {v22.s}[2], [x0], x1 + st1 {v23.s}[0], [x16], x1 + st1 {v23.s}[2], [x0], x1 + st1 {v24.s}[0], [x16], x1 + st1 {v24.s}[2], [x0], x1 + st1 {v25.s}[0], [x16], x1 + st1 {v25.s}[2], [x0], x1 + st1 {v22.s}[1], [x16], x1 + st1 {v22.s}[3], [x0], x1 + st1 {v23.s}[1], [x16], x1 + st1 {v23.s}[3], [x0], x1 + st1 {v24.s}[1], [x16], x1 + st1 {v24.s}[3], [x0], x1 + st1 {v25.s}[1], [x16], x1 + st1 {v25.s}[3], [x0], x1 + add x0, x0, #2 + ret x15 +endfunc + +function lpf_v_8_16_neon + mov x15, x30 + sub x16, x0, x1, lsl #2 + ld1 {v20.16b}, [x16], x1 // p3 + ld1 {v24.16b}, [x0], x1 // q0 + ld1 {v21.16b}, [x16], x1 // p2 + ld1 {v25.16b}, [x0], x1 // q1 + ld1 {v22.16b}, [x16], x1 // p1 + ld1 {v26.16b}, [x0], x1 // q2 + ld1 {v23.16b}, [x16], x1 // p0 + ld1 {v27.16b}, [x0], x1 // q3 + sub x0, x0, x1, lsl #2 + + lpf_16_wd8 + + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + st1 {v21.16b}, [x16], x1 // p2 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v25.16b}, [x0], x1 // q1 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v26.16b}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + ret x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v25.16b}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_8_16_neon + mov x15, x30 + sub x16, x0, #4 + add x0, x16, x1, lsl #3 + ld1 {v20.d}[0], [x16], x1 + ld1 {v20.d}[1], [x0], x1 + ld1 {v21.d}[0], [x16], x1 + ld1 {v21.d}[1], [x0], x1 + ld1 {v22.d}[0], [x16], x1 + ld1 {v22.d}[1], [x0], x1 + ld1 {v23.d}[0], [x16], x1 + ld1 {v23.d}[1], [x0], x1 + ld1 {v24.d}[0], [x16], x1 + ld1 {v24.d}[1], [x0], x1 + ld1 {v25.d}[0], [x16], x1 + ld1 {v25.d}[1], [x0], x1 + ld1 {v26.d}[0], [x16], x1 + ld1 {v26.d}[1], [x0], x1 + ld1 {v27.d}[0], [x16], x1 + ld1 {v27.d}[1], [x0], x1 + add x0, x0, #4 + + transpose_8x16b v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_16_wd8 + + sub x16, x0, x1, lsl #4 + sub x16, x16, #4 + transpose_8x16b v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v20.d}[0], [x16], x1 + st1 {v20.d}[1], [x0], x1 + st1 {v21.d}[0], [x16], x1 + st1 {v21.d}[1], [x0], x1 + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + st1 {v26.d}[0], [x16], x1 + st1 {v26.d}[1], [x0], x1 + st1 {v27.d}[0], [x16], x1 + st1 {v27.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +8: + sub x16, x0, x1, lsl #4 + sub x16, x16, #2 + transpose_4x16b v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v22.s}[0], [x16], x1 + st1 {v22.s}[2], [x0], x1 + st1 {v23.s}[0], [x16], x1 + st1 {v23.s}[2], [x0], x1 + st1 {v24.s}[0], [x16], x1 + st1 {v24.s}[2], [x0], x1 + st1 {v25.s}[0], [x16], x1 + st1 {v25.s}[2], [x0], x1 + st1 {v22.s}[1], [x16], x1 + st1 {v22.s}[3], [x0], x1 + st1 {v23.s}[1], [x16], x1 + st1 {v23.s}[3], [x0], x1 + st1 {v24.s}[1], [x16], x1 + st1 {v24.s}[3], [x0], x1 + st1 {v25.s}[1], [x16], x1 + st1 {v25.s}[3], [x0], x1 + add x0, x0, #2 + ret x15 +endfunc + +function lpf_v_16_16_neon + mov x15, x30 + + sub x16, x0, x1, lsl #3 + add x16, x16, x1 + ld1 {v17.16b}, [x16], x1 // p6 + ld1 {v24.16b}, [x0], x1 // q0 + ld1 {v18.16b}, [x16], x1 // p5 + ld1 {v25.16b}, [x0], x1 // q1 + ld1 {v19.16b}, [x16], x1 // p4 + ld1 {v26.16b}, [x0], x1 // q2 + ld1 {v20.16b}, [x16], x1 // p3 + ld1 {v27.16b}, [x0], x1 // q3 + ld1 {v21.16b}, [x16], x1 // p2 + ld1 {v28.16b}, [x0], x1 // q4 + ld1 {v22.16b}, [x16], x1 // p1 + ld1 {v29.16b}, [x0], x1 // q5 + ld1 {v23.16b}, [x16], x1 // p0 + ld1 {v30.16b}, [x0], x1 // q6 + sub x0, x0, x1, lsl #3 + add x0, x0, x1 + + lpf_16_wd16 + + sub x16, x0, x1, lsl #2 + sub x16, x16, x1, lsl #1 + st1 {v0.16b}, [x16], x1 // p5 + st1 {v6.16b}, [x0], x1 // q0 + st1 {v1.16b}, [x16], x1 // p4 + st1 {v7.16b}, [x0], x1 // q1 + st1 {v2.16b}, [x16], x1 // p3 + st1 {v8.16b}, [x0], x1 // q2 + st1 {v3.16b}, [x16], x1 // p2 + st1 {v9.16b}, [x0], x1 // q3 + st1 {v4.16b}, [x16], x1 // p1 + st1 {v10.16b}, [x0], x1 // q4 + st1 {v5.16b}, [x16], x1 // p0 + st1 {v11.16b}, [x0], x1 // q5 + sub x0, x0, x1, lsl #2 + sub x0, x0, x1, lsl #1 + ret x15 +7: + sub x16, x0, x1 + sub x16, x16, x1, lsl #1 + st1 {v21.16b}, [x16], x1 // p2 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v25.16b}, [x0], x1 // q1 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v26.16b}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + ret x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.16b}, [x16], x1 // p1 + st1 {v24.16b}, [x0], x1 // q0 + st1 {v23.16b}, [x16], x1 // p0 + st1 {v25.16b}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_16_16_neon + mov x15, x30 + sub x16, x0, #8 + ld1 {v16.d}[0], [x16], x1 + ld1 {v24.d}[0], [x0], x1 + ld1 {v17.d}[0], [x16], x1 + ld1 {v25.d}[0], [x0], x1 + ld1 {v18.d}[0], [x16], x1 + ld1 {v26.d}[0], [x0], x1 + ld1 {v19.d}[0], [x16], x1 + ld1 {v27.d}[0], [x0], x1 + ld1 {v20.d}[0], [x16], x1 + ld1 {v28.d}[0], [x0], x1 + ld1 {v21.d}[0], [x16], x1 + ld1 {v29.d}[0], [x0], x1 + ld1 {v22.d}[0], [x16], x1 + ld1 {v30.d}[0], [x0], x1 + ld1 {v23.d}[0], [x16], x1 + ld1 {v31.d}[0], [x0], x1 + ld1 {v16.d}[1], [x16], x1 + ld1 {v24.d}[1], [x0], x1 + ld1 {v17.d}[1], [x16], x1 + ld1 {v25.d}[1], [x0], x1 + ld1 {v18.d}[1], [x16], x1 + ld1 {v26.d}[1], [x0], x1 + ld1 {v19.d}[1], [x16], x1 + ld1 {v27.d}[1], [x0], x1 + ld1 {v20.d}[1], [x16], x1 + ld1 {v28.d}[1], [x0], x1 + ld1 {v21.d}[1], [x16], x1 + ld1 {v29.d}[1], [x0], x1 + ld1 {v22.d}[1], [x16], x1 + ld1 {v30.d}[1], [x0], x1 + ld1 {v23.d}[1], [x16], x1 + ld1 {v31.d}[1], [x0], x1 + + transpose_8x16b v16, v17, v18, v19, v20, v21, v22, v23, v0, v1 + transpose_8x16b v24, v25, v26, v27, v28, v29, v30, v31, v0, v1 + + lpf_16_wd16 + + sub x0, x0, x1, lsl #4 + sub x16, x0, #8 + + transpose_8x16b v16, v17, v0, v1, v2, v3, v4, v5, v18, v19 + transpose_8x16b v6, v7, v8, v9, v10, v11, v30, v31, v18, v19 + + st1 {v16.d}[0], [x16], x1 + st1 {v6.d}[0], [x0], x1 + st1 {v17.d}[0], [x16], x1 + st1 {v7.d}[0], [x0], x1 + st1 {v0.d}[0], [x16], x1 + st1 {v8.d}[0], [x0], x1 + st1 {v1.d}[0], [x16], x1 + st1 {v9.d}[0], [x0], x1 + st1 {v2.d}[0], [x16], x1 + st1 {v10.d}[0], [x0], x1 + st1 {v3.d}[0], [x16], x1 + st1 {v11.d}[0], [x0], x1 + st1 {v4.d}[0], [x16], x1 + st1 {v30.d}[0], [x0], x1 + st1 {v5.d}[0], [x16], x1 + st1 {v31.d}[0], [x0], x1 + st1 {v16.d}[1], [x16], x1 + st1 {v6.d}[1], [x0], x1 + st1 {v17.d}[1], [x16], x1 + st1 {v7.d}[1], [x0], x1 + st1 {v0.d}[1], [x16], x1 + st1 {v8.d}[1], [x0], x1 + st1 {v1.d}[1], [x16], x1 + st1 {v9.d}[1], [x0], x1 + st1 {v2.d}[1], [x16], x1 + st1 {v10.d}[1], [x0], x1 + st1 {v3.d}[1], [x16], x1 + st1 {v11.d}[1], [x0], x1 + st1 {v4.d}[1], [x16], x1 + st1 {v30.d}[1], [x0], x1 + st1 {v5.d}[1], [x16], x1 + st1 {v31.d}[1], [x0], x1 + ret x15 + +7: + sub x16, x0, x1, lsl #4 + sub x16, x16, #4 + transpose_8x16b v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v20.d}[0], [x16], x1 + st1 {v20.d}[1], [x0], x1 + st1 {v21.d}[0], [x16], x1 + st1 {v21.d}[1], [x0], x1 + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + st1 {v26.d}[0], [x16], x1 + st1 {v26.d}[1], [x0], x1 + st1 {v27.d}[0], [x16], x1 + st1 {v27.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +8: + sub x16, x0, x1, lsl #4 + sub x16, x16, #2 + transpose_4x16b v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #3 + + st1 {v22.s}[0], [x16], x1 + st1 {v22.s}[2], [x0], x1 + st1 {v23.s}[0], [x16], x1 + st1 {v23.s}[2], [x0], x1 + st1 {v24.s}[0], [x16], x1 + st1 {v24.s}[2], [x0], x1 + st1 {v25.s}[0], [x16], x1 + st1 {v25.s}[2], [x0], x1 + st1 {v22.s}[1], [x16], x1 + st1 {v22.s}[3], [x0], x1 + st1 {v23.s}[1], [x16], x1 + st1 {v23.s}[3], [x0], x1 + st1 {v24.s}[1], [x16], x1 + st1 {v24.s}[3], [x0], x1 + st1 {v25.s}[1], [x16], x1 + st1 {v25.s}[3], [x0], x1 + add x0, x0, #2 + ret x15 +endfunc + +// void dav1d_lpf_v_sb_y_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w) + +.macro lpf_func dir, type +function lpf_\dir\()_sb_\type\()_8bpc_neon, export=1 + mov x11, x30 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + ldp w6, w7, [x2] // vmask[0], vmask[1] +.ifc \type, y + ldr w2, [x2, #8] // vmask[2] +.endif + add x5, x5, #128 // Move to sharp part of lut +.ifc \type, y + orr w7, w7, w2 // vmask[1] |= vmask[2] +.endif +.ifc \dir, v + sub x4, x3, x4, lsl #2 +.else + sub x3, x3, #4 + lsl x4, x4, #2 +.endif + orr w6, w6, w7 // vmask[0] |= vmask[1] + +1: + tst w6, #0x0f +.ifc \dir, v + ld1 {v0.16b}, [x4], #16 + ld1 {v1.16b}, [x3], #16 +.else + ld2 {v0.s,v1.s}[0], [x3], x4 + ld2 {v0.s,v1.s}[1], [x3], x4 + ld2 {v0.s,v1.s}[2], [x3], x4 + ld2 {v0.s,v1.s}[3], [x3], x4 +.endif + b.eq 7f // if (!(vm & bits)) continue; + + ld1r {v5.16b}, [x5] // sharp[0] + add x5, x5, #8 + movi v2.4s, #0xff + dup v13.4s, w6 // vmask[0] + + and v0.16b, v0.16b, v2.16b // Keep only lowest byte in each 32 bit word + and v1.16b, v1.16b, v2.16b + cmtst v3.16b, v1.16b, v2.16b // Check for nonzero values in l[0][0] + movi v4.16b, #1 + ld1r {v6.16b}, [x5] // sharp[1] + sub x5, x5, #8 + bif v1.16b, v0.16b, v3.16b // if (!l[0][0]) L = l[offset][0] + cmtst v2.4s, v1.4s, v2.4s // L != 0 + mul v1.4s, v1.4s, v4.4s // L +.ifc \type, y + dup v15.4s, w2 // vmask[2] +.endif + dup v14.4s, w7 // vmask[1] + mov x16, v2.d[0] + mov x17, v2.d[1] + adds x16, x16, x17 + b.eq 7f // if (!L) continue; + neg v5.16b, v5.16b // -sharp[0] + movrel x16, word_1248 + ushr v12.16b, v1.16b, #4 // H + ld1 {v16.4s}, [x16] + sshl v3.16b, v1.16b, v5.16b // L >> sharp[0] +.ifc \type, y + cmtst v15.4s, v15.4s, v16.4s // if (vmask[2] & bits) +.endif + movi v7.16b, #2 + umin v3.16b, v3.16b, v6.16b // imin(L >> sharp[0], sharp[1]) + add v0.16b, v1.16b, v7.16b // L + 2 + umax v11.16b, v3.16b, v4.16b // imax(imin(), 1) = limit = I + add v0.16b, v0.16b, v0.16b // 2*(L + 2) + cmtst v14.4s, v14.4s, v16.4s // if (vmask[1] & bits) + add v10.16b, v0.16b, v11.16b // 2*(L + 2) + limit = E + cmtst v13.4s, v13.4s, v16.4s // if (vmask[0] & bits) + and v13.16b, v13.16b, v2.16b // vmask[0] &= L != 0 + +.ifc \type, y + tst w2, #0x0f + b.eq 2f + // wd16 + bl lpf_\dir\()_16_16_neon + b 8f +2: +.endif + tst w7, #0x0f + b.eq 3f +.ifc \type, y + // wd8 + bl lpf_\dir\()_8_16_neon +.else + // wd6 + bl lpf_\dir\()_6_16_neon +.endif + b 8f +3: + // wd4 + bl lpf_\dir\()_4_16_neon +.ifc \dir, h + b 8f +7: + // For dir h, the functions above increment x0. + // If the whole function is skipped, increment it here instead. + add x0, x0, x1, lsl #4 +.else +7: +.endif +8: + lsr w6, w6, #4 // vmask[0] >>= 4 + lsr w7, w7, #4 // vmask[1] >>= 4 +.ifc \type, y + lsr w2, w2, #4 // vmask[2] >>= 4 +.endif +.ifc \dir, v + add x0, x0, #16 +.else + // For dir h, x0 is returned incremented +.endif + cbnz w6, 1b + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + ret x11 +endfunc +.endm + +lpf_func v, y +lpf_func h, y +lpf_func v, uv +lpf_func h, uv + +const word_1248 + .word 1, 2, 4, 8 +endconst diff --git a/third_party/dav1d/src/arm/64/loopfilter16.S b/third_party/dav1d/src/arm/64/loopfilter16.S new file mode 100644 index 0000000000..d181a3e623 --- /dev/null +++ b/third_party/dav1d/src/arm/64/loopfilter16.S @@ -0,0 +1,925 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// depending on how many pixels need to be stored, returns: +// x14 = (1 << 0) : 0 pixels +// x14 = (1 << 4) : inner 4 pixels +// x14 = (1 << 6) : inner 6 pixels +// x14 = 0 : all pixels +.macro loop_filter wd +function lpf_8_wd\wd\()_neon + uabd v0.8h, v22.8h, v23.8h // abs(p1 - p0) + uabd v1.8h, v25.8h, v24.8h // abs(q1 - q0) + uabd v2.8h, v23.8h, v24.8h // abs(p0 - q0) + uabd v3.8h, v22.8h, v25.8h // abs(p1 - q1) +.if \wd >= 6 + uabd v4.8h, v21.8h, v22.8h // abs(p2 - p1) + uabd v5.8h, v26.8h, v25.8h // abs(q2 - q1) +.endif +.if \wd >= 8 + uabd v6.8h, v20.8h, v21.8h // abs(p3 - p2) + uabd v7.8h, v27.8h, v26.8h // abs(q3 - q3) +.endif +.if \wd >= 6 + umax v4.8h, v4.8h, v5.8h +.endif + uqadd v2.8h, v2.8h, v2.8h // abs(p0 - q0) * 2 +.if \wd >= 8 + umax v6.8h, v6.8h, v7.8h +.endif + ushr v3.8h, v3.8h, #1 +.if \wd >= 8 + umax v4.8h, v4.8h, v6.8h +.endif +.if \wd >= 6 + and v4.16b, v4.16b, v14.16b +.endif + umax v0.8h, v0.8h, v1.8h // max(abs(p1 - p0), abs(q1 - q0)) + uqadd v2.8h, v2.8h, v3.8h // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 +.if \wd >= 6 + umax v4.8h, v0.8h, v4.8h + cmhs v1.8h, v11.8h, v4.8h // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I +.else + cmhs v1.8h, v11.8h, v0.8h // max(abs(p1 - p0), abs(q1 - q0)) <= I +.endif + cmhs v2.8h, v10.8h, v2.8h // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E + and v1.16b, v1.16b, v2.16b // fm + and v1.16b, v1.16b, v13.16b // fm && wd >= 4 +.if \wd >= 6 + and v14.16b, v14.16b, v1.16b // fm && wd > 4 +.endif +.if \wd >= 16 + and v15.16b, v15.16b, v1.16b // fm && wd == 16 +.endif + + mov x16, v1.d[0] + mov x17, v1.d[1] + adds x16, x16, x17 + b.ne 9f // if (!fm || wd < 4) return; + mov x14, #(1 << 0) + ret +9: +.if \wd >= 6 + movi v10.8h, #1 + uabd v2.8h, v21.8h, v23.8h // abs(p2 - p0) + uabd v3.8h, v22.8h, v23.8h // abs(p1 - p0) + uabd v4.8h, v25.8h, v24.8h // abs(q1 - q0) + uabd v5.8h, v26.8h, v24.8h // abs(q2 - q0) + dup v9.8h, w9 // bitdepth_min_8 +.if \wd >= 8 + uabd v6.8h, v20.8h, v23.8h // abs(p3 - p0) + uabd v7.8h, v27.8h, v24.8h // abs(q3 - q0) +.endif + umax v2.8h, v2.8h, v3.8h + umax v4.8h, v4.8h, v5.8h +.if \wd >= 8 + umax v6.8h, v6.8h, v7.8h +.endif + umax v2.8h, v2.8h, v4.8h + ushl v10.8h, v10.8h, v9.8h // F = 1 << bitdepth_min_8 +.if \wd >= 8 + umax v2.8h, v2.8h, v6.8h +.endif + +.if \wd == 16 + uabd v3.8h, v17.8h, v23.8h // abs(p6 - p0) + uabd v4.8h, v18.8h, v23.8h // abs(p5 - p0) + uabd v5.8h, v19.8h, v23.8h // abs(p4 - p0) +.endif + cmhs v2.8h, v10.8h, v2.8h // flat8in +.if \wd == 16 + uabd v6.8h, v28.8h, v24.8h // abs(q4 - q0) + uabd v7.8h, v29.8h, v24.8h // abs(q5 - q0) + uabd v8.8h, v30.8h, v24.8h // abs(q6 - q0) +.endif + and v14.16b, v2.16b, v14.16b // flat8in && fm && wd > 4 + bic v1.16b, v1.16b, v14.16b // fm && wd >= 4 && !flat8in +.if \wd == 16 + umax v3.8h, v3.8h, v4.8h + umax v5.8h, v5.8h, v6.8h +.endif + mov x16, v1.d[0] + mov x17, v1.d[1] +.if \wd == 16 + umax v7.8h, v7.8h, v8.8h + umax v3.8h, v3.8h, v5.8h + umax v3.8h, v3.8h, v7.8h + cmhs v3.8h, v10.8h, v3.8h // flat8out +.endif + adds x16, x16, x17 +.if \wd == 16 + and v15.16b, v15.16b, v3.16b // flat8out && fm && wd == 16 + and v15.16b, v15.16b, v14.16b // flat8out && flat8in && fm && wd == 16 + bic v14.16b, v14.16b, v15.16b // flat8in && fm && wd >= 4 && !flat8out +.endif + b.eq 1f // skip wd == 4 case +.endif + + dup v3.8h, w8 // bitdepth_max + sub v2.8h, v22.8h, v25.8h // p1 - q1 + ushr v3.8h, v3.8h, #1 // 128 << bitdepth_min_8 - 1 + cmhi v0.8h, v0.8h, v12.8h // hev + not v9.16b, v3.16b // - 128 * (1 << bitdepth_min_8) + smin v2.8h, v2.8h, v3.8h // iclip_diff(p1 - q1) + smax v2.8h, v2.8h, v9.8h // iclip_diff(p1 - q1) + and v4.16b, v2.16b, v0.16b // if (hev) iclip_diff(p1 - q1) + sub v2.8h, v24.8h, v23.8h + movi v5.8h, #3 + bic v0.16b, v1.16b, v0.16b // (fm && wd >= 4 && !hev) + mul v2.8h, v2.8h, v5.8h + movi v6.8h, #4 + add v2.8h, v2.8h, v4.8h + smin v2.8h, v2.8h, v3.8h // f = iclip_diff() + smax v2.8h, v2.8h, v9.8h // f = iclip_diff() + sqadd v4.8h, v6.8h, v2.8h // f + 4 + sqadd v5.8h, v5.8h, v2.8h // f + 3 + smin v4.8h, v4.8h, v3.8h // imin(f + 4, 128 << bitdepth_min_8 - 1) + smin v5.8h, v5.8h, v3.8h // imin(f + 3, 128 << bitdepth_min_8 - 1) + sshr v4.8h, v4.8h, #3 // f1 + sshr v5.8h, v5.8h, #3 // f2 + movi v9.8h, #0 + dup v3.8h, w8 // bitdepth_max + sqadd v2.8h, v23.8h, v5.8h // p0 + f2 + sqsub v6.8h, v24.8h, v4.8h // q0 - f1 + srshr v4.8h, v4.8h, #1 // (f1 + 1) >> 1 + smin v2.8h, v2.8h, v3.8h // out p0 = iclip_pixel() + smin v6.8h, v6.8h, v3.8h // out q0 = iclip_pixel() + smax v2.8h, v2.8h, v9.8h // out p0 = iclip_pixel() + smax v6.8h, v6.8h, v9.8h // out q0 = iclip_pixel() + bit v23.16b, v2.16b, v1.16b // if (fm && wd >= 4) + bit v24.16b, v6.16b, v1.16b // if (fm && wd >= 4) + sqadd v2.8h, v22.8h, v4.8h // p1 + f + sqsub v6.8h, v25.8h, v4.8h // q1 - f + smin v2.8h, v2.8h, v3.8h // out p1 = iclip_pixel() + smin v6.8h, v6.8h, v3.8h // out q1 = iclip_pixel() + smax v2.8h, v2.8h, v9.8h // out p1 = iclip_pixel() + smax v6.8h, v6.8h, v9.8h // out q1 = iclip_pixel() + bit v22.16b, v2.16b, v0.16b // if (fm && wd >= 4 && !hev) + bit v25.16b, v6.16b, v0.16b // if (fm && wd >= 4 && !hev) +1: + +.if \wd == 6 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 2f // skip if there's no flat8in + + add v0.8h, v21.8h, v21.8h // p2 * 2 + add v2.8h, v21.8h, v22.8h // p2 + p1 + add v4.8h, v22.8h, v23.8h // p1 + p0 + add v6.8h, v23.8h, v24.8h // p0 + q0 + add v8.8h, v0.8h, v2.8h + add v10.8h, v4.8h, v6.8h + add v12.8h, v24.8h, v25.8h // q0 + q1 + add v8.8h, v8.8h, v10.8h + sub v12.8h, v12.8h, v0.8h + add v10.8h, v25.8h, v26.8h // q1 + q2 + urshr v0.8h, v8.8h, #3 // out p1 + + add v8.8h, v8.8h, v12.8h + sub v10.8h, v10.8h, v2.8h + add v12.8h, v26.8h, v26.8h // q2 + q2 + urshr v1.8h, v8.8h, #3 // out p0 + + add v8.8h, v8.8h, v10.8h + sub v12.8h, v12.8h, v4.8h + urshr v2.8h, v8.8h, #3 // out q0 + + bit v22.16b, v0.16b, v14.16b // p1 if (flat8in) + add v8.8h, v8.8h, v12.8h + bit v23.16b, v1.16b, v14.16b // p0 if (flat8in) + urshr v3.8h, v8.8h, #3 // out q1 + bit v24.16b, v2.16b, v14.16b // q0 if (flat8in) + bit v25.16b, v3.16b, v14.16b // q1 if (flat8in) +.elseif \wd >= 8 + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 +.if \wd == 8 + b.eq 8f // skip if there's no flat8in +.else + b.eq 2f // skip if there's no flat8in +.endif + + add v0.8h, v20.8h, v21.8h // p3 + p2 + add v2.8h, v22.8h, v25.8h // p1 + q1 + add v4.8h, v20.8h, v22.8h // p3 + p1 + add v6.8h, v23.8h, v26.8h // p0 + q2 + add v8.8h, v0.8h, v0.8h // 2 * (p3 + p2) + add v9.8h, v23.8h, v24.8h // p0 + q0 + add v8.8h, v8.8h, v4.8h // + p3 + p1 + sub v2.8h, v2.8h, v0.8h // p1 + q1 - p3 - p2 + add v8.8h, v8.8h, v9.8h // + p0 + q0 + sub v6.8h, v6.8h, v4.8h // p0 + q2 - p3 - p1 + urshr v10.8h, v8.8h, #3 // out p2 + + add v8.8h, v8.8h, v2.8h + add v0.8h, v20.8h, v23.8h // p3 + p0 + add v2.8h, v24.8h, v27.8h // q0 + q3 + urshr v11.8h, v8.8h, #3 // out p1 + + add v8.8h, v8.8h, v6.8h + sub v2.8h, v2.8h, v0.8h // q0 + q3 - p3 - p0 + add v4.8h, v21.8h, v24.8h // p2 + q0 + add v6.8h, v25.8h, v27.8h // q1 + q3 + urshr v12.8h, v8.8h, #3 // out p0 + + add v8.8h, v8.8h, v2.8h + sub v6.8h, v6.8h, v4.8h // q1 + q3 - p2 - q0 + add v0.8h, v22.8h, v25.8h // p1 + q1 + add v2.8h, v26.8h, v27.8h // q2 + q3 + urshr v13.8h, v8.8h, #3 // out q0 + + add v8.8h, v8.8h, v6.8h + sub v2.8h, v2.8h, v0.8h // q2 + q3 - p1 - q1 + urshr v0.8h, v8.8h, #3 // out q1 + + add v8.8h, v8.8h, v2.8h + + bit v21.16b, v10.16b, v14.16b + bit v22.16b, v11.16b, v14.16b + bit v23.16b, v12.16b, v14.16b + urshr v1.8h, v8.8h, #3 // out q2 + bit v24.16b, v13.16b, v14.16b + bit v25.16b, v0.16b, v14.16b + bit v26.16b, v1.16b, v14.16b +.endif +2: +.if \wd == 16 + mov x16, v15.d[0] + mov x17, v15.d[1] + adds x16, x16, x17 + b.ne 1f // check if flat8out is needed + mov x16, v14.d[0] + mov x17, v14.d[1] + adds x16, x16, x17 + b.eq 8f // if there was no flat8in, just write the inner 4 pixels + b 7f // if flat8in was used, write the inner 6 pixels +1: + + add v2.8h, v17.8h, v17.8h // p6 + p6 + add v4.8h, v17.8h, v18.8h // p6 + p5 + add v6.8h, v17.8h, v19.8h // p6 + p4 + add v8.8h, v17.8h, v20.8h // p6 + p3 + add v12.8h, v2.8h, v4.8h + add v10.8h, v6.8h, v8.8h + add v6.8h, v17.8h, v21.8h // p6 + p2 + add v12.8h, v12.8h, v10.8h + add v8.8h, v17.8h, v22.8h // p6 + p1 + add v10.8h, v18.8h, v23.8h // p5 + p0 + add v6.8h, v6.8h, v8.8h + add v8.8h, v19.8h, v24.8h // p4 + q0 + add v12.8h, v12.8h, v6.8h + add v10.8h, v10.8h, v8.8h + add v6.8h, v20.8h, v25.8h // p3 + q1 + add v12.8h, v12.8h, v10.8h + sub v6.8h, v6.8h, v2.8h + add v2.8h, v21.8h, v26.8h // p2 + q2 + urshr v0.8h, v12.8h, #4 // out p5 + add v12.8h, v12.8h, v6.8h // - (p6 + p6) + (p3 + q1) + sub v2.8h, v2.8h, v4.8h + add v4.8h, v22.8h, v27.8h // p1 + q3 + add v6.8h, v17.8h, v19.8h // p6 + p4 + urshr v1.8h, v12.8h, #4 // out p4 + add v12.8h, v12.8h, v2.8h // - (p6 + p5) + (p2 + q2) + sub v4.8h, v4.8h, v6.8h + add v6.8h, v23.8h, v28.8h // p0 + q4 + add v8.8h, v17.8h, v20.8h // p6 + p3 + urshr v2.8h, v12.8h, #4 // out p3 + add v12.8h, v12.8h, v4.8h // - (p6 + p4) + (p1 + q3) + sub v6.8h, v6.8h, v8.8h + add v8.8h, v24.8h, v29.8h // q0 + q5 + add v4.8h, v17.8h, v21.8h // p6 + p2 + urshr v3.8h, v12.8h, #4 // out p2 + add v12.8h, v12.8h, v6.8h // - (p6 + p3) + (p0 + q4) + sub v8.8h, v8.8h, v4.8h + add v6.8h, v25.8h, v30.8h // q1 + q6 + add v10.8h, v17.8h, v22.8h // p6 + p1 + urshr v4.8h, v12.8h, #4 // out p1 + add v12.8h, v12.8h, v8.8h // - (p6 + p2) + (q0 + q5) + sub v6.8h, v6.8h, v10.8h + add v8.8h, v26.8h, v30.8h // q2 + q6 + bif v0.16b, v18.16b, v15.16b // out p5 + add v10.8h, v18.8h, v23.8h // p5 + p0 + urshr v5.8h, v12.8h, #4 // out p0 + add v12.8h, v12.8h, v6.8h // - (p6 + p1) + (q1 + q6) + sub v8.8h, v8.8h, v10.8h + add v10.8h, v27.8h, v30.8h // q3 + q6 + bif v1.16b, v19.16b, v15.16b // out p4 + add v18.8h, v19.8h, v24.8h // p4 + q0 + urshr v6.8h, v12.8h, #4 // out q0 + add v12.8h, v12.8h, v8.8h // - (p5 + p0) + (q2 + q6) + sub v10.8h, v10.8h, v18.8h + add v8.8h, v28.8h, v30.8h // q4 + q6 + bif v2.16b, v20.16b, v15.16b // out p3 + add v18.8h, v20.8h, v25.8h // p3 + q1 + urshr v7.8h, v12.8h, #4 // out q1 + add v12.8h, v12.8h, v10.8h // - (p4 + q0) + (q3 + q6) + sub v18.8h, v8.8h, v18.8h + add v10.8h, v29.8h, v30.8h // q5 + q6 + bif v3.16b, v21.16b, v15.16b // out p2 + add v20.8h, v21.8h, v26.8h // p2 + q2 + urshr v8.8h, v12.8h, #4 // out q2 + add v12.8h, v12.8h, v18.8h // - (p3 + q1) + (q4 + q6) + sub v10.8h, v10.8h, v20.8h + add v18.8h, v30.8h, v30.8h // q6 + q6 + bif v4.16b, v22.16b, v15.16b // out p1 + add v20.8h, v22.8h, v27.8h // p1 + q3 + urshr v9.8h, v12.8h, #4 // out q3 + add v12.8h, v12.8h, v10.8h // - (p2 + q2) + (q5 + q6) + sub v18.8h, v18.8h, v20.8h + bif v5.16b, v23.16b, v15.16b // out p0 + urshr v10.8h, v12.8h, #4 // out q4 + add v12.8h, v12.8h, v18.8h // - (p1 + q3) + (q6 + q6) + urshr v11.8h, v12.8h, #4 // out q5 + bif v6.16b, v24.16b, v15.16b // out q0 + bif v7.16b, v25.16b, v15.16b // out q1 + bif v8.16b, v26.16b, v15.16b // out q2 + bif v9.16b, v27.16b, v15.16b // out q3 + bif v10.16b, v28.16b, v15.16b // out q4 + bif v11.16b, v29.16b, v15.16b // out q5 +.endif + + mov x14, #0 + ret +.if \wd == 16 +7: + // Return to a shorter epilogue, writing only the inner 6 pixels + mov x14, #(1 << 6) + ret +.endif +.if \wd >= 8 +8: + // Return to a shorter epilogue, writing only the inner 4 pixels + mov x14, #(1 << 4) + ret +.endif +endfunc +.endm + +loop_filter 16 +loop_filter 8 +loop_filter 6 +loop_filter 4 + +.macro lpf_8_wd16 + bl lpf_8_wd16_neon + cbz x14, 1f + tbnz x14, #6, 7f + tbnz x14, #4, 8f + ret x15 +1: +.endm + +.macro lpf_8_wd8 + bl lpf_8_wd8_neon + cbz x14, 1f + tbnz x14, #4, 8f + ret x15 +1: +.endm + +.macro lpf_8_wd6 + bl lpf_8_wd6_neon + cbz x14, 1f + ret x15 +1: +.endm + +.macro lpf_8_wd4 + bl lpf_8_wd4_neon + cbz x14, 1f + ret x15 +1: +.endm + +function lpf_v_4_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + + lpf_8_wd4 + + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_4_8_neon + mov x15, x30 + sub x16, x0, #4 + add x0, x16, x1, lsl #2 + ld1 {v22.d}[0], [x16], x1 + ld1 {v22.d}[1], [x0], x1 + ld1 {v23.d}[0], [x16], x1 + ld1 {v23.d}[1], [x0], x1 + ld1 {v24.d}[0], [x16], x1 + ld1 {v24.d}[1], [x0], x1 + ld1 {v25.d}[0], [x16], x1 + ld1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd4 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +endfunc + +function lpf_v_6_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + + lpf_8_wd6 + + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_6_8_neon + mov x15, x30 + sub x16, x0, #8 + add x0, x16, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + add x0, x0, #8 + + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd6 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +endfunc + +function lpf_v_8_8_neon + mov x15, x30 + sub x16, x0, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 // p3 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v26.8h}, [x0], x1 // q2 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v27.8h}, [x0], x1 // q3 + sub x0, x0, x1, lsl #2 + + lpf_8_wd8 + + sub x16, x0, x1, lsl #1 + sub x16, x16, x1 + st1 {v21.8h}, [x16], x1 // p2 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v25.8h}, [x0], x1 // q1 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + ret x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_8_8_neon + mov x15, x30 + sub x16, x0, #8 + add x0, x16, x1, lsl #2 + ld1 {v20.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + add x0, x0, #8 + + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + + lpf_8_wd8 + + sub x16, x0, x1, lsl #3 + sub x16, x16, #8 + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v20.8h}, [x16], x1 + st1 {v24.8h}, [x0], x1 + st1 {v21.8h}, [x16], x1 + st1 {v25.8h}, [x0], x1 + st1 {v22.8h}, [x16], x1 + st1 {v26.8h}, [x0], x1 + st1 {v23.8h}, [x16], x1 + st1 {v27.8h}, [x0], x1 + add x0, x0, #8 + ret x15 +8: + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +endfunc + +function lpf_v_16_8_neon + mov x15, x30 + + sub x16, x0, x1, lsl #3 + add x16, x16, x1 + ld1 {v17.8h}, [x16], x1 // p6 + ld1 {v24.8h}, [x0], x1 // q0 + ld1 {v18.8h}, [x16], x1 // p5 + ld1 {v25.8h}, [x0], x1 // q1 + ld1 {v19.8h}, [x16], x1 // p4 + ld1 {v26.8h}, [x0], x1 // q2 + ld1 {v20.8h}, [x16], x1 // p3 + ld1 {v27.8h}, [x0], x1 // q3 + ld1 {v21.8h}, [x16], x1 // p2 + ld1 {v28.8h}, [x0], x1 // q4 + ld1 {v22.8h}, [x16], x1 // p1 + ld1 {v29.8h}, [x0], x1 // q5 + ld1 {v23.8h}, [x16], x1 // p0 + ld1 {v30.8h}, [x0], x1 // q6 + sub x0, x0, x1, lsl #3 + add x0, x0, x1 + + lpf_8_wd16 + + sub x16, x0, x1, lsl #2 + sub x16, x16, x1, lsl #1 + st1 {v0.8h}, [x16], x1 // p5 + st1 {v6.8h}, [x0], x1 // q0 + st1 {v1.8h}, [x16], x1 // p4 + st1 {v7.8h}, [x0], x1 // q1 + st1 {v2.8h}, [x16], x1 // p3 + st1 {v8.8h}, [x0], x1 // q2 + st1 {v3.8h}, [x16], x1 // p2 + st1 {v9.8h}, [x0], x1 // q3 + st1 {v4.8h}, [x16], x1 // p1 + st1 {v10.8h}, [x0], x1 // q4 + st1 {v5.8h}, [x16], x1 // p0 + st1 {v11.8h}, [x0], x1 // q5 + sub x0, x0, x1, lsl #2 + sub x0, x0, x1, lsl #1 + ret x15 +7: + sub x16, x0, x1 + sub x16, x16, x1, lsl #1 + st1 {v21.8h}, [x16], x1 // p2 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v25.8h}, [x0], x1 // q1 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v26.8h}, [x0], x1 // q2 + sub x0, x0, x1, lsl #1 + sub x0, x0, x1 + ret x15 + +8: + sub x16, x0, x1, lsl #1 + st1 {v22.8h}, [x16], x1 // p1 + st1 {v24.8h}, [x0], x1 // q0 + st1 {v23.8h}, [x16], x1 // p0 + st1 {v25.8h}, [x0], x1 // q1 + sub x0, x0, x1, lsl #1 + ret x15 +endfunc + +function lpf_h_16_8_neon + mov x15, x30 + sub x16, x0, #16 + ld1 {v16.8h}, [x16], x1 + ld1 {v24.8h}, [x0], x1 + ld1 {v17.8h}, [x16], x1 + ld1 {v25.8h}, [x0], x1 + ld1 {v18.8h}, [x16], x1 + ld1 {v26.8h}, [x0], x1 + ld1 {v19.8h}, [x16], x1 + ld1 {v27.8h}, [x0], x1 + ld1 {v20.8h}, [x16], x1 + ld1 {v28.8h}, [x0], x1 + ld1 {v21.8h}, [x16], x1 + ld1 {v29.8h}, [x0], x1 + ld1 {v22.8h}, [x16], x1 + ld1 {v30.8h}, [x0], x1 + ld1 {v23.8h}, [x16], x1 + ld1 {v31.8h}, [x0], x1 + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v0, v1 + transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v0, v1 + + lpf_8_wd16 + + sub x0, x0, x1, lsl #3 + sub x16, x0, #16 + + transpose_8x8h v16, v17, v0, v1, v2, v3, v4, v5, v18, v19 + transpose_8x8h v6, v7, v8, v9, v10, v11, v30, v31, v18, v19 + + st1 {v16.8h}, [x16], x1 + st1 {v6.8h}, [x0], x1 + st1 {v17.8h}, [x16], x1 + st1 {v7.8h}, [x0], x1 + st1 {v0.8h}, [x16], x1 + st1 {v8.8h}, [x0], x1 + st1 {v1.8h}, [x16], x1 + st1 {v9.8h}, [x0], x1 + st1 {v2.8h}, [x16], x1 + st1 {v10.8h}, [x0], x1 + st1 {v3.8h}, [x16], x1 + st1 {v11.8h}, [x0], x1 + st1 {v4.8h}, [x16], x1 + st1 {v30.8h}, [x0], x1 + st1 {v5.8h}, [x16], x1 + st1 {v31.8h}, [x0], x1 + ret x15 + +7: + sub x16, x0, x1, lsl #3 + sub x16, x16, #8 + transpose_8x8h v20, v21, v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v20.8h}, [x16], x1 + st1 {v24.8h}, [x0], x1 + st1 {v21.8h}, [x16], x1 + st1 {v25.8h}, [x0], x1 + st1 {v22.8h}, [x16], x1 + st1 {v26.8h}, [x0], x1 + st1 {v23.8h}, [x16], x1 + st1 {v27.8h}, [x0], x1 + add x0, x0, #8 + ret x15 +8: + sub x16, x0, x1, lsl #3 + sub x16, x16, #4 + transpose_4x8h v22, v23, v24, v25, v26, v27, v28, v29 + add x0, x16, x1, lsl #2 + + st1 {v22.d}[0], [x16], x1 + st1 {v22.d}[1], [x0], x1 + st1 {v23.d}[0], [x16], x1 + st1 {v23.d}[1], [x0], x1 + st1 {v24.d}[0], [x16], x1 + st1 {v24.d}[1], [x0], x1 + st1 {v25.d}[0], [x16], x1 + st1 {v25.d}[1], [x0], x1 + add x0, x0, #4 + ret x15 +endfunc + +// void dav1d_lpf_v_sb_y_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint32_t *const vmask, +// const uint8_t (*l)[4], ptrdiff_t b4_stride, +// const Av1FilterLUT *lut, const int w, +// const int bitdepth_max) + +.macro lpf_func dir, type +function lpf_\dir\()_sb_\type\()_16bpc_neon, export=1 + mov x11, x30 + mov w8, w7 // bitdepth_max + clz w9, w8 + mov w10, #24 + sub w9, w10, w9 // bitdepth_min_8 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + ldp w6, w7, [x2] // vmask[0], vmask[1] +.ifc \type, y + ldr w2, [x2, #8] // vmask[2] +.endif + add x5, x5, #128 // Move to sharp part of lut +.ifc \type, y + orr w7, w7, w2 // vmask[1] |= vmask[2] +.endif +.ifc \dir, v + sub x4, x3, x4, lsl #2 +.else + sub x3, x3, #4 + lsl x4, x4, #2 +.endif + orr w6, w6, w7 // vmask[0] |= vmask[1] + +1: + tst w6, #0x03 +.ifc \dir, v + ld1 {v0.8b}, [x4], #8 + ld1 {v1.8b}, [x3], #8 +.else + ld2 {v0.s,v1.s}[0], [x3], x4 + ld2 {v0.s,v1.s}[1], [x3], x4 +.endif + b.eq 7f // if (!(vm & bits)) continue; + + ld1r {v5.8b}, [x5] // sharp[0] + add x5, x5, #8 + movi v2.2s, #0xff + dup v13.2s, w6 // vmask[0] + dup v31.8h, w9 // bitdepth_min_8 + + and v0.8b, v0.8b, v2.8b // Keep only lowest byte in each 32 bit word + and v1.8b, v1.8b, v2.8b + cmtst v3.8b, v1.8b, v2.8b // Check for nonzero values in l[0][0] + movi v4.8b, #1 + ld1r {v6.8b}, [x5] // sharp[1] + sub x5, x5, #8 + bif v1.8b, v0.8b, v3.8b // if (!l[0][0]) L = l[offset][0] + cmtst v2.2s, v1.2s, v2.2s // L != 0 + mul v1.2s, v1.2s, v4.2s // L +.ifc \type, y + dup v15.2s, w2 // vmask[2] +.endif + dup v14.2s, w7 // vmask[1] + mov x16, v2.d[0] + cmp x16, #0 + b.eq 7f // if (!L) continue; + neg v5.8b, v5.8b // -sharp[0] + movrel x16, word_12 + ushr v12.8b, v1.8b, #4 // H + ld1 {v16.2s}, [x16] + sshl v3.8b, v1.8b, v5.8b // L >> sharp[0] +.ifc \type, y + cmtst v15.2s, v15.2s, v16.2s // if (vmask[2] & bits) +.endif + movi v7.8b, #2 + umin v3.8b, v3.8b, v6.8b // imin(L >> sharp[0], sharp[1]) + add v0.8b, v1.8b, v7.8b // L + 2 + umax v11.8b, v3.8b, v4.8b // imax(imin(), 1) = limit = I + add v0.8b, v0.8b, v0.8b // 2*(L + 2) + cmtst v14.2s, v14.2s, v16.2s // if (vmask[1] & bits) + uxtl v12.8h, v12.8b + add v10.8b, v0.8b, v11.8b // 2*(L + 2) + limit = E + cmtst v13.2s, v13.2s, v16.2s // if (vmask[0] & bits) + uxtl v11.8h, v11.8b + uxtl v10.8h, v10.8b + and v13.8b, v13.8b, v2.8b // vmask[0] &= L != 0 + sxtl v14.8h, v14.8b + sxtl v13.8h, v13.8b +.ifc \type, y + sxtl v15.8h, v15.8b +.endif + ushl v12.8h, v12.8h, v31.8h + ushl v11.8h, v11.8h, v31.8h + ushl v10.8h, v10.8h, v31.8h + +.ifc \type, y + tst w2, #0x03 + b.eq 2f + // wd16 + bl lpf_\dir\()_16_8_neon + b 8f +2: +.endif + tst w7, #0x03 + b.eq 3f +.ifc \type, y + // wd8 + bl lpf_\dir\()_8_8_neon +.else + // wd6 + bl lpf_\dir\()_6_8_neon +.endif + b 8f +3: + // wd4 + bl lpf_\dir\()_4_8_neon +.ifc \dir, h + b 8f +7: + // For dir h, the functions above increment x0. + // If the whole function is skipped, increment it here instead. + add x0, x0, x1, lsl #3 +.else +7: +.endif +8: + lsr w6, w6, #2 // vmask[0] >>= 2 + lsr w7, w7, #2 // vmask[1] >>= 2 +.ifc \type, y + lsr w2, w2, #2 // vmask[2] >>= 2 +.endif +.ifc \dir, v + add x0, x0, #16 +.else + // For dir h, x0 is returned incremented +.endif + cbnz w6, 1b + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + ret x11 +endfunc +.endm + +lpf_func v, y +lpf_func h, y +lpf_func v, uv +lpf_func h, uv + +const word_12 + .word 1, 2 +endconst diff --git a/third_party/dav1d/src/arm/64/looprestoration.S b/third_party/dav1d/src/arm/64/looprestoration.S new file mode 100644 index 0000000000..a598b72b03 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration.S @@ -0,0 +1,1336 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +const right_ext_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +right_ext_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void dav1d_wiener_filter7_8bpc_neon(pixel *p, const ptrdiff_t stride, +// const pixel (*left)[4], const pixel *lpf, +// const int w, int h, +// const int16_t filter[2][8], +// const enum LrEdgeFlags edges); +function wiener_filter7_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-16]! + mov x29, sp + ld1 {v0.8h, v1.8h}, [x6] + tst w7, #4 // LR_HAVE_TOP + sub_sp 384*2*6 + + mov w17, #(1 << 14) - (1 << 2) + dup v30.8h, w17 + movi v31.8h, #8, lsl #8 + + // x9 - t6 + // x10 - t5 + // x11 - t4 + // x12 - t3 + // x13 - t2 + // x14 - t1 + // x15 - t0 + mov x14, sp // t1 + b.eq L(no_top_7) + + mov x16, x2 // backup left + mov x2, #0 + bl wiener_filter7_h_8bpc_neon + add x3, x3, x1 // lpf += stride + mov x9, x14 // t6 + mov x10, x14 // t5 + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_8bpc_neon + add x3, x3, x1, lsl #2 + add x3, x3, x1 // lpf += stride*5 + mov x11, x14 // t4 + add x14, x14, #384*2 // t1 += 384*2 + mov x2, x16 // left + mov x16, x3 // backup lpf + mov x3, x0 // lpf = p + bl wiener_filter7_h_8bpc_neon + subs w5, w5, #1 // h-- + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_8bpc_neon + mov x13, x14 // t2 + subs w5, w5, #1 // h-- + b.eq L(v2_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x3, x3, x1 // src += stride + +L(main_7): + add x15, x14, #384*2 // t0 = t1 + 384*2 +L(main_loop_7): + bl wiener_filter7_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_loop_7) + tst w7, #8 // LR_HAVE_BOTTOM + b.eq L(v3_7) + + mov x3, x16 // restore lpf + mov x2, #0 // left = NULL + bl wiener_filter7_hv_8bpc_neon + bl wiener_filter7_hv_8bpc_neon +L(v1_7): + bl wiener_filter7_v_8bpc_neon + + mov sp, x29 + ldp x29, x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(no_top_7): + add x3, x3, x1, lsl #2 + add x16, x3, x1, lsl #1 // lpf += stride*6, backup + mov x3, x0 // lpf = p + + bl wiener_filter7_h_8bpc_neon + subs w5, w5, #1 // h-- + mov x9, x14 // t6 + mov x10, x14 // t5 + mov x11, x14 // t4 + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_8bpc_neon + subs w5, w5, #1 // h-- + mov x13, x14 // t2 + b.eq L(v2_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x3, x3, x1 // src += stride + add x15, x14, #384*2 // t0 = t1 + 384*2 + bl wiener_filter7_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x15, x15, #384*2*4 // t0 += 384*2*4 + bl wiener_filter7_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_7) +L(v3_7): + bl wiener_filter7_v_8bpc_neon +L(v2_7): + bl wiener_filter7_v_8bpc_neon + b L(v1_7) +endfunc + + +function wiener_filter7_h_8bpc_neon + stp x3, x4, [sp, #-32]! + str x14, [sp, #16] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #3 + ld1 {v3.16b}, [x3], #16 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v3.16b}, [x3], #16 + ld1 {v2.s}[3], [x2], #4 + // Move x3 back to account for the last 3 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #3 + ext v3.16b, v2.16b, v3.16b, #13 + b 2f + +1: + ld1 {v3.16b}, [x3], #16 + // !LR_HAVE_LEFT, fill v2 with the leftmost byte + // and shift v3 to have 3x the first byte at the front. + dup v2.16b, v3.b[0] + // Move x3 back to account for the last 3 bytes we loaded before, + // which we shifted out. + sub x3, x3, #3 + ext v3.16b, v2.16b, v3.16b, #13 + +2: + ld1 {v4.8b}, [x3], #8 + uxtl v2.8h, v3.8b + uxtl2 v3.8h, v3.16b + uxtl v4.8h, v4.8b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #19 + b.ge 4f // If w >= 19, all used input pixels are valid + + // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel. + sub w17, w4, #22 + // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -6 + ldr b28, [x3, w17, sxtw] + sub x6, x6, w4, uxtw #1 + dup v28.8h, v28.h[0] + ld1 {v25.16b, v26.16b, v27.16b}, [x6] + + bit v2.16b, v28.16b, v25.16b + bit v3.16b, v28.16b, v26.16b + bit v4.16b, v28.16b, v27.16b + +4: // Loop horizontally + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + ext v17.16b, v2.16b, v3.16b, #4 + ext v19.16b, v2.16b, v3.16b, #8 + ext v16.16b, v2.16b, v3.16b, #2 + ext v20.16b, v2.16b, v3.16b, #10 + ext v21.16b, v2.16b, v3.16b, #12 + ext v18.16b, v2.16b, v3.16b, #6 + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v2.8h + shl v22.8h, v18.8h, #7 + mul v6.8h, v18.8h, v0.h[3] + mla v6.8h, v19.8h, v0.h[4] + mla v6.8h, v20.8h, v0.h[5] + mla v6.8h, v21.8h, v0.h[6] + + ext v17.16b, v3.16b, v4.16b, #4 + ext v19.16b, v3.16b, v4.16b, #8 + ext v16.16b, v3.16b, v4.16b, #2 + ext v20.16b, v3.16b, v4.16b, #10 + ext v21.16b, v3.16b, v4.16b, #12 + ext v18.16b, v3.16b, v4.16b, #6 + + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v3.8h + shl v23.8h, v18.8h, #7 + mul v7.8h, v18.8h, v0.h[3] + mla v7.8h, v19.8h, v0.h[4] + mla v7.8h, v20.8h, v0.h[5] + mla v7.8h, v21.8h, v0.h[6] + + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + sqadd v6.8h, v6.8h, v22.8h + sqadd v7.8h, v7.8h, v23.8h + sshr v6.8h, v6.8h, #3 + sshr v7.8h, v7.8h, #3 + add v6.8h, v6.8h, v31.8h + add v7.8h, v7.8h, v31.8h + + subs w4, w4, #16 + + st1 {v6.8h, v7.8h}, [x14], #32 + + b.le 0f + mov v2.16b, v4.16b + ld1 {v4.16b}, [x3], #16 + tst w7, #2 // LR_HAVE_RIGHT + uxtl v3.8h, v4.8b + uxtl2 v4.8h, v4.16b + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldr x14, [sp, #16] + ldp x3, x4, [sp], #32 + ret +endfunc + +function wiener_filter7_v_8bpc_neon + // Backing up/restoring registers shifted, so that x9 gets the value + // of x10, etc, afterwards. + stp x10, x11, [sp, #-64]! + stp x12, x13, [sp, #16] + stp x14, x14, [sp, #32] + stp x0, x4, [sp, #48] +1: + ld1 {v20.8h, v21.8h}, [x11], #32 + ld1 {v24.8h, v25.8h}, [x13], #32 + + ld1 {v18.8h, v19.8h}, [x10], #32 + add v24.8h, v24.8h, v20.8h + ld1 {v26.8h, v27.8h}, [x14], #32 + + ld1 {v16.8h, v17.8h}, [x9], #32 + add v28.8h, v26.8h, v18.8h + ld1 {v22.8h, v23.8h}, [x12], #32 + + add v16.8h, v26.8h, v16.8h + add v25.8h, v25.8h, v21.8h + + smull v2.4s, v22.4h, v1.h[3] + smlal v2.4s, v24.4h, v1.h[4] + smlal v2.4s, v28.4h, v1.h[5] + smlal v2.4s, v16.4h, v1.h[6] + add v29.8h, v27.8h, v19.8h + smull2 v3.4s, v22.8h, v1.h[3] + smlal2 v3.4s, v24.8h, v1.h[4] + smlal2 v3.4s, v28.8h, v1.h[5] + smlal2 v3.4s, v16.8h, v1.h[6] + add v17.8h, v27.8h, v17.8h + smull v4.4s, v23.4h, v1.h[3] + smlal v4.4s, v25.4h, v1.h[4] + smlal v4.4s, v29.4h, v1.h[5] + smlal v4.4s, v17.4h, v1.h[6] + smull2 v5.4s, v23.8h, v1.h[3] + smlal2 v5.4s, v25.8h, v1.h[4] + smlal2 v5.4s, v29.8h, v1.h[5] + smlal2 v5.4s, v17.8h, v1.h[6] + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + sqrshrun v3.4h, v4.4s, #11 + sqrshrun2 v3.8h, v5.4s, #11 + sqxtun v2.8b, v2.8h + sqxtun2 v2.16b, v3.8h + subs w4, w4, #16 + st1 {v2.16b}, [x0], #16 + b.gt 1b + + ldp x0, x4, [sp, #48] + ldp x13, x14, [sp, #32] + ldp x11, x12, [sp, #16] + ldp x9, x10, [sp], #64 + + add x0, x0, x1 + ret +endfunc + +function wiener_filter7_hv_8bpc_neon + // Backing up/restoring registers shifted, so that x9 gets the value + // of x10, etc, and x15==x9, afterwards. + stp x10, x11, [sp, #-80]! + stp x12, x13, [sp, #16] + stp x14, x15, [sp, #32] + stp x10, x0, [sp, #48] + stp x3, x4, [sp, #64] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #3 + ld1 {v3.16b}, [x3], #16 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v3.16b}, [x3], #16 + ld1 {v2.s}[3], [x2], #4 + // Move x3 back to account for the last 3 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #3 + ext v3.16b, v2.16b, v3.16b, #13 + b 2f +1: + ld1 {v3.16b}, [x3], #16 + // !LR_HAVE_LEFT, fill v2 with the leftmost byte + // and shift v3 to have 3x the first byte at the front. + dup v2.16b, v3.b[0] + // Move x3 back to account for the last 3 bytes we loaded before, + // which we shifted out. + sub x3, x3, #3 + ext v3.16b, v2.16b, v3.16b, #13 + +2: + ld1 {v4.8b}, [x3], #8 + uxtl v2.8h, v3.8b + uxtl2 v3.8h, v3.16b + uxtl v4.8h, v4.8b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #19 + b.ge 4f // If w >= 19, all used input pixels are valid + + // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel. + sub w17, w4, #22 + // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -6 + ldr b28, [x3, w17, sxtw] + sub x6, x6, w4, uxtw #1 + dup v28.8h, v28.h[0] + ld1 {v25.16b, v26.16b, v27.16b}, [x6] + + bit v2.16b, v28.16b, v25.16b + bit v3.16b, v28.16b, v26.16b + bit v4.16b, v28.16b, v27.16b + +4: // Loop horizontally + ext v17.16b, v2.16b, v3.16b, #4 + ext v19.16b, v2.16b, v3.16b, #8 + ext v16.16b, v2.16b, v3.16b, #2 + ext v20.16b, v2.16b, v3.16b, #10 + ext v21.16b, v2.16b, v3.16b, #12 + ext v18.16b, v2.16b, v3.16b, #6 + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v2.8h + shl v22.8h, v18.8h, #7 + mul v6.8h, v18.8h, v0.h[3] + mla v6.8h, v19.8h, v0.h[4] + mla v6.8h, v20.8h, v0.h[5] + mla v6.8h, v21.8h, v0.h[6] + + ext v17.16b, v3.16b, v4.16b, #4 + ext v19.16b, v3.16b, v4.16b, #8 + ext v16.16b, v3.16b, v4.16b, #2 + ext v20.16b, v3.16b, v4.16b, #10 + ext v21.16b, v3.16b, v4.16b, #12 + ext v18.16b, v3.16b, v4.16b, #6 + + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v3.8h + shl v23.8h, v18.8h, #7 + mul v7.8h, v18.8h, v0.h[3] + mla v7.8h, v19.8h, v0.h[4] + mla v7.8h, v20.8h, v0.h[5] + mla v7.8h, v21.8h, v0.h[6] + + ld1 {v20.8h, v21.8h}, [x11], #32 + + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + ld1 {v26.8h, v27.8h}, [x13], #32 + sqadd v6.8h, v6.8h, v22.8h + sqadd v7.8h, v7.8h, v23.8h + ld1 {v18.8h, v19.8h}, [x10], #32 + sshr v6.8h, v6.8h, #3 + sshr v7.8h, v7.8h, #3 + ld1 {v28.8h, v29.8h}, [x14], #32 + add v6.8h, v6.8h, v31.8h + add v7.8h, v7.8h, v31.8h + + ld1 {v16.8h, v17.8h}, [x9], #32 + add v26.8h, v20.8h, v26.8h + + ld1 {v24.8h, v25.8h}, [x12], #32 + add v28.8h, v18.8h, v28.8h + + add v16.8h, v16.8h, v6.8h + add v27.8h, v21.8h, v27.8h + + smull v18.4s, v24.4h, v1.h[3] + smlal v18.4s, v26.4h, v1.h[4] + smlal v18.4s, v28.4h, v1.h[5] + smlal v18.4s, v16.4h, v1.h[6] + add v29.8h, v19.8h, v29.8h + smull2 v19.4s, v24.8h, v1.h[3] + smlal2 v19.4s, v26.8h, v1.h[4] + smlal2 v19.4s, v28.8h, v1.h[5] + smlal2 v19.4s, v16.8h, v1.h[6] + add v17.8h, v17.8h, v7.8h + smull v20.4s, v25.4h, v1.h[3] + smlal v20.4s, v27.4h, v1.h[4] + smlal v20.4s, v29.4h, v1.h[5] + smlal v20.4s, v17.4h, v1.h[6] + smull2 v21.4s, v25.8h, v1.h[3] + smlal2 v21.4s, v27.8h, v1.h[4] + smlal2 v21.4s, v29.8h, v1.h[5] + smlal2 v21.4s, v17.8h, v1.h[6] + sqrshrun v18.4h, v18.4s, #11 + sqrshrun2 v18.8h, v19.4s, #11 + sqrshrun v19.4h, v20.4s, #11 + sqrshrun2 v19.8h, v21.4s, #11 + st1 {v6.8h, v7.8h}, [x15], #32 + sqxtun v18.8b, v18.8h + sqxtun2 v18.16b, v19.8h + subs w4, w4, #16 + + st1 {v18.16b}, [x0], #16 + + b.le 0f + mov v2.16b, v4.16b + ld1 {v4.16b}, [x3], #16 + tst w7, #2 // LR_HAVE_RIGHT + uxtl v3.8h, v4.8b + uxtl2 v4.8h, v4.16b + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldp x3, x4, [sp, #64] + ldp x15, x0, [sp, #48] + ldp x13, x14, [sp, #32] + ldp x11, x12, [sp, #16] + ldp x9, x10, [sp], #80 + + add x3, x3, x1 + add x0, x0, x1 + + ret +endfunc + +// void dav1d_wiener_filter5_8bpc_neon(pixel *p, const ptrdiff_t stride, +// const pixel (*left)[4], const pixel *lpf, +// const int w, int h, +// const int16_t filter[2][8], +// const enum LrEdgeFlags edges); +function wiener_filter5_8bpc_neon, export=1 + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-16]! + mov x29, sp + ld1 {v0.8h, v1.8h}, [x6] + tst w7, #4 // LR_HAVE_TOP + sub_sp 384*2*4 + + mov w17, #(1 << 14) - (1 << 2) + dup v30.8h, w17 + movi v31.8h, #8, lsl #8 + + // x11 - t4 + // x12 - t3 + // x13 - t2 + // x14 - t1 + // x15 - t0 + mov x14, sp // t1 + b.eq L(no_top_5) + + mov x16, x2 // backup left + mov x2, #0 + bl wiener_filter5_h_8bpc_neon + add x3, x3, x1 // lpf += stride + mov x11, x14 // t4 + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_8bpc_neon + add x3, x3, x1, lsl #2 + add x3, x3, x1 // lpf += stride*5 + mov x12, x14 // t3 + add x14, x14, #384*2 // t1 += 384*2 + mov x2, x16 // left + mov x16, x3 // backup lpf + mov x3, x0 // lpf = p + bl wiener_filter5_h_8bpc_neon + subs w5, w5, #1 // h-- + mov x13, x14 // t2 + b.eq L(v1_5) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x3, x3, x1 // src += stride + +L(main_5): + mov x15, x11 // t0 = t4 +L(main_loop_5): + bl wiener_filter5_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_loop_5) + tst w7, #8 // LR_HAVE_BOTTOM + b.eq L(v2_5) + + mov x3, x16 // restore lpf + mov x2, #0 // left = NULL + bl wiener_filter5_hv_8bpc_neon + bl wiener_filter5_hv_8bpc_neon +L(end_5): + + mov sp, x29 + ldp x29, x30, [sp], #16 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(no_top_5): + add x3, x3, x1, lsl #2 + add x16, x3, x1, lsl #1 // lpf += stride*6, backup + mov x3, x0 // lpf = p + + bl wiener_filter5_h_8bpc_neon + subs w5, w5, #1 // h-- + mov x11, x14 // t4 + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_5) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x3, x3, x1 // src += stride + add x15, x14, #384*2 // t0 = t1 + 384*2 + bl wiener_filter5_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x15, x15, #384*2*3 // t0 += 384*2*3 + bl wiener_filter5_hv_8bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_5) +L(v2_5): + bl wiener_filter5_v_8bpc_neon + add x0, x0, x1 + mov x11, x12 + mov x12, x13 + mov x13, x14 +L(v1_5): + bl wiener_filter5_v_8bpc_neon + b L(end_5) +endfunc + + +function wiener_filter5_h_8bpc_neon + stp x3, x4, [sp, #-32]! + str x14, [sp, #16] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #2 + ld1 {v3.16b}, [x3], #16 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v3.16b}, [x3], #16 + ld1 {v2.s}[3], [x2], #4 + // Move x3 back to account for the last 2 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #2 + ext v3.16b, v2.16b, v3.16b, #14 + b 2f + +1: + ld1 {v3.16b}, [x3], #16 + // !LR_HAVE_LEFT, fill v2 with the leftmost byte + // and shift v3 to have 3x the first byte at the front. + dup v2.16b, v3.b[0] + // Move x3 back to account for the last 2 bytes we loaded before, + // which we shifted out. + sub x3, x3, #2 + ext v3.16b, v2.16b, v3.16b, #14 + +2: + ld1 {v4.8b}, [x3], #8 + uxtl v2.8h, v3.8b + uxtl2 v3.8h, v3.16b + uxtl v4.8h, v4.8b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #18 + b.ge 4f // If w >= 18, all used input pixels are valid + + // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel. + sub w17, w4, #23 + // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -4 + ldr b28, [x3, w17, sxtw] + sub x6, x6, w4, uxtw #1 + dup v28.8h, v28.h[0] + ld1 {v25.16b, v26.16b, v27.16b}, [x6] + + bit v2.16b, v28.16b, v25.16b + bit v3.16b, v28.16b, v26.16b + bit v4.16b, v28.16b, v27.16b + +4: // Loop horizontally + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + ext v16.16b, v2.16b, v3.16b, #2 + ext v18.16b, v2.16b, v3.16b, #6 + ext v19.16b, v2.16b, v3.16b, #8 + ext v17.16b, v2.16b, v3.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v2.8h + shl v22.8h, v17.8h, #7 + mul v6.8h, v17.8h, v0.h[3] + mla v6.8h, v18.8h, v0.h[4] + mla v6.8h, v19.8h, v0.h[5] + + ext v16.16b, v3.16b, v4.16b, #2 + ext v18.16b, v3.16b, v4.16b, #6 + ext v19.16b, v3.16b, v4.16b, #8 + ext v17.16b, v3.16b, v4.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v3.8h + shl v23.8h, v17.8h, #7 + mul v7.8h, v17.8h, v0.h[3] + mla v7.8h, v18.8h, v0.h[4] + mla v7.8h, v19.8h, v0.h[5] + + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + sqadd v6.8h, v6.8h, v22.8h + sqadd v7.8h, v7.8h, v23.8h + sshr v6.8h, v6.8h, #3 + sshr v7.8h, v7.8h, #3 + add v6.8h, v6.8h, v31.8h + add v7.8h, v7.8h, v31.8h + + subs w4, w4, #16 + + st1 {v6.8h, v7.8h}, [x14], #32 + + b.le 0f + mov v2.16b, v4.16b + ld1 {v4.16b}, [x3], #16 + tst w7, #2 // LR_HAVE_RIGHT + uxtl v3.8h, v4.8b + uxtl2 v4.8h, v4.16b + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldr x14, [sp, #16] + ldp x3, x4, [sp], #32 + ret +endfunc + +function wiener_filter5_v_8bpc_neon + stp x11, x12, [sp, #-48]! + stp x13, x14, [sp, #16] + stp x0, x4, [sp, #32] +1: + ld1 {v18.8h, v19.8h}, [x12], #32 + ld1 {v22.8h, v23.8h}, [x14], #32 + ld1 {v16.8h, v17.8h}, [x11], #32 + + add v24.8h, v22.8h, v18.8h + ld1 {v20.8h, v21.8h}, [x13], #32 + add v16.8h, v22.8h, v16.8h + add v25.8h, v23.8h, v19.8h + + smull v2.4s, v20.4h, v1.h[3] + smlal v2.4s, v24.4h, v1.h[4] + smlal v2.4s, v16.4h, v1.h[5] + add v17.8h, v23.8h, v17.8h + smull2 v3.4s, v20.8h, v1.h[3] + smlal2 v3.4s, v24.8h, v1.h[4] + smlal2 v3.4s, v16.8h, v1.h[5] + smull v4.4s, v21.4h, v1.h[3] + smlal v4.4s, v25.4h, v1.h[4] + smlal v4.4s, v17.4h, v1.h[5] + smull2 v5.4s, v21.8h, v1.h[3] + smlal2 v5.4s, v25.8h, v1.h[4] + smlal2 v5.4s, v17.8h, v1.h[5] + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + sqrshrun v3.4h, v4.4s, #11 + sqrshrun2 v3.8h, v5.4s, #11 + sqxtun v2.8b, v2.8h + sqxtun2 v2.16b, v3.8h + subs w4, w4, #16 + st1 {v2.16b}, [x0], #16 + b.gt 1b + + ldp x0, x4, [sp, #32] + ldp x13, x14, [sp, #16] + ldp x11, x12, [sp], #48 + + ret +endfunc + +function wiener_filter5_hv_8bpc_neon + // Backing up/restoring registers shifted, so that x11 gets the value + // of x12, etc, and x15==x11, afterwards. + stp x12, x13, [sp, #-64]! + stp x14, x15, [sp, #16] + stp x12, x0, [sp, #32] + stp x3, x4, [sp, #48] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #2 + ld1 {v3.16b}, [x3], #16 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v3.16b}, [x3], #16 + ld1 {v2.s}[3], [x2], #4 + // Move x3 back to account for the last 2 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #2 + ext v3.16b, v2.16b, v3.16b, #14 + b 2f +1: + ld1 {v3.16b}, [x3], #16 + // !LR_HAVE_LEFT, fill v2 with the leftmost byte + // and shift v3 to have 2x the first byte at the front. + dup v2.16b, v3.b[0] + // Move x3 back to account for the last 2 bytes we loaded before, + // which we shifted out. + sub x3, x3, #2 + ext v3.16b, v2.16b, v3.16b, #14 + +2: + ld1 {v4.8b}, [x3], #8 + uxtl v2.8h, v3.8b + uxtl2 v3.8h, v3.16b + uxtl v4.8h, v4.8b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #18 + b.ge 4f // If w >= 18, all used input pixels are valid + + // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+1]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel. + sub w17, w4, #23 + // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -4 + ldr b28, [x3, w17, sxtw] + sub x6, x6, w4, uxtw #1 + dup v28.8h, v28.h[0] + ld1 {v25.16b, v26.16b, v27.16b}, [x6] + + bit v2.16b, v28.16b, v25.16b + bit v3.16b, v28.16b, v26.16b + bit v4.16b, v28.16b, v27.16b + +4: // Loop horizontally + + ext v16.16b, v2.16b, v3.16b, #2 + ext v18.16b, v2.16b, v3.16b, #6 + ext v19.16b, v2.16b, v3.16b, #8 + ext v17.16b, v2.16b, v3.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v2.8h + shl v22.8h, v17.8h, #7 + mul v6.8h, v17.8h, v0.h[3] + mla v6.8h, v18.8h, v0.h[4] + mla v6.8h, v19.8h, v0.h[5] + + ext v16.16b, v3.16b, v4.16b, #2 + ext v18.16b, v3.16b, v4.16b, #6 + ext v19.16b, v3.16b, v4.16b, #8 + ext v17.16b, v3.16b, v4.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v3.8h + shl v23.8h, v17.8h, #7 + mul v7.8h, v17.8h, v0.h[3] + mla v7.8h, v18.8h, v0.h[4] + mla v7.8h, v19.8h, v0.h[5] + + ld1 {v18.8h, v19.8h}, [x12], #32 + + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + ld1 {v24.8h, v25.8h}, [x14], #32 + sqadd v6.8h, v6.8h, v22.8h + sqadd v7.8h, v7.8h, v23.8h + ld1 {v16.8h, v17.8h}, [x11], #32 + sshr v6.8h, v6.8h, #3 + sshr v7.8h, v7.8h, #3 + ld1 {v20.8h, v21.8h}, [x13], #32 + add v6.8h, v6.8h, v31.8h + add v7.8h, v7.8h, v31.8h + + add v24.8h, v24.8h, v18.8h + add v16.8h, v16.8h, v6.8h + + smull v18.4s, v20.4h, v1.h[3] + smlal v18.4s, v24.4h, v1.h[4] + smlal v18.4s, v16.4h, v1.h[5] + add v25.8h, v25.8h, v19.8h + smull2 v19.4s, v20.8h, v1.h[3] + smlal2 v19.4s, v24.8h, v1.h[4] + smlal2 v19.4s, v16.8h, v1.h[5] + add v17.8h, v17.8h, v7.8h + smull v20.4s, v21.4h, v1.h[3] + smlal v20.4s, v25.4h, v1.h[4] + smlal v20.4s, v17.4h, v1.h[5] + smull2 v21.4s, v21.8h, v1.h[3] + smlal2 v21.4s, v25.8h, v1.h[4] + smlal2 v21.4s, v17.8h, v1.h[5] + sqrshrun v18.4h, v18.4s, #11 + sqrshrun2 v18.8h, v19.4s, #11 + sqrshrun v19.4h, v20.4s, #11 + sqrshrun2 v19.8h, v21.4s, #11 + st1 {v6.8h, v7.8h}, [x15], #32 + sqxtun v18.8b, v18.8h + sqxtun2 v18.16b, v19.8h + subs w4, w4, #16 + + st1 {v18.16b}, [x0], #16 + + b.le 0f + mov v2.16b, v4.16b + ld1 {v4.16b}, [x3], #16 + tst w7, #2 // LR_HAVE_RIGHT + uxtl v3.8h, v4.8b + uxtl2 v4.8h, v4.16b + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldp x3, x4, [sp, #48] + ldp x15, x0, [sp, #32] + ldp x13, x14, [sp, #16] + ldp x11, x12, [sp], #64 + + add x3, x3, x1 + add x0, x0, x1 + + ret +endfunc + +#define SUM_STRIDE (384+16) + +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_8bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add w13, w5, #7 + bic w13, w13, #7 + sub x9, x9, w13, uxtw #1 + + // Store the width for the vertical loop + mov w8, w5 + + // Subtract the number of pixels read from the input from the stride + add w13, w13, #8 + sub x4, x4, w13, uxtw + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #2 + sub x12, x12, #2 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 2 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #2 + + +1: // Loop vertically + ld1 {v0.16b}, [x3], #16 + ld1 {v4.16b}, [x12], #16 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v1.s}[3], [x2], #4 + // Move x3/x12 back to account for the last 2 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #2 + sub x12, x12, #2 + ld1 {v5.s}[3], [x2], #4 + ext v0.16b, v1.16b, v0.16b, #14 + ext v4.16b, v5.16b, v4.16b, #14 + b 2f +0: + // !LR_HAVE_LEFT, fill v1 with the leftmost byte + // and shift v0 to have 2x the first byte at the front. + dup v1.16b, v0.b[0] + dup v5.16b, v4.b[0] + // Move x3 back to account for the last 2 bytes we loaded before, + // which we shifted out. + sub x3, x3, #2 + sub x12, x12, #2 + ext v0.16b, v1.16b, v0.16b, #14 + ext v4.16b, v5.16b, v4.16b, #14 + +2: + umull v1.8h, v0.8b, v0.8b + umull2 v2.8h, v0.16b, v0.16b + umull v5.8h, v4.8b, v4.8b + umull2 v6.8h, v4.16b, v4.16b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 2 + 1) + ldr b30, [x3, w13, sxtw] + ldr b31, [x12, w13, sxtw] + // Fill v30/v31 with the right padding pixel + dup v30.16b, v30.b[0] + dup v31.16b, v31.b[0] +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w5, #10 + b.ge 4f // If w >= 10, all used input pixels are valid + + // 1 <= w < 10, w pixels valid in v0. For w=9, this ends up called + // again; it's not strictly needed in those cases (we pad enough here), + // but keeping the code as simple as possible. + + // Insert padding in v0/4.b[w] onwards + movrel x13, right_ext_mask + sub x13, x13, w5, uxtw + ld1 {v29.16b}, [x13] + + bit v0.16b, v30.16b, v29.16b + bit v4.16b, v31.16b, v29.16b + + // Update the precalculated squares + umull v1.8h, v0.8b, v0.8b + umull2 v2.8h, v0.16b, v0.16b + umull v5.8h, v4.8b, v4.8b + umull2 v6.8h, v4.16b, v4.16b + +4: // Loop horizontally + ext v16.16b, v0.16b, v0.16b, #1 + ext v17.16b, v0.16b, v0.16b, #2 + ext v18.16b, v4.16b, v4.16b, #1 + ext v19.16b, v4.16b, v4.16b, #2 + uaddl v3.8h, v0.8b, v16.8b + uaddw v3.8h, v3.8h, v17.8b + uaddl v7.8h, v4.8b, v18.8b + uaddw v7.8h, v7.8h, v19.8b + + ext v20.16b, v1.16b, v2.16b, #2 + ext v21.16b, v1.16b, v2.16b, #4 + ext v22.16b, v5.16b, v6.16b, #2 + ext v23.16b, v5.16b, v6.16b, #4 + + uaddl v26.4s, v1.4h, v20.4h + uaddl2 v27.4s, v1.8h, v20.8h + uaddw v26.4s, v26.4s, v21.4h + uaddw2 v27.4s, v27.4s, v21.8h + + uaddl v28.4s, v5.4h, v22.4h + uaddl2 v29.4s, v5.8h, v22.8h + uaddw v28.4s, v28.4s, v23.4h + uaddw2 v29.4s, v29.4s, v23.8h + + subs w5, w5, #8 + + st1 {v3.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v26.4s,v27.4s}, [x0], #32 + st1 {v28.4s,v29.4s}, [x10], #32 + + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8b}, [x3], #8 + ld1 {v7.8b}, [x12], #8 + mov v1.16b, v2.16b + mov v5.16b, v6.16b + ext v0.16b, v0.16b, v3.16b, #8 + ext v4.16b, v4.16b, v7.16b, #8 + umull v2.8h, v3.8b, v3.8b + umull v6.8h, v7.8b, v7.8b + + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_box5_h_8bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_8bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add w13, w5, #7 + bic w13, w13, #7 + sub x9, x9, w13, uxtw #1 + add w13, w13, #8 + sub x4, x4, w13, uxtw + + // Store the width for the vertical loop + mov w8, w5 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #3 + sub x12, x12, #3 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #3 + +1: // Loop vertically + ld1 {v0.16b}, [x3], #16 + ld1 {v4.16b}, [x12], #16 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v1.s}[3], [x2], #4 + // Move x3/x12 back to account for the last 3 bytes we loaded earlier, + // which we'll shift out. + sub x3, x3, #3 + sub x12, x12, #3 + ld1 {v5.s}[3], [x2], #4 + ext v0.16b, v1.16b, v0.16b, #13 + ext v4.16b, v5.16b, v4.16b, #13 + b 2f +0: + // !LR_HAVE_LEFT, fill v1 with the leftmost byte + // and shift v0 to have 3x the first byte at the front. + dup v1.16b, v0.b[0] + dup v5.16b, v4.b[0] + // Move x3 back to account for the last 3 bytes we loaded before, + // which we shifted out. + sub x3, x3, #3 + sub x12, x12, #3 + ext v0.16b, v1.16b, v0.16b, #13 + ext v4.16b, v5.16b, v4.16b, #13 + +2: + umull v1.8h, v0.8b, v0.8b + umull2 v2.8h, v0.16b, v0.16b + umull v5.8h, v4.8b, v4.8b + umull2 v6.8h, v4.16b, v4.16b + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that byte to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 3 + 1) + ldr b30, [x3, w13, sxtw] + ldr b31, [x12, w13, sxtw] + // Fill v30/v31 with the right padding pixel + dup v30.16b, v30.b[0] + dup v31.16b, v31.b[0] +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w5, #11 + b.ge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+1 pixels valid in v0. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in v0/4.b[w+1] onwards; fuse the +1 into the + // buffer pointer. + movrel x13, right_ext_mask, -1 + sub x13, x13, w5, uxtw + ld1 {v29.16b}, [x13] + + bit v0.16b, v30.16b, v29.16b + bit v4.16b, v31.16b, v29.16b + + // Update the precalculated squares + umull v1.8h, v0.8b, v0.8b + umull2 v2.8h, v0.16b, v0.16b + umull v5.8h, v4.8b, v4.8b + umull2 v6.8h, v4.16b, v4.16b + +4: // Loop horizontally + ext v16.16b, v0.16b, v0.16b, #1 + ext v17.16b, v0.16b, v0.16b, #2 + ext v18.16b, v0.16b, v0.16b, #3 + ext v19.16b, v0.16b, v0.16b, #4 + ext v20.16b, v4.16b, v4.16b, #1 + ext v21.16b, v4.16b, v4.16b, #2 + ext v22.16b, v4.16b, v4.16b, #3 + ext v23.16b, v4.16b, v4.16b, #4 + uaddl v3.8h, v0.8b, v16.8b + uaddl v24.8h, v17.8b, v18.8b + uaddl v7.8h, v4.8b, v20.8b + uaddw v3.8h, v3.8h, v19.8b + uaddl v25.8h, v21.8b, v22.8b + uaddw v7.8h, v7.8h, v23.8b + add v3.8h, v3.8h, v24.8h + add v7.8h, v7.8h, v25.8h + + ext v16.16b, v1.16b, v2.16b, #2 + ext v17.16b, v1.16b, v2.16b, #4 + ext v18.16b, v1.16b, v2.16b, #6 + ext v19.16b, v1.16b, v2.16b, #8 + ext v20.16b, v5.16b, v6.16b, #2 + ext v21.16b, v5.16b, v6.16b, #4 + ext v22.16b, v5.16b, v6.16b, #6 + ext v23.16b, v5.16b, v6.16b, #8 + + uaddl v26.4s, v1.4h, v16.4h + uaddl2 v27.4s, v1.8h, v16.8h + uaddl v16.4s, v17.4h, v18.4h + uaddl2 v17.4s, v17.8h, v18.8h + uaddl v28.4s, v5.4h, v20.4h + uaddl2 v29.4s, v5.8h, v20.8h + uaddw v26.4s, v26.4s, v19.4h + uaddw2 v27.4s, v27.4s, v19.8h + uaddl v20.4s, v21.4h, v22.4h + uaddl2 v21.4s, v21.8h, v22.8h + uaddw v28.4s, v28.4s, v23.4h + uaddw2 v29.4s, v29.4s, v23.8h + add v26.4s, v26.4s, v16.4s + add v27.4s, v27.4s, v17.4s + add v28.4s, v28.4s, v20.4s + add v29.4s, v29.4s, v21.4s + + subs w5, w5, #8 + + st1 {v3.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v26.4s,v27.4s}, [x0], #32 + st1 {v28.4s,v29.4s}, [x10], #32 + + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8b}, [x3], #8 + ld1 {v7.8b}, [x12], #8 + mov v1.16b, v2.16b + mov v5.16b, v6.16b + ext v0.16b, v0.16b, v3.16b, #8 + ext v4.16b, v4.16b, v7.16b, #8 + umull v2.8h, v3.8b, v3.8b + umull v6.8h, v7.8b, v7.8b + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +endfunc + +sgr_funcs 8 diff --git a/third_party/dav1d/src/arm/64/looprestoration16.S b/third_party/dav1d/src/arm/64/looprestoration16.S new file mode 100644 index 0000000000..8954e604cf --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration16.S @@ -0,0 +1,1419 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +const right_ext_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +right_ext_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void dav1d_wiener_filter7_16bpc_neon(pixel *p, const ptrdiff_t p_stride, +// const pixel (*left)[4], const pixel *lpf, +// const int w, int h, +// const int16_t filter[2][8], +// const enum LrEdgeFlags edges, +// const int bitdepth_max); +function wiener_filter7_16bpc_neon, export=1 + ldr w8, [sp] + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-32]! + stp d8, d9, [sp, #16] + mov x29, sp + ld1 {v0.8h, v1.8h}, [x6] + tst w7, #4 // LR_HAVE_TOP + sub_sp 384*2*6 + + dup v28.8h, w8 // bitdepth_max + clz w8, w8 + movi v30.4s, #1 + sub w10, w8, #38 // -(bitdepth + 6) + sub w11, w8, #11 // round_bits_v + sub w8, w8, #25 // -round_bits_h + neg w10, w10 // bitdepth + 6 + neg w11, w11 // -round_bits_v + dup v2.4s, w10 + dup v29.4s, w8 // -round_bits_h + dup v27.4s, w11 // -round_bits_v + movi v31.8h, #0x20, lsl #8 // 1 << 13 = 8192 + ushl v30.4s, v30.4s, v2.4s // 1 << (bitdepth + 6) + + zip1 v0.2d, v0.2d, v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1 + + // x9 - t6 + // x10 - t5 + // x11 - t4 + // x12 - t3 + // x13 - t2 + // x14 - t1 + // x15 - t0 + mov x14, sp // t1 + b.eq L(no_top_7) + + mov x16, x2 // backup left + mov x2, #0 + bl wiener_filter7_h_16bpc_neon + add x3, x3, x1 // lpf += stride + mov x9, x14 // t6 + mov x10, x14 // t5 + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_16bpc_neon + add x3, x3, x1, lsl #2 + add x3, x3, x1 // lpf += stride*5 + mov x11, x14 // t4 + add x14, x14, #384*2 // t1 += 384*2 + mov x2, x16 // left + mov x16, x3 // backup lpf + mov x3, x0 // lpf = p + bl wiener_filter7_h_16bpc_neon + subs w5, w5, #1 // h-- + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_16bpc_neon + mov x13, x14 // t2 + subs w5, w5, #1 // h-- + b.eq L(v2_7) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x3, x3, x1 // src += stride + +L(main_7): + add x15, x14, #384*2 // t0 = t1 + 384*2 +L(main_loop_7): + bl wiener_filter7_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_loop_7) + tst w7, #8 // LR_HAVE_BOTTOM + b.eq L(v3_7) + + mov x3, x16 // restore lpf + mov x2, #0 // left = NULL + bl wiener_filter7_hv_16bpc_neon + bl wiener_filter7_hv_16bpc_neon +L(v1_7): + bl wiener_filter7_v_16bpc_neon + + mov sp, x29 + ldp d8, d9, [sp, #16] + ldp x29, x30, [sp], #32 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(no_top_7): + add x3, x3, x1, lsl #2 + add x16, x3, x1, lsl #1 // lpf += stride*6, backup + mov x3, x0 // lpf = p + + bl wiener_filter7_h_16bpc_neon + subs w5, w5, #1 // h-- + mov x9, x14 // t6 + mov x10, x14 // t5 + mov x11, x14 // t4 + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_7) + add x3, x3, x1 // src += p_stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_16bpc_neon + subs w5, w5, #1 // h-- + mov x13, x14 // t2 + b.eq L(v2_7) + add x3, x3, x1 // src += p_stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter7_h_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x3, x3, x1 // src += p_stride + add x15, x14, #384*2 // t0 = t1 + 384*2 + bl wiener_filter7_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v3_7) + add x15, x15, #384*2*4 // t0 += 384*2*4 + bl wiener_filter7_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_7) +L(v3_7): + bl wiener_filter7_v_16bpc_neon +L(v2_7): + bl wiener_filter7_v_16bpc_neon + b L(v1_7) +endfunc + + +function wiener_filter7_h_16bpc_neon + stp x3, x4, [sp, #-32]! + str x14, [sp, #16] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #6 + ld1 {v2.8h, v3.8h}, [x3], #32 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v2.8h, v3.8h}, [x3], #32 + ld1 {v4.d}[1], [x2], #8 + // Move x3 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #6 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v4.16b, v2.16b, #10 + b 2f + +1: + ld1 {v2.8h, v3.8h}, [x3], #32 + // !LR_HAVE_LEFT, fill v4 with the leftmost pixel + // and shift v3 to have 3x the first pixel at the front. + dup v4.8h, v2.h[0] + // Move x3 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub x3, x3, #6 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v4.16b, v2.16b, #10 + +2: + ld1 {v4.8h}, [x3], #16 + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #19 + b.ge 4f // If w >= 19, all used input pixels are valid + + // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel. + sub w17, w4, #22 + // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -6 + ldr h26, [x3, w17, sxtw #1] + sub x6, x6, w4, uxtw #1 + dup v26.8h, v26.h[0] + ld1 {v23.16b, v24.16b, v25.16b}, [x6] + + bit v2.16b, v26.16b, v23.16b + bit v3.16b, v26.16b, v24.16b + bit v4.16b, v26.16b, v25.16b + +4: // Loop horizontally + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + ext v17.16b, v2.16b, v3.16b, #4 + ext v19.16b, v2.16b, v3.16b, #8 + ext v16.16b, v2.16b, v3.16b, #2 + ext v20.16b, v2.16b, v3.16b, #10 + ext v21.16b, v2.16b, v3.16b, #12 + ext v18.16b, v2.16b, v3.16b, #6 + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v2.8h + smull v6.4s, v18.4h, v0.h[3] + smlal v6.4s, v19.4h, v0.h[2] + smlal v6.4s, v20.4h, v0.h[1] + smlal v6.4s, v21.4h, v0.h[0] + smull2 v7.4s, v18.8h, v0.h[3] + smlal2 v7.4s, v19.8h, v0.h[2] + smlal2 v7.4s, v20.8h, v0.h[1] + smlal2 v7.4s, v21.8h, v0.h[0] + + ext v17.16b, v3.16b, v4.16b, #4 + ext v19.16b, v3.16b, v4.16b, #8 + ext v16.16b, v3.16b, v4.16b, #2 + ext v20.16b, v3.16b, v4.16b, #10 + ext v21.16b, v3.16b, v4.16b, #12 + ext v18.16b, v3.16b, v4.16b, #6 + + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v3.8h + smull v16.4s, v18.4h, v0.h[3] + smlal v16.4s, v19.4h, v0.h[2] + smlal v16.4s, v20.4h, v0.h[1] + smlal v16.4s, v21.4h, v0.h[0] + smull2 v17.4s, v18.8h, v0.h[3] + smlal2 v17.4s, v19.8h, v0.h[2] + smlal2 v17.4s, v20.8h, v0.h[1] + smlal2 v17.4s, v21.8h, v0.h[0] + + mvni v24.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + add v16.4s, v16.4s, v30.4s + add v17.4s, v17.4s, v30.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + srshl v16.4s, v16.4s, v29.4s + srshl v17.4s, v17.4s, v29.4s + sqxtun v6.4h, v6.4s + sqxtun2 v6.8h, v7.4s + sqxtun v7.4h, v16.4s + sqxtun2 v7.8h, v17.4s + umin v6.8h, v6.8h, v24.8h + umin v7.8h, v7.8h, v24.8h + sub v6.8h, v6.8h, v31.8h + sub v7.8h, v7.8h, v31.8h + + subs w4, w4, #16 + + st1 {v6.8h, v7.8h}, [x14], #32 + + b.le 0f + mov v2.16b, v4.16b + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8h, v4.8h}, [x3], #32 + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldr x14, [sp, #16] + ldp x3, x4, [sp], #32 + ret +endfunc + +function wiener_filter7_v_16bpc_neon + // Backing up/restoring registers shifted, so that x9 gets the value + // of x10, etc, afterwards. + stp x10, x11, [sp, #-64]! + stp x12, x13, [sp, #16] + stp x14, x14, [sp, #32] + stp x0, x4, [sp, #48] +1: + ld1 {v16.8h, v17.8h}, [x9], #32 + ld1 {v18.8h, v19.8h}, [x10], #32 + ld1 {v20.8h, v21.8h}, [x11], #32 + ld1 {v22.8h, v23.8h}, [x12], #32 + ld1 {v24.8h, v25.8h}, [x13], #32 + ld1 {v6.8h, v7.8h}, [x14], #32 + + smull v2.4s, v16.4h, v0.h[4] + smlal v2.4s, v18.4h, v0.h[5] + smlal v2.4s, v20.4h, v0.h[6] + smlal v2.4s, v22.4h, v0.h[7] + smlal v2.4s, v24.4h, v0.h[6] + smlal v2.4s, v6.4h, v0.h[5] + smlal v2.4s, v6.4h, v0.h[4] + smull2 v3.4s, v16.8h, v0.h[4] + smlal2 v3.4s, v18.8h, v0.h[5] + smlal2 v3.4s, v20.8h, v0.h[6] + smlal2 v3.4s, v22.8h, v0.h[7] + smlal2 v3.4s, v24.8h, v0.h[6] + smlal2 v3.4s, v6.8h, v0.h[5] + smlal2 v3.4s, v6.8h, v0.h[4] + smull v4.4s, v17.4h, v0.h[4] + smlal v4.4s, v19.4h, v0.h[5] + smlal v4.4s, v21.4h, v0.h[6] + smlal v4.4s, v23.4h, v0.h[7] + smlal v4.4s, v25.4h, v0.h[6] + smlal v4.4s, v7.4h, v0.h[5] + smlal v4.4s, v7.4h, v0.h[4] + smull2 v5.4s, v17.8h, v0.h[4] + smlal2 v5.4s, v19.8h, v0.h[5] + smlal2 v5.4s, v21.8h, v0.h[6] + smlal2 v5.4s, v23.8h, v0.h[7] + smlal2 v5.4s, v25.8h, v0.h[6] + smlal2 v5.4s, v7.8h, v0.h[5] + smlal2 v5.4s, v7.8h, v0.h[4] + srshl v2.4s, v2.4s, v27.4s // -round_bits_v + srshl v3.4s, v3.4s, v27.4s + srshl v4.4s, v4.4s, v27.4s + srshl v5.4s, v5.4s, v27.4s + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v28.8h // bitdepth_max + umin v3.8h, v3.8h, v28.8h + subs w4, w4, #16 + st1 {v2.8h, v3.8h}, [x0], #32 + b.gt 1b + + ldp x0, x4, [sp, #48] + ldp x13, x14, [sp, #32] + ldp x11, x12, [sp, #16] + ldp x9, x10, [sp], #64 + + add x0, x0, x1 + ret +endfunc + +function wiener_filter7_hv_16bpc_neon + // Backing up/restoring registers shifted, so that x9 gets the value + // of x10, etc, and x15==x9, afterwards. + stp x10, x11, [sp, #-80]! + stp x12, x13, [sp, #16] + stp x14, x15, [sp, #32] + stp x10, x0, [sp, #48] + stp x3, x4, [sp, #64] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #6 + ld1 {v2.8h, v3.8h}, [x3], #32 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v2.8h, v3.8h}, [x3], #32 + ld1 {v4.d}[1], [x2], #8 + // Move x3 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #6 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v4.16b, v2.16b, #10 + b 2f +1: + ld1 {v2.8h, v3.8h}, [x3], #32 + // !LR_HAVE_LEFT, fill v4 with the leftmost pixel + // and shift v3 to have 3x the first pixel at the front. + dup v4.8h, v2.h[0] + // Move x3 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub x3, x3, #6 + ext v3.16b, v2.16b, v3.16b, #10 + ext v2.16b, v4.16b, v2.16b, #10 + +2: + ld1 {v4.8h}, [x3], #16 + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #19 + b.ge 4f // If w >= 19, all used input pixels are valid + + // 1 <= w < 19, w+3 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-22] to find the padding pixel. + sub w17, w4, #22 + // Insert padding in v2/3/4.h[w+3] onwards; fuse the +3 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -6 + ldr h26, [x3, w17, sxtw #1] + sub x6, x6, w4, uxtw #1 + dup v26.8h, v26.h[0] + ld1 {v23.16b, v24.16b, v25.16b}, [x6] + + bit v2.16b, v26.16b, v23.16b + bit v3.16b, v26.16b, v24.16b + bit v4.16b, v26.16b, v25.16b + +4: // Loop horizontally + ext v17.16b, v2.16b, v3.16b, #4 + ext v19.16b, v2.16b, v3.16b, #8 + ext v16.16b, v2.16b, v3.16b, #2 + ext v20.16b, v2.16b, v3.16b, #10 + ext v21.16b, v2.16b, v3.16b, #12 + ext v18.16b, v2.16b, v3.16b, #6 + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v2.8h + smull v6.4s, v18.4h, v0.h[3] + smlal v6.4s, v19.4h, v0.h[2] + smlal v6.4s, v20.4h, v0.h[1] + smlal v6.4s, v21.4h, v0.h[0] + smull2 v7.4s, v18.8h, v0.h[3] + smlal2 v7.4s, v19.8h, v0.h[2] + smlal2 v7.4s, v20.8h, v0.h[1] + smlal2 v7.4s, v21.8h, v0.h[0] + + ext v17.16b, v3.16b, v4.16b, #4 + ext v19.16b, v3.16b, v4.16b, #8 + ext v16.16b, v3.16b, v4.16b, #2 + ext v20.16b, v3.16b, v4.16b, #10 + ext v21.16b, v3.16b, v4.16b, #12 + ext v18.16b, v3.16b, v4.16b, #6 + + add v19.8h, v19.8h, v17.8h + add v20.8h, v20.8h, v16.8h + add v21.8h, v21.8h, v3.8h + smull v24.4s, v18.4h, v0.h[3] + smlal v24.4s, v19.4h, v0.h[2] + smlal v24.4s, v20.4h, v0.h[1] + smlal v24.4s, v21.4h, v0.h[0] + smull2 v25.4s, v18.8h, v0.h[3] + smlal2 v25.4s, v19.8h, v0.h[2] + smlal2 v25.4s, v20.8h, v0.h[1] + smlal2 v25.4s, v21.8h, v0.h[0] + + ld1 {v16.8h, v17.8h}, [x9], #32 + + mvni v26.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + add v24.4s, v24.4s, v30.4s + add v25.4s, v25.4s, v30.4s + ld1 {v18.8h, v19.8h}, [x10], #32 + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + srshl v24.4s, v24.4s, v29.4s + srshl v25.4s, v25.4s, v29.4s + ld1 {v20.8h, v21.8h}, [x11], #32 + sqxtun v6.4h, v6.4s + sqxtun2 v6.8h, v7.4s + sqxtun v7.4h, v24.4s + sqxtun2 v7.8h, v25.4s + ld1 {v22.8h, v23.8h}, [x12], #32 + umin v6.8h, v6.8h, v26.8h + umin v7.8h, v7.8h, v26.8h + ld1 {v24.8h, v25.8h}, [x13], #32 + sub v6.8h, v6.8h, v31.8h + sub v7.8h, v7.8h, v31.8h + + ld1 {v8.8h, v9.8h}, [x14], #32 + + smull v1.4s, v16.4h, v0.h[4] + smlal v1.4s, v18.4h, v0.h[5] + smlal v1.4s, v20.4h, v0.h[6] + smlal v1.4s, v22.4h, v0.h[7] + smlal v1.4s, v24.4h, v0.h[6] + smlal v1.4s, v8.4h, v0.h[5] + smlal v1.4s, v6.4h, v0.h[4] + smull2 v5.4s, v16.8h, v0.h[4] + smlal2 v5.4s, v18.8h, v0.h[5] + smlal2 v5.4s, v20.8h, v0.h[6] + smlal2 v5.4s, v22.8h, v0.h[7] + smlal2 v5.4s, v24.8h, v0.h[6] + smlal2 v5.4s, v8.8h, v0.h[5] + smlal2 v5.4s, v6.8h, v0.h[4] + smull v26.4s, v17.4h, v0.h[4] + smlal v26.4s, v19.4h, v0.h[5] + smlal v26.4s, v21.4h, v0.h[6] + smlal v26.4s, v23.4h, v0.h[7] + smlal v26.4s, v25.4h, v0.h[6] + smlal v26.4s, v9.4h, v0.h[5] + smlal v26.4s, v7.4h, v0.h[4] + smull2 v16.4s, v17.8h, v0.h[4] + smlal2 v16.4s, v19.8h, v0.h[5] + smlal2 v16.4s, v21.8h, v0.h[6] + smlal2 v16.4s, v23.8h, v0.h[7] + smlal2 v16.4s, v25.8h, v0.h[6] + smlal2 v16.4s, v9.8h, v0.h[5] + smlal2 v16.4s, v7.8h, v0.h[4] + srshl v1.4s, v1.4s, v27.4s // -round_bits_v + srshl v5.4s, v5.4s, v27.4s + srshl v26.4s, v26.4s, v27.4s + srshl v16.4s, v16.4s, v27.4s + sqxtun v18.4h, v1.4s + sqxtun2 v18.8h, v5.4s + sqxtun v19.4h, v26.4s + sqxtun2 v19.8h, v16.4s + st1 {v6.8h, v7.8h}, [x15], #32 + umin v18.8h, v18.8h, v28.8h // bitdepth_max + umin v19.8h, v19.8h, v28.8h + subs w4, w4, #16 + + st1 {v18.8h, v19.8h}, [x0], #32 + + b.le 0f + mov v2.16b, v4.16b + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8h, v4.8h}, [x3], #32 + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldp x3, x4, [sp, #64] + ldp x15, x0, [sp, #48] + ldp x13, x14, [sp, #32] + ldp x11, x12, [sp, #16] + ldp x9, x10, [sp], #80 + + add x3, x3, x1 + add x0, x0, x1 + + ret +endfunc + +// void dav1d_wiener_filter5_16bpc_neon(pixel *p, const ptrdiff_t p_stride, +// const pixel (*left)[4], const pixel *lpf, +// const int w, int h, +// const int16_t filter[2][8], +// const enum LrEdgeFlags edges, +// const int bitdepth_max); +function wiener_filter5_16bpc_neon, export=1 + ldr w8, [sp] + AARCH64_SIGN_LINK_REGISTER + stp x29, x30, [sp, #-32]! + stp d8, d9, [sp, #16] + mov x29, sp + ld1 {v0.8h, v1.8h}, [x6] + tst w7, #4 // LR_HAVE_TOP + sub_sp 384*2*4 + + dup v28.8h, w8 // bitdepth_max + clz w8, w8 + movi v30.4s, #1 + sub w10, w8, #38 // -(bitdepth + 6) + sub w11, w8, #11 // round_bits_v + sub w8, w8, #25 // -round_bits_h + neg w10, w10 // bitdepth + 6 + neg w11, w11 // -round_bits_v + dup v2.4s, w10 + dup v29.4s, w8 // -round_bits_h + dup v27.4s, w11 // -round_bits_v + movi v31.8h, #0x20, lsl #8 // 1 << 13 = 8192 + ushl v30.4s, v30.4s, v2.4s // 1 << (bitdepth + 6) + + zip1 v0.2d, v0.2d, v1.2d // move vertical coeffs to v0.h[4-7], freeing up v1 + + // x11 - t4 + // x12 - t3 + // x13 - t2 + // x14 - t1 + // x15 - t0 + mov x14, sp // t1 + b.eq L(no_top_5) + + mov x16, x2 // backup left + mov x2, #0 + bl wiener_filter5_h_16bpc_neon + add x3, x3, x1 // lpf += stride + mov x11, x14 // t4 + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_16bpc_neon + add x3, x3, x1, lsl #2 + add x3, x3, x1 // lpf += stride*5 + mov x12, x14 // t3 + add x14, x14, #384*2 // t1 += 384*2 + mov x2, x16 // left + mov x16, x3 // backup lpf + mov x3, x0 // lpf = p + bl wiener_filter5_h_16bpc_neon + subs w5, w5, #1 // h-- + mov x13, x14 // t2 + b.eq L(v1_5) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x3, x3, x1 // src += stride + +L(main_5): + mov x15, x11 // t0 = t4 +L(main_loop_5): + bl wiener_filter5_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_loop_5) + tst w7, #8 // LR_HAVE_BOTTOM + b.eq L(v2_5) + + mov x3, x16 // restore lpf + mov x2, #0 // left = NULL + bl wiener_filter5_hv_16bpc_neon + bl wiener_filter5_hv_16bpc_neon +L(end_5): + + mov sp, x29 + ldp d8, d9, [sp, #16] + ldp x29, x30, [sp], #32 + AARCH64_VALIDATE_LINK_REGISTER + ret + +L(no_top_5): + add x3, x3, x1, lsl #2 + add x16, x3, x1, lsl #1 // lpf += stride*6, backup + mov x3, x0 // lpf = p + + bl wiener_filter5_h_16bpc_neon + subs w5, w5, #1 // h-- + mov x11, x14 // t4 + mov x12, x14 // t3 + mov x13, x14 // t2 + b.eq L(v1_5) + add x3, x3, x1 // src += stride + add x14, x14, #384*2 // t1 += 384*2 + bl wiener_filter5_h_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x3, x3, x1 // src += stride + add x15, x14, #384*2 // t0 = t1 + 384*2 + bl wiener_filter5_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.eq L(v2_5) + add x15, x15, #384*2*3 // t0 += 384*2*3 + bl wiener_filter5_hv_16bpc_neon + subs w5, w5, #1 // h-- + b.ne L(main_5) +L(v2_5): + bl wiener_filter5_v_16bpc_neon + add x0, x0, x1 + mov x11, x12 + mov x12, x13 + mov x13, x14 +L(v1_5): + bl wiener_filter5_v_16bpc_neon + b L(end_5) +endfunc + + +function wiener_filter5_h_16bpc_neon + stp x3, x4, [sp, #-32]! + str x14, [sp, #16] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #4 + ld1 {v2.8h, v3.8h}, [x3], #32 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v2.8h, v3.8h}, [x3], #32 + ld1 {v4.d}[1], [x2], #8 + // Move x3 back to account for the last 2 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #4 + ext v3.16b, v2.16b, v3.16b, #12 + ext v2.16b, v4.16b, v2.16b, #12 + b 2f + +1: + ld1 {v2.8h, v3.8h}, [x3], #32 + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v3 to have 3x the first pixel at the front. + dup v4.8h, v2.h[0] + // Move x3 back to account for the last 2 pixels we loaded before, + // which we shifted out. + sub x3, x3, #4 + ext v3.16b, v2.16b, v3.16b, #12 + ext v2.16b, v4.16b, v2.16b, #12 + +2: + ld1 {v4.8h}, [x3], #16 + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #18 + b.ge 4f // If w >= 18, all used input pixels are valid + + // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+2]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel. + sub w17, w4, #23 + // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -4 + ldr h26, [x3, w17, sxtw #1] + sub x6, x6, w4, uxtw #1 + dup v26.8h, v26.h[0] + ld1 {v23.16b, v24.16b, v25.16b}, [x6] + + bit v2.16b, v26.16b, v23.16b + bit v3.16b, v26.16b, v24.16b + bit v4.16b, v26.16b, v25.16b + +4: // Loop horizontally + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + ext v16.16b, v2.16b, v3.16b, #2 + ext v18.16b, v2.16b, v3.16b, #6 + ext v19.16b, v2.16b, v3.16b, #8 + ext v17.16b, v2.16b, v3.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v2.8h + smull v6.4s, v17.4h, v0.h[3] + smlal v6.4s, v18.4h, v0.h[2] + smlal v6.4s, v19.4h, v0.h[1] + smull2 v7.4s, v17.8h, v0.h[3] + smlal2 v7.4s, v18.8h, v0.h[2] + smlal2 v7.4s, v19.8h, v0.h[1] + + ext v16.16b, v3.16b, v4.16b, #2 + ext v18.16b, v3.16b, v4.16b, #6 + ext v19.16b, v3.16b, v4.16b, #8 + ext v17.16b, v3.16b, v4.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v3.8h + smull v16.4s, v17.4h, v0.h[3] + smlal v16.4s, v18.4h, v0.h[2] + smlal v16.4s, v19.4h, v0.h[1] + smull2 v17.4s, v17.8h, v0.h[3] + smlal2 v17.4s, v18.8h, v0.h[2] + smlal2 v17.4s, v19.8h, v0.h[1] + + mvni v24.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + add v16.4s, v16.4s, v30.4s + add v17.4s, v17.4s, v30.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + srshl v16.4s, v16.4s, v29.4s + srshl v17.4s, v17.4s, v29.4s + sqxtun v6.4h, v6.4s + sqxtun2 v6.8h, v7.4s + sqxtun v7.4h, v16.4s + sqxtun2 v7.8h, v17.4s + umin v6.8h, v6.8h, v24.8h + umin v7.8h, v7.8h, v24.8h + sub v6.8h, v6.8h, v31.8h + sub v7.8h, v7.8h, v31.8h + + subs w4, w4, #16 + + st1 {v6.8h, v7.8h}, [x14], #32 + + b.le 0f + mov v2.16b, v4.16b + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8h, v4.8h}, [x3], #32 + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldr x14, [sp, #16] + ldp x3, x4, [sp], #32 + ret +endfunc + +function wiener_filter5_v_16bpc_neon + stp x11, x12, [sp, #-48]! + stp x13, x14, [sp, #16] + stp x0, x4, [sp, #32] +1: + ld1 {v16.8h, v17.8h}, [x11], #32 + ld1 {v18.8h, v19.8h}, [x12], #32 + ld1 {v20.8h, v21.8h}, [x13], #32 + ld1 {v22.8h, v23.8h}, [x14], #32 + + smull v2.4s, v16.4h, v0.h[5] + smlal v2.4s, v18.4h, v0.h[6] + smlal v2.4s, v20.4h, v0.h[7] + smlal v2.4s, v22.4h, v0.h[6] + smlal v2.4s, v22.4h, v0.h[5] + smull2 v3.4s, v16.8h, v0.h[5] + smlal2 v3.4s, v18.8h, v0.h[6] + smlal2 v3.4s, v20.8h, v0.h[7] + smlal2 v3.4s, v22.8h, v0.h[6] + smlal2 v3.4s, v22.8h, v0.h[5] + smull v4.4s, v17.4h, v0.h[5] + smlal v4.4s, v19.4h, v0.h[6] + smlal v4.4s, v21.4h, v0.h[7] + smlal v4.4s, v23.4h, v0.h[6] + smlal v4.4s, v23.4h, v0.h[5] + smull2 v5.4s, v17.8h, v0.h[5] + smlal2 v5.4s, v19.8h, v0.h[6] + smlal2 v5.4s, v21.8h, v0.h[7] + smlal2 v5.4s, v23.8h, v0.h[6] + smlal2 v5.4s, v23.8h, v0.h[5] + srshl v2.4s, v2.4s, v27.4s // -round_bits_v + srshl v3.4s, v3.4s, v27.4s + srshl v4.4s, v4.4s, v27.4s + srshl v5.4s, v5.4s, v27.4s + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v28.8h // bitdepth_max + umin v3.8h, v3.8h, v28.8h + + subs w4, w4, #16 + st1 {v2.8h, v3.8h}, [x0], #32 + b.gt 1b + + ldp x0, x4, [sp, #32] + ldp x13, x14, [sp, #16] + ldp x11, x12, [sp], #48 + + ret +endfunc + +function wiener_filter5_hv_16bpc_neon + // Backing up/restoring registers shifted, so that x11 gets the value + // of x12, etc, and x15==x11, afterwards. + stp x12, x13, [sp, #-64]! + stp x14, x15, [sp, #16] + stp x12, x0, [sp, #32] + stp x3, x4, [sp, #48] + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 1f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #4 + ld1 {v2.8h, v3.8h}, [x3], #32 + b 2f + +0: + // LR_HAVE_LEFT, left != NULL + ld1 {v2.8h, v3.8h}, [x3], #32 + ld1 {v4.d}[1], [x2], #8 + // Move x3 back to account for the last 2 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #4 + ext v3.16b, v2.16b, v3.16b, #12 + ext v2.16b, v4.16b, v2.16b, #12 + b 2f +1: + ld1 {v2.8h, v3.8h}, [x3], #32 + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v3 to have 2x the first pixel at the front. + dup v4.8h, v2.h[0] + // Move x3 back to account for the last 2 pixels we loaded before, + // which we shifted out. + sub x3, x3, #4 + ext v3.16b, v2.16b, v3.16b, #12 + ext v2.16b, v4.16b, v2.16b, #12 + +2: + ld1 {v4.8h}, [x3], #16 + + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w4, #18 + b.ge 4f // If w >= 18, all used input pixels are valid + + // 1 <= w < 18, w+2 pixels valid in v2-v4. For w>=9, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // The padding pixel is v2/3/4.h[w+1]. x3 points at the next input, ie + // v2/3/4.h[24]. Thus read from x3[w-23] to find the padding pixel. + sub w17, w4, #23 + // Insert padding in v2/3/4.h[w+2] onwards; fuse the +2 (*2) into the + // buffer pointer. + movrel x6, right_ext_mask, -4 + ldr h26, [x3, w17, sxtw #1] + sub x6, x6, w4, uxtw #1 + dup v26.8h, v26.h[0] + ld1 {v23.16b, v24.16b, v25.16b}, [x6] + + bit v2.16b, v26.16b, v23.16b + bit v3.16b, v26.16b, v24.16b + bit v4.16b, v26.16b, v25.16b + +4: // Loop horizontally + ext v16.16b, v2.16b, v3.16b, #2 + ext v18.16b, v2.16b, v3.16b, #6 + ext v19.16b, v2.16b, v3.16b, #8 + ext v17.16b, v2.16b, v3.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v2.8h + smull v6.4s, v17.4h, v0.h[3] + smlal v6.4s, v18.4h, v0.h[2] + smlal v6.4s, v19.4h, v0.h[1] + smull2 v7.4s, v17.8h, v0.h[3] + smlal2 v7.4s, v18.8h, v0.h[2] + smlal2 v7.4s, v19.8h, v0.h[1] + + ext v16.16b, v3.16b, v4.16b, #2 + ext v18.16b, v3.16b, v4.16b, #6 + ext v19.16b, v3.16b, v4.16b, #8 + ext v17.16b, v3.16b, v4.16b, #4 + add v18.8h, v18.8h, v16.8h + add v19.8h, v19.8h, v3.8h + smull v24.4s, v17.4h, v0.h[3] + smlal v24.4s, v18.4h, v0.h[2] + smlal v24.4s, v19.4h, v0.h[1] + smull2 v25.4s, v17.8h, v0.h[3] + smlal2 v25.4s, v18.8h, v0.h[2] + smlal2 v25.4s, v19.8h, v0.h[1] + + ld1 {v16.8h, v17.8h}, [x11], #32 + mvni v26.8h, #0x80, lsl #8 // 0x7fff = (1 << 15) - 1 + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + add v24.4s, v24.4s, v30.4s + add v25.4s, v25.4s, v30.4s + ld1 {v18.8h, v19.8h}, [x12], #32 + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + srshl v24.4s, v24.4s, v29.4s + srshl v25.4s, v25.4s, v29.4s + ld1 {v20.8h, v21.8h}, [x13], #32 + sqxtun v6.4h, v6.4s + sqxtun2 v6.8h, v7.4s + sqxtun v7.4h, v24.4s + sqxtun2 v7.8h, v25.4s + ld1 {v22.8h, v23.8h}, [x14], #32 + umin v6.8h, v6.8h, v26.8h + umin v7.8h, v7.8h, v26.8h + sub v6.8h, v6.8h, v31.8h + sub v7.8h, v7.8h, v31.8h + + smull v8.4s, v16.4h, v0.h[5] + smlal v8.4s, v18.4h, v0.h[6] + smlal v8.4s, v20.4h, v0.h[7] + smlal v8.4s, v22.4h, v0.h[6] + smlal v8.4s, v6.4h, v0.h[5] + smull2 v9.4s, v16.8h, v0.h[5] + smlal2 v9.4s, v18.8h, v0.h[6] + smlal2 v9.4s, v20.8h, v0.h[7] + smlal2 v9.4s, v22.8h, v0.h[6] + smlal2 v9.4s, v6.8h, v0.h[5] + smull v1.4s, v17.4h, v0.h[5] + smlal v1.4s, v19.4h, v0.h[6] + smlal v1.4s, v21.4h, v0.h[7] + smlal v1.4s, v23.4h, v0.h[6] + smlal v1.4s, v7.4h, v0.h[5] + smull2 v5.4s, v17.8h, v0.h[5] + smlal2 v5.4s, v19.8h, v0.h[6] + smlal2 v5.4s, v21.8h, v0.h[7] + smlal2 v5.4s, v23.8h, v0.h[6] + smlal2 v5.4s, v7.8h, v0.h[5] + srshl v8.4s, v8.4s, v27.4s // -round_bits_v + srshl v9.4s, v9.4s, v27.4s + srshl v1.4s, v1.4s, v27.4s + srshl v5.4s, v5.4s, v27.4s + sqxtun v8.4h, v8.4s + sqxtun2 v8.8h, v9.4s + sqxtun v9.4h, v1.4s + sqxtun2 v9.8h, v5.4s + st1 {v6.8h, v7.8h}, [x15], #32 + umin v8.8h, v8.8h, v28.8h // bitdepth_max + umin v9.8h, v9.8h, v28.8h + + subs w4, w4, #16 + + st1 {v8.8h, v9.8h}, [x0], #32 + + b.le 0f + mov v2.16b, v4.16b + tst w7, #2 // LR_HAVE_RIGHT + ld1 {v3.8h, v4.8h}, [x3], #32 + b.ne 4b // If we don't need to pad, just keep filtering. + b 3b // If we need to pad, check how many pixels we have left. + +0: + ldp x3, x4, [sp, #48] + ldp x15, x0, [sp, #32] + ldp x13, x14, [sp, #16] + ldp x11, x12, [sp], #64 + + add x3, x3, x1 + add x0, x0, x1 + + ret +endfunc + +#define SUM_STRIDE (384+16) + +#include "looprestoration_tmpl.S" + +// void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_h_16bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add w13, w5, #7 + bic w13, w13, #7 + sub x9, x9, w13, uxtw #1 + + // Store the width for the vertical loop + mov w8, w5 + + // Subtract the number of pixels read from the input from the stride + add w13, w13, #8 + sub x4, x4, w13, uxtw #1 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #4 + sub x12, x12, #4 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 2 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #4 + + +1: // Loop vertically + ld1 {v0.8h, v1.8h}, [x3], #32 + ld1 {v16.8h, v17.8h}, [x12], #32 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v2.d}[1], [x2], #8 + // Move x3/x12 back to account for the last 2 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #4 + sub x12, x12, #4 + ld1 {v18.d}[1], [x2], #8 + ext v1.16b, v0.16b, v1.16b, #12 + ext v0.16b, v2.16b, v0.16b, #12 + ext v17.16b, v16.16b, v17.16b, #12 + ext v16.16b, v18.16b, v16.16b, #12 + b 2f +0: + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v0/v1 to have 2x the first pixel at the front. + dup v2.8h, v0.h[0] + dup v18.8h, v16.h[0] + // Move x3 back to account for the last 2 pixels we loaded before, + // which we shifted out. + sub x3, x3, #4 + sub x12, x12, #4 + ext v1.16b, v0.16b, v1.16b, #12 + ext v0.16b, v2.16b, v0.16b, #12 + ext v17.16b, v16.16b, v17.16b, #12 + ext v16.16b, v18.16b, v16.16b, #12 + +2: + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that pixel to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 2 + 1) + ldr h30, [x3, w13, sxtw #1] + ldr h31, [x12, w13, sxtw #1] + // Fill v30/v31 with the right padding pixel + dup v30.8h, v30.h[0] + dup v31.8h, v31.h[0] +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w5, #10 + b.ge 4f // If w >= 10, all used input pixels are valid + + // 1 <= w < 10, w pixels valid in v0-v1. For w=9, this ends up called + // again; it's not strictly needed in those cases (we pad enough here), + // but keeping the code as simple as possible. + + // Insert padding in v0/1.h[w] onwards + movrel x13, right_ext_mask + sub x13, x13, w5, uxtw #1 + ld1 {v28.16b, v29.16b}, [x13] + + bit v0.16b, v30.16b, v28.16b + bit v1.16b, v30.16b, v29.16b + bit v16.16b, v31.16b, v28.16b + bit v17.16b, v31.16b, v29.16b + +4: // Loop horizontally + ext v26.16b, v0.16b, v1.16b, #2 + ext v28.16b, v16.16b, v17.16b, #2 + ext v27.16b, v0.16b, v1.16b, #4 + ext v29.16b, v16.16b, v17.16b, #4 + + add v6.8h, v0.8h, v26.8h + umull v22.4s, v0.4h, v0.4h + umlal v22.4s, v26.4h, v26.4h + umlal v22.4s, v27.4h, v27.4h + add v7.8h, v16.8h, v28.8h + umull v24.4s, v16.4h, v16.4h + umlal v24.4s, v28.4h, v28.4h + umlal v24.4s, v29.4h, v29.4h + add v6.8h, v6.8h, v27.8h + umull2 v23.4s, v0.8h, v0.8h + umlal2 v23.4s, v26.8h, v26.8h + umlal2 v23.4s, v27.8h, v27.8h + add v7.8h, v7.8h, v29.8h + umull2 v25.4s, v16.8h, v16.8h + umlal2 v25.4s, v28.8h, v28.8h + umlal2 v25.4s, v29.8h, v29.8h + + subs w5, w5, #8 + + st1 {v6.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v22.4s,v23.4s}, [x0], #32 + st1 {v24.4s,v25.4s}, [x10], #32 + + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + mov v0.16b, v1.16b + mov v16.16b, v17.16b + ld1 {v1.8h}, [x3], #16 + ld1 {v17.8h}, [x12], #16 + + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum, +// const pixel (*left)[4], +// const pixel *src, const ptrdiff_t stride, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_h_16bpc_neon, export=1 + add w5, w5, #2 // w += 2 + + // Set up pointers for reading/writing alternate rows + add x10, x0, #(4*SUM_STRIDE) // sumsq + add x11, x1, #(2*SUM_STRIDE) // sum + add x12, x3, x4 // src + lsl x4, x4, #1 + mov x9, #(2*2*SUM_STRIDE) // double sum stride + + // Subtract the aligned width from the output stride. + add w13, w5, #7 + bic w13, w13, #7 + sub x9, x9, w13, uxtw #1 + add w13, w13, #8 + sub x4, x4, w13, uxtw #1 + + // Store the width for the vertical loop + mov w8, w5 + + // Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL + tst w7, #1 // LR_HAVE_LEFT + b.eq 2f + // LR_HAVE_LEFT + cbnz x2, 0f + // left == NULL + sub x3, x3, #6 + sub x12, x12, #6 + b 1f +0: // LR_HAVE_LEFT, left != NULL +2: // !LR_HAVE_LEFT, increase the stride. + // For this case we don't read the left 3 pixels from the src pointer, + // but shift it as if we had done that. + add x4, x4, #6 + +1: // Loop vertically + ld1 {v0.8h, v1.8h}, [x3], #32 + ld1 {v16.8h, v17.8h}, [x12], #32 + + tst w7, #1 // LR_HAVE_LEFT + b.eq 0f + cbz x2, 2f + // LR_HAVE_LEFT, left != NULL + ld1 {v2.d}[1], [x2], #8 + // Move x3/x12 back to account for the last 3 pixels we loaded earlier, + // which we'll shift out. + sub x3, x3, #6 + sub x12, x12, #6 + ld1 {v18.d}[1], [x2], #8 + ext v1.16b, v0.16b, v1.16b, #10 + ext v0.16b, v2.16b, v0.16b, #10 + ext v17.16b, v16.16b, v17.16b, #10 + ext v16.16b, v18.16b, v16.16b, #10 + b 2f +0: + // !LR_HAVE_LEFT, fill v2 with the leftmost pixel + // and shift v0/v1 to have 3x the first pixel at the front. + dup v2.8h, v0.h[0] + dup v18.8h, v16.h[0] + // Move x3 back to account for the last 3 pixels we loaded before, + // which we shifted out. + sub x3, x3, #6 + sub x12, x12, #6 + ext v1.16b, v0.16b, v1.16b, #10 + ext v0.16b, v2.16b, v0.16b, #10 + ext v17.16b, v16.16b, v17.16b, #10 + ext v16.16b, v18.16b, v16.16b, #10 + +2: + tst w7, #2 // LR_HAVE_RIGHT + b.ne 4f + // If we'll need to pad the right edge, load that pixel to pad with + // here since we can find it pretty easily from here. + sub w13, w5, #(2 + 16 - 3 + 1) + ldr h30, [x3, w13, sxtw #1] + ldr h31, [x12, w13, sxtw #1] + // Fill v30/v31 with the right padding pixel + dup v30.8h, v30.h[0] + dup v31.8h, v31.h[0] +3: // !LR_HAVE_RIGHT + + // Check whether we need to pad the right edge + cmp w5, #11 + b.ge 4f // If w >= 11, all used input pixels are valid + + // 1 <= w < 11, w+1 pixels valid in v0-v1. For w=9 or w=10, + // this ends up called again; it's not strictly needed in those + // cases (we pad enough here), but keeping the code as simple as possible. + + // Insert padding in v0/1.h[w+1] onwards; fuse the +1 into the + // buffer pointer. + movrel x13, right_ext_mask, -2 + sub x13, x13, w5, uxtw #1 + ld1 {v28.16b, v29.16b}, [x13] + + bit v0.16b, v30.16b, v28.16b + bit v1.16b, v30.16b, v29.16b + bit v16.16b, v31.16b, v28.16b + bit v17.16b, v31.16b, v29.16b + +4: // Loop horizontally + ext v26.16b, v0.16b, v1.16b, #2 + ext v28.16b, v16.16b, v17.16b, #2 + ext v27.16b, v0.16b, v1.16b, #4 + ext v29.16b, v16.16b, v17.16b, #4 + + add v6.8h, v0.8h, v26.8h + umull v22.4s, v0.4h, v0.4h + umlal v22.4s, v26.4h, v26.4h + umlal v22.4s, v27.4h, v27.4h + add v7.8h, v16.8h, v28.8h + umull v24.4s, v16.4h, v16.4h + umlal v24.4s, v28.4h, v28.4h + umlal v24.4s, v29.4h, v29.4h + add v6.8h, v6.8h, v27.8h + umull2 v23.4s, v0.8h, v0.8h + umlal2 v23.4s, v26.8h, v26.8h + umlal2 v23.4s, v27.8h, v27.8h + add v7.8h, v7.8h, v29.8h + umull2 v25.4s, v16.8h, v16.8h + umlal2 v25.4s, v28.8h, v28.8h + umlal2 v25.4s, v29.8h, v29.8h + + ext v26.16b, v0.16b, v1.16b, #6 + ext v28.16b, v16.16b, v17.16b, #6 + ext v27.16b, v0.16b, v1.16b, #8 + ext v29.16b, v16.16b, v17.16b, #8 + + add v6.8h, v6.8h, v26.8h + umlal v22.4s, v26.4h, v26.4h + umlal v22.4s, v27.4h, v27.4h + add v7.8h, v7.8h, v28.8h + umlal v24.4s, v28.4h, v28.4h + umlal v24.4s, v29.4h, v29.4h + add v6.8h, v6.8h, v27.8h + umlal2 v23.4s, v26.8h, v26.8h + umlal2 v23.4s, v27.8h, v27.8h + add v7.8h, v7.8h, v29.8h + umlal2 v25.4s, v28.8h, v28.8h + umlal2 v25.4s, v29.8h, v29.8h + + subs w5, w5, #8 + + st1 {v6.8h}, [x1], #16 + st1 {v7.8h}, [x11], #16 + st1 {v22.4s,v23.4s}, [x0], #32 + st1 {v24.4s,v25.4s}, [x10], #32 + + b.le 9f + tst w7, #2 // LR_HAVE_RIGHT + mov v0.16b, v1.16b + mov v16.16b, v17.16b + ld1 {v1.8h}, [x3], #16 + ld1 {v17.8h}, [x12], #16 + + b.ne 4b // If we don't need to pad, just keep summing. + b 3b // If we need to pad, check how many pixels we have left. + +9: + subs w6, w6, #2 + b.le 0f + // Jump to the next row and loop horizontally + add x0, x0, x9, lsl #1 + add x10, x10, x9, lsl #1 + add x1, x1, x9 + add x11, x11, x9 + add x3, x3, x4 + add x12, x12, x4 + mov w5, w8 + b 1b +0: + ret +endfunc + +sgr_funcs 16 diff --git a/third_party/dav1d/src/arm/64/looprestoration_common.S b/third_party/dav1d/src/arm/64/looprestoration_common.S new file mode 100644 index 0000000000..200eb63189 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration_common.S @@ -0,0 +1,432 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define SUM_STRIDE (384+16) + +// void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box3_v_neon, export=1 + add w10, w3, #2 // Number of output rows to move back + mov w11, w3 // Number of input rows to move back + add w2, w2, #2 // Actual summed width + mov x7, #(4*SUM_STRIDE) // sumsq stride + mov x8, #(2*SUM_STRIDE) // sum stride + sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride + sub x1, x1, #(2*SUM_STRIDE) // sum -= stride + + tst w4, #4 // LR_HAVE_TOP + b.eq 0f + // If have top, read from row -2. + sub x5, x0, #(4*SUM_STRIDE) + sub x6, x1, #(2*SUM_STRIDE) + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add x5, x0, #(4*SUM_STRIDE) + add x6, x1, #(2*SUM_STRIDE) +1: + + tst w4, #8 // LR_HAVE_BOTTOM + b.eq 1f + // LR_HAVE_BOTTOM + add w3, w3, #2 // Sum all h+2 lines with the main loop + add w11, w11, #2 +1: + mov w9, w3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into v16-v21 and v24-v26 taking top + // padding into consideration. + tst w4, #4 // LR_HAVE_TOP + ld1 {v16.4s, v17.4s}, [x5], x7 + ld1 {v24.8h}, [x6], x8 + b.eq 2f + // LR_HAVE_TOP + ld1 {v18.4s, v19.4s}, [x5], x7 + ld1 {v25.8h}, [x6], x8 + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b 3f +2: // !LR_HAVE_TOP + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v25.16b, v24.16b + mov v20.16b, v16.16b + mov v21.16b, v17.16b + mov v26.16b, v24.16b + +3: + subs w3, w3, #1 +.macro add3 + add v16.4s, v16.4s, v18.4s + add v17.4s, v17.4s, v19.4s + add v24.8h, v24.8h, v25.8h + add v16.4s, v16.4s, v20.4s + add v17.4s, v17.4s, v21.4s + add v24.8h, v24.8h, v26.8h + st1 {v16.4s, v17.4s}, [x0], x7 + st1 {v24.8h}, [x1], x8 +.endm + add3 + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v24.16b, v25.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v25.16b, v26.16b + b.le 4f + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b 3b + +4: + tst w4, #8 // LR_HAVE_BOTTOM + b.ne 5f + // !LR_HAVE_BOTTOM + // Produce two more rows, extending the already loaded rows. + add3 + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v24.16b, v25.16b + add3 + +5: // End of one vertical slice. + subs w2, w2, #8 + b.le 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + msub x5, x7, x11, x5 + msub x6, x8, x11, x6 + // Output pointers + msub x0, x7, x10, x0 + msub x1, x8, x10, x1 + add x0, x0, #32 + add x1, x1, #16 + add x5, x5, #32 + add x6, x6, #16 + mov w3, w9 + b 1b + +0: + ret +.purgem add3 +endfunc + +// void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, +// const int w, const int h, +// const enum LrEdgeFlags edges); +function sgr_box5_v_neon, export=1 + add w10, w3, #2 // Number of output rows to move back + mov w11, w3 // Number of input rows to move back + add w2, w2, #8 // Actual summed width + mov x7, #(4*SUM_STRIDE) // sumsq stride + mov x8, #(2*SUM_STRIDE) // sum stride + sub x0, x0, #(4*SUM_STRIDE) // sumsq -= stride + sub x1, x1, #(2*SUM_STRIDE) // sum -= stride + + tst w4, #4 // LR_HAVE_TOP + b.eq 0f + // If have top, read from row -2. + sub x5, x0, #(4*SUM_STRIDE) + sub x6, x1, #(2*SUM_STRIDE) + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_TOP + // If we don't have top, read from row 0 even if + // we start writing to row -1. + add x5, x0, #(4*SUM_STRIDE) + add x6, x1, #(2*SUM_STRIDE) +1: + + tst w4, #8 // LR_HAVE_BOTTOM + b.eq 0f + // LR_HAVE_BOTTOM + add w3, w3, #2 // Handle h+2 lines with the main loop + add w11, w11, #2 + b 1f +0: + // !LR_HAVE_BOTTOM + sub w3, w3, #1 // Handle h-1 lines with the main loop +1: + mov w9, w3 // Backup of h for next loops + +1: + // Start of horizontal loop; start one vertical filter slice. + // Start loading rows into v16-v25 and v26-v30 taking top + // padding into consideration. + tst w4, #4 // LR_HAVE_TOP + ld1 {v16.4s, v17.4s}, [x5], x7 + ld1 {v26.8h}, [x6], x8 + b.eq 2f + // LR_HAVE_TOP + ld1 {v20.4s, v21.4s}, [x5], x7 + ld1 {v28.8h}, [x6], x8 + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v27.16b, v26.16b + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + b 3f +2: // !LR_HAVE_TOP + mov v18.16b, v16.16b + mov v19.16b, v17.16b + mov v27.16b, v26.16b + mov v20.16b, v16.16b + mov v21.16b, v17.16b + mov v28.16b, v26.16b + mov v22.16b, v16.16b + mov v23.16b, v17.16b + mov v29.16b, v26.16b + +3: + cbz w3, 4f + ld1 {v24.4s, v25.4s}, [x5], x7 + ld1 {v30.8h}, [x6], x8 + +3: + // Start of vertical loop + subs w3, w3, #2 +.macro add5 + add v16.4s, v16.4s, v18.4s + add v17.4s, v17.4s, v19.4s + add v26.8h, v26.8h, v27.8h + add v0.4s, v20.4s, v22.4s + add v1.4s, v21.4s, v23.4s + add v2.8h, v28.8h, v29.8h + add v16.4s, v16.4s, v24.4s + add v17.4s, v17.4s, v25.4s + add v26.8h, v26.8h, v30.8h + add v16.4s, v16.4s, v0.4s + add v17.4s, v17.4s, v1.4s + add v26.8h, v26.8h, v2.8h + st1 {v16.4s, v17.4s}, [x0], x7 + st1 {v26.8h}, [x1], x8 +.endm + add5 +.macro shift2 + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v26.16b, v28.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b + mov v27.16b, v29.16b + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v28.16b, v30.16b +.endm + shift2 + add x0, x0, x7 + add x1, x1, x8 + b.le 5f + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + ld1 {v24.4s, v25.4s}, [x5], x7 + ld1 {v30.8h}, [x6], x8 + b 3b + +4: + // h == 1, !LR_HAVE_BOTTOM. + // Pad the last row with the only content row, and add. + mov v24.16b, v22.16b + mov v25.16b, v23.16b + mov v30.16b, v29.16b + add5 + shift2 + add x0, x0, x7 + add x1, x1, x8 + add5 + b 6f + +5: + tst w4, #8 // LR_HAVE_BOTTOM + b.ne 6f + // !LR_HAVE_BOTTOM + cbnz w3, 5f + // The intended three edge rows left; output the one at h-2 and + // the past edge one at h. + ld1 {v22.4s, v23.4s}, [x5], x7 + ld1 {v29.8h}, [x6], x8 + // Pad the past-edge row from the last content row. + mov v24.16b, v22.16b + mov v25.16b, v23.16b + mov v30.16b, v29.16b + add5 + shift2 + add x0, x0, x7 + add x1, x1, x8 + // The last two rows are already padded properly here. + add5 + b 6f + +5: + // w3 == -1, two rows left, output one. + // Pad the last two rows from the mid one. + mov v22.16b, v20.16b + mov v23.16b, v21.16b + mov v29.16b, v28.16b + mov v24.16b, v20.16b + mov v25.16b, v21.16b + mov v30.16b, v28.16b + add5 + add x0, x0, x7 + add x1, x1, x8 + b 6f + +6: // End of one vertical slice. + subs w2, w2, #8 + b.le 0f + // Move pointers back up to the top and loop horizontally. + // Input pointers + msub x5, x7, x11, x5 + msub x6, x8, x11, x6 + // Output pointers + msub x0, x7, x10, x0 + msub x1, x8, x10, x1 + add x0, x0, #32 + add x1, x1, #16 + add x5, x5, #32 + add x6, x6, #16 + mov w3, w9 + b 1b + +0: + ret +.purgem add5 +endfunc + +// void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +// void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, +// const int w, const int h, const int strength, +// const int bitdepth_max); +function sgr_calc_ab1_neon, export=1 + clz w9, w5 + add x3, x3, #2 // h += 2 + movi v31.4s, #9 // n + mov x5, #455 + mov x8, #SUM_STRIDE + b sgr_calc_ab_neon +endfunc + +function sgr_calc_ab2_neon, export=1 + clz w9, w5 + add x3, x3, #3 // h += 3 + asr x3, x3, #1 // h /= 2 + movi v31.4s, #25 // n + mov x5, #164 + mov x8, #(2*SUM_STRIDE) +endfunc + +function sgr_calc_ab_neon + sub w9, w9, #24 // -bitdepth_min_8 + movrel x12, X(sgr_x_by_x) + ld1 {v16.16b, v17.16b, v18.16b}, [x12] + dup v6.8h, w9 // -bitdepth_min_8 + movi v19.16b, #5 + movi v20.8b, #55 // idx of last 5 + movi v21.8b, #72 // idx of last 4 + movi v22.8b, #101 // idx of last 3 + movi v23.8b, #169 // idx of last 2 + movi v24.8b, #254 // idx of last 1 + saddl v7.4s, v6.4h, v6.4h // -2*bitdepth_min_8 + add x2, x2, #2 // w += 2 + add x7, x2, #7 + bic x7, x7, #7 // aligned w + sub x7, x8, x7 // increment between rows + movi v29.8h, #1, lsl #8 + dup v28.4s, w4 + dup v30.4s, w5 // one_by_x + sub x0, x0, #(4*(SUM_STRIDE)) + sub x1, x1, #(2*(SUM_STRIDE)) + mov x6, x2 // backup of w + sub v16.16b, v16.16b, v19.16b + sub v17.16b, v17.16b, v19.16b + sub v18.16b, v18.16b, v19.16b +1: + subs x2, x2, #8 + ld1 {v0.4s, v1.4s}, [x0] // a + ld1 {v2.8h}, [x1] // b + srshl v0.4s, v0.4s, v7.4s + srshl v1.4s, v1.4s, v7.4s + srshl v4.8h, v2.8h, v6.8h + mul v0.4s, v0.4s, v31.4s // a * n + mul v1.4s, v1.4s, v31.4s // a * n + umull v3.4s, v4.4h, v4.4h // b * b + umull2 v4.4s, v4.8h, v4.8h // b * b + uqsub v0.4s, v0.4s, v3.4s // imax(a * n - b * b, 0) + uqsub v1.4s, v1.4s, v4.4s // imax(a * n - b * b, 0) + mul v0.4s, v0.4s, v28.4s // p * s + mul v1.4s, v1.4s, v28.4s // p * s + uqshrn v0.4h, v0.4s, #16 + uqshrn2 v0.8h, v1.4s, #16 + uqrshrn v0.8b, v0.8h, #4 // imin(z, 255) + + cmhi v25.8b, v0.8b, v20.8b // = -1 if sgr_x_by_x[v0] < 5 + cmhi v26.8b, v0.8b, v21.8b // = -1 if sgr_x_by_x[v0] < 4 + tbl v1.8b, {v16.16b, v17.16b, v18.16b}, v0.8b + cmhi v27.8b, v0.8b, v22.8b // = -1 if sgr_x_by_x[v0] < 3 + cmhi v4.8b, v0.8b, v23.8b // = -1 if sgr_x_by_x[v0] < 2 + add v25.8b, v25.8b, v26.8b + cmhi v5.8b, v0.8b, v24.8b // = -1 if sgr_x_by_x[v0] < 1 + add v27.8b, v27.8b, v4.8b + add v5.8b, v5.8b, v19.8b + add v25.8b, v25.8b, v27.8b + add v1.8b, v1.8b, v5.8b + add v1.8b, v1.8b, v25.8b + uxtl v1.8h, v1.8b // x + + umull v3.4s, v1.4h, v2.4h // x * BB[i] + umull2 v4.4s, v1.8h, v2.8h // x * BB[i] + mul v3.4s, v3.4s, v30.4s // x * BB[i] * sgr_one_by_x + mul v4.4s, v4.4s, v30.4s // x * BB[i] * sgr_one_by_x + srshr v3.4s, v3.4s, #12 // AA[i] + srshr v4.4s, v4.4s, #12 // AA[i] + sub v2.8h, v29.8h, v1.8h // 256 - x + + st1 {v3.4s, v4.4s}, [x0], #32 + st1 {v2.8h}, [x1], #16 + b.gt 1b + + subs x3, x3, #1 + b.le 0f + add x0, x0, x7, lsl #2 + add x1, x1, x7, lsl #1 + mov x2, x6 + b 1b +0: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/looprestoration_tmpl.S b/third_party/dav1d/src/arm/64/looprestoration_tmpl.S new file mode 100644 index 0000000000..7cdfd6f3f7 --- /dev/null +++ b/third_party/dav1d/src/arm/64/looprestoration_tmpl.S @@ -0,0 +1,597 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" + +#define FILTER_OUT_STRIDE 384 + +.macro sgr_funcs bpc +// void dav1d_sgr_finish_filter1_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter1_\bpc\()bpc_neon, export=1 + sub x7, x3, #(4*SUM_STRIDE) + add x8, x3, #(4*SUM_STRIDE) + sub x9, x4, #(2*SUM_STRIDE) + add x10, x4, #(2*SUM_STRIDE) + mov x11, #SUM_STRIDE + mov x12, #FILTER_OUT_STRIDE + add x13, x5, #7 + bic x13, x13, #7 // Aligned width +.if \bpc == 8 + sub x2, x2, x13 +.else + sub x2, x2, x13, lsl #1 +.endif + sub x12, x12, x13 + sub x11, x11, x13 + sub x11, x11, #4 // We read 4 extra elements from a + sub x14, x11, #4 // We read 8 extra elements from b + mov x13, x5 + movi v6.8h, #3 + movi v7.4s, #3 +1: + ld1 {v0.8h, v1.8h}, [x9], #32 + ld1 {v2.8h, v3.8h}, [x4], #32 + ld1 {v4.8h, v5.8h}, [x10], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x7], #48 + ld1 {v19.4s, v20.4s, v21.4s}, [x3], #48 + ld1 {v22.4s, v23.4s, v24.4s}, [x8], #48 + +2: + subs x5, x5, #8 + ext v25.16b, v0.16b, v1.16b, #2 // -stride + ext v26.16b, v2.16b, v3.16b, #2 // 0 + ext v27.16b, v4.16b, v5.16b, #2 // +stride + ext v28.16b, v0.16b, v1.16b, #4 // +1-stride + ext v29.16b, v2.16b, v3.16b, #4 // +1 + ext v30.16b, v4.16b, v5.16b, #4 // +1+stride + add v2.8h, v2.8h, v25.8h // -1, -stride + add v26.8h, v26.8h, v27.8h // 0, +stride + add v0.8h, v0.8h, v28.8h // -1-stride, +1-stride + add v2.8h, v2.8h, v26.8h + add v4.8h, v4.8h, v30.8h // -1+stride, +1+stride + add v2.8h, v2.8h, v29.8h // +1 + add v0.8h, v0.8h, v4.8h + + ext v25.16b, v16.16b, v17.16b, #4 // -stride + ext v26.16b, v17.16b, v18.16b, #4 + shl v2.8h, v2.8h, #2 + ext v27.16b, v16.16b, v17.16b, #8 // +1-stride + ext v28.16b, v17.16b, v18.16b, #8 + ext v29.16b, v19.16b, v20.16b, #4 // 0 + ext v30.16b, v20.16b, v21.16b, #4 + mla v2.8h, v0.8h, v6.8h // * 3 -> a + add v25.4s, v25.4s, v19.4s // -stride, -1 + add v26.4s, v26.4s, v20.4s + add v16.4s, v16.4s, v27.4s // -1-stride, +1-stride + add v17.4s, v17.4s, v28.4s + ext v27.16b, v19.16b, v20.16b, #8 // +1 + ext v28.16b, v20.16b, v21.16b, #8 + add v16.4s, v16.4s, v22.4s // -1+stride + add v17.4s, v17.4s, v23.4s + add v29.4s, v29.4s, v27.4s // 0, +1 + add v30.4s, v30.4s, v28.4s + add v25.4s, v25.4s, v29.4s + add v26.4s, v26.4s, v30.4s + ext v27.16b, v22.16b, v23.16b, #4 // +stride + ext v28.16b, v23.16b, v24.16b, #4 + ext v29.16b, v22.16b, v23.16b, #8 // +1+stride + ext v30.16b, v23.16b, v24.16b, #8 +.if \bpc == 8 + ld1 {v19.8b}, [x1], #8 // src +.else + ld1 {v19.8h}, [x1], #16 // src +.endif + add v25.4s, v25.4s, v27.4s // +stride + add v26.4s, v26.4s, v28.4s + add v16.4s, v16.4s, v29.4s // +1+stride + add v17.4s, v17.4s, v30.4s + shl v25.4s, v25.4s, #2 + shl v26.4s, v26.4s, #2 + mla v25.4s, v16.4s, v7.4s // * 3 -> b + mla v26.4s, v17.4s, v7.4s +.if \bpc == 8 + uxtl v19.8h, v19.8b // src +.endif + mov v0.16b, v1.16b + umlal v25.4s, v2.4h, v19.4h // b + a * src + umlal2 v26.4s, v2.8h, v19.8h + mov v2.16b, v3.16b + rshrn v25.4h, v25.4s, #9 + rshrn2 v25.8h, v26.4s, #9 + mov v4.16b, v5.16b + st1 {v25.8h}, [x0], #16 + + b.le 3f + mov v16.16b, v18.16b + mov v19.16b, v21.16b + mov v22.16b, v24.16b + ld1 {v1.8h}, [x9], #16 + ld1 {v3.8h}, [x4], #16 + ld1 {v5.8h}, [x10], #16 + ld1 {v17.4s, v18.4s}, [x7], #32 + ld1 {v20.4s, v21.4s}, [x3], #32 + ld1 {v23.4s, v24.4s}, [x8], #32 + b 2b + +3: + subs x6, x6, #1 + b.le 0f + mov x5, x13 + add x0, x0, x12, lsl #1 + add x1, x1, x2 + add x3, x3, x11, lsl #2 + add x7, x7, x11, lsl #2 + add x8, x8, x11, lsl #2 + add x4, x4, x14, lsl #1 + add x9, x9, x14, lsl #1 + add x10, x10, x14, lsl #1 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_finish_filter2_Xbpc_neon(int16_t *tmp, +// const pixel *src, const ptrdiff_t stride, +// const int32_t *a, const int16_t *b, +// const int w, const int h); +function sgr_finish_filter2_\bpc\()bpc_neon, export=1 + add x7, x3, #(4*(SUM_STRIDE)) + sub x3, x3, #(4*(SUM_STRIDE)) + add x8, x4, #(2*(SUM_STRIDE)) + sub x4, x4, #(2*(SUM_STRIDE)) + mov x9, #(2*SUM_STRIDE) + mov x10, #FILTER_OUT_STRIDE + add x11, x5, #7 + bic x11, x11, #7 // Aligned width +.if \bpc == 8 + sub x2, x2, x11 +.else + sub x2, x2, x11, lsl #1 +.endif + sub x10, x10, x11 + sub x9, x9, x11 + sub x9, x9, #4 // We read 4 extra elements from a + sub x12, x9, #4 // We read 8 extra elements from b + mov x11, x5 + movi v4.8h, #5 + movi v5.4s, #5 + movi v6.8h, #6 + movi v7.4s, #6 +1: + ld1 {v0.8h, v1.8h}, [x4], #32 + ld1 {v2.8h, v3.8h}, [x8], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 + ld1 {v19.4s, v20.4s, v21.4s}, [x7], #48 + +2: + subs x5, x5, #8 + ext v24.16b, v0.16b, v1.16b, #4 // +1-stride + ext v25.16b, v2.16b, v3.16b, #4 // +1+stride + ext v22.16b, v0.16b, v1.16b, #2 // -stride + ext v23.16b, v2.16b, v3.16b, #2 // +stride + add v0.8h, v0.8h, v24.8h // -1-stride, +1-stride + add v25.8h, v2.8h, v25.8h // -1+stride, +1+stride + add v2.8h, v22.8h, v23.8h // -stride, +stride + add v0.8h, v0.8h, v25.8h + + ext v22.16b, v16.16b, v17.16b, #4 // -stride + ext v23.16b, v17.16b, v18.16b, #4 + ext v24.16b, v19.16b, v20.16b, #4 // +stride + ext v25.16b, v20.16b, v21.16b, #4 + ext v26.16b, v16.16b, v17.16b, #8 // +1-stride + ext v27.16b, v17.16b, v18.16b, #8 + ext v28.16b, v19.16b, v20.16b, #8 // +1+stride + ext v29.16b, v20.16b, v21.16b, #8 + mul v0.8h, v0.8h, v4.8h // * 5 + mla v0.8h, v2.8h, v6.8h // * 6 +.if \bpc == 8 + ld1 {v31.8b}, [x1], #8 +.else + ld1 {v31.8h}, [x1], #16 +.endif + add v16.4s, v16.4s, v26.4s // -1-stride, +1-stride + add v17.4s, v17.4s, v27.4s + add v19.4s, v19.4s, v28.4s // -1+stride, +1+stride + add v20.4s, v20.4s, v29.4s + add v16.4s, v16.4s, v19.4s + add v17.4s, v17.4s, v20.4s + + add v22.4s, v22.4s, v24.4s // -stride, +stride + add v23.4s, v23.4s, v25.4s + // This is, surprisingly, faster than other variants where the + // mul+mla pairs are further apart, on Cortex A53. + mul v16.4s, v16.4s, v5.4s // * 5 + mla v16.4s, v22.4s, v7.4s // * 6 + mul v17.4s, v17.4s, v5.4s // * 5 + mla v17.4s, v23.4s, v7.4s // * 6 + +.if \bpc == 8 + uxtl v31.8h, v31.8b +.endif + umlal v16.4s, v0.4h, v31.4h // b + a * src + umlal2 v17.4s, v0.8h, v31.8h + mov v0.16b, v1.16b + rshrn v16.4h, v16.4s, #9 + rshrn2 v16.8h, v17.4s, #9 + mov v2.16b, v3.16b + st1 {v16.8h}, [x0], #16 + + b.le 3f + mov v16.16b, v18.16b + mov v19.16b, v21.16b + ld1 {v1.8h}, [x4], #16 + ld1 {v3.8h}, [x8], #16 + ld1 {v17.4s, v18.4s}, [x3], #32 + ld1 {v20.4s, v21.4s}, [x7], #32 + b 2b + +3: + subs x6, x6, #1 + b.le 0f + mov x5, x11 + add x0, x0, x10, lsl #1 + add x1, x1, x2 + add x3, x3, x9, lsl #2 + add x7, x7, x9, lsl #2 + add x4, x4, x12, lsl #1 + add x8, x8, x12, lsl #1 + mov x13, x3 + mov x14, x4 + + ld1 {v0.8h, v1.8h}, [x4], #32 + ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 + +4: + subs x5, x5, #8 + ext v23.16b, v0.16b, v1.16b, #4 // +1 + ext v22.16b, v0.16b, v1.16b, #2 // 0 + add v0.8h, v0.8h, v23.8h // -1, +1 + + ext v24.16b, v16.16b, v17.16b, #4 // 0 + ext v25.16b, v17.16b, v18.16b, #4 + ext v26.16b, v16.16b, v17.16b, #8 // +1 + ext v27.16b, v17.16b, v18.16b, #8 + mul v2.8h, v22.8h, v6.8h // * 6 + mla v2.8h, v0.8h, v4.8h // * 5 -> a +.if \bpc == 8 + ld1 {v31.8b}, [x1], #8 +.else + ld1 {v31.8h}, [x1], #16 +.endif + add v16.4s, v16.4s, v26.4s // -1, +1 + add v17.4s, v17.4s, v27.4s +.if \bpc == 8 + uxtl v31.8h, v31.8b +.endif + // This is, surprisingly, faster than other variants where the + // mul+mla pairs are further apart, on Cortex A53. + mul v24.4s, v24.4s, v7.4s // * 6 + mla v24.4s, v16.4s, v5.4s // * 5 -> b + mul v25.4s, v25.4s, v7.4s // * 6 + mla v25.4s, v17.4s, v5.4s // * 5 -> b + + umlal v24.4s, v2.4h, v31.4h // b + a * src + umlal2 v25.4s, v2.8h, v31.8h + mov v0.16b, v1.16b + rshrn v24.4h, v24.4s, #8 + rshrn2 v24.8h, v25.4s, #8 + mov v16.16b, v18.16b + st1 {v24.8h}, [x0], #16 + + b.le 5f + ld1 {v1.8h}, [x4], #16 + ld1 {v17.4s, v18.4s}, [x3], #32 + b 4b + +5: + subs x6, x6, #1 + b.le 0f + mov x5, x11 + add x0, x0, x10, lsl #1 + add x1, x1, x2 + mov x3, x13 // Rewind x3/x4 to where they started + mov x4, x14 + b 1b +0: + ret +endfunc + +// void dav1d_sgr_weighted1_Xbpc_neon(pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int w, const int h, +// const int wt, const int bitdepth_max); +function sgr_weighted1_\bpc\()bpc_neon, export=1 +.if \bpc == 16 + ldr w8, [sp] +.endif + dup v31.8h, w7 + cmp x6, #2 +.if \bpc == 16 + dup v30.8h, w8 +.endif + add x9, x0, x1 + add x10, x2, x3 + add x11, x4, #2*FILTER_OUT_STRIDE + mov x7, #(4*FILTER_OUT_STRIDE) + lsl x1, x1, #1 + lsl x3, x3, #1 + add x8, x5, #7 + bic x8, x8, #7 // Aligned width +.if \bpc == 8 + sub x1, x1, x8 + sub x3, x3, x8 +.else + sub x1, x1, x8, lsl #1 + sub x3, x3, x8, lsl #1 +.endif + sub x7, x7, x8, lsl #1 + mov x8, x5 + b.lt 2f +1: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 + ld1 {v4.8b}, [x10], #8 +.else + ld1 {v0.8h}, [x2], #16 + ld1 {v4.8h}, [x10], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v5.8h}, [x11], #16 + subs x5, x5, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u + ushll v4.8h, v4.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u + shl v4.8h, v4.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v5.8h, v5.8h, v4.8h // t1 - u + ushll v2.4s, v0.4h, #7 // u << 7 + ushll2 v3.4s, v0.8h, #7 // u << 7 + ushll v6.4s, v4.4h, #7 // u << 7 + ushll2 v7.4s, v4.8h, #7 // u << 7 + smlal v2.4s, v1.4h, v31.4h // v + smlal2 v3.4s, v1.8h, v31.8h // v + smlal v6.4s, v5.4h, v31.4h // v + smlal2 v7.4s, v5.8h, v31.8h // v +.if \bpc == 8 + rshrn v2.4h, v2.4s, #11 + rshrn2 v2.8h, v3.4s, #11 + rshrn v6.4h, v6.4s, #11 + rshrn2 v6.8h, v7.4s, #11 + sqxtun v2.8b, v2.8h + sqxtun v6.8b, v6.8h + st1 {v2.8b}, [x0], #8 + st1 {v6.8b}, [x9], #8 +.else + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + sqrshrun v6.4h, v6.4s, #11 + sqrshrun2 v6.8h, v7.4s, #11 + umin v2.8h, v2.8h, v30.8h + umin v6.8h, v6.8h, v30.8h + st1 {v2.8h}, [x0], #16 + st1 {v6.8h}, [x9], #16 +.endif + b.gt 1b + + sub x6, x6, #2 + cmp x6, #1 + b.lt 0f + mov x5, x8 + add x0, x0, x1 + add x9, x9, x1 + add x2, x2, x3 + add x10, x10, x3 + add x4, x4, x7 + add x11, x11, x7 + b.eq 2f + b 1b + +2: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 +.else + ld1 {v0.8h}, [x2], #16 +.endif + ld1 {v1.8h}, [x4], #16 + subs x5, x5, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + ushll v2.4s, v0.4h, #7 // u << 7 + ushll2 v3.4s, v0.8h, #7 // u << 7 + smlal v2.4s, v1.4h, v31.4h // v + smlal2 v3.4s, v1.8h, v31.8h // v +.if \bpc == 8 + rshrn v2.4h, v2.4s, #11 + rshrn2 v2.8h, v3.4s, #11 + sqxtun v2.8b, v2.8h + st1 {v2.8b}, [x0], #8 +.else + sqrshrun v2.4h, v2.4s, #11 + sqrshrun2 v2.8h, v3.4s, #11 + umin v2.8h, v2.8h, v30.8h + st1 {v2.8h}, [x0], #16 +.endif + b.gt 2b +0: + ret +endfunc + +// void dav1d_sgr_weighted2_Xbpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *t1, const int16_t *t2, +// const int w, const int h, +// const int16_t wt[2], const int bitdepth_max); +function sgr_weighted2_\bpc\()bpc_neon, export=1 +.if \bpc == 8 + ldr x8, [sp] +.else + ldp x8, x9, [sp] +.endif + cmp x7, #2 + add x10, x0, x1 + add x11, x2, x3 + add x12, x4, #2*FILTER_OUT_STRIDE + add x13, x5, #2*FILTER_OUT_STRIDE + ld2r {v30.8h, v31.8h}, [x8] // wt[0], wt[1] +.if \bpc == 16 + dup v29.8h, w9 +.endif + mov x8, #4*FILTER_OUT_STRIDE + lsl x1, x1, #1 + lsl x3, x3, #1 + add x9, x6, #7 + bic x9, x9, #7 // Aligned width +.if \bpc == 8 + sub x1, x1, x9 + sub x3, x3, x9 +.else + sub x1, x1, x9, lsl #1 + sub x3, x3, x9, lsl #1 +.endif + sub x8, x8, x9, lsl #1 + mov x9, x6 + b.lt 2f +1: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 + ld1 {v16.8b}, [x11], #8 +.else + ld1 {v0.8h}, [x2], #16 + ld1 {v16.8h}, [x11], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v17.8h}, [x12], #16 + ld1 {v2.8h}, [x5], #16 + ld1 {v18.8h}, [x13], #16 + subs x6, x6, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u + ushll v16.8h, v16.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u + shl v16.8h, v16.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v2.8h, v2.8h, v0.8h // t2 - u + sub v17.8h, v17.8h, v16.8h // t1 - u + sub v18.8h, v18.8h, v16.8h // t2 - u + ushll v3.4s, v0.4h, #7 // u << 7 + ushll2 v4.4s, v0.8h, #7 // u << 7 + ushll v19.4s, v16.4h, #7 // u << 7 + ushll2 v20.4s, v16.8h, #7 // u << 7 + smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) + smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) + smlal v19.4s, v17.4h, v30.4h // wt[0] * (t1 - u) + smlal v19.4s, v18.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v20.4s, v17.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v20.4s, v18.8h, v31.8h // wt[1] * (t2 - u) +.if \bpc == 8 + rshrn v3.4h, v3.4s, #11 + rshrn2 v3.8h, v4.4s, #11 + rshrn v19.4h, v19.4s, #11 + rshrn2 v19.8h, v20.4s, #11 + sqxtun v3.8b, v3.8h + sqxtun v19.8b, v19.8h + st1 {v3.8b}, [x0], #8 + st1 {v19.8b}, [x10], #8 +.else + sqrshrun v3.4h, v3.4s, #11 + sqrshrun2 v3.8h, v4.4s, #11 + sqrshrun v19.4h, v19.4s, #11 + sqrshrun2 v19.8h, v20.4s, #11 + umin v3.8h, v3.8h, v29.8h + umin v19.8h, v19.8h, v29.8h + st1 {v3.8h}, [x0], #16 + st1 {v19.8h}, [x10], #16 +.endif + b.gt 1b + + subs x7, x7, #2 + cmp x7, #1 + b.lt 0f + mov x6, x9 + add x0, x0, x1 + add x10, x10, x1 + add x2, x2, x3 + add x11, x11, x3 + add x4, x4, x8 + add x12, x12, x8 + add x5, x5, x8 + add x13, x13, x8 + b.eq 2f + b 1b + +2: +.if \bpc == 8 + ld1 {v0.8b}, [x2], #8 +.else + ld1 {v0.8h}, [x2], #16 +.endif + ld1 {v1.8h}, [x4], #16 + ld1 {v2.8h}, [x5], #16 + subs x6, x6, #8 +.if \bpc == 8 + ushll v0.8h, v0.8b, #4 // u +.else + shl v0.8h, v0.8h, #4 // u +.endif + sub v1.8h, v1.8h, v0.8h // t1 - u + sub v2.8h, v2.8h, v0.8h // t2 - u + ushll v3.4s, v0.4h, #7 // u << 7 + ushll2 v4.4s, v0.8h, #7 // u << 7 + smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) + smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) + smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) + smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) +.if \bpc == 8 + rshrn v3.4h, v3.4s, #11 + rshrn2 v3.8h, v4.4s, #11 + sqxtun v3.8b, v3.8h + st1 {v3.8b}, [x0], #8 +.else + sqrshrun v3.4h, v3.4s, #11 + sqrshrun2 v3.8h, v4.4s, #11 + umin v3.8h, v3.8h, v29.8h + st1 {v3.8h}, [x0], #16 +.endif + b.gt 1b +0: + ret +endfunc +.endm diff --git a/third_party/dav1d/src/arm/64/mc.S b/third_party/dav1d/src/arm/64/mc.S new file mode 100644 index 0000000000..9f7b4e7a89 --- /dev/null +++ b/third_party/dav1d/src/arm/64/mc.S @@ -0,0 +1,3310 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * Copyright © 2018, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +.macro avg dst, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + add \t0\().8h, \t0\().8h, \t2\().8h + add \t1\().8h, \t1\().8h, \t3\().8h + sqrshrun \dst\().8b, \t0\().8h, #5 + sqrshrun2 \dst\().16b, \t1\().8h, #5 +.endm + +.macro w_avg dst, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + sub \t0\().8h, \t2\().8h, \t0\().8h + sub \t1\().8h, \t3\().8h, \t1\().8h + sqdmulh \t0\().8h, \t0\().8h, v30.8h + sqdmulh \t1\().8h, \t1\().8h, v30.8h + add \t0\().8h, \t2\().8h, \t0\().8h + add \t1\().8h, \t3\().8h, \t1\().8h + sqrshrun \dst\().8b, \t0\().8h, #4 + sqrshrun2 \dst\().16b, \t1\().8h, #4 +.endm + +.macro mask dst, t0, t1, t2, t3 + ld1 {v30.16b}, [x6], 16 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + mul v30.16b, v30.16b, v31.16b + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + shll v28.8h, v30.8b, #8 + shll2 v29.8h, v30.16b, #8 + sub \t0\().8h, \t2\().8h, \t0\().8h + sub \t1\().8h, \t3\().8h, \t1\().8h + sqdmulh \t0\().8h, \t0\().8h, v28.8h + sqdmulh \t1\().8h, \t1\().8h, v29.8h + add \t0\().8h, \t2\().8h, \t0\().8h + add \t1\().8h, \t3\().8h, \t1\().8h + sqrshrun \dst\().8b, \t0\().8h, #4 + sqrshrun2 \dst\().16b, \t1\().8h, #4 +.endm + +.macro bidir_fn type +function \type\()_8bpc_neon, export=1 + clz w4, w4 +.ifc \type, w_avg + dup v30.8h, w6 + neg v30.8h, v30.8h + shl v30.8h, v30.8h, #11 +.endif +.ifc \type, mask + movi v31.16b, #256-2 +.endif + adr x7, L(\type\()_tbl) + sub w4, w4, #24 + ldrh w4, [x7, x4, lsl #1] + \type v4, v0, v1, v2, v3 + sub x7, x7, w4, uxtw + br x7 +40: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +4: + cmp w5, #4 + st1 {v4.s}[0], [x0], x1 + st1 {v4.s}[1], [x7], x1 + st1 {v4.s}[2], [x0], x1 + st1 {v4.s}[3], [x7], x1 + b.eq 0f + \type v5, v0, v1, v2, v3 + cmp w5, #8 + st1 {v5.s}[0], [x0], x1 + st1 {v5.s}[1], [x7], x1 + st1 {v5.s}[2], [x0], x1 + st1 {v5.s}[3], [x7], x1 + b.eq 0f + \type v4, v0, v1, v2, v3 + st1 {v4.s}[0], [x0], x1 + st1 {v4.s}[1], [x7], x1 + \type v5, v0, v1, v2, v3 + st1 {v4.s}[2], [x0], x1 + st1 {v4.s}[3], [x7], x1 + st1 {v5.s}[0], [x0], x1 + st1 {v5.s}[1], [x7], x1 + st1 {v5.s}[2], [x0], x1 + st1 {v5.s}[3], [x7], x1 + ret +80: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +8: + st1 {v4.d}[0], [x0], x1 + \type v5, v0, v1, v2, v3 + st1 {v4.d}[1], [x7], x1 + st1 {v5.d}[0], [x0], x1 + subs w5, w5, #4 + st1 {v5.d}[1], [x7], x1 + b.le 0f + \type v4, v0, v1, v2, v3 + b 8b +16: + AARCH64_VALID_JUMP_TARGET + \type v5, v0, v1, v2, v3 + st1 {v4.16b}, [x0], x1 + \type v6, v0, v1, v2, v3 + st1 {v5.16b}, [x0], x1 + \type v7, v0, v1, v2, v3 + st1 {v6.16b}, [x0], x1 + subs w5, w5, #4 + st1 {v7.16b}, [x0], x1 + b.le 0f + \type v4, v0, v1, v2, v3 + b 16b +320: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +32: + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 + st1 {v4.16b,v5.16b}, [x0], x1 + \type v7, v0, v1, v2, v3 + subs w5, w5, #2 + st1 {v6.16b,v7.16b}, [x7], x1 + b.le 0f + \type v4, v0, v1, v2, v3 + b 32b +640: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +64: + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 + \type v16, v0, v1, v2, v3 + \type v17, v0, v1, v2, v3 + st1 {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1 + \type v18, v0, v1, v2, v3 + \type v19, v0, v1, v2, v3 + subs w5, w5, #2 + st1 {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1 + b.le 0f + \type v4, v0, v1, v2, v3 + b 64b +1280: + AARCH64_VALID_JUMP_TARGET + add x7, x0, #64 +128: + \type v5, v0, v1, v2, v3 + \type v6, v0, v1, v2, v3 + \type v7, v0, v1, v2, v3 + \type v16, v0, v1, v2, v3 + \type v17, v0, v1, v2, v3 + st1 {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1 + \type v18, v0, v1, v2, v3 + \type v19, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1 + b.le 0f + \type v4, v0, v1, v2, v3 + b 128b +0: + ret +L(\type\()_tbl): + .hword L(\type\()_tbl) - 1280b + .hword L(\type\()_tbl) - 640b + .hword L(\type\()_tbl) - 320b + .hword L(\type\()_tbl) - 16b + .hword L(\type\()_tbl) - 80b + .hword L(\type\()_tbl) - 40b +endfunc +.endm + +bidir_fn avg +bidir_fn w_avg +bidir_fn mask + + +.macro w_mask_fn type +function w_mask_\type\()_8bpc_neon, export=1 + clz w8, w4 + adr x9, L(w_mask_\type\()_tbl) + sub w8, w8, #24 + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + mov w10, #6903 + dup v0.8h, w10 +.if \type == 444 + movi v1.16b, #64 +.elseif \type == 422 + dup v2.8b, w7 + movi v3.8b, #129 + sub v3.8b, v3.8b, v2.8b +.elseif \type == 420 + dup v2.8h, w7 + movi v3.8h, #1, lsl #8 + sub v3.8h, v3.8h, v2.8h +.endif + add x12, x0, x1 + lsl x1, x1, #1 + br x9 +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 (four rows at once) + ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2 (four rows at once) + subs w5, w5, #4 + sub v16.8h, v6.8h, v4.8h + sub v17.8h, v7.8h, v5.8h + sabd v18.8h, v4.8h, v6.8h + sabd v19.8h, v5.8h, v7.8h + uqsub v18.8h, v0.8h, v18.8h + uqsub v19.8h, v0.8h, v19.8h + ushr v18.8h, v18.8h, #8 + ushr v19.8h, v19.8h, #8 + shl v20.8h, v18.8h, #9 + shl v21.8h, v19.8h, #9 + sqdmulh v20.8h, v20.8h, v16.8h + sqdmulh v21.8h, v21.8h, v17.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v5.8h + sqrshrun v22.8b, v20.8h, #4 + sqrshrun v23.8b, v21.8h, #4 +.if \type == 444 + uzp1 v18.16b, v18.16b, v19.16b // Same as xtn, xtn2 + sub v18.16b, v1.16b, v18.16b + st1 {v18.16b}, [x6], #16 +.elseif \type == 422 + addp v18.8h, v18.8h, v19.8h + xtn v18.8b, v18.8h + uhsub v18.8b, v3.8b, v18.8b + st1 {v18.8b}, [x6], #8 +.elseif \type == 420 + trn1 v24.2d, v18.2d, v19.2d + trn2 v25.2d, v18.2d, v19.2d + add v24.8h, v24.8h, v25.8h + addp v18.8h, v24.8h, v24.8h + sub v18.4h, v3.4h, v18.4h + rshrn v18.8b, v18.8h, #2 + st1 {v18.s}[0], [x6], #4 +.endif + st1 {v22.s}[0], [x0], x1 + st1 {v22.s}[1], [x12], x1 + st1 {v23.s}[0], [x0], x1 + st1 {v23.s}[1], [x12], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8h, v5.8h}, [x2], #32 + ld1 {v6.8h, v7.8h}, [x3], #32 + subs w5, w5, #2 + sub v16.8h, v6.8h, v4.8h + sub v17.8h, v7.8h, v5.8h + sabd v18.8h, v4.8h, v6.8h + sabd v19.8h, v5.8h, v7.8h + uqsub v18.8h, v0.8h, v18.8h + uqsub v19.8h, v0.8h, v19.8h + ushr v18.8h, v18.8h, #8 + ushr v19.8h, v19.8h, #8 + shl v20.8h, v18.8h, #9 + shl v21.8h, v19.8h, #9 + sqdmulh v20.8h, v20.8h, v16.8h + sqdmulh v21.8h, v21.8h, v17.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v5.8h + sqrshrun v22.8b, v20.8h, #4 + sqrshrun v23.8b, v21.8h, #4 +.if \type == 444 + uzp1 v18.16b, v18.16b, v19.16b // Same as xtn, xtn2 + sub v18.16b, v1.16b, v18.16b + st1 {v18.16b}, [x6], #16 +.elseif \type == 422 + addp v18.8h, v18.8h, v19.8h + xtn v18.8b, v18.8h + uhsub v18.8b, v3.8b, v18.8b + st1 {v18.8b}, [x6], #8 +.elseif \type == 420 + add v18.8h, v18.8h, v19.8h + addp v18.8h, v18.8h, v18.8h + sub v18.4h, v3.4h, v18.4h + rshrn v18.8b, v18.8h, #2 + st1 {v18.s}[0], [x6], #4 +.endif + st1 {v22.8b}, [x0], x1 + st1 {v23.8b}, [x12], x1 + b.gt 8b + ret +1280: +640: +320: +160: + AARCH64_VALID_JUMP_TARGET + mov w11, w4 + sub x1, x1, w4, uxtw +.if \type == 444 + add x10, x6, w4, uxtw +.elseif \type == 422 + add x10, x6, x11, lsr #1 +.endif + add x9, x3, w4, uxtw #1 + add x7, x2, w4, uxtw #1 +161: + mov w8, w4 +16: + ld1 {v4.8h, v5.8h}, [x2], #32 + ld1 {v6.8h, v7.8h}, [x3], #32 + ld1 {v16.8h, v17.8h}, [x7], #32 + ld1 {v18.8h, v19.8h}, [x9], #32 + subs w8, w8, #16 + sub v6.8h, v6.8h, v4.8h + sub v7.8h, v7.8h, v5.8h + sub v18.8h, v18.8h, v16.8h + sub v19.8h, v19.8h, v17.8h + abs v20.8h, v6.8h + abs v21.8h, v7.8h + abs v22.8h, v18.8h + abs v23.8h, v19.8h + uqsub v20.8h, v0.8h, v20.8h + uqsub v21.8h, v0.8h, v21.8h + uqsub v22.8h, v0.8h, v22.8h + uqsub v23.8h, v0.8h, v23.8h + ushr v20.8h, v20.8h, #8 + ushr v21.8h, v21.8h, #8 + ushr v22.8h, v22.8h, #8 + ushr v23.8h, v23.8h, #8 + shl v24.8h, v20.8h, #9 + shl v25.8h, v21.8h, #9 + shl v26.8h, v22.8h, #9 + shl v27.8h, v23.8h, #9 + sqdmulh v24.8h, v24.8h, v6.8h + sqdmulh v25.8h, v25.8h, v7.8h + sqdmulh v26.8h, v26.8h, v18.8h + sqdmulh v27.8h, v27.8h, v19.8h + add v24.8h, v24.8h, v4.8h + add v25.8h, v25.8h, v5.8h + add v26.8h, v26.8h, v16.8h + add v27.8h, v27.8h, v17.8h + sqrshrun v24.8b, v24.8h, #4 + sqrshrun v25.8b, v25.8h, #4 + sqrshrun v26.8b, v26.8h, #4 + sqrshrun v27.8b, v27.8h, #4 +.if \type == 444 + uzp1 v20.16b, v20.16b, v21.16b // Same as xtn, xtn2 + uzp1 v21.16b, v22.16b, v23.16b // Ditto + sub v20.16b, v1.16b, v20.16b + sub v21.16b, v1.16b, v21.16b + st1 {v20.16b}, [x6], #16 + st1 {v21.16b}, [x10], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h + addp v21.8h, v22.8h, v23.8h + xtn v20.8b, v20.8h + xtn v21.8b, v21.8h + uhsub v20.8b, v3.8b, v20.8b + uhsub v21.8b, v3.8b, v21.8b + st1 {v20.8b}, [x6], #8 + st1 {v21.8b}, [x10], #8 +.elseif \type == 420 + add v20.8h, v20.8h, v22.8h + add v21.8h, v21.8h, v23.8h + addp v20.8h, v20.8h, v21.8h + sub v20.8h, v3.8h, v20.8h + rshrn v20.8b, v20.8h, #2 + st1 {v20.8b}, [x6], #8 +.endif + st1 {v24.8b, v25.8b}, [x0], #16 + st1 {v26.8b, v27.8b}, [x12], #16 + b.gt 16b + subs w5, w5, #2 + add x2, x2, w4, uxtw #1 + add x3, x3, w4, uxtw #1 + add x7, x7, w4, uxtw #1 + add x9, x9, w4, uxtw #1 +.if \type == 444 + add x6, x6, w4, uxtw + add x10, x10, w4, uxtw +.elseif \type == 422 + add x6, x6, x11, lsr #1 + add x10, x10, x11, lsr #1 +.endif + add x0, x0, x1 + add x12, x12, x1 + b.gt 161b + ret +L(w_mask_\type\()_tbl): + .hword L(w_mask_\type\()_tbl) - 1280b + .hword L(w_mask_\type\()_tbl) - 640b + .hword L(w_mask_\type\()_tbl) - 320b + .hword L(w_mask_\type\()_tbl) - 160b + .hword L(w_mask_\type\()_tbl) - 8b + .hword L(w_mask_\type\()_tbl) - 4b +endfunc +.endm + +w_mask_fn 444 +w_mask_fn 422 +w_mask_fn 420 + + +function blend_8bpc_neon, export=1 + adr x6, L(blend_tbl) + clz w3, w3 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + movi v4.16b, #64 + add x8, x0, x1 + lsl x1, x1, #1 + br x6 +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v2.8b}, [x5], #8 + ld1 {v1.d}[0], [x2], #8 + ld1 {v0.s}[0], [x0] + subs w4, w4, #2 + ld1 {v0.s}[1], [x8] + sub v3.8b, v4.8b, v2.8b + umull v5.8h, v1.8b, v2.8b + umlal v5.8h, v0.8b, v3.8b + rshrn v6.8b, v5.8h, #6 + st1 {v6.s}[0], [x0], x1 + st1 {v6.s}[1], [x8], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v2.16b}, [x5], #16 + ld1 {v1.16b}, [x2], #16 + ld1 {v0.d}[0], [x0] + ld1 {v0.d}[1], [x8] + sub v3.16b, v4.16b, v2.16b + subs w4, w4, #2 + umull v5.8h, v1.8b, v2.8b + umlal v5.8h, v0.8b, v3.8b + umull2 v6.8h, v1.16b, v2.16b + umlal2 v6.8h, v0.16b, v3.16b + rshrn v7.8b, v5.8h, #6 + rshrn2 v7.16b, v6.8h, #6 + st1 {v7.d}[0], [x0], x1 + st1 {v7.d}[1], [x8], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld1 {v1.16b, v2.16b}, [x5], #32 + ld1 {v5.16b, v6.16b}, [x2], #32 + ld1 {v0.16b}, [x0] + subs w4, w4, #2 + sub v7.16b, v4.16b, v1.16b + sub v20.16b, v4.16b, v2.16b + ld1 {v3.16b}, [x8] + umull v16.8h, v5.8b, v1.8b + umlal v16.8h, v0.8b, v7.8b + umull2 v17.8h, v5.16b, v1.16b + umlal2 v17.8h, v0.16b, v7.16b + umull v21.8h, v6.8b, v2.8b + umlal v21.8h, v3.8b, v20.8b + umull2 v22.8h, v6.16b, v2.16b + umlal2 v22.8h, v3.16b, v20.16b + rshrn v18.8b, v16.8h, #6 + rshrn2 v18.16b, v17.8h, #6 + rshrn v19.8b, v21.8h, #6 + rshrn2 v19.16b, v22.8h, #6 + st1 {v18.16b}, [x0], x1 + st1 {v19.16b}, [x8], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64 + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], #64 + ld1 {v20.16b, v21.16b}, [x0] + subs w4, w4, #2 + ld1 {v22.16b, v23.16b}, [x8] + sub v5.16b, v4.16b, v0.16b + sub v6.16b, v4.16b, v1.16b + sub v30.16b, v4.16b, v2.16b + sub v31.16b, v4.16b, v3.16b + umull v24.8h, v16.8b, v0.8b + umlal v24.8h, v20.8b, v5.8b + umull2 v26.8h, v16.16b, v0.16b + umlal2 v26.8h, v20.16b, v5.16b + umull v28.8h, v17.8b, v1.8b + umlal v28.8h, v21.8b, v6.8b + umull2 v7.8h, v17.16b, v1.16b + umlal2 v7.8h, v21.16b, v6.16b + umull v27.8h, v18.8b, v2.8b + umlal v27.8h, v22.8b, v30.8b + umull2 v1.8h, v18.16b, v2.16b + umlal2 v1.8h, v22.16b, v30.16b + umull v29.8h, v19.8b, v3.8b + umlal v29.8h, v23.8b, v31.8b + umull2 v21.8h, v19.16b, v3.16b + umlal2 v21.8h, v23.16b, v31.16b + rshrn v24.8b, v24.8h, #6 + rshrn2 v24.16b, v26.8h, #6 + rshrn v25.8b, v28.8h, #6 + rshrn2 v25.16b, v7.8h, #6 + rshrn v27.8b, v27.8h, #6 + rshrn2 v27.16b, v1.8h, #6 + rshrn v28.8b, v29.8h, #6 + rshrn2 v28.16b, v21.8h, #6 + st1 {v24.16b, v25.16b}, [x0], x1 + st1 {v27.16b, v28.16b}, [x8], x1 + b.gt 32b + ret +L(blend_tbl): + .hword L(blend_tbl) - 32b + .hword L(blend_tbl) - 16b + .hword L(blend_tbl) - 8b + .hword L(blend_tbl) - 4b +endfunc + +function blend_h_8bpc_neon, export=1 + adr x6, L(blend_h_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w4, uxtw + sub w4, w4, w4, lsr #2 + clz w7, w3 + movi v4.16b, #64 + add x8, x0, x1 + lsl x1, x1, #1 + sub w7, w7, #24 + ldrh w7, [x6, x7, lsl #1] + sub x6, x6, w7, uxtw + br x6 +2: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.h}[0], [x5], #2 + ld1 {v1.s}[0], [x2], #4 + subs w4, w4, #2 + ld1 {v2.h}[0], [x0] + zip1 v0.8b, v0.8b, v0.8b + sub v3.8b, v4.8b, v0.8b + ld1 {v2.h}[1], [x8] + umull v5.8h, v1.8b, v0.8b + umlal v5.8h, v2.8b, v3.8b + rshrn v5.8b, v5.8h, #6 + st1 {v5.h}[0], [x0], x1 + st1 {v5.h}[1], [x8], x1 + b.gt 2b + ret +4: + AARCH64_VALID_JUMP_TARGET + ld2r {v0.8b, v1.8b}, [x5], #2 + ld1 {v2.8b}, [x2], #8 + subs w4, w4, #2 + ext v0.8b, v0.8b, v1.8b, #4 + ld1 {v3.s}[0], [x0] + sub v5.8b, v4.8b, v0.8b + ld1 {v3.s}[1], [x8] + umull v6.8h, v2.8b, v0.8b + umlal v6.8h, v3.8b, v5.8b + rshrn v6.8b, v6.8h, #6 + st1 {v6.s}[0], [x0], x1 + st1 {v6.s}[1], [x8], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld2r {v0.16b, v1.16b}, [x5], #2 + ld1 {v2.16b}, [x2], #16 + ld1 {v3.d}[0], [x0] + ext v0.16b, v0.16b, v1.16b, #8 + sub v5.16b, v4.16b, v0.16b + ld1 {v3.d}[1], [x8] + subs w4, w4, #2 + umull v6.8h, v0.8b, v2.8b + umlal v6.8h, v3.8b, v5.8b + umull2 v7.8h, v0.16b, v2.16b + umlal2 v7.8h, v3.16b, v5.16b + rshrn v16.8b, v6.8h, #6 + rshrn2 v16.16b, v7.8h, #6 + st1 {v16.d}[0], [x0], x1 + st1 {v16.d}[1], [x8], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld2r {v0.16b, v1.16b}, [x5], #2 + ld1 {v2.16b, v3.16b}, [x2], #32 + ld1 {v5.16b}, [x0] + sub v7.16b, v4.16b, v0.16b + sub v16.16b, v4.16b, v1.16b + ld1 {v6.16b}, [x8] + subs w4, w4, #2 + umull v17.8h, v0.8b, v2.8b + umlal v17.8h, v5.8b, v7.8b + umull2 v18.8h, v0.16b, v2.16b + umlal2 v18.8h, v5.16b, v7.16b + umull v19.8h, v1.8b, v3.8b + umlal v19.8h, v6.8b, v16.8b + umull2 v20.8h, v1.16b, v3.16b + umlal2 v20.8h, v6.16b, v16.16b + rshrn v21.8b, v17.8h, #6 + rshrn2 v21.16b, v18.8h, #6 + rshrn v22.8b, v19.8h, #6 + rshrn2 v22.16b, v20.8h, #6 + st1 {v21.16b}, [x0], x1 + st1 {v22.16b}, [x8], x1 + b.gt 16b + ret +1280: +640: +320: + AARCH64_VALID_JUMP_TARGET + sub x1, x1, w3, uxtw + add x7, x2, w3, uxtw +321: + ld2r {v0.16b, v1.16b}, [x5], #2 + mov w6, w3 + sub v20.16b, v4.16b, v0.16b + sub v21.16b, v4.16b, v1.16b +32: + ld1 {v16.16b, v17.16b}, [x2], #32 + ld1 {v2.16b, v3.16b}, [x0] + subs w6, w6, #32 + umull v23.8h, v0.8b, v16.8b + umlal v23.8h, v2.8b, v20.8b + ld1 {v18.16b, v19.16b}, [x7], #32 + umull2 v27.8h, v0.16b, v16.16b + umlal2 v27.8h, v2.16b, v20.16b + ld1 {v6.16b, v7.16b}, [x8] + umull v24.8h, v0.8b, v17.8b + umlal v24.8h, v3.8b, v20.8b + umull2 v28.8h, v0.16b, v17.16b + umlal2 v28.8h, v3.16b, v20.16b + umull v25.8h, v1.8b, v18.8b + umlal v25.8h, v6.8b, v21.8b + umull2 v5.8h, v1.16b, v18.16b + umlal2 v5.8h, v6.16b, v21.16b + rshrn v29.8b, v23.8h, #6 + rshrn2 v29.16b, v27.8h, #6 + umull v26.8h, v1.8b, v19.8b + umlal v26.8h, v7.8b, v21.8b + umull2 v31.8h, v1.16b, v19.16b + umlal2 v31.8h, v7.16b, v21.16b + rshrn v30.8b, v24.8h, #6 + rshrn2 v30.16b, v28.8h, #6 + rshrn v23.8b, v25.8h, #6 + rshrn2 v23.16b, v5.8h, #6 + rshrn v24.8b, v26.8h, #6 + st1 {v29.16b, v30.16b}, [x0], #32 + rshrn2 v24.16b, v31.8h, #6 + st1 {v23.16b, v24.16b}, [x8], #32 + b.gt 32b + subs w4, w4, #2 + add x0, x0, x1 + add x8, x8, x1 + add x2, x2, w3, uxtw + add x7, x7, w3, uxtw + b.gt 321b + ret +L(blend_h_tbl): + .hword L(blend_h_tbl) - 1280b + .hword L(blend_h_tbl) - 640b + .hword L(blend_h_tbl) - 320b + .hword L(blend_h_tbl) - 16b + .hword L(blend_h_tbl) - 8b + .hword L(blend_h_tbl) - 4b + .hword L(blend_h_tbl) - 2b +endfunc + +function blend_v_8bpc_neon, export=1 + adr x6, L(blend_v_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w3, uxtw + clz w3, w3 + movi v4.16b, #64 + add x8, x0, x1 + lsl x1, x1, #1 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + br x6 +20: + AARCH64_VALID_JUMP_TARGET + ld1r {v0.8b}, [x5] + sub v1.8b, v4.8b, v0.8b +2: + ld1 {v2.h}[0], [x2], #2 + ld1 {v3.b}[0], [x0] + subs w4, w4, #2 + ld1 {v2.b}[1], [x2] + ld1 {v3.b}[1], [x8] + umull v5.8h, v2.8b, v0.8b + umlal v5.8h, v3.8b, v1.8b + rshrn v5.8b, v5.8h, #6 + add x2, x2, #2 + st1 {v5.b}[0], [x0], x1 + st1 {v5.b}[1], [x8], x1 + b.gt 2b + ret +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2s}, [x5] + sub x1, x1, #2 + sub v1.8b, v4.8b, v0.8b +4: + ld1 {v2.8b}, [x2], #8 + ld1 {v3.s}[0], [x0] + ld1 {v3.s}[1], [x8] + subs w4, w4, #2 + umull v5.8h, v2.8b, v0.8b + umlal v5.8h, v3.8b, v1.8b + rshrn v5.8b, v5.8h, #6 + st1 {v5.h}[0], [x0], #2 + st1 {v5.h}[2], [x8], #2 + st1 {v5.b}[2], [x0], x1 + st1 {v5.b}[6], [x8], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1r {v0.2d}, [x5] + sub x1, x1, #4 + sub v1.16b, v4.16b, v0.16b +8: + ld1 {v2.16b}, [x2], #16 + ld1 {v3.d}[0], [x0] + ld1 {v3.d}[1], [x8] + subs w4, w4, #2 + umull v5.8h, v0.8b, v2.8b + umlal v5.8h, v3.8b, v1.8b + umull2 v6.8h, v0.16b, v2.16b + umlal2 v6.8h, v3.16b, v1.16b + rshrn v7.8b, v5.8h, #6 + rshrn2 v7.16b, v6.8h, #6 + st1 {v7.s}[0], [x0], #4 + st1 {v7.s}[2], [x8], #4 + st1 {v7.h}[2], [x0], x1 + st1 {v7.h}[6], [x8], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b}, [x5] + sub x1, x1, #8 + sub v2.16b, v4.16b, v0.16b +16: + ld1 {v5.16b, v6.16b}, [x2], #32 + ld1 {v7.16b}, [x0] + subs w4, w4, #2 + ld1 {v16.16b}, [x8] + umull v17.8h, v5.8b, v0.8b + umlal v17.8h, v7.8b, v2.8b + umull2 v18.8h, v5.16b, v0.16b + umlal2 v18.8h, v7.16b, v2.16b + umull v20.8h, v6.8b, v0.8b + umlal v20.8h, v16.8b, v2.8b + umull2 v21.8h, v6.16b, v0.16b + umlal2 v21.8h, v16.16b, v2.16b + rshrn v19.8b, v17.8h, #6 + rshrn2 v19.16b, v18.8h, #6 + rshrn v22.8b, v20.8h, #6 + rshrn2 v22.16b, v21.8h, #6 + st1 {v19.8b}, [x0], #8 + st1 {v22.8b}, [x8], #8 + st1 {v19.s}[2], [x0], x1 + st1 {v22.s}[2], [x8], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.16b, v1.16b}, [x5] + sub x1, x1, #16 + sub v2.16b, v4.16b, v0.16b + sub v3.8b, v4.8b, v1.8b +32: + ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], #64 + ld1 {v5.16b, v6.16b}, [x0] + subs w4, w4, #2 + ld1 {v20.16b, v21.16b}, [x8] + umull v22.8h, v16.8b, v0.8b + umlal v22.8h, v5.8b, v2.8b + umull2 v23.8h, v16.16b, v0.16b + umlal2 v23.8h, v5.16b, v2.16b + umull v28.8h, v17.8b, v1.8b + umlal v28.8h, v6.8b, v3.8b + umull v30.8h, v18.8b, v0.8b + umlal v30.8h, v20.8b, v2.8b + umull2 v31.8h, v18.16b, v0.16b + umlal2 v31.8h, v20.16b, v2.16b + umull v25.8h, v19.8b, v1.8b + umlal v25.8h, v21.8b, v3.8b + rshrn v24.8b, v22.8h, #6 + rshrn2 v24.16b, v23.8h, #6 + rshrn v28.8b, v28.8h, #6 + rshrn v30.8b, v30.8h, #6 + rshrn2 v30.16b, v31.8h, #6 + rshrn v27.8b, v25.8h, #6 + st1 {v24.16b}, [x0], #16 + st1 {v30.16b}, [x8], #16 + st1 {v28.8b}, [x0], x1 + st1 {v27.8b}, [x8], x1 + b.gt 32b + ret +L(blend_v_tbl): + .hword L(blend_v_tbl) - 320b + .hword L(blend_v_tbl) - 160b + .hword L(blend_v_tbl) - 80b + .hword L(blend_v_tbl) - 40b + .hword L(blend_v_tbl) - 20b +endfunc + + +// This has got the same signature as the put_8tap functions, +// and assumes that x8 is set to (clz(w)-24). +function put_neon + adr x9, L(put_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +2: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.h}[0], [x2], x3 + ld1 {v1.h}[0], [x2], x3 + subs w5, w5, #2 + st1 {v0.h}[0], [x0], x1 + st1 {v1.h}[0], [x0], x1 + b.gt 2b + ret +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x2], x3 + ld1 {v1.s}[0], [x2], x3 + subs w5, w5, #2 + st1 {v0.s}[0], [x0], x1 + st1 {v1.s}[0], [x0], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x2], x3 + ld1 {v1.8b}, [x2], x3 + subs w5, w5, #2 + st1 {v0.8b}, [x0], x1 + st1 {v1.8b}, [x0], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + add x8, x0, x1 + lsl x1, x1, #1 + add x9, x2, x3 + lsl x3, x3, #1 +16: + ld1 {v0.16b}, [x2], x3 + ld1 {v1.16b}, [x9], x3 + subs w5, w5, #2 + st1 {v0.16b}, [x0], x1 + st1 {v1.16b}, [x8], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + subs w5, w5, #1 + stp x8, x9, [x0, #16] + add x2, x2, x3 + add x0, x0, x1 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + ldp x10, x11, [x2, #32] + stp x8, x9, [x0, #16] + subs w5, w5, #1 + ldp x12, x13, [x2, #48] + stp x10, x11, [x0, #32] + stp x12, x13, [x0, #48] + add x2, x2, x3 + add x0, x0, x1 + b.gt 64b + ret +128: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x2] + ldp q2, q3, [x2, #32] + stp q0, q1, [x0] + ldp q4, q5, [x2, #64] + stp q2, q3, [x0, #32] + ldp q6, q7, [x2, #96] + subs w5, w5, #1 + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + add x2, x2, x3 + add x0, x0, x1 + b.gt 128b + ret + +L(put_tbl): + .hword L(put_tbl) - 128b + .hword L(put_tbl) - 64b + .hword L(put_tbl) - 32b + .hword L(put_tbl) - 160b + .hword L(put_tbl) - 8b + .hword L(put_tbl) - 4b + .hword L(put_tbl) - 2b +endfunc + + +// This has got the same signature as the prep_8tap functions, +// and assumes that x8 is set to (clz(w)-24), and x7 to w*2. +function prep_neon + adr x9, L(prep_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x1], x2 + ld1 {v1.s}[0], [x1], x2 + subs w4, w4, #2 + ushll v0.8h, v0.8b, #4 + ushll v1.8h, v1.8b, #4 + st1 {v0.4h, v1.4h}, [x0], #16 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [x1], x2 + ld1 {v1.8b}, [x1], x2 + subs w4, w4, #2 + ushll v0.8h, v0.8b, #4 + ushll v1.8h, v1.8b, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + add x9, x1, x2 + lsl x2, x2, #1 +16: + ld1 {v0.16b}, [x1], x2 + ld1 {v1.16b}, [x9], x2 + subs w4, w4, #2 + ushll v4.8h, v0.8b, #4 + ushll2 v5.8h, v0.16b, #4 + ushll v6.8h, v1.8b, #4 + ushll2 v7.8h, v1.16b, #4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + add x8, x0, w3, uxtw +32: + ld1 {v0.16b, v1.16b}, [x1], x2 + subs w4, w4, #2 + ushll v4.8h, v0.8b, #4 + ushll2 v5.8h, v0.16b, #4 + ld1 {v2.16b, v3.16b}, [x1], x2 + ushll v6.8h, v1.8b, #4 + ushll2 v7.8h, v1.16b, #4 + ushll v16.8h, v2.8b, #4 + st1 {v4.8h, v5.8h}, [x0], x7 + ushll2 v17.8h, v2.16b, #4 + st1 {v6.8h, v7.8h}, [x8], x7 + ushll v18.8h, v3.8b, #4 + st1 {v16.8h, v17.8h}, [x0], x7 + ushll2 v19.8h, v3.16b, #4 + st1 {v18.8h, v19.8h}, [x8], x7 + b.gt 32b + ret +640: + AARCH64_VALID_JUMP_TARGET + add x8, x0, #32 + mov x6, #64 +64: + ldp q0, q1, [x1] + subs w4, w4, #1 + ushll v4.8h, v0.8b, #4 + ushll2 v5.8h, v0.16b, #4 + ldp q2, q3, [x1, #32] + ushll v6.8h, v1.8b, #4 + ushll2 v7.8h, v1.16b, #4 + add x1, x1, x2 + ushll v16.8h, v2.8b, #4 + st1 {v4.8h, v5.8h}, [x0], x6 + ushll2 v17.8h, v2.16b, #4 + ushll v18.8h, v3.8b, #4 + st1 {v6.8h, v7.8h}, [x8], x6 + ushll2 v19.8h, v3.16b, #4 + st1 {v16.8h, v17.8h}, [x0], x6 + st1 {v18.8h, v19.8h}, [x8], x6 + b.gt 64b + ret +1280: + AARCH64_VALID_JUMP_TARGET + add x8, x0, #64 + mov x6, #128 +128: + ldp q0, q1, [x1] + ldp q2, q3, [x1, #32] + ushll v16.8h, v0.8b, #4 + ushll2 v17.8h, v0.16b, #4 + ushll v18.8h, v1.8b, #4 + ushll2 v19.8h, v1.16b, #4 + ushll v20.8h, v2.8b, #4 + ushll2 v21.8h, v2.16b, #4 + ldp q4, q5, [x1, #64] + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x0], x6 + ushll v22.8h, v3.8b, #4 + ushll2 v23.8h, v3.16b, #4 + ushll v24.8h, v4.8b, #4 + ushll2 v25.8h, v4.16b, #4 + ushll v26.8h, v5.8b, #4 + ushll2 v27.8h, v5.16b, #4 + ldp q6, q7, [x1, #96] + st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x8], x6 + ushll v28.8h, v6.8b, #4 + ushll2 v29.8h, v6.16b, #4 + ushll v30.8h, v7.8b, #4 + ushll2 v31.8h, v7.16b, #4 + subs w4, w4, #1 + add x1, x1, x2 + st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x0], x6 + st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x6 + b.gt 128b + ret + +L(prep_tbl): + .hword L(prep_tbl) - 1280b + .hword L(prep_tbl) - 640b + .hword L(prep_tbl) - 320b + .hword L(prep_tbl) - 160b + .hword L(prep_tbl) - 8b + .hword L(prep_tbl) - 4b +endfunc + + +.macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}[0], [\s0], \strd + ld1 {\d1\wd}[0], [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}[0], [\s0], \strd + ld1 {\d3\wd}[0], [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}[0], [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}[0], [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}[0], [\s0], \strd +.endif +.endm +.macro load_reg s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}, [\s0], \strd + ld1 {\d1\wd}, [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}, [\s0], \strd + ld1 {\d3\wd}, [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}, [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}, [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}, [\s0], \strd +.endif +.endm +.macro load_h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, .h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_s s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, .s, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_8b s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .8b, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_16b s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .16b, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro interleave_1 wd, r0, r1, r2, r3, r4 + trn1 \r0\wd, \r0\wd, \r1\wd + trn1 \r1\wd, \r1\wd, \r2\wd +.ifnb \r3 + trn1 \r2\wd, \r2\wd, \r3\wd + trn1 \r3\wd, \r3\wd, \r4\wd +.endif +.endm +.macro interleave_1_h r0, r1, r2, r3, r4 + interleave_1 .4h, \r0, \r1, \r2, \r3, \r4 +.endm +.macro interleave_1_s r0, r1, r2, r3, r4 + interleave_1 .2s, \r0, \r1, \r2, \r3, \r4 +.endm +.macro interleave_2 wd, r0, r1, r2, r3, r4, r5 + trn1 \r0\wd, \r0\wd, \r2\wd + trn1 \r1\wd, \r1\wd, \r3\wd + trn1 \r2\wd, \r2\wd, \r4\wd + trn1 \r3\wd, \r3\wd, \r5\wd +.endm +.macro interleave_2_s r0, r1, r2, r3, r4, r5 + interleave_2 .2s, \r0, \r1, \r2, \r3, \r4, \r5 +.endm +.macro uxtl_b r0, r1, r2, r3, r4, r5, r6 + uxtl \r0\().8h, \r0\().8b + uxtl \r1\().8h, \r1\().8b +.ifnb \r2 + uxtl \r2\().8h, \r2\().8b + uxtl \r3\().8h, \r3\().8b +.endif +.ifnb \r4 + uxtl \r4\().8h, \r4\().8b +.endif +.ifnb \r5 + uxtl \r5\().8h, \r5\().8b +.endif +.ifnb \r6 + uxtl \r6\().8h, \r6\().8b +.endif +.endm +.macro mul_mla_4 d, s0, s1, s2, s3, wd + mul \d\wd, \s0\wd, v0.h[0] + mla \d\wd, \s1\wd, v0.h[1] + mla \d\wd, \s2\wd, v0.h[2] + mla \d\wd, \s3\wd, v0.h[3] +.endm +// Interleaving the mul/mla chains actually hurts performance +// significantly on Cortex A53, thus keeping mul/mla tightly +// chained like this. +.macro mul_mla_8_0_4h d0, s0, s1, s2, s3, s4, s5, s6, s7 + mul \d0\().4h, \s0\().4h, v0.h[0] + mla \d0\().4h, \s1\().4h, v0.h[1] + mla \d0\().4h, \s2\().4h, v0.h[2] + mla \d0\().4h, \s3\().4h, v0.h[3] + mla \d0\().4h, \s4\().4h, v0.h[4] + mla \d0\().4h, \s5\().4h, v0.h[5] + mla \d0\().4h, \s6\().4h, v0.h[6] + mla \d0\().4h, \s7\().4h, v0.h[7] +.endm +.macro mul_mla_8_0 d0, s0, s1, s2, s3, s4, s5, s6, s7 + mul \d0\().8h, \s0\().8h, v0.h[0] + mla \d0\().8h, \s1\().8h, v0.h[1] + mla \d0\().8h, \s2\().8h, v0.h[2] + mla \d0\().8h, \s3\().8h, v0.h[3] + mla \d0\().8h, \s4\().8h, v0.h[4] + mla \d0\().8h, \s5\().8h, v0.h[5] + mla \d0\().8h, \s6\().8h, v0.h[6] + mla \d0\().8h, \s7\().8h, v0.h[7] +.endm +.macro mul_mla_8_1 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8 + mul \d0\().8h, \s0\().8h, v0.h[0] + mla \d0\().8h, \s1\().8h, v0.h[1] + mla \d0\().8h, \s2\().8h, v0.h[2] + mla \d0\().8h, \s3\().8h, v0.h[3] + mla \d0\().8h, \s4\().8h, v0.h[4] + mla \d0\().8h, \s5\().8h, v0.h[5] + mla \d0\().8h, \s6\().8h, v0.h[6] + mla \d0\().8h, \s7\().8h, v0.h[7] + mul \d1\().8h, \s1\().8h, v0.h[0] + mla \d1\().8h, \s2\().8h, v0.h[1] + mla \d1\().8h, \s3\().8h, v0.h[2] + mla \d1\().8h, \s4\().8h, v0.h[3] + mla \d1\().8h, \s5\().8h, v0.h[4] + mla \d1\().8h, \s6\().8h, v0.h[5] + mla \d1\().8h, \s7\().8h, v0.h[6] + mla \d1\().8h, \s8\().8h, v0.h[7] +.endm +.macro mul_mla_8_2 d0, d1, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9 + mul \d0\().8h, \s0\().8h, v0.h[0] + mla \d0\().8h, \s1\().8h, v0.h[1] + mla \d0\().8h, \s2\().8h, v0.h[2] + mla \d0\().8h, \s3\().8h, v0.h[3] + mla \d0\().8h, \s4\().8h, v0.h[4] + mla \d0\().8h, \s5\().8h, v0.h[5] + mla \d0\().8h, \s6\().8h, v0.h[6] + mla \d0\().8h, \s7\().8h, v0.h[7] + mul \d1\().8h, \s2\().8h, v0.h[0] + mla \d1\().8h, \s3\().8h, v0.h[1] + mla \d1\().8h, \s4\().8h, v0.h[2] + mla \d1\().8h, \s5\().8h, v0.h[3] + mla \d1\().8h, \s6\().8h, v0.h[4] + mla \d1\().8h, \s7\().8h, v0.h[5] + mla \d1\().8h, \s8\().8h, v0.h[6] + mla \d1\().8h, \s9\().8h, v0.h[7] +.endm +.macro sqrshrun_b shift, r0, r1, r2, r3 + sqrshrun \r0\().8b, \r0\().8h, #\shift +.ifnb \r1 + sqrshrun \r1\().8b, \r1\().8h, #\shift +.endif +.ifnb \r2 + sqrshrun \r2\().8b, \r2\().8h, #\shift + sqrshrun \r3\().8b, \r3\().8h, #\shift +.endif +.endm +.macro srshr_h shift, r0, r1, r2, r3 + srshr \r0\().8h, \r0\().8h, #\shift +.ifnb \r1 + srshr \r1\().8h, \r1\().8h, #\shift +.endif +.ifnb \r2 + srshr \r2\().8h, \r2\().8h, #\shift + srshr \r3\().8h, \r3\().8h, #\shift +.endif +.endm +.macro st_h strd, reg, lanes + st1 {\reg\().h}[0], [x0], \strd + st1 {\reg\().h}[1], [x8], \strd +.if \lanes > 2 + st1 {\reg\().h}[2], [x0], \strd + st1 {\reg\().h}[3], [x8], \strd +.endif +.endm +.macro st_s strd, r0, r1 + st1 {\r0\().s}[0], [x0], \strd + st1 {\r0\().s}[1], [x8], \strd +.ifnb \r1 + st1 {\r1\().s}[0], [x0], \strd + st1 {\r1\().s}[1], [x8], \strd +.endif +.endm +.macro st_d strd, r0, r1 + st1 {\r0\().d}[0], [x0], \strd + st1 {\r0\().d}[1], [x8], \strd +.ifnb \r1 + st1 {\r1\().d}[0], [x0], \strd + st1 {\r1\().d}[1], [x8], \strd +.endif +.endm +.macro shift_store_4 type, strd, r0, r1 +.ifc \type, put + sqrshrun_b 6, \r0, \r1 + st_s \strd, \r0, \r1 +.else + srshr_h 2, \r0, \r1 + st_d \strd, \r0, \r1 +.endif +.endm +.macro st_reg strd, wd, r0, r1, r2, r3, r4, r5, r6, r7 + st1 {\r0\wd}, [x0], \strd + st1 {\r1\wd}, [x8], \strd +.ifnb \r2 + st1 {\r2\wd}, [x0], \strd + st1 {\r3\wd}, [x8], \strd +.endif +.ifnb \r4 + st1 {\r4\wd}, [x0], \strd + st1 {\r5\wd}, [x8], \strd + st1 {\r6\wd}, [x0], \strd + st1 {\r7\wd}, [x8], \strd +.endif +.endm +.macro st_8b strd, r0, r1, r2, r3, r4, r5, r6, r7 + st_reg \strd, .8b, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endm +.macro st_16b strd, r0, r1, r2, r3, r4, r5, r6, r7 + st_reg \strd, .16b, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endm +.macro shift_store_8 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_b 6, \r0, \r1, \r2, \r3 + st_8b \strd, \r0, \r1, \r2, \r3 +.else + srshr_h 2, \r0, \r1, \r2, \r3 + st_16b \strd, \r0, \r1, \r2, \r3 +.endif +.endm +.macro shift_store_16 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun \r0\().8b, \r0\().8h, #6 + sqrshrun2 \r0\().16b, \r1\().8h, #6 + sqrshrun \r2\().8b, \r2\().8h, #6 + sqrshrun2 \r2\().16b, \r3\().8h, #6 + st_16b \strd, \r0, \r2 +.else + srshr_h 2, \r0, \r1, \r2, \r3 + st1 {\r0\().8h, \r1\().8h}, [x0], \strd + st1 {\r2\().8h, \r3\().8h}, [x8], \strd +.endif +.endm + +.macro make_8tap_fn op, type, type_h, type_v +function \op\()_8tap_\type\()_8bpc_neon, export=1 + mov x8, \type_h + mov x9, \type_v + b \op\()_8tap_neon +endfunc +.endm + +// No spaces in these expressions, due to gas-preprocessor. +#define REGULAR ((0*15<<7)|3*15) +#define SMOOTH ((1*15<<7)|4*15) +#define SHARP ((2*15<<7)|3*15) + +.macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, xmx, my, xmy, ds2, sr2, shift_hv +make_8tap_fn \type, regular, REGULAR, REGULAR +make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH +make_8tap_fn \type, regular_sharp, REGULAR, SHARP +make_8tap_fn \type, smooth, SMOOTH, SMOOTH +make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR +make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP +make_8tap_fn \type, sharp, SHARP, SHARP +make_8tap_fn \type, sharp_regular, SHARP, REGULAR +make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH + +function \type\()_8tap_neon + mov w10, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + mul \mx, \mx, w10 + mul \my, \my, w10 + add \mx, \mx, w8 // mx, 8tap_h, 4tap_h + add \my, \my, w9 // my, 8tap_v, 4tap_v +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + clz w8, \w + tst \mx, #(0x7f << 14) + sub w8, w8, #24 + movrel x10, X(mc_subpel_filters), -8 + b.ne L(\type\()_8tap_h) + tst \my, #(0x7f << 14) + b.ne L(\type\()_8tap_v) + b \type\()_neon + +L(\type\()_8tap_h): + cmp \w, #4 + ubfx w9, \mx, #7, #7 + and \mx, \mx, #0x7f + b.le 4f + mov \mx, w9 +4: + tst \my, #(0x7f << 14) + add \xmx, x10, \mx, uxtw #3 + b.ne L(\type\()_8tap_hv) + + adr x9, L(\type\()_8tap_h_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: // 2xN h + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #1 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +2: + ld1 {v4.8b}, [\src], \s_strd + ld1 {v6.8b}, [\sr2], \s_strd + uxtl v4.8h, v4.8b + uxtl v6.8h, v6.8b + ext v5.16b, v4.16b, v4.16b, #2 + ext v7.16b, v6.16b, v6.16b, #2 + subs \h, \h, #2 + trn1 v3.2s, v4.2s, v6.2s + trn2 v6.2s, v4.2s, v6.2s + trn1 v4.2s, v5.2s, v7.2s + trn2 v7.2s, v5.2s, v7.2s + mul v3.4h, v3.4h, v0.h[0] + mla v3.4h, v4.4h, v0.h[1] + mla v3.4h, v6.4h, v0.h[2] + mla v3.4h, v7.4h, v0.h[3] + srshr v3.4h, v3.4h, #2 + sqrshrun v3.8b, v3.8h, #4 + st1 {v3.h}[0], [\dst], \d_strd + st1 {v3.h}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + AARCH64_VALID_JUMP_TARGET + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #1 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +4: + ld1 {v16.8b}, [\src], \s_strd + ld1 {v20.8b}, [\sr2], \s_strd + uxtl v16.8h, v16.8b + uxtl v20.8h, v20.8b + ext v17.16b, v16.16b, v16.16b, #2 + ext v18.16b, v16.16b, v16.16b, #4 + ext v19.16b, v16.16b, v16.16b, #6 + ext v21.16b, v20.16b, v20.16b, #2 + ext v22.16b, v20.16b, v20.16b, #4 + ext v23.16b, v20.16b, v20.16b, #6 + subs \h, \h, #2 + mul v16.4h, v16.4h, v0.h[0] + mla v16.4h, v17.4h, v0.h[1] + mla v16.4h, v18.4h, v0.h[2] + mla v16.4h, v19.4h, v0.h[3] + mul v20.4h, v20.4h, v0.h[0] + mla v20.4h, v21.4h, v0.h[1] + mla v20.4h, v22.4h, v0.h[2] + mla v20.4h, v23.4h, v0.h[3] + srshr v16.4h, v16.4h, #2 + srshr v20.4h, v20.4h, #2 +.ifc \type, put + sqrshrun v16.8b, v16.8h, #4 + sqrshrun v20.8b, v20.8h, #4 + st1 {v16.s}[0], [\dst], \d_strd + st1 {v20.s}[0], [\ds2], \d_strd +.else + st1 {v16.4h}, [\dst], \d_strd + st1 {v20.4h}, [\ds2], \d_strd +.endif + b.gt 4b + ret + +80: // 8xN h + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmx] + sub \src, \src, #3 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +8: + ld1 {v16.8b, v17.8b}, [\src], \s_strd + ld1 {v20.8b, v21.8b}, [\sr2], \s_strd + uxtl v16.8h, v16.8b + uxtl v17.8h, v17.8b + uxtl v20.8h, v20.8b + uxtl v21.8h, v21.8b + + mul v18.8h, v16.8h, v0.h[0] + mul v22.8h, v20.8h, v0.h[0] +.irpc i, 1234567 + ext v19.16b, v16.16b, v17.16b, #(2*\i) + ext v23.16b, v20.16b, v21.16b, #(2*\i) + mla v18.8h, v19.8h, v0.h[\i] + mla v22.8h, v23.8h, v0.h[\i] +.endr + subs \h, \h, #2 + srshr v18.8h, v18.8h, #2 + srshr v22.8h, v22.8h, #2 +.ifc \type, put + sqrshrun v18.8b, v18.8h, #4 + sqrshrun v22.8b, v22.8h, #4 + st1 {v18.8b}, [\dst], \d_strd + st1 {v22.8b}, [\ds2], \d_strd +.else + st1 {v18.8h}, [\dst], \d_strd + st1 {v22.8h}, [\ds2], \d_strd +.endif + b.gt 8b + ret +160: +320: +640: +1280: // 16xN, 32xN, ... h + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmx] + sub \src, \src, #3 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + sub \s_strd, \s_strd, \w, uxtw + sub \s_strd, \s_strd, #8 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw +.endif +161: + ld1 {v16.8b, v17.8b, v18.8b}, [\src], #24 + ld1 {v20.8b, v21.8b, v22.8b}, [\sr2], #24 + mov \mx, \w + uxtl v16.8h, v16.8b + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v20.8h, v20.8b + uxtl v21.8h, v21.8b + uxtl v22.8h, v22.8b + +16: + mul v24.8h, v16.8h, v0.h[0] + mul v25.8h, v17.8h, v0.h[0] + mul v26.8h, v20.8h, v0.h[0] + mul v27.8h, v21.8h, v0.h[0] +.irpc i, 1234567 + ext v28.16b, v16.16b, v17.16b, #(2*\i) + ext v29.16b, v17.16b, v18.16b, #(2*\i) + ext v30.16b, v20.16b, v21.16b, #(2*\i) + ext v31.16b, v21.16b, v22.16b, #(2*\i) + mla v24.8h, v28.8h, v0.h[\i] + mla v25.8h, v29.8h, v0.h[\i] + mla v26.8h, v30.8h, v0.h[\i] + mla v27.8h, v31.8h, v0.h[\i] +.endr + srshr v24.8h, v24.8h, #2 + srshr v25.8h, v25.8h, #2 + srshr v26.8h, v26.8h, #2 + srshr v27.8h, v27.8h, #2 + subs \mx, \mx, #16 +.ifc \type, put + sqrshrun v24.8b, v24.8h, #4 + sqrshrun2 v24.16b, v25.8h, #4 + sqrshrun v26.8b, v26.8h, #4 + sqrshrun2 v26.16b, v27.8h, #4 + st1 {v24.16b}, [\dst], #16 + st1 {v26.16b}, [\ds2], #16 +.else + st1 {v24.8h, v25.8h}, [\dst], #32 + st1 {v26.8h, v27.8h}, [\ds2], #32 +.endif + b.le 9f + + mov v16.16b, v18.16b + mov v20.16b, v22.16b + ld1 {v17.8b, v18.8b}, [\src], #16 + ld1 {v21.8b, v22.8b}, [\sr2], #16 + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v21.8h, v21.8b + uxtl v22.8h, v22.8b + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 161b + ret + +L(\type\()_8tap_h_tbl): + .hword L(\type\()_8tap_h_tbl) - 1280b + .hword L(\type\()_8tap_h_tbl) - 640b + .hword L(\type\()_8tap_h_tbl) - 320b + .hword L(\type\()_8tap_h_tbl) - 160b + .hword L(\type\()_8tap_h_tbl) - 80b + .hword L(\type\()_8tap_h_tbl) - 40b + .hword L(\type\()_8tap_h_tbl) - 20b + .hword 0 + + +L(\type\()_8tap_v): + cmp \h, #4 + ubfx w9, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w9 +4: + add \xmy, x10, \my, uxtw #3 + + adr x9, L(\type\()_8tap_v_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: // 2xN v + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + b.gt 28f + + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + // 2x2 v + load_h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + interleave_1_h v1, v2, v3, v4, v5 + b.gt 24f + uxtl_b v1, v2, v3, v4 + mul_mla_4 v6, v1, v2, v3, v4, .4h + sqrshrun_b 6, v6 + st_h \d_strd, v6, 2 + ret + +24: // 2x4 v + load_h \sr2, \src, \s_strd, v6, v7 + interleave_1_h v5, v6, v7 + interleave_2_s v1, v2, v3, v4, v5, v6 + uxtl_b v1, v2, v3, v4 + mul_mla_4 v6, v1, v2, v3, v4, .8h + sqrshrun_b 6, v6 + st_h \d_strd, v6, 4 + ret + +28: // 2x6, 2x8, 2x12, 2x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + load_h \src, \sr2, \s_strd, v1, v2, v3, v4, v5, v6, v7 + interleave_1_h v1, v2, v3, v4, v5 + interleave_1_h v5, v6, v7 + interleave_2_s v1, v2, v3, v4, v5, v6 + uxtl_b v1, v2, v3, v4 +216: + subs \h, \h, #4 + load_h \sr2, \src, \s_strd, v16, v17, v18, v19 + interleave_1_h v7, v16, v17, v18, v19 + interleave_2_s v5, v6, v7, v16, v17, v18 + uxtl_b v5, v6, v7, v16 + mul_mla_8_0 v30, v1, v2, v3, v4, v5, v6, v7, v16 + sqrshrun_b 6, v30 + st_h \d_strd, v30, 4 + b.le 0f + cmp \h, #2 + mov v1.16b, v5.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + mov v4.16b, v16.16b + mov v5.16b, v17.16b + mov v6.16b, v18.16b + mov v7.16b, v19.16b + b.eq 26f + b 216b +26: + load_h \sr2, \src, \s_strd, v16, v17 + interleave_1_h v7, v16, v17 + uxtl_b v5, v6, v7, v16 + mul_mla_8_0_4h v30, v1, v2, v3, v4, v5, v6, v7, v16 + sqrshrun_b 6, v30 + st_h \d_strd, v30, 2 +0: + ret +.endif + +40: + AARCH64_VALID_JUMP_TARGET + b.gt 480f + + // 4x2, 4x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + interleave_1_s v1, v2, v3, v4, v5 + uxtl_b v1, v2, v3, v4 + mul_mla_4 v6, v1, v2, v3, v4, .8h + shift_store_4 \type, \d_strd, v6 + b.le 0f + load_s \sr2, \src, \s_strd, v6, v7 + interleave_1_s v5, v6, v7 + uxtl_b v5, v6 + mul_mla_4 v7, v3, v4, v5, v6, .8h + shift_store_4 \type, \d_strd, v7 +0: + ret + +480: // 4x6, 4x8, 4x12, 4x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_s \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + interleave_1_s v16, v17, v18 + interleave_1_s v18, v19, v20, v21, v22 + uxtl_b v16, v17 + uxtl_b v18, v19, v20, v21 + +48: + subs \h, \h, #4 + load_s \sr2, \src, \s_strd, v23, v24, v25, v26 + interleave_1_s v22, v23, v24, v25, v26 + uxtl_b v22, v23, v24, v25 + mul_mla_8_2 v1, v2, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25 + shift_store_4 \type, \d_strd, v1, v2 + b.le 0f + load_s \sr2, \src, \s_strd, v27, v16 + subs \h, \h, #2 + interleave_1_s v26, v27, v16 + uxtl_b v26, v27 + mul_mla_8_0 v1, v20, v21, v22, v23, v24, v25, v26, v27 + shift_store_4 \type, \d_strd, v1 + b.le 0f + load_s \sr2, \src, \s_strd, v17, v18 + subs \h, \h, #2 + interleave_1_s v16, v17, v18 + uxtl_b v16, v17 + mul_mla_8_0 v2, v22, v23, v24, v25, v26, v27, v16, v17 + shift_store_4 \type, \d_strd, v2 + b.le 0f + subs \h, \h, #4 + load_s \sr2, \src, \s_strd, v19, v20, v21, v22 + interleave_1_s v18, v19, v20, v21, v22 + uxtl_b v18, v19, v20, v21 + mul_mla_8_2 v1, v2, v24, v25, v26, v27, v16, v17, v18, v19, v20, v21 + shift_store_4 \type, \d_strd, v1, v2 + b.gt 48b +0: + ret + +80: + AARCH64_VALID_JUMP_TARGET + b.gt 880f + + // 8x2, 8x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_8b \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + uxtl_b v1, v2, v3, v4, v5 + mul_mla_4 v6, v1, v2, v3, v4, .8h + mul_mla_4 v7, v2, v3, v4, v5, .8h + shift_store_8 \type, \d_strd, v6, v7 + b.le 0f + load_8b \sr2, \src, \s_strd, v6, v7 + uxtl_b v6, v7 + mul_mla_4 v1, v3, v4, v5, v6, .8h + mul_mla_4 v2, v4, v5, v6, v7, .8h + shift_store_8 \type, \d_strd, v1, v2 +0: + ret + +880: // 8x6, 8x8, 8x16, 8x32 v +1680: // 16x8, 16x16, ... +320: // 32x8, 32x16, ... +640: +1280: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmy] + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + mov \my, \h +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + load_8b \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + uxtl_b v16, v17, v18, v19, v20, v21, v22 + +88: + subs \h, \h, #2 + load_8b \sr2, \src, \s_strd, v23, v24 + uxtl_b v23, v24 + mul_mla_8_1 v1, v2, v16, v17, v18, v19, v20, v21, v22, v23, v24 + shift_store_8 \type, \d_strd, v1, v2 + b.le 9f + subs \h, \h, #2 + load_8b \sr2, \src, \s_strd, v25, v26 + uxtl_b v25, v26 + mul_mla_8_1 v3, v4, v18, v19, v20, v21, v22, v23, v24, v25, v26 + shift_store_8 \type, \d_strd, v3, v4 + b.le 9f + subs \h, \h, #2 + load_8b \sr2, \src, \s_strd, v27, v16 + uxtl_b v27, v16 + mul_mla_8_1 v1, v2, v20, v21, v22, v23, v24, v25, v26, v27, v16 + shift_store_8 \type, \d_strd, v1, v2 + b.le 9f + subs \h, \h, #2 + load_8b \sr2, \src, \s_strd, v17, v18 + uxtl_b v17, v18 + mul_mla_8_1 v3, v4, v22, v23, v24, v25, v26, v27, v16, v17, v18 + shift_store_8 \type, \d_strd, v3, v4 + b.le 9f + subs \h, \h, #4 + load_8b \sr2, \src, \s_strd, v19, v20, v21, v22 + uxtl_b v19, v20, v21, v22 + mul_mla_8_1 v1, v2, v24, v25, v26, v27, v16, v17, v18, v19, v20 + mul_mla_8_1 v3, v4, v26, v27, v16, v17, v18, v19, v20, v21, v22 + shift_store_8 \type, \d_strd, v1, v2, v3, v4 + b.gt 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 168b +0: + ret + +160: + AARCH64_VALID_JUMP_TARGET + b.gt 1680b + + // 16x2, 16x4 v + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + cmp \h, #2 + load_16b \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + uxtl v16.8h, v1.8b + uxtl v17.8h, v2.8b + uxtl v18.8h, v3.8b + uxtl v19.8h, v4.8b + uxtl v20.8h, v5.8b + uxtl2 v23.8h, v1.16b + uxtl2 v24.8h, v2.16b + uxtl2 v25.8h, v3.16b + uxtl2 v26.8h, v4.16b + uxtl2 v27.8h, v5.16b + mul_mla_4 v1, v16, v17, v18, v19, .8h + mul_mla_4 v16, v17, v18, v19, v20, .8h + mul_mla_4 v2, v23, v24, v25, v26, .8h + mul_mla_4 v17, v24, v25, v26, v27, .8h + shift_store_16 \type, \d_strd, v1, v2, v16, v17 + b.le 0f + load_16b \sr2, \src, \s_strd, v6, v7 + uxtl v21.8h, v6.8b + uxtl v22.8h, v7.8b + uxtl2 v28.8h, v6.16b + uxtl2 v29.8h, v7.16b + mul_mla_4 v1, v18, v19, v20, v21, .8h + mul_mla_4 v3, v19, v20, v21, v22, .8h + mul_mla_4 v2, v25, v26, v27, v28, .8h + mul_mla_4 v4, v26, v27, v28, v29, .8h + shift_store_16 \type, \d_strd, v1, v2, v3, v4 +0: + ret + +L(\type\()_8tap_v_tbl): + .hword L(\type\()_8tap_v_tbl) - 1280b + .hword L(\type\()_8tap_v_tbl) - 640b + .hword L(\type\()_8tap_v_tbl) - 320b + .hword L(\type\()_8tap_v_tbl) - 160b + .hword L(\type\()_8tap_v_tbl) - 80b + .hword L(\type\()_8tap_v_tbl) - 40b + .hword L(\type\()_8tap_v_tbl) - 20b + .hword 0 + +L(\type\()_8tap_hv): + cmp \h, #4 + ubfx w9, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w9 +4: + add \xmy, x10, \my, uxtw #3 + + adr x9, L(\type\()_8tap_hv_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 280f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + + // 2x2, 2x4 hv + sub \sr2, \src, #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v28.8b}, [\src], \s_strd + uxtl v28.8h, v28.8b + ext v29.16b, v28.16b, v28.16b, #2 + mul v28.4h, v28.4h, v0.4h + mul v29.4h, v29.4h, v0.4h + addp v28.4h, v28.4h, v29.4h + addp v16.4h, v28.4h, v28.4h + srshr v16.4h, v16.4h, #2 + bl L(\type\()_8tap_filter_2) + + trn1 v16.2s, v16.2s, v28.2s + mov v17.8b, v28.8b + +2: + bl L(\type\()_8tap_filter_2) + + ext v18.8b, v17.8b, v28.8b, #4 + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v28.4h, v1.h[3] + + sqrshrn v2.4h, v2.4s, #\shift_hv + sqxtun v2.8b, v2.8h + subs \h, \h, #2 + st1 {v2.h}[0], [\dst], \d_strd + st1 {v2.h}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v28.8b + b 2b + +280: // 2x8, 2x16, 2x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #1 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v28.8b}, [\src], \s_strd + uxtl v28.8h, v28.8b + ext v29.16b, v28.16b, v28.16b, #2 + mul v28.4h, v28.4h, v0.4h + mul v29.4h, v29.4h, v0.4h + addp v28.4h, v28.4h, v29.4h + addp v16.4h, v28.4h, v28.4h + srshr v16.4h, v16.4h, #2 + + bl L(\type\()_8tap_filter_2) + trn1 v16.2s, v16.2s, v28.2s + mov v17.8b, v28.8b + bl L(\type\()_8tap_filter_2) + ext v18.8b, v17.8b, v28.8b, #4 + mov v19.8b, v28.8b + bl L(\type\()_8tap_filter_2) + ext v20.8b, v19.8b, v28.8b, #4 + mov v21.8b, v28.8b + +28: + bl L(\type\()_8tap_filter_2) + ext v22.8b, v21.8b, v28.8b, #4 + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal v2.4s, v20.4h, v1.h[4] + smlal v2.4s, v21.4h, v1.h[5] + smlal v2.4s, v22.4h, v1.h[6] + smlal v2.4s, v28.4h, v1.h[7] + + sqrshrn v2.4h, v2.4s, #\shift_hv + sqxtun v2.8b, v2.8h + subs \h, \h, #2 + st1 {v2.h}[0], [\dst], \d_strd + st1 {v2.h}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v19.8b + mov v18.8b, v20.8b + mov v19.8b, v21.8b + mov v20.8b, v22.8b + mov v21.8b, v28.8b + b 28b + +0: + ret x15 + +L(\type\()_8tap_filter_2): + ld1 {v28.8b}, [\sr2], \s_strd + ld1 {v30.8b}, [\src], \s_strd + uxtl v28.8h, v28.8b + uxtl v30.8h, v30.8b + ext v29.16b, v28.16b, v28.16b, #2 + ext v31.16b, v30.16b, v30.16b, #2 + trn1 v27.2s, v28.2s, v30.2s + trn2 v30.2s, v28.2s, v30.2s + trn1 v28.2s, v29.2s, v31.2s + trn2 v31.2s, v29.2s, v31.2s + mul v27.4h, v27.4h, v0.h[0] + mla v27.4h, v28.4h, v0.h[1] + mla v27.4h, v30.4h, v0.h[2] + mla v27.4h, v31.4h, v0.h[3] + srshr v28.4h, v27.4h, #2 + ret +.endif + +40: + AARCH64_VALID_JUMP_TARGET + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 480f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + sub \sr2, \src, #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + // 4x2, 4x4 hv + ld1 {v26.8b}, [\src], \s_strd + uxtl v26.8h, v26.8b + ext v28.16b, v26.16b, v26.16b, #2 + ext v29.16b, v26.16b, v26.16b, #4 + ext v30.16b, v26.16b, v26.16b, #6 + mul v31.4h, v26.4h, v0.h[0] + mla v31.4h, v28.4h, v0.h[1] + mla v31.4h, v29.4h, v0.h[2] + mla v31.4h, v30.4h, v0.h[3] + srshr v16.4h, v31.4h, #2 + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v28.8b + mov v18.8b, v29.8b + +4: + bl L(\type\()_8tap_filter_4) + // Interleaving the mul/mla chains actually hurts performance + // significantly on Cortex A53, thus keeping mul/mla tightly + // chained like this. + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v28.4h, v1.h[3] + smull v3.4s, v17.4h, v1.h[0] + smlal v3.4s, v18.4h, v1.h[1] + smlal v3.4s, v28.4h, v1.h[2] + smlal v3.4s, v29.4h, v1.h[3] + sqrshrn v2.4h, v2.4s, #\shift_hv + sqrshrn v3.4h, v3.4s, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + sqxtun v2.8b, v2.8h + sqxtun v3.8b, v3.8h + st1 {v2.s}[0], [\dst], \d_strd + st1 {v3.s}[0], [\ds2], \d_strd +.else + st1 {v2.4h}, [\dst], \d_strd + st1 {v3.4h}, [\ds2], \d_strd +.endif + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v28.8b + mov v18.8b, v29.8b + b 4b + +480: // 4x8, 4x16, 4x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #1 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v26.8b}, [\src], \s_strd + uxtl v26.8h, v26.8b + ext v28.16b, v26.16b, v26.16b, #2 + ext v29.16b, v26.16b, v26.16b, #4 + ext v30.16b, v26.16b, v26.16b, #6 + mul v31.4h, v26.4h, v0.h[0] + mla v31.4h, v28.4h, v0.h[1] + mla v31.4h, v29.4h, v0.h[2] + mla v31.4h, v30.4h, v0.h[3] + srshr v16.4h, v31.4h, #2 + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v28.8b + mov v18.8b, v29.8b + bl L(\type\()_8tap_filter_4) + mov v19.8b, v28.8b + mov v20.8b, v29.8b + bl L(\type\()_8tap_filter_4) + mov v21.8b, v28.8b + mov v22.8b, v29.8b + +48: + bl L(\type\()_8tap_filter_4) + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal v2.4s, v20.4h, v1.h[4] + smlal v2.4s, v21.4h, v1.h[5] + smlal v2.4s, v22.4h, v1.h[6] + smlal v2.4s, v28.4h, v1.h[7] + smull v3.4s, v17.4h, v1.h[0] + smlal v3.4s, v18.4h, v1.h[1] + smlal v3.4s, v19.4h, v1.h[2] + smlal v3.4s, v20.4h, v1.h[3] + smlal v3.4s, v21.4h, v1.h[4] + smlal v3.4s, v22.4h, v1.h[5] + smlal v3.4s, v28.4h, v1.h[6] + smlal v3.4s, v29.4h, v1.h[7] + sqrshrn v2.4h, v2.4s, #\shift_hv + sqrshrn v3.4h, v3.4s, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + sqxtun v2.8b, v2.8h + sqxtun v3.8b, v3.8h + st1 {v2.s}[0], [\dst], \d_strd + st1 {v3.s}[0], [\ds2], \d_strd +.else + st1 {v2.4h}, [\dst], \d_strd + st1 {v3.4h}, [\ds2], \d_strd +.endif + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v19.8b + mov v18.8b, v20.8b + mov v19.8b, v21.8b + mov v20.8b, v22.8b + mov v21.8b, v28.8b + mov v22.8b, v29.8b + b 48b +0: + ret x15 + +L(\type\()_8tap_filter_4): + ld1 {v26.8b}, [\sr2], \s_strd + ld1 {v27.8b}, [\src], \s_strd + uxtl v26.8h, v26.8b + uxtl v27.8h, v27.8b + ext v28.16b, v26.16b, v26.16b, #2 + ext v29.16b, v26.16b, v26.16b, #4 + ext v30.16b, v26.16b, v26.16b, #6 + mul v31.4h, v26.4h, v0.h[0] + mla v31.4h, v28.4h, v0.h[1] + mla v31.4h, v29.4h, v0.h[2] + mla v31.4h, v30.4h, v0.h[3] + ext v28.16b, v27.16b, v27.16b, #2 + ext v29.16b, v27.16b, v27.16b, #4 + ext v30.16b, v27.16b, v27.16b, #6 + mul v27.4h, v27.4h, v0.h[0] + mla v27.4h, v28.4h, v0.h[1] + mla v27.4h, v29.4h, v0.h[2] + mla v27.4h, v30.4h, v0.h[3] + srshr v28.4h, v31.4h, #2 + srshr v29.4h, v27.4h, #2 + ret + +80: +160: +320: + AARCH64_VALID_JUMP_TARGET + b.gt 880f + add \xmy, \xmy, #2 + ld1 {v0.8b}, [\xmx] + ld1 {v1.s}[0], [\xmy] + sub \src, \src, #3 + sub \src, \src, \s_strd + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + bl L(\type\()_8tap_filter_8_first) + bl L(\type\()_8tap_filter_8) + mov v17.16b, v24.16b + mov v18.16b, v25.16b + +8: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v24.4h, v1.h[2] + smlal2 v5.4s, v24.8h, v1.h[2] + smlal v2.4s, v24.4h, v1.h[3] + smlal2 v3.4s, v24.8h, v1.h[3] + smlal v4.4s, v25.4h, v1.h[3] + smlal2 v5.4s, v25.8h, v1.h[3] + sqrshrn v2.4h, v2.4s, #\shift_hv + sqrshrn2 v2.8h, v3.4s, #\shift_hv + sqrshrn v4.4h, v4.4s, #\shift_hv + sqrshrn2 v4.8h, v5.4s, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + sqxtun v2.8b, v2.8h + sqxtun v4.8b, v4.8h + st1 {v2.8b}, [\dst], \d_strd + st1 {v4.8b}, [\ds2], \d_strd +.else + st1 {v2.8h}, [\dst], \d_strd + st1 {v4.8h}, [\ds2], \d_strd +.endif + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v24.16b + mov v18.16b, v25.16b + b 8b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #2 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 164b + +880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv +640: +1280: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmx] + ld1 {v1.8b}, [\xmy] + sub \src, \src, #3 + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + bl L(\type\()_8tap_filter_8_first) + bl L(\type\()_8tap_filter_8) + mov v17.16b, v24.16b + mov v18.16b, v25.16b + bl L(\type\()_8tap_filter_8) + mov v19.16b, v24.16b + mov v20.16b, v25.16b + bl L(\type\()_8tap_filter_8) + mov v21.16b, v24.16b + mov v22.16b, v25.16b + +88: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v19.4h, v1.h[2] + smlal2 v5.4s, v19.8h, v1.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal2 v3.4s, v19.8h, v1.h[3] + smlal v4.4s, v20.4h, v1.h[3] + smlal2 v5.4s, v20.8h, v1.h[3] + smlal v2.4s, v20.4h, v1.h[4] + smlal2 v3.4s, v20.8h, v1.h[4] + smlal v4.4s, v21.4h, v1.h[4] + smlal2 v5.4s, v21.8h, v1.h[4] + smlal v2.4s, v21.4h, v1.h[5] + smlal2 v3.4s, v21.8h, v1.h[5] + smlal v4.4s, v22.4h, v1.h[5] + smlal2 v5.4s, v22.8h, v1.h[5] + smlal v2.4s, v22.4h, v1.h[6] + smlal2 v3.4s, v22.8h, v1.h[6] + smlal v4.4s, v24.4h, v1.h[6] + smlal2 v5.4s, v24.8h, v1.h[6] + smlal v2.4s, v24.4h, v1.h[7] + smlal2 v3.4s, v24.8h, v1.h[7] + smlal v4.4s, v25.4h, v1.h[7] + smlal2 v5.4s, v25.8h, v1.h[7] + sqrshrn v2.4h, v2.4s, #\shift_hv + sqrshrn2 v2.8h, v3.4s, #\shift_hv + sqrshrn v4.4h, v4.4s, #\shift_hv + sqrshrn2 v4.8h, v5.4s, #\shift_hv + subs \h, \h, #2 +.ifc \type, put + sqxtun v2.8b, v2.8h + sqxtun v4.8b, v4.8h + st1 {v2.8b}, [\dst], \d_strd + st1 {v4.8b}, [\ds2], \d_strd +.else + st1 {v2.8h}, [\dst], \d_strd + st1 {v4.8h}, [\ds2], \d_strd +.endif + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v24.16b + mov v22.16b, v25.16b + b 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 168b +0: + ret x15 + +L(\type\()_8tap_filter_8_first): + ld1 {v28.8b, v29.8b}, [\src], \s_strd + uxtl v28.8h, v28.8b + uxtl v29.8h, v29.8b + mul v16.8h, v28.8h, v0.h[0] + ext v24.16b, v28.16b, v29.16b, #(2*1) + ext v25.16b, v28.16b, v29.16b, #(2*2) + ext v26.16b, v28.16b, v29.16b, #(2*3) + ext v27.16b, v28.16b, v29.16b, #(2*4) + mla v16.8h, v24.8h, v0.h[1] + mla v16.8h, v25.8h, v0.h[2] + mla v16.8h, v26.8h, v0.h[3] + mla v16.8h, v27.8h, v0.h[4] + ext v24.16b, v28.16b, v29.16b, #(2*5) + ext v25.16b, v28.16b, v29.16b, #(2*6) + ext v26.16b, v28.16b, v29.16b, #(2*7) + mla v16.8h, v24.8h, v0.h[5] + mla v16.8h, v25.8h, v0.h[6] + mla v16.8h, v26.8h, v0.h[7] + srshr v16.8h, v16.8h, #2 + ret + +L(\type\()_8tap_filter_8): + ld1 {v28.8b, v29.8b}, [\sr2], \s_strd + ld1 {v30.8b, v31.8b}, [\src], \s_strd + uxtl v28.8h, v28.8b + uxtl v29.8h, v29.8b + uxtl v30.8h, v30.8b + uxtl v31.8h, v31.8b + mul v24.8h, v28.8h, v0.h[0] + mul v25.8h, v30.8h, v0.h[0] +.irpc i, 1234567 + ext v26.16b, v28.16b, v29.16b, #(2*\i) + ext v27.16b, v30.16b, v31.16b, #(2*\i) + mla v24.8h, v26.8h, v0.h[\i] + mla v25.8h, v27.8h, v0.h[\i] +.endr + srshr v24.8h, v24.8h, #2 + srshr v25.8h, v25.8h, #2 + ret + +L(\type\()_8tap_hv_tbl): + .hword L(\type\()_8tap_hv_tbl) - 1280b + .hword L(\type\()_8tap_hv_tbl) - 640b + .hword L(\type\()_8tap_hv_tbl) - 320b + .hword L(\type\()_8tap_hv_tbl) - 160b + .hword L(\type\()_8tap_hv_tbl) - 80b + .hword L(\type\()_8tap_hv_tbl) - 40b + .hword L(\type\()_8tap_hv_tbl) - 20b + .hword 0 +endfunc + + +function \type\()_bilin_8bpc_neon, export=1 + dup v1.16b, \mx + dup v3.16b, \my + mov w9, #16 + sub w8, w9, \mx + sub w9, w9, \my + dup v0.16b, w8 + dup v2.16b, w9 +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + clz w8, \w + sub w8, w8, #24 + cbnz \mx, L(\type\()_bilin_h) + cbnz \my, L(\type\()_bilin_v) + b \type\()_neon + +L(\type\()_bilin_h): + cbnz \my, L(\type\()_bilin_hv) + + adr x9, L(\type\()_bilin_h_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: // 2xN h + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +2: + ld1 {v4.s}[0], [\src], \s_strd + ld1 {v6.s}[0], [\sr2], \s_strd + ext v5.8b, v4.8b, v4.8b, #1 + ext v7.8b, v6.8b, v6.8b, #1 + trn1 v4.4h, v4.4h, v6.4h + trn1 v5.4h, v5.4h, v7.4h + subs \h, \h, #2 + umull v4.8h, v4.8b, v0.8b + umlal v4.8h, v5.8b, v1.8b + uqrshrn v4.8b, v4.8h, #4 + st1 {v4.h}[0], [\dst], \d_strd + st1 {v4.h}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +4: + ld1 {v4.8b}, [\src], \s_strd + ld1 {v6.8b}, [\sr2], \s_strd + ext v5.8b, v4.8b, v4.8b, #1 + ext v7.8b, v6.8b, v6.8b, #1 + trn1 v4.2s, v4.2s, v6.2s + trn1 v5.2s, v5.2s, v7.2s + subs \h, \h, #2 + umull v4.8h, v4.8b, v0.8b + umlal v4.8h, v5.8b, v1.8b +.ifc \type, put + uqrshrn v4.8b, v4.8h, #4 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd +.else + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd +.endif + b.gt 4b + ret + +80: // 8xN h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +8: + ld1 {v4.16b}, [\src], \s_strd + ld1 {v6.16b}, [\sr2], \s_strd + ext v5.16b, v4.16b, v4.16b, #1 + ext v7.16b, v6.16b, v6.16b, #1 + subs \h, \h, #2 + umull v4.8h, v4.8b, v0.8b + umull v6.8h, v6.8b, v0.8b + umlal v4.8h, v5.8b, v1.8b + umlal v6.8h, v7.8b, v1.8b +.ifc \type, put + uqrshrn v4.8b, v4.8h, #4 + uqrshrn v6.8b, v6.8h, #4 + st1 {v4.8b}, [\dst], \d_strd + st1 {v6.8b}, [\ds2], \d_strd +.else + st1 {v4.8h}, [\dst], \d_strd + st1 {v6.8h}, [\ds2], \d_strd +.endif + b.gt 8b + ret +160: +320: +640: +1280: // 16xN, 32xN, ... h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + + sub \s_strd, \s_strd, \w, uxtw + sub \s_strd, \s_strd, #8 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw +.endif +161: + ld1 {v16.d}[1], [\src], #8 + ld1 {v20.d}[1], [\sr2], #8 + mov \mx, \w + +16: + ld1 {v18.16b}, [\src], #16 + ld1 {v22.16b}, [\sr2], #16 + ext v17.16b, v16.16b, v18.16b, #8 + ext v19.16b, v16.16b, v18.16b, #9 + ext v21.16b, v20.16b, v22.16b, #8 + ext v23.16b, v20.16b, v22.16b, #9 + umull v16.8h, v17.8b, v0.8b + umull2 v17.8h, v17.16b, v0.16b + umull v20.8h, v21.8b, v0.8b + umull2 v21.8h, v21.16b, v0.16b + umlal v16.8h, v19.8b, v1.8b + umlal2 v17.8h, v19.16b, v1.16b + umlal v20.8h, v23.8b, v1.8b + umlal2 v21.8h, v23.16b, v1.16b + subs \mx, \mx, #16 +.ifc \type, put + uqrshrn v16.8b, v16.8h, #4 + uqrshrn2 v16.16b, v17.8h, #4 + uqrshrn v20.8b, v20.8h, #4 + uqrshrn2 v20.16b, v21.8h, #4 + st1 {v16.16b}, [\dst], #16 + st1 {v20.16b}, [\ds2], #16 +.else + st1 {v16.8h, v17.8h}, [\dst], #32 + st1 {v20.8h, v21.8h}, [\ds2], #32 +.endif + b.le 9f + + mov v16.16b, v18.16b + mov v20.16b, v22.16b + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 161b + ret + +L(\type\()_bilin_h_tbl): + .hword L(\type\()_bilin_h_tbl) - 1280b + .hword L(\type\()_bilin_h_tbl) - 640b + .hword L(\type\()_bilin_h_tbl) - 320b + .hword L(\type\()_bilin_h_tbl) - 160b + .hword L(\type\()_bilin_h_tbl) - 80b + .hword L(\type\()_bilin_h_tbl) - 40b + .hword L(\type\()_bilin_h_tbl) - 20b + .hword 0 + + +L(\type\()_bilin_v): + cmp \h, #4 + adr x9, L(\type\()_bilin_v_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: // 2xN v + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + cmp \h, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + // 2x2 v + ld1 {v16.h}[0], [\src], \s_strd + b.gt 24f +22: + ld1 {v17.h}[0], [\sr2], \s_strd + ld1 {v18.h}[0], [\src], \s_strd + trn1 v16.4h, v16.4h, v17.4h + trn1 v17.4h, v17.4h, v18.4h + umull v4.8h, v16.8b, v2.8b + umlal v4.8h, v17.8b, v3.8b + uqrshrn v4.8b, v4.8h, #4 + st1 {v4.h}[0], [\dst] + st1 {v4.h}[1], [\ds2] + ret +24: // 2x4, 2x6, 2x8, ... v + ld1 {v17.h}[0], [\sr2], \s_strd + ld1 {v18.h}[0], [\src], \s_strd + ld1 {v19.h}[0], [\sr2], \s_strd + ld1 {v20.h}[0], [\src], \s_strd + sub \h, \h, #4 + trn1 v16.4h, v16.4h, v17.4h + trn1 v17.4h, v17.4h, v18.4h + trn1 v18.4h, v18.4h, v19.4h + trn1 v19.4h, v19.4h, v20.4h + trn1 v16.2s, v16.2s, v18.2s + trn1 v17.2s, v17.2s, v19.2s + umull v4.8h, v16.8b, v2.8b + umlal v4.8h, v17.8b, v3.8b + cmp \h, #2 + uqrshrn v4.8b, v4.8h, #4 + st1 {v4.h}[0], [\dst], \d_strd + st1 {v4.h}[1], [\ds2], \d_strd + st1 {v4.h}[2], [\dst], \d_strd + st1 {v4.h}[3], [\ds2], \d_strd + b.lt 0f + mov v16.8b, v20.8b + b.eq 22b + b 24b +0: + ret +.endif + +40: // 4xN v + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.s}[0], [\src], \s_strd +4: + ld1 {v17.s}[0], [\sr2], \s_strd + ld1 {v18.s}[0], [\src], \s_strd + trn1 v16.2s, v16.2s, v17.2s + trn1 v17.2s, v17.2s, v18.2s + umull v4.8h, v16.8b, v2.8b + umlal v4.8h, v17.8b, v3.8b + subs \h, \h, #2 +.ifc \type, put + uqrshrn v4.8b, v4.8h, #4 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd +.else + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd +.endif + b.le 0f + mov v16.8b, v18.8b + b 4b +0: + ret + +80: // 8xN v + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.8b}, [\src], \s_strd +8: + ld1 {v17.8b}, [\sr2], \s_strd + ld1 {v18.8b}, [\src], \s_strd + umull v4.8h, v16.8b, v2.8b + umull v5.8h, v17.8b, v2.8b + umlal v4.8h, v17.8b, v3.8b + umlal v5.8h, v18.8b, v3.8b + subs \h, \h, #2 +.ifc \type, put + uqrshrn v4.8b, v4.8h, #4 + uqrshrn v5.8b, v5.8h, #4 + st1 {v4.8b}, [\dst], \d_strd + st1 {v5.8b}, [\ds2], \d_strd +.else + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd +.endif + b.le 0f + mov v16.8b, v18.8b + b 8b +0: + ret + +160: // 16xN, 32xN, ... +320: +640: +1280: + AARCH64_VALID_JUMP_TARGET + mov \my, \h +1: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v16.16b}, [\src], \s_strd +2: + ld1 {v17.16b}, [\sr2], \s_strd + ld1 {v18.16b}, [\src], \s_strd + umull v4.8h, v16.8b, v2.8b + umull2 v5.8h, v16.16b, v2.16b + umull v6.8h, v17.8b, v2.8b + umull2 v7.8h, v17.16b, v2.16b + umlal v4.8h, v17.8b, v3.8b + umlal2 v5.8h, v17.16b, v3.16b + umlal v6.8h, v18.8b, v3.8b + umlal2 v7.8h, v18.16b, v3.16b + subs \h, \h, #2 +.ifc \type, put + uqrshrn v4.8b, v4.8h, #4 + uqrshrn2 v4.16b, v5.8h, #4 + uqrshrn v6.8b, v6.8h, #4 + uqrshrn2 v6.16b, v7.8h, #4 + st1 {v4.16b}, [\dst], \d_strd + st1 {v6.16b}, [\ds2], \d_strd +.else + st1 {v4.8h, v5.8h}, [\dst], \d_strd + st1 {v6.8h, v7.8h}, [\ds2], \d_strd +.endif + b.le 9f + mov v16.16b, v18.16b + b 2b +9: + subs \w, \w, #16 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #16 +.ifc \type, put + add \dst, \dst, #16 +.else + add \dst, \dst, #32 +.endif + b 1b +0: + ret + +L(\type\()_bilin_v_tbl): + .hword L(\type\()_bilin_v_tbl) - 1280b + .hword L(\type\()_bilin_v_tbl) - 640b + .hword L(\type\()_bilin_v_tbl) - 320b + .hword L(\type\()_bilin_v_tbl) - 160b + .hword L(\type\()_bilin_v_tbl) - 80b + .hword L(\type\()_bilin_v_tbl) - 40b + .hword L(\type\()_bilin_v_tbl) - 20b + .hword 0 + +L(\type\()_bilin_hv): + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + adr x9, L(\type\()_bilin_hv_tbl) + ldrh w8, [x9, x8, lsl #1] + sub x9, x9, w8, uxtw + br x9 + +20: // 2xN hv + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v28.s}[0], [\src], \s_strd + ext v29.8b, v28.8b, v28.8b, #1 + umull v16.8h, v28.8b, v0.8b + umlal v16.8h, v29.8b, v1.8b + +2: + ld1 {v28.s}[0], [\sr2], \s_strd + ld1 {v30.s}[0], [\src], \s_strd + ext v29.8b, v28.8b, v28.8b, #1 + ext v31.8b, v30.8b, v30.8b, #1 + trn1 v28.4h, v28.4h, v30.4h + trn1 v29.4h, v29.4h, v31.4h + umull v17.8h, v28.8b, v0.8b + umlal v17.8h, v29.8b, v1.8b + + trn1 v16.2s, v16.2s, v17.2s + + mul v4.4h, v16.4h, v2.4h + mla v4.4h, v17.4h, v3.4h + uqrshrn v4.8b, v4.8h, #8 + subs \h, \h, #2 + st1 {v4.h}[0], [\dst], \d_strd + st1 {v4.h}[1], [\ds2], \d_strd + b.le 0f + trn2 v16.2s, v17.2s, v17.2s + b 2b +0: + ret +.endif + +40: // 4xN hv + AARCH64_VALID_JUMP_TARGET + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v28.8b}, [\src], \s_strd + ext v29.8b, v28.8b, v28.8b, #1 + umull v16.8h, v28.8b, v0.8b + umlal v16.8h, v29.8b, v1.8b + +4: + ld1 {v28.8b}, [\sr2], \s_strd + ld1 {v30.8b}, [\src], \s_strd + ext v29.8b, v28.8b, v28.8b, #1 + ext v31.8b, v30.8b, v30.8b, #1 + trn1 v28.2s, v28.2s, v30.2s + trn1 v29.2s, v29.2s, v31.2s + umull v17.8h, v28.8b, v0.8b + umlal v17.8h, v29.8b, v1.8b + + trn1 v16.2d, v16.2d, v17.2d + + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + uqrshrn v4.8b, v4.8h, #8 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd +.else + urshr v4.8h, v4.8h, #4 + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd +.endif + b.le 0f + trn2 v16.2d, v17.2d, v17.2d + b 4b +0: + ret + +80: // 8xN, 16xN, ... hv +160: +320: +640: +1280: + AARCH64_VALID_JUMP_TARGET + mov \my, \h + +1: + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v28.16b}, [\src], \s_strd + ext v29.16b, v28.16b, v28.16b, #1 + umull v16.8h, v28.8b, v0.8b + umlal v16.8h, v29.8b, v1.8b + +2: + ld1 {v28.16b}, [\sr2], \s_strd + ld1 {v30.16b}, [\src], \s_strd + ext v29.16b, v28.16b, v28.16b, #1 + ext v31.16b, v30.16b, v30.16b, #1 + umull v17.8h, v28.8b, v0.8b + umlal v17.8h, v29.8b, v1.8b + umull v18.8h, v30.8b, v0.8b + umlal v18.8h, v31.8b, v1.8b + + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + mul v5.8h, v17.8h, v2.8h + mla v5.8h, v18.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + uqrshrn v4.8b, v4.8h, #8 + uqrshrn v5.8b, v5.8h, #8 + st1 {v4.8b}, [\dst], \d_strd + st1 {v5.8b}, [\ds2], \d_strd +.else + urshr v4.8h, v4.8h, #4 + urshr v5.8h, v5.8h, #4 + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd +.endif + b.le 9f + mov v16.16b, v18.16b + b 2b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #8 +.ifc \type, put + add \dst, \dst, #8 +.else + add \dst, \dst, #16 +.endif + b 1b +0: + ret + +L(\type\()_bilin_hv_tbl): + .hword L(\type\()_bilin_hv_tbl) - 1280b + .hword L(\type\()_bilin_hv_tbl) - 640b + .hword L(\type\()_bilin_hv_tbl) - 320b + .hword L(\type\()_bilin_hv_tbl) - 160b + .hword L(\type\()_bilin_hv_tbl) - 80b + .hword L(\type\()_bilin_hv_tbl) - 40b + .hword L(\type\()_bilin_hv_tbl) - 20b + .hword 0 +endfunc +.endm + +filter_fn put, x0, x1, x2, x3, w4, w5, w6, x6, w7, x7, x8, x9, 10 +filter_fn prep, x0, x7, x1, x2, w3, w4, w5, x5, w6, x6, x8, x9, 6 + +.macro load_filter_row dst, src, inc + asr w13, \src, #10 + add \src, \src, \inc + ldr \dst, [x11, w13, sxtw #3] +.endm + +function warp_filter_horz_neon + add w12, w5, #512 + + ld1 {v16.8b, v17.8b}, [x2], x3 + + load_filter_row d0, w12, w7 + load_filter_row d1, w12, w7 + load_filter_row d2, w12, w7 + load_filter_row d3, w12, w7 + load_filter_row d4, w12, w7 + load_filter_row d5, w12, w7 + load_filter_row d6, w12, w7 + // subtract by 128 to allow using smull + eor v16.8b, v16.8b, v22.8b + eor v17.8b, v17.8b, v22.8b + load_filter_row d7, w12, w7 + + ext v18.8b, v16.8b, v17.8b, #1 + ext v19.8b, v16.8b, v17.8b, #2 + smull v0.8h, v0.8b, v16.8b + smull v1.8h, v1.8b, v18.8b + ext v18.8b, v16.8b, v17.8b, #3 + ext v20.8b, v16.8b, v17.8b, #4 + smull v2.8h, v2.8b, v19.8b + smull v3.8h, v3.8b, v18.8b + ext v18.8b, v16.8b, v17.8b, #5 + ext v19.8b, v16.8b, v17.8b, #6 + smull v4.8h, v4.8b, v20.8b + smull v5.8h, v5.8b, v18.8b + ext v18.8b, v16.8b, v17.8b, #7 + smull v6.8h, v6.8b, v19.8b + smull v7.8h, v7.8b, v18.8b + + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + + addp v0.8h, v0.8h, v4.8h + + add w5, w5, w8 + + ret +endfunc + +// void dav1d_warp_affine_8x8_8bpc_neon( +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *const abcd, int mx, int my) +.macro warp t, shift +function warp_affine_8x8\t\()_8bpc_neon, export=1 + ldr x4, [x4] + sbfx x7, x4, #0, #16 + sbfx x8, x4, #16, #16 + sbfx x9, x4, #32, #16 + sbfx x4, x4, #48, #16 + mov w10, #8 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + sub x2, x2, #3 + movrel x11, X(mc_warp_filter), 64*8 + mov x15, x30 +.ifnb \t + lsl x1, x1, #1 +.endif + + movi v22.8b, #128 +.ifb \t + movi v23.8h, #128 +.else + movi v23.8h, #8, lsl #8 +.endif + + bl warp_filter_horz_neon + srshr v24.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v25.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v26.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v27.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v28.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v29.8h, v0.8h, #3 + bl warp_filter_horz_neon + srshr v30.8h, v0.8h, #3 + +1: + add w14, w6, #512 + bl warp_filter_horz_neon + srshr v31.8h, v0.8h, #3 + + load_filter_row d0, w14, w9 + load_filter_row d1, w14, w9 + load_filter_row d2, w14, w9 + load_filter_row d3, w14, w9 + load_filter_row d4, w14, w9 + load_filter_row d5, w14, w9 + load_filter_row d6, w14, w9 + load_filter_row d7, w14, w9 + transpose_8x8b_xtl v0, v1, v2, v3, v4, v5, v6, v7, sxtl + + // This ordering of smull/smlal/smull2/smlal2 is highly + // beneficial for Cortex A53 here. + smull v16.4s, v24.4h, v0.4h + smlal v16.4s, v25.4h, v1.4h + smlal v16.4s, v26.4h, v2.4h + smlal v16.4s, v27.4h, v3.4h + smlal v16.4s, v28.4h, v4.4h + smlal v16.4s, v29.4h, v5.4h + smlal v16.4s, v30.4h, v6.4h + smlal v16.4s, v31.4h, v7.4h + smull2 v17.4s, v24.8h, v0.8h + smlal2 v17.4s, v25.8h, v1.8h + smlal2 v17.4s, v26.8h, v2.8h + smlal2 v17.4s, v27.8h, v3.8h + smlal2 v17.4s, v28.8h, v4.8h + smlal2 v17.4s, v29.8h, v5.8h + smlal2 v17.4s, v30.8h, v6.8h + smlal2 v17.4s, v31.8h, v7.8h + + mov v24.16b, v25.16b + mov v25.16b, v26.16b + sqrshrn v16.4h, v16.4s, #\shift + mov v26.16b, v27.16b + sqrshrn2 v16.8h, v17.4s, #\shift + mov v27.16b, v28.16b + mov v28.16b, v29.16b + add v16.8h, v16.8h, v23.8h +.ifb \t + sqxtun v16.8b, v16.8h +.endif + mov v29.16b, v30.16b + mov v30.16b, v31.16b + subs w10, w10, #1 +.ifnb \t + st1 {v16.8h}, [x0], x1 +.else + st1 {v16.8b}, [x0], x1 +.endif + + add w6, w6, w4 + b.gt 1b + + ret x15 +endfunc +.endm + +warp , 11 +warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.16b}, [x8] + mov x12, x6 // out = dst + mov x3, x4 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.16b, v1.16b}, [x13], #32 + subs x3, x3, #32 + st1 {v0.16b, v1.16b}, [x12], #32 + b.gt 1b +.if \need_right + add x3, x8, x2 // in + center_w + sub x3, x3, #1 // in + center_w - 1 + add x12, x6, x4 // dst + left_ext + ld1r {v0.16b}, [x3] + add x12, x12, x2 // out = dst + left_ext + center_w + mov x3, x11 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.16b, v1.16b}, [x8], #32 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.16b, v1.16b}, [x14], #32 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/mc16.S b/third_party/dav1d/src/arm/64/mc16.S new file mode 100644 index 0000000000..1bfb12ebb3 --- /dev/null +++ b/third_party/dav1d/src/arm/64/mc16.S @@ -0,0 +1,3611 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define PREP_BIAS 8192 + +.macro avg d0, d1, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + sqadd \t0\().8h, \t0\().8h, \t2\().8h + sqadd \t1\().8h, \t1\().8h, \t3\().8h + smax \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + smax \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sqsub \t0\().8h, \t0\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sqsub \t1\().8h, \t1\().8h, v28.8h // -2*PREP_BIAS - 1 << intermediate_bits + sshl \d0\().8h, \t0\().8h, v29.8h // -(intermediate_bits+1) + sshl \d1\().8h, \t1\().8h, v29.8h // -(intermediate_bits+1) +.endm + +.macro w_avg d0, d1, t0, t1, t2, t3 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + // This difference requires a 17 bit range, and all bits are + // significant for the following multiplication. + ssubl \d0\().4s, \t2\().4h, \t0\().4h + ssubl2 \t0\().4s, \t2\().8h, \t0\().8h + ssubl \d1\().4s, \t3\().4h, \t1\().4h + ssubl2 \t1\().4s, \t3\().8h, \t1\().8h + mul \d0\().4s, \d0\().4s, v27.4s + mul \t0\().4s, \t0\().4s, v27.4s + mul \d1\().4s, \d1\().4s, v27.4s + mul \t1\().4s, \t1\().4s, v27.4s + sshr \d0\().4s, \d0\().4s, #4 + sshr \t0\().4s, \t0\().4s, #4 + sshr \d1\().4s, \d1\().4s, #4 + sshr \t1\().4s, \t1\().4s, #4 + saddw \d0\().4s, \d0\().4s, \t2\().4h + saddw2 \t0\().4s, \t0\().4s, \t2\().8h + saddw \d1\().4s, \d1\().4s, \t3\().4h + saddw2 \t1\().4s, \t1\().4s, \t3\().8h + uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2 + uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto + srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits + srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits + add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits + add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits + smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max + smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max + smax \d0\().8h, \d0\().8h, v30.8h // 0 + smax \d1\().8h, \d1\().8h, v30.8h // 0 +.endm + +.macro mask d0, d1, t0, t1, t2, t3 + ld1 {v27.16b}, [x6], 16 + ld1 {\t0\().8h,\t1\().8h}, [x2], 32 + neg v27.16b, v27.16b + ld1 {\t2\().8h,\t3\().8h}, [x3], 32 + sxtl v26.8h, v27.8b + sxtl2 v27.8h, v27.16b + sxtl v24.4s, v26.4h + sxtl2 v25.4s, v26.8h + sxtl v26.4s, v27.4h + sxtl2 v27.4s, v27.8h + ssubl \d0\().4s, \t2\().4h, \t0\().4h + ssubl2 \t0\().4s, \t2\().8h, \t0\().8h + ssubl \d1\().4s, \t3\().4h, \t1\().4h + ssubl2 \t1\().4s, \t3\().8h, \t1\().8h + mul \d0\().4s, \d0\().4s, v24.4s + mul \t0\().4s, \t0\().4s, v25.4s + mul \d1\().4s, \d1\().4s, v26.4s + mul \t1\().4s, \t1\().4s, v27.4s + sshr \d0\().4s, \d0\().4s, #6 + sshr \t0\().4s, \t0\().4s, #6 + sshr \d1\().4s, \d1\().4s, #6 + sshr \t1\().4s, \t1\().4s, #6 + saddw \d0\().4s, \d0\().4s, \t2\().4h + saddw2 \t0\().4s, \t0\().4s, \t2\().8h + saddw \d1\().4s, \d1\().4s, \t3\().4h + saddw2 \t1\().4s, \t1\().4s, \t3\().8h + uzp1 \d0\().8h, \d0\().8h, \t0\().8h // Same as xtn, xtn2 + uzp1 \d1\().8h, \d1\().8h, \t1\().8h // Ditto + srshl \d0\().8h, \d0\().8h, v29.8h // -intermediate_bits + srshl \d1\().8h, \d1\().8h, v29.8h // -intermediate_bits + add \d0\().8h, \d0\().8h, v28.8h // PREP_BIAS >> intermediate_bits + add \d1\().8h, \d1\().8h, v28.8h // PREP_BIAS >> intermediate_bits + smin \d0\().8h, \d0\().8h, v31.8h // bitdepth_max + smin \d1\().8h, \d1\().8h, v31.8h // bitdepth_max + smax \d0\().8h, \d0\().8h, v30.8h // 0 + smax \d1\().8h, \d1\().8h, v30.8h // 0 +.endm + +.macro bidir_fn type, bdmax +function \type\()_16bpc_neon, export=1 + clz w4, w4 +.ifnc \type, avg + dup v31.8h, \bdmax // bitdepth_max + movi v30.8h, #0 +.endif + clz w7, \bdmax + sub w7, w7, #18 // intermediate_bits = clz(bitdepth_max) - 18 +.ifc \type, avg + mov w9, #1 + mov w8, #-2*PREP_BIAS + lsl w9, w9, w7 // 1 << intermediate_bits + add w7, w7, #1 + sub w8, w8, w9 // -2*PREP_BIAS - 1 << intermediate_bits + neg w7, w7 // -(intermediate_bits+1) + dup v28.8h, w8 // -2*PREP_BIAS - 1 << intermediate_bits + dup v29.8h, w7 // -(intermediate_bits+1) +.else + mov w8, #PREP_BIAS + lsr w8, w8, w7 // PREP_BIAS >> intermediate_bits + neg w7, w7 // -intermediate_bits + dup v28.8h, w8 // PREP_BIAS >> intermediate_bits + dup v29.8h, w7 // -intermediate_bits +.endif +.ifc \type, w_avg + dup v27.4s, w6 + neg v27.4s, v27.4s +.endif + adr x7, L(\type\()_tbl) + sub w4, w4, #24 + \type v4, v5, v0, v1, v2, v3 + ldrh w4, [x7, x4, lsl #1] + sub x7, x7, w4, uxtw + br x7 +40: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +4: + subs w5, w5, #4 + st1 {v4.d}[0], [x0], x1 + st1 {v4.d}[1], [x7], x1 + st1 {v5.d}[0], [x0], x1 + st1 {v5.d}[1], [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 4b +80: + AARCH64_VALID_JUMP_TARGET + add x7, x0, x1 + lsl x1, x1, #1 +8: + st1 {v4.8h}, [x0], x1 + subs w5, w5, #2 + st1 {v5.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 8b +16: + AARCH64_VALID_JUMP_TARGET + \type v6, v7, v0, v1, v2, v3 + st1 {v4.8h, v5.8h}, [x0], x1 + subs w5, w5, #2 + st1 {v6.8h, v7.8h}, [x0], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 16b +32: + AARCH64_VALID_JUMP_TARGET + \type v6, v7, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 32b +640: + AARCH64_VALID_JUMP_TARGET + add x7, x0, #64 +64: + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + \type v18, v19, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 64b +1280: + AARCH64_VALID_JUMP_TARGET + add x7, x0, #64 + mov x8, #128 + sub x1, x1, #128 +128: + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x8 + \type v18, v19, v0, v1, v2, v3 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x8 + \type v4, v5, v0, v1, v2, v3 + \type v6, v7, v0, v1, v2, v3 + \type v16, v17, v0, v1, v2, v3 + subs w5, w5, #1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + \type v18, v19, v0, v1, v2, v3 + st1 {v16.8h,v17.8h,v18.8h,v19.8h}, [x7], x1 + b.le 0f + \type v4, v5, v0, v1, v2, v3 + b 128b +0: + ret +L(\type\()_tbl): + .hword L(\type\()_tbl) - 1280b + .hword L(\type\()_tbl) - 640b + .hword L(\type\()_tbl) - 32b + .hword L(\type\()_tbl) - 16b + .hword L(\type\()_tbl) - 80b + .hword L(\type\()_tbl) - 40b +endfunc +.endm + +bidir_fn avg, w6 +bidir_fn w_avg, w7 +bidir_fn mask, w7 + + +.macro w_mask_fn type +function w_mask_\type\()_16bpc_neon, export=1 + ldr w8, [sp] + clz w9, w4 + adr x10, L(w_mask_\type\()_tbl) + dup v31.8h, w8 // bitdepth_max + sub w9, w9, #24 + clz w8, w8 // clz(bitdepth_max) + ldrh w9, [x10, x9, lsl #1] + sub x10, x10, w9, uxtw + sub w8, w8, #12 // sh = intermediate_bits + 6 = clz(bitdepth_max) - 12 + mov w9, #PREP_BIAS*64 + neg w8, w8 // -sh + mov w11, #27615 // (64 + 1 - 38)<> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + uxtl v22.4s, v20.4h + uxtl2 v23.4s, v20.8h + uxtl v24.4s, v21.4h + uxtl2 v25.4s, v21.8h + mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) + mla v5.4s, v17.4s, v23.4s + mla v6.4s, v18.4s, v24.4s + mla v7.4s, v19.4s, v25.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v6.4s + sqxtun2 v5.8h, v7.4s + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h +.if \type == 444 + uzp1 v20.16b, v20.16b, v21.16b // 64 - m + sub v20.16b, v1.16b, v20.16b // m + st1 {v20.16b}, [x6], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + xtn v20.8b, v20.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + st1 {v20.8b}, [x6], #8 +.elseif \type == 420 + trn1 v24.2d, v20.2d, v21.2d + trn2 v25.2d, v20.2d, v21.2d + add v24.8h, v24.8h, v25.8h // (64 - my1) + (64 - my2) (row wise addition) + addp v20.8h, v24.8h, v24.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.s}[0], [x6], #4 +.endif + st1 {v4.d}[0], [x0], x1 + st1 {v4.d}[1], [x12], x1 + st1 {v5.d}[0], [x0], x1 + st1 {v5.d}[1], [x12], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 + ld1 {v6.8h, v7.8h}, [x3], #32 // tmp2 + subs w5, w5, #2 + sabd v20.8h, v4.8h, v6.8h // abs(tmp1 - tmp2) + sabd v21.8h, v5.8h, v7.8h + ssubl v16.4s, v6.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v17.4s, v6.8h, v4.8h + ssubl v18.4s, v7.4h, v5.4h + ssubl2 v19.4s, v7.8h, v5.8h + uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() + uqsub v21.8h, v0.8h, v21.8h + sshll2 v7.4s, v5.8h, #6 // tmp1 << 6 + sshll v6.4s, v5.4h, #6 + sshll2 v5.4s, v4.8h, #6 + sshll v4.4s, v4.4h, #6 + ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v6.4s, v6.4s, v30.4s + add v7.4s, v7.4s, v30.4s + uxtl v22.4s, v20.4h + uxtl2 v23.4s, v20.8h + uxtl v24.4s, v21.4h + uxtl2 v25.4s, v21.8h + mla v4.4s, v16.4s, v22.4s // (tmp2-tmp1)*(64-m) + mla v5.4s, v17.4s, v23.4s + mla v6.4s, v18.4s, v24.4s + mla v7.4s, v19.4s, v25.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v6.4s, v6.4s, v29.4s + srshl v7.4s, v7.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v6.4s + sqxtun2 v5.8h, v7.4s + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h +.if \type == 444 + uzp1 v20.16b, v20.16b, v21.16b // 64 - m + sub v20.16b, v1.16b, v20.16b // m + st1 {v20.16b}, [x6], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + xtn v20.8b, v20.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + st1 {v20.8b}, [x6], #8 +.elseif \type == 420 + add v20.8h, v20.8h, v21.8h // (64 - my1) + (64 - my2) (row wise addition) + addp v20.8h, v20.8h, v20.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.4h, v3.4h, v20.4h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.s}[0], [x6], #4 +.endif + st1 {v4.8h}, [x0], x1 + st1 {v5.8h}, [x12], x1 + b.gt 8b + ret +1280: +640: +320: +160: + AARCH64_VALID_JUMP_TARGET + mov w11, w4 + sub x1, x1, w4, uxtw #1 +.if \type == 444 + add x10, x6, w4, uxtw +.elseif \type == 422 + add x10, x6, x11, lsr #1 +.endif + add x9, x3, w4, uxtw #1 + add x7, x2, w4, uxtw #1 +161: + mov w8, w4 +16: + ld1 {v4.8h, v5.8h}, [x2], #32 // tmp1 + ld1 {v16.8h, v17.8h}, [x3], #32 // tmp2 + ld1 {v6.8h, v7.8h}, [x7], #32 + ld1 {v18.8h, v19.8h}, [x9], #32 + subs w8, w8, #16 + sabd v20.8h, v4.8h, v16.8h // abs(tmp1 - tmp2) + sabd v21.8h, v5.8h, v17.8h + ssubl v22.4s, v16.4h, v4.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v23.4s, v16.8h, v4.8h + ssubl v24.4s, v17.4h, v5.4h + ssubl2 v25.4s, v17.8h, v5.8h + uqsub v20.8h, v0.8h, v20.8h // 27615 - abs() + uqsub v21.8h, v0.8h, v21.8h + sshll2 v27.4s, v5.8h, #6 // tmp1 << 6 + sshll v26.4s, v5.4h, #6 + sshll2 v5.4s, v4.8h, #6 + sshll v4.4s, v4.4h, #6 + ushr v20.8h, v20.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v21.8h, v21.8h, #10 + add v4.4s, v4.4s, v30.4s // += PREP_BIAS*64 + add v5.4s, v5.4s, v30.4s + add v26.4s, v26.4s, v30.4s + add v27.4s, v27.4s, v30.4s + uxtl v16.4s, v20.4h + uxtl2 v17.4s, v20.8h + uxtl v28.4s, v21.4h + mla v4.4s, v22.4s, v16.4s // (tmp2-tmp1)*(64-m) + uxtl2 v16.4s, v21.8h + mla v5.4s, v23.4s, v17.4s + mla v26.4s, v24.4s, v28.4s + mla v27.4s, v25.4s, v16.4s + srshl v4.4s, v4.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v5.4s, v5.4s, v29.4s + srshl v26.4s, v26.4s, v29.4s + srshl v27.4s, v27.4s, v29.4s + sqxtun v4.4h, v4.4s // iclip_pixel + sqxtun2 v4.8h, v5.4s + sqxtun v5.4h, v26.4s + sqxtun2 v5.8h, v27.4s + + // Start of other half + sabd v22.8h, v6.8h, v18.8h // abs(tmp1 - tmp2) + sabd v23.8h, v7.8h, v19.8h + + umin v4.8h, v4.8h, v31.8h // iclip_pixel + umin v5.8h, v5.8h, v31.8h + + ssubl v16.4s, v18.4h, v6.4h // tmp2 - tmp1 (requires 17 bit) + ssubl2 v17.4s, v18.8h, v6.8h + ssubl v18.4s, v19.4h, v7.4h + ssubl2 v19.4s, v19.8h, v7.8h + uqsub v22.8h, v0.8h, v22.8h // 27615 - abs() + uqsub v23.8h, v0.8h, v23.8h + sshll v24.4s, v6.4h, #6 // tmp1 << 6 + sshll2 v25.4s, v6.8h, #6 + sshll v26.4s, v7.4h, #6 + sshll2 v27.4s, v7.8h, #6 + ushr v22.8h, v22.8h, #10 // 64-m = (27615 - abs()) >> mask_sh + ushr v23.8h, v23.8h, #10 + add v24.4s, v24.4s, v30.4s // += PREP_BIAS*64 + add v25.4s, v25.4s, v30.4s + add v26.4s, v26.4s, v30.4s + add v27.4s, v27.4s, v30.4s + uxtl v6.4s, v22.4h + uxtl2 v7.4s, v22.8h + uxtl v28.4s, v23.4h + mla v24.4s, v16.4s, v6.4s // (tmp2-tmp1)*(64-m) + uxtl2 v6.4s, v23.8h + mla v25.4s, v17.4s, v7.4s + mla v26.4s, v18.4s, v28.4s + mla v27.4s, v19.4s, v6.4s + srshl v24.4s, v24.4s, v29.4s // (tmp1<<6 + (tmp2-tmp1)*(64-m) + (1 << (sh-1)) + PREP_BIAS*64) >> sh + srshl v25.4s, v25.4s, v29.4s + srshl v26.4s, v26.4s, v29.4s + srshl v27.4s, v27.4s, v29.4s + sqxtun v6.4h, v24.4s // iclip_pixel + sqxtun2 v6.8h, v25.4s + sqxtun v7.4h, v26.4s + sqxtun2 v7.8h, v27.4s + umin v6.8h, v6.8h, v31.8h // iclip_pixel + umin v7.8h, v7.8h, v31.8h +.if \type == 444 + uzp1 v20.16b, v20.16b, v21.16b // 64 - m + uzp1 v21.16b, v22.16b, v23.16b + sub v20.16b, v1.16b, v20.16b // m + sub v21.16b, v1.16b, v21.16b + st1 {v20.16b}, [x6], #16 + st1 {v21.16b}, [x10], #16 +.elseif \type == 422 + addp v20.8h, v20.8h, v21.8h // (64 - m) + (64 - n) (column wise addition) + addp v21.8h, v22.8h, v23.8h + xtn v20.8b, v20.8h + xtn v21.8b, v21.8h + uhsub v20.8b, v3.8b, v20.8b // ((129 - sign) - ((64 - m) + (64 - n)) >> 1 + uhsub v21.8b, v3.8b, v21.8b + st1 {v20.8b}, [x6], #8 + st1 {v21.8b}, [x10], #8 +.elseif \type == 420 + add v20.8h, v20.8h, v22.8h // (64 - my1) + (64 - my2) (row wise addition) + add v21.8h, v21.8h, v23.8h + addp v20.8h, v20.8h, v21.8h // (128 - m) + (128 - n) (column wise addition) + sub v20.8h, v3.8h, v20.8h // (256 - sign) - ((128 - m) + (128 - n)) + rshrn v20.8b, v20.8h, #2 // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2 + st1 {v20.8b}, [x6], #8 +.endif + st1 {v4.8h, v5.8h}, [x0], #32 + st1 {v6.8h, v7.8h}, [x12], #32 + b.gt 16b + subs w5, w5, #2 + add x2, x2, w4, uxtw #1 + add x3, x3, w4, uxtw #1 + add x7, x7, w4, uxtw #1 + add x9, x9, w4, uxtw #1 +.if \type == 444 + add x6, x6, w4, uxtw + add x10, x10, w4, uxtw +.elseif \type == 422 + add x6, x6, x11, lsr #1 + add x10, x10, x11, lsr #1 +.endif + add x0, x0, x1 + add x12, x12, x1 + b.gt 161b + ret +L(w_mask_\type\()_tbl): + .hword L(w_mask_\type\()_tbl) - 1280b + .hword L(w_mask_\type\()_tbl) - 640b + .hword L(w_mask_\type\()_tbl) - 320b + .hword L(w_mask_\type\()_tbl) - 160b + .hword L(w_mask_\type\()_tbl) - 8b + .hword L(w_mask_\type\()_tbl) - 4b +endfunc +.endm + +w_mask_fn 444 +w_mask_fn 422 +w_mask_fn 420 + + +function blend_16bpc_neon, export=1 + adr x6, L(blend_tbl) + clz w3, w3 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + add x8, x0, x1 + br x6 +40: + AARCH64_VALID_JUMP_TARGET + lsl x1, x1, #1 +4: + ld1 {v2.8b}, [x5], #8 + ld1 {v1.8h}, [x2], #16 + ld1 {v0.d}[0], [x0] + neg v2.8b, v2.8b // -m + subs w4, w4, #2 + ld1 {v0.d}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.d}[0], [x0], x1 + st1 {v0.d}[1], [x8], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + lsl x1, x1, #1 +8: + ld1 {v4.16b}, [x5], #16 + ld1 {v2.8h, v3.8h}, [x2], #32 + neg v5.16b, v4.16b // -m + ld1 {v0.8h}, [x0] + ld1 {v1.8h}, [x8] + sxtl v4.8h, v5.8b + sxtl2 v5.8h, v5.16b + shl v4.8h, v4.8h, #9 // -m << 9 + shl v5.8h, v5.8h, #9 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + subs w4, w4, #2 + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v5.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.8h}, [x0], x1 + st1 {v1.8h}, [x8], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + lsl x1, x1, #1 +16: + ld1 {v16.16b, v17.16b}, [x5], #32 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + subs w4, w4, #2 + neg v18.16b, v16.16b // -m + neg v19.16b, v17.16b + ld1 {v0.8h, v1.8h}, [x0] + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + ld1 {v2.8h, v3.8h}, [x8] + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + shl v18.8h, v18.8h, #9 + shl v19.8h, v19.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v17.8h + sqrdmulh v6.8h, v6.8h, v18.8h + sqrdmulh v7.8h, v7.8h, v19.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v2.8h, v3.8h}, [x8], x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ld1 {v16.16b, v17.16b}, [x5], #32 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + subs w4, w4, #1 + neg v18.16b, v16.16b // -m + neg v19.16b, v17.16b + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + shl v18.8h, v18.8h, #9 + shl v19.8h, v19.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v17.8h + sqrdmulh v6.8h, v6.8h, v18.8h + sqrdmulh v7.8h, v7.8h, v19.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + b.gt 32b + ret +L(blend_tbl): + .hword L(blend_tbl) - 32b + .hword L(blend_tbl) - 160b + .hword L(blend_tbl) - 80b + .hword L(blend_tbl) - 40b +endfunc + +function blend_h_16bpc_neon, export=1 + adr x6, L(blend_h_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w4, uxtw + sub w4, w4, w4, lsr #2 + clz w7, w3 + add x8, x0, x1 + lsl x1, x1, #1 + sub w7, w7, #24 + ldrh w7, [x6, x7, lsl #1] + sub x6, x6, w7, uxtw + br x6 +2: + AARCH64_VALID_JUMP_TARGET + ld2r {v2.8b, v3.8b}, [x5], #2 + ld1 {v1.4h}, [x2], #8 + ext v2.8b, v2.8b, v3.8b, #6 + subs w4, w4, #2 + neg v2.8b, v2.8b // -m + ld1 {v0.s}[0], [x0] + ld1 {v0.s}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.4h, v2.4h, #9 // -m << 9 + sub v1.4h, v0.4h, v1.4h // a - b + sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 + add v0.4h, v0.4h, v1.4h + st1 {v0.s}[0], [x0], x1 + st1 {v0.s}[1], [x8], x1 + b.gt 2b + ret +4: + AARCH64_VALID_JUMP_TARGET + ld2r {v2.8b, v3.8b}, [x5], #2 + ld1 {v1.8h}, [x2], #16 + ext v2.8b, v2.8b, v3.8b, #4 + subs w4, w4, #2 + neg v2.8b, v2.8b // -m + ld1 {v0.d}[0], [x0] + ld1 {v0.d}[1], [x8] + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.d}[0], [x0], x1 + st1 {v0.d}[1], [x8], x1 + b.gt 4b + ret +8: + AARCH64_VALID_JUMP_TARGET + ld2r {v4.8b, v5.8b}, [x5], #2 + ld1 {v2.8h, v3.8h}, [x2], #32 + neg v4.8b, v4.8b // -m + neg v5.8b, v5.8b + ld1 {v0.8h}, [x0] + subs w4, w4, #2 + sxtl v4.8h, v4.8b + sxtl v5.8h, v5.8b + ld1 {v1.8h}, [x8] + shl v4.8h, v4.8h, #9 // -m << 9 + shl v5.8h, v5.8h, #9 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v5.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.8h}, [x0], x1 + st1 {v1.8h}, [x8], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ld2r {v16.8b, v17.8b}, [x5], #2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + neg v16.8b, v16.8b // -m + neg v17.8b, v17.8b + ld1 {v0.8h, v1.8h}, [x0] + ld1 {v2.8h, v3.8h}, [x8] + subs w4, w4, #2 + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.8h, v17.8h, #9 + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.8h, v1.8h, v5.8h + sub v6.8h, v2.8h, v6.8h + sub v7.8h, v3.8h, v7.8h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.8h, v5.8h, v16.8h + sqrdmulh v6.8h, v6.8h, v17.8h + sqrdmulh v7.8h, v7.8h, v17.8h + add v0.8h, v0.8h, v4.8h + add v1.8h, v1.8h, v5.8h + add v2.8h, v2.8h, v6.8h + add v3.8h, v3.8h, v7.8h + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v2.8h, v3.8h}, [x8], x1 + b.gt 16b + ret +1280: +640: +320: + AARCH64_VALID_JUMP_TARGET + sub x1, x1, w3, uxtw #1 + add x7, x2, w3, uxtw #1 +321: + ld2r {v24.8b, v25.8b}, [x5], #2 + mov w6, w3 + neg v24.8b, v24.8b // -m + neg v25.8b, v25.8b + sxtl v24.8h, v24.8b + sxtl v25.8h, v25.8b + shl v24.8h, v24.8h, #9 // -m << 9 + shl v25.8h, v25.8h, #9 +32: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + subs w6, w6, #32 + sub v16.8h, v0.8h, v16.8h // a - b + sub v17.8h, v1.8h, v17.8h + sub v18.8h, v2.8h, v18.8h + sub v19.8h, v3.8h, v19.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8] + sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v17.8h, v17.8h, v24.8h + sqrdmulh v18.8h, v18.8h, v24.8h + sqrdmulh v19.8h, v19.8h, v24.8h + sub v20.8h, v4.8h, v20.8h // a - b + sub v21.8h, v5.8h, v21.8h + sub v22.8h, v6.8h, v22.8h + sub v23.8h, v7.8h, v23.8h + add v0.8h, v0.8h, v16.8h + add v1.8h, v1.8h, v17.8h + add v2.8h, v2.8h, v18.8h + add v3.8h, v3.8h, v19.8h + sqrdmulh v20.8h, v20.8h, v25.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v21.8h, v21.8h, v25.8h + sqrdmulh v22.8h, v22.8h, v25.8h + sqrdmulh v23.8h, v23.8h, v25.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v4.8h, v4.8h, v20.8h + add v5.8h, v5.8h, v21.8h + add v6.8h, v6.8h, v22.8h + add v7.8h, v7.8h, v23.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x8], #64 + b.gt 32b + subs w4, w4, #2 + add x0, x0, x1 + add x8, x8, x1 + add x2, x2, w3, uxtw #1 + add x7, x7, w3, uxtw #1 + b.gt 321b + ret +L(blend_h_tbl): + .hword L(blend_h_tbl) - 1280b + .hword L(blend_h_tbl) - 640b + .hword L(blend_h_tbl) - 320b + .hword L(blend_h_tbl) - 16b + .hword L(blend_h_tbl) - 8b + .hword L(blend_h_tbl) - 4b + .hword L(blend_h_tbl) - 2b +endfunc + +function blend_v_16bpc_neon, export=1 + adr x6, L(blend_v_tbl) + movrel x5, X(obmc_masks) + add x5, x5, w3, uxtw + clz w3, w3 + add x8, x0, x1 + lsl x1, x1, #1 + sub w3, w3, #26 + ldrh w3, [x6, x3, lsl #1] + sub x6, x6, w3, uxtw + br x6 +20: + AARCH64_VALID_JUMP_TARGET + ld1r {v2.8b}, [x5] + neg v2.8b, v2.8b // -m + sxtl v2.8h, v2.8b + shl v2.4h, v2.4h, #9 // -m << 9 +2: + ld1 {v1.s}[0], [x2], #4 + ld1 {v0.h}[0], [x0] + subs w4, w4, #2 + ld1 {v1.h}[1], [x2] + ld1 {v0.h}[1], [x8] + add x2, x2, #4 + sub v1.4h, v0.4h, v1.4h // a - b + sqrdmulh v1.4h, v1.4h, v2.4h // ((a-b)*-m + 32) >> 6 + add v0.4h, v0.4h, v1.4h + st1 {v0.h}[0], [x0], x1 + st1 {v0.h}[1], [x8], x1 + b.gt 2b + ret +40: + AARCH64_VALID_JUMP_TARGET + ld1r {v2.2s}, [x5] + sub x1, x1, #4 + neg v2.8b, v2.8b // -m + sxtl v2.8h, v2.8b + shl v2.8h, v2.8h, #9 // -m << 9 +4: + ld1 {v1.8h}, [x2], #16 + ld1 {v0.d}[0], [x0] + ld1 {v0.d}[1], [x8] + subs w4, w4, #2 + sub v1.8h, v0.8h, v1.8h // a - b + sqrdmulh v1.8h, v1.8h, v2.8h // ((a-b)*-m + 32) >> 6 + add v0.8h, v0.8h, v1.8h + st1 {v0.s}[0], [x0], #4 + st1 {v0.s}[2], [x8], #4 + st1 {v0.h}[2], [x0], x1 + st1 {v0.h}[6], [x8], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + ld1 {v4.8b}, [x5] + sub x1, x1, #8 + neg v4.8b, v4.8b // -m + sxtl v4.8h, v4.8b + shl v4.8h, v4.8h, #9 // -m << 9 +8: + ld1 {v2.8h, v3.8h}, [x2], #32 + ld1 {v0.8h}, [x0] + ld1 {v1.8h}, [x8] + subs w4, w4, #2 + sub v2.8h, v0.8h, v2.8h // a - b + sub v3.8h, v1.8h, v3.8h + sqrdmulh v2.8h, v2.8h, v4.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v3.8h, v3.8h, v4.8h + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + st1 {v0.d}[0], [x0], #8 + st1 {v1.d}[0], [x8], #8 + st1 {v0.s}[2], [x0], x1 + st1 {v1.s}[2], [x8], x1 + b.gt 8b + ret +160: + AARCH64_VALID_JUMP_TARGET + ld1 {v16.16b}, [x5] + sub x1, x1, #16 + neg v17.16b, v16.16b // -m + sxtl v16.8h, v17.8b + sxtl2 v17.8h, v17.16b + shl v16.8h, v16.8h, #9 // -m << 9 + shl v17.4h, v17.4h, #9 +16: + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + ld1 {v0.8h, v1.8h}, [x0] + subs w4, w4, #2 + ld1 {v2.8h, v3.8h}, [x8] + sub v4.8h, v0.8h, v4.8h // a - b + sub v5.4h, v1.4h, v5.4h + sub v6.8h, v2.8h, v6.8h + sub v7.4h, v3.4h, v7.4h + sqrdmulh v4.8h, v4.8h, v16.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v5.4h, v5.4h, v17.4h + sqrdmulh v6.8h, v6.8h, v16.8h + sqrdmulh v7.4h, v7.4h, v17.4h + add v0.8h, v0.8h, v4.8h + add v1.4h, v1.4h, v5.4h + add v2.8h, v2.8h, v6.8h + add v3.4h, v3.4h, v7.4h + st1 {v0.8h}, [x0], #16 + st1 {v2.8h}, [x8], #16 + st1 {v1.4h}, [x0], x1 + st1 {v3.4h}, [x8], x1 + b.gt 16b + ret +320: + AARCH64_VALID_JUMP_TARGET + ld1 {v24.16b, v25.16b}, [x5] + neg v26.16b, v24.16b // -m + neg v27.8b, v25.8b + sxtl v24.8h, v26.8b + sxtl2 v25.8h, v26.16b + sxtl v26.8h, v27.8b + shl v24.8h, v24.8h, #9 // -m << 9 + shl v25.8h, v25.8h, #9 + shl v26.8h, v26.8h, #9 +32: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x2], #64 + ld1 {v0.8h, v1.8h, v2.8h}, [x0] + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], #64 + ld1 {v4.8h, v5.8h, v6.8h}, [x8] + subs w4, w4, #2 + sub v16.8h, v0.8h, v16.8h // a - b + sub v17.8h, v1.8h, v17.8h + sub v18.8h, v2.8h, v18.8h + sub v20.8h, v4.8h, v20.8h + sub v21.8h, v5.8h, v21.8h + sub v22.8h, v6.8h, v22.8h + sqrdmulh v16.8h, v16.8h, v24.8h // ((a-b)*-m + 32) >> 6 + sqrdmulh v17.8h, v17.8h, v25.8h + sqrdmulh v18.8h, v18.8h, v26.8h + sqrdmulh v20.8h, v20.8h, v24.8h + sqrdmulh v21.8h, v21.8h, v25.8h + sqrdmulh v22.8h, v22.8h, v26.8h + add v0.8h, v0.8h, v16.8h + add v1.8h, v1.8h, v17.8h + add v2.8h, v2.8h, v18.8h + add v4.8h, v4.8h, v20.8h + add v5.8h, v5.8h, v21.8h + add v6.8h, v6.8h, v22.8h + st1 {v0.8h, v1.8h, v2.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h}, [x8], x1 + b.gt 32b + ret +L(blend_v_tbl): + .hword L(blend_v_tbl) - 320b + .hword L(blend_v_tbl) - 160b + .hword L(blend_v_tbl) - 80b + .hword L(blend_v_tbl) - 40b + .hword L(blend_v_tbl) - 20b +endfunc + + +// This has got the same signature as the put_8tap functions, +// and assumes that x9 is set to (clz(w)-24). +function put_neon + adr x10, L(put_tbl) + ldrh w9, [x10, x9, lsl #1] + sub x10, x10, w9, uxtw + br x10 + +2: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.s}[0], [x2], x3 + ld1 {v1.s}[0], [x2], x3 + subs w5, w5, #2 + st1 {v0.s}[0], [x0], x1 + st1 {v1.s}[0], [x0], x1 + b.gt 2b + ret +4: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.4h}, [x2], x3 + ld1 {v1.4h}, [x2], x3 + subs w5, w5, #2 + st1 {v0.4h}, [x0], x1 + st1 {v1.4h}, [x0], x1 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + add x8, x0, x1 + lsl x1, x1, #1 + add x9, x2, x3 + lsl x3, x3, #1 +8: + ld1 {v0.8h}, [x2], x3 + ld1 {v1.8h}, [x9], x3 + subs w5, w5, #2 + st1 {v0.8h}, [x0], x1 + st1 {v1.8h}, [x8], x1 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + subs w5, w5, #1 + stp x8, x9, [x0, #16] + add x2, x2, x3 + add x0, x0, x1 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ldp x6, x7, [x2] + ldp x8, x9, [x2, #16] + stp x6, x7, [x0] + ldp x10, x11, [x2, #32] + stp x8, x9, [x0, #16] + subs w5, w5, #1 + ldp x12, x13, [x2, #48] + stp x10, x11, [x0, #32] + stp x12, x13, [x0, #48] + add x2, x2, x3 + add x0, x0, x1 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x2] + ldp q2, q3, [x2, #32] + stp q0, q1, [x0] + ldp q4, q5, [x2, #64] + stp q2, q3, [x0, #32] + ldp q6, q7, [x2, #96] + subs w5, w5, #1 + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + add x2, x2, x3 + add x0, x0, x1 + b.gt 64b + ret +128: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x2] + ldp q2, q3, [x2, #32] + stp q0, q1, [x0] + ldp q4, q5, [x2, #64] + stp q2, q3, [x0, #32] + ldp q6, q7, [x2, #96] + subs w5, w5, #1 + stp q4, q5, [x0, #64] + ldp q16, q17, [x2, #128] + stp q6, q7, [x0, #96] + ldp q18, q19, [x2, #160] + stp q16, q17, [x0, #128] + ldp q20, q21, [x2, #192] + stp q18, q19, [x0, #160] + ldp q22, q23, [x2, #224] + stp q20, q21, [x0, #192] + stp q22, q23, [x0, #224] + add x2, x2, x3 + add x0, x0, x1 + b.gt 128b + ret + +L(put_tbl): + .hword L(put_tbl) - 128b + .hword L(put_tbl) - 64b + .hword L(put_tbl) - 32b + .hword L(put_tbl) - 16b + .hword L(put_tbl) - 80b + .hword L(put_tbl) - 4b + .hword L(put_tbl) - 2b +endfunc + + +// This has got the same signature as the prep_8tap functions, +// and assumes that x9 is set to (clz(w)-24), w7 to intermediate_bits and +// x8 to w*2. +function prep_neon + adr x10, L(prep_tbl) + ldrh w9, [x10, x9, lsl #1] + dup v31.8h, w7 // intermediate_bits + movi v30.8h, #(PREP_BIAS >> 8), lsl #8 + sub x10, x10, w9, uxtw + br x10 + +40: + AARCH64_VALID_JUMP_TARGET + add x9, x1, x2 + lsl x2, x2, #1 +4: + ld1 {v0.d}[0], [x1], x2 + ld1 {v0.d}[1], [x9], x2 + subs w4, w4, #2 + sshl v0.8h, v0.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + st1 {v0.8h}, [x0], #16 + b.gt 4b + ret +80: + AARCH64_VALID_JUMP_TARGET + add x9, x1, x2 + lsl x2, x2, #1 +8: + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x9], x2 + subs w4, w4, #2 + sshl v0.8h, v0.8h, v31.8h + sshl v1.8h, v1.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 8b + ret +16: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x1] + add x1, x1, x2 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1] + add x1, x1, x2 + subs w4, w4, #2 + sshl v1.8h, v1.8h, v31.8h + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + b.gt 16b + ret +32: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x1] + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + add x1, x1, x2 + sshl v1.8h, v1.8h, v31.8h + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + subs w4, w4, #1 + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + b.gt 32b + ret +64: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x1] + subs w4, w4, #1 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + sshl v1.8h, v1.8h, v31.8h + ldp q4, q5, [x1, #64] + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + ldp q6, q7, [x1, #96] + add x1, x1, x2 + sshl v4.8h, v4.8h, v31.8h + sshl v5.8h, v5.8h, v31.8h + sshl v6.8h, v6.8h, v31.8h + sshl v7.8h, v7.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + stp q0, q1, [x0] + sub v4.8h, v4.8h, v30.8h + sub v5.8h, v5.8h, v30.8h + stp q2, q3, [x0, #32] + sub v6.8h, v6.8h, v30.8h + sub v7.8h, v7.8h, v30.8h + stp q4, q5, [x0, #64] + stp q6, q7, [x0, #96] + add x0, x0, x8 + b.gt 64b + ret +128: + AARCH64_VALID_JUMP_TARGET + ldp q0, q1, [x1] + subs w4, w4, #1 + sshl v0.8h, v0.8h, v31.8h + ldp q2, q3, [x1, #32] + sshl v1.8h, v1.8h, v31.8h + ldp q4, q5, [x1, #64] + sshl v2.8h, v2.8h, v31.8h + sshl v3.8h, v3.8h, v31.8h + ldp q6, q7, [x1, #96] + sshl v4.8h, v4.8h, v31.8h + sshl v5.8h, v5.8h, v31.8h + ldp q16, q17, [x1, #128] + sshl v6.8h, v6.8h, v31.8h + sshl v7.8h, v7.8h, v31.8h + ldp q18, q19, [x1, #160] + sshl v16.8h, v16.8h, v31.8h + sshl v17.8h, v17.8h, v31.8h + ldp q20, q21, [x1, #192] + sshl v18.8h, v18.8h, v31.8h + sshl v19.8h, v19.8h, v31.8h + ldp q22, q23, [x1, #224] + add x1, x1, x2 + sshl v20.8h, v20.8h, v31.8h + sshl v21.8h, v21.8h, v31.8h + sshl v22.8h, v22.8h, v31.8h + sshl v23.8h, v23.8h, v31.8h + sub v0.8h, v0.8h, v30.8h + sub v1.8h, v1.8h, v30.8h + sub v2.8h, v2.8h, v30.8h + sub v3.8h, v3.8h, v30.8h + stp q0, q1, [x0] + sub v4.8h, v4.8h, v30.8h + sub v5.8h, v5.8h, v30.8h + stp q2, q3, [x0, #32] + sub v6.8h, v6.8h, v30.8h + sub v7.8h, v7.8h, v30.8h + stp q4, q5, [x0, #64] + sub v16.8h, v16.8h, v30.8h + sub v17.8h, v17.8h, v30.8h + stp q6, q7, [x0, #96] + sub v18.8h, v18.8h, v30.8h + sub v19.8h, v19.8h, v30.8h + stp q16, q17, [x0, #128] + sub v20.8h, v20.8h, v30.8h + sub v21.8h, v21.8h, v30.8h + stp q18, q19, [x0, #160] + sub v22.8h, v22.8h, v30.8h + sub v23.8h, v23.8h, v30.8h + stp q20, q21, [x0, #192] + stp q22, q23, [x0, #224] + add x0, x0, x8 + b.gt 128b + ret + +L(prep_tbl): + .hword L(prep_tbl) - 128b + .hword L(prep_tbl) - 64b + .hword L(prep_tbl) - 32b + .hword L(prep_tbl) - 16b + .hword L(prep_tbl) - 80b + .hword L(prep_tbl) - 40b +endfunc + + +.macro load_slice s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}[0], [\s0], \strd + ld1 {\d1\wd}[0], [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}[0], [\s0], \strd + ld1 {\d3\wd}[0], [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}[0], [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}[0], [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}[0], [\s0], \strd +.endif +.endm +.macro load_reg s0, s1, strd, wd, d0, d1, d2, d3, d4, d5, d6 + ld1 {\d0\wd}, [\s0], \strd + ld1 {\d1\wd}, [\s1], \strd +.ifnb \d2 + ld1 {\d2\wd}, [\s0], \strd + ld1 {\d3\wd}, [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd}, [\s0], \strd +.endif +.ifnb \d5 + ld1 {\d5\wd}, [\s1], \strd +.endif +.ifnb \d6 + ld1 {\d6\wd}, [\s0], \strd +.endif +.endm +.macro load_regpair s0, s1, strd, wd, d0, d1, d2, d3, d4, d5 + ld1 {\d0\wd, \d1\wd}, [\s0], \strd +.ifnb \d2 + ld1 {\d2\wd, \d3\wd}, [\s1], \strd +.endif +.ifnb \d4 + ld1 {\d4\wd, \d5\wd}, [\s0], \strd +.endif +.endm +.macro load_s s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_slice \s0, \s1, \strd, .s, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_4h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .4h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_8h s0, s1, strd, d0, d1, d2, d3, d4, d5, d6 + load_reg \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5, \d6 +.endm +.macro load_16h s0, s1, strd, d0, d1, d2, d3, d4, d5 + load_regpair \s0, \s1, \strd, .8h, \d0, \d1, \d2, \d3, \d4, \d5 +.endm +.macro interleave_1 wd, r0, r1, r2, r3, r4 + trn1 \r0\wd, \r0\wd, \r1\wd + trn1 \r1\wd, \r1\wd, \r2\wd +.ifnb \r3 + trn1 \r2\wd, \r2\wd, \r3\wd + trn1 \r3\wd, \r3\wd, \r4\wd +.endif +.endm +.macro interleave_1_s r0, r1, r2, r3, r4 + interleave_1 .2s, \r0, \r1, \r2, \r3, \r4 +.endm +.macro umin_h c, wd, r0, r1, r2, r3 + umin \r0\wd, \r0\wd, \c\wd +.ifnb \r1 + umin \r1\wd, \r1\wd, \c\wd +.endif +.ifnb \r2 + umin \r2\wd, \r2\wd, \c\wd + umin \r3\wd, \r3\wd, \c\wd +.endif +.endm +.macro sub_h c, wd, r0, r1, r2, r3 + sub \r0\wd, \r0\wd, \c\wd +.ifnb \r1 + sub \r1\wd, \r1\wd, \c\wd +.endif +.ifnb \r2 + sub \r2\wd, \r2\wd, \c\wd + sub \r3\wd, \r3\wd, \c\wd +.endif +.endm +.macro smull_smlal_4 d, s0, s1, s2, s3 + smull \d\().4s, \s0\().4h, v0.h[0] + smlal \d\().4s, \s1\().4h, v0.h[1] + smlal \d\().4s, \s2\().4h, v0.h[2] + smlal \d\().4s, \s3\().4h, v0.h[3] +.endm +.macro smull2_smlal2_4 d, s0, s1, s2, s3 + smull2 \d\().4s, \s0\().8h, v0.h[0] + smlal2 \d\().4s, \s1\().8h, v0.h[1] + smlal2 \d\().4s, \s2\().8h, v0.h[2] + smlal2 \d\().4s, \s3\().8h, v0.h[3] +.endm +.macro smull_smlal_8 d, s0, s1, s2, s3, s4, s5, s6, s7 + smull \d\().4s, \s0\().4h, v0.h[0] + smlal \d\().4s, \s1\().4h, v0.h[1] + smlal \d\().4s, \s2\().4h, v0.h[2] + smlal \d\().4s, \s3\().4h, v0.h[3] + smlal \d\().4s, \s4\().4h, v0.h[4] + smlal \d\().4s, \s5\().4h, v0.h[5] + smlal \d\().4s, \s6\().4h, v0.h[6] + smlal \d\().4s, \s7\().4h, v0.h[7] +.endm +.macro smull2_smlal2_8 d, s0, s1, s2, s3, s4, s5, s6, s7 + smull2 \d\().4s, \s0\().8h, v0.h[0] + smlal2 \d\().4s, \s1\().8h, v0.h[1] + smlal2 \d\().4s, \s2\().8h, v0.h[2] + smlal2 \d\().4s, \s3\().8h, v0.h[3] + smlal2 \d\().4s, \s4\().8h, v0.h[4] + smlal2 \d\().4s, \s5\().8h, v0.h[5] + smlal2 \d\().4s, \s6\().8h, v0.h[6] + smlal2 \d\().4s, \s7\().8h, v0.h[7] +.endm +.macro sqrshrun_h shift, r0, r1, r2, r3 + sqrshrun \r0\().4h, \r0\().4s, #\shift +.ifnb \r1 + sqrshrun2 \r0\().8h, \r1\().4s, #\shift +.endif +.ifnb \r2 + sqrshrun \r2\().4h, \r2\().4s, #\shift + sqrshrun2 \r2\().8h, \r3\().4s, #\shift +.endif +.endm +.macro xtn_h r0, r1, r2, r3 + uzp1 \r0\().8h, \r0\().8h, \r1\().8h // Same as xtn, xtn2 +.ifnb \r2 + uzp1 \r2\().8h, \r2\().8h, \r3\().8h // Ditto +.endif +.endm +.macro srshl_s shift, r0, r1, r2, r3 + srshl \r0\().4s, \r0\().4s, \shift\().4s + srshl \r1\().4s, \r1\().4s, \shift\().4s +.ifnb \r2 + srshl \r2\().4s, \r2\().4s, \shift\().4s + srshl \r3\().4s, \r3\().4s, \shift\().4s +.endif +.endm +.macro st_s strd, reg, lanes + st1 {\reg\().s}[0], [x0], \strd + st1 {\reg\().s}[1], [x9], \strd +.if \lanes > 2 + st1 {\reg\().s}[2], [x0], \strd + st1 {\reg\().s}[3], [x9], \strd +.endif +.endm +.macro st_d strd, r0, r1 + st1 {\r0\().d}[0], [x0], \strd + st1 {\r0\().d}[1], [x9], \strd +.ifnb \r1 + st1 {\r1\().d}[0], [x0], \strd + st1 {\r1\().d}[1], [x9], \strd +.endif +.endm +.macro shift_store_4 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin_h v31, .8h, \r0, \r2 +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub_h v29, .8h, \r0, \r2 // PREP_BIAS +.endif + st_d \strd, \r0, \r2 +.endm +.macro st_reg strd, wd, r0, r1, r2, r3, r4, r5, r6, r7 + st1 {\r0\wd}, [x0], \strd + st1 {\r1\wd}, [x9], \strd +.ifnb \r2 + st1 {\r2\wd}, [x0], \strd + st1 {\r3\wd}, [x9], \strd +.endif +.ifnb \r4 + st1 {\r4\wd}, [x0], \strd + st1 {\r5\wd}, [x9], \strd + st1 {\r6\wd}, [x0], \strd + st1 {\r7\wd}, [x9], \strd +.endif +.endm +.macro st_8h strd, r0, r1, r2, r3, r4, r5, r6, r7 + st_reg \strd, .8h, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endm +.macro shift_store_8 type, strd, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin_h v31, .8h, \r0, \r2 +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub_h v29, .8h, \r0, \r2 // PREP_BIAS +.endif + st_8h \strd, \r0, \r2 +.endm +.macro shift_store_16 type, strd, dst, r0, r1, r2, r3 +.ifc \type, put + sqrshrun_h 6, \r0, \r1, \r2, \r3 + umin \r0\().8h, \r0\().8h, v31.8h + umin \r1\().8h, \r2\().8h, v31.8h +.else + srshl_s v30, \r0, \r1, \r2, \r3 // -(6-intermediate_bits) + xtn_h \r0, \r1, \r2, \r3 + sub \r0\().8h, \r0\().8h, v29.8h + sub \r1\().8h, \r2\().8h, v29.8h +.endif + st1 {\r0\().8h, \r1\().8h}, [\dst], \strd +.endm + +.macro make_8tap_fn op, type, type_h, type_v +function \op\()_8tap_\type\()_16bpc_neon, export=1 + mov w9, \type_h + mov w10, \type_v + b \op\()_8tap_neon +endfunc +.endm + +// No spaces in these expressions, due to gas-preprocessor. +#define REGULAR ((0*15<<7)|3*15) +#define SMOOTH ((1*15<<7)|4*15) +#define SHARP ((2*15<<7)|3*15) + +.macro filter_fn type, dst, d_strd, src, s_strd, w, h, mx, xmx, my, xmy, bdmax, ds2, sr2 +make_8tap_fn \type, regular, REGULAR, REGULAR +make_8tap_fn \type, regular_smooth, REGULAR, SMOOTH +make_8tap_fn \type, regular_sharp, REGULAR, SHARP +make_8tap_fn \type, smooth, SMOOTH, SMOOTH +make_8tap_fn \type, smooth_regular, SMOOTH, REGULAR +make_8tap_fn \type, smooth_sharp, SMOOTH, SHARP +make_8tap_fn \type, sharp, SHARP, SHARP +make_8tap_fn \type, sharp_regular, SHARP, REGULAR +make_8tap_fn \type, sharp_smooth, SHARP, SMOOTH + +function \type\()_8tap_neon +.ifc \bdmax, w8 + ldr w8, [sp] +.endif + mov w11, #0x4081 // (1 << 14) | (1 << 7) | (1 << 0) + mul \mx, \mx, w11 + mul \my, \my, w11 + add \mx, \mx, w9 // mx, 8tap_h, 4tap_h + add \my, \my, w10 // my, 8tap_v, 4tap_v +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + dup v31.8h, \bdmax // bitdepth_max + clz \bdmax, \bdmax + clz w9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + mov w12, #6 + tst \mx, #(0x7f << 14) + sub w9, w9, #24 + add w13, w12, \bdmax // 6 + intermediate_bits + sub w12, w12, \bdmax // 6 - intermediate_bits + movrel x11, X(mc_subpel_filters), -8 + b.ne L(\type\()_8tap_h) + tst \my, #(0x7f << 14) + b.ne L(\type\()_8tap_v) + b \type\()_neon + +L(\type\()_8tap_h): + cmp \w, #4 + ubfx w10, \mx, #7, #7 + and \mx, \mx, #0x7f + b.le 4f + mov \mx, w10 +4: + tst \my, #(0x7f << 14) + add \xmx, x11, \mx, uxtw #3 + b.ne L(\type\()_8tap_hv) + + adr x10, L(\type\()_8tap_h_tbl) + dup v30.4s, w12 // 6 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v30.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + dup v29.8h, \bdmax // intermediate_bits +.else + movi v28.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v29.8h, v29.8h // -intermediate_bits +.endif + br x10 + +20: // 2xN h + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +2: + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v4.16b, #2 + ext v7.16b, v6.16b, v6.16b, #2 + subs \h, \h, #2 + trn1 v3.2s, v4.2s, v6.2s + trn2 v6.2s, v4.2s, v6.2s + trn1 v4.2s, v5.2s, v7.2s + trn2 v7.2s, v5.2s, v7.2s + smull v3.4s, v3.4h, v0.h[0] + smlal v3.4s, v4.4h, v0.h[1] + smlal v3.4s, v6.4h, v0.h[2] + smlal v3.4s, v7.4h, v0.h[3] + srshl v3.4s, v3.4s, v30.4s // -(6-intermediate_bits) + sqxtun v3.4h, v3.4s + srshl v3.4h, v3.4h, v29.4h // -intermediate_bits + umin v3.4h, v3.4h, v31.4h + st1 {v3.s}[0], [\dst], \d_strd + st1 {v3.s}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + AARCH64_VALID_JUMP_TARGET + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + sub \src, \src, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b +4: + ld1 {v16.8h}, [\src], \s_strd + ld1 {v20.8h}, [\sr2], \s_strd + ext v17.16b, v16.16b, v16.16b, #2 + ext v18.16b, v16.16b, v16.16b, #4 + ext v19.16b, v16.16b, v16.16b, #6 + ext v21.16b, v20.16b, v20.16b, #2 + ext v22.16b, v20.16b, v20.16b, #4 + ext v23.16b, v20.16b, v20.16b, #6 + subs \h, \h, #2 + smull v16.4s, v16.4h, v0.h[0] + smlal v16.4s, v17.4h, v0.h[1] + smlal v16.4s, v18.4h, v0.h[2] + smlal v16.4s, v19.4h, v0.h[3] + smull v20.4s, v20.4h, v0.h[0] + smlal v20.4s, v21.4h, v0.h[1] + smlal v20.4s, v22.4h, v0.h[2] + smlal v20.4s, v23.4h, v0.h[3] + srshl v16.4s, v16.4s, v30.4s // -(6-intermediate_bits) + srshl v20.4s, v20.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + sqxtun v16.4h, v16.4s + sqxtun2 v16.8h, v20.4s + srshl v16.8h, v16.8h, v29.8h // -intermediate_bits + umin v16.8h, v16.8h, v31.8h +.else + uzp1 v16.8h, v16.8h, v20.8h // Same as xtn, xtn2 + sub v16.8h, v16.8h, v28.8h // PREP_BIAS +.endif + st1 {v16.d}[0], [\dst], \d_strd + st1 {v16.d}[1], [\ds2], \d_strd + b.gt 4b + ret + +80: +160: +320: +640: +1280: // 8xN, 16xN, 32xN, ... h + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmx] + sub \src, \src, #6 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + sub \s_strd, \s_strd, \w, uxtw #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw #1 +.endif +81: + ld1 {v16.8h, v17.8h}, [\src], #32 + ld1 {v20.8h, v21.8h}, [\sr2], #32 + mov \mx, \w + +8: + smull v18.4s, v16.4h, v0.h[0] + smull2 v19.4s, v16.8h, v0.h[0] + smull v22.4s, v20.4h, v0.h[0] + smull2 v23.4s, v20.8h, v0.h[0] +.irpc i, 1234567 + ext v24.16b, v16.16b, v17.16b, #(2*\i) + ext v25.16b, v20.16b, v21.16b, #(2*\i) + smlal v18.4s, v24.4h, v0.h[\i] + smlal2 v19.4s, v24.8h, v0.h[\i] + smlal v22.4s, v25.4h, v0.h[\i] + smlal2 v23.4s, v25.8h, v0.h[\i] +.endr + subs \mx, \mx, #8 + srshl v18.4s, v18.4s, v30.4s // -(6-intermediate_bits) + srshl v19.4s, v19.4s, v30.4s // -(6-intermediate_bits) + srshl v22.4s, v22.4s, v30.4s // -(6-intermediate_bits) + srshl v23.4s, v23.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + sqxtun v18.4h, v18.4s + sqxtun2 v18.8h, v19.4s + sqxtun v22.4h, v22.4s + sqxtun2 v22.8h, v23.4s + srshl v18.8h, v18.8h, v29.8h // -intermediate_bits + srshl v22.8h, v22.8h, v29.8h // -intermediate_bits + umin v18.8h, v18.8h, v31.8h + umin v22.8h, v22.8h, v31.8h +.else + uzp1 v18.8h, v18.8h, v19.8h // Same as xtn, xtn2 + uzp1 v22.8h, v22.8h, v23.8h // Ditto + sub v18.8h, v18.8h, v28.8h // PREP_BIAS + sub v22.8h, v22.8h, v28.8h // PREP_BIAS +.endif + st1 {v18.8h}, [\dst], #16 + st1 {v22.8h}, [\ds2], #16 + b.le 9f + + mov v16.16b, v17.16b + mov v20.16b, v21.16b + ld1 {v17.8h}, [\src], #16 + ld1 {v21.8h}, [\sr2], #16 + b 8b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 81b + ret + +L(\type\()_8tap_h_tbl): + .hword L(\type\()_8tap_h_tbl) - 1280b + .hword L(\type\()_8tap_h_tbl) - 640b + .hword L(\type\()_8tap_h_tbl) - 320b + .hword L(\type\()_8tap_h_tbl) - 160b + .hword L(\type\()_8tap_h_tbl) - 80b + .hword L(\type\()_8tap_h_tbl) - 40b + .hword L(\type\()_8tap_h_tbl) - 20b + .hword 0 + + +L(\type\()_8tap_v): + cmp \h, #4 + ubfx w10, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w10 +4: + add \xmy, x11, \my, uxtw #3 + +.ifc \type, prep + dup v30.4s, w12 // 6 - intermediate_bits + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + adr x10, L(\type\()_8tap_v_tbl) + ldrh w9, [x10, x9, lsl #1] +.ifc \type, prep + neg v30.4s, v30.4s // -(6-intermediate_bits) +.endif + sub x10, x10, w9, uxtw + br x10 + +20: // 2xN v + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + b.gt 28f + + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + // 2x2 v + load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + interleave_1_s v1, v2, v3, v4, v5 + b.gt 24f + smull_smlal_4 v6, v1, v2, v3, v4 + sqrshrun_h 6, v6 + umin_h v31, .8h, v6 + st_s \d_strd, v6, 2 + ret + +24: // 2x4 v + load_s \sr2, \src, \s_strd, v6, v7 + interleave_1_s v5, v6, v7 + smull_smlal_4 v16, v1, v2, v3, v4 + smull_smlal_4 v17, v3, v4, v5, v6 + sqrshrun_h 6, v16, v17 + umin_h v31, .8h, v16 + st_s \d_strd, v16, 4 + ret + +28: // 2x6, 2x8, 2x12, 2x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + sxtl v0.8h, v0.8b + + load_s \src, \sr2, \s_strd, v1, v2, v3, v4, v5, v6, v7 + interleave_1_s v1, v2, v3, v4, v5 + interleave_1_s v5, v6, v7 +216: + subs \h, \h, #4 + load_s \sr2, \src, \s_strd, v16, v17, v18, v19 + interleave_1_s v7, v16, v17, v18, v19 + smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16 + smull_smlal_8 v25, v3, v4, v5, v6, v7, v16, v17, v18 + sqrshrun_h 6, v24, v25 + umin_h v31, .8h, v24 + st_s \d_strd, v24, 4 + b.le 0f + cmp \h, #2 + mov v1.16b, v5.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + mov v4.16b, v16.16b + mov v5.16b, v17.16b + mov v6.16b, v18.16b + mov v7.16b, v19.16b + b.eq 26f + b 216b +26: + load_s \sr2, \src, \s_strd, v16, v17 + interleave_1_s v7, v16, v17 + smull_smlal_8 v24, v1, v2, v3, v4, v5, v6, v7, v16 + sqrshrun_h 6, v24 + umin_h v31, .4h, v24 + st_s \d_strd, v24, 2 +0: + ret +.endif + +40: + AARCH64_VALID_JUMP_TARGET + b.gt 480f + + // 4x2, 4x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_4h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + smull_smlal_4 v6, v1, v2, v3, v4 + smull_smlal_4 v7, v2, v3, v4, v5 + shift_store_4 \type, \d_strd, v6, v7 + b.le 0f + load_4h \sr2, \src, \s_strd, v6, v7 + smull_smlal_4 v1, v3, v4, v5, v6 + smull_smlal_4 v2, v4, v5, v6, v7 + shift_store_4 \type, \d_strd, v1, v2 +0: + ret + +480: // 4x6, 4x8, 4x12, 4x16 v + ld1 {v0.8b}, [\xmy] + sub \sr2, \src, \s_strd, lsl #1 + add \ds2, \dst, \d_strd + sub \src, \sr2, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_4h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + +48: + subs \h, \h, #4 + load_4h \sr2, \src, \s_strd, v23, v24, v25, v26 + smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 + smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24 + smull_smlal_8 v3, v18, v19, v20, v21, v22, v23, v24, v25 + smull_smlal_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 + shift_store_4 \type, \d_strd, v1, v2, v3, v4 + b.le 0f + cmp \h, #2 + mov v16.8b, v20.8b + mov v17.8b, v21.8b + mov v18.8b, v22.8b + mov v19.8b, v23.8b + mov v20.8b, v24.8b + mov v21.8b, v25.8b + mov v22.8b, v26.8b + b.eq 46f + b 48b +46: + load_4h \sr2, \src, \s_strd, v23, v24 + smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 + smull_smlal_8 v2, v17, v18, v19, v20, v21, v22, v23, v24 + shift_store_4 \type, \d_strd, v1, v2 +0: + ret + +80: + AARCH64_VALID_JUMP_TARGET + b.gt 880f + + // 8x2, 8x4 v + cmp \h, #2 + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + + load_8h \src, \sr2, \s_strd, v1, v2, v3, v4, v5 + smull_smlal_4 v16, v1, v2, v3, v4 + smull2_smlal2_4 v17, v1, v2, v3, v4 + smull_smlal_4 v18, v2, v3, v4, v5 + smull2_smlal2_4 v19, v2, v3, v4, v5 + shift_store_8 \type, \d_strd, v16, v17, v18, v19 + b.le 0f + load_8h \sr2, \src, \s_strd, v6, v7 + smull_smlal_4 v16, v3, v4, v5, v6 + smull2_smlal2_4 v17, v3, v4, v5, v6 + smull_smlal_4 v18, v4, v5, v6, v7 + smull2_smlal2_4 v19, v4, v5, v6, v7 + shift_store_8 \type, \d_strd, v16, v17, v18, v19 +0: + ret + +880: // 8x6, 8x8, 8x16, 8x32 v +1680: // 16x8, 16x16, ... +320: // 32x8, 32x16, ... +640: +1280: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmy] + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + mov \my, \h +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + load_8h \src, \sr2, \s_strd, v16, v17, v18, v19, v20, v21, v22 + +88: + subs \h, \h, #2 + load_8h \sr2, \src, \s_strd, v23, v24 + smull_smlal_8 v1, v16, v17, v18, v19, v20, v21, v22, v23 + smull2_smlal2_8 v2, v16, v17, v18, v19, v20, v21, v22, v23 + smull_smlal_8 v3, v17, v18, v19, v20, v21, v22, v23, v24 + smull2_smlal2_8 v4, v17, v18, v19, v20, v21, v22, v23, v24 + shift_store_8 \type, \d_strd, v1, v2, v3, v4 + b.le 9f + subs \h, \h, #2 + load_8h \sr2, \src, \s_strd, v25, v26 + smull_smlal_8 v1, v18, v19, v20, v21, v22, v23, v24, v25 + smull2_smlal2_8 v2, v18, v19, v20, v21, v22, v23, v24, v25 + smull_smlal_8 v3, v19, v20, v21, v22, v23, v24, v25, v26 + smull2_smlal2_8 v4, v19, v20, v21, v22, v23, v24, v25, v26 + shift_store_8 \type, \d_strd, v1, v2, v3, v4 + b.le 9f + mov v16.16b, v20.16b + mov v17.16b, v21.16b + mov v18.16b, v22.16b + mov v19.16b, v23.16b + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v22.16b, v26.16b + b 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + ret + +160: + AARCH64_VALID_JUMP_TARGET + b.gt 1680b + + // 16x2, 16x4 v + add \xmy, \xmy, #2 + ld1 {v0.s}[0], [\xmy] + sub \src, \src, \s_strd + sxtl v0.8h, v0.8b + + load_16h \src, \src, \s_strd, v16, v17, v18, v19, v20, v21 +16: + load_16h \src, \src, \s_strd, v22, v23 + subs \h, \h, #1 + smull_smlal_4 v1, v16, v18, v20, v22 + smull2_smlal2_4 v2, v16, v18, v20, v22 + smull_smlal_4 v3, v17, v19, v21, v23 + smull2_smlal2_4 v4, v17, v19, v21, v23 + shift_store_16 \type, \d_strd, x0, v1, v2, v3, v4 + b.le 0f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v23.16b + b 16b +0: + ret + +L(\type\()_8tap_v_tbl): + .hword L(\type\()_8tap_v_tbl) - 1280b + .hword L(\type\()_8tap_v_tbl) - 640b + .hword L(\type\()_8tap_v_tbl) - 320b + .hword L(\type\()_8tap_v_tbl) - 160b + .hword L(\type\()_8tap_v_tbl) - 80b + .hword L(\type\()_8tap_v_tbl) - 40b + .hword L(\type\()_8tap_v_tbl) - 20b + .hword 0 + +L(\type\()_8tap_hv): + cmp \h, #4 + ubfx w10, \my, #7, #7 + and \my, \my, #0x7f + b.le 4f + mov \my, w10 +4: + add \xmy, x11, \my, uxtw #3 + + adr x10, L(\type\()_8tap_hv_tbl) + dup v30.4s, w12 // 6 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v30.4s, v30.4s // -(6-intermediate_bits) +.ifc \type, put + dup v29.4s, w13 // 6 + intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v29.4s, v29.4s // -(6+intermediate_bits) +.endif + br x10 + +20: + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 280f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + + // 2x2, 2x4 hv + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v27.8h}, [\src], \s_strd + ext v28.16b, v27.16b, v27.16b, #2 + smull v27.4s, v27.4h, v0.4h + smull v28.4s, v28.4h, v0.4h + addp v27.4s, v27.4s, v28.4s + addp v16.4s, v27.4s, v27.4s + srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) + bl L(\type\()_8tap_filter_2) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + xtn v16.4h, v16.4s + + trn1 v16.2s, v16.2s, v24.2s + mov v17.8b, v24.8b + +2: + bl L(\type\()_8tap_filter_2) + + ext v18.8b, v17.8b, v24.8b, #4 + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v24.4h, v1.h[3] + + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + umin v2.4h, v2.4h, v31.4h + subs \h, \h, #2 + st1 {v2.s}[0], [\dst], \d_strd + st1 {v2.s}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v24.8b + b 2b + +280: // 2x8, 2x16, 2x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v27.8h}, [\src], \s_strd + ext v28.16b, v27.16b, v27.16b, #2 + smull v27.4s, v27.4h, v0.4h + smull v28.4s, v28.4h, v0.4h + addp v27.4s, v27.4s, v28.4s + addp v16.4s, v27.4s, v27.4s + srshl v16.2s, v16.2s, v30.2s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + + bl L(\type\()_8tap_filter_2) + xtn v16.4h, v16.4s + trn1 v16.2s, v16.2s, v24.2s + mov v17.8b, v24.8b + bl L(\type\()_8tap_filter_2) + ext v18.8b, v17.8b, v24.8b, #4 + mov v19.8b, v24.8b + bl L(\type\()_8tap_filter_2) + ext v20.8b, v19.8b, v24.8b, #4 + mov v21.8b, v24.8b + +28: + bl L(\type\()_8tap_filter_2) + ext v22.8b, v21.8b, v24.8b, #4 + smull v3.4s, v16.4h, v1.h[0] + smlal v3.4s, v17.4h, v1.h[1] + smlal v3.4s, v18.4h, v1.h[2] + smlal v3.4s, v19.4h, v1.h[3] + smlal v3.4s, v20.4h, v1.h[4] + smlal v3.4s, v21.4h, v1.h[5] + smlal v3.4s, v22.4h, v1.h[6] + smlal v3.4s, v24.4h, v1.h[7] + + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + sqxtun v3.4h, v3.4s + umin v3.4h, v3.4h, v31.4h + subs \h, \h, #2 + st1 {v3.s}[0], [\dst], \d_strd + st1 {v3.s}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v19.8b + mov v18.8b, v20.8b + mov v19.8b, v21.8b + mov v20.8b, v22.8b + mov v21.8b, v24.8b + b 28b + +0: + ret x15 + +L(\type\()_8tap_filter_2): + ld1 {v25.8h}, [\sr2], \s_strd + ld1 {v27.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v28.16b, v27.16b, v27.16b, #2 + trn1 v24.2s, v25.2s, v27.2s + trn2 v27.2s, v25.2s, v27.2s + trn1 v25.2s, v26.2s, v28.2s + trn2 v28.2s, v26.2s, v28.2s + smull v24.4s, v24.4h, v0.h[0] + smlal v24.4s, v25.4h, v0.h[1] + smlal v24.4s, v27.4h, v0.h[2] + smlal v24.4s, v28.4h, v0.h[3] + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + xtn v24.4h, v24.4s + ret +.endif + +40: + AARCH64_VALID_JUMP_TARGET + add \xmx, \xmx, #2 + ld1 {v0.s}[0], [\xmx] + b.gt 480f + add \xmy, \xmy, #2 + ld1 {v1.s}[0], [\xmy] + sub \sr2, \src, #2 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + // 4x2, 4x4 hv + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + xtn v16.4h, v16.4s + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v24.8b + mov v18.8b, v25.8b + +4: + bl L(\type\()_8tap_filter_4) + smull v2.4s, v16.4h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal v2.4s, v24.4h, v1.h[3] + smull v3.4s, v17.4h, v1.h[0] + smlal v3.4s, v18.4h, v1.h[1] + smlal v3.4s, v24.4h, v1.h[2] + smlal v3.4s, v25.4h, v1.h[3] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + umin v2.8h, v2.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + + st1 {v2.d}[0], [\dst], \d_strd + st1 {v2.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v24.8b + mov v18.8b, v25.8b + b 4b + +480: // 4x8, 4x16, 4x32 hv + ld1 {v1.8b}, [\xmy] + sub \src, \src, #2 + sub \sr2, \src, \s_strd, lsl #1 + sub \src, \sr2, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v16.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53). + xtn v16.4h, v16.4s + + bl L(\type\()_8tap_filter_4) + mov v17.8b, v24.8b + mov v18.8b, v25.8b + bl L(\type\()_8tap_filter_4) + mov v19.8b, v24.8b + mov v20.8b, v25.8b + bl L(\type\()_8tap_filter_4) + mov v21.8b, v24.8b + mov v22.8b, v25.8b + +48: + bl L(\type\()_8tap_filter_4) + smull v3.4s, v16.4h, v1.h[0] + smlal v3.4s, v17.4h, v1.h[1] + smlal v3.4s, v18.4h, v1.h[2] + smlal v3.4s, v19.4h, v1.h[3] + smlal v3.4s, v20.4h, v1.h[4] + smlal v3.4s, v21.4h, v1.h[5] + smlal v3.4s, v22.4h, v1.h[6] + smlal v3.4s, v24.4h, v1.h[7] + smull v4.4s, v17.4h, v1.h[0] + smlal v4.4s, v18.4h, v1.h[1] + smlal v4.4s, v19.4h, v1.h[2] + smlal v4.4s, v20.4h, v1.h[3] + smlal v4.4s, v21.4h, v1.h[4] + smlal v4.4s, v22.4h, v1.h[5] + smlal v4.4s, v24.4h, v1.h[6] + smlal v4.4s, v25.4h, v1.h[7] +.ifc \type, put + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + sqxtun v3.4h, v3.4s + sqxtun2 v3.8h, v4.4s + umin v3.8h, v3.8h, v31.8h +.else + rshrn v3.4h, v3.4s, #6 + rshrn2 v3.8h, v4.4s, #6 + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v3.d}[0], [\dst], \d_strd + st1 {v3.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + mov v17.8b, v19.8b + mov v18.8b, v20.8b + mov v19.8b, v21.8b + mov v20.8b, v22.8b + mov v21.8b, v24.8b + mov v22.8b, v25.8b + b 48b +0: + ret x15 + +L(\type\()_8tap_filter_4): + ld1 {v24.8h}, [\sr2], \s_strd + ld1 {v25.8h}, [\src], \s_strd + ext v26.16b, v24.16b, v24.16b, #2 + ext v27.16b, v24.16b, v24.16b, #4 + ext v28.16b, v24.16b, v24.16b, #6 + smull v24.4s, v24.4h, v0.h[0] + smlal v24.4s, v26.4h, v0.h[1] + smlal v24.4s, v27.4h, v0.h[2] + smlal v24.4s, v28.4h, v0.h[3] + ext v26.16b, v25.16b, v25.16b, #2 + ext v27.16b, v25.16b, v25.16b, #4 + ext v28.16b, v25.16b, v25.16b, #6 + smull v25.4s, v25.4h, v0.h[0] + smlal v25.4s, v26.4h, v0.h[1] + smlal v25.4s, v27.4h, v0.h[2] + smlal v25.4s, v28.4h, v0.h[3] + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + xtn v24.4h, v24.4s + xtn v25.4h, v25.4s + ret + +80: +160: +320: + AARCH64_VALID_JUMP_TARGET + b.gt 880f + add \xmy, \xmy, #2 + ld1 {v0.8b}, [\xmx] + ld1 {v1.s}[0], [\xmy] + sub \src, \src, #6 + sub \src, \src, \s_strd + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +164: // 8x2, 8x4, 16x2, 16x4, 32x2, 32x4 hv + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + ld1 {v27.8h, v28.8h}, [\src], \s_strd + smull v24.4s, v27.4h, v0.h[0] + smull2 v25.4s, v27.8h, v0.h[0] +.irpc i, 1234567 + ext v26.16b, v27.16b, v28.16b, #(2*\i) + smlal v24.4s, v26.4h, v0.h[\i] + smlal2 v25.4s, v26.8h, v0.h[\i] +.endr + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53), + // and conserves register space (no need to clobber v8-v15). + uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2 + + bl L(\type\()_8tap_filter_8) + mov v17.16b, v23.16b + mov v18.16b, v24.16b + +8: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v23.4h, v1.h[2] + smlal2 v5.4s, v23.8h, v1.h[2] + smlal v2.4s, v23.4h, v1.h[3] + smlal2 v3.4s, v23.8h, v1.h[3] + smlal v4.4s, v24.4h, v1.h[3] + smlal2 v5.4s, v24.8h, v1.h[3] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v2.8h}, [\dst], \d_strd + st1 {v3.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v23.16b + mov v18.16b, v24.16b + b 8b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #2 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 164b + +880: // 8x8, 8x16, ..., 16x8, ..., 32x8, ... hv +640: +1280: + AARCH64_VALID_JUMP_TARGET + ld1 {v0.8b}, [\xmx] + ld1 {v1.8b}, [\xmy] + sub \src, \src, #6 + sub \src, \src, \s_strd + sub \src, \src, \s_strd, lsl #1 + sxtl v0.8h, v0.8b + sxtl v1.8h, v1.8b + mov x15, x30 + mov \my, \h + +168: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 + + ld1 {v27.8h, v28.8h}, [\src], \s_strd + smull v24.4s, v27.4h, v0.h[0] + smull2 v25.4s, v27.8h, v0.h[0] +.irpc i, 1234567 + ext v26.16b, v27.16b, v28.16b, #(2*\i) + smlal v24.4s, v26.4h, v0.h[\i] + smlal2 v25.4s, v26.8h, v0.h[\i] +.endr + srshl v24.4s, v24.4s, v30.4s // -(6-intermediate_bits) + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + // The intermediates from the horizontal pass fit in 16 bit without + // any bias; we could just as well keep them as .4s, but narrowing + // them to .4h gives a significant speedup on out of order cores + // (at the cost of a smaller slowdown on in-order cores such as A53), + // and conserves register space (no need to clobber v8-v15). + uzp1 v16.8h, v24.8h, v25.8h // Same as xtn, xtn2 + + bl L(\type\()_8tap_filter_8) + mov v17.16b, v23.16b + mov v18.16b, v24.16b + bl L(\type\()_8tap_filter_8) + mov v19.16b, v23.16b + mov v20.16b, v24.16b + bl L(\type\()_8tap_filter_8) + mov v21.16b, v23.16b + mov v22.16b, v24.16b + +88: + smull v2.4s, v16.4h, v1.h[0] + smull2 v3.4s, v16.8h, v1.h[0] + bl L(\type\()_8tap_filter_8) + smull v4.4s, v17.4h, v1.h[0] + smull2 v5.4s, v17.8h, v1.h[0] + smlal v2.4s, v17.4h, v1.h[1] + smlal2 v3.4s, v17.8h, v1.h[1] + smlal v4.4s, v18.4h, v1.h[1] + smlal2 v5.4s, v18.8h, v1.h[1] + smlal v2.4s, v18.4h, v1.h[2] + smlal2 v3.4s, v18.8h, v1.h[2] + smlal v4.4s, v19.4h, v1.h[2] + smlal2 v5.4s, v19.8h, v1.h[2] + smlal v2.4s, v19.4h, v1.h[3] + smlal2 v3.4s, v19.8h, v1.h[3] + smlal v4.4s, v20.4h, v1.h[3] + smlal2 v5.4s, v20.8h, v1.h[3] + smlal v2.4s, v20.4h, v1.h[4] + smlal2 v3.4s, v20.8h, v1.h[4] + smlal v4.4s, v21.4h, v1.h[4] + smlal2 v5.4s, v21.8h, v1.h[4] + smlal v2.4s, v21.4h, v1.h[5] + smlal2 v3.4s, v21.8h, v1.h[5] + smlal v4.4s, v22.4h, v1.h[5] + smlal2 v5.4s, v22.8h, v1.h[5] + smlal v2.4s, v22.4h, v1.h[6] + smlal2 v3.4s, v22.8h, v1.h[6] + smlal v4.4s, v23.4h, v1.h[6] + smlal2 v5.4s, v23.8h, v1.h[6] + smlal v2.4s, v23.4h, v1.h[7] + smlal2 v3.4s, v23.8h, v1.h[7] + smlal v4.4s, v24.4h, v1.h[7] + smlal2 v5.4s, v24.8h, v1.h[7] +.ifc \type, put + srshl v2.4s, v2.4s, v29.4s // -(6+intermediate_bits) + srshl v3.4s, v3.4s, v29.4s // -(6+intermediate_bits) + srshl v4.4s, v4.4s, v29.4s // -(6+intermediate_bits) + srshl v5.4s, v5.4s, v29.4s // -(6+intermediate_bits) + sqxtun v2.4h, v2.4s + sqxtun2 v2.8h, v3.4s + sqxtun v3.4h, v4.4s + sqxtun2 v3.8h, v5.4s + umin v2.8h, v2.8h, v31.8h + umin v3.8h, v3.8h, v31.8h +.else + rshrn v2.4h, v2.4s, #6 + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + sub v2.8h, v2.8h, v29.8h // PREP_BIAS + sub v3.8h, v3.8h, v29.8h // PREP_BIAS +.endif + subs \h, \h, #2 + st1 {v2.8h}, [\dst], \d_strd + st1 {v3.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + mov v17.16b, v19.16b + mov v18.16b, v20.16b + mov v19.16b, v21.16b + mov v20.16b, v22.16b + mov v21.16b, v23.16b + mov v22.16b, v24.16b + b 88b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #3 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 168b +0: + ret x15 + +L(\type\()_8tap_filter_8): + ld1 {v4.8h, v5.8h}, [\sr2], \s_strd + ld1 {v6.8h, v7.8h}, [\src], \s_strd + smull v25.4s, v4.4h, v0.h[0] + smull2 v26.4s, v4.8h, v0.h[0] + smull v27.4s, v6.4h, v0.h[0] + smull2 v28.4s, v6.8h, v0.h[0] +.irpc i, 1234567 + ext v23.16b, v4.16b, v5.16b, #(2*\i) + ext v24.16b, v6.16b, v7.16b, #(2*\i) + smlal v25.4s, v23.4h, v0.h[\i] + smlal2 v26.4s, v23.8h, v0.h[\i] + smlal v27.4s, v24.4h, v0.h[\i] + smlal2 v28.4s, v24.8h, v0.h[\i] +.endr + srshl v25.4s, v25.4s, v30.4s // -(6-intermediate_bits) + srshl v26.4s, v26.4s, v30.4s // -(6-intermediate_bits) + srshl v27.4s, v27.4s, v30.4s // -(6-intermediate_bits) + srshl v28.4s, v28.4s, v30.4s // -(6-intermediate_bits) + uzp1 v23.8h, v25.8h, v26.8h // Same as xtn, xtn2 + uzp1 v24.8h, v27.8h, v28.8h // Ditto + ret + +L(\type\()_8tap_hv_tbl): + .hword L(\type\()_8tap_hv_tbl) - 1280b + .hword L(\type\()_8tap_hv_tbl) - 640b + .hword L(\type\()_8tap_hv_tbl) - 320b + .hword L(\type\()_8tap_hv_tbl) - 160b + .hword L(\type\()_8tap_hv_tbl) - 80b + .hword L(\type\()_8tap_hv_tbl) - 40b + .hword L(\type\()_8tap_hv_tbl) - 20b + .hword 0 +endfunc + + +function \type\()_bilin_16bpc_neon, export=1 +.ifc \bdmax, w8 + ldr w8, [sp] +.endif + dup v1.8h, \mx + dup v3.8h, \my + mov w10, #16 + sub w9, w10, \mx + sub w10, w10, \my + dup v0.8h, w9 + dup v2.8h, w10 +.ifc \type, prep + uxtw \d_strd, \w + lsl \d_strd, \d_strd, #1 +.endif + + clz \bdmax, \bdmax // bitdepth_max + clz w9, \w + sub \bdmax, \bdmax, #18 // intermediate_bits = clz(bitdepth_max) - 18 + mov w11, #4 + sub w9, w9, #24 + sub w11, w11, \bdmax // 4 - intermediate_bits + add w12, \bdmax, #4 // 4 + intermediate_bits + cbnz \mx, L(\type\()_bilin_h) + cbnz \my, L(\type\()_bilin_v) + b \type\()_neon + +L(\type\()_bilin_h): + cbnz \my, L(\type\()_bilin_hv) + + adr x10, L(\type\()_bilin_h_tbl) + dup v31.8h, w11 // 4 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v31.8h, v31.8h // -(4-intermediate_bits) +.ifc \type, put + dup v30.8h, \bdmax // intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v30.8h, v30.8h // -intermediate_bits +.endif + br x10 + +20: // 2xN h + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +2: + ld1 {v4.4h}, [\src], \s_strd + ld1 {v6.4h}, [\sr2], \s_strd + ext v5.8b, v4.8b, v4.8b, #2 + ext v7.8b, v6.8b, v6.8b, #2 + trn1 v4.2s, v4.2s, v6.2s + trn1 v5.2s, v5.2s, v7.2s + subs \h, \h, #2 + mul v4.4h, v4.4h, v0.4h + mla v4.4h, v5.4h, v1.4h + urshl v4.4h, v4.4h, v31.4h + urshl v4.4h, v4.4h, v30.4h + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + b.gt 2b + ret +.endif + +40: // 4xN h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +4: + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v4.16b, #2 + ext v7.16b, v6.16b, v6.16b, #2 + trn1 v4.2d, v4.2d, v6.2d + trn1 v5.2d, v5.2d, v7.2d + subs \h, \h, #2 + mul v4.8h, v4.8h, v0.8h + mla v4.8h, v5.8h, v1.8h + urshl v4.8h, v4.8h, v31.8h +.ifc \type, put + urshl v4.8h, v4.8h, v30.8h +.else + sub v4.8h, v4.8h, v29.8h +.endif + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.gt 4b + ret + +80: // 8xN h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \d_strd, \d_strd, #1 + lsl \s_strd, \s_strd, #1 +8: + ldr h5, [\src, #16] + ldr h7, [\sr2, #16] + ld1 {v4.8h}, [\src], \s_strd + ld1 {v6.8h}, [\sr2], \s_strd + ext v5.16b, v4.16b, v5.16b, #2 + ext v7.16b, v6.16b, v7.16b, #2 + subs \h, \h, #2 + mul v4.8h, v4.8h, v0.8h + mla v4.8h, v5.8h, v1.8h + mul v6.8h, v6.8h, v0.8h + mla v6.8h, v7.8h, v1.8h + urshl v4.8h, v4.8h, v31.8h + urshl v6.8h, v6.8h, v31.8h +.ifc \type, put + urshl v4.8h, v4.8h, v30.8h + urshl v6.8h, v6.8h, v30.8h +.else + sub v4.8h, v4.8h, v29.8h + sub v6.8h, v6.8h, v29.8h +.endif + st1 {v4.8h}, [\dst], \d_strd + st1 {v6.8h}, [\ds2], \d_strd + b.gt 8b + ret +160: +320: +640: +1280: // 16xN, 32xN, ... h + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + + sub \s_strd, \s_strd, \w, uxtw #1 + sub \s_strd, \s_strd, #16 +.ifc \type, put + lsl \d_strd, \d_strd, #1 + sub \d_strd, \d_strd, \w, uxtw #1 +.endif +161: + ld1 {v16.8h}, [\src], #16 + ld1 {v21.8h}, [\sr2], #16 + mov \mx, \w + +16: + ld1 {v17.8h, v18.8h}, [\src], #32 + ld1 {v22.8h, v23.8h}, [\sr2], #32 + ext v19.16b, v16.16b, v17.16b, #2 + ext v20.16b, v17.16b, v18.16b, #2 + ext v24.16b, v21.16b, v22.16b, #2 + ext v25.16b, v22.16b, v23.16b, #2 + mul v16.8h, v16.8h, v0.8h + mla v16.8h, v19.8h, v1.8h + mul v17.8h, v17.8h, v0.8h + mla v17.8h, v20.8h, v1.8h + mul v21.8h, v21.8h, v0.8h + mla v21.8h, v24.8h, v1.8h + mul v22.8h, v22.8h, v0.8h + mla v22.8h, v25.8h, v1.8h + urshl v16.8h, v16.8h, v31.8h + urshl v17.8h, v17.8h, v31.8h + urshl v21.8h, v21.8h, v31.8h + urshl v22.8h, v22.8h, v31.8h + subs \mx, \mx, #16 +.ifc \type, put + urshl v16.8h, v16.8h, v30.8h + urshl v17.8h, v17.8h, v30.8h + urshl v21.8h, v21.8h, v30.8h + urshl v22.8h, v22.8h, v30.8h +.else + sub v16.8h, v16.8h, v29.8h + sub v17.8h, v17.8h, v29.8h + sub v21.8h, v21.8h, v29.8h + sub v22.8h, v22.8h, v29.8h +.endif + st1 {v16.8h, v17.8h}, [\dst], #32 + st1 {v21.8h, v22.8h}, [\ds2], #32 + b.le 9f + + mov v16.16b, v18.16b + mov v21.16b, v23.16b + b 16b + +9: + add \dst, \dst, \d_strd + add \ds2, \ds2, \d_strd + add \src, \src, \s_strd + add \sr2, \sr2, \s_strd + + subs \h, \h, #2 + b.gt 161b + ret + +L(\type\()_bilin_h_tbl): + .hword L(\type\()_bilin_h_tbl) - 1280b + .hword L(\type\()_bilin_h_tbl) - 640b + .hword L(\type\()_bilin_h_tbl) - 320b + .hword L(\type\()_bilin_h_tbl) - 160b + .hword L(\type\()_bilin_h_tbl) - 80b + .hword L(\type\()_bilin_h_tbl) - 40b + .hword L(\type\()_bilin_h_tbl) - 20b + .hword 0 + + +L(\type\()_bilin_v): + cmp \h, #4 + adr x10, L(\type\()_bilin_v_tbl) +.ifc \type, prep + dup v31.8h, w11 // 4 - intermediate_bits +.endif + ldrh w9, [x10, x9, lsl #1] +.ifc \type, prep + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 + neg v31.8h, v31.8h // -(4-intermediate_bits) +.endif + sub x10, x10, w9, uxtw + br x10 + +20: // 2xN v + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + cmp \h, #2 + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + // 2x2 v + ld1 {v16.s}[0], [\src], \s_strd + b.gt 24f +22: + ld1 {v17.s}[0], [\sr2], \s_strd + ld1 {v18.s}[0], [\src], \s_strd + trn1 v16.2s, v16.2s, v17.2s + trn1 v17.2s, v17.2s, v18.2s + mul v4.4h, v16.4h, v2.4h + mla v4.4h, v17.4h, v3.4h + urshr v4.8h, v4.8h, #4 + st1 {v4.s}[0], [\dst] + st1 {v4.s}[1], [\ds2] + ret +24: // 2x4, 2x6, 2x8, ... v + ld1 {v17.s}[0], [\sr2], \s_strd + ld1 {v18.s}[0], [\src], \s_strd + ld1 {v19.s}[0], [\sr2], \s_strd + ld1 {v20.s}[0], [\src], \s_strd + sub \h, \h, #4 + trn1 v16.2s, v16.2s, v17.2s + trn1 v17.2s, v17.2s, v18.2s + trn1 v18.2s, v18.2s, v19.2s + trn1 v19.2s, v19.2s, v20.2s + trn1 v16.2d, v16.2d, v18.2d + trn1 v17.2d, v17.2d, v19.2d + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + cmp \h, #2 + urshr v4.8h, v4.8h, #4 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + st1 {v4.s}[2], [\dst], \d_strd + st1 {v4.s}[3], [\ds2], \d_strd + b.lt 0f + mov v16.8b, v20.8b + b.eq 22b + b 24b +0: + ret +.endif + +40: // 4xN v + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.4h}, [\src], \s_strd +4: + ld1 {v17.4h}, [\sr2], \s_strd + ld1 {v18.4h}, [\src], \s_strd + trn1 v16.2d, v16.2d, v17.2d + trn1 v17.2d, v17.2d, v18.2d + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + sub v4.8h, v4.8h, v29.8h +.endif + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.le 0f + mov v16.8b, v18.8b + b 4b +0: + ret + +80: // 8xN v + AARCH64_VALID_JUMP_TARGET + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + ld1 {v16.8h}, [\src], \s_strd +8: + ld1 {v17.8h}, [\sr2], \s_strd + ld1 {v18.8h}, [\src], \s_strd + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v17.8h, v3.8h + mul v5.8h, v17.8h, v2.8h + mla v5.8h, v18.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 + urshr v5.8h, v5.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + urshl v5.8h, v5.8h, v31.8h + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h +.endif + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd + b.le 0f + mov v16.16b, v18.16b + b 8b +0: + ret + +160: // 16xN, 32xN, ... +320: +640: +1280: + AARCH64_VALID_JUMP_TARGET + mov \my, \h +1: + add \ds2, \dst, \d_strd + add \sr2, \src, \s_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v16.8h, v17.8h}, [\src], \s_strd +2: + ld1 {v18.8h, v19.8h}, [\sr2], \s_strd + ld1 {v20.8h, v21.8h}, [\src], \s_strd + mul v4.8h, v16.8h, v2.8h + mla v4.8h, v18.8h, v3.8h + mul v5.8h, v17.8h, v2.8h + mla v5.8h, v19.8h, v3.8h + mul v6.8h, v18.8h, v2.8h + mla v6.8h, v20.8h, v3.8h + mul v7.8h, v19.8h, v2.8h + mla v7.8h, v21.8h, v3.8h + subs \h, \h, #2 +.ifc \type, put + urshr v4.8h, v4.8h, #4 + urshr v5.8h, v5.8h, #4 + urshr v6.8h, v6.8h, #4 + urshr v7.8h, v7.8h, #4 +.else + urshl v4.8h, v4.8h, v31.8h + urshl v5.8h, v5.8h, v31.8h + urshl v6.8h, v6.8h, v31.8h + urshl v7.8h, v7.8h, v31.8h + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h + sub v6.8h, v6.8h, v29.8h + sub v7.8h, v7.8h, v29.8h +.endif + st1 {v4.8h, v5.8h}, [\dst], \d_strd + st1 {v6.8h, v7.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v20.16b + mov v17.16b, v21.16b + b 2b +9: + subs \w, \w, #16 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #32 + add \dst, \dst, #32 + b 1b +0: + ret + +L(\type\()_bilin_v_tbl): + .hword L(\type\()_bilin_v_tbl) - 1280b + .hword L(\type\()_bilin_v_tbl) - 640b + .hword L(\type\()_bilin_v_tbl) - 320b + .hword L(\type\()_bilin_v_tbl) - 160b + .hword L(\type\()_bilin_v_tbl) - 80b + .hword L(\type\()_bilin_v_tbl) - 40b + .hword L(\type\()_bilin_v_tbl) - 20b + .hword 0 + +L(\type\()_bilin_hv): + adr x10, L(\type\()_bilin_hv_tbl) + dup v31.8h, w11 // 4 - intermediate_bits + ldrh w9, [x10, x9, lsl #1] + neg v31.8h, v31.8h // -(4-intermediate_bits) +.ifc \type, put + dup v30.4s, w12 // 4 + intermediate_bits +.else + movi v29.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + sub x10, x10, w9, uxtw +.ifc \type, put + neg v30.4s, v30.4s // -(4+intermediate_bits) +.endif + br x10 + +20: // 2xN hv + AARCH64_VALID_JUMP_TARGET +.ifc \type, put + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v20.4h}, [\src], \s_strd + ext v21.8b, v20.8b, v20.8b, #2 + mul v16.4h, v20.4h, v0.4h + mla v16.4h, v21.4h, v1.4h + urshl v16.4h, v16.4h, v31.4h + +2: + ld1 {v22.4h}, [\sr2], \s_strd + ld1 {v24.4h}, [\src], \s_strd + ext v23.8b, v22.8b, v22.8b, #2 + ext v25.8b, v24.8b, v24.8b, #2 + trn1 v22.2s, v22.2s, v24.2s + trn1 v23.2s, v23.2s, v25.2s + mul v17.4h, v22.4h, v0.4h + mla v17.4h, v23.4h, v1.4h + urshl v17.4h, v17.4h, v31.4h + + trn1 v16.2s, v16.2s, v17.2s + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + urshl v4.4s, v4.4s, v30.4s + xtn v4.4h, v4.4s + subs \h, \h, #2 + st1 {v4.s}[0], [\dst], \d_strd + st1 {v4.s}[1], [\ds2], \d_strd + b.le 0f + trn2 v16.2s, v17.2s, v17.2s + b 2b +0: + ret +.endif + +40: // 4xN hv + AARCH64_VALID_JUMP_TARGET + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ld1 {v20.8h}, [\src], \s_strd + ext v21.16b, v20.16b, v20.16b, #2 + mul v16.4h, v20.4h, v0.4h + mla v16.4h, v21.4h, v1.4h + urshl v16.4h, v16.4h, v31.4h + +4: + ld1 {v22.8h}, [\sr2], \s_strd + ld1 {v24.8h}, [\src], \s_strd + ext v23.16b, v22.16b, v22.16b, #2 + ext v25.16b, v24.16b, v24.16b, #2 + trn1 v22.2d, v22.2d, v24.2d + trn1 v23.2d, v23.2d, v25.2d + mul v17.8h, v22.8h, v0.8h + mla v17.8h, v23.8h, v1.8h + urshl v17.8h, v17.8h, v31.8h + + trn1 v16.2d, v16.2d, v17.2d + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + umull2 v5.4s, v16.8h, v2.8h + umlal2 v5.4s, v17.8h, v3.8h +.ifc \type, put + urshl v4.4s, v4.4s, v30.4s + urshl v5.4s, v5.4s, v30.4s + uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2 +.else + rshrn v4.4h, v4.4s, #4 + rshrn2 v4.8h, v5.4s, #4 + sub v4.8h, v4.8h, v29.8h +.endif + subs \h, \h, #2 + st1 {v4.d}[0], [\dst], \d_strd + st1 {v4.d}[1], [\ds2], \d_strd + b.le 0f + trn2 v16.2d, v17.2d, v17.2d + b 4b +0: + ret + +80: // 8xN, 16xN, ... hv +160: +320: +640: +1280: + AARCH64_VALID_JUMP_TARGET + mov \my, \h + +1: + add \sr2, \src, \s_strd + add \ds2, \dst, \d_strd + lsl \s_strd, \s_strd, #1 + lsl \d_strd, \d_strd, #1 + + ldr h21, [\src, #16] + ld1 {v20.8h}, [\src], \s_strd + ext v21.16b, v20.16b, v21.16b, #2 + mul v16.8h, v20.8h, v0.8h + mla v16.8h, v21.8h, v1.8h + urshl v16.8h, v16.8h, v31.8h + +2: + ldr h23, [\sr2, #16] + ld1 {v22.8h}, [\sr2], \s_strd + ldr h25, [\src, #16] + ld1 {v24.8h}, [\src], \s_strd + ext v23.16b, v22.16b, v23.16b, #2 + ext v25.16b, v24.16b, v25.16b, #2 + mul v17.8h, v22.8h, v0.8h + mla v17.8h, v23.8h, v1.8h + mul v18.8h, v24.8h, v0.8h + mla v18.8h, v25.8h, v1.8h + urshl v17.8h, v17.8h, v31.8h + urshl v18.8h, v18.8h, v31.8h + + umull v4.4s, v16.4h, v2.4h + umlal v4.4s, v17.4h, v3.4h + umull2 v5.4s, v16.8h, v2.8h + umlal2 v5.4s, v17.8h, v3.8h + umull v6.4s, v17.4h, v2.4h + umlal v6.4s, v18.4h, v3.4h + umull2 v7.4s, v17.8h, v2.8h + umlal2 v7.4s, v18.8h, v3.8h +.ifc \type, put + urshl v4.4s, v4.4s, v30.4s + urshl v5.4s, v5.4s, v30.4s + urshl v6.4s, v6.4s, v30.4s + urshl v7.4s, v7.4s, v30.4s + uzp1 v4.8h, v4.8h, v5.8h // Same as xtn, xtn2 + uzp1 v5.8h, v6.8h, v7.8h // Ditto +.else + rshrn v4.4h, v4.4s, #4 + rshrn2 v4.8h, v5.4s, #4 + rshrn v5.4h, v6.4s, #4 + rshrn2 v5.8h, v7.4s, #4 + sub v4.8h, v4.8h, v29.8h + sub v5.8h, v5.8h, v29.8h +.endif + subs \h, \h, #2 + st1 {v4.8h}, [\dst], \d_strd + st1 {v5.8h}, [\ds2], \d_strd + b.le 9f + mov v16.16b, v18.16b + b 2b +9: + subs \w, \w, #8 + b.le 0f + asr \s_strd, \s_strd, #1 + asr \d_strd, \d_strd, #1 + msub \src, \s_strd, \xmy, \src + msub \dst, \d_strd, \xmy, \dst + sub \src, \src, \s_strd, lsl #1 + mov \h, \my + add \src, \src, #16 + add \dst, \dst, #16 + b 1b +0: + ret + +L(\type\()_bilin_hv_tbl): + .hword L(\type\()_bilin_hv_tbl) - 1280b + .hword L(\type\()_bilin_hv_tbl) - 640b + .hword L(\type\()_bilin_hv_tbl) - 320b + .hword L(\type\()_bilin_hv_tbl) - 160b + .hword L(\type\()_bilin_hv_tbl) - 80b + .hword L(\type\()_bilin_hv_tbl) - 40b + .hword L(\type\()_bilin_hv_tbl) - 20b + .hword 0 +endfunc +.endm + +filter_fn put, x0, x1, x2, x3, w4, w5, w6, x6, w7, x7, w8, x9, x10 +filter_fn prep, x0, x8, x1, x2, w3, w4, w5, x5, w6, x6, w7, x9, x10 + +.macro load_filter_row dst, src, inc + asr w13, \src, #10 + add \src, \src, \inc + ldr \dst, [x11, w13, sxtw #3] +.endm + +function warp_filter_horz_neon + add w12, w5, #512 + + ld1 {v16.8h, v17.8h}, [x2], x3 + + load_filter_row d0, w12, w7 + load_filter_row d1, w12, w7 + load_filter_row d2, w12, w7 + sxtl v0.8h, v0.8b + load_filter_row d3, w12, w7 + sxtl v1.8h, v1.8b + load_filter_row d4, w12, w7 + sxtl v2.8h, v2.8b + load_filter_row d5, w12, w7 + sxtl v3.8h, v3.8b + load_filter_row d6, w12, w7 + sxtl v4.8h, v4.8b + load_filter_row d7, w12, w7 + sxtl v5.8h, v5.8b + ext v18.16b, v16.16b, v17.16b, #2*1 + smull v8.4s, v16.4h, v0.4h + smull2 v9.4s, v16.8h, v0.8h + sxtl v6.8h, v6.8b + ext v19.16b, v16.16b, v17.16b, #2*2 + smull v10.4s, v18.4h, v1.4h + smull2 v11.4s, v18.8h, v1.8h + sxtl v7.8h, v7.8b + ext v20.16b, v16.16b, v17.16b, #2*3 + smull v0.4s, v19.4h, v2.4h + smull2 v1.4s, v19.8h, v2.8h + ext v21.16b, v16.16b, v17.16b, #2*4 + addp v8.4s, v8.4s, v9.4s + smull v2.4s, v20.4h, v3.4h + smull2 v3.4s, v20.8h, v3.8h + ext v22.16b, v16.16b, v17.16b, #2*5 + addp v9.4s, v10.4s, v11.4s + smull v10.4s, v21.4h, v4.4h + smull2 v11.4s, v21.8h, v4.8h + ext v23.16b, v16.16b, v17.16b, #2*6 + addp v0.4s, v0.4s, v1.4s + smull v18.4s, v22.4h, v5.4h + smull2 v19.4s, v22.8h, v5.8h + ext v16.16b, v16.16b, v17.16b, #2*7 + addp v1.4s, v2.4s, v3.4s + addp v2.4s, v10.4s, v11.4s + smull v20.4s, v23.4h, v6.4h + smull2 v21.4s, v23.8h, v6.8h + addp v3.4s, v18.4s, v19.4s + smull v22.4s, v16.4h, v7.4h + smull2 v23.4s, v16.8h, v7.8h + addp v4.4s, v20.4s, v21.4s + addp v5.4s, v22.4s, v23.4s + + addp v8.4s, v8.4s, v9.4s + addp v0.4s, v0.4s, v1.4s + addp v2.4s, v2.4s, v3.4s + addp v4.4s, v4.4s, v5.4s + + addp v16.4s, v8.4s, v0.4s + addp v17.4s, v2.4s, v4.4s + + add w5, w5, w8 + + srshl v16.4s, v16.4s, v14.4s // -(7 - intermediate_bits) + srshl v17.4s, v17.4s, v14.4s // -(7 - intermediate_bits) + + ret +endfunc + +// void dav1d_warp_affine_8x8_16bpc_neon( +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *src, const ptrdiff_t src_stride, +// const int16_t *const abcd, int mx, int my, +// const int bitdepth_max) +.macro warp t +function warp_affine_8x8\t\()_16bpc_neon, export=1 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + +.ifb \t + dup v15.8h, w7 // bitdepth_max +.else + movi v15.8h, #(PREP_BIAS >> 8), lsl #8 +.endif + clz w7, w7 + // intermediate_bits = clz(bitdepth_max) - 18 +.ifb \t + sub w8, w7, #11 // 7 + intermediate_bits = clz(bitdepth_max) - 18 + 7 +.endif + sub w7, w7, #25 // -(7 - intermediate_bits) +.ifb \t + neg w8, w8 // -(7 + intermediate_bits) +.endif + dup v14.4s, w7 // -(7 - intermediate_bits) +.ifb \t + dup v13.4s, w8 // -(7 + intermediate_bits) +.endif + + ldr x4, [x4] + sbfx x7, x4, #0, #16 + sbfx x8, x4, #16, #16 + sbfx x9, x4, #32, #16 + sbfx x4, x4, #48, #16 + mov w10, #8 + sub x2, x2, x3, lsl #1 + sub x2, x2, x3 + sub x2, x2, #6 + movrel x11, X(mc_warp_filter), 64*8 + mov x15, x30 +.ifnb \t + lsl x1, x1, #1 +.endif + + bl warp_filter_horz_neon + uzp1 v24.8h, v16.8h, v17.8h // Same as xtn, xtn2 + bl warp_filter_horz_neon + uzp1 v25.8h, v16.8h, v17.8h // Ditto + bl warp_filter_horz_neon + uzp1 v26.8h, v16.8h, v17.8h // Ditto + bl warp_filter_horz_neon + uzp1 v27.8h, v16.8h, v17.8h // Ditto + bl warp_filter_horz_neon + uzp1 v28.8h, v16.8h, v17.8h // Ditto + bl warp_filter_horz_neon + uzp1 v29.8h, v16.8h, v17.8h // Ditto + bl warp_filter_horz_neon + uzp1 v30.8h, v16.8h, v17.8h // Ditto + +1: + add w14, w6, #512 + bl warp_filter_horz_neon + uzp1 v31.8h, v16.8h, v17.8h // Same as xtn, xtn2 + + load_filter_row d0, w14, w9 + load_filter_row d1, w14, w9 + load_filter_row d2, w14, w9 + load_filter_row d3, w14, w9 + load_filter_row d4, w14, w9 + load_filter_row d5, w14, w9 + load_filter_row d6, w14, w9 + load_filter_row d7, w14, w9 + transpose_8x8b_xtl v0, v1, v2, v3, v4, v5, v6, v7, sxtl + + // This ordering of smull/smlal/smull2/smlal2 is highly + // beneficial for Cortex A53 here. + smull v16.4s, v24.4h, v0.4h + smlal v16.4s, v25.4h, v1.4h + smlal v16.4s, v26.4h, v2.4h + smlal v16.4s, v27.4h, v3.4h + smlal v16.4s, v28.4h, v4.4h + smlal v16.4s, v29.4h, v5.4h + smlal v16.4s, v30.4h, v6.4h + smlal v16.4s, v31.4h, v7.4h + smull2 v17.4s, v24.8h, v0.8h + smlal2 v17.4s, v25.8h, v1.8h + smlal2 v17.4s, v26.8h, v2.8h + smlal2 v17.4s, v27.8h, v3.8h + smlal2 v17.4s, v28.8h, v4.8h + smlal2 v17.4s, v29.8h, v5.8h + smlal2 v17.4s, v30.8h, v6.8h + smlal2 v17.4s, v31.8h, v7.8h + + mov v24.16b, v25.16b + mov v25.16b, v26.16b +.ifb \t + srshl v16.4s, v16.4s, v13.4s // -(7 + intermediate_bits) + srshl v17.4s, v17.4s, v13.4s // -(7 + intermediate_bits) +.else + rshrn v16.4h, v16.4s, #7 + rshrn2 v16.8h, v17.4s, #7 +.endif + mov v26.16b, v27.16b +.ifb \t + sqxtun v16.4h, v16.4s + sqxtun2 v16.8h, v17.4s +.else + sub v16.8h, v16.8h, v15.8h // PREP_BIAS +.endif + mov v27.16b, v28.16b + mov v28.16b, v29.16b +.ifb \t + umin v16.8h, v16.8h, v15.8h // bitdepth_max +.endif + mov v29.16b, v30.16b + mov v30.16b, v31.16b + subs w10, w10, #1 + st1 {v16.8h}, [x0], x1 + + add w6, w6, w4 + b.gt 1b + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + + ret x15 +endfunc +.endm + +warp +warp t + +// void dav1d_emu_edge_16bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_16bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13, lsl #1 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.8h}, [x8] + mov x12, x6 // out = dst + mov x3, x4 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4, lsl #1 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x13], #64 + subs x3, x3, #32 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64 + b.gt 1b +.if \need_right + add x3, x8, x2, lsl #1 // in + center_w + sub x3, x3, #2 // in + center_w - 1 + add x12, x6, x4, lsl #1 // dst + left_ext + ld1r {v0.8h}, [x3] + add x12, x12, x2, lsl #1 // out = dst + left_ext + center_w + mov x3, x11 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], #64 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x14], #64 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/msac.S b/third_party/dav1d/src/arm/64/msac.S new file mode 100644 index 0000000000..3a6cf900a9 --- /dev/null +++ b/third_party/dav1d/src/arm/64/msac.S @@ -0,0 +1,480 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define BUF_POS 0 +#define BUF_END 8 +#define DIF 16 +#define RNG 24 +#define CNT 28 +#define ALLOW_UPDATE_CDF 32 + +const coeffs + .short 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0 + .short 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +endconst + +const bits + .short 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + .short 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000 +endconst + +.macro ld1_n d0, d1, src, sz, n +.if \n <= 8 + ld1 {\d0\sz}, [\src] +.else + ld1 {\d0\sz, \d1\sz}, [\src] +.endif +.endm + +.macro st1_n s0, s1, dst, sz, n +.if \n <= 8 + st1 {\s0\sz}, [\dst] +.else + st1 {\s0\sz, \s1\sz}, [\dst] +.endif +.endm + +.macro ushr_n d0, d1, s0, s1, shift, sz, n + ushr \d0\sz, \s0\sz, \shift +.if \n == 16 + ushr \d1\sz, \s1\sz, \shift +.endif +.endm + +.macro add_n d0, d1, s0, s1, s2, s3, sz, n + add \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + add \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro sub_n d0, d1, s0, s1, s2, s3, sz, n + sub \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + sub \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro and_n d0, d1, s0, s1, s2, s3, sz, n + and \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + and \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro cmhs_n d0, d1, s0, s1, s2, s3, sz, n + cmhs \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + cmhs \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro urhadd_n d0, d1, s0, s1, s2, s3, sz, n + urhadd \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + urhadd \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro sshl_n d0, d1, s0, s1, s2, s3, sz, n + sshl \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + sshl \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro sqdmulh_n d0, d1, s0, s1, s2, s3, sz, n + sqdmulh \d0\sz, \s0\sz, \s2\sz +.if \n == 16 + sqdmulh \d1\sz, \s1\sz, \s3\sz +.endif +.endm + +.macro str_n idx0, idx1, dstreg, dstoff, n + str \idx0, [\dstreg, \dstoff] +.if \n == 16 + str \idx1, [\dstreg, \dstoff + 16] +.endif +.endm + +// unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf, +// size_t n_symbols); + +function msac_decode_symbol_adapt4_neon, export=1 +.macro decode_update sz, szb, n + sub sp, sp, #48 + add x8, x0, #RNG + ld1_n v0, v1, x1, \sz, \n // cdf + ld1r {v4\sz}, [x8] // rng + movrel x9, coeffs, 30 + movi v31\sz, #0x7f, lsl #8 // 0x7f00 + sub x9, x9, x2, lsl #1 + mvni v30\sz, #0x3f // 0xffc0 + and v7\szb, v4\szb, v31\szb // rng & 0x7f00 + str h4, [sp, #14] // store original u = s->rng + and_n v2, v3, v0, v1, v30, v30, \szb, \n // cdf & 0xffc0 + + ld1_n v4, v5, x9, \sz, \n // EC_MIN_PROB * (n_symbols - ret) + sqdmulh_n v6, v7, v2, v3, v7, v7, \sz, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add x8, x0, #DIF + 6 + + add_n v4, v5, v2, v3, v4, v5, \sz, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret) + add_n v4, v5, v6, v7, v4, v5, \sz, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + + ld1r {v6.8h}, [x8] // dif >> (EC_WIN_SIZE - 16) + movrel x8, bits + str_n q4, q5, sp, #16, \n // store v values to allow indexed access + + ld1_n v16, v17, x8, .8h, \n + + cmhs_n v2, v3, v6, v6, v4, v5, .8h, \n // c >= v + + and_n v6, v7, v2, v3, v16, v17, .16b, \n // One bit per halfword set in the mask +.if \n == 16 + add v6.8h, v6.8h, v7.8h +.endif + addv h6, v6.8h // Aggregate mask bits + ldr w4, [x0, #ALLOW_UPDATE_CDF] + umov w3, v6.h[0] + rbit w3, w3 + clz w15, w3 // ret + + cbz w4, L(renorm) + // update_cdf + ldrh w3, [x1, x2, lsl #1] // count = cdf[n_symbols] + movi v5\szb, #0xff +.if \n == 16 + mov w4, #-5 +.else + mvn w14, w2 + mov w4, #-4 + cmn w14, #3 // set C if n_symbols <= 2 +.endif + urhadd_n v4, v5, v5, v5, v2, v3, \sz, \n // i >= val ? -1 : 32768 +.if \n == 16 + sub w4, w4, w3, lsr #4 // -((count >> 4) + 5) +.else + lsr w14, w3, #4 // count >> 4 + sbc w4, w4, w14 // -((count >> 4) + (n_symbols > 2) + 4) +.endif + sub_n v4, v5, v4, v5, v0, v1, \sz, \n // (32768 - cdf[i]) or (-1 - cdf[i]) + dup v6\sz, w4 // -rate + + sub w3, w3, w3, lsr #5 // count - (count == 32) + sub_n v0, v1, v0, v1, v2, v3, \sz, \n // cdf + (i >= val ? 1 : 0) + sshl_n v4, v5, v4, v5, v6, v6, \sz, \n // ({32768,-1} - cdf[i]) >> rate + add w3, w3, #1 // count + (count < 32) + add_n v0, v1, v0, v1, v4, v5, \sz, \n // cdf + (32768 - cdf[i]) >> rate + st1_n v0, v1, x1, \sz, \n + strh w3, [x1, x2, lsl #1] +.endm + + decode_update .4h, .8b, 4 + +L(renorm): + add x8, sp, #16 + add x8, x8, w15, uxtw #1 + ldrh w3, [x8] // v + ldurh w4, [x8, #-2] // u + ldr w6, [x0, #CNT] + ldr x7, [x0, #DIF] + sub w4, w4, w3 // rng = u - v + clz w5, w4 // clz(rng) + eor w5, w5, #16 // d = clz(rng) ^ 16 + mvn x7, x7 // ~dif + add x7, x7, x3, lsl #48 // ~dif + (v << 48) +L(renorm2): + lsl w4, w4, w5 // rng << d + subs w6, w6, w5 // cnt -= d + lsl x7, x7, x5 // (~dif + (v << 48)) << d + str w4, [x0, #RNG] + mvn x7, x7 // ~dif + b.hs 9f + + // refill + ldp x3, x4, [x0] // BUF_POS, BUF_END + add x5, x3, #8 + cmp x5, x4 + b.gt 2f + + ldr x3, [x3] // next_bits + add w8, w6, #23 // shift_bits = cnt + 23 + add w6, w6, #16 // cnt += 16 + rev x3, x3 // next_bits = bswap(next_bits) + sub x5, x5, x8, lsr #3 // buf_pos -= shift_bits >> 3 + and w8, w8, #24 // shift_bits &= 24 + lsr x3, x3, x8 // next_bits >>= shift_bits + sub w8, w8, w6 // shift_bits -= 16 + cnt + str x5, [x0, #BUF_POS] + lsl x3, x3, x8 // next_bits <<= shift_bits + mov w4, #48 + sub w6, w4, w8 // cnt = cnt + 64 - shift_bits + eor x7, x7, x3 // dif ^= next_bits + b 9f + +2: // refill_eob + mov w14, #40 + sub w5, w14, w6 // c = 40 - cnt +3: + cmp x3, x4 + b.ge 4f + ldrb w8, [x3], #1 + lsl x8, x8, x5 + eor x7, x7, x8 + subs w5, w5, #8 + b.ge 3b + +4: // refill_eob_end + str x3, [x0, #BUF_POS] + sub w6, w14, w5 // cnt = 40 - c + +9: + str w6, [x0, #CNT] + str x7, [x0, #DIF] + + mov w0, w15 + add sp, sp, #48 + ret +endfunc + +function msac_decode_symbol_adapt8_neon, export=1 + decode_update .8h, .16b, 8 + b L(renorm) +endfunc + +function msac_decode_symbol_adapt16_neon, export=1 + decode_update .8h, .16b, 16 + b L(renorm) +endfunc + +function msac_decode_hi_tok_neon, export=1 + ld1 {v0.4h}, [x1] // cdf + add x16, x0, #RNG + movi v31.4h, #0x7f, lsl #8 // 0x7f00 + movrel x17, coeffs, 30-2*3 + mvni v30.4h, #0x3f // 0xffc0 + ldrh w9, [x1, #6] // count = cdf[n_symbols] + ld1r {v3.4h}, [x16] // rng + movrel x16, bits + ld1 {v29.4h}, [x17] // EC_MIN_PROB * (n_symbols - ret) + add x17, x0, #DIF + 6 + ld1 {v16.8h}, [x16] + mov w13, #-24 + and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 + ldr w10, [x0, #ALLOW_UPDATE_CDF] + ld1r {v1.8h}, [x17] // dif >> (EC_WIN_SIZE - 16) + sub sp, sp, #48 + ldr w6, [x0, #CNT] + ldr x7, [x0, #DIF] +1: + and v7.8b, v3.8b, v31.8b // rng & 0x7f00 + sqdmulh v6.4h, v17.4h, v7.4h // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add v4.4h, v17.4h, v29.4h // v = cdf + EC_MIN_PROB * (n_symbols - ret) + add v4.4h, v6.4h, v4.4h // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + str h3, [sp, #14] // store original u = s->rng + cmhs v2.8h, v1.8h, v4.8h // c >= v + str q4, [sp, #16] // store v values to allow indexed access + and v6.16b, v2.16b, v16.16b // One bit per halfword set in the mask + addv h6, v6.8h // Aggregate mask bits + umov w3, v6.h[0] + add w13, w13, #5 + rbit w3, w3 + add x8, sp, #16 + clz w15, w3 // ret + + cbz w10, 2f + // update_cdf + movi v5.8b, #0xff + mov w4, #-5 + urhadd v4.4h, v5.4h, v2.4h // i >= val ? -1 : 32768 + sub w4, w4, w9, lsr #4 // -((count >> 4) + 5) + sub v4.4h, v4.4h, v0.4h // (32768 - cdf[i]) or (-1 - cdf[i]) + dup v6.4h, w4 // -rate + + sub w9, w9, w9, lsr #5 // count - (count == 32) + sub v0.4h, v0.4h, v2.4h // cdf + (i >= val ? 1 : 0) + sshl v4.4h, v4.4h, v6.4h // ({32768,-1} - cdf[i]) >> rate + add w9, w9, #1 // count + (count < 32) + add v0.4h, v0.4h, v4.4h // cdf + (32768 - cdf[i]) >> rate + st1 {v0.4h}, [x1] + and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 + strh w9, [x1, #6] + +2: + add x8, x8, w15, uxtw #1 + ldrh w3, [x8] // v + ldurh w4, [x8, #-2] // u + sub w4, w4, w3 // rng = u - v + clz w5, w4 // clz(rng) + eor w5, w5, #16 // d = clz(rng) ^ 16 + mvn x7, x7 // ~dif + add x7, x7, x3, lsl #48 // ~dif + (v << 48) + lsl w4, w4, w5 // rng << d + subs w6, w6, w5 // cnt -= d + lsl x7, x7, x5 // (~dif + (v << 48)) << d + str w4, [x0, #RNG] + dup v3.4h, w4 + mvn x7, x7 // ~dif + b.hs 9f + + // refill + ldp x3, x4, [x0] // BUF_POS, BUF_END + add x5, x3, #8 + cmp x5, x4 + b.gt 2f + + ldr x3, [x3] // next_bits + add w8, w6, #23 // shift_bits = cnt + 23 + add w6, w6, #16 // cnt += 16 + rev x3, x3 // next_bits = bswap(next_bits) + sub x5, x5, x8, lsr #3 // buf_pos -= shift_bits >> 3 + and w8, w8, #24 // shift_bits &= 24 + lsr x3, x3, x8 // next_bits >>= shift_bits + sub w8, w8, w6 // shift_bits -= 16 + cnt + str x5, [x0, #BUF_POS] + lsl x3, x3, x8 // next_bits <<= shift_bits + mov w4, #48 + sub w6, w4, w8 // cnt = cnt + 64 - shift_bits + eor x7, x7, x3 // dif ^= next_bits + b 9f + +2: // refill_eob + mov w14, #40 + sub w5, w14, w6 // c = 40 - cnt +3: + cmp x3, x4 + b.ge 4f + ldrb w8, [x3], #1 + lsl x8, x8, x5 + eor x7, x7, x8 + subs w5, w5, #8 + b.ge 3b + +4: // refill_eob_end + str x3, [x0, #BUF_POS] + sub w6, w14, w5 // cnt = 40 - c + +9: + lsl w15, w15, #1 + sub w15, w15, #5 + lsr x12, x7, #48 + adds w13, w13, w15 // carry = tok_br < 3 || tok == 15 + dup v1.8h, w12 + b.cc 1b // loop if !carry + add w13, w13, #30 + str w6, [x0, #CNT] + add sp, sp, #48 + str x7, [x0, #DIF] + lsr w0, w13, #1 + ret +endfunc + +function msac_decode_bool_equi_neon, export=1 + ldp w5, w6, [x0, #RNG] // + CNT + sub sp, sp, #48 + ldr x7, [x0, #DIF] + bic w4, w5, #0xff // r &= 0xff00 + add w4, w4, #8 + subs x8, x7, x4, lsl #47 // dif - vw + lsr w4, w4, #1 // v + sub w5, w5, w4 // r - v + cset w15, lo + csel w4, w5, w4, hs // if (ret) v = r - v; + csel x7, x8, x7, hs // if (ret) dif = dif - vw; + + clz w5, w4 // clz(rng) + mvn x7, x7 // ~dif + eor w5, w5, #16 // d = clz(rng) ^ 16 + b L(renorm2) +endfunc + +function msac_decode_bool_neon, export=1 + ldp w5, w6, [x0, #RNG] // + CNT + sub sp, sp, #48 + ldr x7, [x0, #DIF] + lsr w4, w5, #8 // r >> 8 + bic w1, w1, #0x3f // f &= ~63 + mul w4, w4, w1 + lsr w4, w4, #7 + add w4, w4, #4 // v + subs x8, x7, x4, lsl #48 // dif - vw + sub w5, w5, w4 // r - v + cset w15, lo + csel w4, w5, w4, hs // if (ret) v = r - v; + csel x7, x8, x7, hs // if (ret) dif = dif - vw; + + clz w5, w4 // clz(rng) + mvn x7, x7 // ~dif + eor w5, w5, #16 // d = clz(rng) ^ 16 + b L(renorm2) +endfunc + +function msac_decode_bool_adapt_neon, export=1 + ldr w9, [x1] // cdf[0-1] + ldp w5, w6, [x0, #RNG] // + CNT + sub sp, sp, #48 + ldr x7, [x0, #DIF] + lsr w4, w5, #8 // r >> 8 + and w2, w9, #0xffc0 // f &= ~63 + mul w4, w4, w2 + lsr w4, w4, #7 + add w4, w4, #4 // v + subs x8, x7, x4, lsl #48 // dif - vw + sub w5, w5, w4 // r - v + cset w15, lo + csel w4, w5, w4, hs // if (ret) v = r - v; + csel x7, x8, x7, hs // if (ret) dif = dif - vw; + + ldr w10, [x0, #ALLOW_UPDATE_CDF] + + clz w5, w4 // clz(rng) + mvn x7, x7 // ~dif + eor w5, w5, #16 // d = clz(rng) ^ 16 + + cbz w10, L(renorm2) + + lsr w2, w9, #16 // count = cdf[1] + and w9, w9, #0xffff // cdf[0] + + sub w3, w2, w2, lsr #5 // count - (count >= 32) + lsr w2, w2, #4 // count >> 4 + add w10, w3, #1 // count + (count < 32) + add w2, w2, #4 // rate = (count >> 4) | 4 + + sub w9, w9, w15 // cdf[0] -= bit + sub w11, w9, w15, lsl #15 // {cdf[0], cdf[0] - 32769} + asr w11, w11, w2 // {cdf[0], cdf[0] - 32769} >> rate + sub w9, w9, w11 // cdf[0] + + strh w9, [x1] + strh w10, [x1, #2] + + b L(renorm2) +endfunc diff --git a/third_party/dav1d/src/arm/64/refmvs.S b/third_party/dav1d/src/arm/64/refmvs.S new file mode 100644 index 0000000000..becd4c08f6 --- /dev/null +++ b/third_party/dav1d/src/arm/64/refmvs.S @@ -0,0 +1,91 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void dav1d_splat_mv_neon(refmvs_block **rr, const refmvs_block *rmv, +// int bx4, int bw4, int bh4) + +function splat_mv_neon, export=1 + ld1 {v3.16b}, [x1] + clz w3, w3 + adr x5, L(splat_tbl) + sub w3, w3, #26 + ext v2.16b, v3.16b, v3.16b, #12 + ldrh w3, [x5, w3, uxtw #1] + add w2, w2, w2, lsl #1 + ext v0.16b, v2.16b, v3.16b, #4 + sub x3, x5, w3, uxtw + ext v1.16b, v2.16b, v3.16b, #8 + lsl w2, w2, #2 + ext v2.16b, v2.16b, v3.16b, #12 +1: + ldr x1, [x0], #8 + subs w4, w4, #1 + add x1, x1, x2 + br x3 + +10: + AARCH64_VALID_JUMP_TARGET + st1 {v0.8b}, [x1] + str s2, [x1, #8] + b.gt 1b + ret +20: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b}, [x1] + str d1, [x1, #16] + b.gt 1b + ret +320: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +160: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +80: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +40: + AARCH64_VALID_JUMP_TARGET + st1 {v0.16b, v1.16b, v2.16b}, [x1] + b.gt 1b + ret + +L(splat_tbl): + .hword L(splat_tbl) - 320b + .hword L(splat_tbl) - 160b + .hword L(splat_tbl) - 80b + .hword L(splat_tbl) - 40b + .hword L(splat_tbl) - 20b + .hword L(splat_tbl) - 10b +endfunc diff --git a/third_party/dav1d/src/arm/64/util.S b/third_party/dav1d/src/arm/64/util.S new file mode 100644 index 0000000000..9013fd4b1e --- /dev/null +++ b/third_party/dav1d/src/arm/64/util.S @@ -0,0 +1,229 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2015 Martin Storsjo + * Copyright © 2015 Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#ifndef DAV1D_SRC_ARM_64_UTIL_S +#define DAV1D_SRC_ARM_64_UTIL_S + +#include "config.h" +#include "src/arm/asm.S" + +.macro movrel rd, val, offset=0 +#if defined(__APPLE__) + .if \offset < 0 + adrp \rd, \val@PAGE + add \rd, \rd, \val@PAGEOFF + sub \rd, \rd, -(\offset) + .else + adrp \rd, \val+(\offset)@PAGE + add \rd, \rd, \val+(\offset)@PAGEOFF + .endif +#elif defined(PIC) && defined(_WIN32) + .if \offset < 0 + adrp \rd, \val + add \rd, \rd, :lo12:\val + sub \rd, \rd, -(\offset) + .else + adrp \rd, \val+(\offset) + add \rd, \rd, :lo12:\val+(\offset) + .endif +#elif defined(PIC) + adrp \rd, \val+(\offset) + add \rd, \rd, :lo12:\val+(\offset) +#else + ldr \rd, =\val+\offset +#endif +.endm + +.macro sub_sp space +#ifdef _WIN32 +.if \space > 8192 + // Here, we'd need to touch two (or more) pages while decrementing + // the stack pointer. + .error "sub_sp_align doesn't support values over 8K at the moment" +.elseif \space > 4096 + sub x16, sp, #4096 + ldr xzr, [x16] + sub sp, x16, #(\space - 4096) +.else + sub sp, sp, #\space +.endif +#else +.if \space >= 4096 + sub sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + sub sp, sp, #(\space)%4096 +.endif +#endif +.endm + +.macro transpose_8x8b_xtl r0, r1, r2, r3, r4, r5, r6, r7, xtl + // a0 b0 a1 b1 a2 b2 a3 b3 a4 b4 a5 b5 a6 b6 a7 b7 + zip1 \r0\().16b, \r0\().16b, \r1\().16b + // c0 d0 c1 d1 c2 d2 d3 d3 c4 d4 c5 d5 c6 d6 d7 d7 + zip1 \r2\().16b, \r2\().16b, \r3\().16b + // e0 f0 e1 f1 e2 f2 e3 f3 e4 f4 e5 f5 e6 f6 e7 f7 + zip1 \r4\().16b, \r4\().16b, \r5\().16b + // g0 h0 g1 h1 g2 h2 h3 h3 g4 h4 g5 h5 g6 h6 h7 h7 + zip1 \r6\().16b, \r6\().16b, \r7\().16b + + // a0 b0 c0 d0 a2 b2 c2 d2 a4 b4 c4 d4 a6 b6 c6 d6 + trn1 \r1\().8h, \r0\().8h, \r2\().8h + // a1 b1 c1 d1 a3 b3 c3 d3 a5 b5 c5 d5 a7 b7 c7 d7 + trn2 \r3\().8h, \r0\().8h, \r2\().8h + // e0 f0 g0 h0 e2 f2 g2 h2 e4 f4 g4 h4 e6 f6 g6 h6 + trn1 \r5\().8h, \r4\().8h, \r6\().8h + // e1 f1 g1 h1 e3 f3 g3 h3 e5 f5 g5 h5 e7 f7 g7 h7 + trn2 \r7\().8h, \r4\().8h, \r6\().8h + + // a0 b0 c0 d0 e0 f0 g0 h0 a4 b4 c4 d4 e4 f4 g4 h4 + trn1 \r0\().4s, \r1\().4s, \r5\().4s + // a2 b2 c2 d2 e2 f2 g2 h2 a6 b6 c6 d6 e6 f6 g6 h6 + trn2 \r2\().4s, \r1\().4s, \r5\().4s + // a1 b1 c1 d1 e1 f1 g1 h1 a5 b5 c5 d5 e5 f5 g5 h5 + trn1 \r1\().4s, \r3\().4s, \r7\().4s + // a3 b3 c3 d3 e3 f3 g3 h3 a7 b7 c7 d7 e7 f7 g7 h7 + trn2 \r3\().4s, \r3\().4s, \r7\().4s + + \xtl\()2 \r4\().8h, \r0\().16b + \xtl \r0\().8h, \r0\().8b + \xtl\()2 \r6\().8h, \r2\().16b + \xtl \r2\().8h, \r2\().8b + \xtl\()2 \r5\().8h, \r1\().16b + \xtl \r1\().8h, \r1\().8b + \xtl\()2 \r7\().8h, \r3\().16b + \xtl \r3\().8h, \r3\().8b +.endm + +.macro transpose_8x8h r0, r1, r2, r3, r4, r5, r6, r7, t8, t9 + trn1 \t8\().8h, \r0\().8h, \r1\().8h + trn2 \t9\().8h, \r0\().8h, \r1\().8h + trn1 \r1\().8h, \r2\().8h, \r3\().8h + trn2 \r3\().8h, \r2\().8h, \r3\().8h + trn1 \r0\().8h, \r4\().8h, \r5\().8h + trn2 \r5\().8h, \r4\().8h, \r5\().8h + trn1 \r2\().8h, \r6\().8h, \r7\().8h + trn2 \r7\().8h, \r6\().8h, \r7\().8h + + trn1 \r4\().4s, \r0\().4s, \r2\().4s + trn2 \r2\().4s, \r0\().4s, \r2\().4s + trn1 \r6\().4s, \r5\().4s, \r7\().4s + trn2 \r7\().4s, \r5\().4s, \r7\().4s + trn1 \r5\().4s, \t9\().4s, \r3\().4s + trn2 \t9\().4s, \t9\().4s, \r3\().4s + trn1 \r3\().4s, \t8\().4s, \r1\().4s + trn2 \t8\().4s, \t8\().4s, \r1\().4s + + trn1 \r0\().2d, \r3\().2d, \r4\().2d + trn2 \r4\().2d, \r3\().2d, \r4\().2d + trn1 \r1\().2d, \r5\().2d, \r6\().2d + trn2 \r5\().2d, \r5\().2d, \r6\().2d + trn2 \r6\().2d, \t8\().2d, \r2\().2d + trn1 \r2\().2d, \t8\().2d, \r2\().2d + trn1 \r3\().2d, \t9\().2d, \r7\().2d + trn2 \r7\().2d, \t9\().2d, \r7\().2d +.endm + +.macro transpose_8x16b r0, r1, r2, r3, r4, r5, r6, r7, t8, t9 + trn1 \t8\().16b, \r0\().16b, \r1\().16b + trn2 \t9\().16b, \r0\().16b, \r1\().16b + trn1 \r1\().16b, \r2\().16b, \r3\().16b + trn2 \r3\().16b, \r2\().16b, \r3\().16b + trn1 \r0\().16b, \r4\().16b, \r5\().16b + trn2 \r5\().16b, \r4\().16b, \r5\().16b + trn1 \r2\().16b, \r6\().16b, \r7\().16b + trn2 \r7\().16b, \r6\().16b, \r7\().16b + + trn1 \r4\().8h, \r0\().8h, \r2\().8h + trn2 \r2\().8h, \r0\().8h, \r2\().8h + trn1 \r6\().8h, \r5\().8h, \r7\().8h + trn2 \r7\().8h, \r5\().8h, \r7\().8h + trn1 \r5\().8h, \t9\().8h, \r3\().8h + trn2 \t9\().8h, \t9\().8h, \r3\().8h + trn1 \r3\().8h, \t8\().8h, \r1\().8h + trn2 \t8\().8h, \t8\().8h, \r1\().8h + + trn1 \r0\().4s, \r3\().4s, \r4\().4s + trn2 \r4\().4s, \r3\().4s, \r4\().4s + trn1 \r1\().4s, \r5\().4s, \r6\().4s + trn2 \r5\().4s, \r5\().4s, \r6\().4s + trn2 \r6\().4s, \t8\().4s, \r2\().4s + trn1 \r2\().4s, \t8\().4s, \r2\().4s + trn1 \r3\().4s, \t9\().4s, \r7\().4s + trn2 \r7\().4s, \t9\().4s, \r7\().4s +.endm + +.macro transpose_4x16b r0, r1, r2, r3, t4, t5, t6, t7 + trn1 \t4\().16b, \r0\().16b, \r1\().16b + trn2 \t5\().16b, \r0\().16b, \r1\().16b + trn1 \t6\().16b, \r2\().16b, \r3\().16b + trn2 \t7\().16b, \r2\().16b, \r3\().16b + + trn1 \r0\().8h, \t4\().8h, \t6\().8h + trn2 \r2\().8h, \t4\().8h, \t6\().8h + trn1 \r1\().8h, \t5\().8h, \t7\().8h + trn2 \r3\().8h, \t5\().8h, \t7\().8h +.endm + +.macro transpose_4x4h r0, r1, r2, r3, t4, t5, t6, t7 + trn1 \t4\().4h, \r0\().4h, \r1\().4h + trn2 \t5\().4h, \r0\().4h, \r1\().4h + trn1 \t6\().4h, \r2\().4h, \r3\().4h + trn2 \t7\().4h, \r2\().4h, \r3\().4h + + trn1 \r0\().2s, \t4\().2s, \t6\().2s + trn2 \r2\().2s, \t4\().2s, \t6\().2s + trn1 \r1\().2s, \t5\().2s, \t7\().2s + trn2 \r3\().2s, \t5\().2s, \t7\().2s +.endm + +.macro transpose_4x4s r0, r1, r2, r3, t4, t5, t6, t7 + trn1 \t4\().4s, \r0\().4s, \r1\().4s + trn2 \t5\().4s, \r0\().4s, \r1\().4s + trn1 \t6\().4s, \r2\().4s, \r3\().4s + trn2 \t7\().4s, \r2\().4s, \r3\().4s + + trn1 \r0\().2d, \t4\().2d, \t6\().2d + trn2 \r2\().2d, \t4\().2d, \t6\().2d + trn1 \r1\().2d, \t5\().2d, \t7\().2d + trn2 \r3\().2d, \t5\().2d, \t7\().2d +.endm + +.macro transpose_4x8h r0, r1, r2, r3, t4, t5, t6, t7 + trn1 \t4\().8h, \r0\().8h, \r1\().8h + trn2 \t5\().8h, \r0\().8h, \r1\().8h + trn1 \t6\().8h, \r2\().8h, \r3\().8h + trn2 \t7\().8h, \r2\().8h, \r3\().8h + + trn1 \r0\().4s, \t4\().4s, \t6\().4s + trn2 \r2\().4s, \t4\().4s, \t6\().4s + trn1 \r1\().4s, \t5\().4s, \t7\().4s + trn2 \r3\().4s, \t5\().4s, \t7\().4s +.endm + +#endif /* DAV1D_SRC_ARM_64_UTIL_S */ diff --git a/third_party/dav1d/src/arm/asm-offsets.h b/third_party/dav1d/src/arm/asm-offsets.h new file mode 100644 index 0000000000..2f3c3caa1f --- /dev/null +++ b/third_party/dav1d/src/arm/asm-offsets.h @@ -0,0 +1,43 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ARM_ASM_OFFSETS_H +#define ARM_ASM_OFFSETS_H + +#define FGD_SEED 0 +#define FGD_AR_COEFF_LAG 92 +#define FGD_AR_COEFFS_Y 96 +#define FGD_AR_COEFFS_UV 120 +#define FGD_AR_COEFF_SHIFT 176 +#define FGD_GRAIN_SCALE_SHIFT 184 + +#define FGD_SCALING_SHIFT 88 +#define FGD_UV_MULT 188 +#define FGD_UV_LUMA_MULT 196 +#define FGD_UV_OFFSET 204 +#define FGD_CLIP_TO_RESTRICTED_RANGE 216 + +#endif /* ARM_ASM_OFFSETS_H */ diff --git a/third_party/dav1d/src/arm/asm.S b/third_party/dav1d/src/arm/asm.S new file mode 100644 index 0000000000..dc50415f1f --- /dev/null +++ b/third_party/dav1d/src/arm/asm.S @@ -0,0 +1,291 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_ARM_ASM_S +#define DAV1D_SRC_ARM_ASM_S + +#include "config.h" + +#if ARCH_AARCH64 +#define x18 do_not_use_x18 +#define w18 do_not_use_w18 + +/* Support macros for + * - Armv8.3-A Pointer Authentication and + * - Armv8.5-A Branch Target Identification + * features which require emitting a .note.gnu.property section with the + * appropriate architecture-dependent feature bits set. + * + * |AARCH64_SIGN_LINK_REGISTER| and |AARCH64_VALIDATE_LINK_REGISTER| expand to + * PACIxSP and AUTIxSP, respectively. |AARCH64_SIGN_LINK_REGISTER| should be + * used immediately before saving the LR register (x30) to the stack. + * |AARCH64_VALIDATE_LINK_REGISTER| should be used immediately after restoring + * it. Note |AARCH64_SIGN_LINK_REGISTER|'s modifications to LR must be undone + * with |AARCH64_VALIDATE_LINK_REGISTER| before RET. The SP register must also + * have the same value at the two points. For example: + * + * .global f + * f: + * AARCH64_SIGN_LINK_REGISTER + * stp x29, x30, [sp, #-96]! + * mov x29, sp + * ... + * ldp x29, x30, [sp], #96 + * AARCH64_VALIDATE_LINK_REGISTER + * ret + * + * |AARCH64_VALID_CALL_TARGET| expands to BTI 'c'. Either it, or + * |AARCH64_SIGN_LINK_REGISTER|, must be used at every point that may be an + * indirect call target. In particular, all symbols exported from a file must + * begin with one of these macros. For example, a leaf function that does not + * save LR can instead use |AARCH64_VALID_CALL_TARGET|: + * + * .globl return_zero + * return_zero: + * AARCH64_VALID_CALL_TARGET + * mov x0, #0 + * ret + * + * A non-leaf function which does not immediately save LR may need both macros + * because |AARCH64_SIGN_LINK_REGISTER| appears late. For example, the function + * may jump to an alternate implementation before setting up the stack: + * + * .globl with_early_jump + * with_early_jump: + * AARCH64_VALID_CALL_TARGET + * cmp x0, #128 + * b.lt .Lwith_early_jump_128 + * AARCH64_SIGN_LINK_REGISTER + * stp x29, x30, [sp, #-96]! + * mov x29, sp + * ... + * ldp x29, x30, [sp], #96 + * AARCH64_VALIDATE_LINK_REGISTER + * ret + * + * .Lwith_early_jump_128: + * ... + * ret + * + * These annotations are only required with indirect calls. Private symbols that + * are only the target of direct calls do not require annotations. Also note + * that |AARCH64_VALID_CALL_TARGET| is only valid for indirect calls (BLR), not + * indirect jumps (BR). Indirect jumps in assembly are supported through + * |AARCH64_VALID_JUMP_TARGET|. Landing Pads which shall serve for jumps and + * calls can be created using |AARCH64_VALID_JUMP_CALL_TARGET|. + * + * Although not necessary, it is safe to use these macros in 32-bit ARM + * assembly. This may be used to simplify dual 32-bit and 64-bit files. + * + * References: + * - "ELF for the Arm® 64-bit Architecture" + * https: *github.com/ARM-software/abi-aa/blob/master/aaelf64/aaelf64.rst + * - "Providing protection for complex software" + * https://developer.arm.com/architectures/learn-the-architecture/providing-protection-for-complex-software + */ +#if defined(__ARM_FEATURE_BTI_DEFAULT) && (__ARM_FEATURE_BTI_DEFAULT == 1) +#define GNU_PROPERTY_AARCH64_BTI (1 << 0) // Has Branch Target Identification +#define AARCH64_VALID_JUMP_CALL_TARGET hint #38 // BTI 'jc' +#define AARCH64_VALID_CALL_TARGET hint #34 // BTI 'c' +#define AARCH64_VALID_JUMP_TARGET hint #36 // BTI 'j' +#else +#define GNU_PROPERTY_AARCH64_BTI 0 // No Branch Target Identification +#define AARCH64_VALID_JUMP_CALL_TARGET +#define AARCH64_VALID_CALL_TARGET +#define AARCH64_VALID_JUMP_TARGET +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + +#if ((__ARM_FEATURE_PAC_DEFAULT & (1 << 0)) != 0) // authentication using key A +#define AARCH64_SIGN_LINK_REGISTER paciasp +#define AARCH64_VALIDATE_LINK_REGISTER autiasp +#elif ((__ARM_FEATURE_PAC_DEFAULT & (1 << 1)) != 0) // authentication using key B +#define AARCH64_SIGN_LINK_REGISTER pacibsp +#define AARCH64_VALIDATE_LINK_REGISTER autibsp +#else +#error Pointer authentication defines no valid key! +#endif +#if ((__ARM_FEATURE_PAC_DEFAULT & (1 << 2)) != 0) // authentication of leaf functions +#error Authentication of leaf functions is enabled but not supported in dav1d! +#endif +#define GNU_PROPERTY_AARCH64_PAC (1 << 1) + +#elif defined(__APPLE__) && defined(__arm64e__) + +#define GNU_PROPERTY_AARCH64_PAC 0 +#define AARCH64_SIGN_LINK_REGISTER pacibsp +#define AARCH64_VALIDATE_LINK_REGISTER autibsp + +#else /* __ARM_FEATURE_PAC_DEFAULT */ + +#define GNU_PROPERTY_AARCH64_PAC 0 +#define AARCH64_SIGN_LINK_REGISTER +#define AARCH64_VALIDATE_LINK_REGISTER + +#endif /* !__ARM_FEATURE_PAC_DEFAULT */ + + +#if (GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_PAC != 0) && defined(__ELF__) + .pushsection .note.gnu.property, "a" + .balign 8 + .long 4 + .long 0x10 + .long 0x5 + .asciz "GNU" + .long 0xc0000000 /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4 + .long (GNU_PROPERTY_AARCH64_BTI | GNU_PROPERTY_AARCH64_PAC) + .long 0 + .popsection +#endif /* (GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_PAC != 0) && defined(__ELF__) */ +#endif /* ARCH_AARCH64 */ + +#if ARCH_ARM + .syntax unified +#ifdef __ELF__ + .arch armv7-a + .fpu neon + .eabi_attribute 10, 0 // suppress Tag_FP_arch + .eabi_attribute 12, 0 // suppress Tag_Advanced_SIMD_arch + .section .note.GNU-stack,"",%progbits // Mark stack as non-executable +#endif /* __ELF__ */ + +#ifdef _WIN32 +#define CONFIG_THUMB 1 +#else +#define CONFIG_THUMB 0 +#endif + +#if CONFIG_THUMB + .thumb +#define A @ +#define T +#else +#define A +#define T @ +#endif /* CONFIG_THUMB */ +#endif /* ARCH_ARM */ + +#if !defined(PIC) +#if defined(__PIC__) +#define PIC __PIC__ +#elif defined(__pic__) +#define PIC __pic__ +#endif +#endif + +#ifndef PRIVATE_PREFIX +#define PRIVATE_PREFIX dav1d_ +#endif + +#define PASTE(a,b) a ## b +#define CONCAT(a,b) PASTE(a,b) + +#ifdef PREFIX +#define EXTERN CONCAT(_,PRIVATE_PREFIX) +#else +#define EXTERN PRIVATE_PREFIX +#endif + +.macro function name, export=0, align=2 + .macro endfunc +#ifdef __ELF__ + .size \name, . - \name +#endif +#if HAVE_AS_FUNC + .endfunc +#endif + .purgem endfunc + .endm + .text + .align \align + .if \export + .global EXTERN\name +#ifdef __ELF__ + .type EXTERN\name, %function + .hidden EXTERN\name +#elif defined(__MACH__) + .private_extern EXTERN\name +#endif +#if HAVE_AS_FUNC + .func EXTERN\name +#endif +EXTERN\name: + .else +#ifdef __ELF__ + .type \name, %function +#endif +#if HAVE_AS_FUNC + .func \name +#endif + .endif +\name: +#if ARCH_AARCH64 + .if \export + AARCH64_VALID_CALL_TARGET + .endif +#endif +.endm + +.macro const name, export=0, align=2 + .macro endconst +#ifdef __ELF__ + .size \name, . - \name +#endif + .purgem endconst + .endm +#if defined(_WIN32) + .section .rdata +#elif !defined(__MACH__) + .section .rodata +#else + .const_data +#endif + .align \align + .if \export + .global EXTERN\name +#ifdef __ELF__ + .hidden EXTERN\name +#elif defined(__MACH__) + .private_extern EXTERN\name +#endif +EXTERN\name: + .endif +\name: +.endm + +#ifdef __APPLE__ +#define L(x) L ## x +#else +#define L(x) .L ## x +#endif + +#define X(x) CONCAT(EXTERN, x) + + +#endif /* DAV1D_SRC_ARM_ASM_S */ diff --git a/third_party/dav1d/src/arm/cdef.h b/third_party/dav1d/src/arm/cdef.h new file mode 100644 index 0000000000..2e8c8ab6fb --- /dev/null +++ b/third_party/dav1d/src/arm/cdef.h @@ -0,0 +1,88 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/cdef.h" + +decl_cdef_dir_fn(BF(dav1d_cdef_find_dir, neon)); + +void BF(dav1d_cdef_padding4, neon)(uint16_t *tmp, const pixel *src, + ptrdiff_t src_stride, const pixel (*left)[2], + const pixel *const top, + const pixel *const bottom, int h, + enum CdefEdgeFlags edges); +void BF(dav1d_cdef_padding8, neon)(uint16_t *tmp, const pixel *src, + ptrdiff_t src_stride, const pixel (*left)[2], + const pixel *const top, + const pixel *const bottom, int h, + enum CdefEdgeFlags edges); + +// Passing edges to this function, to allow it to switch to a more +// optimized version for fully edged cases. Using size_t for edges, +// to avoid ABI differences for passing more than one argument on the stack. +void BF(dav1d_cdef_filter4, neon)(pixel *dst, ptrdiff_t dst_stride, + const uint16_t *tmp, int pri_strength, + int sec_strength, int dir, int damping, int h, + size_t edges HIGHBD_DECL_SUFFIX); +void BF(dav1d_cdef_filter8, neon)(pixel *dst, ptrdiff_t dst_stride, + const uint16_t *tmp, int pri_strength, + int sec_strength, int dir, int damping, int h, + size_t edges HIGHBD_DECL_SUFFIX); + +#define DEFINE_FILTER(w, h, tmp_stride) \ +static void \ +cdef_filter_##w##x##h##_neon(pixel *dst, const ptrdiff_t stride, \ + const pixel (*left)[2], \ + const pixel *const top, \ + const pixel *const bottom, \ + const int pri_strength, const int sec_strength, \ + const int dir, const int damping, \ + const enum CdefEdgeFlags edges \ + HIGHBD_DECL_SUFFIX) \ +{ \ + ALIGN_STK_16(uint16_t, tmp_buf, 12 * tmp_stride + 8,); \ + uint16_t *tmp = tmp_buf + 2 * tmp_stride + 8; \ + BF(dav1d_cdef_padding##w, neon)(tmp, dst, stride, \ + left, top, bottom, h, edges); \ + BF(dav1d_cdef_filter##w, neon)(dst, stride, tmp, pri_strength, \ + sec_strength, dir, damping, h, edges \ + HIGHBD_TAIL_SUFFIX); \ +} + +DEFINE_FILTER(8, 8, 16) +DEFINE_FILTER(4, 8, 8) +DEFINE_FILTER(4, 4, 8) + +static ALWAYS_INLINE void cdef_dsp_init_arm(Dav1dCdefDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->dir = BF(dav1d_cdef_find_dir, neon); + c->fb[0] = cdef_filter_8x8_neon; + c->fb[1] = cdef_filter_4x8_neon; + c->fb[2] = cdef_filter_4x4_neon; +} diff --git a/third_party/dav1d/src/arm/cpu.c b/third_party/dav1d/src/arm/cpu.c new file mode 100644 index 0000000000..b7a0d3adbc --- /dev/null +++ b/third_party/dav1d/src/arm/cpu.c @@ -0,0 +1,99 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "common/attributes.h" + +#include "src/arm/cpu.h" + +#if defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32) || ARCH_AARCH64 +// NEON is always available; runtime tests are not needed. +#elif defined(HAVE_GETAUXVAL) && ARCH_ARM +#include + +#ifndef HWCAP_ARM_NEON +#define HWCAP_ARM_NEON (1 << 12) +#endif +#define NEON_HWCAP HWCAP_ARM_NEON + +#elif defined(HAVE_ELF_AUX_INFO) && ARCH_ARM +#include + +#define NEON_HWCAP HWCAP_NEON + +#elif defined(__ANDROID__) +#include +#include + +static unsigned parse_proc_cpuinfo(const char *flag) { + FILE *file = fopen("/proc/cpuinfo", "r"); + if (!file) + return 0; + + char line_buffer[120]; + const char *line; + + while ((line = fgets(line_buffer, sizeof(line_buffer), file))) { + if (strstr(line, flag)) { + fclose(file); + return 1; + } + // if line is incomplete seek back to avoid splitting the search + // string into two buffers + if (!strchr(line, '\n') && strlen(line) > strlen(flag)) { + // use fseek since the 64 bit fseeko is only available since + // Android API level 24 and meson defines _FILE_OFFSET_BITS + // by default 64 + if (fseek(file, -strlen(flag), SEEK_CUR)) + break; + } + } + + fclose(file); + + return 0; +} +#endif + +COLD unsigned dav1d_get_cpu_flags_arm(void) { + unsigned flags = 0; +#if defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32) || ARCH_AARCH64 + flags |= DAV1D_ARM_CPU_FLAG_NEON; +#elif defined(HAVE_GETAUXVAL) && ARCH_ARM + unsigned long hw_cap = getauxval(AT_HWCAP); + flags |= (hw_cap & NEON_HWCAP) ? DAV1D_ARM_CPU_FLAG_NEON : 0; +#elif defined(HAVE_ELF_AUX_INFO) && ARCH_ARM + unsigned long hw_cap = 0; + elf_aux_info(AT_HWCAP, &hw_cap, sizeof(hw_cap)); + flags |= (hw_cap & NEON_HWCAP) ? DAV1D_ARM_CPU_FLAG_NEON : 0; +#elif defined(__ANDROID__) + flags |= parse_proc_cpuinfo("neon") ? DAV1D_ARM_CPU_FLAG_NEON : 0; +#endif + + return flags; +} diff --git a/third_party/dav1d/src/arm/cpu.h b/third_party/dav1d/src/arm/cpu.h new file mode 100644 index 0000000000..8c10a1b6b0 --- /dev/null +++ b/third_party/dav1d/src/arm/cpu.h @@ -0,0 +1,37 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_ARM_CPU_H +#define DAV1D_SRC_ARM_CPU_H + +enum CpuFlags { + DAV1D_ARM_CPU_FLAG_NEON = 1 << 0, +}; + +unsigned dav1d_get_cpu_flags_arm(void); + +#endif /* DAV1D_SRC_ARM_CPU_H */ diff --git a/third_party/dav1d/src/arm/filmgrain.h b/third_party/dav1d/src/arm/filmgrain.h new file mode 100644 index 0000000000..48776ac852 --- /dev/null +++ b/third_party/dav1d/src/arm/filmgrain.h @@ -0,0 +1,204 @@ +/* + * Copyright © 2018, Niklas Haas + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/filmgrain.h" +#include "asm-offsets.h" + +CHECK_OFFSET(Dav1dFilmGrainData, seed, FGD_SEED); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_lag, FGD_AR_COEFF_LAG); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeffs_y, FGD_AR_COEFFS_Y); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeffs_uv, FGD_AR_COEFFS_UV); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_shift, FGD_AR_COEFF_SHIFT); +CHECK_OFFSET(Dav1dFilmGrainData, grain_scale_shift, FGD_GRAIN_SCALE_SHIFT); + +CHECK_OFFSET(Dav1dFilmGrainData, scaling_shift, FGD_SCALING_SHIFT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_mult, FGD_UV_MULT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_luma_mult, FGD_UV_LUMA_MULT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_offset, FGD_UV_OFFSET); +CHECK_OFFSET(Dav1dFilmGrainData, clip_to_restricted_range, FGD_CLIP_TO_RESTRICTED_RANGE); + +void BF(dav1d_generate_grain_y, neon)(entry buf[][GRAIN_WIDTH], + const Dav1dFilmGrainData *const data + HIGHBD_DECL_SUFFIX); + +#define GEN_GRAIN_UV(suff) \ +void BF(dav1d_generate_grain_uv_ ## suff, neon)(entry buf[][GRAIN_WIDTH], \ + const entry buf_y[][GRAIN_WIDTH], \ + const Dav1dFilmGrainData *const data, \ + const intptr_t uv \ + HIGHBD_DECL_SUFFIX) + +GEN_GRAIN_UV(420); +GEN_GRAIN_UV(422); +GEN_GRAIN_UV(444); + +// Use ptrdiff_t instead of int for the last few parameters, to get the +// same layout of parameters on the stack across platforms. +void BF(dav1d_fgy_32x32, neon)(pixel *const dst, + const pixel *const src, + const ptrdiff_t stride, + const uint8_t scaling[SCALING_SIZE], + const int scaling_shift, + const entry grain_lut[][GRAIN_WIDTH], + const int offsets[][2], + const int h, const ptrdiff_t clip, + const ptrdiff_t type + HIGHBD_DECL_SUFFIX); + +static void fgy_32x32xn_neon(pixel *const dst_row, const pixel *const src_row, + const ptrdiff_t stride, + const Dav1dFilmGrainData *const data, const size_t pw, + const uint8_t scaling[SCALING_SIZE], + const entry grain_lut[][GRAIN_WIDTH], + const int bh, const int row_num HIGHBD_DECL_SUFFIX) +{ + const int rows = 1 + (data->overlap_flag && row_num > 0); + + // seed[0] contains the current row, seed[1] contains the previous + unsigned seed[2]; + for (int i = 0; i < rows; i++) { + seed[i] = data->seed; + seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; + seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); + } + + int offsets[2 /* col offset */][2 /* row offset */]; + + // process this row in BLOCK_SIZE^2 blocks + for (unsigned bx = 0; bx < pw; bx += BLOCK_SIZE) { + + if (data->overlap_flag && bx) { + // shift previous offsets left + for (int i = 0; i < rows; i++) + offsets[1][i] = offsets[0][i]; + } + + // update current offsets + for (int i = 0; i < rows; i++) + offsets[0][i] = get_random_number(8, &seed[i]); + + int type = 0; + if (data->overlap_flag && row_num) + type |= 1; /* overlap y */ + if (data->overlap_flag && bx) + type |= 2; /* overlap x */ + + BF(dav1d_fgy_32x32, neon)(dst_row + bx, src_row + bx, stride, + scaling, data->scaling_shift, + grain_lut, offsets, bh, + data->clip_to_restricted_range, type + HIGHBD_TAIL_SUFFIX); + } +} + +// Use ptrdiff_t instead of int for the last few parameters, to get the +// parameters on the stack with the same layout across platforms. +#define FGUV(nm, sx, sy) \ +void BF(dav1d_fguv_32x32_##nm, neon)(pixel *const dst, \ + const pixel *const src, \ + const ptrdiff_t stride, \ + const uint8_t scaling[SCALING_SIZE], \ + const Dav1dFilmGrainData *const data, \ + const entry grain_lut[][GRAIN_WIDTH], \ + const pixel *const luma_row, \ + const ptrdiff_t luma_stride, \ + const int offsets[][2], \ + const ptrdiff_t h, const ptrdiff_t uv, \ + const ptrdiff_t is_id, \ + const ptrdiff_t type \ + HIGHBD_DECL_SUFFIX); \ +static void \ +fguv_32x32xn_##nm##_neon(pixel *const dst_row, const pixel *const src_row, \ + const ptrdiff_t stride, const Dav1dFilmGrainData *const data, \ + const size_t pw, const uint8_t scaling[SCALING_SIZE], \ + const entry grain_lut[][GRAIN_WIDTH], const int bh, \ + const int row_num, const pixel *const luma_row, \ + const ptrdiff_t luma_stride, const int uv, const int is_id \ + HIGHBD_DECL_SUFFIX) \ +{ \ + const int rows = 1 + (data->overlap_flag && row_num > 0); \ + \ + /* seed[0] contains the current row, seed[1] contains the previous */ \ + unsigned seed[2]; \ + for (int i = 0; i < rows; i++) { \ + seed[i] = data->seed; \ + seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; \ + seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); \ + } \ + \ + int offsets[2 /* col offset */][2 /* row offset */]; \ + \ + /* process this row in BLOCK_SIZE^2 blocks (subsampled) */ \ + for (unsigned bx = 0; bx < pw; bx += BLOCK_SIZE >> sx) { \ + if (data->overlap_flag && bx) { \ + /* shift previous offsets left */ \ + for (int i = 0; i < rows; i++) \ + offsets[1][i] = offsets[0][i]; \ + } \ + \ + /* update current offsets */ \ + for (int i = 0; i < rows; i++) \ + offsets[0][i] = get_random_number(8, &seed[i]); \ + \ + int type = 0; \ + if (data->overlap_flag && row_num) \ + type |= 1; /* overlap y */ \ + if (data->overlap_flag && bx) \ + type |= 2; /* overlap x */ \ + if (data->chroma_scaling_from_luma) \ + type |= 4; \ + \ + BF(dav1d_fguv_32x32_##nm, neon)(dst_row + bx, src_row + bx, stride, \ + scaling, data, grain_lut, \ + luma_row + (bx << sx), luma_stride, \ + offsets, bh, uv, is_id, type \ + HIGHBD_TAIL_SUFFIX); \ + } \ +} + +FGUV(420, 1, 1); +FGUV(422, 1, 0); +FGUV(444, 0, 0); + +static ALWAYS_INLINE void film_grain_dsp_init_arm(Dav1dFilmGrainDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->generate_grain_y = BF(dav1d_generate_grain_y, neon); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, neon); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, neon); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, neon); + + c->fgy_32x32xn = fgy_32x32xn_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = fguv_32x32xn_420_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = fguv_32x32xn_422_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = fguv_32x32xn_444_neon; +} diff --git a/third_party/dav1d/src/arm/ipred.h b/third_party/dav1d/src/arm/ipred.h new file mode 100644 index 0000000000..e849d4998b --- /dev/null +++ b/third_party/dav1d/src/arm/ipred.h @@ -0,0 +1,222 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/ipred.h" + +decl_angular_ipred_fn(BF(dav1d_ipred_dc, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_128, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_top, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_left, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_paeth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_filter, neon)); + +decl_cfl_pred_fn(BF(dav1d_ipred_cfl, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_128, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_top, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_left, neon)); + +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_420, neon)); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_422, neon)); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_444, neon)); + +decl_pal_pred_fn(BF(dav1d_pal_pred, neon)); + +#if ARCH_AARCH64 +void BF(dav1d_ipred_z1_upsample_edge, neon)(pixel *out, const int hsz, + const pixel *const in, + const int end HIGHBD_DECL_SUFFIX); +void BF(dav1d_ipred_z1_filter_edge, neon)(pixel *out, const int sz, + const pixel *const in, + const int end, const int strength); +void BF(dav1d_ipred_pixel_set, neon)(pixel *out, const pixel px, + const int n); +void BF(dav1d_ipred_z1_fill1, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const top, const int width, + const int height, const int dx, + const int max_base_x); +void BF(dav1d_ipred_z1_fill2, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const top, const int width, + const int height, const int dx, + const int max_base_x); + +static void ipred_z1_neon(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + int dx = dav1d_dr_intra_derivative[angle >> 1]; + pixel top_out[64 + 64 + (64+15)*2 + 16]; + int max_base_x; + const int upsample_above = enable_intra_edge_filter ? + get_upsample(width + height, 90 - angle, is_sm) : 0; + if (upsample_above) { + BF(dav1d_ipred_z1_upsample_edge, neon)(top_out, width + height, + topleft_in, + width + imin(width, height) + HIGHBD_TAIL_SUFFIX); + max_base_x = 2 * (width + height) - 2; + dx <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, 90 - angle, is_sm) : 0; + if (filter_strength) { + BF(dav1d_ipred_z1_filter_edge, neon)(top_out, width + height, + topleft_in, + width + imin(width, height), + filter_strength); + max_base_x = width + height - 1; + } else { + max_base_x = width + imin(width, height) - 1; + memcpy(top_out, &topleft_in[1], (max_base_x + 1) * sizeof(pixel)); + } + } + const int base_inc = 1 + upsample_above; + int pad_pixels = width + 15; // max(dx >> 6) == 15 + BF(dav1d_ipred_pixel_set, neon)(&top_out[max_base_x + 1], + top_out[max_base_x], pad_pixels * base_inc); + if (upsample_above) + BF(dav1d_ipred_z1_fill2, neon)(dst, stride, top_out, width, height, + dx, max_base_x); + else + BF(dav1d_ipred_z1_fill1, neon)(dst, stride, top_out, width, height, + dx, max_base_x); +} + +void BF(dav1d_ipred_z3_fill1, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const left, const int width, + const int height, const int dy, + const int max_base_y); +void BF(dav1d_ipred_z3_fill2, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const left, const int width, + const int height, const int dy, + const int max_base_y); + +void BF(dav1d_ipred_reverse, neon)(pixel *dst, const pixel *const src, + const int n); + +static void ipred_z3_neon(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + assert(angle > 180); + int dy = dav1d_dr_intra_derivative[(270 - angle) >> 1]; + pixel flipped[64 + 64 + 16]; + pixel left_out[64 + 64 + (64+15)*2]; + int max_base_y; + const int upsample_left = enable_intra_edge_filter ? + get_upsample(width + height, angle - 180, is_sm) : 0; + if (upsample_left) { + flipped[0] = topleft_in[0]; + BF(dav1d_ipred_reverse, neon)(&flipped[1], &topleft_in[0], + height + imax(width, height)); + BF(dav1d_ipred_z1_upsample_edge, neon)(left_out, width + height, + flipped, + height + imin(width, height) + HIGHBD_TAIL_SUFFIX); + max_base_y = 2 * (width + height) - 2; + dy <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, angle - 180, is_sm) : 0; + + if (filter_strength) { + flipped[0] = topleft_in[0]; + BF(dav1d_ipred_reverse, neon)(&flipped[1], &topleft_in[0], + height + imax(width, height)); + BF(dav1d_ipred_z1_filter_edge, neon)(left_out, width + height, + flipped, + height + imin(width, height), + filter_strength); + max_base_y = width + height - 1; + } else { + BF(dav1d_ipred_reverse, neon)(left_out, &topleft_in[0], + height + imin(width, height)); + max_base_y = height + imin(width, height) - 1; + } + } + const int base_inc = 1 + upsample_left; + // The tbx based implementation needs left[] to have 64 bytes intitialized, + // the other implementation can read height + max(dy >> 6) past the end. + int pad_pixels = imax(64 - max_base_y - 1, height + 15); + + BF(dav1d_ipred_pixel_set, neon)(&left_out[max_base_y + 1], + left_out[max_base_y], pad_pixels * base_inc); + if (upsample_left) + BF(dav1d_ipred_z3_fill2, neon)(dst, stride, left_out, width, height, + dy, max_base_y); + else + BF(dav1d_ipred_z3_fill1, neon)(dst, stride, left_out, width, height, + dy, max_base_y); +} +#endif + +static ALWAYS_INLINE void intra_pred_dsp_init_arm(Dav1dIntraPredDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->intra_pred[DC_PRED] = BF(dav1d_ipred_dc, neon); + c->intra_pred[DC_128_PRED] = BF(dav1d_ipred_dc_128, neon); + c->intra_pred[TOP_DC_PRED] = BF(dav1d_ipred_dc_top, neon); + c->intra_pred[LEFT_DC_PRED] = BF(dav1d_ipred_dc_left, neon); + c->intra_pred[HOR_PRED] = BF(dav1d_ipred_h, neon); + c->intra_pred[VERT_PRED] = BF(dav1d_ipred_v, neon); + c->intra_pred[PAETH_PRED] = BF(dav1d_ipred_paeth, neon); + c->intra_pred[SMOOTH_PRED] = BF(dav1d_ipred_smooth, neon); + c->intra_pred[SMOOTH_V_PRED] = BF(dav1d_ipred_smooth_v, neon); + c->intra_pred[SMOOTH_H_PRED] = BF(dav1d_ipred_smooth_h, neon); +#if ARCH_AARCH64 + c->intra_pred[Z1_PRED] = ipred_z1_neon; + c->intra_pred[Z3_PRED] = ipred_z3_neon; +#endif + c->intra_pred[FILTER_PRED] = BF(dav1d_ipred_filter, neon); + + c->cfl_pred[DC_PRED] = BF(dav1d_ipred_cfl, neon); + c->cfl_pred[DC_128_PRED] = BF(dav1d_ipred_cfl_128, neon); + c->cfl_pred[TOP_DC_PRED] = BF(dav1d_ipred_cfl_top, neon); + c->cfl_pred[LEFT_DC_PRED] = BF(dav1d_ipred_cfl_left, neon); + + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_ipred_cfl_ac_420, neon); + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_ipred_cfl_ac_422, neon); + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_ipred_cfl_ac_444, neon); + + c->pal_pred = BF(dav1d_pal_pred, neon); +} diff --git a/third_party/dav1d/src/arm/itx.h b/third_party/dav1d/src/arm/itx.h new file mode 100644 index 0000000000..2ecd086b3b --- /dev/null +++ b/third_party/dav1d/src/arm/itx.h @@ -0,0 +1,141 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/itx.h" + +#define decl_itx2_fns(w, h, opt) \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) + +#define decl_itx12_fns(w, h, opt) \ +decl_itx2_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) + +#define decl_itx16_fns(w, h, opt) \ +decl_itx12_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) + +#define decl_itx17_fns(w, h, opt) \ +decl_itx16_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) + +decl_itx17_fns( 4, 4, neon); +decl_itx16_fns( 4, 8, neon); +decl_itx16_fns( 4, 16, neon); +decl_itx16_fns( 8, 4, neon); +decl_itx16_fns( 8, 8, neon); +decl_itx16_fns( 8, 16, neon); +decl_itx2_fns ( 8, 32, neon); +decl_itx16_fns(16, 4, neon); +decl_itx16_fns(16, 8, neon); +decl_itx12_fns(16, 16, neon); +decl_itx2_fns (16, 32, neon); +decl_itx2_fns (32, 8, neon); +decl_itx2_fns (32, 16, neon); +decl_itx2_fns (32, 32, neon); + +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_16x64, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_32x64, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x16, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, neon)); + +static ALWAYS_INLINE void itx_dsp_init_arm(Dav1dInvTxfmDSPContext *const c, int bpc) { +#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ + c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ + BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) + +#define assign_itx1_fn(pfx, w, h, ext) \ + assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) + +#define assign_itx2_fn(pfx, w, h, ext) \ + assign_itx1_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) + +#define assign_itx12_fn(pfx, w, h, ext) \ + assign_itx2_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ + assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) + +#define assign_itx16_fn(pfx, w, h, ext) \ + assign_itx12_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ + assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) + +#define assign_itx17_fn(pfx, w, h, ext) \ + assign_itx16_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) + + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + if (BITDEPTH == 16 && bpc != 10) return; + + assign_itx17_fn( , 4, 4, neon); + assign_itx16_fn(R, 4, 8, neon); + assign_itx16_fn(R, 4, 16, neon); + assign_itx16_fn(R, 8, 4, neon); + assign_itx16_fn( , 8, 8, neon); + assign_itx16_fn(R, 8, 16, neon); + assign_itx2_fn (R, 8, 32, neon); + assign_itx16_fn(R, 16, 4, neon); + assign_itx16_fn(R, 16, 8, neon); + assign_itx12_fn( , 16, 16, neon); + assign_itx2_fn (R, 16, 32, neon); + assign_itx1_fn (R, 16, 64, neon); + assign_itx2_fn (R, 32, 8, neon); + assign_itx2_fn (R, 32, 16, neon); + assign_itx2_fn ( , 32, 32, neon); + assign_itx1_fn (R, 32, 64, neon); + assign_itx1_fn (R, 64, 16, neon); + assign_itx1_fn (R, 64, 32, neon); + assign_itx1_fn ( , 64, 64, neon); +} diff --git a/third_party/dav1d/src/arm/loopfilter.h b/third_party/dav1d/src/arm/loopfilter.h new file mode 100644 index 0000000000..9ac08d94d2 --- /dev/null +++ b/third_party/dav1d/src/arm/loopfilter.h @@ -0,0 +1,45 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/loopfilter.h" + +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_y, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_y, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_uv, neon)); +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_uv, neon)); + +static ALWAYS_INLINE void loop_filter_dsp_init_arm(Dav1dLoopFilterDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, neon); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, neon); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, neon); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, neon); +} diff --git a/third_party/dav1d/src/arm/looprestoration.h b/third_party/dav1d/src/arm/looprestoration.h new file mode 100644 index 0000000000..7993dbff68 --- /dev/null +++ b/third_party/dav1d/src/arm/looprestoration.h @@ -0,0 +1,265 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/looprestoration.h" + +#if ARCH_AARCH64 +void BF(dav1d_wiener_filter7, neon)(pixel *p, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX); +void BF(dav1d_wiener_filter5, neon)(pixel *p, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX); +#else + +// The 8bpc version calculates things slightly differently than the reference +// C version. That version calculates roughly this: +// int16_t sum = 0; +// for (int i = 0; i < 7; i++) +// sum += src[idx] * fh[i]; +// int16_t sum2 = (src[x] << 7) - (1 << (bitdepth + 6)) + rounding_off_h; +// sum = iclip(sum + sum2, INT16_MIN, INT16_MAX) >> round_bits_h; +// sum += 1 << (bitdepth + 6 - round_bits_h); +// Compared to the reference C version, this is the output of the first pass +// _subtracted_ by 1 << (bitdepth + 6 - round_bits_h) = 2048, i.e. +// with round_offset precompensated. +// The 16bpc version calculates things pretty much the same way as the +// reference C version, but with the end result subtracted by +// 1 << (bitdepth + 6 - round_bits_h). +void BF(dav1d_wiener_filter_h, neon)(int16_t *dst, const pixel (*left)[4], + const pixel *src, ptrdiff_t stride, + const int16_t fh[8], intptr_t w, + int h, enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX); +// This calculates things slightly differently than the reference C version. +// This version calculates roughly this: +// int32_t sum = 0; +// for (int i = 0; i < 7; i++) +// sum += mid[idx] * fv[i]; +// sum = (sum + rounding_off_v) >> round_bits_v; +// This function assumes that the width is a multiple of 8. +void BF(dav1d_wiener_filter_v, neon)(pixel *dst, ptrdiff_t stride, + const int16_t *mid, int w, int h, + const int16_t fv[8], enum LrEdgeFlags edges, + ptrdiff_t mid_stride HIGHBD_DECL_SUFFIX); + +static void wiener_filter_neon(pixel *const dst, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + const int16_t (*const filter)[8] = params->filter; + ALIGN_STK_16(int16_t, mid, 68 * 384,); + int mid_stride = (w + 7) & ~7; + + // Horizontal filter + BF(dav1d_wiener_filter_h, neon)(&mid[2 * mid_stride], left, dst, stride, + filter[0], w, h, edges HIGHBD_TAIL_SUFFIX); + if (edges & LR_HAVE_TOP) + BF(dav1d_wiener_filter_h, neon)(mid, NULL, lpf, stride, + filter[0], w, 2, edges + HIGHBD_TAIL_SUFFIX); + if (edges & LR_HAVE_BOTTOM) + BF(dav1d_wiener_filter_h, neon)(&mid[(2 + h) * mid_stride], NULL, + lpf + 6 * PXSTRIDE(stride), + stride, filter[0], w, 2, edges + HIGHBD_TAIL_SUFFIX); + + // Vertical filter + BF(dav1d_wiener_filter_v, neon)(dst, stride, &mid[2*mid_stride], + w, h, filter[1], edges, + mid_stride * sizeof(*mid) + HIGHBD_TAIL_SUFFIX); +} +#endif + +void BF(dav1d_sgr_box3_h, neon)(int32_t *sumsq, int16_t *sum, + const pixel (*left)[4], + const pixel *src, const ptrdiff_t stride, + const int w, const int h, + const enum LrEdgeFlags edges); +void dav1d_sgr_box3_v_neon(int32_t *sumsq, int16_t *sum, + const int w, const int h, + const enum LrEdgeFlags edges); +void dav1d_sgr_calc_ab1_neon(int32_t *a, int16_t *b, + const int w, const int h, const int strength, + const int bitdepth_max); +void BF(dav1d_sgr_finish_filter1, neon)(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const int32_t *a, const int16_t *b, + const int w, const int h); + +/* filter with a 3x3 box (radius=1) */ +static void dav1d_sgr_filter1_neon(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, const int h, const int strength, + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); + int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; + ALIGN_STK_16(int16_t, sum_mem, (384 + 16) * 68 + 16,); + int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; + + BF(dav1d_sgr_box3_h, neon)(sumsq, sum, left, src, stride, w, h, edges); + if (edges & LR_HAVE_TOP) + BF(dav1d_sgr_box3_h, neon)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], + NULL, lpf, stride, w, 2, edges); + + if (edges & LR_HAVE_BOTTOM) + BF(dav1d_sgr_box3_h, neon)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], + NULL, lpf + 6 * PXSTRIDE(stride), + stride, w, 2, edges); + + dav1d_sgr_box3_v_neon(sumsq, sum, w, h, edges); + dav1d_sgr_calc_ab1_neon(a, b, w, h, strength, BITDEPTH_MAX); + BF(dav1d_sgr_finish_filter1, neon)(tmp, src, stride, a, b, w, h); +} + +void BF(dav1d_sgr_box5_h, neon)(int32_t *sumsq, int16_t *sum, + const pixel (*left)[4], + const pixel *src, const ptrdiff_t stride, + const int w, const int h, + const enum LrEdgeFlags edges); +void dav1d_sgr_box5_v_neon(int32_t *sumsq, int16_t *sum, + const int w, const int h, + const enum LrEdgeFlags edges); +void dav1d_sgr_calc_ab2_neon(int32_t *a, int16_t *b, + const int w, const int h, const int strength, + const int bitdepth_max); +void BF(dav1d_sgr_finish_filter2, neon)(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const int32_t *a, const int16_t *b, + const int w, const int h); + +/* filter with a 5x5 box (radius=2) */ +static void dav1d_sgr_filter2_neon(int16_t *tmp, + const pixel *src, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, const int h, const int strength, + const enum LrEdgeFlags edges + HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); + int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; + ALIGN_STK_16(int16_t, sum_mem, (384 + 16) * 68 + 16,); + int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; + + BF(dav1d_sgr_box5_h, neon)(sumsq, sum, left, src, stride, w, h, edges); + if (edges & LR_HAVE_TOP) + BF(dav1d_sgr_box5_h, neon)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], + NULL, lpf, stride, w, 2, edges); + + if (edges & LR_HAVE_BOTTOM) + BF(dav1d_sgr_box5_h, neon)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], + NULL, lpf + 6 * PXSTRIDE(stride), + stride, w, 2, edges); + + dav1d_sgr_box5_v_neon(sumsq, sum, w, h, edges); + dav1d_sgr_calc_ab2_neon(a, b, w, h, strength, BITDEPTH_MAX); + BF(dav1d_sgr_finish_filter2, neon)(tmp, src, stride, a, b, w, h); +} + +void BF(dav1d_sgr_weighted1, neon)(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *t1, const int w, const int h, + const int wt HIGHBD_DECL_SUFFIX); +void BF(dav1d_sgr_weighted2, neon)(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *t1, const int16_t *t2, + const int w, const int h, + const int16_t wt[2] HIGHBD_DECL_SUFFIX); + +static void sgr_filter_5x5_neon(pixel *const dst, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int16_t, tmp, 64 * 384,); + dav1d_sgr_filter2_neon(tmp, dst, stride, left, lpf, + w, h, params->sgr.s0, edges HIGHBD_TAIL_SUFFIX); + BF(dav1d_sgr_weighted1, neon)(dst, stride, dst, stride, + tmp, w, h, params->sgr.w0 HIGHBD_TAIL_SUFFIX); +} + +static void sgr_filter_3x3_neon(pixel *const dst, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int16_t, tmp, 64 * 384,); + dav1d_sgr_filter1_neon(tmp, dst, stride, left, lpf, + w, h, params->sgr.s1, edges HIGHBD_TAIL_SUFFIX); + BF(dav1d_sgr_weighted1, neon)(dst, stride, dst, stride, + tmp, w, h, params->sgr.w1 HIGHBD_TAIL_SUFFIX); +} + +static void sgr_filter_mix_neon(pixel *const dst, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int16_t, tmp1, 64 * 384,); + ALIGN_STK_16(int16_t, tmp2, 64 * 384,); + dav1d_sgr_filter2_neon(tmp1, dst, stride, left, lpf, + w, h, params->sgr.s0, edges HIGHBD_TAIL_SUFFIX); + dav1d_sgr_filter1_neon(tmp2, dst, stride, left, lpf, + w, h, params->sgr.s1, edges HIGHBD_TAIL_SUFFIX); + const int16_t wt[2] = { params->sgr.w0, params->sgr.w1 }; + BF(dav1d_sgr_weighted2, neon)(dst, stride, dst, stride, + tmp1, tmp2, w, h, wt HIGHBD_TAIL_SUFFIX); +} + +static ALWAYS_INLINE void loop_restoration_dsp_init_arm(Dav1dLoopRestorationDSPContext *const c, int bpc) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + +#if ARCH_AARCH64 + c->wiener[0] = BF(dav1d_wiener_filter7, neon); + c->wiener[1] = BF(dav1d_wiener_filter5, neon); +#else + c->wiener[0] = c->wiener[1] = wiener_filter_neon; +#endif + if (BITDEPTH == 8 || bpc == 10) { + c->sgr[0] = sgr_filter_5x5_neon; + c->sgr[1] = sgr_filter_3x3_neon; + c->sgr[2] = sgr_filter_mix_neon; + } +} diff --git a/third_party/dav1d/src/arm/mc.h b/third_party/dav1d/src/arm/mc.h new file mode 100644 index 0000000000..06cd533a9b --- /dev/null +++ b/third_party/dav1d/src/arm/mc.h @@ -0,0 +1,114 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "src/mc.h" +#include "src/cpu.h" + +decl_mc_fn(BF(dav1d_put_8tap_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_regular_smooth, neon)); +decl_mc_fn(BF(dav1d_put_8tap_regular_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_smooth_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp_regular, neon)); +decl_mc_fn(BF(dav1d_put_8tap_sharp_smooth, neon)); +decl_mc_fn(BF(dav1d_put_bilin, neon)); + +decl_mct_fn(BF(dav1d_prep_8tap_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_regular_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_regular_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_smooth_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp_regular, neon)); +decl_mct_fn(BF(dav1d_prep_8tap_sharp_smooth, neon)); +decl_mct_fn(BF(dav1d_prep_bilin, neon)); + +decl_avg_fn(BF(dav1d_avg, neon)); +decl_w_avg_fn(BF(dav1d_w_avg, neon)); +decl_mask_fn(BF(dav1d_mask, neon)); +decl_blend_fn(BF(dav1d_blend, neon)); +decl_blend_dir_fn(BF(dav1d_blend_h, neon)); +decl_blend_dir_fn(BF(dav1d_blend_v, neon)); + +decl_w_mask_fn(BF(dav1d_w_mask_444, neon)); +decl_w_mask_fn(BF(dav1d_w_mask_422, neon)); +decl_w_mask_fn(BF(dav1d_w_mask_420, neon)); + +decl_warp8x8_fn(BF(dav1d_warp_affine_8x8, neon)); +decl_warp8x8t_fn(BF(dav1d_warp_affine_8x8t, neon)); + +decl_emu_edge_fn(BF(dav1d_emu_edge, neon)); + +static ALWAYS_INLINE void mc_dsp_init_arm(Dav1dMCDSPContext *const c) { +#define init_mc_fn(type, name, suffix) \ + c->mc[type] = BF(dav1d_put_##name, suffix) +#define init_mct_fn(type, name, suffix) \ + c->mct[type] = BF(dav1d_prep_##name, suffix) + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, neon); + init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, neon); + init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, neon); + init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, neon); + init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, neon); + init_mc_fn (FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, neon); + init_mc_fn (FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, neon); + init_mc_fn (FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, neon); + init_mc_fn (FILTER_2D_8TAP_SHARP, 8tap_sharp, neon); + init_mc_fn (FILTER_2D_BILINEAR, bilin, neon); + + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, neon); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, neon); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, neon); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, neon); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, neon); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, neon); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, neon); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, neon); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, neon); + init_mct_fn(FILTER_2D_BILINEAR, bilin, neon); + + c->avg = BF(dav1d_avg, neon); + c->w_avg = BF(dav1d_w_avg, neon); + c->mask = BF(dav1d_mask, neon); + c->blend = BF(dav1d_blend, neon); + c->blend_h = BF(dav1d_blend_h, neon); + c->blend_v = BF(dav1d_blend_v, neon); + c->w_mask[0] = BF(dav1d_w_mask_444, neon); + c->w_mask[1] = BF(dav1d_w_mask_422, neon); + c->w_mask[2] = BF(dav1d_w_mask_420, neon); + c->warp8x8 = BF(dav1d_warp_affine_8x8, neon); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, neon); + c->emu_edge = BF(dav1d_emu_edge, neon); +} diff --git a/third_party/dav1d/src/arm/msac.h b/third_party/dav1d/src/arm/msac.h new file mode 100644 index 0000000000..9db0bf86ae --- /dev/null +++ b/third_party/dav1d/src/arm/msac.h @@ -0,0 +1,52 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_ARM_MSAC_H +#define DAV1D_SRC_ARM_MSAC_H + +unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_symbol_adapt8_neon(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_symbol_adapt16_neon(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_hi_tok_neon(MsacContext *s, uint16_t *cdf); +unsigned dav1d_msac_decode_bool_adapt_neon(MsacContext *s, uint16_t *cdf); +unsigned dav1d_msac_decode_bool_equi_neon(MsacContext *s); +unsigned dav1d_msac_decode_bool_neon(MsacContext *s, unsigned f); + +#if ARCH_AARCH64 || defined(__ARM_NEON) +#define dav1d_msac_decode_symbol_adapt4 dav1d_msac_decode_symbol_adapt4_neon +#define dav1d_msac_decode_symbol_adapt8 dav1d_msac_decode_symbol_adapt8_neon +#define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt16_neon +#define dav1d_msac_decode_hi_tok dav1d_msac_decode_hi_tok_neon +#define dav1d_msac_decode_bool_adapt dav1d_msac_decode_bool_adapt_neon +#define dav1d_msac_decode_bool_equi dav1d_msac_decode_bool_equi_neon +#define dav1d_msac_decode_bool dav1d_msac_decode_bool_neon +#endif + +#endif /* DAV1D_SRC_ARM_MSAC_H */ diff --git a/third_party/dav1d/src/arm/refmvs.h b/third_party/dav1d/src/arm/refmvs.h new file mode 100644 index 0000000000..4c96fc5095 --- /dev/null +++ b/third_party/dav1d/src/arm/refmvs.h @@ -0,0 +1,39 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/refmvs.h" + +decl_splat_mv_fn(dav1d_splat_mv_neon); + +static ALWAYS_INLINE void refmvs_dsp_init_arm(Dav1dRefmvsDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->splat_mv = dav1d_splat_mv_neon; +} diff --git a/third_party/dav1d/src/cdef.h b/third_party/dav1d/src/cdef.h new file mode 100644 index 0000000000..07c84d9ff5 --- /dev/null +++ b/third_party/dav1d/src/cdef.h @@ -0,0 +1,71 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_CDEF_H +#define DAV1D_SRC_CDEF_H + +#include +#include + +#include "common/bitdepth.h" + +enum CdefEdgeFlags { + CDEF_HAVE_LEFT = 1 << 0, + CDEF_HAVE_RIGHT = 1 << 1, + CDEF_HAVE_TOP = 1 << 2, + CDEF_HAVE_BOTTOM = 1 << 3, +}; + +#ifdef BITDEPTH +typedef const pixel (*const_left_pixel_row_2px)[2]; +#else +typedef const void *const_left_pixel_row_2px; +#endif + +// CDEF operates entirely on pre-filter data; if bottom/right edges are +// present (according to $edges), then the pre-filter data is located in +// $dst. However, the edge pixels above $dst may be post-filter, so in +// order to get access to pre-filter top pixels, use $top. +#define decl_cdef_fn(name) \ +void (name)(pixel *dst, ptrdiff_t stride, const_left_pixel_row_2px left, \ + const pixel *top, const pixel *bottom, \ + int pri_strength, int sec_strength, \ + int dir, int damping, enum CdefEdgeFlags edges HIGHBD_DECL_SUFFIX) +typedef decl_cdef_fn(*cdef_fn); + +#define decl_cdef_dir_fn(name) \ +int (name)(const pixel *dst, ptrdiff_t dst_stride, unsigned *var HIGHBD_DECL_SUFFIX) +typedef decl_cdef_dir_fn(*cdef_dir_fn); + +typedef struct Dav1dCdefDSPContext { + cdef_dir_fn dir; + cdef_fn fb[3 /* 444/luma, 422, 420 */]; +} Dav1dCdefDSPContext; + +bitfn_decls(void dav1d_cdef_dsp_init, Dav1dCdefDSPContext *c); + +#endif /* DAV1D_SRC_CDEF_H */ diff --git a/third_party/dav1d/src/cdef_apply.h b/third_party/dav1d/src/cdef_apply.h new file mode 100644 index 0000000000..a9748ee4f6 --- /dev/null +++ b/third_party/dav1d/src/cdef_apply.h @@ -0,0 +1,39 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_CDEF_APPLY_H +#define DAV1D_SRC_CDEF_APPLY_H + +#include "common/bitdepth.h" + +#include "src/internal.h" + +void bytefn(dav1d_cdef_brow)(Dav1dTaskContext *tc, pixel *const p[3], + const Av1Filter *lflvl, int by_start, int by_end, + int sbrow_start, int sby); + +#endif /* DAV1D_SRC_CDEF_APPLY_H */ diff --git a/third_party/dav1d/src/cdef_apply_tmpl.c b/third_party/dav1d/src/cdef_apply_tmpl.c new file mode 100644 index 0000000000..e2d8b83fc7 --- /dev/null +++ b/third_party/dav1d/src/cdef_apply_tmpl.c @@ -0,0 +1,309 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/cdef_apply.h" + +enum Backup2x8Flags { + BACKUP_2X8_Y = 1 << 0, + BACKUP_2X8_UV = 1 << 1, +}; + +static void backup2lines(pixel *const dst[3], /*const*/ pixel *const src[3], + const ptrdiff_t stride[2], + const enum Dav1dPixelLayout layout) +{ + const ptrdiff_t y_stride = PXSTRIDE(stride[0]); + if (y_stride < 0) + pixel_copy(dst[0] + y_stride, src[0] + 7 * y_stride, -2 * y_stride); + else + pixel_copy(dst[0], src[0] + 6 * y_stride, 2 * y_stride); + + if (layout != DAV1D_PIXEL_LAYOUT_I400) { + const ptrdiff_t uv_stride = PXSTRIDE(stride[1]); + if (uv_stride < 0) { + const int uv_off = layout == DAV1D_PIXEL_LAYOUT_I420 ? 3 : 7; + pixel_copy(dst[1] + uv_stride, src[1] + uv_off * uv_stride, -2 * uv_stride); + pixel_copy(dst[2] + uv_stride, src[2] + uv_off * uv_stride, -2 * uv_stride); + } else { + const int uv_off = layout == DAV1D_PIXEL_LAYOUT_I420 ? 2 : 6; + pixel_copy(dst[1], src[1] + uv_off * uv_stride, 2 * uv_stride); + pixel_copy(dst[2], src[2] + uv_off * uv_stride, 2 * uv_stride); + } + } +} + +static void backup2x8(pixel dst[3][8][2], + /*const*/ pixel *const src[3], + const ptrdiff_t src_stride[2], int x_off, + const enum Dav1dPixelLayout layout, + const enum Backup2x8Flags flag) +{ + ptrdiff_t y_off = 0; + if (flag & BACKUP_2X8_Y) { + for (int y = 0; y < 8; y++, y_off += PXSTRIDE(src_stride[0])) + pixel_copy(dst[0][y], &src[0][y_off + x_off - 2], 2); + } + + if (layout == DAV1D_PIXEL_LAYOUT_I400 || !(flag & BACKUP_2X8_UV)) + return; + + const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + + x_off >>= ss_hor; + y_off = 0; + for (int y = 0; y < (8 >> ss_ver); y++, y_off += PXSTRIDE(src_stride[1])) { + pixel_copy(dst[1][y], &src[1][y_off + x_off - 2], 2); + pixel_copy(dst[2][y], &src[2][y_off + x_off - 2], 2); + } +} + +static int adjust_strength(const int strength, const unsigned var) { + if (!var) return 0; + const int i = var >> 6 ? imin(ulog2(var >> 6), 12) : 0; + return (strength * (4 + i) + 8) >> 4; +} + +void bytefn(dav1d_cdef_brow)(Dav1dTaskContext *const tc, + pixel *const p[3], + const Av1Filter *const lflvl, + const int by_start, const int by_end, + const int sbrow_start, const int sby) +{ + Dav1dFrameContext *const f = (Dav1dFrameContext *)tc->f; + const int bitdepth_min_8 = BITDEPTH == 8 ? 0 : f->cur.p.bpc - 8; + const Dav1dDSPContext *const dsp = f->dsp; + enum CdefEdgeFlags edges = CDEF_HAVE_BOTTOM | (by_start > 0 ? CDEF_HAVE_TOP : 0); + pixel *ptrs[3] = { p[0], p[1], p[2] }; + const int sbsz = 16; + const int sb64w = f->sb128w << 1; + const int damping = f->frame_hdr->cdef.damping + bitdepth_min_8; + const enum Dav1dPixelLayout layout = f->cur.p.layout; + const int uv_idx = DAV1D_PIXEL_LAYOUT_I444 - layout; + const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + static const uint8_t uv_dirs[2][8] = { { 0, 1, 2, 3, 4, 5, 6, 7 }, + { 7, 0, 2, 4, 5, 6, 6, 6 } }; + const uint8_t *uv_dir = uv_dirs[layout == DAV1D_PIXEL_LAYOUT_I422]; + const int have_tt = f->c->n_tc > 1; + const int sb128 = f->seq_hdr->sb128; + const int resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; + const ptrdiff_t y_stride = PXSTRIDE(f->cur.stride[0]); + const ptrdiff_t uv_stride = PXSTRIDE(f->cur.stride[1]); + + for (int bit = 0, by = by_start; by < by_end; by += 2, edges |= CDEF_HAVE_TOP) { + const int tf = tc->top_pre_cdef_toggle; + const int by_idx = (by & 30) >> 1; + if (by + 2 >= f->bh) edges &= ~CDEF_HAVE_BOTTOM; + + if ((!have_tt || sbrow_start || by + 2 < by_end) && + edges & CDEF_HAVE_BOTTOM) + { + // backup pre-filter data for next iteration + pixel *const cdef_top_bak[3] = { + f->lf.cdef_line[!tf][0] + have_tt * sby * 4 * y_stride, + f->lf.cdef_line[!tf][1] + have_tt * sby * 8 * uv_stride, + f->lf.cdef_line[!tf][2] + have_tt * sby * 8 * uv_stride + }; + backup2lines(cdef_top_bak, ptrs, f->cur.stride, layout); + } + + ALIGN_STK_16(pixel, lr_bak, 2 /* idx */, [3 /* plane */][8 /* y */][2 /* x */]); + pixel *iptrs[3] = { ptrs[0], ptrs[1], ptrs[2] }; + edges &= ~CDEF_HAVE_LEFT; + edges |= CDEF_HAVE_RIGHT; + enum Backup2x8Flags prev_flag = 0; + for (int sbx = 0, last_skip = 1; sbx < sb64w; sbx++, edges |= CDEF_HAVE_LEFT) { + const int sb128x = sbx >> 1; + const int sb64_idx = ((by & sbsz) >> 3) + (sbx & 1); + const int cdef_idx = lflvl[sb128x].cdef_idx[sb64_idx]; + if (cdef_idx == -1 || + (!f->frame_hdr->cdef.y_strength[cdef_idx] && + !f->frame_hdr->cdef.uv_strength[cdef_idx])) + { + last_skip = 1; + goto next_sb; + } + + // Create a complete 32-bit mask for the sb row ahead of time. + const uint16_t (*noskip_row)[2] = &lflvl[sb128x].noskip_mask[by_idx]; + const unsigned noskip_mask = (unsigned) noskip_row[0][1] << 16 | + noskip_row[0][0]; + + const int y_lvl = f->frame_hdr->cdef.y_strength[cdef_idx]; + const int uv_lvl = f->frame_hdr->cdef.uv_strength[cdef_idx]; + const enum Backup2x8Flags flag = !!y_lvl + (!!uv_lvl << 1); + + const int y_pri_lvl = (y_lvl >> 2) << bitdepth_min_8; + int y_sec_lvl = y_lvl & 3; + y_sec_lvl += y_sec_lvl == 3; + y_sec_lvl <<= bitdepth_min_8; + + const int uv_pri_lvl = (uv_lvl >> 2) << bitdepth_min_8; + int uv_sec_lvl = uv_lvl & 3; + uv_sec_lvl += uv_sec_lvl == 3; + uv_sec_lvl <<= bitdepth_min_8; + + pixel *bptrs[3] = { iptrs[0], iptrs[1], iptrs[2] }; + for (int bx = sbx * sbsz; bx < imin((sbx + 1) * sbsz, f->bw); + bx += 2, edges |= CDEF_HAVE_LEFT) + { + if (bx + 2 >= f->bw) edges &= ~CDEF_HAVE_RIGHT; + + // check if this 8x8 block had any coded coefficients; if not, + // go to the next block + const uint32_t bx_mask = 3U << (bx & 30); + if (!(noskip_mask & bx_mask)) { + last_skip = 1; + goto next_b; + } + const int do_left = last_skip ? flag : (prev_flag ^ flag) & flag; + prev_flag = flag; + if (do_left && edges & CDEF_HAVE_LEFT) { + // we didn't backup the prefilter data because it wasn't + // there, so do it here instead + backup2x8(lr_bak[bit], bptrs, f->cur.stride, 0, layout, do_left); + } + if (edges & CDEF_HAVE_RIGHT) { + // backup pre-filter data for next iteration + backup2x8(lr_bak[!bit], bptrs, f->cur.stride, 8, layout, flag); + } + + int dir; + unsigned variance; + if (y_pri_lvl || uv_pri_lvl) + dir = dsp->cdef.dir(bptrs[0], f->cur.stride[0], + &variance HIGHBD_CALL_SUFFIX); + + const pixel *top, *bot; + ptrdiff_t offset; + + if (!have_tt) goto st_y; + if (sbrow_start && by == by_start) { + if (resize) { + offset = (sby - 1) * 4 * y_stride + bx * 4; + top = &f->lf.cdef_lpf_line[0][offset]; + } else { + offset = (sby * (4 << sb128) - 4) * y_stride + bx * 4; + top = &f->lf.lr_lpf_line[0][offset]; + } + bot = bptrs[0] + 8 * y_stride; + } else if (!sbrow_start && by + 2 >= by_end) { + top = &f->lf.cdef_line[tf][0][sby * 4 * y_stride + bx * 4]; + if (resize) { + offset = (sby * 4 + 2) * y_stride + bx * 4; + bot = &f->lf.cdef_lpf_line[0][offset]; + } else { + const int line = sby * (4 << sb128) + 4 * sb128 + 2; + offset = line * y_stride + bx * 4; + bot = &f->lf.lr_lpf_line[0][offset]; + } + } else { + st_y:; + offset = sby * 4 * y_stride; + top = &f->lf.cdef_line[tf][0][have_tt * offset + bx * 4]; + bot = bptrs[0] + 8 * y_stride; + } + if (y_pri_lvl) { + const int adj_y_pri_lvl = adjust_strength(y_pri_lvl, variance); + if (adj_y_pri_lvl || y_sec_lvl) + dsp->cdef.fb[0](bptrs[0], f->cur.stride[0], lr_bak[bit][0], + top, bot, adj_y_pri_lvl, y_sec_lvl, + dir, damping, edges HIGHBD_CALL_SUFFIX); + } else if (y_sec_lvl) + dsp->cdef.fb[0](bptrs[0], f->cur.stride[0], lr_bak[bit][0], + top, bot, 0, y_sec_lvl, 0, damping, + edges HIGHBD_CALL_SUFFIX); + + if (!uv_lvl) goto skip_uv; + assert(layout != DAV1D_PIXEL_LAYOUT_I400); + + const int uvdir = uv_pri_lvl ? uv_dir[dir] : 0; + for (int pl = 1; pl <= 2; pl++) { + if (!have_tt) goto st_uv; + if (sbrow_start && by == by_start) { + if (resize) { + offset = (sby - 1) * 4 * uv_stride + (bx * 4 >> ss_hor); + top = &f->lf.cdef_lpf_line[pl][offset]; + } else { + const int line = sby * (4 << sb128) - 4; + offset = line * uv_stride + (bx * 4 >> ss_hor); + top = &f->lf.lr_lpf_line[pl][offset]; + } + bot = bptrs[pl] + (8 >> ss_ver) * uv_stride; + } else if (!sbrow_start && by + 2 >= by_end) { + const ptrdiff_t top_offset = sby * 8 * uv_stride + + (bx * 4 >> ss_hor); + top = &f->lf.cdef_line[tf][pl][top_offset]; + if (resize) { + offset = (sby * 4 + 2) * uv_stride + (bx * 4 >> ss_hor); + bot = &f->lf.cdef_lpf_line[pl][offset]; + } else { + const int line = sby * (4 << sb128) + 4 * sb128 + 2; + offset = line * uv_stride + (bx * 4 >> ss_hor); + bot = &f->lf.lr_lpf_line[pl][offset]; + } + } else { + st_uv:; + const ptrdiff_t offset = sby * 8 * uv_stride; + top = &f->lf.cdef_line[tf][pl][have_tt * offset + (bx * 4 >> ss_hor)]; + bot = bptrs[pl] + (8 >> ss_ver) * uv_stride; + } + dsp->cdef.fb[uv_idx](bptrs[pl], f->cur.stride[1], + lr_bak[bit][pl], top, bot, + uv_pri_lvl, uv_sec_lvl, uvdir, + damping - 1, edges HIGHBD_CALL_SUFFIX); + } + + skip_uv: + bit ^= 1; + last_skip = 0; + + next_b: + bptrs[0] += 8; + bptrs[1] += 8 >> ss_hor; + bptrs[2] += 8 >> ss_hor; + } + + next_sb: + iptrs[0] += sbsz * 4; + iptrs[1] += sbsz * 4 >> ss_hor; + iptrs[2] += sbsz * 4 >> ss_hor; + } + + ptrs[0] += 8 * PXSTRIDE(f->cur.stride[0]); + ptrs[1] += 8 * PXSTRIDE(f->cur.stride[1]) >> ss_ver; + ptrs[2] += 8 * PXSTRIDE(f->cur.stride[1]) >> ss_ver; + tc->top_pre_cdef_toggle ^= 1; + } +} diff --git a/third_party/dav1d/src/cdef_tmpl.c b/third_party/dav1d/src/cdef_tmpl.c new file mode 100644 index 0000000000..59439457a1 --- /dev/null +++ b/third_party/dav1d/src/cdef_tmpl.c @@ -0,0 +1,331 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/cdef.h" +#include "src/tables.h" + +static inline int constrain(const int diff, const int threshold, + const int shift) +{ + const int adiff = abs(diff); + return apply_sign(imin(adiff, imax(0, threshold - (adiff >> shift))), diff); +} + +static inline void fill(int16_t *tmp, const ptrdiff_t stride, + const int w, const int h) +{ + /* Use a value that's a large positive number when interpreted as unsigned, + * and a large negative number when interpreted as signed. */ + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) + tmp[x] = INT16_MIN; + tmp += stride; + } +} + +static void padding(int16_t *tmp, const ptrdiff_t tmp_stride, + const pixel *src, const ptrdiff_t src_stride, + const pixel (*left)[2], + const pixel *top, const pixel *bottom, + const int w, const int h, const enum CdefEdgeFlags edges) +{ + // fill extended input buffer + int x_start = -2, x_end = w + 2, y_start = -2, y_end = h + 2; + if (!(edges & CDEF_HAVE_TOP)) { + fill(tmp - 2 - 2 * tmp_stride, tmp_stride, w + 4, 2); + y_start = 0; + } + if (!(edges & CDEF_HAVE_BOTTOM)) { + fill(tmp + h * tmp_stride - 2, tmp_stride, w + 4, 2); + y_end -= 2; + } + if (!(edges & CDEF_HAVE_LEFT)) { + fill(tmp + y_start * tmp_stride - 2, tmp_stride, 2, y_end - y_start); + x_start = 0; + } + if (!(edges & CDEF_HAVE_RIGHT)) { + fill(tmp + y_start * tmp_stride + w, tmp_stride, 2, y_end - y_start); + x_end -= 2; + } + + for (int y = y_start; y < 0; y++) { + for (int x = x_start; x < x_end; x++) + tmp[x + y * tmp_stride] = top[x]; + top += PXSTRIDE(src_stride); + } + for (int y = 0; y < h; y++) + for (int x = x_start; x < 0; x++) + tmp[x + y * tmp_stride] = left[y][2 + x]; + for (int y = 0; y < h; y++) { + for (int x = (y < h) ? 0 : x_start; x < x_end; x++) + tmp[x] = src[x]; + src += PXSTRIDE(src_stride); + tmp += tmp_stride; + } + for (int y = h; y < y_end; y++) { + for (int x = x_start; x < x_end; x++) + tmp[x] = bottom[x]; + bottom += PXSTRIDE(src_stride); + tmp += tmp_stride; + } + +} + +static NOINLINE void +cdef_filter_block_c(pixel *dst, const ptrdiff_t dst_stride, + const pixel (*left)[2], + const pixel *const top, const pixel *const bottom, + const int pri_strength, const int sec_strength, + const int dir, const int damping, const int w, int h, + const enum CdefEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + const ptrdiff_t tmp_stride = 12; + assert((w == 4 || w == 8) && (h == 4 || h == 8)); + int16_t tmp_buf[144]; // 12*12 is the maximum value of tmp_stride * (h + 4) + int16_t *tmp = tmp_buf + 2 * tmp_stride + 2; + + padding(tmp, tmp_stride, dst, dst_stride, left, top, bottom, w, h, edges); + + if (pri_strength) { + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const int pri_tap = 4 - ((pri_strength >> bitdepth_min_8) & 1); + const int pri_shift = imax(0, damping - ulog2(pri_strength)); + if (sec_strength) { + const int sec_shift = damping - ulog2(sec_strength); + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + int max = px, min = px; + int pri_tap_k = pri_tap; + for (int k = 0; k < 2; k++) { + const int off1 = dav1d_cdef_directions[dir + 2][k]; // dir + const int p0 = tmp[x + off1]; + const int p1 = tmp[x - off1]; + sum += pri_tap_k * constrain(p0 - px, pri_strength, pri_shift); + sum += pri_tap_k * constrain(p1 - px, pri_strength, pri_shift); + // if pri_tap_k == 4 then it becomes 2 else it remains 3 + pri_tap_k = (pri_tap_k & 3) | 2; + min = umin(p0, min); + max = imax(p0, max); + min = umin(p1, min); + max = imax(p1, max); + const int off2 = dav1d_cdef_directions[dir + 4][k]; // dir + 2 + const int off3 = dav1d_cdef_directions[dir + 0][k]; // dir - 2 + const int s0 = tmp[x + off2]; + const int s1 = tmp[x - off2]; + const int s2 = tmp[x + off3]; + const int s3 = tmp[x - off3]; + // sec_tap starts at 2 and becomes 1 + const int sec_tap = 2 - k; + sum += sec_tap * constrain(s0 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s1 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s2 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s3 - px, sec_strength, sec_shift); + min = umin(s0, min); + max = imax(s0, max); + min = umin(s1, min); + max = imax(s1, max); + min = umin(s2, min); + max = imax(s2, max); + min = umin(s3, min); + max = imax(s3, max); + } + dst[x] = iclip(px + ((sum - (sum < 0) + 8) >> 4), min, max); + } + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); + } else { // pri_strength only + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + int pri_tap_k = pri_tap; + for (int k = 0; k < 2; k++) { + const int off = dav1d_cdef_directions[dir + 2][k]; // dir + const int p0 = tmp[x + off]; + const int p1 = tmp[x - off]; + sum += pri_tap_k * constrain(p0 - px, pri_strength, pri_shift); + sum += pri_tap_k * constrain(p1 - px, pri_strength, pri_shift); + pri_tap_k = (pri_tap_k & 3) | 2; + } + dst[x] = px + ((sum - (sum < 0) + 8) >> 4); + } + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); + } + } else { // sec_strength only + assert(sec_strength); + const int sec_shift = damping - ulog2(sec_strength); + do { + for (int x = 0; x < w; x++) { + const int px = dst[x]; + int sum = 0; + for (int k = 0; k < 2; k++) { + const int off1 = dav1d_cdef_directions[dir + 4][k]; // dir + 2 + const int off2 = dav1d_cdef_directions[dir + 0][k]; // dir - 2 + const int s0 = tmp[x + off1]; + const int s1 = tmp[x - off1]; + const int s2 = tmp[x + off2]; + const int s3 = tmp[x - off2]; + const int sec_tap = 2 - k; + sum += sec_tap * constrain(s0 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s1 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s2 - px, sec_strength, sec_shift); + sum += sec_tap * constrain(s3 - px, sec_strength, sec_shift); + } + dst[x] = px + ((sum - (sum < 0) + 8) >> 4); + } + dst += PXSTRIDE(dst_stride); + tmp += tmp_stride; + } while (--h); + } +} + +#define cdef_fn(w, h) \ +static void cdef_filter_block_##w##x##h##_c(pixel *const dst, \ + const ptrdiff_t stride, \ + const pixel (*left)[2], \ + const pixel *const top, \ + const pixel *const bottom, \ + const int pri_strength, \ + const int sec_strength, \ + const int dir, \ + const int damping, \ + const enum CdefEdgeFlags edges \ + HIGHBD_DECL_SUFFIX) \ +{ \ + cdef_filter_block_c(dst, stride, left, top, bottom, \ + pri_strength, sec_strength, dir, damping, w, h, edges HIGHBD_TAIL_SUFFIX); \ +} + +cdef_fn(4, 4); +cdef_fn(4, 8); +cdef_fn(8, 8); + +static int cdef_find_dir_c(const pixel *img, const ptrdiff_t stride, + unsigned *const var HIGHBD_DECL_SUFFIX) +{ + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + int partial_sum_hv[2][8] = { { 0 } }; + int partial_sum_diag[2][15] = { { 0 } }; + int partial_sum_alt[4][11] = { { 0 } }; + + for (int y = 0; y < 8; y++) { + for (int x = 0; x < 8; x++) { + const int px = (img[x] >> bitdepth_min_8) - 128; + + partial_sum_diag[0][ y + x ] += px; + partial_sum_alt [0][ y + (x >> 1)] += px; + partial_sum_hv [0][ y ] += px; + partial_sum_alt [1][3 + y - (x >> 1)] += px; + partial_sum_diag[1][7 + y - x ] += px; + partial_sum_alt [2][3 - (y >> 1) + x ] += px; + partial_sum_hv [1][ x ] += px; + partial_sum_alt [3][ (y >> 1) + x ] += px; + } + img += PXSTRIDE(stride); + } + + unsigned cost[8] = { 0 }; + for (int n = 0; n < 8; n++) { + cost[2] += partial_sum_hv[0][n] * partial_sum_hv[0][n]; + cost[6] += partial_sum_hv[1][n] * partial_sum_hv[1][n]; + } + cost[2] *= 105; + cost[6] *= 105; + + static const uint16_t div_table[7] = { 840, 420, 280, 210, 168, 140, 120 }; + for (int n = 0; n < 7; n++) { + const int d = div_table[n]; + cost[0] += (partial_sum_diag[0][n] * partial_sum_diag[0][n] + + partial_sum_diag[0][14 - n] * partial_sum_diag[0][14 - n]) * d; + cost[4] += (partial_sum_diag[1][n] * partial_sum_diag[1][n] + + partial_sum_diag[1][14 - n] * partial_sum_diag[1][14 - n]) * d; + } + cost[0] += partial_sum_diag[0][7] * partial_sum_diag[0][7] * 105; + cost[4] += partial_sum_diag[1][7] * partial_sum_diag[1][7] * 105; + + for (int n = 0; n < 4; n++) { + unsigned *const cost_ptr = &cost[n * 2 + 1]; + for (int m = 0; m < 5; m++) + *cost_ptr += partial_sum_alt[n][3 + m] * partial_sum_alt[n][3 + m]; + *cost_ptr *= 105; + for (int m = 0; m < 3; m++) { + const int d = div_table[2 * m + 1]; + *cost_ptr += (partial_sum_alt[n][m] * partial_sum_alt[n][m] + + partial_sum_alt[n][10 - m] * partial_sum_alt[n][10 - m]) * d; + } + } + + int best_dir = 0; + unsigned best_cost = cost[0]; + for (int n = 1; n < 8; n++) { + if (cost[n] > best_cost) { + best_cost = cost[n]; + best_dir = n; + } + } + + *var = (best_cost - (cost[best_dir ^ 4])) >> 10; + return best_dir; +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/cdef.h" +#elif ARCH_PPC64LE +#include "src/ppc/cdef.h" +#elif ARCH_X86 +#include "src/x86/cdef.h" +#endif +#endif + +COLD void bitfn(dav1d_cdef_dsp_init)(Dav1dCdefDSPContext *const c) { + c->dir = cdef_find_dir_c; + c->fb[0] = cdef_filter_block_8x8_c; + c->fb[1] = cdef_filter_block_4x8_c; + c->fb[2] = cdef_filter_block_4x4_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + cdef_dsp_init_arm(c); +#elif ARCH_PPC64LE + cdef_dsp_init_ppc(c); +#elif ARCH_X86 + cdef_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/cdf.c b/third_party/dav1d/src/cdf.c new file mode 100644 index 0000000000..e0f2132e00 --- /dev/null +++ b/third_party/dav1d/src/cdf.c @@ -0,0 +1,4123 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/frame.h" + +#include "src/internal.h" +#include "src/tables.h" + +#define CDF1(x) (32768-(x)) + +#define CDF2(a,b) \ + CDF1(a), CDF1(b) +#define CDF3(a,b,c) \ + CDF1(a), CDF2(b,c) +#define CDF4(a,b,c,d) \ + CDF1(a), CDF3(b,c,d) +#define CDF5(a,b,c,d,e) \ + CDF1(a), CDF4(b,c,d,e) +#define CDF6(a,b,c,d,e,f) \ + CDF1(a), CDF5(b,c,d,e,f) +#define CDF7(a,b,c,d,e,f,g) \ + CDF1(a), CDF6(b,c,d,e,f,g) +#define CDF8(a,b,c,d,e,f,g,h) \ + CDF1(a), CDF7(b,c,d,e,f,g,h) +#define CDF9(a,b,c,d,e,f,g,h,i) \ + CDF1(a), CDF8(b,c,d,e,f,g,h,i) +#define CDF10(a,b,c,d,e,f,g,h,i,j) \ + CDF1(a), CDF9(b,c,d,e,f,g,h,i,j) +#define CDF11(a,b,c,d,e,f,g,h,i,j,k) \ + CDF1(a), CDF10(b,c,d,e,f,g,h,i,j,k) +#define CDF12(a,b,c,d,e,f,g,h,i,j,k,l) \ + CDF1(a), CDF11(b,c,d,e,f,g,h,i,j,k,l) +#define CDF13(a,b,c,d,e,f,g,h,i,j,k,l,m) \ + CDF1(a), CDF12(b,c,d,e,f,g,h,i,j,k,l,m) +#define CDF14(a,b,c,d,e,f,g,h,i,j,k,l,m,n) \ + CDF1(a), CDF13(b,c,d,e,f,g,h,i,j,k,l,m,n) +#define CDF15(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o) \ + CDF1(a), CDF14(b,c,d,e,f,g,h,i,j,k,l,m,n,o) + +static const CdfModeContext av1_default_cdf = { + .y_mode = { + { CDF12(22801, 23489, 24293, 24756, 25601, 26123, + 26606, 27418, 27945, 29228, 29685, 30349) }, + { CDF12(18673, 19845, 22631, 23318, 23950, 24649, + 25527, 27364, 28152, 29701, 29984, 30852) }, + { CDF12(19770, 20979, 23396, 23939, 24241, 24654, + 25136, 27073, 27830, 29360, 29730, 30659) }, + { CDF12(20155, 21301, 22838, 23178, 23261, 23533, + 23703, 24804, 25352, 26575, 27016, 28049) }, + }, .use_filter_intra = { + [BS_4x4] = { CDF1( 4621) }, + [BS_4x8] = { CDF1( 6743) }, + [BS_8x4] = { CDF1( 5893) }, + [BS_8x8] = { CDF1( 7866) }, + [BS_8x16] = { CDF1(12551) }, + [BS_16x8] = { CDF1( 9394) }, + [BS_16x16] = { CDF1(12408) }, + [BS_16x32] = { CDF1(14301) }, + [BS_32x16] = { CDF1(12756) }, + [BS_32x32] = { CDF1(22343) }, + [BS_32x64] = { CDF1(16384) }, + [BS_64x32] = { CDF1(16384) }, + [BS_64x64] = { CDF1(16384) }, + [BS_64x128] = { CDF1(16384) }, + [BS_128x64] = { CDF1(16384) }, + [BS_128x128] = { CDF1(16384) }, + [BS_4x16] = { CDF1(12770) }, + [BS_16x4] = { CDF1(10368) }, + [BS_8x32] = { CDF1(20229) }, + [BS_32x8] = { CDF1(18101) }, + [BS_16x64] = { CDF1(16384) }, + [BS_64x16] = { CDF1(16384) }, + }, .filter_intra = { + CDF4(8949, 12776, 17211, 29558), + }, .uv_mode = { + { + { CDF12(22631, 24152, 25378, 25661, 25986, 26520, + 27055, 27923, 28244, 30059, 30941, 31961) }, + { CDF12( 9513, 26881, 26973, 27046, 27118, 27664, + 27739, 27824, 28359, 29505, 29800, 31796) }, + { CDF12( 9845, 9915, 28663, 28704, 28757, 28780, + 29198, 29822, 29854, 30764, 31777, 32029) }, + { CDF12(13639, 13897, 14171, 25331, 25606, 25727, + 25953, 27148, 28577, 30612, 31355, 32493) }, + { CDF12( 9764, 9835, 9930, 9954, 25386, 27053, + 27958, 28148, 28243, 31101, 31744, 32363) }, + { CDF12(11825, 13589, 13677, 13720, 15048, 29213, + 29301, 29458, 29711, 31161, 31441, 32550) }, + { CDF12(14175, 14399, 16608, 16821, 17718, 17775, + 28551, 30200, 30245, 31837, 32342, 32667) }, + { CDF12(12885, 13038, 14978, 15590, 15673, 15748, + 16176, 29128, 29267, 30643, 31961, 32461) }, + { CDF12(12026, 13661, 13874, 15305, 15490, 15726, + 15995, 16273, 28443, 30388, 30767, 32416) }, + { CDF12(19052, 19840, 20579, 20916, 21150, 21467, + 21885, 22719, 23174, 28861, 30379, 32175) }, + { CDF12(18627, 19649, 20974, 21219, 21492, 21816, + 22199, 23119, 23527, 27053, 31397, 32148) }, + { CDF12(17026, 19004, 19997, 20339, 20586, 21103, + 21349, 21907, 22482, 25896, 26541, 31819) }, + { CDF12(12124, 13759, 14959, 14992, 15007, 15051, + 15078, 15166, 15255, 15753, 16039, 16606) }, + }, { + { CDF13(10407, 11208, 12900, 13181, 13823, 14175, 14899, + 15656, 15986, 20086, 20995, 22455, 24212) }, + { CDF13( 4532, 19780, 20057, 20215, 20428, 21071, 21199, + 21451, 22099, 24228, 24693, 27032, 29472) }, + { CDF13( 5273, 5379, 20177, 20270, 20385, 20439, 20949, + 21695, 21774, 23138, 24256, 24703, 26679) }, + { CDF13( 6740, 7167, 7662, 14152, 14536, 14785, 15034, + 16741, 18371, 21520, 22206, 23389, 24182) }, + { CDF13( 4987, 5368, 5928, 6068, 19114, 20315, 21857, + 22253, 22411, 24911, 25380, 26027, 26376) }, + { CDF13( 5370, 6889, 7247, 7393, 9498, 21114, 21402, + 21753, 21981, 24780, 25386, 26517, 27176) }, + { CDF13( 4816, 4961, 7204, 7326, 8765, 8930, 20169, + 20682, 20803, 23188, 23763, 24455, 24940) }, + { CDF13( 6608, 6740, 8529, 9049, 9257, 9356, 9735, + 18827, 19059, 22336, 23204, 23964, 24793) }, + { CDF13( 5998, 7419, 7781, 8933, 9255, 9549, 9753, + 10417, 18898, 22494, 23139, 24764, 25989) }, + { CDF13(10660, 11298, 12550, 12957, 13322, 13624, 14040, + 15004, 15534, 20714, 21789, 23443, 24861) }, + { CDF13(10522, 11530, 12552, 12963, 13378, 13779, 14245, + 15235, 15902, 20102, 22696, 23774, 25838) }, + { CDF13(10099, 10691, 12639, 13049, 13386, 13665, 14125, + 15163, 15636, 19676, 20474, 23519, 25208) }, + { CDF13( 3144, 5087, 7382, 7504, 7593, 7690, 7801, + 8064, 8232, 9248, 9875, 10521, 29048) }, + }, + }, .angle_delta = { + { CDF6( 2180, 5032, 7567, 22776, 26989, 30217) }, + { CDF6( 2301, 5608, 8801, 23487, 26974, 30330) }, + { CDF6( 3780, 11018, 13699, 19354, 23083, 31286) }, + { CDF6( 4581, 11226, 15147, 17138, 21834, 28397) }, + { CDF6( 1737, 10927, 14509, 19588, 22745, 28823) }, + { CDF6( 2664, 10176, 12485, 17650, 21600, 30495) }, + { CDF6( 2240, 11096, 15453, 20341, 22561, 28917) }, + { CDF6( 3605, 10428, 12459, 17676, 21244, 30655) }, + }, .filter = { + { + { CDF2(31935, 32720) }, { CDF2( 5568, 32719) }, + { CDF2( 422, 2938) }, { CDF2(28244, 32608) }, + { CDF2(31206, 31953) }, { CDF2( 4862, 32121) }, + { CDF2( 770, 1152) }, { CDF2(20889, 25637) }, + }, { + { CDF2(31910, 32724) }, { CDF2( 4120, 32712) }, + { CDF2( 305, 2247) }, { CDF2(27403, 32636) }, + { CDF2(31022, 32009) }, { CDF2( 2963, 32093) }, + { CDF2( 601, 943) }, { CDF2(14969, 21398) }, + }, + }, .newmv_mode = { + { CDF1(24035) }, { CDF1(16630) }, { CDF1(15339) }, + { CDF1( 8386) }, { CDF1(12222) }, { CDF1( 4676) }, + }, .globalmv_mode = { + { CDF1( 2175) }, { CDF1( 1054) }, + }, .refmv_mode = { + { CDF1(23974) }, { CDF1(24188) }, { CDF1(17848) }, + { CDF1(28622) }, { CDF1(24312) }, { CDF1(19923) }, + }, .drl_bit = { + { CDF1(13104) }, { CDF1(24560) }, { CDF1(18945) }, + }, .comp_inter_mode = { + { CDF7( 7760, 13823, 15808, 17641, 19156, 20666, 26891) }, + { CDF7(10730, 19452, 21145, 22749, 24039, 25131, 28724) }, + { CDF7(10664, 20221, 21588, 22906, 24295, 25387, 28436) }, + { CDF7(13298, 16984, 20471, 24182, 25067, 25736, 26422) }, + { CDF7(18904, 23325, 25242, 27432, 27898, 28258, 30758) }, + { CDF7(10725, 17454, 20124, 22820, 24195, 25168, 26046) }, + { CDF7(17125, 24273, 25814, 27492, 28214, 28704, 30592) }, + { CDF7(13046, 23214, 24505, 25942, 27435, 28442, 29330) }, + }, .intra = { + { CDF1( 806) }, { CDF1(16662) }, { CDF1(20186) }, + { CDF1(26538) }, + }, .comp = { + { CDF1(26828) }, { CDF1(24035) }, { CDF1(12031) }, + { CDF1(10640) }, { CDF1( 2901) }, + }, .comp_dir = { + { CDF1( 1198) }, { CDF1( 2070) }, { CDF1( 9166) }, + { CDF1( 7499) }, { CDF1(22475) }, + }, .jnt_comp = { + { CDF1(18244) }, { CDF1(12865) }, { CDF1( 7053) }, + { CDF1(13259) }, { CDF1( 9334) }, { CDF1( 4644) }, + }, .mask_comp = { + { CDF1(26607) }, { CDF1(22891) }, { CDF1(18840) }, + { CDF1(24594) }, { CDF1(19934) }, { CDF1(22674) }, + }, .wedge_comp = { + { CDF1(23431) }, { CDF1(13171) }, { CDF1(11470) }, + { CDF1( 9770) }, { CDF1( 9100) }, { CDF1( 8233) }, + { CDF1( 6172) }, { CDF1(11820) }, { CDF1( 7701) }, + }, .wedge_idx = { + { CDF15( 2438, 4440, 6599, 8663, 11005, 12874, 15751, 18094, + 20359, 22362, 24127, 25702, 27752, 29450, 31171) }, + { CDF15( 806, 3266, 6005, 6738, 7218, 7367, 7771, 14588, + 16323, 17367, 18452, 19422, 22839, 26127, 29629) }, + { CDF15( 2779, 3738, 4683, 7213, 7775, 8017, 8655, 14357, + 17939, 21332, 24520, 27470, 29456, 30529, 31656) }, + { CDF15( 1684, 3625, 5675, 7108, 9302, 11274, 14429, 17144, + 19163, 20961, 22884, 24471, 26719, 28714, 30877) }, + { CDF15( 1142, 3491, 6277, 7314, 8089, 8355, 9023, 13624, + 15369, 16730, 18114, 19313, 22521, 26012, 29550) }, + { CDF15( 2742, 4195, 5727, 8035, 8980, 9336, 10146, 14124, + 17270, 20533, 23434, 25972, 27944, 29570, 31416) }, + { CDF15( 1727, 3948, 6101, 7796, 9841, 12344, 15766, 18944, + 20638, 22038, 23963, 25311, 26988, 28766, 31012) }, + { CDF15( 154, 987, 1925, 2051, 2088, 2111, 2151, 23033, + 23703, 24284, 24985, 25684, 27259, 28883, 30911) }, + { CDF15( 1135, 1322, 1493, 2635, 2696, 2737, 2770, 21016, + 22935, 25057, 27251, 29173, 30089, 30960, 31933) }, + }, .interintra = { + { CDF1(16384) }, { CDF1(26887) }, { CDF1(27597) }, + { CDF1(30237) }, + }, .interintra_mode = { + { CDF3(8192, 16384, 24576) }, + { CDF3(1875, 11082, 27332) }, + { CDF3(2473, 9996, 26388) }, + { CDF3(4238, 11537, 25926) }, + }, .interintra_wedge = { + { CDF1(20036) }, { CDF1(24957) }, { CDF1(26704) }, + { CDF1(27530) }, { CDF1(29564) }, { CDF1(29444) }, + { CDF1(26872) }, + }, .ref = { + { { CDF1( 4897) }, { CDF1(16973) }, { CDF1(29744) } }, + { { CDF1( 1555) }, { CDF1(16751) }, { CDF1(30279) } }, + { { CDF1( 4236) }, { CDF1(19647) }, { CDF1(31194) } }, + { { CDF1( 8650) }, { CDF1(24773) }, { CDF1(31895) } }, + { { CDF1( 904) }, { CDF1(11014) }, { CDF1(26875) } }, + { { CDF1( 1444) }, { CDF1(15087) }, { CDF1(30304) } }, + }, .comp_fwd_ref = { + { { CDF1( 4946) }, { CDF1(19891) }, { CDF1(30731) } }, + { { CDF1( 9468) }, { CDF1(22441) }, { CDF1(31059) } }, + { { CDF1( 1503) }, { CDF1(15160) }, { CDF1(27544) } }, + }, .comp_bwd_ref = { + { { CDF1( 2235) }, { CDF1(17182) }, { CDF1(30606) } }, + { { CDF1( 1423) }, { CDF1(15175) }, { CDF1(30489) } }, + }, .comp_uni_ref = { + { { CDF1( 5284) }, { CDF1(23152) }, { CDF1(31774) } }, + { { CDF1( 3865) }, { CDF1(14173) }, { CDF1(25120) } }, + { { CDF1( 3128) }, { CDF1(15270) }, { CDF1(26710) } }, + }, .txsz = { + { + { CDF1(19968) }, { CDF1(19968) }, { CDF1(24320) }, + }, { + { CDF2(12272, 30172) }, { CDF2(12272, 30172) }, + { CDF2(18677, 30848) }, + }, { + { CDF2(12986, 15180) }, { CDF2(12986, 15180) }, + { CDF2(24302, 25602) }, + }, { + { CDF2( 5782, 11475) }, { CDF2( 5782, 11475) }, + { CDF2(16803, 22759) }, + }, + }, .txpart = { + { { CDF1(28581) }, { CDF1(23846) }, { CDF1(20847) } }, + { { CDF1(24315) }, { CDF1(18196) }, { CDF1(12133) } }, + { { CDF1(18791) }, { CDF1(10887) }, { CDF1(11005) } }, + { { CDF1(27179) }, { CDF1(20004) }, { CDF1(11281) } }, + { { CDF1(26549) }, { CDF1(19308) }, { CDF1(14224) } }, + { { CDF1(28015) }, { CDF1(21546) }, { CDF1(14400) } }, + { { CDF1(28165) }, { CDF1(22401) }, { CDF1(16088) } }, + }, .txtp_inter1 = { + { CDF15( 4458, 5560, 7695, 9709, 13330, 14789, 17537, 20266, + 21504, 22848, 23934, 25474, 27727, 28915, 30631) }, + { CDF15( 1645, 2573, 4778, 5711, 7807, 8622, 10522, 15357, + 17674, 20408, 22517, 25010, 27116, 28856, 30749) }, + }, .txtp_inter2 = { + CDF11( 770, 2421, 5225, 12907, 15819, 18927, + 21561, 24089, 26595, 28526, 30529) + }, .txtp_inter3 = { + { CDF1(16384) }, { CDF1( 4167) }, { CDF1( 1998) }, { CDF1( 748) }, + }, .txtp_intra1 = { + { + { CDF6( 1535, 8035, 9461, 12751, 23467, 27825) }, + { CDF6( 564, 3335, 9709, 10870, 18143, 28094) }, + { CDF6( 672, 3247, 3676, 11982, 19415, 23127) }, + { CDF6( 5279, 13885, 15487, 18044, 23527, 30252) }, + { CDF6( 4423, 6074, 7985, 10416, 25693, 29298) }, + { CDF6( 1486, 4241, 9460, 10662, 16456, 27694) }, + { CDF6( 439, 2838, 3522, 6737, 18058, 23754) }, + { CDF6( 1190, 4233, 4855, 11670, 20281, 24377) }, + { CDF6( 1045, 4312, 8647, 10159, 18644, 29335) }, + { CDF6( 202, 3734, 4747, 7298, 17127, 24016) }, + { CDF6( 447, 4312, 6819, 8884, 16010, 23858) }, + { CDF6( 277, 4369, 5255, 8905, 16465, 22271) }, + { CDF6( 3409, 5436, 10599, 15599, 19687, 24040) }, + }, { + { CDF6( 1870, 13742, 14530, 16498, 23770, 27698) }, + { CDF6( 326, 8796, 14632, 15079, 19272, 27486) }, + { CDF6( 484, 7576, 7712, 14443, 19159, 22591) }, + { CDF6( 1126, 15340, 15895, 17023, 20896, 30279) }, + { CDF6( 655, 4854, 5249, 5913, 22099, 27138) }, + { CDF6( 1299, 6458, 8885, 9290, 14851, 25497) }, + { CDF6( 311, 5295, 5552, 6885, 16107, 22672) }, + { CDF6( 883, 8059, 8270, 11258, 17289, 21549) }, + { CDF6( 741, 7580, 9318, 10345, 16688, 29046) }, + { CDF6( 110, 7406, 7915, 9195, 16041, 23329) }, + { CDF6( 363, 7974, 9357, 10673, 15629, 24474) }, + { CDF6( 153, 7647, 8112, 9936, 15307, 19996) }, + { CDF6( 3511, 6332, 11165, 15335, 19323, 23594) }, + }, + }, .txtp_intra2 = { + { + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + }, { + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + { CDF4( 6554, 13107, 19661, 26214) }, + }, { + { CDF4( 1127, 12814, 22772, 27483) }, + { CDF4( 145, 6761, 11980, 26667) }, + { CDF4( 362, 5887, 11678, 16725) }, + { CDF4( 385, 15213, 18587, 30693) }, + { CDF4( 25, 2914, 23134, 27903) }, + { CDF4( 60, 4470, 11749, 23991) }, + { CDF4( 37, 3332, 14511, 21448) }, + { CDF4( 157, 6320, 13036, 17439) }, + { CDF4( 119, 6719, 12906, 29396) }, + { CDF4( 47, 5537, 12576, 21499) }, + { CDF4( 269, 6076, 11258, 23115) }, + { CDF4( 83, 5615, 12001, 17228) }, + { CDF4( 1968, 5556, 12023, 18547) }, + }, + }, .skip = { + { CDF1(31671) }, { CDF1(16515) }, { CDF1( 4576) }, + }, .skip_mode = { + { CDF1(32621) }, { CDF1(20708) }, { CDF1( 8127) }, + }, .partition = { + { + // 128x128 -> 64x64 + { CDF7(27899, 28219, 28529, 32484, 32539, 32619, 32639) }, + { CDF7( 6607, 6990, 8268, 32060, 32219, 32338, 32371) }, + { CDF7( 5429, 6676, 7122, 32027, 32227, 32531, 32582) }, + { CDF7( 711, 966, 1172, 32448, 32538, 32617, 32664) }, + }, { + // 64x64 -> 32x32 + { CDF9(20137, 21547, 23078, 29566, 29837, + 30261, 30524, 30892, 31724) }, + { CDF9( 6732, 7490, 9497, 27944, 28250, + 28515, 28969, 29630, 30104) }, + { CDF9( 5945, 7663, 8348, 28683, 29117, + 29749, 30064, 30298, 32238) }, + { CDF9( 870, 1212, 1487, 31198, 31394, + 31574, 31743, 31881, 32332) }, + }, { + // 32x32 -> 16x16 + { CDF9(18462, 20920, 23124, 27647, 28227, + 29049, 29519, 30178, 31544) }, + { CDF9( 7689, 9060, 12056, 24992, 25660, + 26182, 26951, 28041, 29052) }, + { CDF9( 6015, 9009, 10062, 24544, 25409, + 26545, 27071, 27526, 32047) }, + { CDF9( 1394, 2208, 2796, 28614, 29061, + 29466, 29840, 30185, 31899) }, + }, { + // 16x16 -> 8x8 + { CDF9(15597, 20929, 24571, 26706, 27664, + 28821, 29601, 30571, 31902) }, + { CDF9( 7925, 11043, 16785, 22470, 23971, + 25043, 26651, 28701, 29834) }, + { CDF9( 5414, 13269, 15111, 20488, 22360, + 24500, 25537, 26336, 32117) }, + { CDF9( 2662, 6362, 8614, 20860, 23053, + 24778, 26436, 27829, 31171) }, + }, { + // 8x8 -> 4x4 only supports the four legacy partition types + { CDF3(19132, 25510, 30392) }, + { CDF3(13928, 19855, 28540) }, + { CDF3(12522, 23679, 28629) }, + { CDF3( 9896, 18783, 25853) }, + }, + }, .seg_pred = { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + }, .seg_id = { + { CDF7( 5622, 7893, 16093, 18233, 27809, 28373, 32533) }, + { CDF7(14274, 18230, 22557, 24935, 29980, 30851, 32344) }, + { CDF7(27527, 28487, 28723, 28890, 32397, 32647, 32679) }, + }, .cfl_sign = { + CDF7( 1418, 2123, 13340, 18405, 26972, 28343, 32294) + }, .cfl_alpha = { + { CDF15( 7637, 20719, 31401, 32481, 32657, 32688, 32692, 32696, + 32700, 32704, 32708, 32712, 32716, 32720, 32724) }, + { CDF15(14365, 23603, 28135, 31168, 32167, 32395, 32487, 32573, + 32620, 32647, 32668, 32672, 32676, 32680, 32684) }, + { CDF15(11532, 22380, 28445, 31360, 32349, 32523, 32584, 32649, + 32673, 32677, 32681, 32685, 32689, 32693, 32697) }, + { CDF15(26990, 31402, 32282, 32571, 32692, 32696, 32700, 32704, + 32708, 32712, 32716, 32720, 32724, 32728, 32732) }, + { CDF15(17248, 26058, 28904, 30608, 31305, 31877, 32126, 32321, + 32394, 32464, 32516, 32560, 32576, 32593, 32622) }, + { CDF15(14738, 21678, 25779, 27901, 29024, 30302, 30980, 31843, + 32144, 32413, 32520, 32594, 32622, 32656, 32660) }, + }, .restore_wiener = { + CDF1(11570) + }, .restore_sgrproj = { + CDF1(16855) + }, .restore_switchable = { + CDF2( 9413, 22581) + }, .delta_q = { + CDF3(28160, 32120, 32677) + }, .delta_lf = { + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + { CDF3(28160, 32120, 32677) }, + }, .motion_mode = { + [BS_8x8] = { CDF2( 7651, 24760) }, + [BS_8x16] = { CDF2( 4738, 24765) }, + [BS_8x32] = { CDF2(28799, 31390) }, + [BS_16x8] = { CDF2( 5391, 25528) }, + [BS_16x16] = { CDF2(19419, 26810) }, + [BS_16x32] = { CDF2( 5123, 23606) }, + [BS_16x64] = { CDF2(28973, 31594) }, + [BS_32x8] = { CDF2(26431, 30774) }, + [BS_32x16] = { CDF2(11606, 24308) }, + [BS_32x32] = { CDF2(26260, 29116) }, + [BS_32x64] = { CDF2(20360, 28062) }, + [BS_64x16] = { CDF2(29742, 31203) }, + [BS_64x32] = { CDF2(21679, 26830) }, + [BS_64x64] = { CDF2(29516, 30701) }, + [BS_64x128] = { CDF2(28898, 30397) }, + [BS_128x64] = { CDF2(30878, 31335) }, + [BS_128x128] = { CDF2(32507, 32558) }, + }, .obmc = { + [BS_8x8] = { CDF1(10437) }, + [BS_8x16] = { CDF1( 9371) }, + [BS_8x32] = { CDF1(23664) }, + [BS_16x8] = { CDF1( 9301) }, + [BS_16x16] = { CDF1(17432) }, + [BS_16x32] = { CDF1(14423) }, + [BS_16x64] = { CDF1(24008) }, + [BS_32x8] = { CDF1(20901) }, + [BS_32x16] = { CDF1(15142) }, + [BS_32x32] = { CDF1(25817) }, + [BS_32x64] = { CDF1(22823) }, + [BS_64x16] = { CDF1(26879) }, + [BS_64x32] = { CDF1(22083) }, + [BS_64x64] = { CDF1(30128) }, + [BS_64x128] = { CDF1(31014) }, + [BS_128x64] = { CDF1(31560) }, + [BS_128x128] = { CDF1(32638) }, + }, .pal_y = { + { { CDF1(31676) }, { CDF1( 3419) }, { CDF1( 1261) } }, + { { CDF1(31912) }, { CDF1( 2859) }, { CDF1( 980) } }, + { { CDF1(31823) }, { CDF1( 3400) }, { CDF1( 781) } }, + { { CDF1(32030) }, { CDF1( 3561) }, { CDF1( 904) } }, + { { CDF1(32309) }, { CDF1( 7337) }, { CDF1( 1462) } }, + { { CDF1(32265) }, { CDF1( 4015) }, { CDF1( 1521) } }, + { { CDF1(32450) }, { CDF1( 7946) }, { CDF1( 129) } }, + }, .pal_sz = { + { + { CDF6( 7952, 13000, 18149, 21478, 25527, 29241) }, + { CDF6( 7139, 11421, 16195, 19544, 23666, 28073) }, + { CDF6( 7788, 12741, 17325, 20500, 24315, 28530) }, + { CDF6( 8271, 14064, 18246, 21564, 25071, 28533) }, + { CDF6(12725, 19180, 21863, 24839, 27535, 30120) }, + { CDF6( 9711, 14888, 16923, 21052, 25661, 27875) }, + { CDF6(14940, 20797, 21678, 24186, 27033, 28999) }, + }, { + { CDF6( 8713, 19979, 27128, 29609, 31331, 32272) }, + { CDF6( 5839, 15573, 23581, 26947, 29848, 31700) }, + { CDF6( 4426, 11260, 17999, 21483, 25863, 29430) }, + { CDF6( 3228, 9464, 14993, 18089, 22523, 27420) }, + { CDF6( 3768, 8886, 13091, 17852, 22495, 27207) }, + { CDF6( 2464, 8451, 12861, 21632, 25525, 28555) }, + { CDF6( 1269, 5435, 10433, 18963, 21700, 25865) }, + }, + }, .pal_uv = { + { CDF1(32461) }, { CDF1(21488) }, + }, .color_map = { + { /* y */ + { + { CDF1(28710) }, { CDF1(16384) }, { CDF1(10553) }, + { CDF1(27036) }, { CDF1(31603) }, + }, { + { CDF2(27877, 30490) }, { CDF2(11532, 25697) }, + { CDF2( 6544, 30234) }, { CDF2(23018, 28072) }, + { CDF2(31915, 32385) }, + }, { + { CDF3(25572, 28046, 30045) }, + { CDF3( 9478, 21590, 27256) }, + { CDF3( 7248, 26837, 29824) }, + { CDF3(19167, 24486, 28349) }, + { CDF3(31400, 31825, 32250) }, + }, { + { CDF4(24779, 26955, 28576, 30282) }, + { CDF4( 8669, 20364, 24073, 28093) }, + { CDF4( 4255, 27565, 29377, 31067) }, + { CDF4(19864, 23674, 26716, 29530) }, + { CDF4(31646, 31893, 32147, 32426) }, + }, { + { CDF5(23132, 25407, 26970, 28435, 30073) }, + { CDF5( 7443, 17242, 20717, 24762, 27982) }, + { CDF5( 6300, 24862, 26944, 28784, 30671) }, + { CDF5(18916, 22895, 25267, 27435, 29652) }, + { CDF5(31270, 31550, 31808, 32059, 32353) }, + }, { + { CDF6(23105, 25199, 26464, 27684, 28931, 30318) }, + { CDF6( 6950, 15447, 18952, 22681, 25567, 28563) }, + { CDF6( 7560, 23474, 25490, 27203, 28921, 30708) }, + { CDF6(18544, 22373, 24457, 26195, 28119, 30045) }, + { CDF6(31198, 31451, 31670, 31882, 32123, 32391) }, + }, { + { CDF7(21689, 23883, 25163, 26352, 27506, 28827, 30195) }, + { CDF7( 6892, 15385, 17840, 21606, 24287, 26753, 29204) }, + { CDF7( 5651, 23182, 25042, 26518, 27982, 29392, 30900) }, + { CDF7(19349, 22578, 24418, 25994, 27524, 29031, 30448) }, + { CDF7(31028, 31270, 31504, 31705, 31927, 32153, 32392) }, + }, + }, { /* uv */ + { + { CDF1(29089) }, { CDF1(16384) }, { CDF1( 8713) }, + { CDF1(29257) }, { CDF1(31610) }, + }, { + { CDF2(25257, 29145) }, { CDF2(12287, 27293) }, + { CDF2( 7033, 27960) }, { CDF2(20145, 25405) }, + { CDF2(30608, 31639) }, + }, { + { CDF3(24210, 27175, 29903) }, + { CDF3( 9888, 22386, 27214) }, + { CDF3( 5901, 26053, 29293) }, + { CDF3(18318, 22152, 28333) }, + { CDF3(30459, 31136, 31926) }, + }, { + { CDF4(22980, 25479, 27781, 29986) }, + { CDF4( 8413, 21408, 24859, 28874) }, + { CDF4( 2257, 29449, 30594, 31598) }, + { CDF4(19189, 21202, 25915, 28620) }, + { CDF4(31844, 32044, 32281, 32518) }, + }, { + { CDF5(22217, 24567, 26637, 28683, 30548) }, + { CDF5( 7307, 16406, 19636, 24632, 28424) }, + { CDF5( 4441, 25064, 26879, 28942, 30919) }, + { CDF5(17210, 20528, 23319, 26750, 29582) }, + { CDF5(30674, 30953, 31396, 31735, 32207) }, + }, { + { CDF6(21239, 23168, 25044, 26962, 28705, 30506) }, + { CDF6( 6545, 15012, 18004, 21817, 25503, 28701) }, + { CDF6( 3448, 26295, 27437, 28704, 30126, 31442) }, + { CDF6(15889, 18323, 21704, 24698, 26976, 29690) }, + { CDF6(30988, 31204, 31479, 31734, 31983, 32325) }, + }, { + { CDF7(21442, 23288, 24758, 26246, 27649, 28980, 30563) }, + { CDF7( 5863, 14933, 17552, 20668, 23683, 26411, 29273) }, + { CDF7( 3415, 25810, 26877, 27990, 29223, 30394, 31618) }, + { CDF7(17965, 20084, 22232, 23974, 26274, 28402, 30390) }, + { CDF7(31190, 31329, 31516, 31679, 31825, 32026, 32322) }, + }, + }, + }, .intrabc = { + CDF1(30531) + }, +}; + +static const CdfMvComponent default_mv_component_cdf = { + .classes = { + CDF10(28672, 30976, 31858, 32320, 32551, + 32656, 32740, 32757, 32762, 32767) + }, .class0 = { + CDF1(27648) + }, .classN = { + { CDF1(17408) }, { CDF1(17920) }, { CDF1(18944) }, + { CDF1(20480) }, { CDF1(22528) }, { CDF1(24576) }, + { CDF1(28672) }, { CDF1(29952) }, { CDF1(29952) }, + { CDF1(30720) }, + }, .class0_fp = { + { CDF3(16384, 24576, 26624) }, + { CDF3(12288, 21248, 24128) }, + }, .classN_fp = { + CDF3( 8192, 17408, 21248) + }, .class0_hp = { + CDF1(20480) + }, .classN_hp = { + CDF1(16384) + }, .sign = { + CDF1(16384) + }, +}; + +static const uint16_t ALIGN(default_mv_joint_cdf[N_MV_JOINTS], 8) = { + CDF3( 4096, 11264, 19328) +}; + +static const uint16_t ALIGN(default_kf_y_mode_cdf[5][5][N_INTRA_PRED_MODES + 3], 32) = { + { + { CDF12(15588, 17027, 19338, 20218, 20682, 21110, + 21825, 23244, 24189, 28165, 29093, 30466) }, + { CDF12(12016, 18066, 19516, 20303, 20719, 21444, + 21888, 23032, 24434, 28658, 30172, 31409) }, + { CDF12(10052, 10771, 22296, 22788, 23055, 23239, + 24133, 25620, 26160, 29336, 29929, 31567) }, + { CDF12(14091, 15406, 16442, 18808, 19136, 19546, + 19998, 22096, 24746, 29585, 30958, 32462) }, + { CDF12(12122, 13265, 15603, 16501, 18609, 20033, + 22391, 25583, 26437, 30261, 31073, 32475) }, + }, { + { CDF12(10023, 19585, 20848, 21440, 21832, 22760, + 23089, 24023, 25381, 29014, 30482, 31436) }, + { CDF12( 5983, 24099, 24560, 24886, 25066, 25795, + 25913, 26423, 27610, 29905, 31276, 31794) }, + { CDF12( 7444, 12781, 20177, 20728, 21077, 21607, + 22170, 23405, 24469, 27915, 29090, 30492) }, + { CDF12( 8537, 14689, 15432, 17087, 17408, 18172, + 18408, 19825, 24649, 29153, 31096, 32210) }, + { CDF12( 7543, 14231, 15496, 16195, 17905, 20717, + 21984, 24516, 26001, 29675, 30981, 31994) }, + }, { + { CDF12(12613, 13591, 21383, 22004, 22312, 22577, + 23401, 25055, 25729, 29538, 30305, 32077) }, + { CDF12( 9687, 13470, 18506, 19230, 19604, 20147, + 20695, 22062, 23219, 27743, 29211, 30907) }, + { CDF12( 6183, 6505, 26024, 26252, 26366, 26434, + 27082, 28354, 28555, 30467, 30794, 32086) }, + { CDF12(10718, 11734, 14954, 17224, 17565, 17924, + 18561, 21523, 23878, 28975, 30287, 32252) }, + { CDF12( 9194, 9858, 16501, 17263, 18424, 19171, + 21563, 25961, 26561, 30072, 30737, 32463) }, + }, { + { CDF12(12602, 14399, 15488, 18381, 18778, 19315, + 19724, 21419, 25060, 29696, 30917, 32409) }, + { CDF12( 8203, 13821, 14524, 17105, 17439, 18131, + 18404, 19468, 25225, 29485, 31158, 32342) }, + { CDF12( 8451, 9731, 15004, 17643, 18012, 18425, + 19070, 21538, 24605, 29118, 30078, 32018) }, + { CDF12( 7714, 9048, 9516, 16667, 16817, 16994, + 17153, 18767, 26743, 30389, 31536, 32528) }, + { CDF12( 8843, 10280, 11496, 15317, 16652, 17943, + 19108, 22718, 25769, 29953, 30983, 32485) }, + }, { + { CDF12(12578, 13671, 15979, 16834, 19075, 20913, + 22989, 25449, 26219, 30214, 31150, 32477) }, + { CDF12( 9563, 13626, 15080, 15892, 17756, 20863, + 22207, 24236, 25380, 29653, 31143, 32277) }, + { CDF12( 8356, 8901, 17616, 18256, 19350, 20106, + 22598, 25947, 26466, 29900, 30523, 32261) }, + { CDF12(10835, 11815, 13124, 16042, 17018, 18039, + 18947, 22753, 24615, 29489, 30883, 32482) }, + { CDF12( 7618, 8288, 9859, 10509, 15386, 18657, + 22903, 28776, 29180, 31355, 31802, 32593) }, + }, +}; + +static const CdfCoefContext av1_default_coef_cdf[4] = { + [0] = { + .skip = { + { + { CDF1(31849) }, { CDF1( 5892) }, { CDF1(12112) }, + { CDF1(21935) }, { CDF1(20289) }, { CDF1(27473) }, + { CDF1(32487) }, { CDF1( 7654) }, { CDF1(19473) }, + { CDF1(29984) }, { CDF1( 9961) }, { CDF1(30242) }, + { CDF1(32117) }, + }, { + { CDF1(31548) }, { CDF1( 1549) }, { CDF1(10130) }, + { CDF1(16656) }, { CDF1(18591) }, { CDF1(26308) }, + { CDF1(32537) }, { CDF1( 5403) }, { CDF1(18096) }, + { CDF1(30003) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(29957) }, { CDF1( 5391) }, { CDF1(18039) }, + { CDF1(23566) }, { CDF1(22431) }, { CDF1(25822) }, + { CDF1(32197) }, { CDF1( 3778) }, { CDF1(15336) }, + { CDF1(28981) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(17920) }, { CDF1( 1818) }, { CDF1( 7282) }, + { CDF1(25273) }, { CDF1(10923) }, { CDF1(31554) }, + { CDF1(32624) }, { CDF1( 1366) }, { CDF1(15628) }, + { CDF1(30462) }, { CDF1( 146) }, { CDF1( 5132) }, + { CDF1(31657) }, + }, { + { CDF1( 6308) }, { CDF1( 117) }, { CDF1( 1638) }, + { CDF1( 2161) }, { CDF1(16384) }, { CDF1(10923) }, + { CDF1(30247) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, + }, .eob_bin_16 = { + { + { CDF4( 840, 1039, 1980, 4895) }, + { CDF4( 370, 671, 1883, 4471) }, + }, { + { CDF4( 3247, 4950, 9688, 14563) }, + { CDF4( 1904, 3354, 7763, 14647) }, + }, + }, .eob_bin_32 = { + { + { CDF5( 400, 520, 977, 2102, 6542) }, + { CDF5( 210, 405, 1315, 3326, 7537) }, + }, { + { CDF5( 2636, 4273, 7588, 11794, 20401) }, + { CDF5( 1786, 3179, 6902, 11357, 19054) }, + }, + }, .eob_bin_64 = { + { + { CDF6( 329, 498, 1101, 1784, 3265, 7758) }, + { CDF6( 335, 730, 1459, 5494, 8755, 12997) }, + }, { + { CDF6( 3505, 5304, 10086, 13814, 17684, 23370) }, + { CDF6( 1563, 2700, 4876, 10911, 14706, 22480) }, + }, + }, .eob_bin_128 = { + { + { CDF7( 219, 482, 1140, 2091, 3680, 6028, 12586) }, + { CDF7( 371, 699, 1254, 4830, 9479, 12562, 17497) }, + }, { + { CDF7( 5245, 7456, 12880, 15852, 20033, 23932, 27608) }, + { CDF7( 2054, 3472, 5869, 14232, 18242, 20590, 26752) }, + }, + }, .eob_bin_256 = { + { + { CDF8( 310, 584, 1887, 3589, + 6168, 8611, 11352, 15652) }, + { CDF8( 998, 1850, 2998, 5604, + 17341, 19888, 22899, 25583) }, + }, { + { CDF8( 2520, 3240, 5952, 8870, + 12577, 17558, 19954, 24168) }, + { CDF8( 2203, 4130, 7435, 10739, + 20652, 23681, 25609, 27261) }, + }, + }, .eob_bin_512 = { + { CDF9( 641, 983, 3707, 5430, 10234, + 14958, 18788, 23412, 26061) }, + { CDF9( 5095, 6446, 9996, 13354, 16017, + 17986, 20919, 26129, 29140) }, + }, .eob_bin_1024 = { + { CDF10( 393, 421, 751, 1623, 3160, + 6352, 13345, 18047, 22571, 25830) }, + { CDF10( 1865, 1988, 2930, 4242, 10533, + 16538, 21354, 27255, 28546, 31784) }, + }, .eob_hi_bit = { + { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16961) }, + { CDF1(17223) }, { CDF1( 7621) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(19069) }, + { CDF1(22525) }, { CDF1(13377) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20401) }, + { CDF1(17025) }, { CDF1(12845) }, { CDF1(12873) }, + { CDF1(14094) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20681) }, + { CDF1(20701) }, { CDF1(15250) }, { CDF1(15017) }, + { CDF1(14928) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(23905) }, + { CDF1(17194) }, { CDF1(16170) }, { CDF1(17695) }, + { CDF1(13826) }, { CDF1(15810) }, { CDF1(12036) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(23959) }, + { CDF1(20799) }, { CDF1(19021) }, { CDF1(16203) }, + { CDF1(17886) }, { CDF1(14144) }, { CDF1(12010) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(27399) }, + { CDF1(16327) }, { CDF1(18071) }, { CDF1(19584) }, + { CDF1(20721) }, { CDF1(18432) }, { CDF1(19560) }, + { CDF1(10150) }, { CDF1( 8805) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(24932) }, + { CDF1(20833) }, { CDF1(12027) }, { CDF1(16670) }, + { CDF1(19914) }, { CDF1(15106) }, { CDF1(17662) }, + { CDF1(13783) }, { CDF1(28756) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(23406) }, + { CDF1(21845) }, { CDF1(18432) }, { CDF1(16384) }, + { CDF1(17096) }, { CDF1(12561) }, { CDF1(17320) }, + { CDF1(22395) }, { CDF1(21370) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, + }, .eob_base_tok = { + { + { + { CDF2(17837, 29055) }, { CDF2(29600, 31446) }, + { CDF2(30844, 31878) }, { CDF2(24926, 28948) }, + }, { + { CDF2(21365, 30026) }, { CDF2(30512, 32423) }, + { CDF2(31658, 32621) }, { CDF2(29630, 31881) }, + }, + }, { + { + { CDF2( 5717, 26477) }, { CDF2(30491, 31703) }, + { CDF2(31550, 32158) }, { CDF2(29648, 31491) }, + }, { + { CDF2(12608, 27820) }, { CDF2(30680, 32225) }, + { CDF2(30809, 32335) }, { CDF2(31299, 32423) }, + }, + }, { + { + { CDF2( 1786, 12612) }, { CDF2(30663, 31625) }, + { CDF2(32339, 32468) }, { CDF2(31148, 31833) }, + }, { + { CDF2(18857, 23865) }, { CDF2(31428, 32428) }, + { CDF2(31744, 32373) }, { CDF2(31775, 32526) }, + }, + }, { + { + { CDF2( 1787, 2532) }, { CDF2(30832, 31662) }, + { CDF2(31824, 32682) }, { CDF2(32133, 32569) }, + }, { + { CDF2(13751, 22235) }, { CDF2(32089, 32409) }, + { CDF2(27084, 27920) }, { CDF2(29291, 32594) }, + }, + }, { + { + { CDF2( 1725, 3449) }, { CDF2(31102, 31935) }, + { CDF2(32457, 32613) }, { CDF2(32412, 32649) }, + }, { + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + }, + }, + }, .base_tok = { + { + { + { CDF3( 4034, 8930, 12727) }, + { CDF3(18082, 29741, 31877) }, + { CDF3(12596, 26124, 30493) }, + { CDF3( 9446, 21118, 27005) }, + { CDF3( 6308, 15141, 21279) }, + { CDF3( 2463, 6357, 9783) }, + { CDF3(20667, 30546, 31929) }, + { CDF3(13043, 26123, 30134) }, + { CDF3( 8151, 18757, 24778) }, + { CDF3( 5255, 12839, 18632) }, + { CDF3( 2820, 7206, 11161) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(15736, 27553, 30604) }, + { CDF3(11210, 23794, 28787) }, + { CDF3( 5947, 13874, 19701) }, + { CDF3( 4215, 9323, 13891) }, + { CDF3( 2833, 6462, 10059) }, + { CDF3(19605, 30393, 31582) }, + { CDF3(13523, 26252, 30248) }, + { CDF3( 8446, 18622, 24512) }, + { CDF3( 3818, 10343, 15974) }, + { CDF3( 1481, 4117, 6796) }, + { CDF3(22649, 31302, 32190) }, + { CDF3(14829, 27127, 30449) }, + { CDF3( 8313, 17702, 23304) }, + { CDF3( 3022, 8301, 12786) }, + { CDF3( 1536, 4412, 7184) }, + { CDF3(22354, 29774, 31372) }, + { CDF3(14723, 25472, 29214) }, + { CDF3( 6673, 13745, 18662) }, + { CDF3( 2068, 5766, 9322) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 6302, 16444, 21761) }, + { CDF3(23040, 31538, 32475) }, + { CDF3(15196, 28452, 31496) }, + { CDF3(10020, 22946, 28514) }, + { CDF3( 6533, 16862, 23501) }, + { CDF3( 3538, 9816, 15076) }, + { CDF3(24444, 31875, 32525) }, + { CDF3(15881, 28924, 31635) }, + { CDF3( 9922, 22873, 28466) }, + { CDF3( 6527, 16966, 23691) }, + { CDF3( 4114, 11303, 17220) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(20201, 30770, 32209) }, + { CDF3(14754, 28071, 31258) }, + { CDF3( 8378, 20186, 26517) }, + { CDF3( 5916, 15299, 21978) }, + { CDF3( 4268, 11583, 17901) }, + { CDF3(24361, 32025, 32581) }, + { CDF3(18673, 30105, 31943) }, + { CDF3(10196, 22244, 27576) }, + { CDF3( 5495, 14349, 20417) }, + { CDF3( 2676, 7415, 11498) }, + { CDF3(24678, 31958, 32585) }, + { CDF3(18629, 29906, 31831) }, + { CDF3( 9364, 20724, 26315) }, + { CDF3( 4641, 12318, 18094) }, + { CDF3( 2758, 7387, 11579) }, + { CDF3(25433, 31842, 32469) }, + { CDF3(18795, 29289, 31411) }, + { CDF3( 7644, 17584, 23592) }, + { CDF3( 3408, 9014, 15047) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 4536, 10072, 14001) }, + { CDF3(25459, 31416, 32206) }, + { CDF3(16605, 28048, 30818) }, + { CDF3(11008, 22857, 27719) }, + { CDF3( 6915, 16268, 22315) }, + { CDF3( 2625, 6812, 10537) }, + { CDF3(24257, 31788, 32499) }, + { CDF3(16880, 29454, 31879) }, + { CDF3(11958, 25054, 29778) }, + { CDF3( 7916, 18718, 25084) }, + { CDF3( 3383, 8777, 13446) }, + { CDF3(22720, 31603, 32393) }, + { CDF3(14960, 28125, 31335) }, + { CDF3( 9731, 22210, 27928) }, + { CDF3( 6304, 15832, 22277) }, + { CDF3( 2910, 7818, 12166) }, + { CDF3(20375, 30627, 32131) }, + { CDF3(13904, 27284, 30887) }, + { CDF3( 9368, 21558, 27144) }, + { CDF3( 5937, 14966, 21119) }, + { CDF3( 2667, 7225, 11319) }, + { CDF3(23970, 31470, 32378) }, + { CDF3(17173, 29734, 32018) }, + { CDF3(12795, 25441, 29965) }, + { CDF3( 8981, 19680, 25893) }, + { CDF3( 4728, 11372, 16902) }, + { CDF3(24287, 31797, 32439) }, + { CDF3(16703, 29145, 31696) }, + { CDF3(10833, 23554, 28725) }, + { CDF3( 6468, 16566, 23057) }, + { CDF3( 2415, 6562, 10278) }, + { CDF3(26610, 32395, 32659) }, + { CDF3(18590, 30498, 32117) }, + { CDF3(12420, 25756, 29950) }, + { CDF3( 7639, 18746, 24710) }, + { CDF3( 3001, 8086, 12347) }, + { CDF3(25076, 32064, 32580) }, + { CDF3(17946, 30128, 32028) }, + { CDF3(12024, 24985, 29378) }, + { CDF3( 7517, 18390, 24304) }, + { CDF3( 3243, 8781, 13331) }, + }, { + { CDF3( 6037, 16771, 21957) }, + { CDF3(24774, 31704, 32426) }, + { CDF3(16830, 28589, 31056) }, + { CDF3(10602, 22828, 27760) }, + { CDF3( 6733, 16829, 23071) }, + { CDF3( 3250, 8914, 13556) }, + { CDF3(25582, 32220, 32668) }, + { CDF3(18659, 30342, 32223) }, + { CDF3(12546, 26149, 30515) }, + { CDF3( 8420, 20451, 26801) }, + { CDF3( 4636, 12420, 18344) }, + { CDF3(27581, 32362, 32639) }, + { CDF3(18987, 30083, 31978) }, + { CDF3(11327, 24248, 29084) }, + { CDF3( 7264, 17719, 24120) }, + { CDF3( 3995, 10768, 16169) }, + { CDF3(25893, 31831, 32487) }, + { CDF3(16577, 28587, 31379) }, + { CDF3(10189, 22748, 28182) }, + { CDF3( 6832, 17094, 23556) }, + { CDF3( 3708, 10110, 15334) }, + { CDF3(25904, 32282, 32656) }, + { CDF3(19721, 30792, 32276) }, + { CDF3(12819, 26243, 30411) }, + { CDF3( 8572, 20614, 26891) }, + { CDF3( 5364, 14059, 20467) }, + { CDF3(26580, 32438, 32677) }, + { CDF3(20852, 31225, 32340) }, + { CDF3(12435, 25700, 29967) }, + { CDF3( 8691, 20825, 26976) }, + { CDF3( 4446, 12209, 17269) }, + { CDF3(27350, 32429, 32696) }, + { CDF3(21372, 30977, 32272) }, + { CDF3(12673, 25270, 29853) }, + { CDF3( 9208, 20925, 26640) }, + { CDF3( 5018, 13351, 18732) }, + { CDF3(27351, 32479, 32713) }, + { CDF3(21398, 31209, 32387) }, + { CDF3(12162, 25047, 29842) }, + { CDF3( 7896, 18691, 25319) }, + { CDF3( 4670, 12882, 18881) }, + }, + }, { + { + { CDF3( 5487, 10460, 13708) }, + { CDF3(21597, 28303, 30674) }, + { CDF3(11037, 21953, 26476) }, + { CDF3( 8147, 17962, 22952) }, + { CDF3( 5242, 13061, 18532) }, + { CDF3( 1889, 5208, 8182) }, + { CDF3(26774, 32133, 32590) }, + { CDF3(17844, 29564, 31767) }, + { CDF3(11690, 24438, 29171) }, + { CDF3( 7542, 18215, 24459) }, + { CDF3( 2993, 8050, 12319) }, + { CDF3(28023, 32328, 32591) }, + { CDF3(18651, 30126, 31954) }, + { CDF3(12164, 25146, 29589) }, + { CDF3( 7762, 18530, 24771) }, + { CDF3( 3492, 9183, 13920) }, + { CDF3(27591, 32008, 32491) }, + { CDF3(17149, 28853, 31510) }, + { CDF3(11485, 24003, 28860) }, + { CDF3( 7697, 18086, 24210) }, + { CDF3( 3075, 7999, 12218) }, + { CDF3(28268, 32482, 32654) }, + { CDF3(19631, 31051, 32404) }, + { CDF3(13860, 27260, 31020) }, + { CDF3( 9605, 21613, 27594) }, + { CDF3( 4876, 12162, 17908) }, + { CDF3(27248, 32316, 32576) }, + { CDF3(18955, 30457, 32075) }, + { CDF3(11824, 23997, 28795) }, + { CDF3( 7346, 18196, 24647) }, + { CDF3( 3403, 9247, 14111) }, + { CDF3(29711, 32655, 32735) }, + { CDF3(21169, 31394, 32417) }, + { CDF3(13487, 27198, 30957) }, + { CDF3( 8828, 21683, 27614) }, + { CDF3( 4270, 11451, 17038) }, + { CDF3(28708, 32578, 32731) }, + { CDF3(20120, 31241, 32482) }, + { CDF3(13692, 27550, 31321) }, + { CDF3( 9418, 22514, 28439) }, + { CDF3( 4999, 13283, 19462) }, + }, { + { CDF3( 5673, 14302, 19711) }, + { CDF3(26251, 30701, 31834) }, + { CDF3(12782, 23783, 27803) }, + { CDF3( 9127, 20657, 25808) }, + { CDF3( 6368, 16208, 21462) }, + { CDF3( 2465, 7177, 10822) }, + { CDF3(29961, 32563, 32719) }, + { CDF3(18318, 29891, 31949) }, + { CDF3(11361, 24514, 29357) }, + { CDF3( 7900, 19603, 25607) }, + { CDF3( 4002, 10590, 15546) }, + { CDF3(29637, 32310, 32595) }, + { CDF3(18296, 29913, 31809) }, + { CDF3(10144, 21515, 26871) }, + { CDF3( 5358, 14322, 20394) }, + { CDF3( 3067, 8362, 13346) }, + { CDF3(28652, 32470, 32676) }, + { CDF3(17538, 30771, 32209) }, + { CDF3(13924, 26882, 30494) }, + { CDF3(10496, 22837, 27869) }, + { CDF3( 7236, 16396, 21621) }, + { CDF3(30743, 32687, 32746) }, + { CDF3(23006, 31676, 32489) }, + { CDF3(14494, 27828, 31120) }, + { CDF3(10174, 22801, 28352) }, + { CDF3( 6242, 15281, 21043) }, + { CDF3(25817, 32243, 32720) }, + { CDF3(18618, 31367, 32325) }, + { CDF3(13997, 28318, 31878) }, + { CDF3(12255, 26534, 31383) }, + { CDF3( 9561, 21588, 28450) }, + { CDF3(28188, 32635, 32724) }, + { CDF3(22060, 32365, 32728) }, + { CDF3(18102, 30690, 32528) }, + { CDF3(14196, 28864, 31999) }, + { CDF3(12262, 25792, 30865) }, + { CDF3(24176, 32109, 32628) }, + { CDF3(18280, 29681, 31963) }, + { CDF3(10205, 23703, 29664) }, + { CDF3( 7889, 20025, 27676) }, + { CDF3( 6060, 16743, 23970) }, + }, + }, { + { + { CDF3( 5141, 7096, 8260) }, + { CDF3(27186, 29022, 29789) }, + { CDF3( 6668, 12568, 15682) }, + { CDF3( 2172, 6181, 8638) }, + { CDF3( 1126, 3379, 4531) }, + { CDF3( 443, 1361, 2254) }, + { CDF3(26083, 31153, 32436) }, + { CDF3(13486, 24603, 28483) }, + { CDF3( 6508, 14840, 19910) }, + { CDF3( 3386, 8800, 13286) }, + { CDF3( 1530, 4322, 7054) }, + { CDF3(29639, 32080, 32548) }, + { CDF3(15897, 27552, 30290) }, + { CDF3( 8588, 20047, 25383) }, + { CDF3( 4889, 13339, 19269) }, + { CDF3( 2240, 6871, 10498) }, + { CDF3(28165, 32197, 32517) }, + { CDF3(20735, 30427, 31568) }, + { CDF3(14325, 24671, 27692) }, + { CDF3( 5119, 12554, 17805) }, + { CDF3( 1810, 5441, 8261) }, + { CDF3(31212, 32724, 32748) }, + { CDF3(23352, 31766, 32545) }, + { CDF3(14669, 27570, 31059) }, + { CDF3( 8492, 20894, 27272) }, + { CDF3( 3644, 10194, 15204) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 2461, 7013, 9371) }, + { CDF3(24749, 29600, 30986) }, + { CDF3( 9466, 19037, 22417) }, + { CDF3( 3584, 9280, 14400) }, + { CDF3( 1505, 3929, 5433) }, + { CDF3( 677, 1500, 2736) }, + { CDF3(23987, 30702, 32117) }, + { CDF3(13554, 24571, 29263) }, + { CDF3( 6211, 14556, 21155) }, + { CDF3( 3135, 10972, 15625) }, + { CDF3( 2435, 7127, 11427) }, + { CDF3(31300, 32532, 32550) }, + { CDF3(14757, 30365, 31954) }, + { CDF3( 4405, 11612, 18553) }, + { CDF3( 580, 4132, 7322) }, + { CDF3( 1695, 10169, 14124) }, + { CDF3(30008, 32282, 32591) }, + { CDF3(19244, 30108, 31748) }, + { CDF3(11180, 24158, 29555) }, + { CDF3( 5650, 14972, 19209) }, + { CDF3( 2114, 5109, 8456) }, + { CDF3(31856, 32716, 32748) }, + { CDF3(23012, 31664, 32572) }, + { CDF3(13694, 26656, 30636) }, + { CDF3( 8142, 19508, 26093) }, + { CDF3( 4253, 10955, 16724) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 601, 983, 1311) }, + { CDF3(18725, 23406, 28087) }, + { CDF3( 5461, 8192, 10923) }, + { CDF3( 3781, 15124, 21425) }, + { CDF3( 2587, 7761, 12072) }, + { CDF3( 106, 458, 810) }, + { CDF3(22282, 29710, 31894) }, + { CDF3( 8508, 20926, 25984) }, + { CDF3( 3726, 12713, 18083) }, + { CDF3( 1620, 7112, 10893) }, + { CDF3( 729, 2236, 3495) }, + { CDF3(30163, 32474, 32684) }, + { CDF3(18304, 30464, 32000) }, + { CDF3(11443, 26526, 29647) }, + { CDF3( 6007, 15292, 21299) }, + { CDF3( 2234, 6703, 8937) }, + { CDF3(30954, 32177, 32571) }, + { CDF3(17363, 29562, 31076) }, + { CDF3( 9686, 22464, 27410) }, + { CDF3( 8192, 16384, 21390) }, + { CDF3( 1755, 8046, 11264) }, + { CDF3(31168, 32734, 32748) }, + { CDF3(22486, 31441, 32471) }, + { CDF3(12833, 25627, 29738) }, + { CDF3( 6980, 17379, 23122) }, + { CDF3( 3111, 8887, 13479) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, + }, .dc_sign = { + { { CDF1(16000) }, { CDF1(13056) }, { CDF1(18816) } }, + { { CDF1(15232) }, { CDF1(12928) }, { CDF1(17280) } }, + }, .br_tok = { + { + { + { CDF3(14298, 20718, 24174) }, + { CDF3(12536, 19601, 23789) }, + { CDF3( 8712, 15051, 19503) }, + { CDF3( 6170, 11327, 15434) }, + { CDF3( 4742, 8926, 12538) }, + { CDF3( 3803, 7317, 10546) }, + { CDF3( 1696, 3317, 4871) }, + { CDF3(14392, 19951, 22756) }, + { CDF3(15978, 23218, 26818) }, + { CDF3(12187, 19474, 23889) }, + { CDF3( 9176, 15640, 20259) }, + { CDF3( 7068, 12655, 17028) }, + { CDF3( 5656, 10442, 14472) }, + { CDF3( 2580, 4992, 7244) }, + { CDF3(12136, 18049, 21426) }, + { CDF3(13784, 20721, 24481) }, + { CDF3(10836, 17621, 21900) }, + { CDF3( 8372, 14444, 18847) }, + { CDF3( 6523, 11779, 16000) }, + { CDF3( 5337, 9898, 13760) }, + { CDF3( 3034, 5860, 8462) }, + }, { + { CDF3(15967, 22905, 26286) }, + { CDF3(13534, 20654, 24579) }, + { CDF3( 9504, 16092, 20535) }, + { CDF3( 6975, 12568, 16903) }, + { CDF3( 5364, 10091, 14020) }, + { CDF3( 4357, 8370, 11857) }, + { CDF3( 2506, 4934, 7218) }, + { CDF3(23032, 28815, 30936) }, + { CDF3(19540, 26704, 29719) }, + { CDF3(15158, 22969, 27097) }, + { CDF3(11408, 18865, 23650) }, + { CDF3( 8885, 15448, 20250) }, + { CDF3( 7108, 12853, 17416) }, + { CDF3( 4231, 8041, 11480) }, + { CDF3(19823, 26490, 29156) }, + { CDF3(18890, 25929, 28932) }, + { CDF3(15660, 23491, 27433) }, + { CDF3(12147, 19776, 24488) }, + { CDF3( 9728, 16774, 21649) }, + { CDF3( 7919, 14277, 19066) }, + { CDF3( 5440, 10170, 14185) }, + }, + }, { + { + { CDF3(14406, 20862, 24414) }, + { CDF3(11824, 18907, 23109) }, + { CDF3( 8257, 14393, 18803) }, + { CDF3( 5860, 10747, 14778) }, + { CDF3( 4475, 8486, 11984) }, + { CDF3( 3606, 6954, 10043) }, + { CDF3( 1736, 3410, 5048) }, + { CDF3(14430, 20046, 22882) }, + { CDF3(15593, 22899, 26709) }, + { CDF3(12102, 19368, 23811) }, + { CDF3( 9059, 15584, 20262) }, + { CDF3( 6999, 12603, 17048) }, + { CDF3( 5684, 10497, 14553) }, + { CDF3( 2822, 5438, 7862) }, + { CDF3(15785, 21585, 24359) }, + { CDF3(18347, 25229, 28266) }, + { CDF3(14974, 22487, 26389) }, + { CDF3(11423, 18681, 23271) }, + { CDF3( 8863, 15350, 20008) }, + { CDF3( 7153, 12852, 17278) }, + { CDF3( 3707, 7036, 9982) }, + }, { + { CDF3(15460, 21696, 25469) }, + { CDF3(12170, 19249, 23191) }, + { CDF3( 8723, 15027, 19332) }, + { CDF3( 6428, 11704, 15874) }, + { CDF3( 4922, 9292, 13052) }, + { CDF3( 4139, 7695, 11010) }, + { CDF3( 2291, 4508, 6598) }, + { CDF3(19856, 26920, 29828) }, + { CDF3(17923, 25289, 28792) }, + { CDF3(14278, 21968, 26297) }, + { CDF3(10910, 18136, 22950) }, + { CDF3( 8423, 14815, 19627) }, + { CDF3( 6771, 12283, 16774) }, + { CDF3( 4074, 7750, 11081) }, + { CDF3(19852, 26074, 28672) }, + { CDF3(19371, 26110, 28989) }, + { CDF3(16265, 23873, 27663) }, + { CDF3(12758, 20378, 24952) }, + { CDF3(10095, 17098, 21961) }, + { CDF3( 8250, 14628, 19451) }, + { CDF3( 5205, 9745, 13622) }, + }, + }, { + { + { CDF3(10563, 16233, 19763) }, + { CDF3( 9794, 16022, 19804) }, + { CDF3( 6750, 11945, 15759) }, + { CDF3( 4963, 9186, 12752) }, + { CDF3( 3845, 7435, 10627) }, + { CDF3( 3051, 6085, 8834) }, + { CDF3( 1311, 2596, 3830) }, + { CDF3(11246, 16404, 19689) }, + { CDF3(12315, 18911, 22731) }, + { CDF3(10557, 17095, 21289) }, + { CDF3( 8136, 14006, 18249) }, + { CDF3( 6348, 11474, 15565) }, + { CDF3( 5196, 9655, 13400) }, + { CDF3( 2349, 4526, 6587) }, + { CDF3(13337, 18730, 21569) }, + { CDF3(19306, 26071, 28882) }, + { CDF3(15952, 23540, 27254) }, + { CDF3(12409, 19934, 24430) }, + { CDF3( 9760, 16706, 21389) }, + { CDF3( 8004, 14220, 18818) }, + { CDF3( 4138, 7794, 10961) }, + }, { + { CDF3(10870, 16684, 20949) }, + { CDF3( 9664, 15230, 18680) }, + { CDF3( 6886, 12109, 15408) }, + { CDF3( 4825, 8900, 12305) }, + { CDF3( 3630, 7162, 10314) }, + { CDF3( 3036, 6429, 9387) }, + { CDF3( 1671, 3296, 4940) }, + { CDF3(13819, 19159, 23026) }, + { CDF3(11984, 19108, 23120) }, + { CDF3(10690, 17210, 21663) }, + { CDF3( 7984, 14154, 18333) }, + { CDF3( 6868, 12294, 16124) }, + { CDF3( 5274, 8994, 12868) }, + { CDF3( 2988, 5771, 8424) }, + { CDF3(19736, 26647, 29141) }, + { CDF3(18933, 26070, 28984) }, + { CDF3(15779, 23048, 27200) }, + { CDF3(12638, 20061, 24532) }, + { CDF3(10692, 17545, 22220) }, + { CDF3( 9217, 15251, 20054) }, + { CDF3( 5078, 9284, 12594) }, + }, + }, { + { + { CDF3( 2331, 3662, 5244) }, + { CDF3( 2891, 4771, 6145) }, + { CDF3( 4598, 7623, 9729) }, + { CDF3( 3520, 6845, 9199) }, + { CDF3( 3417, 6119, 9324) }, + { CDF3( 2601, 5412, 7385) }, + { CDF3( 600, 1173, 1744) }, + { CDF3( 7672, 13286, 17469) }, + { CDF3( 4232, 7792, 10793) }, + { CDF3( 2915, 5317, 7397) }, + { CDF3( 2318, 4356, 6152) }, + { CDF3( 2127, 4000, 5554) }, + { CDF3( 1850, 3478, 5275) }, + { CDF3( 977, 1933, 2843) }, + { CDF3(18280, 24387, 27989) }, + { CDF3(15852, 22671, 26185) }, + { CDF3(13845, 20951, 24789) }, + { CDF3(11055, 17966, 22129) }, + { CDF3( 9138, 15422, 19801) }, + { CDF3( 7454, 13145, 17456) }, + { CDF3( 3370, 6393, 9013) }, + }, { + { CDF3( 5842, 9229, 10838) }, + { CDF3( 2313, 3491, 4276) }, + { CDF3( 2998, 6104, 7496) }, + { CDF3( 2420, 7447, 9868) }, + { CDF3( 3034, 8495, 10923) }, + { CDF3( 4076, 8937, 10975) }, + { CDF3( 1086, 2370, 3299) }, + { CDF3( 9714, 17254, 20444) }, + { CDF3( 8543, 13698, 17123) }, + { CDF3( 4918, 9007, 11910) }, + { CDF3( 4129, 7532, 10553) }, + { CDF3( 2364, 5533, 8058) }, + { CDF3( 1834, 3546, 5563) }, + { CDF3( 1473, 2908, 4133) }, + { CDF3(15405, 21193, 25619) }, + { CDF3(15691, 21952, 26561) }, + { CDF3(12962, 19194, 24165) }, + { CDF3(10272, 17855, 22129) }, + { CDF3( 8588, 15270, 20718) }, + { CDF3( 8682, 14669, 19500) }, + { CDF3( 4870, 9636, 13205) }, + }, + }, + }, + }, [1] = { + .skip = { + { + { CDF1(30371) }, { CDF1( 7570) }, { CDF1(13155) }, + { CDF1(20751) }, { CDF1(20969) }, { CDF1(27067) }, + { CDF1(32013) }, { CDF1( 5495) }, { CDF1(17942) }, + { CDF1(28280) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(31782) }, { CDF1( 1836) }, { CDF1(10689) }, + { CDF1(17604) }, { CDF1(21622) }, { CDF1(27518) }, + { CDF1(32399) }, { CDF1( 4419) }, { CDF1(16294) }, + { CDF1(28345) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(31901) }, { CDF1(10311) }, { CDF1(18047) }, + { CDF1(24806) }, { CDF1(23288) }, { CDF1(27914) }, + { CDF1(32296) }, { CDF1( 4215) }, { CDF1(15756) }, + { CDF1(28341) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(26726) }, { CDF1( 1045) }, { CDF1(11703) }, + { CDF1(20590) }, { CDF1(18554) }, { CDF1(25970) }, + { CDF1(31938) }, { CDF1( 5583) }, { CDF1(21313) }, + { CDF1(29390) }, { CDF1( 641) }, { CDF1(22265) }, + { CDF1(31452) }, + }, { + { CDF1(26584) }, { CDF1( 188) }, { CDF1( 8847) }, + { CDF1(24519) }, { CDF1(22938) }, { CDF1(30583) }, + { CDF1(32608) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, + }, .eob_bin_16 = { + { + { CDF4( 2125, 2551, 5165, 8946) }, + { CDF4( 513, 765, 1859, 6339) }, + }, { + { CDF4( 7637, 9498, 14259, 19108) }, + { CDF4( 2497, 4096, 8866, 16993) }, + }, + }, .eob_bin_32 = { + { + { CDF5( 989, 1249, 2019, 4151, 10785) }, + { CDF5( 313, 441, 1099, 2917, 8562) }, + }, { + { CDF5( 8394, 10352, 13932, 18855, 26014) }, + { CDF5( 2578, 4124, 8181, 13670, 24234) }, + }, + }, .eob_bin_64 = { + { + { CDF6( 1260, 1446, 2253, 3712, 6652, 13369) }, + { CDF6( 401, 605, 1029, 2563, 5845, 12626) }, + }, { + { CDF6( 8609, 10612, 14624, 18714, 22614, 29024) }, + { CDF6( 1923, 3127, 5867, 9703, 14277, 27100) }, + }, + }, .eob_bin_128 = { + { + { CDF7( 685, 933, 1488, 2714, 4766, 8562, 19254) }, + { CDF7( 217, 352, 618, 2303, 5261, 9969, 17472) }, + }, { + { CDF7( 8045, 11200, 15497, 19595, 23948, 27408, 30938) }, + { CDF7( 2310, 4160, 7471, 14997, 17931, 20768, 30240) }, + }, + }, .eob_bin_256 = { + { + { CDF8( 1448, 2109, 4151, 6263, + 9329, 13260, 17944, 23300) }, + { CDF8( 399, 1019, 1749, 3038, + 10444, 15546, 22739, 27294) }, + }, { + { CDF8( 6402, 8148, 12623, 15072, + 18728, 22847, 26447, 29377) }, + { CDF8( 1674, 3252, 5734, 10159, + 22397, 23802, 24821, 30940) }, + }, + }, .eob_bin_512 = { + { CDF9( 1230, 2278, 5035, 7776, 11871, + 15346, 19590, 24584, 28749) }, + { CDF9( 7265, 9979, 15819, 19250, 21780, + 23846, 26478, 28396, 31811) }, + }, .eob_bin_1024 = { + { CDF10( 696, 948, 3145, 5702, 9706, + 13217, 17851, 21856, 25692, 28034) }, + { CDF10( 2672, 3591, 9330, 17084, 22725, + 24284, 26527, 28027, 28377, 30876) }, + }, .eob_hi_bit = { + { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(17471) }, + { CDF1(20223) }, { CDF1(11357) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20335) }, + { CDF1(21667) }, { CDF1(14818) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20430) }, + { CDF1(20662) }, { CDF1(15367) }, { CDF1(16970) }, + { CDF1(14657) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(22117) }, + { CDF1(22028) }, { CDF1(18650) }, { CDF1(16042) }, + { CDF1(15885) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(22409) }, + { CDF1(21012) }, { CDF1(15650) }, { CDF1(17395) }, + { CDF1(15469) }, { CDF1(20205) }, { CDF1(19511) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(24220) }, + { CDF1(22480) }, { CDF1(17737) }, { CDF1(18916) }, + { CDF1(19268) }, { CDF1(18412) }, { CDF1(18844) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(25991) }, + { CDF1(20314) }, { CDF1(17731) }, { CDF1(19678) }, + { CDF1(18649) }, { CDF1(17307) }, { CDF1(21798) }, + { CDF1(17549) }, { CDF1(15630) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(26585) }, + { CDF1(21469) }, { CDF1(20432) }, { CDF1(17735) }, + { CDF1(19280) }, { CDF1(15235) }, { CDF1(20297) }, + { CDF1(22471) }, { CDF1(28997) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(26605) }, + { CDF1(11304) }, { CDF1(16726) }, { CDF1(16560) }, + { CDF1(20866) }, { CDF1(23524) }, { CDF1(19878) }, + { CDF1(13469) }, { CDF1(23084) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, + }, .eob_base_tok = { + { + { + { CDF2(17560, 29888) }, { CDF2(29671, 31549) }, + { CDF2(31007, 32056) }, { CDF2(27286, 30006) }, + }, { + { CDF2(26594, 31212) }, { CDF2(31208, 32582) }, + { CDF2(31835, 32637) }, { CDF2(30595, 32206) }, + }, + }, { + { + { CDF2(15239, 29932) }, { CDF2(31315, 32095) }, + { CDF2(32130, 32434) }, { CDF2(30864, 31996) }, + }, { + { CDF2(26279, 30968) }, { CDF2(31142, 32495) }, + { CDF2(31713, 32540) }, { CDF2(31929, 32594) }, + }, + }, { + { + { CDF2( 2644, 25198) }, { CDF2(32038, 32451) }, + { CDF2(32639, 32695) }, { CDF2(32166, 32518) }, + }, { + { CDF2(17187, 27668) }, { CDF2(31714, 32550) }, + { CDF2(32283, 32678) }, { CDF2(31930, 32563) }, + }, + }, { + { + { CDF2( 1044, 2257) }, { CDF2(30755, 31923) }, + { CDF2(32208, 32693) }, { CDF2(32244, 32615) }, + }, { + { CDF2(21317, 26207) }, { CDF2(29133, 30868) }, + { CDF2(29311, 31231) }, { CDF2(29657, 31087) }, + }, + }, { + { + { CDF2( 478, 1834) }, { CDF2(31005, 31987) }, + { CDF2(32317, 32724) }, { CDF2(30865, 32648) }, + }, { + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + }, + }, + }, .base_tok = { + { + { + { CDF3( 6041, 11854, 15927) }, + { CDF3(20326, 30905, 32251) }, + { CDF3(14164, 26831, 30725) }, + { CDF3( 9760, 20647, 26585) }, + { CDF3( 6416, 14953, 21219) }, + { CDF3( 2966, 7151, 10891) }, + { CDF3(23567, 31374, 32254) }, + { CDF3(14978, 27416, 30946) }, + { CDF3( 9434, 20225, 26254) }, + { CDF3( 6658, 14558, 20535) }, + { CDF3( 3916, 8677, 12989) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(18088, 29545, 31587) }, + { CDF3(13062, 25843, 30073) }, + { CDF3( 8940, 16827, 22251) }, + { CDF3( 7654, 13220, 17973) }, + { CDF3( 5733, 10316, 14456) }, + { CDF3(22879, 31388, 32114) }, + { CDF3(15215, 27993, 30955) }, + { CDF3( 9397, 19445, 24978) }, + { CDF3( 3442, 9813, 15344) }, + { CDF3( 1368, 3936, 6532) }, + { CDF3(25494, 32033, 32406) }, + { CDF3(16772, 27963, 30718) }, + { CDF3( 9419, 18165, 23260) }, + { CDF3( 2677, 7501, 11797) }, + { CDF3( 1516, 4344, 7170) }, + { CDF3(26556, 31454, 32101) }, + { CDF3(17128, 27035, 30108) }, + { CDF3( 8324, 15344, 20249) }, + { CDF3( 1903, 5696, 9469) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 8455, 19003, 24368) }, + { CDF3(23563, 32021, 32604) }, + { CDF3(16237, 29446, 31935) }, + { CDF3(10724, 23999, 29358) }, + { CDF3( 6725, 17528, 24416) }, + { CDF3( 3927, 10927, 16825) }, + { CDF3(26313, 32288, 32634) }, + { CDF3(17430, 30095, 32095) }, + { CDF3(11116, 24606, 29679) }, + { CDF3( 7195, 18384, 25269) }, + { CDF3( 4726, 12852, 19315) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(22822, 31648, 32483) }, + { CDF3(16724, 29633, 31929) }, + { CDF3(10261, 23033, 28725) }, + { CDF3( 7029, 17840, 24528) }, + { CDF3( 4867, 13886, 21502) }, + { CDF3(25298, 31892, 32491) }, + { CDF3(17809, 29330, 31512) }, + { CDF3( 9668, 21329, 26579) }, + { CDF3( 4774, 12956, 18976) }, + { CDF3( 2322, 7030, 11540) }, + { CDF3(25472, 31920, 32543) }, + { CDF3(17957, 29387, 31632) }, + { CDF3( 9196, 20593, 26400) }, + { CDF3( 4680, 12705, 19202) }, + { CDF3( 2917, 8456, 13436) }, + { CDF3(26471, 32059, 32574) }, + { CDF3(18458, 29783, 31909) }, + { CDF3( 8400, 19464, 25956) }, + { CDF3( 3812, 10973, 17206) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 6779, 13743, 17678) }, + { CDF3(24806, 31797, 32457) }, + { CDF3(17616, 29047, 31372) }, + { CDF3(11063, 23175, 28003) }, + { CDF3( 6521, 16110, 22324) }, + { CDF3( 2764, 7504, 11654) }, + { CDF3(25266, 32367, 32637) }, + { CDF3(19054, 30553, 32175) }, + { CDF3(12139, 25212, 29807) }, + { CDF3( 7311, 18162, 24704) }, + { CDF3( 3397, 9164, 14074) }, + { CDF3(25988, 32208, 32522) }, + { CDF3(16253, 28912, 31526) }, + { CDF3( 9151, 21387, 27372) }, + { CDF3( 5688, 14915, 21496) }, + { CDF3( 2717, 7627, 12004) }, + { CDF3(23144, 31855, 32443) }, + { CDF3(16070, 28491, 31325) }, + { CDF3( 8702, 20467, 26517) }, + { CDF3( 5243, 13956, 20367) }, + { CDF3( 2621, 7335, 11567) }, + { CDF3(26636, 32340, 32630) }, + { CDF3(19990, 31050, 32341) }, + { CDF3(13243, 26105, 30315) }, + { CDF3( 8588, 19521, 25918) }, + { CDF3( 4717, 11585, 17304) }, + { CDF3(25844, 32292, 32582) }, + { CDF3(19090, 30635, 32097) }, + { CDF3(11963, 24546, 28939) }, + { CDF3( 6218, 16087, 22354) }, + { CDF3( 2340, 6608, 10426) }, + { CDF3(28046, 32576, 32694) }, + { CDF3(21178, 31313, 32296) }, + { CDF3(13486, 26184, 29870) }, + { CDF3( 7149, 17871, 23723) }, + { CDF3( 2833, 7958, 12259) }, + { CDF3(27710, 32528, 32686) }, + { CDF3(20674, 31076, 32268) }, + { CDF3(12413, 24955, 29243) }, + { CDF3( 6676, 16927, 23097) }, + { CDF3( 2966, 8333, 12919) }, + }, { + { CDF3( 8639, 19339, 24429) }, + { CDF3(24404, 31837, 32525) }, + { CDF3(16997, 29425, 31784) }, + { CDF3(11253, 24234, 29149) }, + { CDF3( 6751, 17394, 24028) }, + { CDF3( 3490, 9830, 15191) }, + { CDF3(26283, 32471, 32714) }, + { CDF3(19599, 31168, 32442) }, + { CDF3(13146, 26954, 30893) }, + { CDF3( 8214, 20588, 26890) }, + { CDF3( 4699, 13081, 19300) }, + { CDF3(28212, 32458, 32669) }, + { CDF3(18594, 30316, 32100) }, + { CDF3(11219, 24408, 29234) }, + { CDF3( 6865, 17656, 24149) }, + { CDF3( 3678, 10362, 16006) }, + { CDF3(25825, 32136, 32616) }, + { CDF3(17313, 29853, 32021) }, + { CDF3(11197, 24471, 29472) }, + { CDF3( 6947, 17781, 24405) }, + { CDF3( 3768, 10660, 16261) }, + { CDF3(27352, 32500, 32706) }, + { CDF3(20850, 31468, 32469) }, + { CDF3(14021, 27707, 31133) }, + { CDF3( 8964, 21748, 27838) }, + { CDF3( 5437, 14665, 21187) }, + { CDF3(26304, 32492, 32698) }, + { CDF3(20409, 31380, 32385) }, + { CDF3(13682, 27222, 30632) }, + { CDF3( 8974, 21236, 26685) }, + { CDF3( 4234, 11665, 16934) }, + { CDF3(26273, 32357, 32711) }, + { CDF3(20672, 31242, 32441) }, + { CDF3(14172, 27254, 30902) }, + { CDF3( 9870, 21898, 27275) }, + { CDF3( 5164, 13506, 19270) }, + { CDF3(26725, 32459, 32728) }, + { CDF3(20991, 31442, 32527) }, + { CDF3(13071, 26434, 30811) }, + { CDF3( 8184, 20090, 26742) }, + { CDF3( 4803, 13255, 19895) }, + }, + }, { + { + { CDF3( 7555, 14942, 18501) }, + { CDF3(24410, 31178, 32287) }, + { CDF3(14394, 26738, 30253) }, + { CDF3( 8413, 19554, 25195) }, + { CDF3( 4766, 12924, 18785) }, + { CDF3( 2029, 5806, 9207) }, + { CDF3(26776, 32364, 32663) }, + { CDF3(18732, 29967, 31931) }, + { CDF3(11005, 23786, 28852) }, + { CDF3( 6466, 16909, 23510) }, + { CDF3( 3044, 8638, 13419) }, + { CDF3(29208, 32582, 32704) }, + { CDF3(20068, 30857, 32208) }, + { CDF3(12003, 25085, 29595) }, + { CDF3( 6947, 17750, 24189) }, + { CDF3( 3245, 9103, 14007) }, + { CDF3(27359, 32465, 32669) }, + { CDF3(19421, 30614, 32174) }, + { CDF3(11915, 25010, 29579) }, + { CDF3( 6950, 17676, 24074) }, + { CDF3( 3007, 8473, 13096) }, + { CDF3(29002, 32676, 32735) }, + { CDF3(22102, 31849, 32576) }, + { CDF3(14408, 28009, 31405) }, + { CDF3( 9027, 21679, 27931) }, + { CDF3( 4694, 12678, 18748) }, + { CDF3(28216, 32528, 32682) }, + { CDF3(20849, 31264, 32318) }, + { CDF3(12756, 25815, 29751) }, + { CDF3( 7565, 18801, 24923) }, + { CDF3( 3509, 9533, 14477) }, + { CDF3(30133, 32687, 32739) }, + { CDF3(23063, 31910, 32515) }, + { CDF3(14588, 28051, 31132) }, + { CDF3( 9085, 21649, 27457) }, + { CDF3( 4261, 11654, 17264) }, + { CDF3(29518, 32691, 32748) }, + { CDF3(22451, 31959, 32613) }, + { CDF3(14864, 28722, 31700) }, + { CDF3( 9695, 22964, 28716) }, + { CDF3( 4932, 13358, 19502) }, + }, { + { CDF3( 6465, 16958, 21688) }, + { CDF3(25199, 31514, 32360) }, + { CDF3(14774, 27149, 30607) }, + { CDF3( 9257, 21438, 26972) }, + { CDF3( 5723, 15183, 21882) }, + { CDF3( 3150, 8879, 13731) }, + { CDF3(26989, 32262, 32682) }, + { CDF3(17396, 29937, 32085) }, + { CDF3(11387, 24901, 29784) }, + { CDF3( 7289, 18821, 25548) }, + { CDF3( 3734, 10577, 16086) }, + { CDF3(29728, 32501, 32695) }, + { CDF3(17431, 29701, 31903) }, + { CDF3( 9921, 22826, 28300) }, + { CDF3( 5896, 15434, 22068) }, + { CDF3( 3430, 9646, 14757) }, + { CDF3(28614, 32511, 32705) }, + { CDF3(19364, 30638, 32263) }, + { CDF3(13129, 26254, 30402) }, + { CDF3( 8754, 20484, 26440) }, + { CDF3( 4378, 11607, 17110) }, + { CDF3(30292, 32671, 32744) }, + { CDF3(21780, 31603, 32501) }, + { CDF3(14314, 27829, 31291) }, + { CDF3( 9611, 22327, 28263) }, + { CDF3( 4890, 13087, 19065) }, + { CDF3(25862, 32567, 32733) }, + { CDF3(20794, 32050, 32567) }, + { CDF3(17243, 30625, 32254) }, + { CDF3(13283, 27628, 31474) }, + { CDF3( 9669, 22532, 28918) }, + { CDF3(27435, 32697, 32748) }, + { CDF3(24922, 32390, 32714) }, + { CDF3(21449, 31504, 32536) }, + { CDF3(16392, 29729, 31832) }, + { CDF3(11692, 24884, 29076) }, + { CDF3(24193, 32290, 32735) }, + { CDF3(18909, 31104, 32563) }, + { CDF3(12236, 26841, 31403) }, + { CDF3( 8171, 21840, 29082) }, + { CDF3( 7224, 17280, 25275) }, + }, + }, { + { + { CDF3( 3078, 6839, 9890) }, + { CDF3(13837, 20450, 24479) }, + { CDF3( 5914, 14222, 19328) }, + { CDF3( 3866, 10267, 14762) }, + { CDF3( 2612, 7208, 11042) }, + { CDF3( 1067, 2991, 4776) }, + { CDF3(25817, 31646, 32529) }, + { CDF3(13708, 26338, 30385) }, + { CDF3( 7328, 18585, 24870) }, + { CDF3( 4691, 13080, 19276) }, + { CDF3( 1825, 5253, 8352) }, + { CDF3(29386, 32315, 32624) }, + { CDF3(17160, 29001, 31360) }, + { CDF3( 9602, 21862, 27396) }, + { CDF3( 5915, 15772, 22148) }, + { CDF3( 2786, 7779, 12047) }, + { CDF3(29246, 32450, 32663) }, + { CDF3(18696, 29929, 31818) }, + { CDF3(10510, 23369, 28560) }, + { CDF3( 6229, 16499, 23125) }, + { CDF3( 2608, 7448, 11705) }, + { CDF3(30753, 32710, 32748) }, + { CDF3(21638, 31487, 32503) }, + { CDF3(12937, 26854, 30870) }, + { CDF3( 8182, 20596, 26970) }, + { CDF3( 3637, 10269, 15497) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 5244, 12150, 16906) }, + { CDF3(20486, 26858, 29701) }, + { CDF3( 7756, 18317, 23735) }, + { CDF3( 3452, 9256, 13146) }, + { CDF3( 2020, 5206, 8229) }, + { CDF3( 1801, 4993, 7903) }, + { CDF3(27051, 31858, 32531) }, + { CDF3(15988, 27531, 30619) }, + { CDF3( 9188, 21484, 26719) }, + { CDF3( 6273, 17186, 23800) }, + { CDF3( 3108, 9355, 14764) }, + { CDF3(31076, 32520, 32680) }, + { CDF3(18119, 30037, 31850) }, + { CDF3(10244, 22969, 27472) }, + { CDF3( 4692, 14077, 19273) }, + { CDF3( 3694, 11677, 17556) }, + { CDF3(30060, 32581, 32720) }, + { CDF3(21011, 30775, 32120) }, + { CDF3(11931, 24820, 29289) }, + { CDF3( 7119, 17662, 24356) }, + { CDF3( 3833, 10706, 16304) }, + { CDF3(31954, 32731, 32748) }, + { CDF3(23913, 31724, 32489) }, + { CDF3(15520, 28060, 31286) }, + { CDF3(11517, 23008, 28571) }, + { CDF3( 6193, 14508, 20629) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 1035, 2807, 4156) }, + { CDF3(13162, 18138, 20939) }, + { CDF3( 2696, 6633, 8755) }, + { CDF3( 1373, 4161, 6853) }, + { CDF3( 1099, 2746, 4716) }, + { CDF3( 340, 1021, 1599) }, + { CDF3(22826, 30419, 32135) }, + { CDF3(10395, 21762, 26942) }, + { CDF3( 4726, 12407, 17361) }, + { CDF3( 2447, 7080, 10593) }, + { CDF3( 1227, 3717, 6011) }, + { CDF3(28156, 31424, 31934) }, + { CDF3(16915, 27754, 30373) }, + { CDF3( 9148, 20990, 26431) }, + { CDF3( 5950, 15515, 21148) }, + { CDF3( 2492, 7327, 11526) }, + { CDF3(30602, 32477, 32670) }, + { CDF3(20026, 29955, 31568) }, + { CDF3(11220, 23628, 28105) }, + { CDF3( 6652, 17019, 22973) }, + { CDF3( 3064, 8536, 13043) }, + { CDF3(31769, 32724, 32748) }, + { CDF3(22230, 30887, 32373) }, + { CDF3(12234, 25079, 29731) }, + { CDF3( 7326, 18816, 25353) }, + { CDF3( 3933, 10907, 16616) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, + }, .dc_sign = { + { { CDF1(16000) }, { CDF1(13056) }, { CDF1(18816) } }, + { { CDF1(15232) }, { CDF1(12928) }, { CDF1(17280) } }, + }, .br_tok = { + { + { + { CDF3(14995, 21341, 24749) }, + { CDF3(13158, 20289, 24601) }, + { CDF3( 8941, 15326, 19876) }, + { CDF3( 6297, 11541, 15807) }, + { CDF3( 4817, 9029, 12776) }, + { CDF3( 3731, 7273, 10627) }, + { CDF3( 1847, 3617, 5354) }, + { CDF3(14472, 19659, 22343) }, + { CDF3(16806, 24162, 27533) }, + { CDF3(12900, 20404, 24713) }, + { CDF3( 9411, 16112, 20797) }, + { CDF3( 7056, 12697, 17148) }, + { CDF3( 5544, 10339, 14460) }, + { CDF3( 2954, 5704, 8319) }, + { CDF3(12464, 18071, 21354) }, + { CDF3(15482, 22528, 26034) }, + { CDF3(12070, 19269, 23624) }, + { CDF3( 8953, 15406, 20106) }, + { CDF3( 7027, 12730, 17220) }, + { CDF3( 5887, 10913, 15140) }, + { CDF3( 3793, 7278, 10447) }, + }, { + { CDF3(15571, 22232, 25749) }, + { CDF3(14506, 21575, 25374) }, + { CDF3(10189, 17089, 21569) }, + { CDF3( 7316, 13301, 17915) }, + { CDF3( 5783, 10912, 15190) }, + { CDF3( 4760, 9155, 13088) }, + { CDF3( 2993, 5966, 8774) }, + { CDF3(23424, 28903, 30778) }, + { CDF3(20775, 27666, 30290) }, + { CDF3(16474, 24410, 28299) }, + { CDF3(12471, 20180, 24987) }, + { CDF3( 9410, 16487, 21439) }, + { CDF3( 7536, 13614, 18529) }, + { CDF3( 5048, 9586, 13549) }, + { CDF3(21090, 27290, 29756) }, + { CDF3(20796, 27402, 30026) }, + { CDF3(17819, 25485, 28969) }, + { CDF3(13860, 21909, 26462) }, + { CDF3(11002, 18494, 23529) }, + { CDF3( 8953, 15929, 20897) }, + { CDF3( 6448, 11918, 16454) }, + }, + }, { + { + { CDF3(15999, 22208, 25449) }, + { CDF3(13050, 19988, 24122) }, + { CDF3( 8594, 14864, 19378) }, + { CDF3( 6033, 11079, 15238) }, + { CDF3( 4554, 8683, 12347) }, + { CDF3( 3672, 7139, 10337) }, + { CDF3( 1900, 3771, 5576) }, + { CDF3(15788, 21340, 23949) }, + { CDF3(16825, 24235, 27758) }, + { CDF3(12873, 20402, 24810) }, + { CDF3( 9590, 16363, 21094) }, + { CDF3( 7352, 13209, 17733) }, + { CDF3( 5960, 10989, 15184) }, + { CDF3( 3232, 6234, 9007) }, + { CDF3(15761, 20716, 23224) }, + { CDF3(19318, 25989, 28759) }, + { CDF3(15529, 23094, 26929) }, + { CDF3(11662, 18989, 23641) }, + { CDF3( 8955, 15568, 20366) }, + { CDF3( 7281, 13106, 17708) }, + { CDF3( 4248, 8059, 11440) }, + }, { + { CDF3(14899, 21217, 24503) }, + { CDF3(13519, 20283, 24047) }, + { CDF3( 9429, 15966, 20365) }, + { CDF3( 6700, 12355, 16652) }, + { CDF3( 5088, 9704, 13716) }, + { CDF3( 4243, 8154, 11731) }, + { CDF3( 2702, 5364, 7861) }, + { CDF3(22745, 28388, 30454) }, + { CDF3(20235, 27146, 29922) }, + { CDF3(15896, 23715, 27637) }, + { CDF3(11840, 19350, 24131) }, + { CDF3( 9122, 15932, 20880) }, + { CDF3( 7488, 13581, 18362) }, + { CDF3( 5114, 9568, 13370) }, + { CDF3(20845, 26553, 28932) }, + { CDF3(20981, 27372, 29884) }, + { CDF3(17781, 25335, 28785) }, + { CDF3(13760, 21708, 26297) }, + { CDF3(10975, 18415, 23365) }, + { CDF3( 9045, 15789, 20686) }, + { CDF3( 6130, 11199, 15423) }, + }, + }, { + { + { CDF3(13549, 19724, 23158) }, + { CDF3(11844, 18382, 22246) }, + { CDF3( 7919, 13619, 17773) }, + { CDF3( 5486, 10143, 13946) }, + { CDF3( 4166, 7983, 11324) }, + { CDF3( 3364, 6506, 9427) }, + { CDF3( 1598, 3160, 4674) }, + { CDF3(15281, 20979, 23781) }, + { CDF3(14939, 22119, 25952) }, + { CDF3(11363, 18407, 22812) }, + { CDF3( 8609, 14857, 19370) }, + { CDF3( 6737, 12184, 16480) }, + { CDF3( 5506, 10263, 14262) }, + { CDF3( 2990, 5786, 8380) }, + { CDF3(20249, 25253, 27417) }, + { CDF3(21070, 27518, 30001) }, + { CDF3(16854, 24469, 28074) }, + { CDF3(12864, 20486, 25000) }, + { CDF3( 9962, 16978, 21778) }, + { CDF3( 8074, 14338, 19048) }, + { CDF3( 4494, 8479, 11906) }, + }, { + { CDF3(13960, 19617, 22829) }, + { CDF3(11150, 17341, 21228) }, + { CDF3( 7150, 12964, 17190) }, + { CDF3( 5331, 10002, 13867) }, + { CDF3( 4167, 7744, 11057) }, + { CDF3( 3480, 6629, 9646) }, + { CDF3( 1883, 3784, 5686) }, + { CDF3(18752, 25660, 28912) }, + { CDF3(16968, 24586, 28030) }, + { CDF3(13520, 21055, 25313) }, + { CDF3(10453, 17626, 22280) }, + { CDF3( 8386, 14505, 19116) }, + { CDF3( 6742, 12595, 17008) }, + { CDF3( 4273, 8140, 11499) }, + { CDF3(22120, 27827, 30233) }, + { CDF3(20563, 27358, 29895) }, + { CDF3(17076, 24644, 28153) }, + { CDF3(13362, 20942, 25309) }, + { CDF3(10794, 17965, 22695) }, + { CDF3( 9014, 15652, 20319) }, + { CDF3( 5708, 10512, 14497) }, + }, + }, { + { + { CDF3( 5705, 10930, 15725) }, + { CDF3( 7946, 12765, 16115) }, + { CDF3( 6801, 12123, 16226) }, + { CDF3( 5462, 10135, 14200) }, + { CDF3( 4189, 8011, 11507) }, + { CDF3( 3191, 6229, 9408) }, + { CDF3( 1057, 2137, 3212) }, + { CDF3(10018, 17067, 21491) }, + { CDF3( 7380, 12582, 16453) }, + { CDF3( 6068, 10845, 14339) }, + { CDF3( 5098, 9198, 12555) }, + { CDF3( 4312, 8010, 11119) }, + { CDF3( 3700, 6966, 9781) }, + { CDF3( 1693, 3326, 4887) }, + { CDF3(18757, 24930, 27774) }, + { CDF3(17648, 24596, 27817) }, + { CDF3(14707, 22052, 26026) }, + { CDF3(11720, 18852, 23292) }, + { CDF3( 9357, 15952, 20525) }, + { CDF3( 7810, 13753, 18210) }, + { CDF3( 3879, 7333, 10328) }, + }, { + { CDF3( 8278, 13242, 15922) }, + { CDF3(10547, 15867, 18919) }, + { CDF3( 9106, 15842, 20609) }, + { CDF3( 6833, 13007, 17218) }, + { CDF3( 4811, 9712, 13923) }, + { CDF3( 3985, 7352, 11128) }, + { CDF3( 1688, 3458, 5262) }, + { CDF3(12951, 21861, 26510) }, + { CDF3( 9788, 16044, 20276) }, + { CDF3( 6309, 11244, 14870) }, + { CDF3( 5183, 9349, 12566) }, + { CDF3( 4389, 8229, 11492) }, + { CDF3( 3633, 6945, 10620) }, + { CDF3( 3600, 6847, 9907) }, + { CDF3(21748, 28137, 30255) }, + { CDF3(19436, 26581, 29560) }, + { CDF3(16359, 24201, 27953) }, + { CDF3(13961, 21693, 25871) }, + { CDF3(11544, 18686, 23322) }, + { CDF3( 9372, 16462, 20952) }, + { CDF3( 6138, 11210, 15390) }, + }, + }, + }, + }, [2] = { + .skip = { + { + { CDF1(29614) }, { CDF1( 9068) }, { CDF1(12924) }, + { CDF1(19538) }, { CDF1(17737) }, { CDF1(24619) }, + { CDF1(30642) }, { CDF1( 4119) }, { CDF1(16026) }, + { CDF1(25657) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(31957) }, { CDF1( 3230) }, { CDF1(11153) }, + { CDF1(18123) }, { CDF1(20143) }, { CDF1(26536) }, + { CDF1(31986) }, { CDF1( 3050) }, { CDF1(14603) }, + { CDF1(25155) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(32363) }, { CDF1(10692) }, { CDF1(19090) }, + { CDF1(24357) }, { CDF1(24442) }, { CDF1(28312) }, + { CDF1(32169) }, { CDF1( 3648) }, { CDF1(15690) }, + { CDF1(26815) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(30669) }, { CDF1( 3832) }, { CDF1(11663) }, + { CDF1(18889) }, { CDF1(19782) }, { CDF1(23313) }, + { CDF1(31330) }, { CDF1( 5124) }, { CDF1(18719) }, + { CDF1(28468) }, { CDF1( 3082) }, { CDF1(20982) }, + { CDF1(29443) }, + }, { + { CDF1(28573) }, { CDF1( 3183) }, { CDF1(17802) }, + { CDF1(25977) }, { CDF1(26677) }, { CDF1(27832) }, + { CDF1(32387) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, + }, .eob_bin_16 = { + { + { CDF4( 4016, 4897, 8881, 14968) }, + { CDF4( 716, 1105, 2646, 10056) }, + }, { + { CDF4(11139, 13270, 18241, 23566) }, + { CDF4( 3192, 5032, 10297, 19755) }, + }, + }, .eob_bin_32 = { + { + { CDF5( 2515, 3003, 4452, 8162, 16041) }, + { CDF5( 574, 821, 1836, 5089, 13128) }, + }, { + { CDF5(13468, 16303, 20361, 25105, 29281) }, + { CDF5( 3542, 5502, 10415, 16760, 25644) }, + }, + }, .eob_bin_64 = { + { + { CDF6( 2374, 2772, 4583, 7276, 12288, 19706) }, + { CDF6( 497, 810, 1315, 3000, 7004, 15641) }, + }, { + { CDF6(15050, 17126, 21410, 24886, 28156, 30726) }, + { CDF6( 4034, 6290, 10235, 14982, 21214, 28491) }, + }, + }, .eob_bin_128 = { + { + { CDF7( 1366, 1738, 2527, 5016, 9355, 15797, 24643) }, + { CDF7( 354, 558, 944, 2760, 7287, 14037, 21779) }, + }, { + { CDF7(13627, 16246, 20173, 24429, 27948, 30415, 31863) }, + { CDF7( 6275, 9889, 14769, 23164, 27988, 30493, 32272) }, + }, + }, .eob_bin_256 = { + { + { CDF8( 3089, 3920, 6038, 9460, + 14266, 19881, 25766, 29176) }, + { CDF8( 1084, 2358, 3488, 5122, + 11483, 18103, 26023, 29799) }, + }, { + { CDF8(11514, 13794, 17480, 20754, + 24361, 27378, 29492, 31277) }, + { CDF8( 6571, 9610, 15516, 21826, + 29092, 30829, 31842, 32708) }, + }, + }, .eob_bin_512 = { + { CDF9( 2624, 3936, 6480, 9686, 13979, + 17726, 23267, 28410, 31078) }, + { CDF9(12015, 14769, 19588, 22052, 24222, + 25812, 27300, 29219, 32114) }, + }, .eob_bin_1024 = { + { CDF10( 2784, 3831, 7041, 10521, 14847, + 18844, 23155, 26682, 29229, 31045) }, + { CDF10( 9577, 12466, 17739, 20750, 22061, + 23215, 24601, 25483, 25843, 32056) }, + }, .eob_hi_bit = { + { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(18983) }, + { CDF1(20512) }, { CDF1(14885) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20090) }, + { CDF1(19444) }, { CDF1(17286) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(19139) }, + { CDF1(21487) }, { CDF1(18959) }, { CDF1(20910) }, + { CDF1(19089) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20536) }, + { CDF1(20664) }, { CDF1(20625) }, { CDF1(19123) }, + { CDF1(14862) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(19833) }, + { CDF1(21502) }, { CDF1(17485) }, { CDF1(20267) }, + { CDF1(18353) }, { CDF1(23329) }, { CDF1(21478) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(22041) }, + { CDF1(23434) }, { CDF1(20001) }, { CDF1(20554) }, + { CDF1(20951) }, { CDF1(20145) }, { CDF1(15562) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(23312) }, + { CDF1(21607) }, { CDF1(16526) }, { CDF1(18957) }, + { CDF1(18034) }, { CDF1(18934) }, { CDF1(24247) }, + { CDF1(16921) }, { CDF1(17080) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(26579) }, + { CDF1(24910) }, { CDF1(18637) }, { CDF1(19800) }, + { CDF1(20388) }, { CDF1( 9887) }, { CDF1(15642) }, + { CDF1(30198) }, { CDF1(24721) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(26998) }, + { CDF1(16737) }, { CDF1(17838) }, { CDF1(18922) }, + { CDF1(19515) }, { CDF1(18636) }, { CDF1(17333) }, + { CDF1(15776) }, { CDF1(22658) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, + }, .eob_base_tok = { + { + { + { CDF2(20092, 30774) }, { CDF2(30695, 32020) }, + { CDF2(31131, 32103) }, { CDF2(28666, 30870) }, + }, { + { CDF2(27258, 31095) }, { CDF2(31804, 32623) }, + { CDF2(31763, 32528) }, { CDF2(31438, 32506) }, + }, + }, { + { + { CDF2(18049, 30489) }, { CDF2(31706, 32286) }, + { CDF2(32163, 32473) }, { CDF2(31550, 32184) }, + }, { + { CDF2(27116, 30842) }, { CDF2(31971, 32598) }, + { CDF2(32088, 32576) }, { CDF2(32067, 32664) }, + }, + }, { + { + { CDF2(12854, 29093) }, { CDF2(32272, 32558) }, + { CDF2(32667, 32729) }, { CDF2(32306, 32585) }, + }, { + { CDF2(25476, 30366) }, { CDF2(32169, 32687) }, + { CDF2(32479, 32689) }, { CDF2(31673, 32634) }, + }, + }, { + { + { CDF2( 2809, 19301) }, { CDF2(32205, 32622) }, + { CDF2(32338, 32730) }, { CDF2(31786, 32616) }, + }, { + { CDF2(22737, 29105) }, { CDF2(30810, 32362) }, + { CDF2(30014, 32627) }, { CDF2(30528, 32574) }, + }, + }, { + { + { CDF2( 935, 3382) }, { CDF2(30789, 31909) }, + { CDF2(32466, 32756) }, { CDF2(30860, 32513) }, + }, { + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + }, + }, + }, .base_tok = { + { + { + { CDF3( 8896, 16227, 20630) }, + { CDF3(23629, 31782, 32527) }, + { CDF3(15173, 27755, 31321) }, + { CDF3(10158, 21233, 27382) }, + { CDF3( 6420, 14857, 21558) }, + { CDF3( 3269, 8155, 12646) }, + { CDF3(24835, 32009, 32496) }, + { CDF3(16509, 28421, 31579) }, + { CDF3(10957, 21514, 27418) }, + { CDF3( 7881, 15930, 22096) }, + { CDF3( 5388, 10960, 15918) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(20745, 30773, 32093) }, + { CDF3(15200, 27221, 30861) }, + { CDF3(13032, 20873, 25667) }, + { CDF3(12285, 18663, 23494) }, + { CDF3(11563, 17481, 21489) }, + { CDF3(26260, 31982, 32320) }, + { CDF3(15397, 28083, 31100) }, + { CDF3( 9742, 19217, 24824) }, + { CDF3( 3261, 9629, 15362) }, + { CDF3( 1480, 4322, 7499) }, + { CDF3(27599, 32256, 32460) }, + { CDF3(16857, 27659, 30774) }, + { CDF3( 9551, 18290, 23748) }, + { CDF3( 3052, 8933, 14103) }, + { CDF3( 2021, 5910, 9787) }, + { CDF3(29005, 32015, 32392) }, + { CDF3(17677, 27694, 30863) }, + { CDF3( 9204, 17356, 23219) }, + { CDF3( 2403, 7516, 12814) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3(10808, 22056, 26896) }, + { CDF3(25739, 32313, 32676) }, + { CDF3(17288, 30203, 32221) }, + { CDF3(11359, 24878, 29896) }, + { CDF3( 6949, 17767, 24893) }, + { CDF3( 4287, 11796, 18071) }, + { CDF3(27880, 32521, 32705) }, + { CDF3(19038, 31004, 32414) }, + { CDF3(12564, 26345, 30768) }, + { CDF3( 8269, 19947, 26779) }, + { CDF3( 5674, 14657, 21674) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(25742, 32319, 32671) }, + { CDF3(19557, 31164, 32454) }, + { CDF3(13381, 26381, 30755) }, + { CDF3(10101, 21466, 26722) }, + { CDF3( 9209, 19650, 26825) }, + { CDF3(27107, 31917, 32432) }, + { CDF3(18056, 28893, 31203) }, + { CDF3(10200, 21434, 26764) }, + { CDF3( 4660, 12913, 19502) }, + { CDF3( 2368, 6930, 12504) }, + { CDF3(26960, 32158, 32613) }, + { CDF3(18628, 30005, 32031) }, + { CDF3(10233, 22442, 28232) }, + { CDF3( 5471, 14630, 21516) }, + { CDF3( 3235, 10767, 17109) }, + { CDF3(27696, 32440, 32692) }, + { CDF3(20032, 31167, 32438) }, + { CDF3( 8700, 21341, 28442) }, + { CDF3( 5662, 14831, 21795) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 9704, 17294, 21132) }, + { CDF3(26762, 32278, 32633) }, + { CDF3(18382, 29620, 31819) }, + { CDF3(10891, 23475, 28723) }, + { CDF3( 6358, 16583, 23309) }, + { CDF3( 3248, 9118, 14141) }, + { CDF3(27204, 32573, 32699) }, + { CDF3(19818, 30824, 32329) }, + { CDF3(11772, 25120, 30041) }, + { CDF3( 6995, 18033, 25039) }, + { CDF3( 3752, 10442, 16098) }, + { CDF3(27222, 32256, 32559) }, + { CDF3(15356, 28399, 31475) }, + { CDF3( 8821, 20635, 27057) }, + { CDF3( 5511, 14404, 21239) }, + { CDF3( 2935, 8222, 13051) }, + { CDF3(24875, 32120, 32529) }, + { CDF3(15233, 28265, 31445) }, + { CDF3( 8605, 20570, 26932) }, + { CDF3( 5431, 14413, 21196) }, + { CDF3( 2994, 8341, 13223) }, + { CDF3(28201, 32604, 32700) }, + { CDF3(21041, 31446, 32456) }, + { CDF3(13221, 26213, 30475) }, + { CDF3( 8255, 19385, 26037) }, + { CDF3( 4930, 12585, 18830) }, + { CDF3(28768, 32448, 32627) }, + { CDF3(19705, 30561, 32021) }, + { CDF3(11572, 23589, 28220) }, + { CDF3( 5532, 15034, 21446) }, + { CDF3( 2460, 7150, 11456) }, + { CDF3(29874, 32619, 32699) }, + { CDF3(21621, 31071, 32201) }, + { CDF3(12511, 24747, 28992) }, + { CDF3( 6281, 16395, 22748) }, + { CDF3( 3246, 9278, 14497) }, + { CDF3(29715, 32625, 32712) }, + { CDF3(20958, 31011, 32283) }, + { CDF3(11233, 23671, 28806) }, + { CDF3( 6012, 16128, 22868) }, + { CDF3( 3427, 9851, 15414) }, + }, { + { CDF3(11016, 22111, 26794) }, + { CDF3(25946, 32357, 32677) }, + { CDF3(17890, 30452, 32252) }, + { CDF3(11678, 25142, 29816) }, + { CDF3( 6720, 17534, 24584) }, + { CDF3( 4230, 11665, 17820) }, + { CDF3(28400, 32623, 32747) }, + { CDF3(21164, 31668, 32575) }, + { CDF3(13572, 27388, 31182) }, + { CDF3( 8234, 20750, 27358) }, + { CDF3( 5065, 14055, 20897) }, + { CDF3(28981, 32547, 32705) }, + { CDF3(18681, 30543, 32239) }, + { CDF3(10919, 24075, 29286) }, + { CDF3( 6431, 17199, 24077) }, + { CDF3( 3819, 10464, 16618) }, + { CDF3(26870, 32467, 32693) }, + { CDF3(19041, 30831, 32347) }, + { CDF3(11794, 25211, 30016) }, + { CDF3( 6888, 18019, 24970) }, + { CDF3( 4370, 12363, 18992) }, + { CDF3(29578, 32670, 32744) }, + { CDF3(23159, 32007, 32613) }, + { CDF3(15315, 28669, 31676) }, + { CDF3( 9298, 22607, 28782) }, + { CDF3( 6144, 15913, 22968) }, + { CDF3(28110, 32499, 32669) }, + { CDF3(21574, 30937, 32015) }, + { CDF3(12759, 24818, 28727) }, + { CDF3( 6545, 16761, 23042) }, + { CDF3( 3649, 10597, 16833) }, + { CDF3(28163, 32552, 32728) }, + { CDF3(22101, 31469, 32464) }, + { CDF3(13160, 25472, 30143) }, + { CDF3( 7303, 18684, 25468) }, + { CDF3( 5241, 13975, 20955) }, + { CDF3(28400, 32631, 32744) }, + { CDF3(22104, 31793, 32603) }, + { CDF3(13557, 26571, 30846) }, + { CDF3( 7749, 19861, 26675) }, + { CDF3( 4873, 14030, 21234) }, + }, + }, { + { + { CDF3( 9800, 17635, 21073) }, + { CDF3(26153, 31885, 32527) }, + { CDF3(15038, 27852, 31006) }, + { CDF3( 8718, 20564, 26486) }, + { CDF3( 5128, 14076, 20514) }, + { CDF3( 2636, 7566, 11925) }, + { CDF3(27551, 32504, 32701) }, + { CDF3(18310, 30054, 32100) }, + { CDF3(10211, 23420, 29082) }, + { CDF3( 6222, 16876, 23916) }, + { CDF3( 3462, 9954, 15498) }, + { CDF3(29991, 32633, 32721) }, + { CDF3(19883, 30751, 32201) }, + { CDF3(11141, 24184, 29285) }, + { CDF3( 6420, 16940, 23774) }, + { CDF3( 3392, 9753, 15118) }, + { CDF3(28465, 32616, 32712) }, + { CDF3(19850, 30702, 32244) }, + { CDF3(10983, 24024, 29223) }, + { CDF3( 6294, 16770, 23582) }, + { CDF3( 3244, 9283, 14509) }, + { CDF3(30023, 32717, 32748) }, + { CDF3(22940, 32032, 32626) }, + { CDF3(14282, 27928, 31473) }, + { CDF3( 8562, 21327, 27914) }, + { CDF3( 4846, 13393, 19919) }, + { CDF3(29981, 32590, 32695) }, + { CDF3(20465, 30963, 32166) }, + { CDF3(11479, 23579, 28195) }, + { CDF3( 5916, 15648, 22073) }, + { CDF3( 3031, 8605, 13398) }, + { CDF3(31146, 32691, 32739) }, + { CDF3(23106, 31724, 32444) }, + { CDF3(13783, 26738, 30439) }, + { CDF3( 7852, 19468, 25807) }, + { CDF3( 3860, 11124, 16853) }, + { CDF3(31014, 32724, 32748) }, + { CDF3(23629, 32109, 32628) }, + { CDF3(14747, 28115, 31403) }, + { CDF3( 8545, 21242, 27478) }, + { CDF3( 4574, 12781, 19067) }, + }, { + { CDF3( 9185, 19694, 24688) }, + { CDF3(26081, 31985, 32621) }, + { CDF3(16015, 29000, 31787) }, + { CDF3(10542, 23690, 29206) }, + { CDF3( 6732, 17945, 24677) }, + { CDF3( 3916, 11039, 16722) }, + { CDF3(28224, 32566, 32744) }, + { CDF3(19100, 31138, 32485) }, + { CDF3(12528, 26620, 30879) }, + { CDF3( 7741, 20277, 26885) }, + { CDF3( 4566, 12845, 18990) }, + { CDF3(29933, 32593, 32718) }, + { CDF3(17670, 30333, 32155) }, + { CDF3(10385, 23600, 28909) }, + { CDF3( 6243, 16236, 22407) }, + { CDF3( 3976, 10389, 16017) }, + { CDF3(28377, 32561, 32738) }, + { CDF3(19366, 31175, 32482) }, + { CDF3(13327, 27175, 31094) }, + { CDF3( 8258, 20769, 27143) }, + { CDF3( 4703, 13198, 19527) }, + { CDF3(31086, 32706, 32748) }, + { CDF3(22853, 31902, 32583) }, + { CDF3(14759, 28186, 31419) }, + { CDF3( 9284, 22382, 28348) }, + { CDF3( 5585, 15192, 21868) }, + { CDF3(28291, 32652, 32746) }, + { CDF3(19849, 32107, 32571) }, + { CDF3(14834, 26818, 29214) }, + { CDF3(10306, 22594, 28672) }, + { CDF3( 6615, 17384, 23384) }, + { CDF3(28947, 32604, 32745) }, + { CDF3(25625, 32289, 32646) }, + { CDF3(18758, 28672, 31403) }, + { CDF3(10017, 23430, 28523) }, + { CDF3( 6862, 15269, 22131) }, + { CDF3(23933, 32509, 32739) }, + { CDF3(19927, 31495, 32631) }, + { CDF3(11903, 26023, 30621) }, + { CDF3( 7026, 20094, 27252) }, + { CDF3( 5998, 18106, 24437) }, + }, + }, { + { + { CDF3( 4456, 11274, 15533) }, + { CDF3(21219, 29079, 31616) }, + { CDF3(11173, 23774, 28567) }, + { CDF3( 7282, 18293, 24263) }, + { CDF3( 4890, 13286, 19115) }, + { CDF3( 1890, 5508, 8659) }, + { CDF3(26651, 32136, 32647) }, + { CDF3(14630, 28254, 31455) }, + { CDF3( 8716, 21287, 27395) }, + { CDF3( 5615, 15331, 22008) }, + { CDF3( 2675, 7700, 12150) }, + { CDF3(29954, 32526, 32690) }, + { CDF3(16126, 28982, 31633) }, + { CDF3( 9030, 21361, 27352) }, + { CDF3( 5411, 14793, 21271) }, + { CDF3( 2943, 8422, 13163) }, + { CDF3(29539, 32601, 32730) }, + { CDF3(18125, 30385, 32201) }, + { CDF3(10422, 24090, 29468) }, + { CDF3( 6468, 17487, 24438) }, + { CDF3( 2970, 8653, 13531) }, + { CDF3(30912, 32715, 32748) }, + { CDF3(20666, 31373, 32497) }, + { CDF3(12509, 26640, 30917) }, + { CDF3( 8058, 20629, 27290) }, + { CDF3( 4231, 12006, 18052) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3(10202, 20633, 25484) }, + { CDF3(27336, 31445, 32352) }, + { CDF3(12420, 24384, 28552) }, + { CDF3( 7648, 18115, 23856) }, + { CDF3( 5662, 14341, 19902) }, + { CDF3( 3611, 10328, 15390) }, + { CDF3(30945, 32616, 32736) }, + { CDF3(18682, 30505, 32253) }, + { CDF3(11513, 25336, 30203) }, + { CDF3( 7449, 19452, 26148) }, + { CDF3( 4482, 13051, 18886) }, + { CDF3(32022, 32690, 32747) }, + { CDF3(18578, 30501, 32146) }, + { CDF3(11249, 23368, 28631) }, + { CDF3( 5645, 16958, 22158) }, + { CDF3( 5009, 11444, 16637) }, + { CDF3(31357, 32710, 32748) }, + { CDF3(21552, 31494, 32504) }, + { CDF3(13891, 27677, 31340) }, + { CDF3( 9051, 22098, 28172) }, + { CDF3( 5190, 13377, 19486) }, + { CDF3(32364, 32740, 32748) }, + { CDF3(24839, 31907, 32551) }, + { CDF3(17160, 28779, 31696) }, + { CDF3(12452, 24137, 29602) }, + { CDF3( 6165, 15389, 22477) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 2575, 7281, 11077) }, + { CDF3(14002, 20866, 25402) }, + { CDF3( 6343, 15056, 19658) }, + { CDF3( 4474, 11858, 17041) }, + { CDF3( 2865, 8299, 12534) }, + { CDF3( 1344, 3949, 6391) }, + { CDF3(24720, 31239, 32459) }, + { CDF3(12585, 25356, 29968) }, + { CDF3( 7181, 18246, 24444) }, + { CDF3( 5025, 13667, 19885) }, + { CDF3( 2521, 7304, 11605) }, + { CDF3(29908, 32252, 32584) }, + { CDF3(17421, 29156, 31575) }, + { CDF3( 9889, 22188, 27782) }, + { CDF3( 5878, 15647, 22123) }, + { CDF3( 2814, 8665, 13323) }, + { CDF3(30183, 32568, 32713) }, + { CDF3(18528, 30195, 32049) }, + { CDF3(10982, 24606, 29657) }, + { CDF3( 6957, 18165, 25231) }, + { CDF3( 3508, 10118, 15468) }, + { CDF3(31761, 32736, 32748) }, + { CDF3(21041, 31328, 32546) }, + { CDF3(12568, 26732, 31166) }, + { CDF3( 8052, 20720, 27733) }, + { CDF3( 4336, 12192, 18396) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, + }, .dc_sign = { + { { CDF1(16000) }, { CDF1(13056) }, { CDF1(18816) } }, + { { CDF1(15232) }, { CDF1(12928) }, { CDF1(17280) } }, + }, .br_tok = { + { + { + { CDF3(16138, 22223, 25509) }, + { CDF3(15347, 22430, 26332) }, + { CDF3( 9614, 16736, 21332) }, + { CDF3( 6600, 12275, 16907) }, + { CDF3( 4811, 9424, 13547) }, + { CDF3( 3748, 7809, 11420) }, + { CDF3( 2254, 4587, 6890) }, + { CDF3(15196, 20284, 23177) }, + { CDF3(18317, 25469, 28451) }, + { CDF3(13918, 21651, 25842) }, + { CDF3(10052, 17150, 21995) }, + { CDF3( 7499, 13630, 18587) }, + { CDF3( 6158, 11417, 16003) }, + { CDF3( 4014, 7785, 11252) }, + { CDF3(15048, 21067, 24384) }, + { CDF3(18202, 25346, 28553) }, + { CDF3(14302, 22019, 26356) }, + { CDF3(10839, 18139, 23166) }, + { CDF3( 8715, 15744, 20806) }, + { CDF3( 7536, 13576, 18544) }, + { CDF3( 5413, 10335, 14498) }, + }, { + { CDF3(17394, 24501, 27895) }, + { CDF3(15889, 23420, 27185) }, + { CDF3(11561, 19133, 23870) }, + { CDF3( 8285, 14812, 19844) }, + { CDF3( 6496, 12043, 16550) }, + { CDF3( 4771, 9574, 13677) }, + { CDF3( 3603, 6830, 10144) }, + { CDF3(21656, 27704, 30200) }, + { CDF3(21324, 27915, 30511) }, + { CDF3(17327, 25336, 28997) }, + { CDF3(13417, 21381, 26033) }, + { CDF3(10132, 17425, 22338) }, + { CDF3( 8580, 15016, 19633) }, + { CDF3( 5694, 11477, 16411) }, + { CDF3(24116, 29780, 31450) }, + { CDF3(23853, 29695, 31591) }, + { CDF3(20085, 27614, 30428) }, + { CDF3(15326, 24335, 28575) }, + { CDF3(11814, 19472, 24810) }, + { CDF3(10221, 18611, 24767) }, + { CDF3( 7689, 14558, 20321) }, + }, + }, { + { + { CDF3(16214, 22380, 25770) }, + { CDF3(14213, 21304, 25295) }, + { CDF3( 9213, 15823, 20455) }, + { CDF3( 6395, 11758, 16139) }, + { CDF3( 4779, 9187, 13066) }, + { CDF3( 3821, 7501, 10953) }, + { CDF3( 2293, 4567, 6795) }, + { CDF3(15859, 21283, 23820) }, + { CDF3(18404, 25602, 28726) }, + { CDF3(14325, 21980, 26206) }, + { CDF3(10669, 17937, 22720) }, + { CDF3( 8297, 14642, 19447) }, + { CDF3( 6746, 12389, 16893) }, + { CDF3( 4324, 8251, 11770) }, + { CDF3(16532, 21631, 24475) }, + { CDF3(20667, 27150, 29668) }, + { CDF3(16728, 24510, 28175) }, + { CDF3(12861, 20645, 25332) }, + { CDF3(10076, 17361, 22417) }, + { CDF3( 8395, 14940, 19963) }, + { CDF3( 5731, 10683, 14912) }, + }, { + { CDF3(14433, 21155, 24938) }, + { CDF3(14658, 21716, 25545) }, + { CDF3( 9923, 16824, 21557) }, + { CDF3( 6982, 13052, 17721) }, + { CDF3( 5419, 10503, 15050) }, + { CDF3( 4852, 9162, 13014) }, + { CDF3( 3271, 6395, 9630) }, + { CDF3(22210, 27833, 30109) }, + { CDF3(20750, 27368, 29821) }, + { CDF3(16894, 24828, 28573) }, + { CDF3(13247, 21276, 25757) }, + { CDF3(10038, 17265, 22563) }, + { CDF3( 8587, 14947, 20327) }, + { CDF3( 5645, 11371, 15252) }, + { CDF3(22027, 27526, 29714) }, + { CDF3(23098, 29146, 31221) }, + { CDF3(19886, 27341, 30272) }, + { CDF3(15609, 23747, 28046) }, + { CDF3(11993, 20065, 24939) }, + { CDF3( 9637, 18267, 23671) }, + { CDF3( 7625, 13801, 19144) }, + }, + }, { + { + { CDF3(14438, 20798, 24089) }, + { CDF3(12621, 19203, 23097) }, + { CDF3( 8177, 14125, 18402) }, + { CDF3( 5674, 10501, 14456) }, + { CDF3( 4236, 8239, 11733) }, + { CDF3( 3447, 6750, 9806) }, + { CDF3( 1986, 3950, 5864) }, + { CDF3(16208, 22099, 24930) }, + { CDF3(16537, 24025, 27585) }, + { CDF3(12780, 20381, 24867) }, + { CDF3( 9767, 16612, 21416) }, + { CDF3( 7686, 13738, 18398) }, + { CDF3( 6333, 11614, 15964) }, + { CDF3( 3941, 7571, 10836) }, + { CDF3(22819, 27422, 29202) }, + { CDF3(22224, 28514, 30721) }, + { CDF3(17660, 25433, 28913) }, + { CDF3(13574, 21482, 26002) }, + { CDF3(10629, 17977, 22938) }, + { CDF3( 8612, 15298, 20265) }, + { CDF3( 5607, 10491, 14596) }, + }, { + { CDF3(13569, 19800, 23206) }, + { CDF3(13128, 19924, 23869) }, + { CDF3( 8329, 14841, 19403) }, + { CDF3( 6130, 10976, 15057) }, + { CDF3( 4682, 8839, 12518) }, + { CDF3( 3656, 7409, 10588) }, + { CDF3( 2577, 5099, 7412) }, + { CDF3(22427, 28684, 30585) }, + { CDF3(20913, 27750, 30139) }, + { CDF3(15840, 24109, 27834) }, + { CDF3(12308, 20029, 24569) }, + { CDF3(10216, 16785, 21458) }, + { CDF3( 8309, 14203, 19113) }, + { CDF3( 6043, 11168, 15307) }, + { CDF3(23166, 28901, 30998) }, + { CDF3(21899, 28405, 30751) }, + { CDF3(18413, 26091, 29443) }, + { CDF3(15233, 23114, 27352) }, + { CDF3(12683, 20472, 25288) }, + { CDF3(10702, 18259, 23409) }, + { CDF3( 8125, 14464, 19226) }, + }, + }, { + { + { CDF3( 9040, 14786, 18360) }, + { CDF3( 9979, 15718, 19415) }, + { CDF3( 7913, 13918, 18311) }, + { CDF3( 5859, 10889, 15184) }, + { CDF3( 4593, 8677, 12510) }, + { CDF3( 3820, 7396, 10791) }, + { CDF3( 1730, 3471, 5192) }, + { CDF3(11803, 18365, 22709) }, + { CDF3(11419, 18058, 22225) }, + { CDF3( 9418, 15774, 20243) }, + { CDF3( 7539, 13325, 17657) }, + { CDF3( 6233, 11317, 15384) }, + { CDF3( 5137, 9656, 13545) }, + { CDF3( 2977, 5774, 8349) }, + { CDF3(21207, 27246, 29640) }, + { CDF3(19547, 26578, 29497) }, + { CDF3(16169, 23871, 27690) }, + { CDF3(12820, 20458, 25018) }, + { CDF3(10224, 17332, 22214) }, + { CDF3( 8526, 15048, 19884) }, + { CDF3( 5037, 9410, 13118) }, + }, { + { CDF3(12339, 17329, 20140) }, + { CDF3(13505, 19895, 23225) }, + { CDF3( 9847, 16944, 21564) }, + { CDF3( 7280, 13256, 18348) }, + { CDF3( 4712, 10009, 14454) }, + { CDF3( 4361, 7914, 12477) }, + { CDF3( 2870, 5628, 7995) }, + { CDF3(20061, 25504, 28526) }, + { CDF3(15235, 22878, 26145) }, + { CDF3(12985, 19958, 24155) }, + { CDF3( 9782, 16641, 21403) }, + { CDF3( 9456, 16360, 20760) }, + { CDF3( 6855, 12940, 18557) }, + { CDF3( 5661, 10564, 15002) }, + { CDF3(25656, 30602, 31894) }, + { CDF3(22570, 29107, 31092) }, + { CDF3(18917, 26423, 29541) }, + { CDF3(15940, 23649, 27754) }, + { CDF3(12803, 20581, 25219) }, + { CDF3(11082, 18695, 23376) }, + { CDF3( 7939, 14373, 19005) }, + }, + }, + }, + }, [3] = { + .skip = { + { + { CDF1(26887) }, { CDF1( 6729) }, { CDF1(10361) }, + { CDF1(17442) }, { CDF1(15045) }, { CDF1(22478) }, + { CDF1(29072) }, { CDF1( 2713) }, { CDF1(11861) }, + { CDF1(20773) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(31903) }, { CDF1( 2044) }, { CDF1( 7528) }, + { CDF1(14618) }, { CDF1(16182) }, { CDF1(24168) }, + { CDF1(31037) }, { CDF1( 2786) }, { CDF1(11194) }, + { CDF1(20155) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(32510) }, { CDF1( 8430) }, { CDF1(17318) }, + { CDF1(24154) }, { CDF1(23674) }, { CDF1(28789) }, + { CDF1(32139) }, { CDF1( 3440) }, { CDF1(13117) }, + { CDF1(22702) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, { + { CDF1(31671) }, { CDF1( 2056) }, { CDF1(11746) }, + { CDF1(16852) }, { CDF1(18635) }, { CDF1(24715) }, + { CDF1(31484) }, { CDF1( 4656) }, { CDF1(16074) }, + { CDF1(24704) }, { CDF1( 1806) }, { CDF1(14645) }, + { CDF1(25336) }, + }, { + { CDF1(31539) }, { CDF1( 8433) }, { CDF1(20576) }, + { CDF1(27904) }, { CDF1(27852) }, { CDF1(30026) }, + { CDF1(32441) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, + }, + }, .eob_bin_16 = { + { + { CDF4( 6708, 8958, 14746, 22133) }, + { CDF4( 1222, 2074, 4783, 15410) }, + }, { + { CDF4(19575, 21766, 26044, 29709) }, + { CDF4( 7297, 10767, 19273, 28194) }, + }, + }, .eob_bin_32 = { + { + { CDF5( 4617, 5709, 8446, 13584, 23135) }, + { CDF5( 1156, 1702, 3675, 9274, 20539) }, + }, { + { CDF5(22086, 24282, 27010, 29770, 31743) }, + { CDF5( 7699, 10897, 20891, 26926, 31628) }, + }, + }, .eob_bin_64 = { + { + { CDF6( 6307, 7541, 12060, 16358, 22553, 27865) }, + { CDF6( 1289, 2320, 3971, 7926, 14153, 24291) }, + }, { + { CDF6(24212, 25708, 28268, 30035, 31307, 32049) }, + { CDF6( 8726, 12378, 19409, 26450, 30038, 32462) }, + }, + }, .eob_bin_128 = { + { + { CDF7( 3472, 4885, 7489, 12481, 18517, 24536, 29635) }, + { CDF7( 886, 1731, 3271, 8469, 15569, 22126, 28383) }, + }, { + { CDF7(24313, 26062, 28385, 30107, 31217, 31898, 32345) }, + { CDF7( 9165, 13282, 21150, 30286, 31894, 32571, 32712) }, + }, + }, .eob_bin_256 = { + { + { CDF8( 5348, 7113, 11820, 15924, + 22106, 26777, 30334, 31757) }, + { CDF8( 2453, 4474, 6307, 8777, + 16474, 22975, 29000, 31547) }, + }, { + { CDF8(23110, 24597, 27140, 28894, + 30167, 30927, 31392, 32094) }, + { CDF8( 9998, 17661, 25178, 28097, + 31308, 32038, 32403, 32695) }, + }, + }, .eob_bin_512 = { + { CDF9( 5927, 7809, 10923, 14597, 19439, + 24135, 28456, 31142, 32060) }, + { CDF9(21093, 23043, 25742, 27658, 29097, + 29716, 30073, 30820, 31956) }, + }, .eob_bin_1024 = { + { CDF10( 6698, 8334, 11961, 15762, 20186, + 23862, 27434, 29326, 31082, 32050) }, + { CDF10(20569, 22426, 25569, 26859, 28053, + 28913, 29486, 29724, 29807, 32570) }, + }, .eob_hi_bit = { + { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20177) }, + { CDF1(20789) }, { CDF1(20262) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(21416) }, + { CDF1(20855) }, { CDF1(23410) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20238) }, + { CDF1(21057) }, { CDF1(19159) }, { CDF1(22337) }, + { CDF1(20159) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(20125) }, + { CDF1(20559) }, { CDF1(21707) }, { CDF1(22296) }, + { CDF1(17333) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(19941) }, + { CDF1(20527) }, { CDF1(21470) }, { CDF1(22487) }, + { CDF1(19558) }, { CDF1(22354) }, { CDF1(20331) }, + { CDF1(16384) }, { CDF1(16384) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(22752) }, + { CDF1(25006) }, { CDF1(22075) }, { CDF1(21576) }, + { CDF1(17740) }, { CDF1(21690) }, { CDF1(19211) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(21442) }, + { CDF1(22358) }, { CDF1(18503) }, { CDF1(20291) }, + { CDF1(19945) }, { CDF1(21294) }, { CDF1(21178) }, + { CDF1(19400) }, { CDF1(10556) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(24648) }, + { CDF1(24949) }, { CDF1(20708) }, { CDF1(23905) }, + { CDF1(20501) }, { CDF1( 9558) }, { CDF1( 9423) }, + { CDF1(30365) }, { CDF1(19253) }, + }, + }, { + { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(26064) }, + { CDF1(22098) }, { CDF1(19613) }, { CDF1(20525) }, + { CDF1(17595) }, { CDF1(16618) }, { CDF1(20497) }, + { CDF1(18989) }, { CDF1(15513) }, + }, { + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, { CDF1(16384) }, + { CDF1(16384) }, { CDF1(16384) }, + }, + }, + }, .eob_base_tok = { + { + { + { CDF2(22497, 31198) }, { CDF2(31715, 32495) }, + { CDF2(31606, 32337) }, { CDF2(30388, 31990) }, + }, { + { CDF2(27877, 31584) }, { CDF2(32170, 32728) }, + { CDF2(32155, 32688) }, { CDF2(32219, 32702) }, + }, + }, { + { + { CDF2(21457, 31043) }, { CDF2(31951, 32483) }, + { CDF2(32153, 32562) }, { CDF2(31473, 32215) }, + }, { + { CDF2(27558, 31151) }, { CDF2(32020, 32640) }, + { CDF2(32097, 32575) }, { CDF2(32242, 32719) }, + }, + }, { + { + { CDF2(19980, 30591) }, { CDF2(32219, 32597) }, + { CDF2(32581, 32706) }, { CDF2(31803, 32287) }, + }, { + { CDF2(26473, 30507) }, { CDF2(32431, 32723) }, + { CDF2(32196, 32611) }, { CDF2(31588, 32528) }, + }, + }, { + { + { CDF2(24647, 30463) }, { CDF2(32412, 32695) }, + { CDF2(32468, 32720) }, { CDF2(31269, 32523) }, + }, { + { CDF2(28482, 31505) }, { CDF2(32152, 32701) }, + { CDF2(31732, 32598) }, { CDF2(31767, 32712) }, + }, + }, { + { + { CDF2(12358, 24977) }, { CDF2(31331, 32385) }, + { CDF2(32634, 32756) }, { CDF2(30411, 32548) }, + }, { + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + { CDF2(10923, 21845) }, { CDF2(10923, 21845) }, + }, + }, + }, .base_tok = { + { + { + { CDF3( 7062, 16472, 22319) }, + { CDF3(24538, 32261, 32674) }, + { CDF3(13675, 28041, 31779) }, + { CDF3( 8590, 20674, 27631) }, + { CDF3( 5685, 14675, 22013) }, + { CDF3( 3655, 9898, 15731) }, + { CDF3(26493, 32418, 32658) }, + { CDF3(16376, 29342, 32090) }, + { CDF3(10594, 22649, 28970) }, + { CDF3( 8176, 17170, 24303) }, + { CDF3( 5605, 12694, 19139) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(23888, 31902, 32542) }, + { CDF3(18612, 29687, 31987) }, + { CDF3(16245, 24852, 29249) }, + { CDF3(15765, 22608, 27559) }, + { CDF3(19895, 24699, 27510) }, + { CDF3(28401, 32212, 32457) }, + { CDF3(15274, 27825, 30980) }, + { CDF3( 9364, 18128, 24332) }, + { CDF3( 2283, 8193, 15082) }, + { CDF3( 1228, 3972, 7881) }, + { CDF3(29455, 32469, 32620) }, + { CDF3(17981, 28245, 31388) }, + { CDF3(10921, 20098, 26240) }, + { CDF3( 3743, 11829, 18657) }, + { CDF3( 2374, 9593, 15715) }, + { CDF3(31068, 32466, 32635) }, + { CDF3(20321, 29572, 31971) }, + { CDF3(10771, 20255, 27119) }, + { CDF3( 2795, 10410, 17361) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 9320, 22102, 27840) }, + { CDF3(27057, 32464, 32724) }, + { CDF3(16331, 30268, 32309) }, + { CDF3(10319, 23935, 29720) }, + { CDF3( 6189, 16448, 24106) }, + { CDF3( 3589, 10884, 18808) }, + { CDF3(29026, 32624, 32748) }, + { CDF3(19226, 31507, 32587) }, + { CDF3(12692, 26921, 31203) }, + { CDF3( 7049, 19532, 27635) }, + { CDF3( 7727, 15669, 23252) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3(28056, 32625, 32748) }, + { CDF3(22383, 32075, 32669) }, + { CDF3(15417, 27098, 31749) }, + { CDF3(18127, 26493, 27190) }, + { CDF3( 5461, 16384, 21845) }, + { CDF3(27982, 32091, 32584) }, + { CDF3(19045, 29868, 31972) }, + { CDF3(10397, 22266, 27932) }, + { CDF3( 5990, 13697, 21500) }, + { CDF3( 1792, 6912, 15104) }, + { CDF3(28198, 32501, 32718) }, + { CDF3(21534, 31521, 32569) }, + { CDF3(11109, 25217, 30017) }, + { CDF3( 5671, 15124, 26151) }, + { CDF3( 4681, 14043, 18725) }, + { CDF3(28688, 32580, 32741) }, + { CDF3(22576, 32079, 32661) }, + { CDF3(10627, 22141, 28340) }, + { CDF3( 9362, 14043, 28087) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 7754, 16948, 22142) }, + { CDF3(25670, 32330, 32691) }, + { CDF3(15663, 29225, 31994) }, + { CDF3( 9878, 23288, 29158) }, + { CDF3( 6419, 17088, 24336) }, + { CDF3( 3859, 11003, 17039) }, + { CDF3(27562, 32595, 32725) }, + { CDF3(17575, 30588, 32399) }, + { CDF3(10819, 24838, 30309) }, + { CDF3( 7124, 18686, 25916) }, + { CDF3( 4479, 12688, 19340) }, + { CDF3(28385, 32476, 32673) }, + { CDF3(15306, 29005, 31938) }, + { CDF3( 8937, 21615, 28322) }, + { CDF3( 5982, 15603, 22786) }, + { CDF3( 3620, 10267, 16136) }, + { CDF3(27280, 32464, 32667) }, + { CDF3(15607, 29160, 32004) }, + { CDF3( 9091, 22135, 28740) }, + { CDF3( 6232, 16632, 24020) }, + { CDF3( 4047, 11377, 17672) }, + { CDF3(29220, 32630, 32718) }, + { CDF3(19650, 31220, 32462) }, + { CDF3(13050, 26312, 30827) }, + { CDF3( 9228, 20870, 27468) }, + { CDF3( 6146, 15149, 21971) }, + { CDF3(30169, 32481, 32623) }, + { CDF3(17212, 29311, 31554) }, + { CDF3( 9911, 21311, 26882) }, + { CDF3( 4487, 13314, 20372) }, + { CDF3( 2570, 7772, 12889) }, + { CDF3(30924, 32613, 32708) }, + { CDF3(19490, 30206, 32107) }, + { CDF3(11232, 23998, 29276) }, + { CDF3( 6769, 17955, 25035) }, + { CDF3( 4398, 12623, 19214) }, + { CDF3(30609, 32627, 32722) }, + { CDF3(19370, 30582, 32287) }, + { CDF3(10457, 23619, 29409) }, + { CDF3( 6443, 17637, 24834) }, + { CDF3( 4645, 13236, 20106) }, + }, { + { CDF3( 8626, 20271, 26216) }, + { CDF3(26707, 32406, 32711) }, + { CDF3(16999, 30329, 32286) }, + { CDF3(11445, 25123, 30286) }, + { CDF3( 6411, 18828, 25601) }, + { CDF3( 6801, 12458, 20248) }, + { CDF3(29918, 32682, 32748) }, + { CDF3(20649, 31739, 32618) }, + { CDF3(12879, 27773, 31581) }, + { CDF3( 7896, 21751, 28244) }, + { CDF3( 5260, 14870, 23698) }, + { CDF3(29252, 32593, 32731) }, + { CDF3(17072, 30460, 32294) }, + { CDF3(10653, 24143, 29365) }, + { CDF3( 6536, 17490, 23983) }, + { CDF3( 4929, 13170, 20085) }, + { CDF3(28137, 32518, 32715) }, + { CDF3(18171, 30784, 32407) }, + { CDF3(11437, 25436, 30459) }, + { CDF3( 7252, 18534, 26176) }, + { CDF3( 4126, 13353, 20978) }, + { CDF3(31162, 32726, 32748) }, + { CDF3(23017, 32222, 32701) }, + { CDF3(15629, 29233, 32046) }, + { CDF3( 9387, 22621, 29480) }, + { CDF3( 6922, 17616, 25010) }, + { CDF3(28838, 32265, 32614) }, + { CDF3(19701, 30206, 31920) }, + { CDF3(11214, 22410, 27933) }, + { CDF3( 5320, 14177, 23034) }, + { CDF3( 5049, 12881, 17827) }, + { CDF3(27484, 32471, 32734) }, + { CDF3(21076, 31526, 32561) }, + { CDF3(12707, 26303, 31211) }, + { CDF3( 8169, 21722, 28219) }, + { CDF3( 6045, 19406, 27042) }, + { CDF3(27753, 32572, 32745) }, + { CDF3(20832, 31878, 32653) }, + { CDF3(13250, 27356, 31674) }, + { CDF3( 7718, 21508, 29858) }, + { CDF3( 7209, 18350, 25559) }, + }, + }, { + { + { CDF3( 7876, 16901, 21741) }, + { CDF3(24001, 31898, 32625) }, + { CDF3(14529, 27959, 31451) }, + { CDF3( 8273, 20818, 27258) }, + { CDF3( 5278, 14673, 21510) }, + { CDF3( 2983, 8843, 14039) }, + { CDF3(28016, 32574, 32732) }, + { CDF3(17471, 30306, 32301) }, + { CDF3(10224, 24063, 29728) }, + { CDF3( 6602, 17954, 25052) }, + { CDF3( 4002, 11585, 17759) }, + { CDF3(30190, 32634, 32739) }, + { CDF3(17497, 30282, 32270) }, + { CDF3(10229, 23729, 29538) }, + { CDF3( 6344, 17211, 24440) }, + { CDF3( 3849, 11189, 17108) }, + { CDF3(28570, 32583, 32726) }, + { CDF3(17521, 30161, 32238) }, + { CDF3(10153, 23565, 29378) }, + { CDF3( 6455, 17341, 24443) }, + { CDF3( 3907, 11042, 17024) }, + { CDF3(30689, 32715, 32748) }, + { CDF3(21546, 31840, 32610) }, + { CDF3(13547, 27581, 31459) }, + { CDF3( 8912, 21757, 28309) }, + { CDF3( 5548, 15080, 22046) }, + { CDF3(30783, 32540, 32685) }, + { CDF3(17540, 29528, 31668) }, + { CDF3(10160, 21468, 26783) }, + { CDF3( 4724, 13393, 20054) }, + { CDF3( 2702, 8174, 13102) }, + { CDF3(31648, 32686, 32742) }, + { CDF3(20954, 31094, 32337) }, + { CDF3(12420, 25698, 30179) }, + { CDF3( 7304, 19320, 26248) }, + { CDF3( 4366, 12261, 18864) }, + { CDF3(31581, 32723, 32748) }, + { CDF3(21373, 31586, 32525) }, + { CDF3(12744, 26625, 30885) }, + { CDF3( 7431, 20322, 26950) }, + { CDF3( 4692, 13323, 20111) }, + }, { + { CDF3( 7833, 18369, 24095) }, + { CDF3(26650, 32273, 32702) }, + { CDF3(16371, 29961, 32191) }, + { CDF3(11055, 24082, 29629) }, + { CDF3( 6892, 18644, 25400) }, + { CDF3( 5006, 13057, 19240) }, + { CDF3(29834, 32666, 32748) }, + { CDF3(19577, 31335, 32570) }, + { CDF3(12253, 26509, 31122) }, + { CDF3( 7991, 20772, 27711) }, + { CDF3( 5677, 15910, 23059) }, + { CDF3(30109, 32532, 32720) }, + { CDF3(16747, 30166, 32252) }, + { CDF3(10134, 23542, 29184) }, + { CDF3( 5791, 16176, 23556) }, + { CDF3( 4362, 10414, 17284) }, + { CDF3(29492, 32626, 32748) }, + { CDF3(19894, 31402, 32525) }, + { CDF3(12942, 27071, 30869) }, + { CDF3( 8346, 21216, 27405) }, + { CDF3( 6572, 17087, 23859) }, + { CDF3(32035, 32735, 32748) }, + { CDF3(22957, 31838, 32618) }, + { CDF3(14724, 28572, 31772) }, + { CDF3(10364, 23999, 29553) }, + { CDF3( 7004, 18433, 25655) }, + { CDF3(27528, 32277, 32681) }, + { CDF3(16959, 31171, 32096) }, + { CDF3(10486, 23593, 27962) }, + { CDF3( 8192, 16384, 23211) }, + { CDF3( 8937, 17873, 20852) }, + { CDF3(27715, 32002, 32615) }, + { CDF3(15073, 29491, 31676) }, + { CDF3(11264, 24576, 28672) }, + { CDF3( 2341, 18725, 23406) }, + { CDF3( 7282, 18204, 25486) }, + { CDF3(28547, 32213, 32657) }, + { CDF3(20788, 29773, 32239) }, + { CDF3( 6780, 21469, 30508) }, + { CDF3( 5958, 14895, 23831) }, + { CDF3(16384, 21845, 27307) }, + }, + }, { + { + { CDF3( 5992, 14304, 19765) }, + { CDF3(22612, 31238, 32456) }, + { CDF3(13456, 27162, 31087) }, + { CDF3( 8001, 20062, 26504) }, + { CDF3( 5168, 14105, 20764) }, + { CDF3( 2632, 7771, 12385) }, + { CDF3(27034, 32344, 32709) }, + { CDF3(15850, 29415, 31997) }, + { CDF3( 9494, 22776, 28841) }, + { CDF3( 6151, 16830, 23969) }, + { CDF3( 3461, 10039, 15722) }, + { CDF3(30134, 32569, 32731) }, + { CDF3(15638, 29422, 31945) }, + { CDF3( 9150, 21865, 28218) }, + { CDF3( 5647, 15719, 22676) }, + { CDF3( 3402, 9772, 15477) }, + { CDF3(28530, 32586, 32735) }, + { CDF3(17139, 30298, 32292) }, + { CDF3(10200, 24039, 29685) }, + { CDF3( 6419, 17674, 24786) }, + { CDF3( 3544, 10225, 15824) }, + { CDF3(31333, 32726, 32748) }, + { CDF3(20618, 31487, 32544) }, + { CDF3(12901, 27217, 31232) }, + { CDF3( 8624, 21734, 28171) }, + { CDF3( 5104, 14191, 20748) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3(11206, 21090, 26561) }, + { CDF3(28759, 32279, 32671) }, + { CDF3(14171, 27952, 31569) }, + { CDF3( 9743, 22907, 29141) }, + { CDF3( 6871, 17886, 24868) }, + { CDF3( 4960, 13152, 19315) }, + { CDF3(31077, 32661, 32748) }, + { CDF3(19400, 31195, 32515) }, + { CDF3(12752, 26858, 31040) }, + { CDF3( 8370, 22098, 28591) }, + { CDF3( 5457, 15373, 22298) }, + { CDF3(31697, 32706, 32748) }, + { CDF3(17860, 30657, 32333) }, + { CDF3(12510, 24812, 29261) }, + { CDF3( 6180, 19124, 24722) }, + { CDF3( 5041, 13548, 17959) }, + { CDF3(31552, 32716, 32748) }, + { CDF3(21908, 31769, 32623) }, + { CDF3(14470, 28201, 31565) }, + { CDF3( 9493, 22982, 28608) }, + { CDF3( 6858, 17240, 24137) }, + { CDF3(32543, 32752, 32756) }, + { CDF3(24286, 32097, 32666) }, + { CDF3(15958, 29217, 32024) }, + { CDF3(10207, 24234, 29958) }, + { CDF3( 6929, 18305, 25652) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, { + { + { CDF3( 4137, 10847, 15682) }, + { CDF3(17824, 27001, 30058) }, + { CDF3(10204, 22796, 28291) }, + { CDF3( 6076, 15935, 22125) }, + { CDF3( 3852, 10937, 16816) }, + { CDF3( 2252, 6324, 10131) }, + { CDF3(25840, 32016, 32662) }, + { CDF3(15109, 28268, 31531) }, + { CDF3( 9385, 22231, 28340) }, + { CDF3( 6082, 16672, 23479) }, + { CDF3( 3318, 9427, 14681) }, + { CDF3(30594, 32574, 32718) }, + { CDF3(16836, 29552, 31859) }, + { CDF3( 9556, 22542, 28356) }, + { CDF3( 6305, 16725, 23540) }, + { CDF3( 3376, 9895, 15184) }, + { CDF3(29383, 32617, 32745) }, + { CDF3(18891, 30809, 32401) }, + { CDF3(11688, 25942, 30687) }, + { CDF3( 7468, 19469, 26651) }, + { CDF3( 3909, 11358, 17012) }, + { CDF3(31564, 32736, 32748) }, + { CDF3(20906, 31611, 32600) }, + { CDF3(13191, 27621, 31537) }, + { CDF3( 8768, 22029, 28676) }, + { CDF3( 5079, 14109, 20906) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, { + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + { CDF3( 8192, 16384, 24576) }, + }, + }, + }, .dc_sign = { + { { CDF1(16000) }, { CDF1(13056) }, { CDF1(18816) } }, + { { CDF1(15232) }, { CDF1(12928) }, { CDF1(17280) } }, + }, .br_tok = { + { + { + { CDF3(18315, 24289, 27551) }, + { CDF3(16854, 24068, 27835) }, + { CDF3(10140, 17927, 23173) }, + { CDF3( 6722, 12982, 18267) }, + { CDF3( 4661, 9826, 14706) }, + { CDF3( 3832, 8165, 12294) }, + { CDF3( 2795, 6098, 9245) }, + { CDF3(17145, 23326, 26672) }, + { CDF3(20733, 27680, 30308) }, + { CDF3(16032, 24461, 28546) }, + { CDF3(11653, 20093, 25081) }, + { CDF3( 9290, 16429, 22086) }, + { CDF3( 7796, 14598, 19982) }, + { CDF3( 6502, 12378, 17441) }, + { CDF3(21681, 27732, 30320) }, + { CDF3(22389, 29044, 31261) }, + { CDF3(19027, 26731, 30087) }, + { CDF3(14739, 23755, 28624) }, + { CDF3(11358, 20778, 25511) }, + { CDF3(10995, 18073, 24190) }, + { CDF3( 9162, 14990, 20617) }, + }, { + { CDF3(21425, 27952, 30388) }, + { CDF3(18062, 25838, 29034) }, + { CDF3(11956, 19881, 24808) }, + { CDF3( 7718, 15000, 20980) }, + { CDF3( 5702, 11254, 16143) }, + { CDF3( 4898, 9088, 16864) }, + { CDF3( 3679, 6776, 11907) }, + { CDF3(23294, 30160, 31663) }, + { CDF3(24397, 29896, 31836) }, + { CDF3(19245, 27128, 30593) }, + { CDF3(13202, 19825, 26404) }, + { CDF3(11578, 19297, 23957) }, + { CDF3( 8073, 13297, 21370) }, + { CDF3( 5461, 10923, 19745) }, + { CDF3(27367, 30521, 31934) }, + { CDF3(24904, 30671, 31940) }, + { CDF3(23075, 28460, 31299) }, + { CDF3(14400, 23658, 30417) }, + { CDF3(13885, 23882, 28325) }, + { CDF3(14746, 22938, 27853) }, + { CDF3( 5461, 16384, 27307) }, + }, + }, { + { + { CDF3(18274, 24813, 27890) }, + { CDF3(15537, 23149, 27003) }, + { CDF3( 9449, 16740, 21827) }, + { CDF3( 6700, 12498, 17261) }, + { CDF3( 4988, 9866, 14198) }, + { CDF3( 4236, 8147, 11902) }, + { CDF3( 2867, 5860, 8654) }, + { CDF3(17124, 23171, 26101) }, + { CDF3(20396, 27477, 30148) }, + { CDF3(16573, 24629, 28492) }, + { CDF3(12749, 20846, 25674) }, + { CDF3(10233, 17878, 22818) }, + { CDF3( 8525, 15332, 20363) }, + { CDF3( 6283, 11632, 16255) }, + { CDF3(20466, 26511, 29286) }, + { CDF3(23059, 29174, 31191) }, + { CDF3(19481, 27263, 30241) }, + { CDF3(15458, 23631, 28137) }, + { CDF3(12416, 20608, 25693) }, + { CDF3(10261, 18011, 23261) }, + { CDF3( 8016, 14655, 19666) }, + }, { + { CDF3(17616, 24586, 28112) }, + { CDF3(15809, 23299, 27155) }, + { CDF3(10767, 18890, 23793) }, + { CDF3( 7727, 14255, 18865) }, + { CDF3( 6129, 11926, 16882) }, + { CDF3( 4482, 9704, 14861) }, + { CDF3( 3277, 7452, 11522) }, + { CDF3(22956, 28551, 30730) }, + { CDF3(22724, 28937, 30961) }, + { CDF3(18467, 26324, 29580) }, + { CDF3(13234, 20713, 25649) }, + { CDF3(11181, 17592, 22481) }, + { CDF3( 8291, 18358, 24576) }, + { CDF3( 7568, 11881, 14984) }, + { CDF3(24948, 29001, 31147) }, + { CDF3(25674, 30619, 32151) }, + { CDF3(20841, 26793, 29603) }, + { CDF3(14669, 24356, 28666) }, + { CDF3(11334, 23593, 28219) }, + { CDF3( 8922, 14762, 22873) }, + { CDF3( 8301, 13544, 20535) }, + }, + }, { + { + { CDF3(17113, 23733, 27081) }, + { CDF3(14139, 21406, 25452) }, + { CDF3( 8552, 15002, 19776) }, + { CDF3( 5871, 11120, 15378) }, + { CDF3( 4455, 8616, 12253) }, + { CDF3( 3469, 6910, 10386) }, + { CDF3( 2255, 4553, 6782) }, + { CDF3(18224, 24376, 27053) }, + { CDF3(19290, 26710, 29614) }, + { CDF3(14936, 22991, 27184) }, + { CDF3(11238, 18951, 23762) }, + { CDF3( 8786, 15617, 20588) }, + { CDF3( 7317, 13228, 18003) }, + { CDF3( 5101, 9512, 13493) }, + { CDF3(22639, 28222, 30210) }, + { CDF3(23216, 29331, 31307) }, + { CDF3(19075, 26762, 29895) }, + { CDF3(15014, 23113, 27457) }, + { CDF3(11938, 19857, 24752) }, + { CDF3( 9942, 17280, 22282) }, + { CDF3( 7167, 13144, 17752) }, + }, { + { CDF3(15820, 22738, 26488) }, + { CDF3(13530, 20885, 25216) }, + { CDF3( 8395, 15530, 20452) }, + { CDF3( 6574, 12321, 16380) }, + { CDF3( 5353, 10419, 14568) }, + { CDF3( 4613, 8446, 12381) }, + { CDF3( 3440, 7158, 9903) }, + { CDF3(24247, 29051, 31224) }, + { CDF3(22118, 28058, 30369) }, + { CDF3(16498, 24768, 28389) }, + { CDF3(12920, 21175, 26137) }, + { CDF3(10730, 18619, 25352) }, + { CDF3(10187, 16279, 22791) }, + { CDF3( 9310, 14631, 22127) }, + { CDF3(24970, 30558, 32057) }, + { CDF3(24801, 29942, 31698) }, + { CDF3(22432, 28453, 30855) }, + { CDF3(19054, 25680, 29580) }, + { CDF3(14392, 23036, 28109) }, + { CDF3(12495, 20947, 26650) }, + { CDF3(12442, 20326, 26214) }, + }, + }, { + { + { CDF3(12162, 18785, 22648) }, + { CDF3(12749, 19697, 23806) }, + { CDF3( 8580, 15297, 20346) }, + { CDF3( 6169, 11749, 16543) }, + { CDF3( 4836, 9391, 13448) }, + { CDF3( 3821, 7711, 11613) }, + { CDF3( 2228, 4601, 7070) }, + { CDF3(16319, 24725, 28280) }, + { CDF3(15698, 23277, 27168) }, + { CDF3(12726, 20368, 25047) }, + { CDF3( 9912, 17015, 21976) }, + { CDF3( 7888, 14220, 19179) }, + { CDF3( 6777, 12284, 17018) }, + { CDF3( 4492, 8590, 12252) }, + { CDF3(23249, 28904, 30947) }, + { CDF3(21050, 27908, 30512) }, + { CDF3(17440, 25340, 28949) }, + { CDF3(14059, 22018, 26541) }, + { CDF3(11288, 18903, 23898) }, + { CDF3( 9411, 16342, 21428) }, + { CDF3( 6278, 11588, 15944) }, + }, { + { CDF3(13981, 20067, 23226) }, + { CDF3(16922, 23580, 26783) }, + { CDF3(11005, 19039, 24487) }, + { CDF3( 7389, 14218, 19798) }, + { CDF3( 5598, 11505, 17206) }, + { CDF3( 6090, 11213, 15659) }, + { CDF3( 3820, 7371, 10119) }, + { CDF3(21082, 26925, 29675) }, + { CDF3(21262, 28627, 31128) }, + { CDF3(18392, 26454, 30437) }, + { CDF3(14870, 22910, 27096) }, + { CDF3(12620, 19484, 24908) }, + { CDF3( 9290, 16553, 22802) }, + { CDF3( 6668, 14288, 20004) }, + { CDF3(27704, 31055, 31949) }, + { CDF3(24709, 29978, 31788) }, + { CDF3(21668, 29264, 31657) }, + { CDF3(18295, 26968, 30074) }, + { CDF3(16399, 24422, 29313) }, + { CDF3(14347, 23026, 28104) }, + { CDF3(12370, 19806, 24477) }, + }, + }, + }, + } +}; + +void dav1d_cdf_thread_update(const Dav1dFrameHeader *const hdr, + CdfContext *const dst, + const CdfContext *const src) +{ +#define update_cdf_1d(n1d, name) \ + do { \ + memcpy(dst->name, src->name, sizeof(dst->name)); \ + dst->name[n1d] = 0; \ + } while (0) + +#define update_cdf_2d(n1d, n2d, name) \ + for (int j = 0; j < (n1d); j++) update_cdf_1d(n2d, name[j]) +#define update_cdf_3d(n1d, n2d, n3d, name) \ + for (int k = 0; k < (n1d); k++) update_cdf_2d(n2d, n3d, name[k]) +#define update_cdf_4d(n1d, n2d, n3d, n4d, name) \ + for (int l = 0; l < (n1d); l++) update_cdf_3d(n2d, n3d, n4d, name[l]) + +#define update_bit_0d(name) \ + do { \ + dst->name[0] = src->name[0]; \ + dst->name[1] = 0; \ + } while (0) + +#define update_bit_1d(n1d, name) \ + for (int i = 0; i < (n1d); i++) update_bit_0d(name[i]) +#define update_bit_2d(n1d, n2d, name) \ + for (int j = 0; j < (n1d); j++) update_bit_1d(n2d, name[j]) +#define update_bit_3d(n1d, n2d, n3d, name) \ + for (int k = 0; k < (n1d); k++) update_bit_2d(n2d, n3d, name[k]) + + update_bit_1d(N_BS_SIZES, m.use_filter_intra); + update_cdf_1d(4, m.filter_intra); + update_cdf_3d(2, N_INTRA_PRED_MODES, N_UV_INTRA_PRED_MODES - 1 - !k, m.uv_mode); + update_cdf_2d(8, 6, m.angle_delta); + update_cdf_3d(N_TX_SIZES - 1, 3, imin(k + 1, 2), m.txsz); + update_cdf_3d(2, N_INTRA_PRED_MODES, 6, m.txtp_intra1); + update_cdf_3d(3, N_INTRA_PRED_MODES, 4, m.txtp_intra2); + update_bit_1d(3, m.skip); + update_cdf_3d(N_BL_LEVELS, 4, dav1d_partition_type_count[k], m.partition); + update_bit_2d(N_TX_SIZES, 13, coef.skip); + update_cdf_3d(2, 2, 4, coef.eob_bin_16); + update_cdf_3d(2, 2, 5, coef.eob_bin_32); + update_cdf_3d(2, 2, 6, coef.eob_bin_64); + update_cdf_3d(2, 2, 7, coef.eob_bin_128); + update_cdf_3d(2, 2, 8, coef.eob_bin_256); + update_cdf_2d(2, 9, coef.eob_bin_512); + update_cdf_2d(2, 10, coef.eob_bin_1024); + update_bit_3d(N_TX_SIZES, 2, 11 /*22*/, coef.eob_hi_bit); + update_cdf_4d(N_TX_SIZES, 2, 4, 2, coef.eob_base_tok); + update_cdf_4d(N_TX_SIZES, 2, 41 /*42*/, 3, coef.base_tok); + update_bit_2d(2, 3, coef.dc_sign); + update_cdf_4d(4, 2, 21, 3, coef.br_tok); + update_cdf_2d(3, DAV1D_MAX_SEGMENTS - 1, m.seg_id); + update_cdf_1d(7, m.cfl_sign); + update_cdf_2d(6, 15, m.cfl_alpha); + update_bit_0d(m.restore_wiener); + update_bit_0d(m.restore_sgrproj); + update_cdf_1d(2, m.restore_switchable); + update_cdf_1d(3, m.delta_q); + update_cdf_2d(5, 3, m.delta_lf); + update_bit_2d(7, 3, m.pal_y); + update_bit_1d(2, m.pal_uv); + update_cdf_3d(2, 7, 6, m.pal_sz); + update_cdf_4d(2, 7, 5, k + 1, m.color_map); + update_bit_2d(7, 3, m.txpart); + update_cdf_2d(2, 15, m.txtp_inter1); + update_cdf_1d(11, m.txtp_inter2); + update_bit_1d(4, m.txtp_inter3); + + if (IS_KEY_OR_INTRA(hdr)) { + update_bit_0d(m.intrabc); + + update_cdf_1d(N_MV_JOINTS - 1, dmv.joint); + for (int k = 0; k < 2; k++) { + update_cdf_1d(10, dmv.comp[k].classes); + update_bit_0d(dmv.comp[k].class0); + update_bit_1d(10, dmv.comp[k].classN); + update_bit_0d(dmv.comp[k].sign); + } + return; + } + + update_bit_1d(3, m.skip_mode); + update_cdf_2d(4, N_INTRA_PRED_MODES - 1, m.y_mode); + update_cdf_3d(2, 8, DAV1D_N_SWITCHABLE_FILTERS - 1, m.filter); + update_bit_1d(6, m.newmv_mode); + update_bit_1d(2, m.globalmv_mode); + update_bit_1d(6, m.refmv_mode); + update_bit_1d(3, m.drl_bit); + update_cdf_2d(8, N_COMP_INTER_PRED_MODES - 1, m.comp_inter_mode); + update_bit_1d(4, m.intra); + update_bit_1d(5, m.comp); + update_bit_1d(5, m.comp_dir); + update_bit_1d(6, m.jnt_comp); + update_bit_1d(6, m.mask_comp); + update_bit_1d(9, m.wedge_comp); + update_cdf_2d(9, 15, m.wedge_idx); + update_bit_2d(6, 3, m.ref); + update_bit_2d(3, 3, m.comp_fwd_ref); + update_bit_2d(2, 3, m.comp_bwd_ref); + update_bit_2d(3, 3, m.comp_uni_ref); + update_bit_1d(3, m.seg_pred); + update_bit_1d(4, m.interintra); + update_bit_1d(7, m.interintra_wedge); + update_cdf_2d(4, 3, m.interintra_mode); + update_cdf_2d(N_BS_SIZES, 2, m.motion_mode); + update_bit_1d(N_BS_SIZES, m.obmc); + + update_cdf_1d(N_MV_JOINTS - 1, mv.joint); + for (int k = 0; k < 2; k++) { + update_cdf_1d(10, mv.comp[k].classes); + update_bit_0d(mv.comp[k].class0); + update_bit_1d(10, mv.comp[k].classN); + update_cdf_2d(2, 3, mv.comp[k].class0_fp); + update_cdf_1d(3, mv.comp[k].classN_fp); + update_bit_0d(mv.comp[k].class0_hp); + update_bit_0d(mv.comp[k].classN_hp); + update_bit_0d(mv.comp[k].sign); + } +} + +/* + * CDF threading wrappers. + */ +static inline int get_qcat_idx(const int q) { + if (q <= 20) return 0; + if (q <= 60) return 1; + if (q <= 120) return 2; + return 3; +} + +void dav1d_cdf_thread_init_static(CdfThreadContext *const cdf, const int qidx) { + cdf->ref = NULL; + cdf->data.qcat = get_qcat_idx(qidx); +} + +void dav1d_cdf_thread_copy(CdfContext *const dst, const CdfThreadContext *const src) { + if (src->ref) { + memcpy(dst, src->data.cdf, sizeof(*dst)); + } else { + dst->m = av1_default_cdf; + memcpy(dst->kfym, default_kf_y_mode_cdf, sizeof(default_kf_y_mode_cdf)); + dst->coef = av1_default_coef_cdf[src->data.qcat]; + memcpy(dst->mv.joint, default_mv_joint_cdf, sizeof(default_mv_joint_cdf)); + memcpy(dst->dmv.joint, default_mv_joint_cdf, sizeof(default_mv_joint_cdf)); + dst->mv.comp[0] = dst->mv.comp[1] = dst->dmv.comp[0] = dst->dmv.comp[1] = + default_mv_component_cdf; + } +} + +int dav1d_cdf_thread_alloc(Dav1dContext *const c, CdfThreadContext *const cdf, + const int have_frame_mt) +{ + cdf->ref = dav1d_ref_create_using_pool(c->cdf_pool, + sizeof(CdfContext) + sizeof(atomic_uint)); + if (!cdf->ref) return DAV1D_ERR(ENOMEM); + cdf->data.cdf = cdf->ref->data; + if (have_frame_mt) { + cdf->progress = (atomic_uint *) &cdf->data.cdf[1]; + atomic_init(cdf->progress, 0); + } + return 0; +} + +void dav1d_cdf_thread_ref(CdfThreadContext *const dst, + CdfThreadContext *const src) +{ + *dst = *src; + if (src->ref) + dav1d_ref_inc(src->ref); +} + +void dav1d_cdf_thread_unref(CdfThreadContext *const cdf) { + memset(&cdf->data, 0, sizeof(*cdf) - offsetof(CdfThreadContext, data)); + dav1d_ref_dec(&cdf->ref); +} diff --git a/third_party/dav1d/src/cdf.h b/third_party/dav1d/src/cdf.h new file mode 100644 index 0000000000..4b30474baa --- /dev/null +++ b/third_party/dav1d/src/cdf.h @@ -0,0 +1,150 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_CDF_H +#define DAV1D_SRC_CDF_H + +#include + +#include "src/levels.h" +#include "src/ref.h" +#include "src/thread_data.h" + +/* Buffers padded to [8] or [16] for SIMD where needed. */ + +typedef struct CdfModeContext { + ALIGN(uint16_t y_mode[4][N_INTRA_PRED_MODES + 3], 32); + ALIGN(uint16_t uv_mode[2][N_INTRA_PRED_MODES][N_UV_INTRA_PRED_MODES + 2], 32); + ALIGN(uint16_t wedge_idx[9][16], 32); + ALIGN(uint16_t partition[N_BL_LEVELS][4][N_PARTITIONS + 6], 32); + ALIGN(uint16_t cfl_alpha[6][16], 32); + ALIGN(uint16_t txtp_inter1[2][16], 32); + ALIGN(uint16_t txtp_inter2[12 + 4], 32); + ALIGN(uint16_t txtp_intra1[2][N_INTRA_PRED_MODES][7 + 1], 16); + ALIGN(uint16_t txtp_intra2[3][N_INTRA_PRED_MODES][5 + 3], 16); + ALIGN(uint16_t cfl_sign[8], 16); + ALIGN(uint16_t angle_delta[8][8], 16); + ALIGN(uint16_t filter_intra[5 + 3], 16); + ALIGN(uint16_t comp_inter_mode[8][N_COMP_INTER_PRED_MODES], 16); + ALIGN(uint16_t seg_id[3][DAV1D_MAX_SEGMENTS], 16); + ALIGN(uint16_t pal_sz[2][7][7 + 1], 16); + ALIGN(uint16_t color_map[2][7][5][8], 16); + ALIGN(uint16_t filter[2][8][DAV1D_N_SWITCHABLE_FILTERS + 1], 8); + ALIGN(uint16_t txsz[N_TX_SIZES - 1][3][4], 8); + ALIGN(uint16_t motion_mode[N_BS_SIZES][3 + 1], 8); + ALIGN(uint16_t delta_q[4], 8); + ALIGN(uint16_t delta_lf[5][4], 8); + ALIGN(uint16_t interintra_mode[4][4], 8); + ALIGN(uint16_t restore_switchable[3 + 1], 8); + ALIGN(uint16_t restore_wiener[2], 4); + ALIGN(uint16_t restore_sgrproj[2], 4); + ALIGN(uint16_t interintra[7][2], 4); + ALIGN(uint16_t interintra_wedge[7][2], 4); + ALIGN(uint16_t txtp_inter3[4][2], 4); + ALIGN(uint16_t use_filter_intra[N_BS_SIZES][2], 4); + ALIGN(uint16_t newmv_mode[6][2], 4); + ALIGN(uint16_t globalmv_mode[2][2], 4); + ALIGN(uint16_t refmv_mode[6][2], 4); + ALIGN(uint16_t drl_bit[3][2], 4); + ALIGN(uint16_t intra[4][2], 4); + ALIGN(uint16_t comp[5][2], 4); + ALIGN(uint16_t comp_dir[5][2], 4); + ALIGN(uint16_t jnt_comp[6][2], 4); + ALIGN(uint16_t mask_comp[6][2], 4); + ALIGN(uint16_t wedge_comp[9][2], 4); + ALIGN(uint16_t ref[6][3][2], 4); + ALIGN(uint16_t comp_fwd_ref[3][3][2], 4); + ALIGN(uint16_t comp_bwd_ref[2][3][2], 4); + ALIGN(uint16_t comp_uni_ref[3][3][2], 4); + ALIGN(uint16_t txpart[7][3][2], 4); + ALIGN(uint16_t skip[3][2], 4); + ALIGN(uint16_t skip_mode[3][2], 4); + ALIGN(uint16_t seg_pred[3][2], 4); + ALIGN(uint16_t obmc[N_BS_SIZES][2], 4); + ALIGN(uint16_t pal_y[7][3][2], 4); + ALIGN(uint16_t pal_uv[2][2], 4); + ALIGN(uint16_t intrabc[2], 4); +} CdfModeContext; + +typedef struct CdfCoefContext { + ALIGN(uint16_t eob_bin_16[2][2][5 + 3], 16); + ALIGN(uint16_t eob_bin_32[2][2][6 + 2], 16); + ALIGN(uint16_t eob_bin_64[2][2][7 + 1], 16); + ALIGN(uint16_t eob_bin_128[2][2][8 + 0], 16); + ALIGN(uint16_t eob_bin_256[2][2][9 + 7], 32); + ALIGN(uint16_t eob_bin_512[2][10 + 6], 32); + ALIGN(uint16_t eob_bin_1024[2][11 + 5], 32); + ALIGN(uint16_t eob_base_tok[N_TX_SIZES][2][4][4], 8); + ALIGN(uint16_t base_tok[N_TX_SIZES][2][41][4], 8); + ALIGN(uint16_t br_tok[4 /*5*/][2][21][4], 8); + ALIGN(uint16_t eob_hi_bit[N_TX_SIZES][2][11 /*22*/][2], 4); + ALIGN(uint16_t skip[N_TX_SIZES][13][2], 4); + ALIGN(uint16_t dc_sign[2][3][2], 4); +} CdfCoefContext; + +typedef struct CdfMvComponent { + ALIGN(uint16_t classes[11 + 5], 32); + ALIGN(uint16_t class0_fp[2][4], 8); + ALIGN(uint16_t classN_fp[4], 8); + ALIGN(uint16_t class0_hp[2], 4); + ALIGN(uint16_t classN_hp[2], 4); + ALIGN(uint16_t class0[2], 4); + ALIGN(uint16_t classN[10][2], 4); + ALIGN(uint16_t sign[2], 4); +} CdfMvComponent; + +typedef struct CdfMvContext { + CdfMvComponent comp[2]; + ALIGN(uint16_t joint[N_MV_JOINTS], 8); +} CdfMvContext; + +typedef struct CdfContext { + CdfModeContext m; + ALIGN(uint16_t kfym[5][5][N_INTRA_PRED_MODES + 3], 32); + CdfCoefContext coef; + CdfMvContext mv, dmv; +} CdfContext; + +typedef struct CdfThreadContext { + Dav1dRef *ref; ///< allocation origin + union { + CdfContext *cdf; // if ref != NULL + unsigned qcat; // if ref == NULL, from static CDF tables + } data; + atomic_uint *progress; +} CdfThreadContext; + +void dav1d_cdf_thread_init_static(CdfThreadContext *cdf, int qidx); +int dav1d_cdf_thread_alloc(Dav1dContext *c, CdfThreadContext *cdf, + const int have_frame_mt); +void dav1d_cdf_thread_copy(CdfContext *dst, const CdfThreadContext *src); +void dav1d_cdf_thread_ref(CdfThreadContext *dst, CdfThreadContext *src); +void dav1d_cdf_thread_unref(CdfThreadContext *cdf); +void dav1d_cdf_thread_update(const Dav1dFrameHeader *hdr, CdfContext *dst, + const CdfContext *src); + +#endif /* DAV1D_SRC_CDF_H */ diff --git a/third_party/dav1d/src/cpu.c b/third_party/dav1d/src/cpu.c new file mode 100644 index 0000000000..d24148c352 --- /dev/null +++ b/third_party/dav1d/src/cpu.c @@ -0,0 +1,101 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "config.h" + +#include + +#include "src/cpu.h" +#include "src/log.h" + +#ifdef _WIN32 +#include +#elif defined(__APPLE__) +#include +#include +#else +#include +#include +#endif + +#ifdef HAVE_PTHREAD_NP_H +#include +#endif +#if defined(__FreeBSD__) +#define cpu_set_t cpuset_t +#endif + +unsigned dav1d_cpu_flags = 0U; +unsigned dav1d_cpu_flags_mask = ~0U; + +COLD void dav1d_init_cpu(void) { +#if HAVE_ASM && !__has_feature(memory_sanitizer) +// memory sanitizer is inherently incompatible with asm +#if ARCH_AARCH64 || ARCH_ARM + dav1d_cpu_flags = dav1d_get_cpu_flags_arm(); +#elif ARCH_PPC64LE + dav1d_cpu_flags = dav1d_get_cpu_flags_ppc(); +#elif ARCH_X86 + dav1d_cpu_flags = dav1d_get_cpu_flags_x86(); +#endif +#endif +} + +COLD void dav1d_set_cpu_flags_mask(const unsigned mask) { + dav1d_cpu_flags_mask = mask; +} + +COLD int dav1d_num_logical_processors(Dav1dContext *const c) { +#ifdef _WIN32 +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + GROUP_AFFINITY affinity; + if (GetThreadGroupAffinity(GetCurrentThread(), &affinity)) { + int num_processors = 1; + while (affinity.Mask &= affinity.Mask - 1) + num_processors++; + return num_processors; + } +#else + SYSTEM_INFO system_info; + GetNativeSystemInfo(&system_info); + return system_info.dwNumberOfProcessors; +#endif +#elif defined(HAVE_PTHREAD_GETAFFINITY_NP) && defined(CPU_COUNT) + cpu_set_t affinity; + if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) + return CPU_COUNT(&affinity); +#elif defined(__APPLE__) + int num_processors; + size_t length = sizeof(num_processors); + if (!sysctlbyname("hw.logicalcpu", &num_processors, &length, NULL, 0)) + return num_processors; +#elif defined(_SC_NPROCESSORS_ONLN) + return (int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + if (c) + dav1d_log(c, "Unable to detect thread count, defaulting to single-threaded mode\n"); + return 1; +} diff --git a/third_party/dav1d/src/cpu.h b/third_party/dav1d/src/cpu.h new file mode 100644 index 0000000000..8f70fefe54 --- /dev/null +++ b/third_party/dav1d/src/cpu.h @@ -0,0 +1,102 @@ +/* + * Copyright © 2018-2022, VideoLAN and dav1d authors + * Copyright © 2018-2022, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_CPU_H +#define DAV1D_SRC_CPU_H + +#include "config.h" + +#include "common/attributes.h" + +#include "dav1d/common.h" +#include "dav1d/dav1d.h" + +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/cpu.h" +#elif ARCH_PPC64LE +#include "src/ppc/cpu.h" +#elif ARCH_X86 +#include "src/x86/cpu.h" +#endif + +EXTERN unsigned dav1d_cpu_flags; +EXTERN unsigned dav1d_cpu_flags_mask; + +void dav1d_init_cpu(void); +DAV1D_API void dav1d_set_cpu_flags_mask(unsigned mask); +int dav1d_num_logical_processors(Dav1dContext *c); + +static ALWAYS_INLINE unsigned dav1d_get_cpu_flags(void) { + unsigned flags = dav1d_cpu_flags & dav1d_cpu_flags_mask; + +#if TRIM_DSP_FUNCTIONS +/* Since this function is inlined, unconditionally setting a flag here will + * enable dead code elimination in the calling function. */ +#if ARCH_AARCH64 || ARCH_ARM +#if defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32) || ARCH_AARCH64 + flags |= DAV1D_ARM_CPU_FLAG_NEON; +#endif +#elif ARCH_PPC64LE +#if defined(__VSX__) + flags |= DAV1D_PPC_CPU_FLAG_VSX; +#endif +#elif ARCH_X86 +#if defined(__AVX512F__) && defined(__AVX512CD__) && \ + defined(__AVX512BW__) && defined(__AVX512DQ__) && \ + defined(__AVX512VL__) && defined(__AVX512VNNI__) && \ + defined(__AVX512IFMA__) && defined(__AVX512VBMI__) && \ + defined(__AVX512VBMI2__) && defined(__AVX512VPOPCNTDQ__) && \ + defined(__AVX512BITALG__) && defined(__GFNI__) && \ + defined(__VAES__) && defined(__VPCLMULQDQ__) + flags |= DAV1D_X86_CPU_FLAG_AVX512ICL | + DAV1D_X86_CPU_FLAG_AVX2 | + DAV1D_X86_CPU_FLAG_SSE41 | + DAV1D_X86_CPU_FLAG_SSSE3 | + DAV1D_X86_CPU_FLAG_SSE2; +#elif defined(__AVX2__) + flags |= DAV1D_X86_CPU_FLAG_AVX2 | + DAV1D_X86_CPU_FLAG_SSE41 | + DAV1D_X86_CPU_FLAG_SSSE3 | + DAV1D_X86_CPU_FLAG_SSE2; +#elif defined(__SSE4_1__) || defined(__AVX__) + flags |= DAV1D_X86_CPU_FLAG_SSE41 | + DAV1D_X86_CPU_FLAG_SSSE3 | + DAV1D_X86_CPU_FLAG_SSE2; +#elif defined(__SSSE3__) + flags |= DAV1D_X86_CPU_FLAG_SSSE3 | + DAV1D_X86_CPU_FLAG_SSE2; +#elif ARCH_X86_64 || defined(__SSE2__) || \ + (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + flags |= DAV1D_X86_CPU_FLAG_SSE2; +#endif +#endif +#endif + + return flags; +} + +#endif /* DAV1D_SRC_CPU_H */ diff --git a/third_party/dav1d/src/ctx.h b/third_party/dav1d/src/ctx.h new file mode 100644 index 0000000000..d0e1f310ae --- /dev/null +++ b/third_party/dav1d/src/ctx.h @@ -0,0 +1,91 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_CTX_H +#define DAV1D_SRC_CTX_H + +#include + +#include "common/attributes.h" + +union alias64 { uint64_t u64; uint8_t u8[8]; } ATTR_ALIAS; +union alias32 { uint32_t u32; uint8_t u8[4]; } ATTR_ALIAS; +union alias16 { uint16_t u16; uint8_t u8[2]; } ATTR_ALIAS; +union alias8 { uint8_t u8; } ATTR_ALIAS; + +#define set_ctx_rep4(type, var, off, val) do { \ + const uint64_t const_val = val; \ + ((union alias64 *) &var[off + 0])->u64 = const_val; \ + ((union alias64 *) &var[off + 8])->u64 = const_val; \ + ((union alias64 *) &var[off + 16])->u64 = const_val; \ + ((union alias64 *) &var[off + 24])->u64 = const_val; \ + } while (0) +#define set_ctx_rep2(type, var, off, val) do { \ + const uint64_t const_val = val; \ + ((union alias64 *) &var[off + 0])->u64 = const_val; \ + ((union alias64 *) &var[off + 8])->u64 = const_val; \ + } while (0) +#define set_ctx_rep1(typesz, var, off, val) \ + ((union alias##typesz *) &var[off])->u##typesz = val +#define case_set(var, dir, diridx, off) \ + switch (var) { \ + case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ + case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ + case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ + case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ + case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ + case 32: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep4); break; \ + } +#define case_set_upto16(var, dir, diridx, off) \ + switch (var) { \ + case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ + case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ + case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ + case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ + case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ + } +#define case_set_upto32_with_default(var, dir, diridx, off) \ + switch (var) { \ + case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ + case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ + case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ + case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ + case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ + case 32: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep4); break; \ + default: default_memset(dir, diridx, off, var); break; \ + } +#define case_set_upto16_with_default(var, dir, diridx, off) \ + switch (var) { \ + case 1: set_ctx( 8, dir, diridx, off, 0x01, set_ctx_rep1); break; \ + case 2: set_ctx(16, dir, diridx, off, 0x0101, set_ctx_rep1); break; \ + case 4: set_ctx(32, dir, diridx, off, 0x01010101U, set_ctx_rep1); break; \ + case 8: set_ctx(64, dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep1); break; \ + case 16: set_ctx( , dir, diridx, off, 0x0101010101010101ULL, set_ctx_rep2); break; \ + default: default_memset(dir, diridx, off, var); break; \ + } + +#endif /* DAV1D_SRC_CTX_H */ diff --git a/third_party/dav1d/src/data.c b/third_party/dav1d/src/data.c new file mode 100644 index 0000000000..8a1386ad95 --- /dev/null +++ b/third_party/dav1d/src/data.c @@ -0,0 +1,145 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "dav1d/data.h" + +#include "common/attributes.h" +#include "common/validate.h" + +#include "src/data.h" +#include "src/ref.h" + +uint8_t *dav1d_data_create_internal(Dav1dData *const buf, const size_t sz) { + validate_input_or_ret(buf != NULL, NULL); + + if (sz > SIZE_MAX / 2) return NULL; + buf->ref = dav1d_ref_create(sz); + if (!buf->ref) return NULL; + buf->data = buf->ref->const_data; + buf->sz = sz; + dav1d_data_props_set_defaults(&buf->m); + buf->m.size = sz; + + return buf->ref->data; +} + +int dav1d_data_wrap_internal(Dav1dData *const buf, const uint8_t *const ptr, + const size_t sz, + void (*const free_callback)(const uint8_t *data, + void *cookie), + void *const cookie) +{ + validate_input_or_ret(buf != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(ptr != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(free_callback != NULL, DAV1D_ERR(EINVAL)); + + buf->ref = dav1d_ref_wrap(ptr, free_callback, cookie); + if (!buf->ref) return DAV1D_ERR(ENOMEM); + buf->data = ptr; + buf->sz = sz; + dav1d_data_props_set_defaults(&buf->m); + buf->m.size = sz; + + return 0; +} + +int dav1d_data_wrap_user_data_internal(Dav1dData *const buf, + const uint8_t *const user_data, + void (*const free_callback)(const uint8_t *user_data, + void *cookie), + void *const cookie) +{ + validate_input_or_ret(buf != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(free_callback != NULL, DAV1D_ERR(EINVAL)); + + buf->m.user_data.ref = dav1d_ref_wrap(user_data, free_callback, cookie); + if (!buf->m.user_data.ref) return DAV1D_ERR(ENOMEM); + buf->m.user_data.data = user_data; + + return 0; +} + + +void dav1d_data_ref(Dav1dData *const dst, const Dav1dData *const src) { + validate_input(dst != NULL); + validate_input(dst->data == NULL); + validate_input(src != NULL); + + if (src->ref) { + validate_input(src->data != NULL); + dav1d_ref_inc(src->ref); + } + if (src->m.user_data.ref) dav1d_ref_inc(src->m.user_data.ref); + *dst = *src; +} + +void dav1d_data_props_copy(Dav1dDataProps *const dst, + const Dav1dDataProps *const src) +{ + assert(dst != NULL); + assert(src != NULL); + + dav1d_ref_dec(&dst->user_data.ref); + *dst = *src; + if (dst->user_data.ref) dav1d_ref_inc(dst->user_data.ref); +} + +void dav1d_data_props_set_defaults(Dav1dDataProps *const props) { + assert(props != NULL); + + memset(props, 0, sizeof(*props)); + props->timestamp = INT64_MIN; + props->offset = -1; +} + +void dav1d_data_props_unref_internal(Dav1dDataProps *const props) { + validate_input(props != NULL); + + struct Dav1dRef *user_data_ref = props->user_data.ref; + dav1d_data_props_set_defaults(props); + dav1d_ref_dec(&user_data_ref); +} + +void dav1d_data_unref_internal(Dav1dData *const buf) { + validate_input(buf != NULL); + + struct Dav1dRef *user_data_ref = buf->m.user_data.ref; + if (buf->ref) { + validate_input(buf->data != NULL); + dav1d_ref_dec(&buf->ref); + } + memset(buf, 0, sizeof(*buf)); + dav1d_data_props_set_defaults(&buf->m); + dav1d_ref_dec(&user_data_ref); +} diff --git a/third_party/dav1d/src/data.h b/third_party/dav1d/src/data.h new file mode 100644 index 0000000000..b34c1db702 --- /dev/null +++ b/third_party/dav1d/src/data.h @@ -0,0 +1,56 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_DATA_H +#define DAV1D_SRC_DATA_H + +#include "dav1d/data.h" + +void dav1d_data_ref(Dav1dData *dst, const Dav1dData *src); + +/** + * Copy the source properties to the destination and increase the + * user_data's reference count (if it's not NULL). + */ +void dav1d_data_props_copy(Dav1dDataProps *dst, const Dav1dDataProps *src); + +void dav1d_data_props_set_defaults(Dav1dDataProps *props); + +uint8_t *dav1d_data_create_internal(Dav1dData *buf, size_t sz); +int dav1d_data_wrap_internal(Dav1dData *buf, const uint8_t *ptr, size_t sz, + void (*free_callback)(const uint8_t *data, + void *user_data), + void *user_data); +int dav1d_data_wrap_user_data_internal(Dav1dData *buf, + const uint8_t *user_data, + void (*free_callback)(const uint8_t *user_data, + void *cookie), + void *cookie); +void dav1d_data_unref_internal(Dav1dData *buf); +void dav1d_data_props_unref_internal(Dav1dDataProps *props); + +#endif /* DAV1D_SRC_DATA_H */ diff --git a/third_party/dav1d/src/dav1d.rc.in b/third_party/dav1d/src/dav1d.rc.in new file mode 100644 index 0000000000..ad6aab481d --- /dev/null +++ b/third_party/dav1d/src/dav1d.rc.in @@ -0,0 +1,32 @@ +#define API_VERSION_NUMBER @API_VERSION_MAJOR@,@API_VERSION_MINOR@,@API_VERSION_REVISION@,0 +#define API_VERSION_NUMBER_STR "@API_VERSION_MAJOR@.@API_VERSION_MINOR@.@API_VERSION_REVISION@" +#define PROJECT_VERSION_NUMBER @PROJECT_VERSION_MAJOR@,@PROJECT_VERSION_MINOR@,@PROJECT_VERSION_REVISION@,0 +#define PROJECT_VERSION_NUMBER_STR "@PROJECT_VERSION_MAJOR@.@PROJECT_VERSION_MINOR@.@PROJECT_VERSION_REVISION@" + +#include + +1 VERSIONINFO +FILETYPE VFT_DLL +FILEOS VOS_NT_WINDOWS32 +PRODUCTVERSION PROJECT_VERSION_NUMBER +FILEVERSION API_VERSION_NUMBER +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904E4" + BEGIN + VALUE "CompanyName", "VideoLAN" + VALUE "ProductName", "dav1d" + VALUE "ProductVersion", PROJECT_VERSION_NUMBER_STR + VALUE "FileVersion", API_VERSION_NUMBER_STR + VALUE "FileDescription", "dav1d " PROJECT_VERSION_NUMBER_STR " - AV1 decoder" + VALUE "InternalName", "dav1d" + VALUE "OriginalFilename", "libdav1d.dll" + VALUE "LegalCopyright", "Copyright \251 @COPYRIGHT_YEARS@ VideoLAN and dav1d Authors" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c new file mode 100644 index 0000000000..b4853088e2 --- /dev/null +++ b/third_party/dav1d/src/decode.c @@ -0,0 +1,3910 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "dav1d/data.h" + +#include "common/frame.h" +#include "common/intops.h" + +#include "src/ctx.h" +#include "src/decode.h" +#include "src/dequant_tables.h" +#include "src/env.h" +#include "src/filmgrain.h" +#include "src/log.h" +#include "src/qm.h" +#include "src/recon.h" +#include "src/ref.h" +#include "src/tables.h" +#include "src/thread_task.h" +#include "src/warpmv.h" + +static void init_quant_tables(const Dav1dSequenceHeader *const seq_hdr, + const Dav1dFrameHeader *const frame_hdr, + const int qidx, uint16_t (*dq)[3][2]) +{ + for (int i = 0; i < (frame_hdr->segmentation.enabled ? 8 : 1); i++) { + const int yac = frame_hdr->segmentation.enabled ? + iclip_u8(qidx + frame_hdr->segmentation.seg_data.d[i].delta_q) : qidx; + const int ydc = iclip_u8(yac + frame_hdr->quant.ydc_delta); + const int uac = iclip_u8(yac + frame_hdr->quant.uac_delta); + const int udc = iclip_u8(yac + frame_hdr->quant.udc_delta); + const int vac = iclip_u8(yac + frame_hdr->quant.vac_delta); + const int vdc = iclip_u8(yac + frame_hdr->quant.vdc_delta); + + dq[i][0][0] = dav1d_dq_tbl[seq_hdr->hbd][ydc][0]; + dq[i][0][1] = dav1d_dq_tbl[seq_hdr->hbd][yac][1]; + dq[i][1][0] = dav1d_dq_tbl[seq_hdr->hbd][udc][0]; + dq[i][1][1] = dav1d_dq_tbl[seq_hdr->hbd][uac][1]; + dq[i][2][0] = dav1d_dq_tbl[seq_hdr->hbd][vdc][0]; + dq[i][2][1] = dav1d_dq_tbl[seq_hdr->hbd][vac][1]; + } +} + +static int read_mv_component_diff(Dav1dTaskContext *const t, + CdfMvComponent *const mv_comp, + const int have_fp) +{ + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + const int have_hp = f->frame_hdr->hp; + const int sign = dav1d_msac_decode_bool_adapt(&ts->msac, mv_comp->sign); + const int cl = dav1d_msac_decode_symbol_adapt16(&ts->msac, + mv_comp->classes, 10); + int up, fp, hp; + + if (!cl) { + up = dav1d_msac_decode_bool_adapt(&ts->msac, mv_comp->class0); + if (have_fp) { + fp = dav1d_msac_decode_symbol_adapt4(&ts->msac, + mv_comp->class0_fp[up], 3); + hp = have_hp ? dav1d_msac_decode_bool_adapt(&ts->msac, + mv_comp->class0_hp) : 1; + } else { + fp = 3; + hp = 1; + } + } else { + up = 1 << cl; + for (int n = 0; n < cl; n++) + up |= dav1d_msac_decode_bool_adapt(&ts->msac, + mv_comp->classN[n]) << n; + if (have_fp) { + fp = dav1d_msac_decode_symbol_adapt4(&ts->msac, + mv_comp->classN_fp, 3); + hp = have_hp ? dav1d_msac_decode_bool_adapt(&ts->msac, + mv_comp->classN_hp) : 1; + } else { + fp = 3; + hp = 1; + } + } + + const int diff = ((up << 3) | (fp << 1) | hp) + 1; + + return sign ? -diff : diff; +} + +static void read_mv_residual(Dav1dTaskContext *const t, mv *const ref_mv, + CdfMvContext *const mv_cdf, const int have_fp) +{ + switch (dav1d_msac_decode_symbol_adapt4(&t->ts->msac, t->ts->cdf.mv.joint, + N_MV_JOINTS - 1)) + { + case MV_JOINT_HV: + ref_mv->y += read_mv_component_diff(t, &mv_cdf->comp[0], have_fp); + ref_mv->x += read_mv_component_diff(t, &mv_cdf->comp[1], have_fp); + break; + case MV_JOINT_H: + ref_mv->x += read_mv_component_diff(t, &mv_cdf->comp[1], have_fp); + break; + case MV_JOINT_V: + ref_mv->y += read_mv_component_diff(t, &mv_cdf->comp[0], have_fp); + break; + default: + break; + } +} + +static void read_tx_tree(Dav1dTaskContext *const t, + const enum RectTxfmSize from, + const int depth, uint16_t *const masks, + const int x_off, const int y_off) +{ + const Dav1dFrameContext *const f = t->f; + const int bx4 = t->bx & 31, by4 = t->by & 31; + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[from]; + const int txw = t_dim->lw, txh = t_dim->lh; + int is_split; + + if (depth < 2 && from > (int) TX_4X4) { + const int cat = 2 * (TX_64X64 - t_dim->max) - depth; + const int a = t->a->tx[bx4] < txw; + const int l = t->l.tx[by4] < txh; + + is_split = dav1d_msac_decode_bool_adapt(&t->ts->msac, + t->ts->cdf.m.txpart[cat][a + l]); + if (is_split) + masks[depth] |= 1 << (y_off * 4 + x_off); + } else { + is_split = 0; + } + + if (is_split && t_dim->max > TX_8X8) { + const enum RectTxfmSize sub = t_dim->sub; + const TxfmInfo *const sub_t_dim = &dav1d_txfm_dimensions[sub]; + const int txsw = sub_t_dim->w, txsh = sub_t_dim->h; + + read_tx_tree(t, sub, depth + 1, masks, x_off * 2 + 0, y_off * 2 + 0); + t->bx += txsw; + if (txw >= txh && t->bx < f->bw) + read_tx_tree(t, sub, depth + 1, masks, x_off * 2 + 1, y_off * 2 + 0); + t->bx -= txsw; + t->by += txsh; + if (txh >= txw && t->by < f->bh) { + read_tx_tree(t, sub, depth + 1, masks, x_off * 2 + 0, y_off * 2 + 1); + t->bx += txsw; + if (txw >= txh && t->bx < f->bw) + read_tx_tree(t, sub, depth + 1, masks, + x_off * 2 + 1, y_off * 2 + 1); + t->bx -= txsw; + } + t->by -= txsh; + } else { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx, off, is_split ? TX_4X4 : mul * txh) + case_set_upto16(t_dim->h, l., 1, by4); +#undef set_ctx +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx, off, is_split ? TX_4X4 : mul * txw) + case_set_upto16(t_dim->w, a->, 0, bx4); +#undef set_ctx + } +} + +static int neg_deinterleave(int diff, int ref, int max) { + if (!ref) return diff; + if (ref >= (max - 1)) return max - diff - 1; + if (2 * ref < max) { + if (diff <= 2 * ref) { + if (diff & 1) + return ref + ((diff + 1) >> 1); + else + return ref - (diff >> 1); + } + return diff; + } else { + if (diff <= 2 * (max - ref - 1)) { + if (diff & 1) + return ref + ((diff + 1) >> 1); + else + return ref - (diff >> 1); + } + return max - (diff + 1); + } +} + +static void find_matching_ref(const Dav1dTaskContext *const t, + const enum EdgeFlags intra_edge_flags, + const int bw4, const int bh4, + const int w4, const int h4, + const int have_left, const int have_top, + const int ref, uint64_t masks[2]) +{ + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; + int count = 0; + int have_topleft = have_top && have_left; + int have_topright = imax(bw4, bh4) < 32 && + have_top && t->bx + bw4 < t->ts->tiling.col_end && + (intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT); + +#define bs(rp) dav1d_block_dimensions[(rp)->bs] +#define matches(rp) ((rp)->ref.ref[0] == ref + 1 && (rp)->ref.ref[1] == -1) + + if (have_top) { + const refmvs_block *r2 = &r[-1][t->bx]; + if (matches(r2)) { + masks[0] |= 1; + count = 1; + } + int aw4 = bs(r2)[0]; + if (aw4 >= bw4) { + const int off = t->bx & (aw4 - 1); + if (off) have_topleft = 0; + if (aw4 - off > bw4) have_topright = 0; + } else { + unsigned mask = 1 << aw4; + for (int x = aw4; x < w4; x += aw4) { + r2 += aw4; + if (matches(r2)) { + masks[0] |= mask; + if (++count >= 8) return; + } + aw4 = bs(r2)[0]; + mask <<= aw4; + } + } + } + if (have_left) { + /*const*/ refmvs_block *const *r2 = r; + if (matches(&r2[0][t->bx - 1])) { + masks[1] |= 1; + if (++count >= 8) return; + } + int lh4 = bs(&r2[0][t->bx - 1])[1]; + if (lh4 >= bh4) { + if (t->by & (lh4 - 1)) have_topleft = 0; + } else { + unsigned mask = 1 << lh4; + for (int y = lh4; y < h4; y += lh4) { + r2 += lh4; + if (matches(&r2[0][t->bx - 1])) { + masks[1] |= mask; + if (++count >= 8) return; + } + lh4 = bs(&r2[0][t->bx - 1])[1]; + mask <<= lh4; + } + } + } + if (have_topleft && matches(&r[-1][t->bx - 1])) { + masks[1] |= 1ULL << 32; + if (++count >= 8) return; + } + if (have_topright && matches(&r[-1][t->bx + bw4])) { + masks[0] |= 1ULL << 32; + } +#undef matches +} + +static void derive_warpmv(const Dav1dTaskContext *const t, + const int bw4, const int bh4, + const uint64_t masks[2], const union mv mv, + Dav1dWarpedMotionParams *const wmp) +{ + int pts[8][2 /* in, out */][2 /* x, y */], np = 0; + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; + +#define add_sample(dx, dy, sx, sy, rp) do { \ + pts[np][0][0] = 16 * (2 * dx + sx * bs(rp)[0]) - 8; \ + pts[np][0][1] = 16 * (2 * dy + sy * bs(rp)[1]) - 8; \ + pts[np][1][0] = pts[np][0][0] + (rp)->mv.mv[0].x; \ + pts[np][1][1] = pts[np][0][1] + (rp)->mv.mv[0].y; \ + np++; \ +} while (0) + + // use masks[] to find the projectable motion vectors in the edges + if ((unsigned) masks[0] == 1 && !(masks[1] >> 32)) { + const int off = t->bx & (bs(&r[-1][t->bx])[0] - 1); + add_sample(-off, 0, 1, -1, &r[-1][t->bx]); + } else for (unsigned off = 0, xmask = (uint32_t) masks[0]; np < 8 && xmask;) { // top + const int tz = ctz(xmask); + off += tz; + xmask >>= tz; + add_sample(off, 0, 1, -1, &r[-1][t->bx + off]); + xmask &= ~1; + } + if (np < 8 && masks[1] == 1) { + const int off = t->by & (bs(&r[0][t->bx - 1])[1] - 1); + add_sample(0, -off, -1, 1, &r[-off][t->bx - 1]); + } else for (unsigned off = 0, ymask = (uint32_t) masks[1]; np < 8 && ymask;) { // left + const int tz = ctz(ymask); + off += tz; + ymask >>= tz; + add_sample(0, off, -1, 1, &r[off][t->bx - 1]); + ymask &= ~1; + } + if (np < 8 && masks[1] >> 32) // top/left + add_sample(0, 0, -1, -1, &r[-1][t->bx - 1]); + if (np < 8 && masks[0] >> 32) // top/right + add_sample(bw4, 0, 1, -1, &r[-1][t->bx + bw4]); + assert(np > 0 && np <= 8); +#undef bs + + // select according to motion vector difference against a threshold + int mvd[8], ret = 0; + const int thresh = 4 * iclip(imax(bw4, bh4), 4, 28); + for (int i = 0; i < np; i++) { + mvd[i] = abs(pts[i][1][0] - pts[i][0][0] - mv.x) + + abs(pts[i][1][1] - pts[i][0][1] - mv.y); + if (mvd[i] > thresh) + mvd[i] = -1; + else + ret++; + } + if (!ret) { + ret = 1; + } else for (int i = 0, j = np - 1, k = 0; k < np - ret; k++, i++, j--) { + while (mvd[i] != -1) i++; + while (mvd[j] == -1) j--; + assert(i != j); + if (i > j) break; + // replace the discarded samples; + mvd[i] = mvd[j]; + memcpy(pts[i], pts[j], sizeof(*pts)); + } + + if (!dav1d_find_affine_int(pts, ret, bw4, bh4, mv, wmp, t->bx, t->by) && + !dav1d_get_shear_params(wmp)) + { + wmp->type = DAV1D_WM_TYPE_AFFINE; + } else + wmp->type = DAV1D_WM_TYPE_IDENTITY; +} + +static inline int findoddzero(const uint8_t *buf, int len) { + for (int n = 0; n < len; n++) + if (!buf[n * 2]) return 1; + return 0; +} + +static void read_pal_plane(Dav1dTaskContext *const t, Av1Block *const b, + const int pl, const int sz_ctx, + const int bx4, const int by4) +{ + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + const int pal_sz = b->pal_sz[pl] = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.pal_sz[pl][sz_ctx], 6) + 2; + uint16_t cache[16], used_cache[8]; + int l_cache = pl ? t->pal_sz_uv[1][by4] : t->l.pal_sz[by4]; + int n_cache = 0; + // don't reuse above palette outside SB64 boundaries + int a_cache = by4 & 15 ? pl ? t->pal_sz_uv[0][bx4] : t->a->pal_sz[bx4] : 0; + const uint16_t *l = t->al_pal[1][by4][pl], *a = t->al_pal[0][bx4][pl]; + + // fill/sort cache + while (l_cache && a_cache) { + if (*l < *a) { + if (!n_cache || cache[n_cache - 1] != *l) + cache[n_cache++] = *l; + l++; + l_cache--; + } else { + if (*a == *l) { + l++; + l_cache--; + } + if (!n_cache || cache[n_cache - 1] != *a) + cache[n_cache++] = *a; + a++; + a_cache--; + } + } + if (l_cache) { + do { + if (!n_cache || cache[n_cache - 1] != *l) + cache[n_cache++] = *l; + l++; + } while (--l_cache > 0); + } else if (a_cache) { + do { + if (!n_cache || cache[n_cache - 1] != *a) + cache[n_cache++] = *a; + a++; + } while (--a_cache > 0); + } + + // find reused cache entries + int i = 0; + for (int n = 0; n < n_cache && i < pal_sz; n++) + if (dav1d_msac_decode_bool_equi(&ts->msac)) + used_cache[i++] = cache[n]; + const int n_used_cache = i; + + // parse new entries + uint16_t *const pal = t->frame_thread.pass ? + f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + + ((t->bx >> 1) + (t->by & 1))][pl] : t->scratch.pal[pl]; + if (i < pal_sz) { + int prev = pal[i++] = dav1d_msac_decode_bools(&ts->msac, f->cur.p.bpc); + + if (i < pal_sz) { + int bits = f->cur.p.bpc - 3 + dav1d_msac_decode_bools(&ts->msac, 2); + const int max = (1 << f->cur.p.bpc) - 1; + + do { + const int delta = dav1d_msac_decode_bools(&ts->msac, bits); + prev = pal[i++] = imin(prev + delta + !pl, max); + if (prev + !pl >= max) { + for (; i < pal_sz; i++) + pal[i] = max; + break; + } + bits = imin(bits, 1 + ulog2(max - prev - !pl)); + } while (i < pal_sz); + } + + // merge cache+new entries + int n = 0, m = n_used_cache; + for (i = 0; i < pal_sz; i++) { + if (n < n_used_cache && (m >= pal_sz || used_cache[n] <= pal[m])) { + pal[i] = used_cache[n++]; + } else { + assert(m < pal_sz); + pal[i] = pal[m++]; + } + } + } else { + memcpy(pal, used_cache, n_used_cache * sizeof(*used_cache)); + } + + if (DEBUG_BLOCK_INFO) { + printf("Post-pal[pl=%d,sz=%d,cache_size=%d,used_cache=%d]: r=%d, cache=", + pl, pal_sz, n_cache, n_used_cache, ts->msac.rng); + for (int n = 0; n < n_cache; n++) + printf("%c%02x", n ? ' ' : '[', cache[n]); + printf("%s, pal=", n_cache ? "]" : "[]"); + for (int n = 0; n < pal_sz; n++) + printf("%c%02x", n ? ' ' : '[', pal[n]); + printf("]\n"); + } +} + +static void read_pal_uv(Dav1dTaskContext *const t, Av1Block *const b, + const int sz_ctx, const int bx4, const int by4) +{ + read_pal_plane(t, b, 1, sz_ctx, bx4, by4); + + // V pal coding + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + uint16_t *const pal = t->frame_thread.pass ? + f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + + ((t->bx >> 1) + (t->by & 1))][2] : t->scratch.pal[2]; + if (dav1d_msac_decode_bool_equi(&ts->msac)) { + const int bits = f->cur.p.bpc - 4 + + dav1d_msac_decode_bools(&ts->msac, 2); + int prev = pal[0] = dav1d_msac_decode_bools(&ts->msac, f->cur.p.bpc); + const int max = (1 << f->cur.p.bpc) - 1; + for (int i = 1; i < b->pal_sz[1]; i++) { + int delta = dav1d_msac_decode_bools(&ts->msac, bits); + if (delta && dav1d_msac_decode_bool_equi(&ts->msac)) delta = -delta; + prev = pal[i] = (prev + delta) & max; + } + } else { + for (int i = 0; i < b->pal_sz[1]; i++) + pal[i] = dav1d_msac_decode_bools(&ts->msac, f->cur.p.bpc); + } + if (DEBUG_BLOCK_INFO) { + printf("Post-pal[pl=2]: r=%d ", ts->msac.rng); + for (int n = 0; n < b->pal_sz[1]; n++) + printf("%c%02x", n ? ' ' : '[', pal[n]); + printf("]\n"); + } +} + +// meant to be SIMD'able, so that theoretical complexity of this function +// times block size goes from w4*h4 to w4+h4-1 +// a and b are previous two lines containing (a) top/left entries or (b) +// top/left entries, with a[0] being either the first top or first left entry, +// depending on top_offset being 1 or 0, and b being the first top/left entry +// for whichever has one. left_offset indicates whether the (len-1)th entry +// has a left neighbour. +// output is order[] and ctx for each member of this diagonal. +static void order_palette(const uint8_t *pal_idx, const ptrdiff_t stride, + const int i, const int first, const int last, + uint8_t (*const order)[8], uint8_t *const ctx) +{ + int have_top = i > first; + + assert(pal_idx); + pal_idx += first + (i - first) * stride; + for (int j = first, n = 0; j >= last; have_top = 1, j--, n++, pal_idx += stride - 1) { + const int have_left = j > 0; + + assert(have_left || have_top); + +#define add(v_in) do { \ + const int v = v_in; \ + assert((unsigned)v < 8U); \ + order[n][o_idx++] = v; \ + mask |= 1 << v; \ + } while (0) + + unsigned mask = 0; + int o_idx = 0; + if (!have_left) { + ctx[n] = 0; + add(pal_idx[-stride]); + } else if (!have_top) { + ctx[n] = 0; + add(pal_idx[-1]); + } else { + const int l = pal_idx[-1], t = pal_idx[-stride], tl = pal_idx[-(stride + 1)]; + const int same_t_l = t == l; + const int same_t_tl = t == tl; + const int same_l_tl = l == tl; + const int same_all = same_t_l & same_t_tl & same_l_tl; + + if (same_all) { + ctx[n] = 4; + add(t); + } else if (same_t_l) { + ctx[n] = 3; + add(t); + add(tl); + } else if (same_t_tl | same_l_tl) { + ctx[n] = 2; + add(tl); + add(same_t_tl ? l : t); + } else { + ctx[n] = 1; + add(imin(t, l)); + add(imax(t, l)); + add(tl); + } + } + for (unsigned m = 1, bit = 0; m < 0x100; m <<= 1, bit++) + if (!(mask & m)) + order[n][o_idx++] = bit; + assert(o_idx == 8); +#undef add + } +} + +static void read_pal_indices(Dav1dTaskContext *const t, + uint8_t *const pal_idx, + const Av1Block *const b, const int pl, + const int w4, const int h4, + const int bw4, const int bh4) +{ + Dav1dTileState *const ts = t->ts; + const ptrdiff_t stride = bw4 * 4; + assert(pal_idx); + pal_idx[0] = dav1d_msac_decode_uniform(&ts->msac, b->pal_sz[pl]); + uint16_t (*const color_map_cdf)[8] = + ts->cdf.m.color_map[pl][b->pal_sz[pl] - 2]; + uint8_t (*const order)[8] = t->scratch.pal_order; + uint8_t *const ctx = t->scratch.pal_ctx; + for (int i = 1; i < 4 * (w4 + h4) - 1; i++) { + // top/left-to-bottom/right diagonals ("wave-front") + const int first = imin(i, w4 * 4 - 1); + const int last = imax(0, i - h4 * 4 + 1); + order_palette(pal_idx, stride, i, first, last, order, ctx); + for (int j = first, m = 0; j >= last; j--, m++) { + const int color_idx = dav1d_msac_decode_symbol_adapt8(&ts->msac, + color_map_cdf[ctx[m]], b->pal_sz[pl] - 1); + pal_idx[(i - j) * stride + j] = order[m][color_idx]; + } + } + // fill invisible edges + if (bw4 > w4) + for (int y = 0; y < 4 * h4; y++) + memset(&pal_idx[y * stride + 4 * w4], + pal_idx[y * stride + 4 * w4 - 1], 4 * (bw4 - w4)); + if (h4 < bh4) { + const uint8_t *const src = &pal_idx[stride * (4 * h4 - 1)]; + for (int y = h4 * 4; y < bh4 * 4; y++) + memcpy(&pal_idx[y * stride], src, bw4 * 4); + } +} + +static void read_vartx_tree(Dav1dTaskContext *const t, + Av1Block *const b, const enum BlockSize bs, + const int bx4, const int by4) +{ + const Dav1dFrameContext *const f = t->f; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + + // var-tx tree coding + uint16_t tx_split[2] = { 0 }; + b->max_ytx = dav1d_max_txfm_size_for_bs[bs][0]; + if (!b->skip && (f->frame_hdr->segmentation.lossless[b->seg_id] || + b->max_ytx == TX_4X4)) + { + b->max_ytx = b->uvtx = TX_4X4; + if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx, off, TX_4X4) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + } + } else if (f->frame_hdr->txfm_mode != DAV1D_TX_SWITCHABLE || b->skip) { + if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx, off, mul * b_dim[2 + diridx]) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + } + b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; + } else { + assert(bw4 <= 16 || bh4 <= 16 || b->max_ytx == TX_64X64); + int y, x, y_off, x_off; + const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; + for (y = 0, y_off = 0; y < bh4; y += ytx->h, y_off++) { + for (x = 0, x_off = 0; x < bw4; x += ytx->w, x_off++) { + read_tx_tree(t, b->max_ytx, 0, tx_split, x_off, y_off); + // contexts are updated inside read_tx_tree() + t->bx += ytx->w; + } + t->bx -= x; + t->by += ytx->h; + } + t->by -= y; + if (DEBUG_BLOCK_INFO) + printf("Post-vartxtree[%x/%x]: r=%d\n", + tx_split[0], tx_split[1], t->ts->msac.rng); + b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; + } + assert(!(tx_split[0] & ~0x33)); + b->tx_split0 = (uint8_t)tx_split[0]; + b->tx_split1 = tx_split[1]; +} + +static inline unsigned get_prev_frame_segid(const Dav1dFrameContext *const f, + const int by, const int bx, + const int w4, int h4, + const uint8_t *ref_seg_map, + const ptrdiff_t stride) +{ + assert(f->frame_hdr->primary_ref_frame != DAV1D_PRIMARY_REF_NONE); + + unsigned seg_id = 8; + ref_seg_map += by * stride + bx; + do { + for (int x = 0; x < w4; x++) + seg_id = imin(seg_id, ref_seg_map[x]); + ref_seg_map += stride; + } while (--h4 > 0 && seg_id); + assert(seg_id < 8); + + return seg_id; +} + +static inline void splat_oneref_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + const enum InterPredMode mode = b->inter_mode; + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { b->ref[0] + 1, b->interintra_type ? 0 : -1 }, + .mv.mv[0] = b->mv[0], + .bs = bs, + .mf = (mode == GLOBALMV && imin(bw4, bh4) >= 2) | ((mode == NEWMV) * 2), + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_intrabc_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0] = b->mv[0], + .bs = bs, + .mf = 0, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_tworef_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + assert(bw4 >= 2 && bh4 >= 2); + const enum CompInterPredMode mode = b->inter_mode; + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { b->ref[0] + 1, b->ref[1] + 1 }, + .mv.mv = { b->mv[0], b->mv[1] }, + .bs = bs, + .mf = (mode == GLOBALMV_GLOBALMV) | !!((1 << mode) & (0xbc)) * 2, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_intraref(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const int bw4, const int bh4) +{ + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0].n = INVALID_MV, + .bs = bs, + .mf = 0, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static void mc_lowest_px(int *const dst, const int by4, const int bh4, + const int mvy, const int ss_ver, + const struct ScalableMotionParams *const smp) +{ + const int v_mul = 4 >> ss_ver; + if (!smp->scale) { + const int my = mvy >> (3 + ss_ver), dy = mvy & (15 >> !ss_ver); + *dst = imax(*dst, (by4 + bh4) * v_mul + my + 4 * !!dy); + } else { + int y = (by4 * v_mul << 4) + mvy * (1 << !ss_ver); + const int64_t tmp = (int64_t)(y) * smp->scale + (smp->scale - 0x4000) * 8; + y = apply_sign64((int)((llabs(tmp) + 128) >> 8), tmp) + 32; + const int bottom = ((y + (bh4 * v_mul - 1) * smp->step) >> 10) + 1 + 4; + *dst = imax(*dst, bottom); + } +} + +static ALWAYS_INLINE void affine_lowest_px(Dav1dTaskContext *const t, int *const dst, + const uint8_t *const b_dim, + const Dav1dWarpedMotionParams *const wmp, + const int ss_ver, const int ss_hor) +{ + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + assert(!((b_dim[0] * h_mul) & 7) && !((b_dim[1] * v_mul) & 7)); + const int32_t *const mat = wmp->matrix; + const int y = b_dim[1] * v_mul - 8; // lowest y + + const int src_y = t->by * 4 + ((y + 4) << ss_ver); + const int64_t mat5_y = (int64_t) mat[5] * src_y + mat[1]; + // check left- and right-most blocks + for (int x = 0; x < b_dim[0] * h_mul; x += imax(8, b_dim[0] * h_mul - 8)) { + // calculate transformation relative to center of 8x8 block in + // luma pixel units + const int src_x = t->bx * 4 + ((x + 4) << ss_hor); + const int64_t mvy = ((int64_t) mat[4] * src_x + mat5_y) >> ss_ver; + const int dy = (int) (mvy >> 16) - 4; + *dst = imax(*dst, dy + 4 + 8); + } +} + +static NOINLINE void affine_lowest_px_luma(Dav1dTaskContext *const t, int *const dst, + const uint8_t *const b_dim, + const Dav1dWarpedMotionParams *const wmp) +{ + affine_lowest_px(t, dst, b_dim, wmp, 0, 0); +} + +static NOINLINE void affine_lowest_px_chroma(Dav1dTaskContext *const t, int *const dst, + const uint8_t *const b_dim, + const Dav1dWarpedMotionParams *const wmp) +{ + const Dav1dFrameContext *const f = t->f; + assert(f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400); + if (f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I444) + affine_lowest_px_luma(t, dst, b_dim, wmp); + else + affine_lowest_px(t, dst, b_dim, wmp, f->cur.p.layout & DAV1D_PIXEL_LAYOUT_I420, 1); +} + +static void obmc_lowest_px(Dav1dTaskContext *const t, + int (*const dst)[2], const int is_chroma, + const uint8_t *const b_dim, + const int bx4, const int by4, const int w4, const int h4) +{ + assert(!(t->bx & 1) && !(t->by & 1)); + const Dav1dFrameContext *const f = t->f; + /*const*/ refmvs_block **r = &t->rt.r[(t->by & 31) + 5]; + const int ss_ver = is_chroma && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = is_chroma && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + + if (t->by > t->ts->tiling.row_start && + (!is_chroma || b_dim[0] * h_mul + b_dim[1] * v_mul >= 16)) + { + for (int i = 0, x = 0; x < w4 && i < imin(b_dim[2], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const a_r = &r[-1][t->bx + x + 1]; + const uint8_t *const a_b_dim = dav1d_block_dimensions[a_r->bs]; + + if (a_r->ref.ref[0] > 0) { + const int oh4 = imin(b_dim[1], 16) >> 1; + mc_lowest_px(&dst[a_r->ref.ref[0] - 1][is_chroma], t->by, + (oh4 * 3 + 3) >> 2, a_r->mv.mv[0].y, ss_ver, + &f->svc[a_r->ref.ref[0] - 1][1]); + i++; + } + x += imax(a_b_dim[0], 2); + } + } + + if (t->bx > t->ts->tiling.col_start) + for (int i = 0, y = 0; y < h4 && i < imin(b_dim[3], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const l_r = &r[y + 1][t->bx - 1]; + const uint8_t *const l_b_dim = dav1d_block_dimensions[l_r->bs]; + + if (l_r->ref.ref[0] > 0) { + const int oh4 = iclip(l_b_dim[1], 2, b_dim[1]); + mc_lowest_px(&dst[l_r->ref.ref[0] - 1][is_chroma], + t->by + y, oh4, l_r->mv.mv[0].y, ss_ver, + &f->svc[l_r->ref.ref[0] - 1][1]); + i++; + } + y += imax(l_b_dim[1], 2); + } +} + +static int decode_b(Dav1dTaskContext *const t, + const enum BlockLevel bl, + const enum BlockSize bs, + const enum BlockPartition bp, + const enum EdgeFlags intra_edge_flags) +{ + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + Av1Block b_mem, *const b = t->frame_thread.pass ? + &f->frame_thread.b[t->by * f->b4_stride + t->bx] : &b_mem; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bx4 = t->bx & 31, by4 = t->by & 31; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int cbx4 = bx4 >> ss_hor, cby4 = by4 >> ss_ver; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); + const int cbw4 = (bw4 + ss_hor) >> ss_hor, cbh4 = (bh4 + ss_ver) >> ss_ver; + const int have_left = t->bx > ts->tiling.col_start; + const int have_top = t->by > ts->tiling.row_start; + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 && + (bw4 > ss_hor || t->bx & 1) && + (bh4 > ss_ver || t->by & 1); + + if (t->frame_thread.pass == 2) { + if (b->intra) { + f->bd_fn.recon_b_intra(t, bs, intra_edge_flags, b); + + const enum IntraPredMode y_mode_nofilt = + b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ + rep_macro(type, t->dir intra, off, mul) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = 0; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = 0; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } + + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + } else { + if (IS_INTER_OR_SWITCH(f->frame_hdr) /* not intrabc */ && + b->comp_type == COMP_INTER_NONE && b->motion_mode == MM_WARP) + { + if (b->matrix[0] == SHRT_MIN) { + t->warpmv.type = DAV1D_WM_TYPE_IDENTITY; + } else { + t->warpmv.type = DAV1D_WM_TYPE_AFFINE; + t->warpmv.matrix[2] = b->matrix[0] + 0x10000; + t->warpmv.matrix[3] = b->matrix[1]; + t->warpmv.matrix[4] = b->matrix[2]; + t->warpmv.matrix[5] = b->matrix[3] + 0x10000; + dav1d_set_affine_mv2d(bw4, bh4, b->mv2d, &t->warpmv, + t->bx, t->by); + dav1d_get_shear_params(&t->warpmv); +#define signabs(v) v < 0 ? '-' : ' ', abs(v) + if (DEBUG_BLOCK_INFO) + printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, mv=y:%d,x:%d\n", + signabs(t->warpmv.matrix[0]), + signabs(t->warpmv.matrix[1]), + signabs(t->warpmv.matrix[2]), + signabs(t->warpmv.matrix[3]), + signabs(t->warpmv.matrix[4]), + signabs(t->warpmv.matrix[5]), + signabs(t->warpmv.u.p.alpha), + signabs(t->warpmv.u.p.beta), + signabs(t->warpmv.u.p.gamma), + signabs(t->warpmv.u.p.delta), + b->mv2d.y, b->mv2d.x); +#undef signabs + } + } + if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; + + const uint8_t *const filter = dav1d_filter_dir[b->filter2d]; +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir filter[0], off, mul * filter[0]); \ + rep_macro(type, t->dir filter[1], off, mul * filter[1]); \ + rep_macro(type, t->dir intra, off, 0) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = b->ref[0] + 1; + r[x].mv.mv[0] = b->mv[0]; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = b->ref[0] + 1; + rr[y][t->bx + bw4 - 1].mv.mv[0] = b->mv[0]; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } + + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir uvmode, off, mul * DC_PRED) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + } + return 0; + } + + const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; + + b->bl = bl; + b->bp = bp; + b->bs = bs; + + const Dav1dSegmentationData *seg = NULL; + + // segment_id (if seg_feature for skip/ref/gmv is enabled) + int seg_pred = 0; + if (f->frame_hdr->segmentation.enabled) { + if (!f->frame_hdr->segmentation.update_map) { + if (f->prev_segmap) { + unsigned seg_id = get_prev_frame_segid(f, t->by, t->bx, w4, h4, + f->prev_segmap, + f->b4_stride); + if (seg_id >= 8) return -1; + b->seg_id = seg_id; + } else { + b->seg_id = 0; + } + seg = &f->frame_hdr->segmentation.seg_data.d[b->seg_id]; + } else if (f->frame_hdr->segmentation.seg_data.preskip) { + if (f->frame_hdr->segmentation.temporal && + (seg_pred = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.seg_pred[t->a->seg_pred[bx4] + + t->l.seg_pred[by4]]))) + { + // temporal predicted seg_id + if (f->prev_segmap) { + unsigned seg_id = get_prev_frame_segid(f, t->by, t->bx, + w4, h4, + f->prev_segmap, + f->b4_stride); + if (seg_id >= 8) return -1; + b->seg_id = seg_id; + } else { + b->seg_id = 0; + } + } else { + int seg_ctx; + const unsigned pred_seg_id = + get_cur_frame_segid(t->by, t->bx, have_top, have_left, + &seg_ctx, f->cur_segmap, f->b4_stride); + const unsigned diff = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.seg_id[seg_ctx], + DAV1D_MAX_SEGMENTS - 1); + const unsigned last_active_seg_id = + f->frame_hdr->segmentation.seg_data.last_active_segid; + b->seg_id = neg_deinterleave(diff, pred_seg_id, + last_active_seg_id + 1); + if (b->seg_id > last_active_seg_id) b->seg_id = 0; // error? + if (b->seg_id >= DAV1D_MAX_SEGMENTS) b->seg_id = 0; // error? + } + + if (DEBUG_BLOCK_INFO) + printf("Post-segid[preskip;%d]: r=%d\n", + b->seg_id, ts->msac.rng); + + seg = &f->frame_hdr->segmentation.seg_data.d[b->seg_id]; + } + } else { + b->seg_id = 0; + } + + // skip_mode + if ((!seg || (!seg->globalmv && seg->ref == -1 && !seg->skip)) && + f->frame_hdr->skip_mode_enabled && imin(bw4, bh4) > 1) + { + const int smctx = t->a->skip_mode[bx4] + t->l.skip_mode[by4]; + b->skip_mode = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.skip_mode[smctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-skipmode[%d]: r=%d\n", b->skip_mode, ts->msac.rng); + } else { + b->skip_mode = 0; + } + + // skip + if (b->skip_mode || (seg && seg->skip)) { + b->skip = 1; + } else { + const int sctx = t->a->skip[bx4] + t->l.skip[by4]; + b->skip = dav1d_msac_decode_bool_adapt(&ts->msac, ts->cdf.m.skip[sctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-skip[%d]: r=%d\n", b->skip, ts->msac.rng); + } + + // segment_id + if (f->frame_hdr->segmentation.enabled && + f->frame_hdr->segmentation.update_map && + !f->frame_hdr->segmentation.seg_data.preskip) + { + if (!b->skip && f->frame_hdr->segmentation.temporal && + (seg_pred = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.seg_pred[t->a->seg_pred[bx4] + + t->l.seg_pred[by4]]))) + { + // temporal predicted seg_id + if (f->prev_segmap) { + unsigned seg_id = get_prev_frame_segid(f, t->by, t->bx, w4, h4, + f->prev_segmap, + f->b4_stride); + if (seg_id >= 8) return -1; + b->seg_id = seg_id; + } else { + b->seg_id = 0; + } + } else { + int seg_ctx; + const unsigned pred_seg_id = + get_cur_frame_segid(t->by, t->bx, have_top, have_left, + &seg_ctx, f->cur_segmap, f->b4_stride); + if (b->skip) { + b->seg_id = pred_seg_id; + } else { + const unsigned diff = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.seg_id[seg_ctx], + DAV1D_MAX_SEGMENTS - 1); + const unsigned last_active_seg_id = + f->frame_hdr->segmentation.seg_data.last_active_segid; + b->seg_id = neg_deinterleave(diff, pred_seg_id, + last_active_seg_id + 1); + if (b->seg_id > last_active_seg_id) b->seg_id = 0; // error? + } + if (b->seg_id >= DAV1D_MAX_SEGMENTS) b->seg_id = 0; // error? + } + + seg = &f->frame_hdr->segmentation.seg_data.d[b->seg_id]; + + if (DEBUG_BLOCK_INFO) + printf("Post-segid[postskip;%d]: r=%d\n", + b->seg_id, ts->msac.rng); + } + + // cdef index + if (!b->skip) { + const int idx = f->seq_hdr->sb128 ? ((t->bx & 16) >> 4) + + ((t->by & 16) >> 3) : 0; + if (t->cur_sb_cdef_idx_ptr[idx] == -1) { + const int v = dav1d_msac_decode_bools(&ts->msac, + f->frame_hdr->cdef.n_bits); + t->cur_sb_cdef_idx_ptr[idx] = v; + if (bw4 > 16) t->cur_sb_cdef_idx_ptr[idx + 1] = v; + if (bh4 > 16) t->cur_sb_cdef_idx_ptr[idx + 2] = v; + if (bw4 == 32 && bh4 == 32) t->cur_sb_cdef_idx_ptr[idx + 3] = v; + + if (DEBUG_BLOCK_INFO) + printf("Post-cdef_idx[%d]: r=%d\n", + *t->cur_sb_cdef_idx_ptr, ts->msac.rng); + } + } + + // delta-q/lf + if (!(t->bx & (31 >> !f->seq_hdr->sb128)) && + !(t->by & (31 >> !f->seq_hdr->sb128))) + { + const int prev_qidx = ts->last_qidx; + const int have_delta_q = f->frame_hdr->delta.q.present && + (bs != (f->seq_hdr->sb128 ? BS_128x128 : BS_64x64) || !b->skip); + + int8_t prev_delta_lf[4]; + memcpy(prev_delta_lf, ts->last_delta_lf, 4); + + if (have_delta_q) { + int delta_q = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.delta_q, 3); + if (delta_q == 3) { + const int n_bits = 1 + dav1d_msac_decode_bools(&ts->msac, 3); + delta_q = dav1d_msac_decode_bools(&ts->msac, n_bits) + + 1 + (1 << n_bits); + } + if (delta_q) { + if (dav1d_msac_decode_bool_equi(&ts->msac)) delta_q = -delta_q; + delta_q *= 1 << f->frame_hdr->delta.q.res_log2; + } + ts->last_qidx = iclip(ts->last_qidx + delta_q, 1, 255); + if (have_delta_q && DEBUG_BLOCK_INFO) + printf("Post-delta_q[%d->%d]: r=%d\n", + delta_q, ts->last_qidx, ts->msac.rng); + + if (f->frame_hdr->delta.lf.present) { + const int n_lfs = f->frame_hdr->delta.lf.multi ? + f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 ? 4 : 2 : 1; + + for (int i = 0; i < n_lfs; i++) { + int delta_lf = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.delta_lf[i + f->frame_hdr->delta.lf.multi], 3); + if (delta_lf == 3) { + const int n_bits = 1 + dav1d_msac_decode_bools(&ts->msac, 3); + delta_lf = dav1d_msac_decode_bools(&ts->msac, n_bits) + + 1 + (1 << n_bits); + } + if (delta_lf) { + if (dav1d_msac_decode_bool_equi(&ts->msac)) + delta_lf = -delta_lf; + delta_lf *= 1 << f->frame_hdr->delta.lf.res_log2; + } + ts->last_delta_lf[i] = + iclip(ts->last_delta_lf[i] + delta_lf, -63, 63); + if (have_delta_q && DEBUG_BLOCK_INFO) + printf("Post-delta_lf[%d:%d]: r=%d\n", i, delta_lf, + ts->msac.rng); + } + } + } + if (ts->last_qidx == f->frame_hdr->quant.yac) { + // assign frame-wide q values to this sb + ts->dq = f->dq; + } else if (ts->last_qidx != prev_qidx) { + // find sb-specific quant parameters + init_quant_tables(f->seq_hdr, f->frame_hdr, ts->last_qidx, ts->dqmem); + ts->dq = ts->dqmem; + } + if (!memcmp(ts->last_delta_lf, (int8_t[4]) { 0, 0, 0, 0 }, 4)) { + // assign frame-wide lf values to this sb + ts->lflvl = f->lf.lvl; + } else if (memcmp(ts->last_delta_lf, prev_delta_lf, 4)) { + // find sb-specific lf lvl parameters + dav1d_calc_lf_values(ts->lflvlmem, f->frame_hdr, ts->last_delta_lf); + ts->lflvl = ts->lflvlmem; + } + } + + if (b->skip_mode) { + b->intra = 0; + } else if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + if (seg && (seg->ref >= 0 || seg->globalmv)) { + b->intra = !seg->ref; + } else { + const int ictx = get_intra_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->intra = !dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.intra[ictx]); + if (DEBUG_BLOCK_INFO) + printf("Post-intra[%d]: r=%d\n", b->intra, ts->msac.rng); + } + } else if (f->frame_hdr->allow_intrabc) { + b->intra = !dav1d_msac_decode_bool_adapt(&ts->msac, ts->cdf.m.intrabc); + if (DEBUG_BLOCK_INFO) + printf("Post-intrabcflag[%d]: r=%d\n", b->intra, ts->msac.rng); + } else { + b->intra = 1; + } + + // intra/inter-specific stuff + if (b->intra) { + uint16_t *const ymode_cdf = IS_INTER_OR_SWITCH(f->frame_hdr) ? + ts->cdf.m.y_mode[dav1d_ymode_size_context[bs]] : + ts->cdf.kfym[dav1d_intra_mode_context[t->a->mode[bx4]]] + [dav1d_intra_mode_context[t->l.mode[by4]]]; + b->y_mode = dav1d_msac_decode_symbol_adapt16(&ts->msac, ymode_cdf, + N_INTRA_PRED_MODES - 1); + if (DEBUG_BLOCK_INFO) + printf("Post-ymode[%d]: r=%d\n", b->y_mode, ts->msac.rng); + + // angle delta + if (b_dim[2] + b_dim[3] >= 2 && b->y_mode >= VERT_PRED && + b->y_mode <= VERT_LEFT_PRED) + { + uint16_t *const acdf = ts->cdf.m.angle_delta[b->y_mode - VERT_PRED]; + const int angle = dav1d_msac_decode_symbol_adapt8(&ts->msac, acdf, 6); + b->y_angle = angle - 3; + } else { + b->y_angle = 0; + } + + if (has_chroma) { + const int cfl_allowed = f->frame_hdr->segmentation.lossless[b->seg_id] ? + cbw4 == 1 && cbh4 == 1 : !!(cfl_allowed_mask & (1 << bs)); + uint16_t *const uvmode_cdf = ts->cdf.m.uv_mode[cfl_allowed][b->y_mode]; + b->uv_mode = dav1d_msac_decode_symbol_adapt16(&ts->msac, uvmode_cdf, + N_UV_INTRA_PRED_MODES - 1 - !cfl_allowed); + if (DEBUG_BLOCK_INFO) + printf("Post-uvmode[%d]: r=%d\n", b->uv_mode, ts->msac.rng); + + b->uv_angle = 0; + if (b->uv_mode == CFL_PRED) { +#define SIGN(a) (!!(a) + ((a) > 0)) + const int sign = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.cfl_sign, 7) + 1; + const int sign_u = sign * 0x56 >> 8, sign_v = sign - sign_u * 3; + assert(sign_u == sign / 3); + if (sign_u) { + const int ctx = (sign_u == 2) * 3 + sign_v; + b->cfl_alpha[0] = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.cfl_alpha[ctx], 15) + 1; + if (sign_u == 1) b->cfl_alpha[0] = -b->cfl_alpha[0]; + } else { + b->cfl_alpha[0] = 0; + } + if (sign_v) { + const int ctx = (sign_v == 2) * 3 + sign_u; + b->cfl_alpha[1] = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.cfl_alpha[ctx], 15) + 1; + if (sign_v == 1) b->cfl_alpha[1] = -b->cfl_alpha[1]; + } else { + b->cfl_alpha[1] = 0; + } +#undef SIGN + if (DEBUG_BLOCK_INFO) + printf("Post-uvalphas[%d/%d]: r=%d\n", + b->cfl_alpha[0], b->cfl_alpha[1], ts->msac.rng); + } else if (b_dim[2] + b_dim[3] >= 2 && b->uv_mode >= VERT_PRED && + b->uv_mode <= VERT_LEFT_PRED) + { + uint16_t *const acdf = ts->cdf.m.angle_delta[b->uv_mode - VERT_PRED]; + const int angle = dav1d_msac_decode_symbol_adapt8(&ts->msac, acdf, 6); + b->uv_angle = angle - 3; + } + } + + b->pal_sz[0] = b->pal_sz[1] = 0; + if (f->frame_hdr->allow_screen_content_tools && + imax(bw4, bh4) <= 16 && bw4 + bh4 >= 4) + { + const int sz_ctx = b_dim[2] + b_dim[3] - 2; + if (b->y_mode == DC_PRED) { + const int pal_ctx = (t->a->pal_sz[bx4] > 0) + (t->l.pal_sz[by4] > 0); + const int use_y_pal = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.pal_y[sz_ctx][pal_ctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-y_pal[%d]: r=%d\n", use_y_pal, ts->msac.rng); + if (use_y_pal) + read_pal_plane(t, b, 0, sz_ctx, bx4, by4); + } + + if (has_chroma && b->uv_mode == DC_PRED) { + const int pal_ctx = b->pal_sz[0] > 0; + const int use_uv_pal = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.pal_uv[pal_ctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-uv_pal[%d]: r=%d\n", use_uv_pal, ts->msac.rng); + if (use_uv_pal) // see aomedia bug 2183 for why we use luma coordinates + read_pal_uv(t, b, sz_ctx, bx4, by4); + } + } + + if (b->y_mode == DC_PRED && !b->pal_sz[0] && + imax(b_dim[2], b_dim[3]) <= 3 && f->seq_hdr->filter_intra) + { + const int is_filter = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.use_filter_intra[bs]); + if (is_filter) { + b->y_mode = FILTER_PRED; + b->y_angle = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.filter_intra, 4); + } + if (DEBUG_BLOCK_INFO) + printf("Post-filterintramode[%d/%d]: r=%d\n", + b->y_mode, b->y_angle, ts->msac.rng); + } + + if (b->pal_sz[0]) { + uint8_t *pal_idx; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += bw4 * bh4 * 16; + } else + pal_idx = t->scratch.pal_idx; + read_pal_indices(t, pal_idx, b, 0, w4, h4, bw4, bh4); + if (DEBUG_BLOCK_INFO) + printf("Post-y-pal-indices: r=%d\n", ts->msac.rng); + } + + if (has_chroma && b->pal_sz[1]) { + uint8_t *pal_idx; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += cbw4 * cbh4 * 16; + } else + pal_idx = &t->scratch.pal_idx[bw4 * bh4 * 16]; + read_pal_indices(t, pal_idx, b, 1, cw4, ch4, cbw4, cbh4); + if (DEBUG_BLOCK_INFO) + printf("Post-uv-pal-indices: r=%d\n", ts->msac.rng); + } + + const TxfmInfo *t_dim; + if (f->frame_hdr->segmentation.lossless[b->seg_id]) { + b->tx = b->uvtx = (int) TX_4X4; + t_dim = &dav1d_txfm_dimensions[TX_4X4]; + } else { + b->tx = dav1d_max_txfm_size_for_bs[bs][0]; + b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; + t_dim = &dav1d_txfm_dimensions[b->tx]; + if (f->frame_hdr->txfm_mode == DAV1D_TX_SWITCHABLE && t_dim->max > TX_4X4) { + const int tctx = get_tx_ctx(t->a, &t->l, t_dim, by4, bx4); + uint16_t *const tx_cdf = ts->cdf.m.txsz[t_dim->max - 1][tctx]; + int depth = dav1d_msac_decode_symbol_adapt4(&ts->msac, tx_cdf, + imin(t_dim->max, 2)); + + while (depth--) { + b->tx = t_dim->sub; + t_dim = &dav1d_txfm_dimensions[b->tx]; + } + } + if (DEBUG_BLOCK_INFO) + printf("Post-tx[%d]: r=%d\n", b->tx, ts->msac.rng); + } + + // reconstruction + if (t->frame_thread.pass == 1) { + f->bd_fn.read_coef_blocks(t, bs, b); + } else { + f->bd_fn.recon_b_intra(t, bs, intra_edge_flags, b); + } + + if (f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]) + { + dav1d_create_lf_mask_intra(t->lf_mask, f->lf.level, f->b4_stride, + (const uint8_t (*)[8][2]) + &ts->lflvl[b->seg_id][0][0][0], + t->bx, t->by, f->w4, f->h4, bs, + b->tx, b->uvtx, f->cur.p.layout, + &t->a->tx_lpf_y[bx4], &t->l.tx_lpf_y[by4], + has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, + has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); + } + + // update contexts +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx_intra, off, mul * (((uint8_t *) &t_dim->lw)[diridx])); \ + rep_macro(type, t->dir tx, off, mul * (((uint8_t *) &t_dim->lw)[diridx])); \ + rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ + rep_macro(type, t->dir pal_sz, off, mul * b->pal_sz[0]); \ + rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ + rep_macro(type, t->dir skip_mode, off, 0); \ + rep_macro(type, t->dir intra, off, mul); \ + rep_macro(type, t->dir skip, off, mul * b->skip); \ + /* see aomedia bug 2183 for why we use luma coordinates here */ \ + rep_macro(type, t->pal_sz_uv[diridx], off, mul * (has_chroma ? b->pal_sz[1] : 0)); \ + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { \ + rep_macro(type, t->dir comp_type, off, mul * COMP_INTER_NONE); \ + rep_macro(type, t->dir ref[0], off, mul * ((uint8_t) -1)); \ + rep_macro(type, t->dir ref[1], off, mul * ((uint8_t) -1)); \ + rep_macro(type, t->dir filter[0], off, mul * DAV1D_N_SWITCHABLE_FILTERS); \ + rep_macro(type, t->dir filter[1], off, mul * DAV1D_N_SWITCHABLE_FILTERS); \ + } + const enum IntraPredMode y_mode_nofilt = + b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + if (b->pal_sz[0]) { + uint16_t *const pal = t->frame_thread.pass ? + f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + + ((t->bx >> 1) + (t->by & 1))][0] : t->scratch.pal[0]; + for (int x = 0; x < bw4; x++) + memcpy(t->al_pal[0][bx4 + x][0], pal, 16); + for (int y = 0; y < bh4; y++) + memcpy(t->al_pal[1][by4 + y][0], pal, 16); + } + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + if (b->pal_sz[1]) { + const uint16_t (*const pal)[8] = t->frame_thread.pass ? + f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * + (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))] : + t->scratch.pal; + // see aomedia bug 2183 for why we use luma coordinates here + for (int pl = 1; pl <= 2; pl++) { + for (int x = 0; x < bw4; x++) + memcpy(t->al_pal[0][bx4 + x][pl], pal[pl], 16); + for (int y = 0; y < bh4; y++) + memcpy(t->al_pal[1][by4 + y][pl], pal[pl], 16); + } + } + } + if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) + splat_intraref(f->c, t, bs, bw4, bh4); + } else if (IS_KEY_OR_INTRA(f->frame_hdr)) { + // intra block copy + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { 0, -1 }}, + bs, intra_edge_flags, t->by, t->bx); + + if (mvstack[0].mv.mv[0].n) + b->mv[0] = mvstack[0].mv.mv[0]; + else if (mvstack[1].mv.mv[0].n) + b->mv[0] = mvstack[1].mv.mv[0]; + else { + if (t->by - (16 << f->seq_hdr->sb128) < ts->tiling.row_start) { + b->mv[0].y = 0; + b->mv[0].x = -(512 << f->seq_hdr->sb128) - 2048; + } else { + b->mv[0].y = -(512 << f->seq_hdr->sb128); + b->mv[0].x = 0; + } + } + + const union mv ref = b->mv[0]; + read_mv_residual(t, &b->mv[0], &ts->cdf.dmv, 0); + + // clip intrabc motion vector to decoded parts of current tile + int border_left = ts->tiling.col_start * 4; + int border_top = ts->tiling.row_start * 4; + if (has_chroma) { + if (bw4 < 2 && ss_hor) + border_left += 4; + if (bh4 < 2 && ss_ver) + border_top += 4; + } + int src_left = t->bx * 4 + (b->mv[0].x >> 3); + int src_top = t->by * 4 + (b->mv[0].y >> 3); + int src_right = src_left + bw4 * 4; + int src_bottom = src_top + bh4 * 4; + const int border_right = ((ts->tiling.col_end + (bw4 - 1)) & ~(bw4 - 1)) * 4; + + // check against left or right tile boundary and adjust if necessary + if (src_left < border_left) { + src_right += border_left - src_left; + src_left += border_left - src_left; + } else if (src_right > border_right) { + src_left -= src_right - border_right; + src_right -= src_right - border_right; + } + // check against top tile boundary and adjust if necessary + if (src_top < border_top) { + src_bottom += border_top - src_top; + src_top += border_top - src_top; + } + + const int sbx = (t->bx >> (4 + f->seq_hdr->sb128)) << (6 + f->seq_hdr->sb128); + const int sby = (t->by >> (4 + f->seq_hdr->sb128)) << (6 + f->seq_hdr->sb128); + const int sb_size = 1 << (6 + f->seq_hdr->sb128); + // check for overlap with current superblock + if (src_bottom > sby && src_right > sbx) { + if (src_top - border_top >= src_bottom - sby) { + // if possible move src up into the previous suberblock row + src_top -= src_bottom - sby; + src_bottom -= src_bottom - sby; + } else if (src_left - border_left >= src_right - sbx) { + // if possible move src left into the previous suberblock + src_left -= src_right - sbx; + src_right -= src_right - sbx; + } + } + // move src up if it is below current superblock row + if (src_bottom > sby + sb_size) { + src_top -= src_bottom - (sby + sb_size); + src_bottom -= src_bottom - (sby + sb_size); + } + // error out if mv still overlaps with the current superblock + if (src_bottom > sby && src_right > sbx) + return -1; + + b->mv[0].x = (src_left - t->bx * 4) * 8; + b->mv[0].y = (src_top - t->by * 4) * 8; + + if (DEBUG_BLOCK_INFO) + printf("Post-dmv[%d/%d,ref=%d/%d|%d/%d]: r=%d\n", + b->mv[0].y, b->mv[0].x, ref.y, ref.x, + mvstack[0].mv.mv[0].y, mvstack[0].mv.mv[0].x, ts->msac.rng); + read_vartx_tree(t, b, bs, bx4, by4); + + // reconstruction + if (t->frame_thread.pass == 1) { + f->bd_fn.read_coef_blocks(t, bs, b); + b->filter2d = FILTER_2D_BILINEAR; + } else { + if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; + } + + splat_intrabc_mv(f->c, t, bs, b, bw4, bh4); + +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ + rep_macro(type, t->dir mode, off, mul * DC_PRED); \ + rep_macro(type, t->dir pal_sz, off, 0); \ + /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ + rep_macro(type, t->pal_sz_uv[diridx], off, 0); \ + rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ + rep_macro(type, t->dir skip_mode, off, 0); \ + rep_macro(type, t->dir intra, off, 0); \ + rep_macro(type, t->dir skip, off, mul * b->skip) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir uvmode, off, mul * DC_PRED) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + } else { + // inter-specific mode/mv coding + int is_comp, has_subpel_filter; + + if (b->skip_mode) { + is_comp = 1; + } else if ((!seg || (seg->ref == -1 && !seg->globalmv && !seg->skip)) && + f->frame_hdr->switchable_comp_refs && imin(bw4, bh4) > 1) + { + const int ctx = get_comp_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + is_comp = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp[ctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-compflag[%d]: r=%d\n", is_comp, ts->msac.rng); + } else { + is_comp = 0; + } + + if (b->skip_mode) { + b->ref[0] = f->frame_hdr->skip_mode_refs[0]; + b->ref[1] = f->frame_hdr->skip_mode_refs[1]; + b->comp_type = COMP_INTER_AVG; + b->inter_mode = NEARESTMV_NEARESTMV; + b->drl_idx = NEAREST_DRL; + has_subpel_filter = 0; + + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); + + b->mv[0] = mvstack[0].mv.mv[0]; + b->mv[1] = mvstack[0].mv.mv[1]; + fix_mv_precision(f->frame_hdr, &b->mv[0]); + fix_mv_precision(f->frame_hdr, &b->mv[1]); + if (DEBUG_BLOCK_INFO) + printf("Post-skipmodeblock[mv=1:y=%d,x=%d,2:y=%d,x=%d,refs=%d+%d\n", + b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, + b->ref[0], b->ref[1]); + } else if (is_comp) { + const int dir_ctx = get_comp_dir_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_dir[dir_ctx])) + { + // bidir - first reference (fw) + const int ctx1 = av1_get_fwd_ref_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_fwd_ref[0][ctx1])) + { + const int ctx2 = av1_get_fwd_ref_2_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = 2 + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_fwd_ref[2][ctx2]); + } else { + const int ctx2 = av1_get_fwd_ref_1_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_fwd_ref[1][ctx2]); + } + + // second reference (bw) + const int ctx3 = av1_get_bwd_ref_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_bwd_ref[0][ctx3])) + { + b->ref[1] = 6; + } else { + const int ctx4 = av1_get_bwd_ref_1_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[1] = 4 + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_bwd_ref[1][ctx4]); + } + } else { + // unidir + const int uctx_p = av1_get_uni_p_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_uni_ref[0][uctx_p])) + { + b->ref[0] = 4; + b->ref[1] = 6; + } else { + const int uctx_p1 = av1_get_uni_p1_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = 0; + b->ref[1] = 1 + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_uni_ref[1][uctx_p1]); + if (b->ref[1] == 2) { + const int uctx_p2 = av1_get_uni_p2_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[1] += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.comp_uni_ref[2][uctx_p2]); + } + } + } + if (DEBUG_BLOCK_INFO) + printf("Post-refs[%d/%d]: r=%d\n", + b->ref[0], b->ref[1], ts->msac.rng); + + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); + + b->inter_mode = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.comp_inter_mode[ctx], + N_COMP_INTER_PRED_MODES - 1); + if (DEBUG_BLOCK_INFO) + printf("Post-compintermode[%d,ctx=%d,n_mvs=%d]: r=%d\n", + b->inter_mode, ctx, n_mvs, ts->msac.rng); + + const uint8_t *const im = dav1d_comp_inter_pred_modes[b->inter_mode]; + b->drl_idx = NEAREST_DRL; + if (b->inter_mode == NEWMV_NEWMV) { + if (n_mvs > 1) { // NEARER, NEAR or NEARISH + const int drl_ctx_v1 = get_drl_context(mvstack, 0); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v1]); + if (b->drl_idx == NEARER_DRL && n_mvs > 2) { + const int drl_ctx_v2 = get_drl_context(mvstack, 1); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v2]); + } + if (DEBUG_BLOCK_INFO) + printf("Post-drlidx[%d,n_mvs=%d]: r=%d\n", + b->drl_idx, n_mvs, ts->msac.rng); + } + } else if (im[0] == NEARMV || im[1] == NEARMV) { + b->drl_idx = NEARER_DRL; + if (n_mvs > 2) { // NEAR or NEARISH + const int drl_ctx_v2 = get_drl_context(mvstack, 1); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v2]); + if (b->drl_idx == NEAR_DRL && n_mvs > 3) { + const int drl_ctx_v3 = get_drl_context(mvstack, 2); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v3]); + } + if (DEBUG_BLOCK_INFO) + printf("Post-drlidx[%d,n_mvs=%d]: r=%d\n", + b->drl_idx, n_mvs, ts->msac.rng); + } + } + assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); + +#define assign_comp_mv(idx) \ + switch (im[idx]) { \ + case NEARMV: \ + case NEARESTMV: \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ + fix_mv_precision(f->frame_hdr, &b->mv[idx]); \ + break; \ + case GLOBALMV: \ + has_subpel_filter |= \ + f->frame_hdr->gmv[b->ref[idx]].type == DAV1D_WM_TYPE_TRANSLATION; \ + b->mv[idx] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[idx]], \ + t->bx, t->by, bw4, bh4, f->frame_hdr); \ + break; \ + case NEWMV: \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ + read_mv_residual(t, &b->mv[idx], &ts->cdf.mv, \ + !f->frame_hdr->force_integer_mv); \ + break; \ + } + has_subpel_filter = imin(bw4, bh4) == 1 || + b->inter_mode != GLOBALMV_GLOBALMV; + assign_comp_mv(0); + assign_comp_mv(1); +#undef assign_comp_mv + if (DEBUG_BLOCK_INFO) + printf("Post-residual_mv[1:y=%d,x=%d,2:y=%d,x=%d]: r=%d\n", + b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, + ts->msac.rng); + + // jnt_comp vs. seg vs. wedge + int is_segwedge = 0; + if (f->seq_hdr->masked_compound) { + const int mask_ctx = get_mask_comp_ctx(t->a, &t->l, by4, bx4); + + is_segwedge = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.mask_comp[mask_ctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-segwedge_vs_jntavg[%d,ctx=%d]: r=%d\n", + is_segwedge, mask_ctx, ts->msac.rng); + } + + if (!is_segwedge) { + if (f->seq_hdr->jnt_comp) { + const int jnt_ctx = + get_jnt_comp_ctx(f->seq_hdr->order_hint_n_bits, + f->cur.frame_hdr->frame_offset, + f->refp[b->ref[0]].p.frame_hdr->frame_offset, + f->refp[b->ref[1]].p.frame_hdr->frame_offset, + t->a, &t->l, by4, bx4); + b->comp_type = COMP_INTER_WEIGHTED_AVG + + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.jnt_comp[jnt_ctx]); + if (DEBUG_BLOCK_INFO) + printf("Post-jnt_comp[%d,ctx=%d[ac:%d,ar:%d,lc:%d,lr:%d]]: r=%d\n", + b->comp_type == COMP_INTER_AVG, + jnt_ctx, t->a->comp_type[bx4], t->a->ref[0][bx4], + t->l.comp_type[by4], t->l.ref[0][by4], + ts->msac.rng); + } else { + b->comp_type = COMP_INTER_AVG; + } + } else { + if (wedge_allowed_mask & (1 << bs)) { + const int ctx = dav1d_wedge_ctx_lut[bs]; + b->comp_type = COMP_INTER_WEDGE - + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.wedge_comp[ctx]); + if (b->comp_type == COMP_INTER_WEDGE) + b->wedge_idx = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.wedge_idx[ctx], 15); + } else { + b->comp_type = COMP_INTER_SEG; + } + b->mask_sign = dav1d_msac_decode_bool_equi(&ts->msac); + if (DEBUG_BLOCK_INFO) + printf("Post-seg/wedge[%d,wedge_idx=%d,sign=%d]: r=%d\n", + b->comp_type == COMP_INTER_WEDGE, + b->wedge_idx, b->mask_sign, ts->msac.rng); + } + } else { + b->comp_type = COMP_INTER_NONE; + + // ref + if (seg && seg->ref > 0) { + b->ref[0] = seg->ref - 1; + } else if (seg && (seg->globalmv || seg->skip)) { + b->ref[0] = 0; + } else { + const int ctx1 = av1_get_ref_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[0][ctx1])) + { + const int ctx2 = av1_get_ref_2_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[1][ctx2])) + { + b->ref[0] = 6; + } else { + const int ctx3 = av1_get_ref_6_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = 4 + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[5][ctx3]); + } + } else { + const int ctx2 = av1_get_ref_3_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[2][ctx2])) + { + const int ctx3 = av1_get_ref_5_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = 2 + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[4][ctx3]); + } else { + const int ctx3 = av1_get_ref_4_ctx(t->a, &t->l, by4, bx4, + have_top, have_left); + b->ref[0] = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.ref[3][ctx3]); + } + } + if (DEBUG_BLOCK_INFO) + printf("Post-ref[%d]: r=%d\n", b->ref[0], ts->msac.rng); + } + b->ref[1] = -1; + + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { b->ref[0] + 1, -1 }}, + bs, intra_edge_flags, t->by, t->bx); + + // mode parsing and mv derivation from ref_mvs + if ((seg && (seg->skip || seg->globalmv)) || + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.newmv_mode[ctx & 7])) + { + if ((seg && (seg->skip || seg->globalmv)) || + !dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.globalmv_mode[(ctx >> 3) & 1])) + { + b->inter_mode = GLOBALMV; + b->mv[0] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[0]], + t->bx, t->by, bw4, bh4, f->frame_hdr); + has_subpel_filter = imin(bw4, bh4) == 1 || + f->frame_hdr->gmv[b->ref[0]].type == DAV1D_WM_TYPE_TRANSLATION; + } else { + has_subpel_filter = 1; + if (dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.refmv_mode[(ctx >> 4) & 15])) + { // NEAREST, NEARER, NEAR or NEARISH + b->inter_mode = NEARMV; + b->drl_idx = NEARER_DRL; + if (n_mvs > 2) { // NEARER, NEAR or NEARISH + const int drl_ctx_v2 = get_drl_context(mvstack, 1); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v2]); + if (b->drl_idx == NEAR_DRL && n_mvs > 3) { // NEAR or NEARISH + const int drl_ctx_v3 = + get_drl_context(mvstack, 2); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v3]); + } + } + } else { + b->inter_mode = NEARESTMV; + b->drl_idx = NEAREST_DRL; + } + assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; + if (b->drl_idx < NEAR_DRL) + fix_mv_precision(f->frame_hdr, &b->mv[0]); + } + + if (DEBUG_BLOCK_INFO) + printf("Post-intermode[%d,drl=%d,mv=y:%d,x:%d,n_mvs=%d]: r=%d\n", + b->inter_mode, b->drl_idx, b->mv[0].y, b->mv[0].x, n_mvs, + ts->msac.rng); + } else { + has_subpel_filter = 1; + b->inter_mode = NEWMV; + b->drl_idx = NEAREST_DRL; + if (n_mvs > 1) { // NEARER, NEAR or NEARISH + const int drl_ctx_v1 = get_drl_context(mvstack, 0); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v1]); + if (b->drl_idx == NEARER_DRL && n_mvs > 2) { // NEAR or NEARISH + const int drl_ctx_v2 = get_drl_context(mvstack, 1); + b->drl_idx += dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.drl_bit[drl_ctx_v2]); + } + } + assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); + if (n_mvs > 1) { + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; + } else { + assert(!b->drl_idx); + b->mv[0] = mvstack[0].mv.mv[0]; + fix_mv_precision(f->frame_hdr, &b->mv[0]); + } + if (DEBUG_BLOCK_INFO) + printf("Post-intermode[%d,drl=%d]: r=%d\n", + b->inter_mode, b->drl_idx, ts->msac.rng); + read_mv_residual(t, &b->mv[0], &ts->cdf.mv, + !f->frame_hdr->force_integer_mv); + if (DEBUG_BLOCK_INFO) + printf("Post-residualmv[mv=y:%d,x:%d]: r=%d\n", + b->mv[0].y, b->mv[0].x, ts->msac.rng); + } + + // interintra flags + const int ii_sz_grp = dav1d_ymode_size_context[bs]; + if (f->seq_hdr->inter_intra && + interintra_allowed_mask & (1 << bs) && + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.interintra[ii_sz_grp])) + { + b->interintra_mode = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.interintra_mode[ii_sz_grp], + N_INTER_INTRA_PRED_MODES - 1); + const int wedge_ctx = dav1d_wedge_ctx_lut[bs]; + b->interintra_type = INTER_INTRA_BLEND + + dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.interintra_wedge[wedge_ctx]); + if (b->interintra_type == INTER_INTRA_WEDGE) + b->wedge_idx = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.wedge_idx[wedge_ctx], 15); + } else { + b->interintra_type = INTER_INTRA_NONE; + } + if (DEBUG_BLOCK_INFO && f->seq_hdr->inter_intra && + interintra_allowed_mask & (1 << bs)) + { + printf("Post-interintra[t=%d,m=%d,w=%d]: r=%d\n", + b->interintra_type, b->interintra_mode, + b->wedge_idx, ts->msac.rng); + } + + // motion variation + if (f->frame_hdr->switchable_motion_mode && + b->interintra_type == INTER_INTRA_NONE && imin(bw4, bh4) >= 2 && + // is not warped global motion + !(!f->frame_hdr->force_integer_mv && b->inter_mode == GLOBALMV && + f->frame_hdr->gmv[b->ref[0]].type > DAV1D_WM_TYPE_TRANSLATION) && + // has overlappable neighbours + ((have_left && findoddzero(&t->l.intra[by4 + 1], h4 >> 1)) || + (have_top && findoddzero(&t->a->intra[bx4 + 1], w4 >> 1)))) + { + // reaching here means the block allows obmc - check warp by + // finding matching-ref blocks in top/left edges + uint64_t mask[2] = { 0, 0 }; + find_matching_ref(t, intra_edge_flags, bw4, bh4, w4, h4, + have_left, have_top, b->ref[0], mask); + const int allow_warp = !f->svc[b->ref[0]][0].scale && + !f->frame_hdr->force_integer_mv && + f->frame_hdr->warp_motion && (mask[0] | mask[1]); + + b->motion_mode = allow_warp ? + dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.motion_mode[bs], 2) : + dav1d_msac_decode_bool_adapt(&ts->msac, ts->cdf.m.obmc[bs]); + if (b->motion_mode == MM_WARP) { + has_subpel_filter = 0; + derive_warpmv(t, bw4, bh4, mask, b->mv[0], &t->warpmv); +#define signabs(v) v < 0 ? '-' : ' ', abs(v) + if (DEBUG_BLOCK_INFO) + printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, " + "mv=y:%d,x:%d\n", + signabs(t->warpmv.matrix[0]), + signabs(t->warpmv.matrix[1]), + signabs(t->warpmv.matrix[2]), + signabs(t->warpmv.matrix[3]), + signabs(t->warpmv.matrix[4]), + signabs(t->warpmv.matrix[5]), + signabs(t->warpmv.u.p.alpha), + signabs(t->warpmv.u.p.beta), + signabs(t->warpmv.u.p.gamma), + signabs(t->warpmv.u.p.delta), + b->mv[0].y, b->mv[0].x); +#undef signabs + if (t->frame_thread.pass) { + if (t->warpmv.type == DAV1D_WM_TYPE_AFFINE) { + b->matrix[0] = t->warpmv.matrix[2] - 0x10000; + b->matrix[1] = t->warpmv.matrix[3]; + b->matrix[2] = t->warpmv.matrix[4]; + b->matrix[3] = t->warpmv.matrix[5] - 0x10000; + } else { + b->matrix[0] = SHRT_MIN; + } + } + } + + if (DEBUG_BLOCK_INFO) + printf("Post-motionmode[%d]: r=%d [mask: 0x%" PRIx64 "/0x%" + PRIx64 "]\n", b->motion_mode, ts->msac.rng, mask[0], + mask[1]); + } else { + b->motion_mode = MM_TRANSLATION; + } + } + + // subpel filter + enum Dav1dFilterMode filter[2]; + if (f->frame_hdr->subpel_filter_mode == DAV1D_FILTER_SWITCHABLE) { + if (has_subpel_filter) { + const int comp = b->comp_type != COMP_INTER_NONE; + const int ctx1 = get_filter_ctx(t->a, &t->l, comp, 0, b->ref[0], + by4, bx4); + filter[0] = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.filter[0][ctx1], + DAV1D_N_SWITCHABLE_FILTERS - 1); + if (f->seq_hdr->dual_filter) { + const int ctx2 = get_filter_ctx(t->a, &t->l, comp, 1, + b->ref[0], by4, bx4); + if (DEBUG_BLOCK_INFO) + printf("Post-subpel_filter1[%d,ctx=%d]: r=%d\n", + filter[0], ctx1, ts->msac.rng); + filter[1] = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.filter[1][ctx2], + DAV1D_N_SWITCHABLE_FILTERS - 1); + if (DEBUG_BLOCK_INFO) + printf("Post-subpel_filter2[%d,ctx=%d]: r=%d\n", + filter[1], ctx2, ts->msac.rng); + } else { + filter[1] = filter[0]; + if (DEBUG_BLOCK_INFO) + printf("Post-subpel_filter[%d,ctx=%d]: r=%d\n", + filter[0], ctx1, ts->msac.rng); + } + } else { + filter[0] = filter[1] = DAV1D_FILTER_8TAP_REGULAR; + } + } else { + filter[0] = filter[1] = f->frame_hdr->subpel_filter_mode; + } + b->filter2d = dav1d_filter_2d[filter[1]][filter[0]]; + + read_vartx_tree(t, b, bs, bx4, by4); + + // reconstruction + if (t->frame_thread.pass == 1) { + f->bd_fn.read_coef_blocks(t, bs, b); + } else { + if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; + } + + if (f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]) + { + const int is_globalmv = + b->inter_mode == (is_comp ? GLOBALMV_GLOBALMV : GLOBALMV); + const uint8_t (*const lf_lvls)[8][2] = (const uint8_t (*)[8][2]) + &ts->lflvl[b->seg_id][0][b->ref[0] + 1][!is_globalmv]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; + enum RectTxfmSize ytx = b->max_ytx, uvtx = b->uvtx; + if (f->frame_hdr->segmentation.lossless[b->seg_id]) { + ytx = (enum RectTxfmSize) TX_4X4; + uvtx = (enum RectTxfmSize) TX_4X4; + } + dav1d_create_lf_mask_inter(t->lf_mask, f->lf.level, f->b4_stride, lf_lvls, + t->bx, t->by, f->w4, f->h4, b->skip, bs, + ytx, tx_split, uvtx, f->cur.p.layout, + &t->a->tx_lpf_y[bx4], &t->l.tx_lpf_y[by4], + has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, + has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); + } + + // context updates + if (is_comp) + splat_tworef_mv(f->c, t, bs, b, bw4, bh4); + else + splat_oneref_mv(f->c, t, bs, b, bw4, bh4); + +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ + rep_macro(type, t->dir skip_mode, off, mul * b->skip_mode); \ + rep_macro(type, t->dir intra, off, 0); \ + rep_macro(type, t->dir skip, off, mul * b->skip); \ + rep_macro(type, t->dir pal_sz, off, 0); \ + /* see aomedia bug 2183 for why this is outside if (has_chroma) */ \ + rep_macro(type, t->pal_sz_uv[diridx], off, 0); \ + rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ + rep_macro(type, t->dir comp_type, off, mul * b->comp_type); \ + rep_macro(type, t->dir filter[0], off, mul * filter[0]); \ + rep_macro(type, t->dir filter[1], off, mul * filter[1]); \ + rep_macro(type, t->dir mode, off, mul * b->inter_mode); \ + rep_macro(type, t->dir ref[0], off, mul * b->ref[0]); \ + rep_macro(type, t->dir ref[1], off, mul * ((uint8_t) b->ref[1])) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir uvmode, off, mul * DC_PRED) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + } + + // update contexts + if (f->frame_hdr->segmentation.enabled && + f->frame_hdr->segmentation.update_map) + { + uint8_t *seg_ptr = &f->cur_segmap[t->by * f->b4_stride + t->bx]; +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + for (int y = 0; y < bh4; y++) { \ + rep_macro(type, seg_ptr, 0, mul * b->seg_id); \ + seg_ptr += f->b4_stride; \ + } + case_set(bw4, NULL, 0, 0); +#undef set_ctx + } + if (!b->skip) { + uint16_t (*noskip_mask)[2] = &t->lf_mask->noskip_mask[by4 >> 1]; + const unsigned mask = (~0U >> (32 - bw4)) << (bx4 & 15); + const int bx_idx = (bx4 & 16) >> 4; + for (int y = 0; y < bh4; y += 2, noskip_mask++) { + (*noskip_mask)[bx_idx] |= mask; + if (bw4 == 32) // this should be mask >> 16, but it's 0xffffffff anyway + (*noskip_mask)[1] |= mask; + } + } + + if (t->frame_thread.pass == 1 && !b->intra && IS_INTER_OR_SWITCH(f->frame_hdr)) { + const int sby = (t->by - ts->tiling.row_start) >> f->sb_shift; + int (*const lowest_px)[2] = ts->lowest_pixel[sby]; + + // keep track of motion vectors for each reference + if (b->comp_type == COMP_INTER_NONE) { + // y + if (imin(bw4, bh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + affine_lowest_px_luma(t, &lowest_px[b->ref[0]][0], b_dim, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + } else { + mc_lowest_px(&lowest_px[b->ref[0]][0], t->by, bh4, b->mv[0].y, + 0, &f->svc[b->ref[0]][1]); + if (b->motion_mode == MM_OBMC) { + obmc_lowest_px(t, lowest_px, 0, b_dim, bx4, by4, w4, h4); + } + } + + // uv + if (has_chroma) { + // sub8x8 derivation + int is_sub8x8 = bw4 == ss_hor || bh4 == ss_ver; + refmvs_block *const *r; + if (is_sub8x8) { + assert(ss_hor == 1); + r = &t->rt.r[(t->by & 31) + 5]; + if (bw4 == 1) is_sub8x8 &= r[0][t->bx - 1].ref.ref[0] > 0; + if (bh4 == ss_ver) is_sub8x8 &= r[-1][t->bx].ref.ref[0] > 0; + if (bw4 == 1 && bh4 == ss_ver) + is_sub8x8 &= r[-1][t->bx - 1].ref.ref[0] > 0; + } + + // chroma prediction + if (is_sub8x8) { + assert(ss_hor == 1); + if (bw4 == 1 && bh4 == ss_ver) { + const refmvs_block *const rr = &r[-1][t->bx - 1]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by - 1, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + if (bw4 == 1) { + const refmvs_block *const rr = &r[0][t->bx - 1]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + if (bh4 == ss_ver) { + const refmvs_block *const rr = &r[-1][t->bx]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by - 1, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + mc_lowest_px(&lowest_px[b->ref[0]][1], t->by, bh4, + b->mv[0].y, ss_ver, &f->svc[b->ref[0]][1]); + } else { + if (imin(cbw4, cbh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + affine_lowest_px_chroma(t, &lowest_px[b->ref[0]][1], b_dim, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + } else { + mc_lowest_px(&lowest_px[b->ref[0]][1], + t->by & ~ss_ver, bh4 << (bh4 == ss_ver), + b->mv[0].y, ss_ver, &f->svc[b->ref[0]][1]); + if (b->motion_mode == MM_OBMC) { + obmc_lowest_px(t, lowest_px, 1, b_dim, bx4, by4, w4, h4); + } + } + } + } + } else { + // y + for (int i = 0; i < 2; i++) { + if (b->inter_mode == GLOBALMV_GLOBALMV && f->gmv_warp_allowed[b->ref[i]]) { + affine_lowest_px_luma(t, &lowest_px[b->ref[i]][0], b_dim, + &f->frame_hdr->gmv[b->ref[i]]); + } else { + mc_lowest_px(&lowest_px[b->ref[i]][0], t->by, bh4, + b->mv[i].y, 0, &f->svc[b->ref[i]][1]); + } + } + + // uv + if (has_chroma) for (int i = 0; i < 2; i++) { + if (b->inter_mode == GLOBALMV_GLOBALMV && + imin(cbw4, cbh4) > 1 && f->gmv_warp_allowed[b->ref[i]]) + { + affine_lowest_px_chroma(t, &lowest_px[b->ref[i]][1], b_dim, + &f->frame_hdr->gmv[b->ref[i]]); + } else { + mc_lowest_px(&lowest_px[b->ref[i]][1], t->by, bh4, + b->mv[i].y, ss_ver, &f->svc[b->ref[i]][1]); + } + } + } + } + + return 0; +} + +#if __has_feature(memory_sanitizer) + +#include + +static int checked_decode_b(Dav1dTaskContext *const t, + const enum BlockLevel bl, + const enum BlockSize bs, + const enum BlockPartition bp, + const enum EdgeFlags intra_edge_flags) +{ + const Dav1dFrameContext *const f = t->f; + const int err = decode_b(t, bl, bs, bp, intra_edge_flags); + + if (err == 0 && !(t->frame_thread.pass & 1)) { + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); + const int has_chroma = f->seq_hdr->layout != DAV1D_PIXEL_LAYOUT_I400 && + (bw4 > ss_hor || t->bx & 1) && + (bh4 > ss_ver || t->by & 1); + + for (int p = 0; p < 1 + 2 * has_chroma; p++) { + const int ss_ver = p && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = p && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const ptrdiff_t stride = f->cur.stride[!!p]; + const int bx = t->bx & ~ss_hor; + const int by = t->by & ~ss_ver; + const int width = w4 << (2 - ss_hor + (bw4 == ss_hor)); + const int height = h4 << (2 - ss_ver + (bh4 == ss_ver)); + + const uint8_t *data = f->cur.data[p] + (by << (2 - ss_ver)) * stride + + (bx << (2 - ss_hor + !!f->seq_hdr->hbd)); + + for (int y = 0; y < height; data += stride, y++) { + const size_t line_sz = width << !!f->seq_hdr->hbd; + if (__msan_test_shadow(data, line_sz) != -1) { + fprintf(stderr, "B[%d](%d, %d) w4:%d, h4:%d, row:%d\n", + p, bx, by, w4, h4, y); + __msan_check_mem_is_initialized(data, line_sz); + } + } + } + } + + return err; +} + +#define decode_b checked_decode_b + +#endif /* defined(__has_feature) */ + +static int decode_sb(Dav1dTaskContext *const t, const enum BlockLevel bl, + const EdgeNode *const node) +{ + const Dav1dFrameContext *const f = t->f; + Dav1dTileState *const ts = t->ts; + const int hsz = 16 >> bl; + const int have_h_split = f->bw > t->bx + hsz; + const int have_v_split = f->bh > t->by + hsz; + + if (!have_h_split && !have_v_split) { + assert(bl < BL_8X8); + return decode_sb(t, bl + 1, ((const EdgeBranch *) node)->split[0]); + } + + uint16_t *pc; + enum BlockPartition bp; + int ctx, bx8, by8; + if (t->frame_thread.pass != 2) { + if (0 && bl == BL_64X64) + printf("poc=%d,y=%d,x=%d,bl=%d,r=%d\n", + f->frame_hdr->frame_offset, t->by, t->bx, bl, ts->msac.rng); + bx8 = (t->bx & 31) >> 1; + by8 = (t->by & 31) >> 1; + ctx = get_partition_ctx(t->a, &t->l, bl, by8, bx8); + pc = ts->cdf.m.partition[bl][ctx]; + } + + if (have_h_split && have_v_split) { + if (t->frame_thread.pass == 2) { + const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; + bp = b->bl == bl ? b->bp : PARTITION_SPLIT; + } else { + bp = dav1d_msac_decode_symbol_adapt16(&ts->msac, pc, + dav1d_partition_type_count[bl]); + if (f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I422 && + (bp == PARTITION_V || bp == PARTITION_V4 || + bp == PARTITION_T_LEFT_SPLIT || bp == PARTITION_T_RIGHT_SPLIT)) + { + return 1; + } + if (DEBUG_BLOCK_INFO) + printf("poc=%d,y=%d,x=%d,bl=%d,ctx=%d,bp=%d: r=%d\n", + f->frame_hdr->frame_offset, t->by, t->bx, bl, ctx, bp, + ts->msac.rng); + } + const uint8_t *const b = dav1d_block_sizes[bl][bp]; + + switch (bp) { + case PARTITION_NONE: + if (decode_b(t, bl, b[0], PARTITION_NONE, node->o)) + return -1; + break; + case PARTITION_H: + if (decode_b(t, bl, b[0], PARTITION_H, node->h[0])) + return -1; + t->by += hsz; + if (decode_b(t, bl, b[0], PARTITION_H, node->h[1])) + return -1; + t->by -= hsz; + break; + case PARTITION_V: + if (decode_b(t, bl, b[0], PARTITION_V, node->v[0])) + return -1; + t->bx += hsz; + if (decode_b(t, bl, b[0], PARTITION_V, node->v[1])) + return -1; + t->bx -= hsz; + break; + case PARTITION_SPLIT: + if (bl == BL_8X8) { + const EdgeTip *const tip = (const EdgeTip *) node; + assert(hsz == 1); + if (decode_b(t, bl, BS_4x4, PARTITION_SPLIT, tip->split[0])) + return -1; + const enum Filter2d tl_filter = t->tl_4x4_filter; + t->bx++; + if (decode_b(t, bl, BS_4x4, PARTITION_SPLIT, tip->split[1])) + return -1; + t->bx--; + t->by++; + if (decode_b(t, bl, BS_4x4, PARTITION_SPLIT, tip->split[2])) + return -1; + t->bx++; + t->tl_4x4_filter = tl_filter; + if (decode_b(t, bl, BS_4x4, PARTITION_SPLIT, tip->split[3])) + return -1; + t->bx--; + t->by--; +#if ARCH_X86_64 + if (t->frame_thread.pass) { + /* In 8-bit mode with 2-pass decoding the coefficient buffer + * can end up misaligned due to skips here. Work around + * the issue by explicitly realigning the buffer. */ + const int p = t->frame_thread.pass & 1; + ts->frame_thread[p].cf = + (void*)(((uintptr_t)ts->frame_thread[p].cf + 63) & ~63); + } +#endif + } else { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_sb(t, bl + 1, branch->split[0])) + return 1; + t->bx += hsz; + if (decode_sb(t, bl + 1, branch->split[1])) + return 1; + t->bx -= hsz; + t->by += hsz; + if (decode_sb(t, bl + 1, branch->split[2])) + return 1; + t->bx += hsz; + if (decode_sb(t, bl + 1, branch->split[3])) + return 1; + t->bx -= hsz; + t->by -= hsz; + } + break; + case PARTITION_T_TOP_SPLIT: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_T_TOP_SPLIT, branch->tts[0])) + return -1; + t->bx += hsz; + if (decode_b(t, bl, b[0], PARTITION_T_TOP_SPLIT, branch->tts[1])) + return -1; + t->bx -= hsz; + t->by += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_TOP_SPLIT, branch->tts[2])) + return -1; + t->by -= hsz; + break; + } + case PARTITION_T_BOTTOM_SPLIT: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_T_BOTTOM_SPLIT, branch->tbs[0])) + return -1; + t->by += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_BOTTOM_SPLIT, branch->tbs[1])) + return -1; + t->bx += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_BOTTOM_SPLIT, branch->tbs[2])) + return -1; + t->bx -= hsz; + t->by -= hsz; + break; + } + case PARTITION_T_LEFT_SPLIT: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_T_LEFT_SPLIT, branch->tls[0])) + return -1; + t->by += hsz; + if (decode_b(t, bl, b[0], PARTITION_T_LEFT_SPLIT, branch->tls[1])) + return -1; + t->by -= hsz; + t->bx += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_LEFT_SPLIT, branch->tls[2])) + return -1; + t->bx -= hsz; + break; + } + case PARTITION_T_RIGHT_SPLIT: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_T_RIGHT_SPLIT, branch->trs[0])) + return -1; + t->bx += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_RIGHT_SPLIT, branch->trs[1])) + return -1; + t->by += hsz; + if (decode_b(t, bl, b[1], PARTITION_T_RIGHT_SPLIT, branch->trs[2])) + return -1; + t->by -= hsz; + t->bx -= hsz; + break; + } + case PARTITION_H4: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_H4, branch->h4[0])) + return -1; + t->by += hsz >> 1; + if (decode_b(t, bl, b[0], PARTITION_H4, branch->h4[1])) + return -1; + t->by += hsz >> 1; + if (decode_b(t, bl, b[0], PARTITION_H4, branch->h4[2])) + return -1; + t->by += hsz >> 1; + if (t->by < f->bh) + if (decode_b(t, bl, b[0], PARTITION_H4, branch->h4[3])) + return -1; + t->by -= hsz * 3 >> 1; + break; + } + case PARTITION_V4: { + const EdgeBranch *const branch = (const EdgeBranch *) node; + if (decode_b(t, bl, b[0], PARTITION_V4, branch->v4[0])) + return -1; + t->bx += hsz >> 1; + if (decode_b(t, bl, b[0], PARTITION_V4, branch->v4[1])) + return -1; + t->bx += hsz >> 1; + if (decode_b(t, bl, b[0], PARTITION_V4, branch->v4[2])) + return -1; + t->bx += hsz >> 1; + if (t->bx < f->bw) + if (decode_b(t, bl, b[0], PARTITION_V4, branch->v4[3])) + return -1; + t->bx -= hsz * 3 >> 1; + break; + } + default: assert(0); + } + } else if (have_h_split) { + unsigned is_split; + if (t->frame_thread.pass == 2) { + const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; + is_split = b->bl != bl; + } else { + is_split = dav1d_msac_decode_bool(&ts->msac, + gather_top_partition_prob(pc, bl)); + if (DEBUG_BLOCK_INFO) + printf("poc=%d,y=%d,x=%d,bl=%d,ctx=%d,bp=%d: r=%d\n", + f->frame_hdr->frame_offset, t->by, t->bx, bl, ctx, + is_split ? PARTITION_SPLIT : PARTITION_H, ts->msac.rng); + } + + assert(bl < BL_8X8); + if (is_split) { + const EdgeBranch *const branch = (const EdgeBranch *) node; + bp = PARTITION_SPLIT; + if (decode_sb(t, bl + 1, branch->split[0])) return 1; + t->bx += hsz; + if (decode_sb(t, bl + 1, branch->split[1])) return 1; + t->bx -= hsz; + } else { + bp = PARTITION_H; + if (decode_b(t, bl, dav1d_block_sizes[bl][PARTITION_H][0], + PARTITION_H, node->h[0])) + return -1; + } + } else { + assert(have_v_split); + unsigned is_split; + if (t->frame_thread.pass == 2) { + const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; + is_split = b->bl != bl; + } else { + is_split = dav1d_msac_decode_bool(&ts->msac, + gather_left_partition_prob(pc, bl)); + if (f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I422 && !is_split) + return 1; + if (DEBUG_BLOCK_INFO) + printf("poc=%d,y=%d,x=%d,bl=%d,ctx=%d,bp=%d: r=%d\n", + f->frame_hdr->frame_offset, t->by, t->bx, bl, ctx, + is_split ? PARTITION_SPLIT : PARTITION_V, ts->msac.rng); + } + + assert(bl < BL_8X8); + if (is_split) { + const EdgeBranch *const branch = (const EdgeBranch *) node; + bp = PARTITION_SPLIT; + if (decode_sb(t, bl + 1, branch->split[0])) return 1; + t->by += hsz; + if (decode_sb(t, bl + 1, branch->split[2])) return 1; + t->by -= hsz; + } else { + bp = PARTITION_V; + if (decode_b(t, bl, dav1d_block_sizes[bl][PARTITION_V][0], + PARTITION_V, node->v[0])) + return -1; + } + } + + if (t->frame_thread.pass != 2 && (bp != PARTITION_SPLIT || bl == BL_8X8)) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->a->partition, bx8, mul * dav1d_al_part_ctx[0][bl][bp]); \ + rep_macro(type, t->l.partition, by8, mul * dav1d_al_part_ctx[1][bl][bp]) + case_set_upto16(hsz,,,); +#undef set_ctx + } + + return 0; +} + +static void reset_context(BlockContext *const ctx, const int keyframe, const int pass) { + memset(ctx->intra, keyframe, sizeof(ctx->intra)); + memset(ctx->uvmode, DC_PRED, sizeof(ctx->uvmode)); + if (keyframe) + memset(ctx->mode, DC_PRED, sizeof(ctx->mode)); + + if (pass == 2) return; + + memset(ctx->partition, 0, sizeof(ctx->partition)); + memset(ctx->skip, 0, sizeof(ctx->skip)); + memset(ctx->skip_mode, 0, sizeof(ctx->skip_mode)); + memset(ctx->tx_lpf_y, 2, sizeof(ctx->tx_lpf_y)); + memset(ctx->tx_lpf_uv, 1, sizeof(ctx->tx_lpf_uv)); + memset(ctx->tx_intra, -1, sizeof(ctx->tx_intra)); + memset(ctx->tx, TX_64X64, sizeof(ctx->tx)); + if (!keyframe) { + memset(ctx->ref, -1, sizeof(ctx->ref)); + memset(ctx->comp_type, 0, sizeof(ctx->comp_type)); + memset(ctx->mode, NEARESTMV, sizeof(ctx->mode)); + } + memset(ctx->lcoef, 0x40, sizeof(ctx->lcoef)); + memset(ctx->ccoef, 0x40, sizeof(ctx->ccoef)); + memset(ctx->filter, DAV1D_N_SWITCHABLE_FILTERS, sizeof(ctx->filter)); + memset(ctx->seg_pred, 0, sizeof(ctx->seg_pred)); + memset(ctx->pal_sz, 0, sizeof(ctx->pal_sz)); +} + +// { Y+U+V, Y+U } * 4 +static const uint8_t ss_size_mul[4][2] = { + [DAV1D_PIXEL_LAYOUT_I400] = { 4, 4 }, + [DAV1D_PIXEL_LAYOUT_I420] = { 6, 5 }, + [DAV1D_PIXEL_LAYOUT_I422] = { 8, 6 }, + [DAV1D_PIXEL_LAYOUT_I444] = { 12, 8 }, +}; + +static void setup_tile(Dav1dTileState *const ts, + const Dav1dFrameContext *const f, + const uint8_t *const data, const size_t sz, + const int tile_row, const int tile_col, + const int tile_start_off) +{ + const int col_sb_start = f->frame_hdr->tiling.col_start_sb[tile_col]; + const int col_sb128_start = col_sb_start >> !f->seq_hdr->sb128; + const int col_sb_end = f->frame_hdr->tiling.col_start_sb[tile_col + 1]; + const int row_sb_start = f->frame_hdr->tiling.row_start_sb[tile_row]; + const int row_sb_end = f->frame_hdr->tiling.row_start_sb[tile_row + 1]; + const int sb_shift = f->sb_shift; + + const uint8_t *const size_mul = ss_size_mul[f->cur.p.layout]; + for (int p = 0; p < 2; p++) { + ts->frame_thread[p].pal_idx = f->frame_thread.pal_idx ? + &f->frame_thread.pal_idx[(size_t)tile_start_off * size_mul[1] / 4] : + NULL; + ts->frame_thread[p].cf = f->frame_thread.cf ? + (uint8_t*)f->frame_thread.cf + + (((size_t)tile_start_off * size_mul[0]) >> !f->seq_hdr->hbd) : + NULL; + } + + dav1d_cdf_thread_copy(&ts->cdf, &f->in_cdf); + ts->last_qidx = f->frame_hdr->quant.yac; + memset(ts->last_delta_lf, 0, sizeof(ts->last_delta_lf)); + + dav1d_msac_init(&ts->msac, data, sz, f->frame_hdr->disable_cdf_update); + + ts->tiling.row = tile_row; + ts->tiling.col = tile_col; + ts->tiling.col_start = col_sb_start << sb_shift; + ts->tiling.col_end = imin(col_sb_end << sb_shift, f->bw); + ts->tiling.row_start = row_sb_start << sb_shift; + ts->tiling.row_end = imin(row_sb_end << sb_shift, f->bh); + + // Reference Restoration Unit (used for exp coding) + int sb_idx, unit_idx; + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { + // vertical components only + sb_idx = (ts->tiling.row_start >> 5) * f->sr_sb128w; + unit_idx = (ts->tiling.row_start & 16) >> 3; + } else { + sb_idx = (ts->tiling.row_start >> 5) * f->sb128w + col_sb128_start; + unit_idx = ((ts->tiling.row_start & 16) >> 3) + + ((ts->tiling.col_start & 16) >> 4); + } + for (int p = 0; p < 3; p++) { + if (!((f->lf.restore_planes >> p) & 1U)) + continue; + + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { + const int ss_hor = p && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int d = f->frame_hdr->super_res.width_scale_denominator; + const int unit_size_log2 = f->frame_hdr->restoration.unit_size[!!p]; + const int rnd = (8 << unit_size_log2) - 1, shift = unit_size_log2 + 3; + const int x = ((4 * ts->tiling.col_start * d >> ss_hor) + rnd) >> shift; + const int px_x = x << (unit_size_log2 + ss_hor); + const int u_idx = unit_idx + ((px_x & 64) >> 6); + const int sb128x = px_x >> 7; + if (sb128x >= f->sr_sb128w) continue; + ts->lr_ref[p] = &f->lf.lr_mask[sb_idx + sb128x].lr[p][u_idx]; + } else { + ts->lr_ref[p] = &f->lf.lr_mask[sb_idx].lr[p][unit_idx]; + } + + ts->lr_ref[p]->filter_v[0] = 3; + ts->lr_ref[p]->filter_v[1] = -7; + ts->lr_ref[p]->filter_v[2] = 15; + ts->lr_ref[p]->filter_h[0] = 3; + ts->lr_ref[p]->filter_h[1] = -7; + ts->lr_ref[p]->filter_h[2] = 15; + ts->lr_ref[p]->sgr_weights[0] = -32; + ts->lr_ref[p]->sgr_weights[1] = 31; + } + + if (f->c->n_tc > 1) { + for (int p = 0; p < 2; p++) + atomic_init(&ts->progress[p], row_sb_start); + } +} + +static void read_restoration_info(Dav1dTaskContext *const t, + Av1RestorationUnit *const lr, const int p, + const enum Dav1dRestorationType frame_type) +{ + const Dav1dFrameContext *const f = t->f; + Dav1dTileState *const ts = t->ts; + + if (frame_type == DAV1D_RESTORATION_SWITCHABLE) { + const int filter = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.restore_switchable, 2); + lr->type = filter ? filter == 2 ? DAV1D_RESTORATION_SGRPROJ : + DAV1D_RESTORATION_WIENER : + DAV1D_RESTORATION_NONE; + } else { + const unsigned type = + dav1d_msac_decode_bool_adapt(&ts->msac, + frame_type == DAV1D_RESTORATION_WIENER ? + ts->cdf.m.restore_wiener : ts->cdf.m.restore_sgrproj); + lr->type = type ? frame_type : DAV1D_RESTORATION_NONE; + } + + if (lr->type == DAV1D_RESTORATION_WIENER) { + lr->filter_v[0] = p ? 0 : + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_v[0] + 5, 16, 1) - 5; + lr->filter_v[1] = + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_v[1] + 23, 32, 2) - 23; + lr->filter_v[2] = + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_v[2] + 17, 64, 3) - 17; + + lr->filter_h[0] = p ? 0 : + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_h[0] + 5, 16, 1) - 5; + lr->filter_h[1] = + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_h[1] + 23, 32, 2) - 23; + lr->filter_h[2] = + dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->filter_h[2] + 17, 64, 3) - 17; + memcpy(lr->sgr_weights, ts->lr_ref[p]->sgr_weights, sizeof(lr->sgr_weights)); + ts->lr_ref[p] = lr; + if (DEBUG_BLOCK_INFO) + printf("Post-lr_wiener[pl=%d,v[%d,%d,%d],h[%d,%d,%d]]: r=%d\n", + p, lr->filter_v[0], lr->filter_v[1], + lr->filter_v[2], lr->filter_h[0], + lr->filter_h[1], lr->filter_h[2], ts->msac.rng); + } else if (lr->type == DAV1D_RESTORATION_SGRPROJ) { + const unsigned idx = dav1d_msac_decode_bools(&ts->msac, 4); + const uint16_t *const sgr_params = dav1d_sgr_params[idx]; + lr->sgr_idx = idx; + lr->sgr_weights[0] = sgr_params[0] ? dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->sgr_weights[0] + 96, 128, 4) - 96 : 0; + lr->sgr_weights[1] = sgr_params[1] ? dav1d_msac_decode_subexp(&ts->msac, + ts->lr_ref[p]->sgr_weights[1] + 32, 128, 4) - 32 : 95; + memcpy(lr->filter_v, ts->lr_ref[p]->filter_v, sizeof(lr->filter_v)); + memcpy(lr->filter_h, ts->lr_ref[p]->filter_h, sizeof(lr->filter_h)); + ts->lr_ref[p] = lr; + if (DEBUG_BLOCK_INFO) + printf("Post-lr_sgrproj[pl=%d,idx=%d,w[%d,%d]]: r=%d\n", + p, lr->sgr_idx, lr->sgr_weights[0], + lr->sgr_weights[1], ts->msac.rng); + } +} + +int dav1d_decode_tile_sbrow(Dav1dTaskContext *const t) { + const Dav1dFrameContext *const f = t->f; + const enum BlockLevel root_bl = f->seq_hdr->sb128 ? BL_128X128 : BL_64X64; + Dav1dTileState *const ts = t->ts; + const Dav1dContext *const c = f->c; + const int sb_step = f->sb_step; + const int tile_row = ts->tiling.row, tile_col = ts->tiling.col; + const int col_sb_start = f->frame_hdr->tiling.col_start_sb[tile_col]; + const int col_sb128_start = col_sb_start >> !f->seq_hdr->sb128; + + if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { + dav1d_refmvs_tile_sbrow_init(&t->rt, &f->rf, ts->tiling.col_start, + ts->tiling.col_end, ts->tiling.row_start, + ts->tiling.row_end, t->by >> f->sb_shift, + ts->tiling.row, t->frame_thread.pass); + } + + if (IS_INTER_OR_SWITCH(f->frame_hdr) && c->n_fc > 1) { + const int sby = (t->by - ts->tiling.row_start) >> f->sb_shift; + int (*const lowest_px)[2] = ts->lowest_pixel[sby]; + for (int n = 0; n < 7; n++) + for (int m = 0; m < 2; m++) + lowest_px[n][m] = INT_MIN; + } + + reset_context(&t->l, IS_KEY_OR_INTRA(f->frame_hdr), t->frame_thread.pass); + if (t->frame_thread.pass == 2) { + const int off_2pass = c->n_tc > 1 ? f->sb128w * f->frame_hdr->tiling.rows : 0; + for (t->bx = ts->tiling.col_start, + t->a = f->a + off_2pass + col_sb128_start + tile_row * f->sb128w; + t->bx < ts->tiling.col_end; t->bx += sb_step) + { + if (atomic_load_explicit(c->flush, memory_order_acquire)) + return 1; + if (decode_sb(t, root_bl, c->intra_edge.root[root_bl])) + return 1; + if (t->bx & 16 || f->seq_hdr->sb128) + t->a++; + } + f->bd_fn.backup_ipred_edge(t); + return 0; + } + + // error out on symbol decoder overread + if (ts->msac.cnt < -15) return 1; + + if (f->c->n_tc > 1 && f->frame_hdr->use_ref_frame_mvs) { + f->c->refmvs_dsp.load_tmvs(&f->rf, ts->tiling.row, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); + } + memset(t->pal_sz_uv[1], 0, sizeof(*t->pal_sz_uv)); + const int sb128y = t->by >> 5; + for (t->bx = ts->tiling.col_start, t->a = f->a + col_sb128_start + tile_row * f->sb128w, + t->lf_mask = f->lf.mask + sb128y * f->sb128w + col_sb128_start; + t->bx < ts->tiling.col_end; t->bx += sb_step) + { + if (atomic_load_explicit(c->flush, memory_order_acquire)) + return 1; + if (root_bl == BL_128X128) { + t->cur_sb_cdef_idx_ptr = t->lf_mask->cdef_idx; + t->cur_sb_cdef_idx_ptr[0] = -1; + t->cur_sb_cdef_idx_ptr[1] = -1; + t->cur_sb_cdef_idx_ptr[2] = -1; + t->cur_sb_cdef_idx_ptr[3] = -1; + } else { + t->cur_sb_cdef_idx_ptr = + &t->lf_mask->cdef_idx[((t->bx & 16) >> 4) + + ((t->by & 16) >> 3)]; + t->cur_sb_cdef_idx_ptr[0] = -1; + } + // Restoration filter + for (int p = 0; p < 3; p++) { + if (!((f->lf.restore_planes >> p) & 1U)) + continue; + + const int ss_ver = p && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = p && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int unit_size_log2 = f->frame_hdr->restoration.unit_size[!!p]; + const int y = t->by * 4 >> ss_ver; + const int h = (f->cur.p.h + ss_ver) >> ss_ver; + + const int unit_size = 1 << unit_size_log2; + const unsigned mask = unit_size - 1; + if (y & mask) continue; + const int half_unit = unit_size >> 1; + // Round half up at frame boundaries, if there's more than one + // restoration unit + if (y && y + half_unit > h) continue; + + const enum Dav1dRestorationType frame_type = f->frame_hdr->restoration.type[p]; + + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { + const int w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; + const int n_units = imax(1, (w + half_unit) >> unit_size_log2); + + const int d = f->frame_hdr->super_res.width_scale_denominator; + const int rnd = unit_size * 8 - 1, shift = unit_size_log2 + 3; + const int x0 = ((4 * t->bx * d >> ss_hor) + rnd) >> shift; + const int x1 = ((4 * (t->bx + sb_step) * d >> ss_hor) + rnd) >> shift; + + for (int x = x0; x < imin(x1, n_units); x++) { + const int px_x = x << (unit_size_log2 + ss_hor); + const int sb_idx = (t->by >> 5) * f->sr_sb128w + (px_x >> 7); + const int unit_idx = ((t->by & 16) >> 3) + ((px_x & 64) >> 6); + Av1RestorationUnit *const lr = &f->lf.lr_mask[sb_idx].lr[p][unit_idx]; + + read_restoration_info(t, lr, p, frame_type); + } + } else { + const int x = 4 * t->bx >> ss_hor; + if (x & mask) continue; + const int w = (f->cur.p.w + ss_hor) >> ss_hor; + // Round half up at frame boundaries, if there's more than one + // restoration unit + if (x && x + half_unit > w) continue; + const int sb_idx = (t->by >> 5) * f->sr_sb128w + (t->bx >> 5); + const int unit_idx = ((t->by & 16) >> 3) + ((t->bx & 16) >> 4); + Av1RestorationUnit *const lr = &f->lf.lr_mask[sb_idx].lr[p][unit_idx]; + + read_restoration_info(t, lr, p, frame_type); + } + } + if (decode_sb(t, root_bl, c->intra_edge.root[root_bl])) + return 1; + if (t->bx & 16 || f->seq_hdr->sb128) { + t->a++; + t->lf_mask++; + } + } + + if (f->seq_hdr->ref_frame_mvs && f->c->n_tc > 1 && IS_INTER_OR_SWITCH(f->frame_hdr)) { + dav1d_refmvs_save_tmvs(&f->c->refmvs_dsp, &t->rt, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); + } + + // backup pre-loopfilter pixels for intra prediction of the next sbrow + if (t->frame_thread.pass != 1) + f->bd_fn.backup_ipred_edge(t); + + // backup t->a/l.tx_lpf_y/uv at tile boundaries to use them to "fix" + // up the initial value in neighbour tiles when running the loopfilter + int align_h = (f->bh + 31) & ~31; + memcpy(&f->lf.tx_lpf_right_edge[0][align_h * tile_col + t->by], + &t->l.tx_lpf_y[t->by & 16], sb_step); + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + align_h >>= ss_ver; + memcpy(&f->lf.tx_lpf_right_edge[1][align_h * tile_col + (t->by >> ss_ver)], + &t->l.tx_lpf_uv[(t->by & 16) >> ss_ver], sb_step >> ss_ver); + + return 0; +} + +int dav1d_decode_frame_init(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + int retval = DAV1D_ERR(ENOMEM); + + if (f->sbh > f->lf.start_of_tile_row_sz) { + free(f->lf.start_of_tile_row); + f->lf.start_of_tile_row = malloc(f->sbh * sizeof(uint8_t)); + if (!f->lf.start_of_tile_row) { + f->lf.start_of_tile_row_sz = 0; + goto error; + } + f->lf.start_of_tile_row_sz = f->sbh; + } + int sby = 0; + for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { + f->lf.start_of_tile_row[sby++] = tile_row; + while (sby < f->frame_hdr->tiling.row_start_sb[tile_row + 1]) + f->lf.start_of_tile_row[sby++] = 0; + } + + const int n_ts = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; + if (n_ts != f->n_ts) { + if (c->n_fc > 1) { + freep(&f->frame_thread.tile_start_off); + f->frame_thread.tile_start_off = + malloc(sizeof(*f->frame_thread.tile_start_off) * n_ts); + if (!f->frame_thread.tile_start_off) { + f->n_ts = 0; + goto error; + } + } + dav1d_free_aligned(f->ts); + f->ts = dav1d_alloc_aligned(sizeof(*f->ts) * n_ts, 32); + if (!f->ts) goto error; + f->n_ts = n_ts; + } + + const int a_sz = f->sb128w * f->frame_hdr->tiling.rows * (1 + (c->n_fc > 1 && c->n_tc > 1)); + if (a_sz != f->a_sz) { + freep(&f->a); + f->a = malloc(sizeof(*f->a) * a_sz); + if (!f->a) { + f->a_sz = 0; + goto error; + } + f->a_sz = a_sz; + } + + const int num_sb128 = f->sb128w * f->sb128h; + const uint8_t *const size_mul = ss_size_mul[f->cur.p.layout]; + const int hbd = !!f->seq_hdr->hbd; + if (c->n_fc > 1) { + int tile_idx = 0; + for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { + int row_off = f->frame_hdr->tiling.row_start_sb[tile_row] * + f->sb_step * 4 * f->sb128w * 128; + int b_diff = (f->frame_hdr->tiling.row_start_sb[tile_row + 1] - + f->frame_hdr->tiling.row_start_sb[tile_row]) * f->sb_step * 4; + for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { + f->frame_thread.tile_start_off[tile_idx++] = row_off + b_diff * + f->frame_hdr->tiling.col_start_sb[tile_col] * f->sb_step * 4; + } + } + + const int lowest_pixel_mem_sz = f->frame_hdr->tiling.cols * f->sbh; + if (lowest_pixel_mem_sz != f->tile_thread.lowest_pixel_mem_sz) { + free(f->tile_thread.lowest_pixel_mem); + f->tile_thread.lowest_pixel_mem = + malloc(lowest_pixel_mem_sz * sizeof(*f->tile_thread.lowest_pixel_mem)); + if (!f->tile_thread.lowest_pixel_mem) { + f->tile_thread.lowest_pixel_mem_sz = 0; + goto error; + } + f->tile_thread.lowest_pixel_mem_sz = lowest_pixel_mem_sz; + } + int (*lowest_pixel_ptr)[7][2] = f->tile_thread.lowest_pixel_mem; + for (int tile_row = 0, tile_row_base = 0; tile_row < f->frame_hdr->tiling.rows; + tile_row++, tile_row_base += f->frame_hdr->tiling.cols) + { + const int tile_row_sb_h = f->frame_hdr->tiling.row_start_sb[tile_row + 1] - + f->frame_hdr->tiling.row_start_sb[tile_row]; + for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { + f->ts[tile_row_base + tile_col].lowest_pixel = lowest_pixel_ptr; + lowest_pixel_ptr += tile_row_sb_h; + } + } + + const int cf_sz = (num_sb128 * size_mul[0]) << hbd; + if (cf_sz != f->frame_thread.cf_sz) { + dav1d_freep_aligned(&f->frame_thread.cf); + f->frame_thread.cf = + dav1d_alloc_aligned((size_t)cf_sz * 128 * 128 / 2, 64); + if (!f->frame_thread.cf) { + f->frame_thread.cf_sz = 0; + goto error; + } + memset(f->frame_thread.cf, 0, (size_t)cf_sz * 128 * 128 / 2); + f->frame_thread.cf_sz = cf_sz; + } + + if (f->frame_hdr->allow_screen_content_tools) { + if (num_sb128 != f->frame_thread.pal_sz) { + dav1d_freep_aligned(&f->frame_thread.pal); + f->frame_thread.pal = + dav1d_alloc_aligned(sizeof(*f->frame_thread.pal) * + num_sb128 * 16 * 16, 64); + if (!f->frame_thread.pal) { + f->frame_thread.pal_sz = 0; + goto error; + } + f->frame_thread.pal_sz = num_sb128; + } + + const int pal_idx_sz = num_sb128 * size_mul[1]; + if (pal_idx_sz != f->frame_thread.pal_idx_sz) { + dav1d_freep_aligned(&f->frame_thread.pal_idx); + f->frame_thread.pal_idx = + dav1d_alloc_aligned(sizeof(*f->frame_thread.pal_idx) * + pal_idx_sz * 128 * 128 / 4, 64); + if (!f->frame_thread.pal_idx) { + f->frame_thread.pal_idx_sz = 0; + goto error; + } + f->frame_thread.pal_idx_sz = pal_idx_sz; + } + } else if (f->frame_thread.pal) { + dav1d_freep_aligned(&f->frame_thread.pal); + dav1d_freep_aligned(&f->frame_thread.pal_idx); + f->frame_thread.pal_sz = f->frame_thread.pal_idx_sz = 0; + } + } + + // update allocation of block contexts for above + ptrdiff_t y_stride = f->cur.stride[0], uv_stride = f->cur.stride[1]; + const int has_resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; + const int need_cdef_lpf_copy = c->n_tc > 1 && has_resize; + if (y_stride * f->sbh * 4 != f->lf.cdef_buf_plane_sz[0] || + uv_stride * f->sbh * 8 != f->lf.cdef_buf_plane_sz[1] || + need_cdef_lpf_copy != f->lf.need_cdef_lpf_copy || + f->sbh != f->lf.cdef_buf_sbh) + { + dav1d_free_aligned(f->lf.cdef_line_buf); + size_t alloc_sz = 64; + alloc_sz += (size_t)llabs(y_stride) * 4 * f->sbh << need_cdef_lpf_copy; + alloc_sz += (size_t)llabs(uv_stride) * 8 * f->sbh << need_cdef_lpf_copy; + uint8_t *ptr = f->lf.cdef_line_buf = dav1d_alloc_aligned(alloc_sz, 32); + if (!ptr) { + f->lf.cdef_buf_plane_sz[0] = f->lf.cdef_buf_plane_sz[1] = 0; + goto error; + } + + ptr += 32; + if (y_stride < 0) { + f->lf.cdef_line[0][0] = ptr - y_stride * (f->sbh * 4 - 1); + f->lf.cdef_line[1][0] = ptr - y_stride * (f->sbh * 4 - 3); + } else { + f->lf.cdef_line[0][0] = ptr + y_stride * 0; + f->lf.cdef_line[1][0] = ptr + y_stride * 2; + } + ptr += llabs(y_stride) * f->sbh * 4; + if (uv_stride < 0) { + f->lf.cdef_line[0][1] = ptr - uv_stride * (f->sbh * 8 - 1); + f->lf.cdef_line[0][2] = ptr - uv_stride * (f->sbh * 8 - 3); + f->lf.cdef_line[1][1] = ptr - uv_stride * (f->sbh * 8 - 5); + f->lf.cdef_line[1][2] = ptr - uv_stride * (f->sbh * 8 - 7); + } else { + f->lf.cdef_line[0][1] = ptr + uv_stride * 0; + f->lf.cdef_line[0][2] = ptr + uv_stride * 2; + f->lf.cdef_line[1][1] = ptr + uv_stride * 4; + f->lf.cdef_line[1][2] = ptr + uv_stride * 6; + } + + if (need_cdef_lpf_copy) { + ptr += llabs(uv_stride) * f->sbh * 8; + if (y_stride < 0) + f->lf.cdef_lpf_line[0] = ptr - y_stride * (f->sbh * 4 - 1); + else + f->lf.cdef_lpf_line[0] = ptr; + ptr += llabs(y_stride) * f->sbh * 4; + if (uv_stride < 0) { + f->lf.cdef_lpf_line[1] = ptr - uv_stride * (f->sbh * 4 - 1); + f->lf.cdef_lpf_line[2] = ptr - uv_stride * (f->sbh * 8 - 1); + } else { + f->lf.cdef_lpf_line[1] = ptr; + f->lf.cdef_lpf_line[2] = ptr + uv_stride * f->sbh * 4; + } + } + + f->lf.cdef_buf_plane_sz[0] = (int) y_stride * f->sbh * 4; + f->lf.cdef_buf_plane_sz[1] = (int) uv_stride * f->sbh * 8; + f->lf.need_cdef_lpf_copy = need_cdef_lpf_copy; + f->lf.cdef_buf_sbh = f->sbh; + } + + const int sb128 = f->seq_hdr->sb128; + const int num_lines = c->n_tc > 1 ? f->sbh * 4 << sb128 : 12; + y_stride = f->sr_cur.p.stride[0], uv_stride = f->sr_cur.p.stride[1]; + if (y_stride * num_lines != f->lf.lr_buf_plane_sz[0] || + uv_stride * num_lines * 2 != f->lf.lr_buf_plane_sz[1]) + { + dav1d_free_aligned(f->lf.lr_line_buf); + // lr simd may overread the input, so slightly over-allocate the lpf buffer + size_t alloc_sz = 128; + alloc_sz += (size_t)llabs(y_stride) * num_lines; + alloc_sz += (size_t)llabs(uv_stride) * num_lines * 2; + uint8_t *ptr = f->lf.lr_line_buf = dav1d_alloc_aligned(alloc_sz, 64); + if (!ptr) { + f->lf.lr_buf_plane_sz[0] = f->lf.lr_buf_plane_sz[1] = 0; + goto error; + } + + ptr += 64; + if (y_stride < 0) + f->lf.lr_lpf_line[0] = ptr - y_stride * (num_lines - 1); + else + f->lf.lr_lpf_line[0] = ptr; + ptr += llabs(y_stride) * num_lines; + if (uv_stride < 0) { + f->lf.lr_lpf_line[1] = ptr - uv_stride * (num_lines * 1 - 1); + f->lf.lr_lpf_line[2] = ptr - uv_stride * (num_lines * 2 - 1); + } else { + f->lf.lr_lpf_line[1] = ptr; + f->lf.lr_lpf_line[2] = ptr + uv_stride * num_lines; + } + + f->lf.lr_buf_plane_sz[0] = (int) y_stride * num_lines; + f->lf.lr_buf_plane_sz[1] = (int) uv_stride * num_lines * 2; + } + + // update allocation for loopfilter masks + if (num_sb128 != f->lf.mask_sz) { + freep(&f->lf.mask); + freep(&f->lf.level); + f->lf.mask = malloc(sizeof(*f->lf.mask) * num_sb128); + // over-allocate by 3 bytes since some of the SIMD implementations + // index this from the level type and can thus over-read by up to 3 + f->lf.level = malloc(sizeof(*f->lf.level) * num_sb128 * 32 * 32 + 3); + if (!f->lf.mask || !f->lf.level) { + f->lf.mask_sz = 0; + goto error; + } + if (c->n_fc > 1) { + freep(&f->frame_thread.b); + freep(&f->frame_thread.cbi); + f->frame_thread.b = malloc(sizeof(*f->frame_thread.b) * + num_sb128 * 32 * 32); + f->frame_thread.cbi = malloc(sizeof(*f->frame_thread.cbi) * + num_sb128 * 32 * 32); + if (!f->frame_thread.b || !f->frame_thread.cbi) { + f->lf.mask_sz = 0; + goto error; + } + } + f->lf.mask_sz = num_sb128; + } + + f->sr_sb128w = (f->sr_cur.p.p.w + 127) >> 7; + const int lr_mask_sz = f->sr_sb128w * f->sb128h; + if (lr_mask_sz != f->lf.lr_mask_sz) { + freep(&f->lf.lr_mask); + f->lf.lr_mask = malloc(sizeof(*f->lf.lr_mask) * lr_mask_sz); + if (!f->lf.lr_mask) { + f->lf.lr_mask_sz = 0; + goto error; + } + f->lf.lr_mask_sz = lr_mask_sz; + } + f->lf.restore_planes = + ((f->frame_hdr->restoration.type[0] != DAV1D_RESTORATION_NONE) << 0) + + ((f->frame_hdr->restoration.type[1] != DAV1D_RESTORATION_NONE) << 1) + + ((f->frame_hdr->restoration.type[2] != DAV1D_RESTORATION_NONE) << 2); + if (f->frame_hdr->loopfilter.sharpness != f->lf.last_sharpness) { + dav1d_calc_eih(&f->lf.lim_lut, f->frame_hdr->loopfilter.sharpness); + f->lf.last_sharpness = f->frame_hdr->loopfilter.sharpness; + } + dav1d_calc_lf_values(f->lf.lvl, f->frame_hdr, (int8_t[4]) { 0, 0, 0, 0 }); + memset(f->lf.mask, 0, sizeof(*f->lf.mask) * num_sb128); + + const int ipred_edge_sz = f->sbh * f->sb128w << hbd; + if (ipred_edge_sz != f->ipred_edge_sz) { + dav1d_freep_aligned(&f->ipred_edge[0]); + uint8_t *ptr = f->ipred_edge[0] = + dav1d_alloc_aligned(ipred_edge_sz * 128 * 3, 64); + if (!ptr) { + f->ipred_edge_sz = 0; + goto error; + } + f->ipred_edge[1] = ptr + ipred_edge_sz * 128 * 1; + f->ipred_edge[2] = ptr + ipred_edge_sz * 128 * 2; + f->ipred_edge_sz = ipred_edge_sz; + } + + const int re_sz = f->sb128h * f->frame_hdr->tiling.cols; + if (re_sz != f->lf.re_sz) { + freep(&f->lf.tx_lpf_right_edge[0]); + f->lf.tx_lpf_right_edge[0] = malloc(re_sz * 32 * 2); + if (!f->lf.tx_lpf_right_edge[0]) { + f->lf.re_sz = 0; + goto error; + } + f->lf.tx_lpf_right_edge[1] = f->lf.tx_lpf_right_edge[0] + re_sz * 32; + f->lf.re_sz = re_sz; + } + + // init ref mvs + if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { + const int ret = + dav1d_refmvs_init_frame(&f->rf, f->seq_hdr, f->frame_hdr, + f->refpoc, f->mvs, f->refrefpoc, f->ref_mvs, + f->c->n_tc, f->c->n_fc); + if (ret < 0) goto error; + } + + // setup dequant tables + init_quant_tables(f->seq_hdr, f->frame_hdr, f->frame_hdr->quant.yac, f->dq); + if (f->frame_hdr->quant.qm) + for (int i = 0; i < N_RECT_TX_SIZES; i++) { + f->qm[i][0] = dav1d_qm_tbl[f->frame_hdr->quant.qm_y][0][i]; + f->qm[i][1] = dav1d_qm_tbl[f->frame_hdr->quant.qm_u][1][i]; + f->qm[i][2] = dav1d_qm_tbl[f->frame_hdr->quant.qm_v][1][i]; + } + else + memset(f->qm, 0, sizeof(f->qm)); + + // setup jnt_comp weights + if (f->frame_hdr->switchable_comp_refs) { + for (int i = 0; i < 7; i++) { + const unsigned ref0poc = f->refp[i].p.frame_hdr->frame_offset; + + for (int j = i + 1; j < 7; j++) { + const unsigned ref1poc = f->refp[j].p.frame_hdr->frame_offset; + + const unsigned d1 = + imin(abs(get_poc_diff(f->seq_hdr->order_hint_n_bits, ref0poc, + f->cur.frame_hdr->frame_offset)), 31); + const unsigned d0 = + imin(abs(get_poc_diff(f->seq_hdr->order_hint_n_bits, ref1poc, + f->cur.frame_hdr->frame_offset)), 31); + const int order = d0 <= d1; + + static const uint8_t quant_dist_weight[3][2] = { + { 2, 3 }, { 2, 5 }, { 2, 7 } + }; + static const uint8_t quant_dist_lookup_table[4][2] = { + { 9, 7 }, { 11, 5 }, { 12, 4 }, { 13, 3 } + }; + + int k; + for (k = 0; k < 3; k++) { + const int c0 = quant_dist_weight[k][order]; + const int c1 = quant_dist_weight[k][!order]; + const int d0_c0 = d0 * c0; + const int d1_c1 = d1 * c1; + if ((d0 > d1 && d0_c0 < d1_c1) || (d0 <= d1 && d0_c0 > d1_c1)) break; + } + + f->jnt_weights[i][j] = quant_dist_lookup_table[k][order]; + } + } + } + + /* Init loopfilter pointers. Increasing NULL pointers is technically UB, + * so just point the chroma pointers in 4:0:0 to the luma plane here to + * avoid having additional in-loop branches in various places. We never + * dereference those pointers so it doesn't really matter what they + * point at, as long as the pointers are valid. */ + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400; + f->lf.mask_ptr = f->lf.mask; + f->lf.p[0] = f->cur.data[0]; + f->lf.p[1] = f->cur.data[has_chroma ? 1 : 0]; + f->lf.p[2] = f->cur.data[has_chroma ? 2 : 0]; + f->lf.sr_p[0] = f->sr_cur.p.data[0]; + f->lf.sr_p[1] = f->sr_cur.p.data[has_chroma ? 1 : 0]; + f->lf.sr_p[2] = f->sr_cur.p.data[has_chroma ? 2 : 0]; + + retval = 0; +error: + return retval; +} + +int dav1d_decode_frame_init_cdf(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + int retval = DAV1D_ERR(EINVAL); + + if (f->frame_hdr->refresh_context) + dav1d_cdf_thread_copy(f->out_cdf.data.cdf, &f->in_cdf); + + // parse individual tiles per tile group + int tile_row = 0, tile_col = 0; + f->task_thread.update_set = 0; + for (int i = 0; i < f->n_tile_data; i++) { + const uint8_t *data = f->tile[i].data.data; + size_t size = f->tile[i].data.sz; + + for (int j = f->tile[i].start; j <= f->tile[i].end; j++) { + size_t tile_sz; + if (j == f->tile[i].end) { + tile_sz = size; + } else { + if (f->frame_hdr->tiling.n_bytes > size) goto error; + tile_sz = 0; + for (unsigned k = 0; k < f->frame_hdr->tiling.n_bytes; k++) + tile_sz |= (unsigned)*data++ << (k * 8); + tile_sz++; + size -= f->frame_hdr->tiling.n_bytes; + if (tile_sz > size) goto error; + } + + setup_tile(&f->ts[j], f, data, tile_sz, tile_row, tile_col++, + c->n_fc > 1 ? f->frame_thread.tile_start_off[j] : 0); + + if (tile_col == f->frame_hdr->tiling.cols) { + tile_col = 0; + tile_row++; + } + if (j == f->frame_hdr->tiling.update && f->frame_hdr->refresh_context) + f->task_thread.update_set = 1; + data += tile_sz; + size -= tile_sz; + } + } + + if (c->n_tc > 1) { + const int uses_2pass = c->n_fc > 1; + for (int n = 0; n < f->sb128w * f->frame_hdr->tiling.rows * (1 + uses_2pass); n++) + reset_context(&f->a[n], IS_KEY_OR_INTRA(f->frame_hdr), + uses_2pass ? 1 + (n >= f->sb128w * f->frame_hdr->tiling.rows) : 0); + } + + retval = 0; +error: + return retval; +} + +int dav1d_decode_frame_main(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + int retval = DAV1D_ERR(EINVAL); + + assert(f->c->n_tc == 1); + + Dav1dTaskContext *const t = &c->tc[f - c->fc]; + t->f = f; + t->frame_thread.pass = 0; + + for (int n = 0; n < f->sb128w * f->frame_hdr->tiling.rows; n++) + reset_context(&f->a[n], IS_KEY_OR_INTRA(f->frame_hdr), 0); + + // no threading - we explicitly interleave tile/sbrow decoding + // and post-filtering, so that the full process runs in-line + for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { + const int sbh_end = + imin(f->frame_hdr->tiling.row_start_sb[tile_row + 1], f->sbh); + for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; + sby < sbh_end; sby++) + { + t->by = sby << (4 + f->seq_hdr->sb128); + const int by_end = (t->by + f->sb_step) >> 1; + if (f->frame_hdr->use_ref_frame_mvs) { + f->c->refmvs_dsp.load_tmvs(&f->rf, tile_row, + 0, f->bw >> 1, t->by >> 1, by_end); + } + for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { + t->ts = &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; + if (dav1d_decode_tile_sbrow(t)) goto error; + } + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + dav1d_refmvs_save_tmvs(&f->c->refmvs_dsp, &t->rt, + 0, f->bw >> 1, t->by >> 1, by_end); + } + + // loopfilter + cdef + restoration + f->bd_fn.filter_sbrow(f, sby); + } + } + + retval = 0; +error: + return retval; +} + +void dav1d_decode_frame_exit(Dav1dFrameContext *const f, const int retval) { + const Dav1dContext *const c = f->c; + + if (f->sr_cur.p.data[0]) + atomic_init(&f->task_thread.error, 0); + + if (c->n_fc > 1 && retval && f->frame_thread.cf) { + memset(f->frame_thread.cf, 0, + (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); + } + for (int i = 0; i < 7; i++) { + if (f->refp[i].p.frame_hdr) + dav1d_thread_picture_unref(&f->refp[i]); + dav1d_ref_dec(&f->ref_mvs_ref[i]); + } + + dav1d_picture_unref_internal(&f->cur); + dav1d_thread_picture_unref(&f->sr_cur); + dav1d_cdf_thread_unref(&f->in_cdf); + if (f->frame_hdr && f->frame_hdr->refresh_context) { + if (f->out_cdf.progress) + atomic_store(f->out_cdf.progress, retval == 0 ? 1 : TILE_ERROR); + dav1d_cdf_thread_unref(&f->out_cdf); + } + dav1d_ref_dec(&f->cur_segmap_ref); + dav1d_ref_dec(&f->prev_segmap_ref); + dav1d_ref_dec(&f->mvs_ref); + dav1d_ref_dec(&f->seq_hdr_ref); + dav1d_ref_dec(&f->frame_hdr_ref); + + for (int i = 0; i < f->n_tile_data; i++) + dav1d_data_unref_internal(&f->tile[i].data); + f->task_thread.retval = retval; +} + +int dav1d_decode_frame(Dav1dFrameContext *const f) { + assert(f->c->n_fc == 1); + // if n_tc > 1 (but n_fc == 1), we could run init/exit in the task + // threads also. Not sure it makes a measurable difference. + int res = dav1d_decode_frame_init(f); + if (!res) res = dav1d_decode_frame_init_cdf(f); + // wait until all threads have completed + if (!res) { + if (f->c->n_tc > 1) { + res = dav1d_task_create_tile_sbrow(f, 0, 1); + pthread_mutex_lock(&f->task_thread.ttd->lock); + pthread_cond_signal(&f->task_thread.ttd->cond); + if (!res) { + while (!f->task_thread.done[0] || + atomic_load(&f->task_thread.task_counter) > 0) + { + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); + } + } + pthread_mutex_unlock(&f->task_thread.ttd->lock); + res = f->task_thread.retval; + } else { + res = dav1d_decode_frame_main(f); + if (!res && f->frame_hdr->refresh_context && f->task_thread.update_set) { + dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, + &f->ts[f->frame_hdr->tiling.update].cdf); + } + } + } + dav1d_decode_frame_exit(f, res); + f->n_tile_data = 0; + return res; +} + +static int get_upscale_x0(const int in_w, const int out_w, const int step) { + const int err = out_w * step - (in_w << 14); + const int x0 = (-((out_w - in_w) << 13) + (out_w >> 1)) / out_w + 128 - (err / 2); + return x0 & 0x3fff; +} + +int dav1d_submit_frame(Dav1dContext *const c) { + Dav1dFrameContext *f; + int res = -1; + + // wait for c->out_delayed[next] and move into c->out if visible + Dav1dThreadPicture *out_delayed; + if (c->n_fc > 1) { + pthread_mutex_lock(&c->task_thread.lock); + const unsigned next = c->frame_thread.next++; + if (c->frame_thread.next == c->n_fc) + c->frame_thread.next = 0; + + f = &c->fc[next]; + while (f->n_tile_data > 0) + pthread_cond_wait(&f->task_thread.cond, + &c->task_thread.lock); + out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + unsigned first = atomic_load(&c->task_thread.first); + if (first + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + atomic_compare_exchange_strong(&c->task_thread.reset_task_cur, + &first, UINT_MAX); + if (c->task_thread.cur && c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + } + const int error = f->task_thread.retval; + if (error) { + f->task_thread.retval = 0; + c->cached_error = error; + dav1d_data_props_copy(&c->cached_error_props, &out_delayed->p.m); + dav1d_thread_picture_unref(out_delayed); + } else if (out_delayed->p.data[0]) { + const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], + memory_order_relaxed); + if ((out_delayed->visible || c->output_invisible_frames) && + progress != FRAME_ERROR) + { + dav1d_thread_picture_ref(&c->out, out_delayed); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } + dav1d_thread_picture_unref(out_delayed); + } + } else { + f = c->fc; + } + + f->seq_hdr = c->seq_hdr; + f->seq_hdr_ref = c->seq_hdr_ref; + dav1d_ref_inc(f->seq_hdr_ref); + f->frame_hdr = c->frame_hdr; + f->frame_hdr_ref = c->frame_hdr_ref; + c->frame_hdr = NULL; + c->frame_hdr_ref = NULL; + f->dsp = &c->dsp[f->seq_hdr->hbd]; + + const int bpc = 8 + 2 * f->seq_hdr->hbd; + + if (!f->dsp->ipred.intra_pred[DC_PRED]) { + Dav1dDSPContext *const dsp = &c->dsp[f->seq_hdr->hbd]; + + switch (bpc) { +#define assign_bitdepth_case(bd) \ + dav1d_cdef_dsp_init_##bd##bpc(&dsp->cdef); \ + dav1d_intra_pred_dsp_init_##bd##bpc(&dsp->ipred); \ + dav1d_itx_dsp_init_##bd##bpc(&dsp->itx, bpc); \ + dav1d_loop_filter_dsp_init_##bd##bpc(&dsp->lf); \ + dav1d_loop_restoration_dsp_init_##bd##bpc(&dsp->lr, bpc); \ + dav1d_mc_dsp_init_##bd##bpc(&dsp->mc); \ + dav1d_film_grain_dsp_init_##bd##bpc(&dsp->fg); \ + break +#if CONFIG_8BPC + case 8: + assign_bitdepth_case(8); +#endif +#if CONFIG_16BPC + case 10: + case 12: + assign_bitdepth_case(16); +#endif +#undef assign_bitdepth_case + default: + dav1d_log(c, "Compiled without support for %d-bit decoding\n", + 8 + 2 * f->seq_hdr->hbd); + res = DAV1D_ERR(ENOPROTOOPT); + goto error; + } + } + +#define assign_bitdepth_case(bd) \ + f->bd_fn.recon_b_inter = dav1d_recon_b_inter_##bd##bpc; \ + f->bd_fn.recon_b_intra = dav1d_recon_b_intra_##bd##bpc; \ + f->bd_fn.filter_sbrow = dav1d_filter_sbrow_##bd##bpc; \ + f->bd_fn.filter_sbrow_deblock_cols = dav1d_filter_sbrow_deblock_cols_##bd##bpc; \ + f->bd_fn.filter_sbrow_deblock_rows = dav1d_filter_sbrow_deblock_rows_##bd##bpc; \ + f->bd_fn.filter_sbrow_cdef = dav1d_filter_sbrow_cdef_##bd##bpc; \ + f->bd_fn.filter_sbrow_resize = dav1d_filter_sbrow_resize_##bd##bpc; \ + f->bd_fn.filter_sbrow_lr = dav1d_filter_sbrow_lr_##bd##bpc; \ + f->bd_fn.backup_ipred_edge = dav1d_backup_ipred_edge_##bd##bpc; \ + f->bd_fn.read_coef_blocks = dav1d_read_coef_blocks_##bd##bpc + if (!f->seq_hdr->hbd) { +#if CONFIG_8BPC + assign_bitdepth_case(8); +#endif + } else { +#if CONFIG_16BPC + assign_bitdepth_case(16); +#endif + } +#undef assign_bitdepth_case + + int ref_coded_width[7]; + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + if (f->frame_hdr->primary_ref_frame != DAV1D_PRIMARY_REF_NONE) { + const int pri_ref = f->frame_hdr->refidx[f->frame_hdr->primary_ref_frame]; + if (!c->refs[pri_ref].p.p.data[0]) { + res = DAV1D_ERR(EINVAL); + goto error; + } + } + for (int i = 0; i < 7; i++) { + const int refidx = f->frame_hdr->refidx[i]; + if (!c->refs[refidx].p.p.data[0] || + f->frame_hdr->width[0] * 2 < c->refs[refidx].p.p.p.w || + f->frame_hdr->height * 2 < c->refs[refidx].p.p.p.h || + f->frame_hdr->width[0] > c->refs[refidx].p.p.p.w * 16 || + f->frame_hdr->height > c->refs[refidx].p.p.p.h * 16 || + f->seq_hdr->layout != c->refs[refidx].p.p.p.layout || + bpc != c->refs[refidx].p.p.p.bpc) + { + for (int j = 0; j < i; j++) + dav1d_thread_picture_unref(&f->refp[j]); + res = DAV1D_ERR(EINVAL); + goto error; + } + dav1d_thread_picture_ref(&f->refp[i], &c->refs[refidx].p); + ref_coded_width[i] = c->refs[refidx].p.p.frame_hdr->width[0]; + if (f->frame_hdr->width[0] != c->refs[refidx].p.p.p.w || + f->frame_hdr->height != c->refs[refidx].p.p.p.h) + { +#define scale_fac(ref_sz, this_sz) \ + ((((ref_sz) << 14) + ((this_sz) >> 1)) / (this_sz)) + f->svc[i][0].scale = scale_fac(c->refs[refidx].p.p.p.w, + f->frame_hdr->width[0]); + f->svc[i][1].scale = scale_fac(c->refs[refidx].p.p.p.h, + f->frame_hdr->height); + f->svc[i][0].step = (f->svc[i][0].scale + 8) >> 4; + f->svc[i][1].step = (f->svc[i][1].scale + 8) >> 4; + } else { + f->svc[i][0].scale = f->svc[i][1].scale = 0; + } + f->gmv_warp_allowed[i] = f->frame_hdr->gmv[i].type > DAV1D_WM_TYPE_TRANSLATION && + !f->frame_hdr->force_integer_mv && + !dav1d_get_shear_params(&f->frame_hdr->gmv[i]) && + !f->svc[i][0].scale; + } + } + + // setup entropy + if (f->frame_hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { + dav1d_cdf_thread_init_static(&f->in_cdf, f->frame_hdr->quant.yac); + } else { + const int pri_ref = f->frame_hdr->refidx[f->frame_hdr->primary_ref_frame]; + dav1d_cdf_thread_ref(&f->in_cdf, &c->cdf[pri_ref]); + } + if (f->frame_hdr->refresh_context) { + res = dav1d_cdf_thread_alloc(c, &f->out_cdf, c->n_fc > 1); + if (res < 0) goto error; + } + + // FIXME qsort so tiles are in order (for frame threading) + if (f->n_tile_data_alloc < c->n_tile_data) { + freep(&f->tile); + assert(c->n_tile_data < INT_MAX / (int)sizeof(*f->tile)); + f->tile = malloc(c->n_tile_data * sizeof(*f->tile)); + if (!f->tile) { + f->n_tile_data_alloc = f->n_tile_data = 0; + res = DAV1D_ERR(ENOMEM); + goto error; + } + f->n_tile_data_alloc = c->n_tile_data; + } + memcpy(f->tile, c->tile, c->n_tile_data * sizeof(*f->tile)); + memset(c->tile, 0, c->n_tile_data * sizeof(*c->tile)); + f->n_tile_data = c->n_tile_data; + c->n_tile_data = 0; + + // allocate frame + res = dav1d_thread_picture_alloc(c, f, bpc); + if (res < 0) goto error; + + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { + res = dav1d_picture_alloc_copy(c, &f->cur, f->frame_hdr->width[0], &f->sr_cur.p); + if (res < 0) goto error; + } else { + dav1d_picture_ref(&f->cur, &f->sr_cur.p); + } + + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { + f->resize_step[0] = scale_fac(f->cur.p.w, f->sr_cur.p.p.w); + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int in_cw = (f->cur.p.w + ss_hor) >> ss_hor; + const int out_cw = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; + f->resize_step[1] = scale_fac(in_cw, out_cw); +#undef scale_fac + f->resize_start[0] = get_upscale_x0(f->cur.p.w, f->sr_cur.p.p.w, f->resize_step[0]); + f->resize_start[1] = get_upscale_x0(in_cw, out_cw, f->resize_step[1]); + } + + // move f->cur into output queue + if (c->n_fc == 1) { + if (f->frame_hdr->show_frame || c->output_invisible_frames) { + dav1d_thread_picture_ref(&c->out, &f->sr_cur); + c->event_flags |= dav1d_picture_get_event_flags(&f->sr_cur); + } + } else { + dav1d_thread_picture_ref(out_delayed, &f->sr_cur); + } + + f->w4 = (f->frame_hdr->width[0] + 3) >> 2; + f->h4 = (f->frame_hdr->height + 3) >> 2; + f->bw = ((f->frame_hdr->width[0] + 7) >> 3) << 1; + f->bh = ((f->frame_hdr->height + 7) >> 3) << 1; + f->sb128w = (f->bw + 31) >> 5; + f->sb128h = (f->bh + 31) >> 5; + f->sb_shift = 4 + f->seq_hdr->sb128; + f->sb_step = 16 << f->seq_hdr->sb128; + f->sbh = (f->bh + f->sb_step - 1) >> f->sb_shift; + f->b4_stride = (f->bw + 31) & ~31; + f->bitdepth_max = (1 << f->cur.p.bpc) - 1; + atomic_init(&f->task_thread.error, 0); + const int uses_2pass = c->n_fc > 1; + const int cols = f->frame_hdr->tiling.cols; + const int rows = f->frame_hdr->tiling.rows; + atomic_store(&f->task_thread.task_counter, + (cols * rows + f->sbh) << uses_2pass); + + // ref_mvs + if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { + f->mvs_ref = dav1d_ref_create_using_pool(c->refmvs_pool, + sizeof(*f->mvs) * f->sb128h * 16 * (f->b4_stride >> 1)); + if (!f->mvs_ref) { + res = DAV1D_ERR(ENOMEM); + goto error; + } + f->mvs = f->mvs_ref->data; + if (!f->frame_hdr->allow_intrabc) { + for (int i = 0; i < 7; i++) + f->refpoc[i] = f->refp[i].p.frame_hdr->frame_offset; + } else { + memset(f->refpoc, 0, sizeof(f->refpoc)); + } + if (f->frame_hdr->use_ref_frame_mvs) { + for (int i = 0; i < 7; i++) { + const int refidx = f->frame_hdr->refidx[i]; + const int ref_w = ((ref_coded_width[i] + 7) >> 3) << 1; + const int ref_h = ((f->refp[i].p.p.h + 7) >> 3) << 1; + if (c->refs[refidx].refmvs != NULL && + ref_w == f->bw && ref_h == f->bh) + { + f->ref_mvs_ref[i] = c->refs[refidx].refmvs; + dav1d_ref_inc(f->ref_mvs_ref[i]); + f->ref_mvs[i] = c->refs[refidx].refmvs->data; + } else { + f->ref_mvs[i] = NULL; + f->ref_mvs_ref[i] = NULL; + } + memcpy(f->refrefpoc[i], c->refs[refidx].refpoc, + sizeof(*f->refrefpoc)); + } + } else { + memset(f->ref_mvs_ref, 0, sizeof(f->ref_mvs_ref)); + } + } else { + f->mvs_ref = NULL; + memset(f->ref_mvs_ref, 0, sizeof(f->ref_mvs_ref)); + } + + // segmap + if (f->frame_hdr->segmentation.enabled) { + // By default, the previous segmentation map is not initialised. + f->prev_segmap_ref = NULL; + f->prev_segmap = NULL; + + // We might need a previous frame's segmentation map. This + // happens if there is either no update or a temporal update. + if (f->frame_hdr->segmentation.temporal || !f->frame_hdr->segmentation.update_map) { + const int pri_ref = f->frame_hdr->primary_ref_frame; + assert(pri_ref != DAV1D_PRIMARY_REF_NONE); + const int ref_w = ((ref_coded_width[pri_ref] + 7) >> 3) << 1; + const int ref_h = ((f->refp[pri_ref].p.p.h + 7) >> 3) << 1; + if (ref_w == f->bw && ref_h == f->bh) { + f->prev_segmap_ref = c->refs[f->frame_hdr->refidx[pri_ref]].segmap; + if (f->prev_segmap_ref) { + dav1d_ref_inc(f->prev_segmap_ref); + f->prev_segmap = f->prev_segmap_ref->data; + } + } + } + + if (f->frame_hdr->segmentation.update_map) { + // We're updating an existing map, but need somewhere to + // put the new values. Allocate them here (the data + // actually gets set elsewhere) + f->cur_segmap_ref = dav1d_ref_create_using_pool(c->segmap_pool, + sizeof(*f->cur_segmap) * f->b4_stride * 32 * f->sb128h); + if (!f->cur_segmap_ref) { + dav1d_ref_dec(&f->prev_segmap_ref); + res = DAV1D_ERR(ENOMEM); + goto error; + } + f->cur_segmap = f->cur_segmap_ref->data; + } else if (f->prev_segmap_ref) { + // We're not updating an existing map, and we have a valid + // reference. Use that. + f->cur_segmap_ref = f->prev_segmap_ref; + dav1d_ref_inc(f->cur_segmap_ref); + f->cur_segmap = f->prev_segmap_ref->data; + } else { + // We need to make a new map. Allocate one here and zero it out. + const size_t segmap_size = sizeof(*f->cur_segmap) * f->b4_stride * 32 * f->sb128h; + f->cur_segmap_ref = dav1d_ref_create_using_pool(c->segmap_pool, segmap_size); + if (!f->cur_segmap_ref) { + res = DAV1D_ERR(ENOMEM); + goto error; + } + f->cur_segmap = f->cur_segmap_ref->data; + memset(f->cur_segmap, 0, segmap_size); + } + } else { + f->cur_segmap = NULL; + f->cur_segmap_ref = NULL; + f->prev_segmap_ref = NULL; + } + + // update references etc. + const unsigned refresh_frame_flags = f->frame_hdr->refresh_frame_flags; + for (int i = 0; i < 8; i++) { + if (refresh_frame_flags & (1 << i)) { + if (c->refs[i].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[i].p); + dav1d_thread_picture_ref(&c->refs[i].p, &f->sr_cur); + + dav1d_cdf_thread_unref(&c->cdf[i]); + if (f->frame_hdr->refresh_context) { + dav1d_cdf_thread_ref(&c->cdf[i], &f->out_cdf); + } else { + dav1d_cdf_thread_ref(&c->cdf[i], &f->in_cdf); + } + + dav1d_ref_dec(&c->refs[i].segmap); + c->refs[i].segmap = f->cur_segmap_ref; + if (f->cur_segmap_ref) + dav1d_ref_inc(f->cur_segmap_ref); + dav1d_ref_dec(&c->refs[i].refmvs); + if (!f->frame_hdr->allow_intrabc) { + c->refs[i].refmvs = f->mvs_ref; + if (f->mvs_ref) + dav1d_ref_inc(f->mvs_ref); + } + memcpy(c->refs[i].refpoc, f->refpoc, sizeof(f->refpoc)); + } + } + + if (c->n_fc == 1) { + if ((res = dav1d_decode_frame(f)) < 0) { + dav1d_thread_picture_unref(&c->out); + for (int i = 0; i < 8; i++) { + if (refresh_frame_flags & (1 << i)) { + if (c->refs[i].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[i].p); + dav1d_cdf_thread_unref(&c->cdf[i]); + dav1d_ref_dec(&c->refs[i].segmap); + dav1d_ref_dec(&c->refs[i].refmvs); + } + } + goto error; + } + } else { + dav1d_task_frame_init(f); + pthread_mutex_unlock(&c->task_thread.lock); + } + + return 0; +error: + atomic_init(&f->task_thread.error, 1); + dav1d_cdf_thread_unref(&f->in_cdf); + if (f->frame_hdr->refresh_context) + dav1d_cdf_thread_unref(&f->out_cdf); + for (int i = 0; i < 7; i++) { + if (f->refp[i].p.frame_hdr) + dav1d_thread_picture_unref(&f->refp[i]); + dav1d_ref_dec(&f->ref_mvs_ref[i]); + } + if (c->n_fc == 1) + dav1d_thread_picture_unref(&c->out); + else + dav1d_thread_picture_unref(out_delayed); + dav1d_picture_unref_internal(&f->cur); + dav1d_thread_picture_unref(&f->sr_cur); + dav1d_ref_dec(&f->mvs_ref); + dav1d_ref_dec(&f->seq_hdr_ref); + dav1d_ref_dec(&f->frame_hdr_ref); + dav1d_data_props_copy(&c->cached_error_props, &c->in.m); + + for (int i = 0; i < f->n_tile_data; i++) + dav1d_data_unref_internal(&f->tile[i].data); + f->n_tile_data = 0; + + if (c->n_fc > 1) + pthread_mutex_unlock(&c->task_thread.lock); + + return res; +} diff --git a/third_party/dav1d/src/decode.h b/third_party/dav1d/src/decode.h new file mode 100644 index 0000000000..1eae5850a5 --- /dev/null +++ b/third_party/dav1d/src/decode.h @@ -0,0 +1,35 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_DECODE_H +#define DAV1D_SRC_DECODE_H + +#include "src/internal.h" + +int dav1d_submit_frame(Dav1dContext *c); + +#endif /* DAV1D_SRC_DECODE_H */ diff --git a/third_party/dav1d/src/dequant_tables.c b/third_party/dav1d/src/dequant_tables.c new file mode 100644 index 0000000000..520d727b03 --- /dev/null +++ b/third_party/dav1d/src/dequant_tables.c @@ -0,0 +1,229 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "src/dequant_tables.h" + +const uint16_t dav1d_dq_tbl[3][QINDEX_RANGE][2] = { + { + { 4, 4, }, { 8, 8, }, { 8, 9, }, { 9, 10, }, + { 10, 11, }, { 11, 12, }, { 12, 13, }, { 12, 14, }, + { 13, 15, }, { 14, 16, }, { 15, 17, }, { 16, 18, }, + { 17, 19, }, { 18, 20, }, { 19, 21, }, { 19, 22, }, + { 20, 23, }, { 21, 24, }, { 22, 25, }, { 23, 26, }, + { 24, 27, }, { 25, 28, }, { 26, 29, }, { 26, 30, }, + { 27, 31, }, { 28, 32, }, { 29, 33, }, { 30, 34, }, + { 31, 35, }, { 32, 36, }, { 32, 37, }, { 33, 38, }, + { 34, 39, }, { 35, 40, }, { 36, 41, }, { 37, 42, }, + { 38, 43, }, { 38, 44, }, { 39, 45, }, { 40, 46, }, + { 41, 47, }, { 42, 48, }, { 43, 49, }, { 43, 50, }, + { 44, 51, }, { 45, 52, }, { 46, 53, }, { 47, 54, }, + { 48, 55, }, { 48, 56, }, { 49, 57, }, { 50, 58, }, + { 51, 59, }, { 52, 60, }, { 53, 61, }, { 53, 62, }, + { 54, 63, }, { 55, 64, }, { 56, 65, }, { 57, 66, }, + { 57, 67, }, { 58, 68, }, { 59, 69, }, { 60, 70, }, + { 61, 71, }, { 62, 72, }, { 62, 73, }, { 63, 74, }, + { 64, 75, }, { 65, 76, }, { 66, 77, }, { 66, 78, }, + { 67, 79, }, { 68, 80, }, { 69, 81, }, { 70, 82, }, + { 70, 83, }, { 71, 84, }, { 72, 85, }, { 73, 86, }, + { 74, 87, }, { 74, 88, }, { 75, 89, }, { 76, 90, }, + { 77, 91, }, { 78, 92, }, { 78, 93, }, { 79, 94, }, + { 80, 95, }, { 81, 96, }, { 81, 97, }, { 82, 98, }, + { 83, 99, }, { 84, 100, }, { 85, 101, }, { 85, 102, }, + { 87, 104, }, { 88, 106, }, { 90, 108, }, { 92, 110, }, + { 93, 112, }, { 95, 114, }, { 96, 116, }, { 98, 118, }, + { 99, 120, }, { 101, 122, }, { 102, 124, }, { 104, 126, }, + { 105, 128, }, { 107, 130, }, { 108, 132, }, { 110, 134, }, + { 111, 136, }, { 113, 138, }, { 114, 140, }, { 116, 142, }, + { 117, 144, }, { 118, 146, }, { 120, 148, }, { 121, 150, }, + { 123, 152, }, { 125, 155, }, { 127, 158, }, { 129, 161, }, + { 131, 164, }, { 134, 167, }, { 136, 170, }, { 138, 173, }, + { 140, 176, }, { 142, 179, }, { 144, 182, }, { 146, 185, }, + { 148, 188, }, { 150, 191, }, { 152, 194, }, { 154, 197, }, + { 156, 200, }, { 158, 203, }, { 161, 207, }, { 164, 211, }, + { 166, 215, }, { 169, 219, }, { 172, 223, }, { 174, 227, }, + { 177, 231, }, { 180, 235, }, { 182, 239, }, { 185, 243, }, + { 187, 247, }, { 190, 251, }, { 192, 255, }, { 195, 260, }, + { 199, 265, }, { 202, 270, }, { 205, 275, }, { 208, 280, }, + { 211, 285, }, { 214, 290, }, { 217, 295, }, { 220, 300, }, + { 223, 305, }, { 226, 311, }, { 230, 317, }, { 233, 323, }, + { 237, 329, }, { 240, 335, }, { 243, 341, }, { 247, 347, }, + { 250, 353, }, { 253, 359, }, { 257, 366, }, { 261, 373, }, + { 265, 380, }, { 269, 387, }, { 272, 394, }, { 276, 401, }, + { 280, 408, }, { 284, 416, }, { 288, 424, }, { 292, 432, }, + { 296, 440, }, { 300, 448, }, { 304, 456, }, { 309, 465, }, + { 313, 474, }, { 317, 483, }, { 322, 492, }, { 326, 501, }, + { 330, 510, }, { 335, 520, }, { 340, 530, }, { 344, 540, }, + { 349, 550, }, { 354, 560, }, { 359, 571, }, { 364, 582, }, + { 369, 593, }, { 374, 604, }, { 379, 615, }, { 384, 627, }, + { 389, 639, }, { 395, 651, }, { 400, 663, }, { 406, 676, }, + { 411, 689, }, { 417, 702, }, { 423, 715, }, { 429, 729, }, + { 435, 743, }, { 441, 757, }, { 447, 771, }, { 454, 786, }, + { 461, 801, }, { 467, 816, }, { 475, 832, }, { 482, 848, }, + { 489, 864, }, { 497, 881, }, { 505, 898, }, { 513, 915, }, + { 522, 933, }, { 530, 951, }, { 539, 969, }, { 549, 988, }, + { 559, 1007, }, { 569, 1026, }, { 579, 1046, }, { 590, 1066, }, + { 602, 1087, }, { 614, 1108, }, { 626, 1129, }, { 640, 1151, }, + { 654, 1173, }, { 668, 1196, }, { 684, 1219, }, { 700, 1243, }, + { 717, 1267, }, { 736, 1292, }, { 755, 1317, }, { 775, 1343, }, + { 796, 1369, }, { 819, 1396, }, { 843, 1423, }, { 869, 1451, }, + { 896, 1479, }, { 925, 1508, }, { 955, 1537, }, { 988, 1567, }, + { 1022, 1597, }, { 1058, 1628, }, { 1098, 1660, }, { 1139, 1692, }, + { 1184, 1725, }, { 1232, 1759, }, { 1282, 1793, }, { 1336, 1828, }, + }, { + { 4, 4, }, { 9, 9, }, { 10, 11, }, { 13, 13, }, + { 15, 16, }, { 17, 18, }, { 20, 21, }, { 22, 24, }, + { 25, 27, }, { 28, 30, }, { 31, 33, }, { 34, 37, }, + { 37, 40, }, { 40, 44, }, { 43, 48, }, { 47, 51, }, + { 50, 55, }, { 53, 59, }, { 57, 63, }, { 60, 67, }, + { 64, 71, }, { 68, 75, }, { 71, 79, }, { 75, 83, }, + { 78, 88, }, { 82, 92, }, { 86, 96, }, { 90, 100, }, + { 93, 105, }, { 97, 109, }, { 101, 114, }, { 105, 118, }, + { 109, 122, }, { 113, 127, }, { 116, 131, }, { 120, 136, }, + { 124, 140, }, { 128, 145, }, { 132, 149, }, { 136, 154, }, + { 140, 158, }, { 143, 163, }, { 147, 168, }, { 151, 172, }, + { 155, 177, }, { 159, 181, }, { 163, 186, }, { 166, 190, }, + { 170, 195, }, { 174, 199, }, { 178, 204, }, { 182, 208, }, + { 185, 213, }, { 189, 217, }, { 193, 222, }, { 197, 226, }, + { 200, 231, }, { 204, 235, }, { 208, 240, }, { 212, 244, }, + { 215, 249, }, { 219, 253, }, { 223, 258, }, { 226, 262, }, + { 230, 267, }, { 233, 271, }, { 237, 275, }, { 241, 280, }, + { 244, 284, }, { 248, 289, }, { 251, 293, }, { 255, 297, }, + { 259, 302, }, { 262, 306, }, { 266, 311, }, { 269, 315, }, + { 273, 319, }, { 276, 324, }, { 280, 328, }, { 283, 332, }, + { 287, 337, }, { 290, 341, }, { 293, 345, }, { 297, 349, }, + { 300, 354, }, { 304, 358, }, { 307, 362, }, { 310, 367, }, + { 314, 371, }, { 317, 375, }, { 321, 379, }, { 324, 384, }, + { 327, 388, }, { 331, 392, }, { 334, 396, }, { 337, 401, }, + { 343, 409, }, { 350, 417, }, { 356, 425, }, { 362, 433, }, + { 369, 441, }, { 375, 449, }, { 381, 458, }, { 387, 466, }, + { 394, 474, }, { 400, 482, }, { 406, 490, }, { 412, 498, }, + { 418, 506, }, { 424, 514, }, { 430, 523, }, { 436, 531, }, + { 442, 539, }, { 448, 547, }, { 454, 555, }, { 460, 563, }, + { 466, 571, }, { 472, 579, }, { 478, 588, }, { 484, 596, }, + { 490, 604, }, { 499, 616, }, { 507, 628, }, { 516, 640, }, + { 525, 652, }, { 533, 664, }, { 542, 676, }, { 550, 688, }, + { 559, 700, }, { 567, 713, }, { 576, 725, }, { 584, 737, }, + { 592, 749, }, { 601, 761, }, { 609, 773, }, { 617, 785, }, + { 625, 797, }, { 634, 809, }, { 644, 825, }, { 655, 841, }, + { 666, 857, }, { 676, 873, }, { 687, 889, }, { 698, 905, }, + { 708, 922, }, { 718, 938, }, { 729, 954, }, { 739, 970, }, + { 749, 986, }, { 759, 1002, }, { 770, 1018, }, { 782, 1038, }, + { 795, 1058, }, { 807, 1078, }, { 819, 1098, }, { 831, 1118, }, + { 844, 1138, }, { 856, 1158, }, { 868, 1178, }, { 880, 1198, }, + { 891, 1218, }, { 906, 1242, }, { 920, 1266, }, { 933, 1290, }, + { 947, 1314, }, { 961, 1338, }, { 975, 1362, }, { 988, 1386, }, + { 1001, 1411, }, { 1015, 1435, }, { 1030, 1463, }, { 1045, 1491, }, + { 1061, 1519, }, { 1076, 1547, }, { 1090, 1575, }, { 1105, 1603, }, + { 1120, 1631, }, { 1137, 1663, }, { 1153, 1695, }, { 1170, 1727, }, + { 1186, 1759, }, { 1202, 1791, }, { 1218, 1823, }, { 1236, 1859, }, + { 1253, 1895, }, { 1271, 1931, }, { 1288, 1967, }, { 1306, 2003, }, + { 1323, 2039, }, { 1342, 2079, }, { 1361, 2119, }, { 1379, 2159, }, + { 1398, 2199, }, { 1416, 2239, }, { 1436, 2283, }, { 1456, 2327, }, + { 1476, 2371, }, { 1496, 2415, }, { 1516, 2459, }, { 1537, 2507, }, + { 1559, 2555, }, { 1580, 2603, }, { 1601, 2651, }, { 1624, 2703, }, + { 1647, 2755, }, { 1670, 2807, }, { 1692, 2859, }, { 1717, 2915, }, + { 1741, 2971, }, { 1766, 3027, }, { 1791, 3083, }, { 1817, 3143, }, + { 1844, 3203, }, { 1871, 3263, }, { 1900, 3327, }, { 1929, 3391, }, + { 1958, 3455, }, { 1990, 3523, }, { 2021, 3591, }, { 2054, 3659, }, + { 2088, 3731, }, { 2123, 3803, }, { 2159, 3876, }, { 2197, 3952, }, + { 2236, 4028, }, { 2276, 4104, }, { 2319, 4184, }, { 2363, 4264, }, + { 2410, 4348, }, { 2458, 4432, }, { 2508, 4516, }, { 2561, 4604, }, + { 2616, 4692, }, { 2675, 4784, }, { 2737, 4876, }, { 2802, 4972, }, + { 2871, 5068, }, { 2944, 5168, }, { 3020, 5268, }, { 3102, 5372, }, + { 3188, 5476, }, { 3280, 5584, }, { 3375, 5692, }, { 3478, 5804, }, + { 3586, 5916, }, { 3702, 6032, }, { 3823, 6148, }, { 3953, 6268, }, + { 4089, 6388, }, { 4236, 6512, }, { 4394, 6640, }, { 4559, 6768, }, + { 4737, 6900, }, { 4929, 7036, }, { 5130, 7172, }, { 5347, 7312, }, + }, { + { 4, 4 }, { 12, 13 }, { 18, 19 }, { 25, 27 }, + { 33, 35 }, { 41, 44 }, { 50, 54 }, { 60, 64 }, + { 70, 75 }, { 80, 87 }, { 91, 99 }, { 103, 112 }, + { 115, 126 }, { 127, 139 }, { 140, 154 }, { 153, 168 }, + { 166, 183 }, { 180, 199 }, { 194, 214 }, { 208, 230 }, + { 222, 247 }, { 237, 263 }, { 251, 280 }, { 266, 297 }, + { 281, 314 }, { 296, 331 }, { 312, 349 }, { 327, 366 }, + { 343, 384 }, { 358, 402 }, { 374, 420 }, { 390, 438 }, + { 405, 456 }, { 421, 475 }, { 437, 493 }, { 453, 511 }, + { 469, 530 }, { 484, 548 }, { 500, 567 }, { 516, 586 }, + { 532, 604 }, { 548, 623 }, { 564, 642 }, { 580, 660 }, + { 596, 679 }, { 611, 698 }, { 627, 716 }, { 643, 735 }, + { 659, 753 }, { 674, 772 }, { 690, 791 }, { 706, 809 }, + { 721, 828 }, { 737, 846 }, { 752, 865 }, { 768, 884 }, + { 783, 902 }, { 798, 920 }, { 814, 939 }, { 829, 957 }, + { 844, 976 }, { 859, 994 }, { 874, 1012 }, { 889, 1030 }, + { 904, 1049 }, { 919, 1067 }, { 934, 1085 }, { 949, 1103 }, + { 964, 1121 }, { 978, 1139 }, { 993, 1157 }, { 1008, 1175 }, + { 1022, 1193 }, { 1037, 1211 }, { 1051, 1229 }, { 1065, 1246 }, + { 1080, 1264 }, { 1094, 1282 }, { 1108, 1299 }, { 1122, 1317 }, + { 1136, 1335 }, { 1151, 1352 }, { 1165, 1370 }, { 1179, 1387 }, + { 1192, 1405 }, { 1206, 1422 }, { 1220, 1440 }, { 1234, 1457 }, + { 1248, 1474 }, { 1261, 1491 }, { 1275, 1509 }, { 1288, 1526 }, + { 1302, 1543 }, { 1315, 1560 }, { 1329, 1577 }, { 1342, 1595 }, + { 1368, 1627 }, { 1393, 1660 }, { 1419, 1693 }, { 1444, 1725 }, + { 1469, 1758 }, { 1494, 1791 }, { 1519, 1824 }, { 1544, 1856 }, + { 1569, 1889 }, { 1594, 1922 }, { 1618, 1954 }, { 1643, 1987 }, + { 1668, 2020 }, { 1692, 2052 }, { 1717, 2085 }, { 1741, 2118 }, + { 1765, 2150 }, { 1789, 2183 }, { 1814, 2216 }, { 1838, 2248 }, + { 1862, 2281 }, { 1885, 2313 }, { 1909, 2346 }, { 1933, 2378 }, + { 1957, 2411 }, { 1992, 2459 }, { 2027, 2508 }, { 2061, 2556 }, + { 2096, 2605 }, { 2130, 2653 }, { 2165, 2701 }, { 2199, 2750 }, + { 2233, 2798 }, { 2267, 2847 }, { 2300, 2895 }, { 2334, 2943 }, + { 2367, 2992 }, { 2400, 3040 }, { 2434, 3088 }, { 2467, 3137 }, + { 2499, 3185 }, { 2532, 3234 }, { 2575, 3298 }, { 2618, 3362 }, + { 2661, 3426 }, { 2704, 3491 }, { 2746, 3555 }, { 2788, 3619 }, + { 2830, 3684 }, { 2872, 3748 }, { 2913, 3812 }, { 2954, 3876 }, + { 2995, 3941 }, { 3036, 4005 }, { 3076, 4069 }, { 3127, 4149 }, + { 3177, 4230 }, { 3226, 4310 }, { 3275, 4390 }, { 3324, 4470 }, + { 3373, 4550 }, { 3421, 4631 }, { 3469, 4711 }, { 3517, 4791 }, + { 3565, 4871 }, { 3621, 4967 }, { 3677, 5064 }, { 3733, 5160 }, + { 3788, 5256 }, { 3843, 5352 }, { 3897, 5448 }, { 3951, 5544 }, + { 4005, 5641 }, { 4058, 5737 }, { 4119, 5849 }, { 4181, 5961 }, + { 4241, 6073 }, { 4301, 6185 }, { 4361, 6297 }, { 4420, 6410 }, + { 4479, 6522 }, { 4546, 6650 }, { 4612, 6778 }, { 4677, 6906 }, + { 4742, 7034 }, { 4807, 7162 }, { 4871, 7290 }, { 4942, 7435 }, + { 5013, 7579 }, { 5083, 7723 }, { 5153, 7867 }, { 5222, 8011 }, + { 5291, 8155 }, { 5367, 8315 }, { 5442, 8475 }, { 5517, 8635 }, + { 5591, 8795 }, { 5665, 8956 }, { 5745, 9132 }, { 5825, 9308 }, + { 5905, 9484 }, { 5984, 9660 }, { 6063, 9836 }, { 6149, 10028 }, + { 6234, 10220 }, { 6319, 10412 }, { 6404, 10604 }, { 6495, 10812 }, + { 6587, 11020 }, { 6678, 11228 }, { 6769, 11437 }, { 6867, 11661 }, + { 6966, 11885 }, { 7064, 12109 }, { 7163, 12333 }, { 7269, 12573 }, + { 7376, 12813 }, { 7483, 13053 }, { 7599, 13309 }, { 7715, 13565 }, + { 7832, 13821 }, { 7958, 14093 }, { 8085, 14365 }, { 8214, 14637 }, + { 8352, 14925 }, { 8492, 15213 }, { 8635, 15502 }, { 8788, 15806 }, + { 8945, 16110 }, { 9104, 16414 }, { 9275, 16734 }, { 9450, 17054 }, + { 9639, 17390 }, { 9832, 17726 }, { 10031, 18062 }, { 10245, 18414 }, + { 10465, 18766 }, { 10702, 19134 }, { 10946, 19502 }, { 11210, 19886 }, + { 11482, 20270 }, { 11776, 20670 }, { 12081, 21070 }, { 12409, 21486 }, + { 12750, 21902 }, { 13118, 22334 }, { 13501, 22766 }, { 13913, 23214 }, + { 14343, 23662 }, { 14807, 24126 }, { 15290, 24590 }, { 15812, 25070 }, + { 16356, 25551 }, { 16943, 26047 }, { 17575, 26559 }, { 18237, 27071 }, + { 18949, 27599 }, { 19718, 28143 }, { 20521, 28687 }, { 21387, 29247 }, + } +}; diff --git a/third_party/dav1d/src/dequant_tables.h b/third_party/dav1d/src/dequant_tables.h new file mode 100644 index 0000000000..17763377bc --- /dev/null +++ b/third_party/dav1d/src/dequant_tables.h @@ -0,0 +1,37 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_DEQUANT_TABLES_H +#define DAV1D_SRC_DEQUANT_TABLES_H + +#include + +#include "src/levels.h" + +EXTERN const uint16_t dav1d_dq_tbl[3][QINDEX_RANGE][2]; + +#endif /* DAV1D_SRC_DEQUANT_TABLES_H */ diff --git a/third_party/dav1d/src/env.h b/third_party/dav1d/src/env.h new file mode 100644 index 0000000000..7b91c4cab6 --- /dev/null +++ b/third_party/dav1d/src/env.h @@ -0,0 +1,521 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_ENV_H +#define DAV1D_SRC_ENV_H + +#include +#include +#include + +#include "src/levels.h" +#include "src/refmvs.h" +#include "src/tables.h" + +typedef struct BlockContext { + uint8_t ALIGN(mode[32], 8); + uint8_t ALIGN(lcoef[32], 8); + uint8_t ALIGN(ccoef[2][32], 8); + uint8_t ALIGN(seg_pred[32], 8); + uint8_t ALIGN(skip[32], 8); + uint8_t ALIGN(skip_mode[32], 8); + uint8_t ALIGN(intra[32], 8); + uint8_t ALIGN(comp_type[32], 8); + int8_t ALIGN(ref[2][32], 8); // -1 means intra + uint8_t ALIGN(filter[2][32], 8); // 3 means unset + int8_t ALIGN(tx_intra[32], 8); + int8_t ALIGN(tx[32], 8); + uint8_t ALIGN(tx_lpf_y[32], 8); + uint8_t ALIGN(tx_lpf_uv[32], 8); + uint8_t ALIGN(partition[16], 8); + uint8_t ALIGN(uvmode[32], 8); + uint8_t ALIGN(pal_sz[32], 8); +} BlockContext; + +static inline int get_intra_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + if (have_left) { + if (have_top) { + const int ctx = l->intra[yb4] + a->intra[xb4]; + return ctx + (ctx == 2); + } else + return l->intra[yb4] * 2; + } else { + return have_top ? a->intra[xb4] * 2 : 0; + } +} + +static inline int get_tx_ctx(const BlockContext *const a, + const BlockContext *const l, + const TxfmInfo *const max_tx, + const int yb4, const int xb4) +{ + return (l->tx_intra[yb4] >= max_tx->lh) + (a->tx_intra[xb4] >= max_tx->lw); +} + +static inline int get_partition_ctx(const BlockContext *const a, + const BlockContext *const l, + const enum BlockLevel bl, + const int yb8, const int xb8) +{ + return ((a->partition[xb8] >> (4 - bl)) & 1) + + (((l->partition[yb8] >> (4 - bl)) & 1) << 1); +} + +static inline unsigned gather_left_partition_prob(const uint16_t *const in, + const enum BlockLevel bl) +{ + unsigned out = in[PARTITION_H - 1] - in[PARTITION_H]; + // Exploit the fact that cdfs for PARTITION_SPLIT, PARTITION_T_TOP_SPLIT, + // PARTITION_T_BOTTOM_SPLIT and PARTITION_T_LEFT_SPLIT are neighbors. + out += in[PARTITION_SPLIT - 1] - in[PARTITION_T_LEFT_SPLIT]; + if (bl != BL_128X128) + out += in[PARTITION_H4 - 1] - in[PARTITION_H4]; + return out; +} + +static inline unsigned gather_top_partition_prob(const uint16_t *const in, + const enum BlockLevel bl) +{ + // Exploit the fact that cdfs for PARTITION_V, PARTITION_SPLIT and + // PARTITION_T_TOP_SPLIT are neighbors. + unsigned out = in[PARTITION_V - 1] - in[PARTITION_T_TOP_SPLIT]; + // Exploit the facts that cdfs for PARTITION_T_LEFT_SPLIT and + // PARTITION_T_RIGHT_SPLIT are neighbors, the probability for + // PARTITION_V4 is always zero, and the probability for + // PARTITION_T_RIGHT_SPLIT is zero in 128x128 blocks. + out += in[PARTITION_T_LEFT_SPLIT - 1]; + if (bl != BL_128X128) + out += in[PARTITION_V4 - 1] - in[PARTITION_T_RIGHT_SPLIT]; + return out; +} + +static inline enum TxfmType get_uv_inter_txtp(const TxfmInfo *const uvt_dim, + const enum TxfmType ytxtp) +{ + if (uvt_dim->max == TX_32X32) + return ytxtp == IDTX ? IDTX : DCT_DCT; + if (uvt_dim->min == TX_16X16 && + ((1 << ytxtp) & ((1 << H_FLIPADST) | (1 << V_FLIPADST) | + (1 << H_ADST) | (1 << V_ADST)))) + { + return DCT_DCT; + } + + return ytxtp; +} + +static inline int get_filter_ctx(const BlockContext *const a, + const BlockContext *const l, + const int comp, const int dir, const int ref, + const int yb4, const int xb4) +{ + const int a_filter = (a->ref[0][xb4] == ref || a->ref[1][xb4] == ref) ? + a->filter[dir][xb4] : DAV1D_N_SWITCHABLE_FILTERS; + const int l_filter = (l->ref[0][yb4] == ref || l->ref[1][yb4] == ref) ? + l->filter[dir][yb4] : DAV1D_N_SWITCHABLE_FILTERS; + + if (a_filter == l_filter) { + return comp * 4 + a_filter; + } else if (a_filter == DAV1D_N_SWITCHABLE_FILTERS) { + return comp * 4 + l_filter; + } else if (l_filter == DAV1D_N_SWITCHABLE_FILTERS) { + return comp * 4 + a_filter; + } else { + return comp * 4 + DAV1D_N_SWITCHABLE_FILTERS; + } +} + +static inline int get_comp_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + if (have_top) { + if (have_left) { + if (a->comp_type[xb4]) { + if (l->comp_type[yb4]) { + return 4; + } else { + // 4U means intra (-1) or bwd (>= 4) + return 2 + ((unsigned)l->ref[0][yb4] >= 4U); + } + } else if (l->comp_type[yb4]) { + // 4U means intra (-1) or bwd (>= 4) + return 2 + ((unsigned)a->ref[0][xb4] >= 4U); + } else { + return (l->ref[0][yb4] >= 4) ^ (a->ref[0][xb4] >= 4); + } + } else { + return a->comp_type[xb4] ? 3 : a->ref[0][xb4] >= 4; + } + } else if (have_left) { + return l->comp_type[yb4] ? 3 : l->ref[0][yb4] >= 4; + } else { + return 1; + } +} + +static inline int get_comp_dir_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ +#define has_uni_comp(edge, off) \ + ((edge->ref[0][off] < 4) == (edge->ref[1][off] < 4)) + + if (have_top && have_left) { + const int a_intra = a->intra[xb4], l_intra = l->intra[yb4]; + + if (a_intra && l_intra) return 2; + if (a_intra || l_intra) { + const BlockContext *const edge = a_intra ? l : a; + const int off = a_intra ? yb4 : xb4; + + if (edge->comp_type[off] == COMP_INTER_NONE) return 2; + return 1 + 2 * has_uni_comp(edge, off); + } + + const int a_comp = a->comp_type[xb4] != COMP_INTER_NONE; + const int l_comp = l->comp_type[yb4] != COMP_INTER_NONE; + const int a_ref0 = a->ref[0][xb4], l_ref0 = l->ref[0][yb4]; + + if (!a_comp && !l_comp) { + return 1 + 2 * ((a_ref0 >= 4) == (l_ref0 >= 4)); + } else if (!a_comp || !l_comp) { + const BlockContext *const edge = a_comp ? a : l; + const int off = a_comp ? xb4 : yb4; + + if (!has_uni_comp(edge, off)) return 1; + return 3 + ((a_ref0 >= 4) == (l_ref0 >= 4)); + } else { + const int a_uni = has_uni_comp(a, xb4), l_uni = has_uni_comp(l, yb4); + + if (!a_uni && !l_uni) return 0; + if (!a_uni || !l_uni) return 2; + return 3 + ((a_ref0 == 4) == (l_ref0 == 4)); + } + } else if (have_top || have_left) { + const BlockContext *const edge = have_left ? l : a; + const int off = have_left ? yb4 : xb4; + + if (edge->intra[off]) return 2; + if (edge->comp_type[off] == COMP_INTER_NONE) return 2; + return 4 * has_uni_comp(edge, off); + } else { + return 2; + } +} + +static inline int get_poc_diff(const int order_hint_n_bits, + const int poc0, const int poc1) +{ + if (!order_hint_n_bits) return 0; + const int mask = 1 << (order_hint_n_bits - 1); + const int diff = poc0 - poc1; + return (diff & (mask - 1)) - (diff & mask); +} + +static inline int get_jnt_comp_ctx(const int order_hint_n_bits, + const unsigned poc, const unsigned ref0poc, + const unsigned ref1poc, + const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4) +{ + const unsigned d0 = abs(get_poc_diff(order_hint_n_bits, ref0poc, poc)); + const unsigned d1 = abs(get_poc_diff(order_hint_n_bits, poc, ref1poc)); + const int offset = d0 == d1; + const int a_ctx = a->comp_type[xb4] >= COMP_INTER_AVG || + a->ref[0][xb4] == 6; + const int l_ctx = l->comp_type[yb4] >= COMP_INTER_AVG || + l->ref[0][yb4] == 6; + + return 3 * offset + a_ctx + l_ctx; +} + +static inline int get_mask_comp_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4) +{ + const int a_ctx = a->comp_type[xb4] >= COMP_INTER_SEG ? 1 : + a->ref[0][xb4] == 6 ? 3 : 0; + const int l_ctx = l->comp_type[yb4] >= COMP_INTER_SEG ? 1 : + l->ref[0][yb4] == 6 ? 3 : 0; + + return imin(a_ctx + l_ctx, 5); +} + +#define av1_get_ref_2_ctx av1_get_bwd_ref_ctx +#define av1_get_ref_3_ctx av1_get_fwd_ref_ctx +#define av1_get_ref_4_ctx av1_get_fwd_ref_1_ctx +#define av1_get_ref_5_ctx av1_get_fwd_ref_2_ctx +#define av1_get_ref_6_ctx av1_get_bwd_ref_1_ctx +#define av1_get_uni_p_ctx av1_get_ref_ctx +#define av1_get_uni_p2_ctx av1_get_fwd_ref_2_ctx + +static inline int av1_get_ref_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + int have_top, int have_left) +{ + int cnt[2] = { 0 }; + + if (have_top && !a->intra[xb4]) { + cnt[a->ref[0][xb4] >= 4]++; + if (a->comp_type[xb4]) cnt[a->ref[1][xb4] >= 4]++; + } + + if (have_left && !l->intra[yb4]) { + cnt[l->ref[0][yb4] >= 4]++; + if (l->comp_type[yb4]) cnt[l->ref[1][yb4] >= 4]++; + } + + return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; +} + +static inline int av1_get_fwd_ref_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[4] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if (a->ref[0][xb4] < 4) cnt[a->ref[0][xb4]]++; + if (a->comp_type[xb4] && a->ref[1][xb4] < 4) cnt[a->ref[1][xb4]]++; + } + + if (have_left && !l->intra[yb4]) { + if (l->ref[0][yb4] < 4) cnt[l->ref[0][yb4]]++; + if (l->comp_type[yb4] && l->ref[1][yb4] < 4) cnt[l->ref[1][yb4]]++; + } + + cnt[0] += cnt[1]; + cnt[2] += cnt[3]; + + return cnt[0] == cnt[2] ? 1 : cnt[0] < cnt[2] ? 0 : 2; +} + +static inline int av1_get_fwd_ref_1_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[2] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if (a->ref[0][xb4] < 2) cnt[a->ref[0][xb4]]++; + if (a->comp_type[xb4] && a->ref[1][xb4] < 2) cnt[a->ref[1][xb4]]++; + } + + if (have_left && !l->intra[yb4]) { + if (l->ref[0][yb4] < 2) cnt[l->ref[0][yb4]]++; + if (l->comp_type[yb4] && l->ref[1][yb4] < 2) cnt[l->ref[1][yb4]]++; + } + + return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; +} + +static inline int av1_get_fwd_ref_2_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[2] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if ((a->ref[0][xb4] ^ 2U) < 2) cnt[a->ref[0][xb4] - 2]++; + if (a->comp_type[xb4] && (a->ref[1][xb4] ^ 2U) < 2) cnt[a->ref[1][xb4] - 2]++; + } + + if (have_left && !l->intra[yb4]) { + if ((l->ref[0][yb4] ^ 2U) < 2) cnt[l->ref[0][yb4] - 2]++; + if (l->comp_type[yb4] && (l->ref[1][yb4] ^ 2U) < 2) cnt[l->ref[1][yb4] - 2]++; + } + + return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; +} + +static inline int av1_get_bwd_ref_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[3] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if (a->ref[0][xb4] >= 4) cnt[a->ref[0][xb4] - 4]++; + if (a->comp_type[xb4] && a->ref[1][xb4] >= 4) cnt[a->ref[1][xb4] - 4]++; + } + + if (have_left && !l->intra[yb4]) { + if (l->ref[0][yb4] >= 4) cnt[l->ref[0][yb4] - 4]++; + if (l->comp_type[yb4] && l->ref[1][yb4] >= 4) cnt[l->ref[1][yb4] - 4]++; + } + + cnt[1] += cnt[0]; + + return cnt[2] == cnt[1] ? 1 : cnt[1] < cnt[2] ? 0 : 2; +} + +static inline int av1_get_bwd_ref_1_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[3] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if (a->ref[0][xb4] >= 4) cnt[a->ref[0][xb4] - 4]++; + if (a->comp_type[xb4] && a->ref[1][xb4] >= 4) cnt[a->ref[1][xb4] - 4]++; + } + + if (have_left && !l->intra[yb4]) { + if (l->ref[0][yb4] >= 4) cnt[l->ref[0][yb4] - 4]++; + if (l->comp_type[yb4] && l->ref[1][yb4] >= 4) cnt[l->ref[1][yb4] - 4]++; + } + + return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; +} + +static inline int av1_get_uni_p1_ctx(const BlockContext *const a, + const BlockContext *const l, + const int yb4, const int xb4, + const int have_top, const int have_left) +{ + int cnt[3] = { 0 }; + + if (have_top && !a->intra[xb4]) { + if (a->ref[0][xb4] - 1U < 3) cnt[a->ref[0][xb4] - 1]++; + if (a->comp_type[xb4] && a->ref[1][xb4] - 1U < 3) cnt[a->ref[1][xb4] - 1]++; + } + + if (have_left && !l->intra[yb4]) { + if (l->ref[0][yb4] - 1U < 3) cnt[l->ref[0][yb4] - 1]++; + if (l->comp_type[yb4] && l->ref[1][yb4] - 1U < 3) cnt[l->ref[1][yb4] - 1]++; + } + + cnt[1] += cnt[2]; + + return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; +} + +static inline int get_drl_context(const refmvs_candidate *const ref_mv_stack, + const int ref_idx) +{ + if (ref_mv_stack[ref_idx].weight >= 640) + return ref_mv_stack[ref_idx + 1].weight < 640; + + return ref_mv_stack[ref_idx + 1].weight < 640 ? 2 : 0; +} + +static inline unsigned get_cur_frame_segid(const int by, const int bx, + const int have_top, + const int have_left, + int *const seg_ctx, + const uint8_t *cur_seg_map, + const ptrdiff_t stride) +{ + cur_seg_map += bx + by * stride; + if (have_left && have_top) { + const int l = cur_seg_map[-1]; + const int a = cur_seg_map[-stride]; + const int al = cur_seg_map[-(stride + 1)]; + + if (l == a && al == l) *seg_ctx = 2; + else if (l == a || al == l || a == al) *seg_ctx = 1; + else *seg_ctx = 0; + return a == al ? a : l; + } else { + *seg_ctx = 0; + return have_left ? cur_seg_map[-1] : have_top ? cur_seg_map[-stride] : 0; + } +} + +static inline void fix_int_mv_precision(mv *const mv) { + mv->x = (mv->x - (mv->x >> 15) + 3) & ~7U; + mv->y = (mv->y - (mv->y >> 15) + 3) & ~7U; +} + +static inline void fix_mv_precision(const Dav1dFrameHeader *const hdr, + mv *const mv) +{ + if (hdr->force_integer_mv) { + fix_int_mv_precision(mv); + } else if (!hdr->hp) { + mv->x = (mv->x - (mv->x >> 15)) & ~1U; + mv->y = (mv->y - (mv->y >> 15)) & ~1U; + } +} + +static inline mv get_gmv_2d(const Dav1dWarpedMotionParams *const gmv, + const int bx4, const int by4, + const int bw4, const int bh4, + const Dav1dFrameHeader *const hdr) +{ + switch (gmv->type) { + case DAV1D_WM_TYPE_ROT_ZOOM: + assert(gmv->matrix[5] == gmv->matrix[2]); + assert(gmv->matrix[4] == -gmv->matrix[3]); + // fall-through + default: + case DAV1D_WM_TYPE_AFFINE: { + const int x = bx4 * 4 + bw4 * 2 - 1; + const int y = by4 * 4 + bh4 * 2 - 1; + const int xc = (gmv->matrix[2] - (1 << 16)) * x + + gmv->matrix[3] * y + gmv->matrix[0]; + const int yc = (gmv->matrix[5] - (1 << 16)) * y + + gmv->matrix[4] * x + gmv->matrix[1]; + const int shift = 16 - (3 - !hdr->hp); + const int round = (1 << shift) >> 1; + mv res = (mv) { + .y = apply_sign(((abs(yc) + round) >> shift) << !hdr->hp, yc), + .x = apply_sign(((abs(xc) + round) >> shift) << !hdr->hp, xc), + }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; + } + case DAV1D_WM_TYPE_TRANSLATION: { + mv res = (mv) { + .y = gmv->matrix[0] >> 13, + .x = gmv->matrix[1] >> 13, + }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; + } + case DAV1D_WM_TYPE_IDENTITY: + return (mv) { .x = 0, .y = 0 }; + } +} + +#endif /* DAV1D_SRC_ENV_H */ diff --git a/third_party/dav1d/src/ext/x86/x86inc.asm b/third_party/dav1d/src/ext/x86/x86inc.asm new file mode 100644 index 0000000000..68b1f74f4b --- /dev/null +++ b/third_party/dav1d/src/ext/x86/x86inc.asm @@ -0,0 +1,1902 @@ +;***************************************************************************** +;* x86inc.asm: x86 abstraction layer +;***************************************************************************** +;* Copyright (C) 2005-2022 x264 project +;* +;* Authors: Loren Merritt +;* Henrik Gramner +;* Anton Mitrofanov +;* Fiona Glaser +;* +;* Permission to use, copy, modify, and/or distribute this software for any +;* purpose with or without fee is hereby granted, provided that the above +;* copyright notice and this permission notice appear in all copies. +;* +;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +;***************************************************************************** + +; This is a header file for the x86inc.asm assembly language, which uses +; NASM/YASM syntax combined with a large number of macros to provide easy +; abstraction between different calling conventions (x86_32, win64, linux64). +; It also has various other useful features to simplify writing the kind of +; DSP functions that are most often used. + +%ifndef private_prefix + %error private_prefix not defined +%endif + +%ifndef public_prefix + %define public_prefix private_prefix +%endif + +%ifndef STACK_ALIGNMENT + %if ARCH_X86_64 + %define STACK_ALIGNMENT 16 + %else + %define STACK_ALIGNMENT 4 + %endif +%endif + +%define WIN64 0 +%define UNIX64 0 +%if ARCH_X86_64 + %ifidn __OUTPUT_FORMAT__,win32 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,win64 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,x64 + %define WIN64 1 + %else + %define UNIX64 1 + %endif +%endif + +%define FORMAT_ELF 0 +%define FORMAT_MACHO 0 +%ifidn __OUTPUT_FORMAT__,elf + %define FORMAT_ELF 1 +%elifidn __OUTPUT_FORMAT__,elf32 + %define FORMAT_ELF 1 +%elifidn __OUTPUT_FORMAT__,elf64 + %define FORMAT_ELF 1 +%elifidn __OUTPUT_FORMAT__,macho + %define FORMAT_MACHO 1 +%elifidn __OUTPUT_FORMAT__,macho32 + %define FORMAT_MACHO 1 +%elifidn __OUTPUT_FORMAT__,macho64 + %define FORMAT_MACHO 1 +%endif + +%ifdef PREFIX + %define mangle(x) _ %+ x +%else + %define mangle(x) x +%endif + +; Use VEX-encoding even in non-AVX functions +%ifndef FORCE_VEX_ENCODING + %define FORCE_VEX_ENCODING 0 +%endif + +%macro SECTION_RODATA 0-1 16 + %ifidn __OUTPUT_FORMAT__,win32 + SECTION .rdata align=%1 + %elif WIN64 + SECTION .rdata align=%1 + %else + SECTION .rodata align=%1 + %endif +%endmacro + +%if ARCH_X86_64 + %define PIC 1 ; always use PIC on x86-64 + default rel +%elifidn __OUTPUT_FORMAT__,win32 + %define PIC 0 ; PIC isn't used on 32-bit Windows +%elifndef PIC + %define PIC 0 +%endif + +%define HAVE_PRIVATE_EXTERN 1 +%ifdef __NASM_VER__ + %use smartalign + %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14 + %define HAVE_PRIVATE_EXTERN 0 + %endif +%endif + +; Macros to eliminate most code duplication between x86_32 and x86_64: +; Currently this works only for leaf functions which load all their arguments +; into registers at the start, and make no other use of the stack. Luckily that +; covers most use cases. + +; PROLOGUE: +; %1 = number of arguments. loads them from stack if needed. +; %2 = number of registers used. pushes callee-saved regs if needed. +; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. +; %4 = (optional) stack size to be allocated. The stack will be aligned before +; allocating the specified stack size. If the required stack alignment is +; larger than the known stack alignment the stack will be manually aligned +; and an extra register will be allocated to hold the original stack +; pointer (to not invalidate r0m etc.). To prevent the use of an extra +; register as stack pointer, request a negative stack size. +; %4+/%5+ = list of names to define to registers +; PROLOGUE can also be invoked by adding the same options to cglobal + +; e.g. +; cglobal foo, 2,3,7,0x40, dst, src, tmp +; declares a function (foo) that automatically loads two arguments (dst and +; src) into registers, uses one additional register (tmp) plus 7 vector +; registers (m0-m6) and allocates 0x40 bytes of stack space. + +; TODO Some functions can use some args directly from the stack. If they're the +; last args then you can just not declare them, but if they're in the middle +; we need more flexible macro. + +; RET: +; Pops anything that was pushed by PROLOGUE, and returns. + +; REP_RET: +; Use this instead of RET if it's a branch target. + +; registers: +; rN and rNq are the native-size register holding function argument N +; rNd, rNw, rNb are dword, word, and byte size +; rNh is the high 8 bits of the word size +; rNm is the original location of arg N (a register or on the stack), dword +; rNmp is native size + +%macro DECLARE_REG 2-3 + %define r%1q %2 + %define r%1d %2d + %define r%1w %2w + %define r%1b %2b + %define r%1h %2h + %define %2q %2 + %if %0 == 2 + %define r%1m %2d + %define r%1mp %2 + %elif ARCH_X86_64 ; memory + %define r%1m [rstk + stack_offset + %3] + %define r%1mp qword r %+ %1 %+ m + %else + %define r%1m [rstk + stack_offset + %3] + %define r%1mp dword r %+ %1 %+ m + %endif + %define r%1 %2 +%endmacro + +%macro DECLARE_REG_SIZE 3 + %define r%1q r%1 + %define e%1q r%1 + %define r%1d e%1 + %define e%1d e%1 + %define r%1w %1 + %define e%1w %1 + %define r%1h %3 + %define e%1h %3 + %define r%1b %2 + %define e%1b %2 + %if ARCH_X86_64 == 0 + %define r%1 e%1 + %endif +%endmacro + +DECLARE_REG_SIZE ax, al, ah +DECLARE_REG_SIZE bx, bl, bh +DECLARE_REG_SIZE cx, cl, ch +DECLARE_REG_SIZE dx, dl, dh +DECLARE_REG_SIZE si, sil, null +DECLARE_REG_SIZE di, dil, null +DECLARE_REG_SIZE bp, bpl, null + +; t# defines for when per-arch register allocation is more complex than just function arguments + +%macro DECLARE_REG_TMP 1-* + %assign %%i 0 + %rep %0 + CAT_XDEFINE t, %%i, r%1 + %assign %%i %%i+1 + %rotate 1 + %endrep +%endmacro + +%macro DECLARE_REG_TMP_SIZE 0-* + %rep %0 + %define t%1q t%1 %+ q + %define t%1d t%1 %+ d + %define t%1w t%1 %+ w + %define t%1h t%1 %+ h + %define t%1b t%1 %+ b + %rotate 1 + %endrep +%endmacro + +DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 + +%if ARCH_X86_64 + %define gprsize 8 +%else + %define gprsize 4 +%endif + +%macro LEA 2 +%if ARCH_X86_64 + lea %1, [%2] +%elif PIC + call $+5 ; special-cased to not affect the RSB on most CPU:s + pop %1 + add %1, (%2)-$+1 +%else + mov %1, %2 +%endif +%endmacro + +; Repeats an instruction/operation for multiple arguments. +; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3" +%macro REPX 2-* ; operation, args + %xdefine %%f(x) %1 + %rep %0 - 1 + %rotate 1 + %%f(%1) + %endrep +%endmacro + +%macro PUSH 1 + push %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset+gprsize + %endif +%endmacro + +%macro POP 1 + pop %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset-gprsize + %endif +%endmacro + +%macro PUSH_IF_USED 1-* + %rep %0 + %if %1 < regs_used + PUSH r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro POP_IF_USED 1-* + %rep %0 + %if %1 < regs_used + pop r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro LOAD_IF_USED 1-* + %rep %0 + %if %1 < num_args + mov r%1, r %+ %1 %+ mp + %endif + %rotate 1 + %endrep +%endmacro + +%macro SUB 2 + sub %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset+(%2) + %endif +%endmacro + +%macro ADD 2 + add %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset-(%2) + %endif +%endmacro + +%macro movifnidn 2 + %ifnidn %1, %2 + mov %1, %2 + %endif +%endmacro + +%if ARCH_X86_64 == 0 + %define movsxd movifnidn +%endif + +%macro movsxdifnidn 2 + %ifnidn %1, %2 + movsxd %1, %2 + %endif +%endmacro + +%macro ASSERT 1 + %if (%1) == 0 + %error assertion ``%1'' failed + %endif +%endmacro + +%macro DEFINE_ARGS 0-* + %ifdef n_arg_names + %assign %%i 0 + %rep n_arg_names + CAT_UNDEF arg_name %+ %%i, q + CAT_UNDEF arg_name %+ %%i, d + CAT_UNDEF arg_name %+ %%i, w + CAT_UNDEF arg_name %+ %%i, h + CAT_UNDEF arg_name %+ %%i, b + CAT_UNDEF arg_name %+ %%i, m + CAT_UNDEF arg_name %+ %%i, mp + CAT_UNDEF arg_name, %%i + %assign %%i %%i+1 + %endrep + %endif + + %xdefine %%stack_offset stack_offset + %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine + %assign %%i 0 + %rep %0 + %xdefine %1q r %+ %%i %+ q + %xdefine %1d r %+ %%i %+ d + %xdefine %1w r %+ %%i %+ w + %xdefine %1h r %+ %%i %+ h + %xdefine %1b r %+ %%i %+ b + %xdefine %1m r %+ %%i %+ m + %xdefine %1mp r %+ %%i %+ mp + CAT_XDEFINE arg_name, %%i, %1 + %assign %%i %%i+1 + %rotate 1 + %endrep + %xdefine stack_offset %%stack_offset + %assign n_arg_names %0 +%endmacro + +%define required_stack_alignment ((mmsize + 15) & ~15) +%define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512))) +%define high_mm_regs (16*cpuflag(avx512)) + +; Large stack allocations on Windows need to use stack probing in order +; to guarantee that all stack memory is committed before accessing it. +; This is done by ensuring that the guard page(s) at the end of the +; currently committed pages are touched prior to any pages beyond that. +%if WIN64 + %assign STACK_PROBE_SIZE 8192 +%elifidn __OUTPUT_FORMAT__, win32 + %assign STACK_PROBE_SIZE 4096 +%else + %assign STACK_PROBE_SIZE 0 +%endif + +%macro PROBE_STACK 1 ; stack_size + %if STACK_PROBE_SIZE + %assign %%i STACK_PROBE_SIZE + %rep %1 / STACK_PROBE_SIZE + mov eax, [rsp-%%i] + %assign %%i %%i+STACK_PROBE_SIZE + %endrep + %endif +%endmacro + +%macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs (for win64 only) + %ifnum %1 + %if %1 != 0 + %assign %%pad 0 + %assign stack_size %1 + %if stack_size < 0 + %assign stack_size -stack_size + %endif + %if WIN64 + %assign %%pad %%pad + 32 ; shadow space + %if mmsize != 8 + %assign xmm_regs_used %2 + %if xmm_regs_used > 8 + %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers + %endif + %endif + %endif + %if required_stack_alignment <= STACK_ALIGNMENT + ; maintain the current stack alignment + %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) + PROBE_STACK stack_size_padded + SUB rsp, stack_size_padded + %else + %assign %%reg_num (regs_used - 1) + %xdefine rstk r %+ %%reg_num + ; align stack, and save original stack location directly above + ; it, i.e. in [rsp+stack_size_padded], so we can restore the + ; stack in a single instruction (i.e. mov rsp, rstk or mov + ; rsp, [rsp+stack_size_padded]) + %if %1 < 0 ; need to store rsp on stack + %xdefine rstkm [rsp + stack_size + %%pad] + %assign %%pad %%pad + gprsize + %else ; can keep rsp in rstk during whole function + %xdefine rstkm rstk + %endif + %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1)) + PROBE_STACK stack_size_padded + mov rstk, rsp + and rsp, ~(required_stack_alignment-1) + sub rsp, stack_size_padded + movifnidn rstkm, rstk + %endif + WIN64_PUSH_XMM + %endif + %endif +%endmacro + +%macro SETUP_STACK_POINTER 0-1 0 + %ifnum %1 + %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT + %if %1 > 0 + ; Reserve an additional register for storing the original stack pointer, but avoid using + ; eax/rax for this purpose since it can potentially get overwritten as a return value. + %assign regs_used (regs_used + 1) + %if ARCH_X86_64 && regs_used == 7 + %assign regs_used 8 + %elif ARCH_X86_64 == 0 && regs_used == 1 + %assign regs_used 2 + %endif + %endif + %if ARCH_X86_64 && regs_used < 5 + UNIX64 * 3 + ; Ensure that we don't clobber any registers containing arguments. For UNIX64 we also preserve r6 (rax) + ; since it's used as a hidden argument in vararg functions to specify the number of vector registers used. + %assign regs_used 5 + UNIX64 * 3 + %endif + %endif + %endif +%endmacro + +%if WIN64 ; Windows x64 ;================================================= + +DECLARE_REG 0, rcx +DECLARE_REG 1, rdx +DECLARE_REG 2, R8 +DECLARE_REG 3, R9 +DECLARE_REG 4, R10, 40 +DECLARE_REG 5, R11, 48 +DECLARE_REG 6, rax, 56 +DECLARE_REG 7, rdi, 64 +DECLARE_REG 8, rsi, 72 +DECLARE_REG 9, rbx, 80 +DECLARE_REG 10, rbp, 88 +DECLARE_REG 11, R14, 96 +DECLARE_REG 12, R15, 104 +DECLARE_REG 13, R12, 112 +DECLARE_REG 14, R13, 120 + +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4, %3 + %if mmsize != 8 && stack_size == 0 + WIN64_SPILL_XMM %3 + %endif + LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif +%endmacro + +%macro WIN64_PUSH_XMM 0 + ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. + %if xmm_regs_used > 6 + high_mm_regs + movaps [rstk + stack_offset + 8], xmm6 + %endif + %if xmm_regs_used > 7 + high_mm_regs + movaps [rstk + stack_offset + 24], xmm7 + %endif + %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 + %if %%xmm_regs_on_stack > 0 + %assign %%i 8 + %rep %%xmm_regs_on_stack + movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +%macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 + ASSERT xmm_regs_used <= 16 + high_mm_regs + %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 + %if %%xmm_regs_on_stack > 0 + ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. + %assign %%pad %%xmm_regs_on_stack*16 + 32 + %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) + SUB rsp, stack_size_padded + %endif + WIN64_PUSH_XMM +%endmacro + +%macro WIN64_RESTORE_XMM_INTERNAL 0 + %assign %%pad_size 0 + %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 + %if %%xmm_regs_on_stack > 0 + %assign %%i xmm_regs_used - high_mm_regs + %rep %%xmm_regs_on_stack + %assign %%i %%i-1 + movaps xmm %+ %%i, [rsp + (%%i-8)*16 + stack_size + 32] + %endrep + %endif + %if stack_size_padded > 0 + %if stack_size > 0 && required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add rsp, stack_size_padded + %assign %%pad_size stack_size_padded + %endif + %endif + %if xmm_regs_used > 7 + high_mm_regs + movaps xmm7, [rsp + stack_offset - %%pad_size + 24] + %endif + %if xmm_regs_used > 6 + high_mm_regs + movaps xmm6, [rsp + stack_offset - %%pad_size + 8] + %endif +%endmacro + +%macro WIN64_RESTORE_XMM 0 + WIN64_RESTORE_XMM_INTERNAL + %assign stack_offset (stack_offset-stack_size_padded) + %assign stack_size_padded 0 + %assign xmm_regs_used 0 +%endmacro + +%define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs + +%macro RET 0 + WIN64_RESTORE_XMM_INTERNAL + POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 + %if vzeroupper_required + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%elif ARCH_X86_64 ; *nix x64 ;============================================= + +DECLARE_REG 0, rdi +DECLARE_REG 1, rsi +DECLARE_REG 2, rdx +DECLARE_REG 3, rcx +DECLARE_REG 4, R8 +DECLARE_REG 5, R9 +DECLARE_REG 6, rax, 8 +DECLARE_REG 7, R10, 16 +DECLARE_REG 8, R11, 24 +DECLARE_REG 9, rbx, 32 +DECLARE_REG 10, rbp, 40 +DECLARE_REG 11, R14, 48 +DECLARE_REG 12, R15, 56 +DECLARE_REG 13, R12, 64 +DECLARE_REG 14, R13, 72 + +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + %assign xmm_regs_used %3 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4 + LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif +%endmacro + +%define has_epilogue regs_used > 9 || stack_size > 0 || vzeroupper_required + +%macro RET 0 + %if stack_size_padded > 0 + %if required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add rsp, stack_size_padded + %endif + %endif + POP_IF_USED 14, 13, 12, 11, 10, 9 + %if vzeroupper_required + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%else ; X86_32 ;============================================================== + +DECLARE_REG 0, eax, 4 +DECLARE_REG 1, ecx, 8 +DECLARE_REG 2, edx, 12 +DECLARE_REG 3, ebx, 16 +DECLARE_REG 4, esi, 20 +DECLARE_REG 5, edi, 24 +DECLARE_REG 6, ebp, 28 +%define rsp esp + +%macro DECLARE_ARG 1-* + %rep %0 + %define r%1m [rstk + stack_offset + 4*%1 + 4] + %define r%1mp dword r%1m + %rotate 1 + %endrep +%endmacro + +DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 + +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + %if num_args > 7 + %assign num_args 7 + %endif + %if regs_used > 7 + %assign regs_used 7 + %endif + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 7 + PUSH_IF_USED 3, 4, 5, 6 + ALLOC_STACK %4 + LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif +%endmacro + +%define has_epilogue regs_used > 3 || stack_size > 0 || vzeroupper_required + +%macro RET 0 + %if stack_size_padded > 0 + %if required_stack_alignment > STACK_ALIGNMENT + mov rsp, rstkm + %else + add rsp, stack_size_padded + %endif + %endif + POP_IF_USED 6, 5, 4, 3 + %if vzeroupper_required + vzeroupper + %endif + AUTO_REP_RET +%endmacro + +%endif ;====================================================================== + +%if WIN64 == 0 + %macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 + %endmacro + %macro WIN64_RESTORE_XMM 0 + %assign xmm_regs_used 0 + %endmacro + %macro WIN64_PUSH_XMM 0 + %endmacro +%endif + +; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either +; a branch or a branch target. So switch to a 2-byte form of ret in that case. +; We can automatically detect "follows a branch", but not a branch target. +; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.) +%macro REP_RET 0 + %if has_epilogue || cpuflag(ssse3) + RET + %else + rep ret + %endif + annotate_function_size +%endmacro + +%define last_branch_adr $$ +%macro AUTO_REP_RET 0 + %if notcpuflag(ssse3) + times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ == last_branch_adr. + %endif + ret + annotate_function_size +%endmacro + +%macro BRANCH_INSTR 0-* + %rep %0 + %macro %1 1-2 %1 + %2 %1 + %if notcpuflag(ssse3) + %%branch_instr equ $ + %xdefine last_branch_adr %%branch_instr + %endif + %endmacro + %rotate 1 + %endrep +%endmacro + +BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp + +%macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent + %if has_epilogue + call %1 + RET + %elif %2 + jmp %1 + %endif + annotate_function_size +%endmacro + +;============================================================================= +; arch-independent part +;============================================================================= + +%assign function_align 16 + +; Begin a function. +; Applies any symbol mangling needed for C linkage, and sets up a define such that +; subsequent uses of the function name automatically refer to the mangled version. +; Appends cpuflags to the function name if cpuflags has been specified. +; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX +; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2). +%macro cglobal 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 1, %1 %+ SUFFIX, %2 +%endmacro +%macro cvisible 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 0, %1 %+ SUFFIX, %2 +%endmacro +%macro cglobal_internal 2-3+ + annotate_function_size + %ifndef cglobaled_%2 + %if %1 + %xdefine %2 mangle(private_prefix %+ _ %+ %2) + %else + %xdefine %2 mangle(public_prefix %+ _ %+ %2) + %endif + %xdefine %2.skip_prologue %2 %+ .skip_prologue + CAT_XDEFINE cglobaled_, %2, 1 + %endif + %xdefine current_function %2 + %xdefine current_function_section __SECT__ + %if FORMAT_ELF + %if %1 + global %2:function hidden + %else + global %2:function + %endif + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1 + global %2:private_extern + %else + global %2 + %endif + align function_align + %2: + RESET_MM_PERMUTATION ; needed for x86-64, also makes disassembly somewhat nicer + %xdefine rstk rsp ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required + %assign stack_offset 0 ; stack pointer offset relative to the return address + %assign stack_size 0 ; amount of stack space that can be freely used inside a function + %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding + %assign xmm_regs_used 0 ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 and vzeroupper + %ifnidn %3, "" + PROLOGUE %3 + %endif +%endmacro + +; Create a global symbol from a local label with the correct name mangling and type +%macro cglobal_label 1 + %if FORMAT_ELF + global current_function %+ %1:function hidden + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN + global current_function %+ %1:private_extern + %else + global current_function %+ %1 + %endif + %1: +%endmacro + +%macro cextern 1 + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +; like cextern, but without the prefix +%macro cextern_naked 1 + %ifdef PREFIX + %xdefine %1 mangle(%1) + %endif + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +%macro const 1-2+ + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + %if FORMAT_ELF + global %1:data hidden + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN + global %1:private_extern + %else + global %1 + %endif + %1: %2 +%endmacro + +; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default. +%if FORMAT_ELF + [SECTION .note.GNU-stack noalloc noexec nowrite progbits] +%endif + +; Tell debuggers how large the function was. +; This may be invoked multiple times per function; we rely on later instances overriding earlier ones. +; This is invoked by RET and similar macros, and also cglobal does it for the previous function, +; but if the last function in a source file doesn't use any of the standard macros for its epilogue, +; then its size might be unspecified. +%macro annotate_function_size 0 + %ifdef __YASM_VER__ + %ifdef current_function + %if FORMAT_ELF + current_function_section + %%ecf equ $ + size current_function %%ecf - current_function + __SECT__ + %endif + %endif + %endif +%endmacro + +; cpuflags + +%assign cpuflags_mmx (1<<0) +%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx +%assign cpuflags_3dnow (1<<2) | cpuflags_mmx +%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow +%assign cpuflags_sse (1<<4) | cpuflags_mmx2 +%assign cpuflags_sse2 (1<<5) | cpuflags_sse +%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 +%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 +%assign cpuflags_sse3 (1<<8) | cpuflags_sse2 +%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 +%assign cpuflags_sse4 (1<<10) | cpuflags_ssse3 +%assign cpuflags_sse42 (1<<11) | cpuflags_sse4 +%assign cpuflags_aesni (1<<12) | cpuflags_sse42 +%assign cpuflags_gfni (1<<13) | cpuflags_sse42 +%assign cpuflags_avx (1<<14) | cpuflags_sse42 +%assign cpuflags_xop (1<<15) | cpuflags_avx +%assign cpuflags_fma4 (1<<16) | cpuflags_avx +%assign cpuflags_fma3 (1<<17) | cpuflags_avx +%assign cpuflags_bmi1 (1<<18) | cpuflags_avx|cpuflags_lzcnt +%assign cpuflags_bmi2 (1<<19) | cpuflags_bmi1 +%assign cpuflags_avx2 (1<<20) | cpuflags_fma3|cpuflags_bmi2 +%assign cpuflags_avx512 (1<<21) | cpuflags_avx2 ; F, CD, BW, DQ, VL +%assign cpuflags_avx512icl (1<<22) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ + +%assign cpuflags_cache32 (1<<23) +%assign cpuflags_cache64 (1<<24) +%assign cpuflags_aligned (1<<25) ; not a cpu feature, but a function variant +%assign cpuflags_atom (1<<26) + +; Returns a boolean value expressing whether or not the specified cpuflag is enabled. +%define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) +%define notcpuflag(x) (cpuflag(x) ^ 1) + +; Takes an arbitrary number of cpuflags from the above list. +; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. +; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. +%macro INIT_CPUFLAGS 0-* + %xdefine SUFFIX + %undef cpuname + %assign cpuflags 0 + + %if %0 >= 1 + %rep %0 + %ifdef cpuname + %xdefine cpuname cpuname %+ _%1 + %else + %xdefine cpuname %1 + %endif + %assign cpuflags cpuflags | cpuflags_%1 + %rotate 1 + %endrep + %xdefine SUFFIX _ %+ cpuname + + %if cpuflag(avx) + %assign avx_enabled 1 + %endif + %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2)) + %define mova movaps + %define movu movups + %define movnta movntps + %endif + %if cpuflag(aligned) + %define movu mova + %elif cpuflag(sse3) && notcpuflag(ssse3) + %define movu lddqu + %endif + %endif + + %if ARCH_X86_64 || cpuflag(sse2) + %ifdef __NASM_VER__ + ALIGNMODE p6 + %else + CPU amdnop + %endif + %else + %ifdef __NASM_VER__ + ALIGNMODE nop + %else + CPU basicnop + %endif + %endif +%endmacro + +; Merge mmx, sse*, and avx* +; m# is a simd register of the currently selected size +; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m# +; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m# +; zm# is the corresponding zmm register if mmsize >= 64, otherwise the same as m# +; (All 4 remain in sync through SWAP.) + +%macro CAT_XDEFINE 3 + %xdefine %1%2 %3 +%endmacro + +%macro CAT_UNDEF 2 + %undef %1%2 +%endmacro + +%macro DEFINE_MMREGS 1 ; mmtype + %assign %%prev_mmregs 0 + %ifdef num_mmregs + %assign %%prev_mmregs num_mmregs + %endif + + %assign num_mmregs 8 + %if ARCH_X86_64 && mmsize >= 16 + %assign num_mmregs 16 + %if cpuflag(avx512) || mmsize == 64 + %assign num_mmregs 32 + %endif + %endif + + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, %1 %+ %%i + CAT_XDEFINE nn%1, %%i, %%i + %assign %%i %%i+1 + %endrep + %if %%prev_mmregs > num_mmregs + %rep %%prev_mmregs - num_mmregs + CAT_UNDEF m, %%i + CAT_UNDEF nn %+ mmtype, %%i + %assign %%i %%i+1 + %endrep + %endif + %xdefine mmtype %1 +%endmacro + +; Prefer registers 16-31 over 0-15 to avoid having to use vzeroupper +%macro AVX512_MM_PERMUTATION 0-1 0 ; start_reg + %if ARCH_X86_64 && cpuflag(avx512) + %assign %%i %1 + %rep 16-%1 + %assign %%i_high %%i+16 + SWAP %%i, %%i_high + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +%macro INIT_MMX 0-1+ + %assign avx_enabled 0 + %define RESET_MM_PERMUTATION INIT_MMX %1 + %define mmsize 8 + %define mova movq + %define movu movq + %define movh movd + %define movnta movntq + INIT_CPUFLAGS %1 + DEFINE_MMREGS mm +%endmacro + +%macro INIT_XMM 0-1+ + %assign avx_enabled FORCE_VEX_ENCODING + %define RESET_MM_PERMUTATION INIT_XMM %1 + %define mmsize 16 + %define mova movdqa + %define movu movdqu + %define movh movq + %define movnta movntdq + INIT_CPUFLAGS %1 + DEFINE_MMREGS xmm + %if WIN64 + AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers + %endif + %xdefine bcstd 1to4 + %xdefine bcstq 1to2 +%endmacro + +%macro INIT_YMM 0-1+ + %assign avx_enabled 1 + %define RESET_MM_PERMUTATION INIT_YMM %1 + %define mmsize 32 + %define mova movdqa + %define movu movdqu + %undef movh + %define movnta movntdq + INIT_CPUFLAGS %1 + DEFINE_MMREGS ymm + AVX512_MM_PERMUTATION + %xdefine bcstd 1to8 + %xdefine bcstq 1to4 +%endmacro + +%macro INIT_ZMM 0-1+ + %assign avx_enabled 1 + %define RESET_MM_PERMUTATION INIT_ZMM %1 + %define mmsize 64 + %define mova movdqa + %define movu movdqu + %undef movh + %define movnta movntdq + INIT_CPUFLAGS %1 + DEFINE_MMREGS zmm + AVX512_MM_PERMUTATION + %xdefine bcstd 1to16 + %xdefine bcstq 1to8 +%endmacro + +INIT_XMM + +%macro DECLARE_MMCAST 1 + %define mmmm%1 mm%1 + %define mmxmm%1 mm%1 + %define mmymm%1 mm%1 + %define mmzmm%1 mm%1 + %define xmmmm%1 mm%1 + %define xmmxmm%1 xmm%1 + %define xmmymm%1 xmm%1 + %define xmmzmm%1 xmm%1 + %define ymmmm%1 mm%1 + %define ymmxmm%1 xmm%1 + %define ymmymm%1 ymm%1 + %define ymmzmm%1 ymm%1 + %define zmmmm%1 mm%1 + %define zmmxmm%1 xmm%1 + %define zmmymm%1 ymm%1 + %define zmmzmm%1 zmm%1 + %define xm%1 xmm %+ m%1 + %define ym%1 ymm %+ m%1 + %define zm%1 zmm %+ m%1 +%endmacro + +%assign i 0 +%rep 32 + DECLARE_MMCAST i + %assign i i+1 +%endrep + +; I often want to use macros that permute their arguments. e.g. there's no +; efficient way to implement butterfly or transpose or dct without swapping some +; arguments. +; +; I would like to not have to manually keep track of the permutations: +; If I insert a permutation in the middle of a function, it should automatically +; change everything that follows. For more complex macros I may also have multiple +; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. +; +; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that +; permutes its arguments. It's equivalent to exchanging the contents of the +; registers, except that this way you exchange the register names instead, so it +; doesn't cost any cycles. + +%macro PERMUTE 2-* ; takes a list of pairs to swap + %rep %0/2 + %xdefine %%tmp%2 m%2 + %rotate 2 + %endrep + %rep %0/2 + %xdefine m%1 %%tmp%2 + CAT_XDEFINE nn, m%1, %1 + %rotate 2 + %endrep +%endmacro + +%macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs) + %ifnum %1 ; SWAP 0, 1, ... + SWAP_INTERNAL_NUM %1, %2 + %else ; SWAP m0, m1, ... + SWAP_INTERNAL_NAME %1, %2 + %endif +%endmacro + +%macro SWAP_INTERNAL_NUM 2-* + %rep %0-1 + %xdefine %%tmp m%1 + %xdefine m%1 m%2 + %xdefine m%2 %%tmp + CAT_XDEFINE nn, m%1, %1 + CAT_XDEFINE nn, m%2, %2 + %rotate 1 + %endrep +%endmacro + +%macro SWAP_INTERNAL_NAME 2-* + %xdefine %%args nn %+ %1 + %rep %0-1 + %xdefine %%args %%args, nn %+ %2 + %rotate 1 + %endrep + SWAP_INTERNAL_NUM %%args +%endmacro + +; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later +; calls to that function will automatically load the permutation, so values can +; be returned in mmregs. +%macro SAVE_MM_PERMUTATION 0-1 + %if %0 + %xdefine %%f %1_m + %else + %xdefine %%f current_function %+ _m + %endif + %assign %%i 0 + %rep num_mmregs + %xdefine %%tmp m %+ %%i + CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp + %assign %%i %%i+1 + %endrep +%endmacro + +%macro LOAD_MM_PERMUTATION 0-1 ; name to load from + %if %0 + %xdefine %%f %1_m + %else + %xdefine %%f current_function %+ _m + %endif + %xdefine %%tmp %%f %+ 0 + %ifnum %%tmp + DEFINE_MMREGS mmtype + %assign %%i 0 + %rep num_mmregs + %xdefine %%tmp %%f %+ %%i + CAT_XDEFINE %%m, %%i, m %+ %%tmp + %assign %%i %%i+1 + %endrep + %rep num_mmregs + %assign %%i %%i-1 + CAT_XDEFINE m, %%i, %%m %+ %%i + CAT_XDEFINE nn, m %+ %%i, %%i + %endrep + %endif +%endmacro + +; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't +%macro call 1 + %ifid %1 + call_internal %1 %+ SUFFIX, %1 + %else + call %1 + %endif +%endmacro +%macro call_internal 2 + %xdefine %%i %2 + %ifndef cglobaled_%2 + %ifdef cglobaled_%1 + %xdefine %%i %1 + %endif + %endif + call %%i + LOAD_MM_PERMUTATION %%i +%endmacro + +; Substitutions that reduce instruction size but are functionally equivalent +%macro add 2 + %ifnum %2 + %if %2==128 + sub %1, -128 + %else + add %1, %2 + %endif + %else + add %1, %2 + %endif +%endmacro + +%macro sub 2 + %ifnum %2 + %if %2==128 + add %1, -128 + %else + sub %1, %2 + %endif + %else + sub %1, %2 + %endif +%endmacro + +;============================================================================= +; AVX abstraction layer +;============================================================================= + +%assign i 0 +%rep 32 + %if i < 8 + CAT_XDEFINE sizeofmm, i, 8 + CAT_XDEFINE regnumofmm, i, i + %endif + CAT_XDEFINE sizeofxmm, i, 16 + CAT_XDEFINE sizeofymm, i, 32 + CAT_XDEFINE sizeofzmm, i, 64 + CAT_XDEFINE regnumofxmm, i, i + CAT_XDEFINE regnumofymm, i, i + CAT_XDEFINE regnumofzmm, i, i + %assign i i+1 +%endrep +%undef i + +%macro CHECK_AVX_INSTR_EMU 3-* + %xdefine %%opcode %1 + %xdefine %%dst %2 + %rep %0-2 + %ifidn %%dst, %3 + %error non-avx emulation of ``%%opcode'' is not supported + %endif + %rotate 1 + %endrep +%endmacro + +;%1 == instruction +;%2 == minimal instruction set +;%3 == 1 if float, 0 if int +;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation) +;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +;%6+: operands +%macro RUN_AVX_INSTR 6-9+ + %ifnum sizeof%7 + %assign __sizeofreg sizeof%7 + %elifnum sizeof%6 + %assign __sizeofreg sizeof%6 + %else + %assign __sizeofreg mmsize + %endif + %assign __emulate_avx 0 + %if avx_enabled && __sizeofreg >= 16 + %xdefine __instr v%1 + %else + %xdefine __instr %1 + %if %0 >= 8+%4 + %assign __emulate_avx 1 + %endif + %endif + %ifnidn %2, fnord + %ifdef cpuname + %if notcpuflag(%2) + %error use of ``%1'' %2 instruction in cpuname function: current_function + %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2) + %error use of ``%1'' sse2 instruction in cpuname function: current_function + %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2) + %error use of ``%1'' avx2 instruction in cpuname function: current_function + %elif __sizeofreg == 16 && notcpuflag(sse) + %error use of ``%1'' sse instruction in cpuname function: current_function + %elif __sizeofreg == 32 && notcpuflag(avx) + %error use of ``%1'' avx instruction in cpuname function: current_function + %elif __sizeofreg == 64 && notcpuflag(avx512) + %error use of ``%1'' avx512 instruction in cpuname function: current_function + %elifidn %1, pextrw ; special case because the base instruction is mmx2, + %ifnid %6 ; but sse4 is required for memory operands + %if notcpuflag(sse4) + %error use of ``%1'' sse4 instruction in cpuname function: current_function + %endif + %endif + %endif + %endif + %endif + + %if __emulate_avx + %xdefine __src1 %7 + %xdefine __src2 %8 + %if %5 && %4 == 0 + %ifnidn %6, %7 + %ifidn %6, %8 + %xdefine __src1 %8 + %xdefine __src2 %7 + %elifnnum sizeof%8 + ; 3-operand AVX instructions with a memory arg can only have it in src2, + ; whereas SSE emulation prefers to have it in src1 (i.e. the mov). + ; So, if the instruction is commutative with a memory arg, swap them. + %xdefine __src1 %8 + %xdefine __src2 %7 + %endif + %endif + %endif + %ifnidn %6, __src1 + %if %0 >= 9 + CHECK_AVX_INSTR_EMU {%1 %6, %7, %8, %9}, %6, __src2, %9 + %else + CHECK_AVX_INSTR_EMU {%1 %6, %7, %8}, %6, __src2 + %endif + %if __sizeofreg == 8 + MOVQ %6, __src1 + %elif %3 + MOVAPS %6, __src1 + %else + MOVDQA %6, __src1 + %endif + %endif + %if %0 >= 9 + %1 %6, __src2, %9 + %else + %1 %6, __src2 + %endif + %elif %0 >= 9 + %if avx_enabled && __sizeofreg >= 16 && %4 == 1 + %ifnnum regnumof%7 + %if %3 + vmovaps %6, %7 + %else + vmovdqa %6, %7 + %endif + __instr %6, %6, %8, %9 + %else + __instr %6, %7, %8, %9 + %endif + %else + __instr %6, %7, %8, %9 + %endif + %elif %0 == 8 + %if avx_enabled && __sizeofreg >= 16 && %4 == 0 + %xdefine __src1 %7 + %xdefine __src2 %8 + %if %5 + %ifnum regnumof%7 + %ifnum regnumof%8 + %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32 + ; Most VEX-encoded instructions require an additional byte to encode when + ; src2 is a high register (e.g. m8..15). If the instruction is commutative + ; we can swap src1 and src2 when doing so reduces the instruction length. + %xdefine __src1 %8 + %xdefine __src2 %7 + %endif + %endif + %elifnum regnumof%8 ; put memory operands in src2 when possible + %xdefine __src1 %8 + %xdefine __src2 %7 + %else + %assign __emulate_avx 1 + %endif + %elifnnum regnumof%7 + ; EVEX allows imm8 shift instructions to be used with memory operands, + ; but VEX does not. This handles those special cases. + %ifnnum %8 + %assign __emulate_avx 1 + %elif notcpuflag(avx512) + %assign __emulate_avx 1 + %endif + %endif + %if __emulate_avx ; a separate load is required + %if %3 + vmovaps %6, %7 + %else + vmovdqa %6, %7 + %endif + __instr %6, %6, %8 + %else + __instr %6, __src1, __src2 + %endif + %else + __instr %6, %7, %8 + %endif + %elif %0 == 7 + %if avx_enabled && __sizeofreg >= 16 && %5 + %xdefine __src1 %6 + %xdefine __src2 %7 + %ifnum regnumof%6 + %ifnum regnumof%7 + %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32 + %xdefine __src1 %7 + %xdefine __src2 %6 + %endif + %endif + %endif + __instr %6, __src1, __src2 + %else + __instr %6, %7 + %endif + %else + __instr %6 + %endif +%endmacro + +;%1 == instruction +;%2 == minimal instruction set +;%3 == 1 if float, 0 if int +;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation) +;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +%macro AVX_INSTR 1-5 fnord, 0, 255, 0 + %macro %1 1-10 fnord, fnord, fnord, fnord, %1, %2, %3, %4, %5 + %ifidn %2, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1 + %elifidn %3, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2 + %elifidn %4, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3 + %elifidn %5, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4 + %else + RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4, %5 + %endif + %endmacro +%endmacro + +; Instructions with both VEX/EVEX and legacy encodings +; Non-destructive instructions are written without parameters +AVX_INSTR addpd, sse2, 1, 0, 1 +AVX_INSTR addps, sse, 1, 0, 1 +AVX_INSTR addsd, sse2, 1, 0, 0 +AVX_INSTR addss, sse, 1, 0, 0 +AVX_INSTR addsubpd, sse3, 1, 0, 0 +AVX_INSTR addsubps, sse3, 1, 0, 0 +AVX_INSTR aesdec, aesni, 0, 0, 0 +AVX_INSTR aesdeclast, aesni, 0, 0, 0 +AVX_INSTR aesenc, aesni, 0, 0, 0 +AVX_INSTR aesenclast, aesni, 0, 0, 0 +AVX_INSTR aesimc, aesni +AVX_INSTR aeskeygenassist, aesni +AVX_INSTR andnpd, sse2, 1, 0, 0 +AVX_INSTR andnps, sse, 1, 0, 0 +AVX_INSTR andpd, sse2, 1, 0, 1 +AVX_INSTR andps, sse, 1, 0, 1 +AVX_INSTR blendpd, sse4, 1, 1, 0 +AVX_INSTR blendps, sse4, 1, 1, 0 +AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding +AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding +AVX_INSTR cmpeqpd, sse2, 1, 0, 1 +AVX_INSTR cmpeqps, sse, 1, 0, 1 +AVX_INSTR cmpeqsd, sse2, 1, 0, 0 +AVX_INSTR cmpeqss, sse, 1, 0, 0 +AVX_INSTR cmplepd, sse2, 1, 0, 0 +AVX_INSTR cmpleps, sse, 1, 0, 0 +AVX_INSTR cmplesd, sse2, 1, 0, 0 +AVX_INSTR cmpless, sse, 1, 0, 0 +AVX_INSTR cmpltpd, sse2, 1, 0, 0 +AVX_INSTR cmpltps, sse, 1, 0, 0 +AVX_INSTR cmpltsd, sse2, 1, 0, 0 +AVX_INSTR cmpltss, sse, 1, 0, 0 +AVX_INSTR cmpneqpd, sse2, 1, 0, 1 +AVX_INSTR cmpneqps, sse, 1, 0, 1 +AVX_INSTR cmpneqsd, sse2, 1, 0, 0 +AVX_INSTR cmpneqss, sse, 1, 0, 0 +AVX_INSTR cmpnlepd, sse2, 1, 0, 0 +AVX_INSTR cmpnleps, sse, 1, 0, 0 +AVX_INSTR cmpnlesd, sse2, 1, 0, 0 +AVX_INSTR cmpnless, sse, 1, 0, 0 +AVX_INSTR cmpnltpd, sse2, 1, 0, 0 +AVX_INSTR cmpnltps, sse, 1, 0, 0 +AVX_INSTR cmpnltsd, sse2, 1, 0, 0 +AVX_INSTR cmpnltss, sse, 1, 0, 0 +AVX_INSTR cmpordpd, sse2 1, 0, 1 +AVX_INSTR cmpordps, sse 1, 0, 1 +AVX_INSTR cmpordsd, sse2 1, 0, 0 +AVX_INSTR cmpordss, sse 1, 0, 0 +AVX_INSTR cmppd, sse2, 1, 1, 0 +AVX_INSTR cmpps, sse, 1, 1, 0 +AVX_INSTR cmpsd, sse2, 1, 1, 0 +AVX_INSTR cmpss, sse, 1, 1, 0 +AVX_INSTR cmpunordpd, sse2, 1, 0, 1 +AVX_INSTR cmpunordps, sse, 1, 0, 1 +AVX_INSTR cmpunordsd, sse2, 1, 0, 0 +AVX_INSTR cmpunordss, sse, 1, 0, 0 +AVX_INSTR comisd, sse2, 1 +AVX_INSTR comiss, sse, 1 +AVX_INSTR cvtdq2pd, sse2, 1 +AVX_INSTR cvtdq2ps, sse2, 1 +AVX_INSTR cvtpd2dq, sse2, 1 +AVX_INSTR cvtpd2ps, sse2, 1 +AVX_INSTR cvtps2dq, sse2, 1 +AVX_INSTR cvtps2pd, sse2, 1 +AVX_INSTR cvtsd2si, sse2, 1 +AVX_INSTR cvtsd2ss, sse2, 1, 0, 0 +AVX_INSTR cvtsi2sd, sse2, 1, 0, 0 +AVX_INSTR cvtsi2ss, sse, 1, 0, 0 +AVX_INSTR cvtss2sd, sse2, 1, 0, 0 +AVX_INSTR cvtss2si, sse, 1 +AVX_INSTR cvttpd2dq, sse2, 1 +AVX_INSTR cvttps2dq, sse2, 1 +AVX_INSTR cvttsd2si, sse2, 1 +AVX_INSTR cvttss2si, sse, 1 +AVX_INSTR divpd, sse2, 1, 0, 0 +AVX_INSTR divps, sse, 1, 0, 0 +AVX_INSTR divsd, sse2, 1, 0, 0 +AVX_INSTR divss, sse, 1, 0, 0 +AVX_INSTR dppd, sse4, 1, 1, 0 +AVX_INSTR dpps, sse4, 1, 1, 0 +AVX_INSTR extractps, sse4, 1 +AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0 +AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0 +AVX_INSTR gf2p8mulb, gfni, 0, 0, 0 +AVX_INSTR haddpd, sse3, 1, 0, 0 +AVX_INSTR haddps, sse3, 1, 0, 0 +AVX_INSTR hsubpd, sse3, 1, 0, 0 +AVX_INSTR hsubps, sse3, 1, 0, 0 +AVX_INSTR insertps, sse4, 1, 1, 0 +AVX_INSTR lddqu, sse3 +AVX_INSTR ldmxcsr, sse, 1 +AVX_INSTR maskmovdqu, sse2 +AVX_INSTR maxpd, sse2, 1, 0, 1 +AVX_INSTR maxps, sse, 1, 0, 1 +AVX_INSTR maxsd, sse2, 1, 0, 0 +AVX_INSTR maxss, sse, 1, 0, 0 +AVX_INSTR minpd, sse2, 1, 0, 1 +AVX_INSTR minps, sse, 1, 0, 1 +AVX_INSTR minsd, sse2, 1, 0, 0 +AVX_INSTR minss, sse, 1, 0, 0 +AVX_INSTR movapd, sse2, 1 +AVX_INSTR movaps, sse, 1 +AVX_INSTR movd, mmx +AVX_INSTR movddup, sse3, 1 +AVX_INSTR movdqa, sse2 +AVX_INSTR movdqu, sse2 +AVX_INSTR movhlps, sse, 1, 0, 0 +AVX_INSTR movhpd, sse2, 1, 0, 0 +AVX_INSTR movhps, sse, 1, 0, 0 +AVX_INSTR movlhps, sse, 1, 0, 0 +AVX_INSTR movlpd, sse2, 1, 0, 0 +AVX_INSTR movlps, sse, 1, 0, 0 +AVX_INSTR movmskpd, sse2, 1 +AVX_INSTR movmskps, sse, 1 +AVX_INSTR movntdq, sse2 +AVX_INSTR movntdqa, sse4 +AVX_INSTR movntpd, sse2, 1 +AVX_INSTR movntps, sse, 1 +AVX_INSTR movq, mmx +AVX_INSTR movsd, sse2, 1, 0, 0 +AVX_INSTR movshdup, sse3, 1 +AVX_INSTR movsldup, sse3, 1 +AVX_INSTR movss, sse, 1, 0, 0 +AVX_INSTR movupd, sse2, 1 +AVX_INSTR movups, sse, 1 +AVX_INSTR mpsadbw, sse4, 0, 1, 0 +AVX_INSTR mulpd, sse2, 1, 0, 1 +AVX_INSTR mulps, sse, 1, 0, 1 +AVX_INSTR mulsd, sse2, 1, 0, 0 +AVX_INSTR mulss, sse, 1, 0, 0 +AVX_INSTR orpd, sse2, 1, 0, 1 +AVX_INSTR orps, sse, 1, 0, 1 +AVX_INSTR pabsb, ssse3 +AVX_INSTR pabsd, ssse3 +AVX_INSTR pabsw, ssse3 +AVX_INSTR packssdw, mmx, 0, 0, 0 +AVX_INSTR packsswb, mmx, 0, 0, 0 +AVX_INSTR packusdw, sse4, 0, 0, 0 +AVX_INSTR packuswb, mmx, 0, 0, 0 +AVX_INSTR paddb, mmx, 0, 0, 1 +AVX_INSTR paddd, mmx, 0, 0, 1 +AVX_INSTR paddq, sse2, 0, 0, 1 +AVX_INSTR paddsb, mmx, 0, 0, 1 +AVX_INSTR paddsw, mmx, 0, 0, 1 +AVX_INSTR paddusb, mmx, 0, 0, 1 +AVX_INSTR paddusw, mmx, 0, 0, 1 +AVX_INSTR paddw, mmx, 0, 0, 1 +AVX_INSTR palignr, ssse3, 0, 1, 0 +AVX_INSTR pand, mmx, 0, 0, 1 +AVX_INSTR pandn, mmx, 0, 0, 0 +AVX_INSTR pavgb, mmx2, 0, 0, 1 +AVX_INSTR pavgw, mmx2, 0, 0, 1 +AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding +AVX_INSTR pblendw, sse4, 0, 1, 0 +AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0 +AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0 +AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0 +AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0 +AVX_INSTR pclmulqdq, fnord, 0, 1, 0 +AVX_INSTR pcmpeqb, mmx, 0, 0, 1 +AVX_INSTR pcmpeqd, mmx, 0, 0, 1 +AVX_INSTR pcmpeqq, sse4, 0, 0, 1 +AVX_INSTR pcmpeqw, mmx, 0, 0, 1 +AVX_INSTR pcmpestri, sse42 +AVX_INSTR pcmpestrm, sse42 +AVX_INSTR pcmpgtb, mmx, 0, 0, 0 +AVX_INSTR pcmpgtd, mmx, 0, 0, 0 +AVX_INSTR pcmpgtq, sse42, 0, 0, 0 +AVX_INSTR pcmpgtw, mmx, 0, 0, 0 +AVX_INSTR pcmpistri, sse42 +AVX_INSTR pcmpistrm, sse42 +AVX_INSTR pextrb, sse4 +AVX_INSTR pextrd, sse4 +AVX_INSTR pextrq, sse4 +AVX_INSTR pextrw, mmx2 +AVX_INSTR phaddd, ssse3, 0, 0, 0 +AVX_INSTR phaddsw, ssse3, 0, 0, 0 +AVX_INSTR phaddw, ssse3, 0, 0, 0 +AVX_INSTR phminposuw, sse4 +AVX_INSTR phsubd, ssse3, 0, 0, 0 +AVX_INSTR phsubsw, ssse3, 0, 0, 0 +AVX_INSTR phsubw, ssse3, 0, 0, 0 +AVX_INSTR pinsrb, sse4, 0, 1, 0 +AVX_INSTR pinsrd, sse4, 0, 1, 0 +AVX_INSTR pinsrq, sse4, 0, 1, 0 +AVX_INSTR pinsrw, mmx2, 0, 1, 0 +AVX_INSTR pmaddubsw, ssse3, 0, 0, 0 +AVX_INSTR pmaddwd, mmx, 0, 0, 1 +AVX_INSTR pmaxsb, sse4, 0, 0, 1 +AVX_INSTR pmaxsd, sse4, 0, 0, 1 +AVX_INSTR pmaxsw, mmx2, 0, 0, 1 +AVX_INSTR pmaxub, mmx2, 0, 0, 1 +AVX_INSTR pmaxud, sse4, 0, 0, 1 +AVX_INSTR pmaxuw, sse4, 0, 0, 1 +AVX_INSTR pminsb, sse4, 0, 0, 1 +AVX_INSTR pminsd, sse4, 0, 0, 1 +AVX_INSTR pminsw, mmx2, 0, 0, 1 +AVX_INSTR pminub, mmx2, 0, 0, 1 +AVX_INSTR pminud, sse4, 0, 0, 1 +AVX_INSTR pminuw, sse4, 0, 0, 1 +AVX_INSTR pmovmskb, mmx2 +AVX_INSTR pmovsxbd, sse4 +AVX_INSTR pmovsxbq, sse4 +AVX_INSTR pmovsxbw, sse4 +AVX_INSTR pmovsxdq, sse4 +AVX_INSTR pmovsxwd, sse4 +AVX_INSTR pmovsxwq, sse4 +AVX_INSTR pmovzxbd, sse4 +AVX_INSTR pmovzxbq, sse4 +AVX_INSTR pmovzxbw, sse4 +AVX_INSTR pmovzxdq, sse4 +AVX_INSTR pmovzxwd, sse4 +AVX_INSTR pmovzxwq, sse4 +AVX_INSTR pmuldq, sse4, 0, 0, 1 +AVX_INSTR pmulhrsw, ssse3, 0, 0, 1 +AVX_INSTR pmulhuw, mmx2, 0, 0, 1 +AVX_INSTR pmulhw, mmx, 0, 0, 1 +AVX_INSTR pmulld, sse4, 0, 0, 1 +AVX_INSTR pmullw, mmx, 0, 0, 1 +AVX_INSTR pmuludq, sse2, 0, 0, 1 +AVX_INSTR por, mmx, 0, 0, 1 +AVX_INSTR psadbw, mmx2, 0, 0, 1 +AVX_INSTR pshufb, ssse3, 0, 0, 0 +AVX_INSTR pshufd, sse2 +AVX_INSTR pshufhw, sse2 +AVX_INSTR pshuflw, sse2 +AVX_INSTR psignb, ssse3, 0, 0, 0 +AVX_INSTR psignd, ssse3, 0, 0, 0 +AVX_INSTR psignw, ssse3, 0, 0, 0 +AVX_INSTR pslld, mmx, 0, 0, 0 +AVX_INSTR pslldq, sse2, 0, 0, 0 +AVX_INSTR psllq, mmx, 0, 0, 0 +AVX_INSTR psllw, mmx, 0, 0, 0 +AVX_INSTR psrad, mmx, 0, 0, 0 +AVX_INSTR psraw, mmx, 0, 0, 0 +AVX_INSTR psrld, mmx, 0, 0, 0 +AVX_INSTR psrldq, sse2, 0, 0, 0 +AVX_INSTR psrlq, mmx, 0, 0, 0 +AVX_INSTR psrlw, mmx, 0, 0, 0 +AVX_INSTR psubb, mmx, 0, 0, 0 +AVX_INSTR psubd, mmx, 0, 0, 0 +AVX_INSTR psubq, sse2, 0, 0, 0 +AVX_INSTR psubsb, mmx, 0, 0, 0 +AVX_INSTR psubsw, mmx, 0, 0, 0 +AVX_INSTR psubusb, mmx, 0, 0, 0 +AVX_INSTR psubusw, mmx, 0, 0, 0 +AVX_INSTR psubw, mmx, 0, 0, 0 +AVX_INSTR ptest, sse4 +AVX_INSTR punpckhbw, mmx, 0, 0, 0 +AVX_INSTR punpckhdq, mmx, 0, 0, 0 +AVX_INSTR punpckhqdq, sse2, 0, 0, 0 +AVX_INSTR punpckhwd, mmx, 0, 0, 0 +AVX_INSTR punpcklbw, mmx, 0, 0, 0 +AVX_INSTR punpckldq, mmx, 0, 0, 0 +AVX_INSTR punpcklqdq, sse2, 0, 0, 0 +AVX_INSTR punpcklwd, mmx, 0, 0, 0 +AVX_INSTR pxor, mmx, 0, 0, 1 +AVX_INSTR rcpps, sse, 1 +AVX_INSTR rcpss, sse, 1, 0, 0 +AVX_INSTR roundpd, sse4, 1 +AVX_INSTR roundps, sse4, 1 +AVX_INSTR roundsd, sse4, 1, 1, 0 +AVX_INSTR roundss, sse4, 1, 1, 0 +AVX_INSTR rsqrtps, sse, 1 +AVX_INSTR rsqrtss, sse, 1, 0, 0 +AVX_INSTR shufpd, sse2, 1, 1, 0 +AVX_INSTR shufps, sse, 1, 1, 0 +AVX_INSTR sqrtpd, sse2, 1 +AVX_INSTR sqrtps, sse, 1 +AVX_INSTR sqrtsd, sse2, 1, 0, 0 +AVX_INSTR sqrtss, sse, 1, 0, 0 +AVX_INSTR stmxcsr, sse, 1 +AVX_INSTR subpd, sse2, 1, 0, 0 +AVX_INSTR subps, sse, 1, 0, 0 +AVX_INSTR subsd, sse2, 1, 0, 0 +AVX_INSTR subss, sse, 1, 0, 0 +AVX_INSTR ucomisd, sse2, 1 +AVX_INSTR ucomiss, sse, 1 +AVX_INSTR unpckhpd, sse2, 1, 0, 0 +AVX_INSTR unpckhps, sse, 1, 0, 0 +AVX_INSTR unpcklpd, sse2, 1, 0, 0 +AVX_INSTR unpcklps, sse, 1, 0, 0 +AVX_INSTR xorpd, sse2, 1, 0, 1 +AVX_INSTR xorps, sse, 1, 0, 1 + +; 3DNow instructions, for sharing code between AVX, SSE and 3DN +AVX_INSTR pfadd, 3dnow, 1, 0, 1 +AVX_INSTR pfmul, 3dnow, 1, 0, 1 +AVX_INSTR pfsub, 3dnow, 1, 0, 0 + +;%1 == instruction +;%2 == minimal instruction set +%macro GPR_INSTR 2 + %macro %1 2-5 fnord, %1, %2 + %ifdef cpuname + %if notcpuflag(%5) + %error use of ``%4'' %5 instruction in cpuname function: current_function + %endif + %endif + %ifidn %3, fnord + %4 %1, %2 + %else + %4 %1, %2, %3 + %endif + %endmacro +%endmacro + +GPR_INSTR andn, bmi1 +GPR_INSTR bextr, bmi1 +GPR_INSTR blsi, bmi1 +GPR_INSTR blsmsk, bmi1 +GPR_INSTR blsr, bmi1 +GPR_INSTR bzhi, bmi2 +GPR_INSTR mulx, bmi2 +GPR_INSTR pdep, bmi2 +GPR_INSTR pext, bmi2 +GPR_INSTR popcnt, sse42 +GPR_INSTR rorx, bmi2 +GPR_INSTR sarx, bmi2 +GPR_INSTR shlx, bmi2 +GPR_INSTR shrx, bmi2 + +; base-4 constants for shuffles +%assign i 0 +%rep 256 + %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3) + %if j < 10 + CAT_XDEFINE q000, j, i + %elif j < 100 + CAT_XDEFINE q00, j, i + %elif j < 1000 + CAT_XDEFINE q0, j, i + %else + CAT_XDEFINE q, j, i + %endif + %assign i i+1 +%endrep +%undef i +%undef j + +%macro FMA_INSTR 3 + %macro %1 4-7 %1, %2, %3 + %if cpuflag(xop) + v%5 %1, %2, %3, %4 + %elifnidn %1, %4 + %6 %1, %2, %3 + %7 %1, %4 + %else + %error non-xop emulation of ``%5 %1, %2, %3, %4'' is not supported + %endif + %endmacro +%endmacro + +FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation +FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation +FMA_INSTR pmacsww, pmullw, paddw +FMA_INSTR pmadcswd, pmaddwd, paddd + +; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax. +; FMA3 is only possible if dst is the same as one of the src registers. +; Either src2 or src3 can be a memory operand. +%macro FMA4_INSTR 2-* + %push fma4_instr + %xdefine %$prefix %1 + %rep %0 - 1 + %macro %$prefix%2 4-6 %$prefix, %2 + %if notcpuflag(fma3) && notcpuflag(fma4) + %error use of ``%5%6'' fma instruction in cpuname function: current_function + %elif cpuflag(fma4) + v%5%6 %1, %2, %3, %4 + %elifidn %1, %2 + ; If %3 or %4 is a memory operand it needs to be encoded as the last operand. + %ifnum sizeof%3 + v%{5}213%6 %2, %3, %4 + %else + v%{5}132%6 %2, %4, %3 + %endif + %elifidn %1, %3 + v%{5}213%6 %3, %2, %4 + %elifidn %1, %4 + v%{5}231%6 %4, %2, %3 + %else + %error fma3 emulation of ``%5%6 %1, %2, %3, %4'' is not supported + %endif + %endmacro + %rotate 1 + %endrep + %pop +%endmacro + +FMA4_INSTR fmadd, pd, ps, sd, ss +FMA4_INSTR fmaddsub, pd, ps +FMA4_INSTR fmsub, pd, ps, sd, ss +FMA4_INSTR fmsubadd, pd, ps +FMA4_INSTR fnmadd, pd, ps, sd, ss +FMA4_INSTR fnmsub, pd, ps, sd, ss + +; Macros for converting VEX instructions to equivalent EVEX ones. +%macro EVEX_INSTR 2-3 0 ; vex, evex, prefer_evex + %macro %1 2-7 fnord, fnord, %1, %2, %3 + %ifidn %3, fnord + %define %%args %1, %2 + %elifidn %4, fnord + %define %%args %1, %2, %3 + %else + %define %%args %1, %2, %3, %4 + %endif + %assign %%evex_required cpuflag(avx512) & %7 + %ifnum regnumof%1 + %if regnumof%1 >= 16 || sizeof%1 > 32 + %assign %%evex_required 1 + %endif + %endif + %ifnum regnumof%2 + %if regnumof%2 >= 16 || sizeof%2 > 32 + %assign %%evex_required 1 + %endif + %endif + %ifnum regnumof%3 + %if regnumof%3 >= 16 || sizeof%3 > 32 + %assign %%evex_required 1 + %endif + %endif + %if %%evex_required + %6 %%args + %else + %5 %%args ; Prefer VEX over EVEX due to shorter instruction length + %endif + %endmacro +%endmacro + +EVEX_INSTR vbroadcastf128, vbroadcastf32x4 +EVEX_INSTR vbroadcasti128, vbroadcasti32x4 +EVEX_INSTR vextractf128, vextractf32x4 +EVEX_INSTR vextracti128, vextracti32x4 +EVEX_INSTR vinsertf128, vinsertf32x4 +EVEX_INSTR vinserti128, vinserti32x4 +EVEX_INSTR vmovdqa, vmovdqa32 +EVEX_INSTR vmovdqu, vmovdqu32 +EVEX_INSTR vpand, vpandd +EVEX_INSTR vpandn, vpandnd +EVEX_INSTR vpor, vpord +EVEX_INSTR vpxor, vpxord +EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision +EVEX_INSTR vrcpss, vrcp14ss, 1 +EVEX_INSTR vrsqrtps, vrsqrt14ps, 1 +EVEX_INSTR vrsqrtss, vrsqrt14ss, 1 diff --git a/third_party/dav1d/src/fg_apply.h b/third_party/dav1d/src/fg_apply.h new file mode 100644 index 0000000000..be6685d801 --- /dev/null +++ b/third_party/dav1d/src/fg_apply.h @@ -0,0 +1,58 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_FG_APPLY_H +#define DAV1D_SRC_FG_APPLY_H + +#include "dav1d/picture.h" + +#include "common/bitdepth.h" + +#include "src/filmgrain.h" + +#ifdef BITDEPTH +# define array_decl(type, name, sz) type name sz +#else +# define array_decl(type, name, sz) void *name +#endif + +bitfn_decls(void dav1d_apply_grain, + const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, const Dav1dPicture *const in); +bitfn_decls(void dav1d_prep_grain, + const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, const Dav1dPicture *const in, + array_decl(uint8_t, scaling, [3][SCALING_SIZE]), + array_decl(entry, grain_lut, [3][GRAIN_HEIGHT+1][GRAIN_WIDTH])); +bitfn_decls(void dav1d_apply_grain_row, + const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, const Dav1dPicture *const in, + array_decl(const uint8_t, scaling, [3][SCALING_SIZE]), + array_decl(const entry, grain_lut, [3][GRAIN_HEIGHT+1][GRAIN_WIDTH]), + const int row); + +#endif /* DAV1D_SRC_FG_APPLY_H */ diff --git a/third_party/dav1d/src/fg_apply_tmpl.c b/third_party/dav1d/src/fg_apply_tmpl.c new file mode 100644 index 0000000000..e1b1655f44 --- /dev/null +++ b/third_party/dav1d/src/fg_apply_tmpl.c @@ -0,0 +1,239 @@ +/* + * Copyright © 2018, Niklas Haas + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "dav1d/common.h" +#include "dav1d/picture.h" + +#include "common/intops.h" +#include "common/bitdepth.h" + +#include "src/fg_apply.h" +#include "src/ref.h" + +static void generate_scaling(const int bitdepth, + const uint8_t points[][2], const int num, + uint8_t scaling[SCALING_SIZE]) +{ +#if BITDEPTH == 8 + const int shift_x = 0; + const int scaling_size = SCALING_SIZE; +#else + assert(bitdepth > 8); + const int shift_x = bitdepth - 8; + const int scaling_size = 1 << bitdepth; +#endif + + if (num == 0) { + memset(scaling, 0, scaling_size); + return; + } + + // Fill up the preceding entries with the initial value + memset(scaling, points[0][1], points[0][0] << shift_x); + + // Linearly interpolate the values in the middle + for (int i = 0; i < num - 1; i++) { + const int bx = points[i][0]; + const int by = points[i][1]; + const int ex = points[i+1][0]; + const int ey = points[i+1][1]; + const int dx = ex - bx; + const int dy = ey - by; + assert(dx > 0); + const int delta = dy * ((0x10000 + (dx >> 1)) / dx); + for (int x = 0, d = 0x8000; x < dx; x++) { + scaling[(bx + x) << shift_x] = by + (d >> 16); + d += delta; + } + } + + // Fill up the remaining entries with the final value + const int n = points[num - 1][0] << shift_x; + memset(&scaling[n], points[num - 1][1], scaling_size - n); + +#if BITDEPTH != 8 + const int pad = 1 << shift_x, rnd = pad >> 1; + for (int i = 0; i < num - 1; i++) { + const int bx = points[i][0] << shift_x; + const int ex = points[i+1][0] << shift_x; + const int dx = ex - bx; + for (int x = 0; x < dx; x += pad) { + const int range = scaling[bx + x + pad] - scaling[bx + x]; + for (int n = 1, r = rnd; n < pad; n++) { + r += range; + scaling[bx + x + n] = scaling[bx + x] + (r >> shift_x); + } + } + } +#endif +} + +#ifndef UNIT_TEST +void bitfn(dav1d_prep_grain)(const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, + const Dav1dPicture *const in, + uint8_t scaling[3][SCALING_SIZE], + entry grain_lut[3][GRAIN_HEIGHT+1][GRAIN_WIDTH]) +{ + const Dav1dFilmGrainData *const data = &out->frame_hdr->film_grain.data; +#if BITDEPTH != 8 + const int bitdepth_max = (1 << out->p.bpc) - 1; +#endif + + // Generate grain LUTs as needed + dsp->generate_grain_y(grain_lut[0], data HIGHBD_TAIL_SUFFIX); // always needed + if (data->num_uv_points[0] || data->chroma_scaling_from_luma) + dsp->generate_grain_uv[in->p.layout - 1](grain_lut[1], grain_lut[0], + data, 0 HIGHBD_TAIL_SUFFIX); + if (data->num_uv_points[1] || data->chroma_scaling_from_luma) + dsp->generate_grain_uv[in->p.layout - 1](grain_lut[2], grain_lut[0], + data, 1 HIGHBD_TAIL_SUFFIX); + + // Generate scaling LUTs as needed + if (data->num_y_points || data->chroma_scaling_from_luma) + generate_scaling(in->p.bpc, data->y_points, data->num_y_points, scaling[0]); + if (data->num_uv_points[0]) + generate_scaling(in->p.bpc, data->uv_points[0], data->num_uv_points[0], scaling[1]); + if (data->num_uv_points[1]) + generate_scaling(in->p.bpc, data->uv_points[1], data->num_uv_points[1], scaling[2]); + + // Create new references for the non-modified planes + assert(out->stride[0] == in->stride[0]); + if (!data->num_y_points) { + struct Dav1dRef **out_plane_ref = out->ref->user_data; + struct Dav1dRef **in_plane_ref = in->ref->user_data; + dav1d_ref_dec(&out_plane_ref[0]); + out_plane_ref[0] = in_plane_ref[0]; + dav1d_ref_inc(out_plane_ref[0]); + out->data[0] = in->data[0]; + } + + if (in->p.layout != DAV1D_PIXEL_LAYOUT_I400 && !data->chroma_scaling_from_luma) { + assert(out->stride[1] == in->stride[1]); + struct Dav1dRef **out_plane_ref = out->ref->user_data; + struct Dav1dRef **in_plane_ref = in->ref->user_data; + if (!data->num_uv_points[0]) { + dav1d_ref_dec(&out_plane_ref[1]); + out_plane_ref[1] = in_plane_ref[1]; + dav1d_ref_inc(out_plane_ref[1]); + out->data[1] = in->data[1]; + } + if (!data->num_uv_points[1]) { + dav1d_ref_dec(&out_plane_ref[2]); + out_plane_ref[2] = in_plane_ref[2]; + dav1d_ref_inc(out_plane_ref[2]); + out->data[2] = in->data[2]; + } + } +} + +void bitfn(dav1d_apply_grain_row)(const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, + const Dav1dPicture *const in, + const uint8_t scaling[3][SCALING_SIZE], + const entry grain_lut[3][GRAIN_HEIGHT+1][GRAIN_WIDTH], + const int row) +{ + // Synthesize grain for the affected planes + const Dav1dFilmGrainData *const data = &out->frame_hdr->film_grain.data; + const int ss_y = in->p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_x = in->p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int cpw = (out->p.w + ss_x) >> ss_x; + const int is_id = out->seq_hdr->mtrx == DAV1D_MC_IDENTITY; + pixel *const luma_src = + ((pixel *) in->data[0]) + row * BLOCK_SIZE * PXSTRIDE(in->stride[0]); +#if BITDEPTH != 8 + const int bitdepth_max = (1 << out->p.bpc) - 1; +#endif + + if (data->num_y_points) { + const int bh = imin(out->p.h - row * BLOCK_SIZE, BLOCK_SIZE); + dsp->fgy_32x32xn(((pixel *) out->data[0]) + row * BLOCK_SIZE * PXSTRIDE(out->stride[0]), + luma_src, out->stride[0], data, + out->p.w, scaling[0], grain_lut[0], bh, row HIGHBD_TAIL_SUFFIX); + } + + if (!data->num_uv_points[0] && !data->num_uv_points[1] && + !data->chroma_scaling_from_luma) + { + return; + } + + const int bh = (imin(out->p.h - row * BLOCK_SIZE, BLOCK_SIZE) + ss_y) >> ss_y; + + // extend padding pixels + if (out->p.w & ss_x) { + pixel *ptr = luma_src; + for (int y = 0; y < bh; y++) { + ptr[out->p.w] = ptr[out->p.w - 1]; + ptr += PXSTRIDE(in->stride[0]) << ss_y; + } + } + + const ptrdiff_t uv_off = row * BLOCK_SIZE * PXSTRIDE(out->stride[1]) >> ss_y; + if (data->chroma_scaling_from_luma) { + for (int pl = 0; pl < 2; pl++) + dsp->fguv_32x32xn[in->p.layout - 1](((pixel *) out->data[1 + pl]) + uv_off, + ((const pixel *) in->data[1 + pl]) + uv_off, + in->stride[1], data, cpw, + scaling[0], grain_lut[1 + pl], + bh, row, luma_src, in->stride[0], + pl, is_id HIGHBD_TAIL_SUFFIX); + } else { + for (int pl = 0; pl < 2; pl++) + if (data->num_uv_points[pl]) + dsp->fguv_32x32xn[in->p.layout - 1](((pixel *) out->data[1 + pl]) + uv_off, + ((const pixel *) in->data[1 + pl]) + uv_off, + in->stride[1], data, cpw, + scaling[1 + pl], grain_lut[1 + pl], + bh, row, luma_src, in->stride[0], + pl, is_id HIGHBD_TAIL_SUFFIX); + } +} + +void bitfn(dav1d_apply_grain)(const Dav1dFilmGrainDSPContext *const dsp, + Dav1dPicture *const out, + const Dav1dPicture *const in) +{ + ALIGN_STK_16(entry, grain_lut, 3,[GRAIN_HEIGHT + 1][GRAIN_WIDTH]); +#if ARCH_X86_64 && BITDEPTH == 8 + ALIGN_STK_64(uint8_t, scaling, 3,[SCALING_SIZE]); +#else + uint8_t scaling[3][SCALING_SIZE]; +#endif + const int rows = (out->p.h + 31) >> 5; + + bitfn(dav1d_prep_grain)(dsp, out, in, scaling, grain_lut); + for (int row = 0; row < rows; row++) + bitfn(dav1d_apply_grain_row)(dsp, out, in, scaling, grain_lut, row); +} +#endif diff --git a/third_party/dav1d/src/filmgrain.h b/third_party/dav1d/src/filmgrain.h new file mode 100644 index 0000000000..a5d6be6d44 --- /dev/null +++ b/third_party/dav1d/src/filmgrain.h @@ -0,0 +1,84 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_FILM_GRAIN_H +#define DAV1D_SRC_FILM_GRAIN_H + +#include "common/bitdepth.h" + +#include "src/levels.h" + +#define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 +#define BLOCK_SIZE 32 +#if !defined(BITDEPTH) || BITDEPTH == 8 +#define SCALING_SIZE 256 +typedef int8_t entry; +#else +#define SCALING_SIZE 4096 +typedef int16_t entry; +#endif + +#define decl_generate_grain_y_fn(name) \ +void (name)(entry buf[][GRAIN_WIDTH], \ + const Dav1dFilmGrainData *const data HIGHBD_DECL_SUFFIX) +typedef decl_generate_grain_y_fn(*generate_grain_y_fn); + +#define decl_generate_grain_uv_fn(name) \ +void (name)(entry buf[][GRAIN_WIDTH], \ + const entry buf_y[][GRAIN_WIDTH], \ + const Dav1dFilmGrainData *const data, const intptr_t uv HIGHBD_DECL_SUFFIX) +typedef decl_generate_grain_uv_fn(*generate_grain_uv_fn); + +#define decl_fgy_32x32xn_fn(name) \ +void (name)(pixel *dst_row, const pixel *src_row, ptrdiff_t stride, \ + const Dav1dFilmGrainData *data, \ + size_t pw, const uint8_t scaling[SCALING_SIZE], \ + const entry grain_lut[][GRAIN_WIDTH], \ + int bh, int row_num HIGHBD_DECL_SUFFIX) +typedef decl_fgy_32x32xn_fn(*fgy_32x32xn_fn); + +#define decl_fguv_32x32xn_fn(name) \ +void (name)(pixel *dst_row, const pixel *src_row, ptrdiff_t stride, \ + const Dav1dFilmGrainData *data, size_t pw, \ + const uint8_t scaling[SCALING_SIZE], \ + const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num, \ + const pixel *luma_row, ptrdiff_t luma_stride, \ + int uv_pl, int is_id HIGHBD_DECL_SUFFIX) +typedef decl_fguv_32x32xn_fn(*fguv_32x32xn_fn); + +typedef struct Dav1dFilmGrainDSPContext { + generate_grain_y_fn generate_grain_y; + generate_grain_uv_fn generate_grain_uv[3]; + + fgy_32x32xn_fn fgy_32x32xn; + fguv_32x32xn_fn fguv_32x32xn[3]; +} Dav1dFilmGrainDSPContext; + +bitfn_decls(void dav1d_film_grain_dsp_init, Dav1dFilmGrainDSPContext *c); + +#endif /* DAV1D_SRC_FILM_GRAIN_H */ diff --git a/third_party/dav1d/src/filmgrain_tmpl.c b/third_party/dav1d/src/filmgrain_tmpl.c new file mode 100644 index 0000000000..0986ac2a58 --- /dev/null +++ b/third_party/dav1d/src/filmgrain_tmpl.c @@ -0,0 +1,441 @@ +/* + * Copyright © 2018, Niklas Haas + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/filmgrain.h" +#include "src/tables.h" + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +static inline int get_random_number(const int bits, unsigned *const state) { + const int r = *state; + unsigned bit = ((r >> 0) ^ (r >> 1) ^ (r >> 3) ^ (r >> 12)) & 1; + *state = (r >> 1) | (bit << 15); + + return (*state >> (16 - bits)) & ((1 << bits) - 1); +} + +static inline int round2(const int x, const uint64_t shift) { + return (x + ((1 << shift) >> 1)) >> shift; +} + +static void generate_grain_y_c(entry buf[][GRAIN_WIDTH], + const Dav1dFilmGrainData *const data + HIGHBD_DECL_SUFFIX) +{ + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + unsigned seed = data->seed; + const int shift = 4 - bitdepth_min_8 + data->grain_scale_shift; + const int grain_ctr = 128 << bitdepth_min_8; + const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; + + for (int y = 0; y < GRAIN_HEIGHT; y++) { + for (int x = 0; x < GRAIN_WIDTH; x++) { + const int value = get_random_number(11, &seed); + buf[y][x] = round2(dav1d_gaussian_sequence[ value ], shift); + } + } + + const int ar_pad = 3; + const int ar_lag = data->ar_coeff_lag; + + for (int y = ar_pad; y < GRAIN_HEIGHT; y++) { + for (int x = ar_pad; x < GRAIN_WIDTH - ar_pad; x++) { + const int8_t *coeff = data->ar_coeffs_y; + int sum = 0; + for (int dy = -ar_lag; dy <= 0; dy++) { + for (int dx = -ar_lag; dx <= ar_lag; dx++) { + if (!dx && !dy) + break; + sum += *(coeff++) * buf[y + dy][x + dx]; + } + } + + const int grain = buf[y][x] + round2(sum, data->ar_coeff_shift); + buf[y][x] = iclip(grain, grain_min, grain_max); + } + } +} + +static NOINLINE void +generate_grain_uv_c(entry buf[][GRAIN_WIDTH], + const entry buf_y[][GRAIN_WIDTH], + const Dav1dFilmGrainData *const data, const intptr_t uv, + const int subx, const int suby HIGHBD_DECL_SUFFIX) +{ + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + unsigned seed = data->seed ^ (uv ? 0x49d8 : 0xb524); + const int shift = 4 - bitdepth_min_8 + data->grain_scale_shift; + const int grain_ctr = 128 << bitdepth_min_8; + const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; + + const int chromaW = subx ? SUB_GRAIN_WIDTH : GRAIN_WIDTH; + const int chromaH = suby ? SUB_GRAIN_HEIGHT : GRAIN_HEIGHT; + + for (int y = 0; y < chromaH; y++) { + for (int x = 0; x < chromaW; x++) { + const int value = get_random_number(11, &seed); + buf[y][x] = round2(dav1d_gaussian_sequence[ value ], shift); + } + } + + const int ar_pad = 3; + const int ar_lag = data->ar_coeff_lag; + + for (int y = ar_pad; y < chromaH; y++) { + for (int x = ar_pad; x < chromaW - ar_pad; x++) { + const int8_t *coeff = data->ar_coeffs_uv[uv]; + int sum = 0; + for (int dy = -ar_lag; dy <= 0; dy++) { + for (int dx = -ar_lag; dx <= ar_lag; dx++) { + // For the final (current) pixel, we need to add in the + // contribution from the luma grain texture + if (!dx && !dy) { + if (!data->num_y_points) + break; + int luma = 0; + const int lumaX = ((x - ar_pad) << subx) + ar_pad; + const int lumaY = ((y - ar_pad) << suby) + ar_pad; + for (int i = 0; i <= suby; i++) { + for (int j = 0; j <= subx; j++) { + luma += buf_y[lumaY + i][lumaX + j]; + } + } + luma = round2(luma, subx + suby); + sum += luma * (*coeff); + break; + } + + sum += *(coeff++) * buf[y + dy][x + dx]; + } + } + + const int grain = buf[y][x] + round2(sum, data->ar_coeff_shift); + buf[y][x] = iclip(grain, grain_min, grain_max); + } + } +} + +#define gnuv_ss_fn(nm, ss_x, ss_y) \ +static decl_generate_grain_uv_fn(generate_grain_uv_##nm##_c) { \ + generate_grain_uv_c(buf, buf_y, data, uv, ss_x, ss_y HIGHBD_TAIL_SUFFIX); \ +} + +gnuv_ss_fn(420, 1, 1); +gnuv_ss_fn(422, 1, 0); +gnuv_ss_fn(444, 0, 0); + +// samples from the correct block of a grain LUT, while taking into account the +// offsets provided by the offsets cache +static inline entry sample_lut(const entry grain_lut[][GRAIN_WIDTH], + const int offsets[2][2], const int subx, const int suby, + const int bx, const int by, const int x, const int y) +{ + const int randval = offsets[bx][by]; + const int offx = 3 + (2 >> subx) * (3 + (randval >> 4)); + const int offy = 3 + (2 >> suby) * (3 + (randval & 0xF)); + return grain_lut[offy + y + (BLOCK_SIZE >> suby) * by] + [offx + x + (BLOCK_SIZE >> subx) * bx]; +} + +static void fgy_32x32xn_c(pixel *const dst_row, const pixel *const src_row, + const ptrdiff_t stride, + const Dav1dFilmGrainData *const data, const size_t pw, + const uint8_t scaling[SCALING_SIZE], + const entry grain_lut[][GRAIN_WIDTH], + const int bh, const int row_num HIGHBD_DECL_SUFFIX) +{ + const int rows = 1 + (data->overlap_flag && row_num > 0); + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const int grain_ctr = 128 << bitdepth_min_8; + const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; + + int min_value, max_value; + if (data->clip_to_restricted_range) { + min_value = 16 << bitdepth_min_8; + max_value = 235 << bitdepth_min_8; + } else { + min_value = 0; + max_value = BITDEPTH_MAX; + } + + // seed[0] contains the current row, seed[1] contains the previous + unsigned seed[2]; + for (int i = 0; i < rows; i++) { + seed[i] = data->seed; + seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; + seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); + } + + assert(stride % (BLOCK_SIZE * sizeof(pixel)) == 0); + + int offsets[2 /* col offset */][2 /* row offset */]; + + // process this row in BLOCK_SIZE^2 blocks + for (unsigned bx = 0; bx < pw; bx += BLOCK_SIZE) { + const int bw = imin(BLOCK_SIZE, (int) pw - bx); + + if (data->overlap_flag && bx) { + // shift previous offsets left + for (int i = 0; i < rows; i++) + offsets[1][i] = offsets[0][i]; + } + + // update current offsets + for (int i = 0; i < rows; i++) + offsets[0][i] = get_random_number(8, &seed[i]); + + // x/y block offsets to compensate for overlapped regions + const int ystart = data->overlap_flag && row_num ? imin(2, bh) : 0; + const int xstart = data->overlap_flag && bx ? imin(2, bw) : 0; + + static const int w[2][2] = { { 27, 17 }, { 17, 27 } }; + +#define add_noise_y(x, y, grain) \ + const pixel *const src = src_row + (y) * PXSTRIDE(stride) + (x) + bx; \ + pixel *const dst = dst_row + (y) * PXSTRIDE(stride) + (x) + bx; \ + const int noise = round2(scaling[ *src ] * (grain), data->scaling_shift); \ + *dst = iclip(*src + noise, min_value, max_value); + + for (int y = ystart; y < bh; y++) { + // Non-overlapped image region (straightforward) + for (int x = xstart; x < bw; x++) { + int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); + add_noise_y(x, y, grain); + } + + // Special case for overlapped column + for (int x = 0; x < xstart; x++) { + int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); + int old = sample_lut(grain_lut, offsets, 0, 0, 1, 0, x, y); + grain = round2(old * w[x][0] + grain * w[x][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_y(x, y, grain); + } + } + + for (int y = 0; y < ystart; y++) { + // Special case for overlapped row (sans corner) + for (int x = xstart; x < bw; x++) { + int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); + int old = sample_lut(grain_lut, offsets, 0, 0, 0, 1, x, y); + grain = round2(old * w[y][0] + grain * w[y][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_y(x, y, grain); + } + + // Special case for doubly-overlapped corner + for (int x = 0; x < xstart; x++) { + // Blend the top pixel with the top left block + int top = sample_lut(grain_lut, offsets, 0, 0, 0, 1, x, y); + int old = sample_lut(grain_lut, offsets, 0, 0, 1, 1, x, y); + top = round2(old * w[x][0] + top * w[x][1], 5); + top = iclip(top, grain_min, grain_max); + + // Blend the current pixel with the left block + int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); + old = sample_lut(grain_lut, offsets, 0, 0, 1, 0, x, y); + grain = round2(old * w[x][0] + grain * w[x][1], 5); + grain = iclip(grain, grain_min, grain_max); + + // Mix the row rows together and apply grain + grain = round2(top * w[y][0] + grain * w[y][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_y(x, y, grain); + } + } + } +} + +static NOINLINE void +fguv_32x32xn_c(pixel *const dst_row, const pixel *const src_row, + const ptrdiff_t stride, const Dav1dFilmGrainData *const data, + const size_t pw, const uint8_t scaling[SCALING_SIZE], + const entry grain_lut[][GRAIN_WIDTH], const int bh, + const int row_num, const pixel *const luma_row, + const ptrdiff_t luma_stride, const int uv, const int is_id, + const int sx, const int sy HIGHBD_DECL_SUFFIX) +{ + const int rows = 1 + (data->overlap_flag && row_num > 0); + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const int grain_ctr = 128 << bitdepth_min_8; + const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; + + int min_value, max_value; + if (data->clip_to_restricted_range) { + min_value = 16 << bitdepth_min_8; + max_value = (is_id ? 235 : 240) << bitdepth_min_8; + } else { + min_value = 0; + max_value = BITDEPTH_MAX; + } + + // seed[0] contains the current row, seed[1] contains the previous + unsigned seed[2]; + for (int i = 0; i < rows; i++) { + seed[i] = data->seed; + seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; + seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); + } + + assert(stride % (BLOCK_SIZE * sizeof(pixel)) == 0); + + int offsets[2 /* col offset */][2 /* row offset */]; + + // process this row in BLOCK_SIZE^2 blocks (subsampled) + for (unsigned bx = 0; bx < pw; bx += BLOCK_SIZE >> sx) { + const int bw = imin(BLOCK_SIZE >> sx, (int)(pw - bx)); + if (data->overlap_flag && bx) { + // shift previous offsets left + for (int i = 0; i < rows; i++) + offsets[1][i] = offsets[0][i]; + } + + // update current offsets + for (int i = 0; i < rows; i++) + offsets[0][i] = get_random_number(8, &seed[i]); + + // x/y block offsets to compensate for overlapped regions + const int ystart = data->overlap_flag && row_num ? imin(2 >> sy, bh) : 0; + const int xstart = data->overlap_flag && bx ? imin(2 >> sx, bw) : 0; + + static const int w[2 /* sub */][2 /* off */][2] = { + { { 27, 17 }, { 17, 27 } }, + { { 23, 22 } }, + }; + +#define add_noise_uv(x, y, grain) \ + const int lx = (bx + x) << sx; \ + const int ly = y << sy; \ + const pixel *const luma = luma_row + ly * PXSTRIDE(luma_stride) + lx; \ + pixel avg = luma[0]; \ + if (sx) \ + avg = (avg + luma[1] + 1) >> 1; \ + const pixel *const src = src_row + (y) * PXSTRIDE(stride) + (bx + (x)); \ + pixel *const dst = dst_row + (y) * PXSTRIDE(stride) + (bx + (x)); \ + int val = avg; \ + if (!data->chroma_scaling_from_luma) { \ + const int combined = avg * data->uv_luma_mult[uv] + \ + *src * data->uv_mult[uv]; \ + val = iclip_pixel( (combined >> 6) + \ + (data->uv_offset[uv] * (1 << bitdepth_min_8)) ); \ + } \ + const int noise = round2(scaling[ val ] * (grain), data->scaling_shift); \ + *dst = iclip(*src + noise, min_value, max_value); + + for (int y = ystart; y < bh; y++) { + // Non-overlapped image region (straightforward) + for (int x = xstart; x < bw; x++) { + int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); + add_noise_uv(x, y, grain); + } + + // Special case for overlapped column + for (int x = 0; x < xstart; x++) { + int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); + int old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); + grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_uv(x, y, grain); + } + } + + for (int y = 0; y < ystart; y++) { + // Special case for overlapped row (sans corner) + for (int x = xstart; x < bw; x++) { + int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); + int old = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); + grain = round2(old * w[sy][y][0] + grain * w[sy][y][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_uv(x, y, grain); + } + + // Special case for doubly-overlapped corner + for (int x = 0; x < xstart; x++) { + // Blend the top pixel with the top left block + int top = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); + int old = sample_lut(grain_lut, offsets, sx, sy, 1, 1, x, y); + top = round2(old * w[sx][x][0] + top * w[sx][x][1], 5); + top = iclip(top, grain_min, grain_max); + + // Blend the current pixel with the left block + int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); + old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); + grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); + grain = iclip(grain, grain_min, grain_max); + + // Mix the row rows together and apply to image + grain = round2(top * w[sy][y][0] + grain * w[sy][y][1], 5); + grain = iclip(grain, grain_min, grain_max); + add_noise_uv(x, y, grain); + } + } + } +} + +#define fguv_ss_fn(nm, ss_x, ss_y) \ +static decl_fguv_32x32xn_fn(fguv_32x32xn_##nm##_c) { \ + fguv_32x32xn_c(dst_row, src_row, stride, data, pw, scaling, grain_lut, bh, \ + row_num, luma_row, luma_stride, uv_pl, is_id, ss_x, ss_y \ + HIGHBD_TAIL_SUFFIX); \ +} + +fguv_ss_fn(420, 1, 1); +fguv_ss_fn(422, 1, 0); +fguv_ss_fn(444, 0, 0); + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/filmgrain.h" +#elif ARCH_X86 +#include "src/x86/filmgrain.h" +#endif +#endif + +COLD void bitfn(dav1d_film_grain_dsp_init)(Dav1dFilmGrainDSPContext *const c) { + c->generate_grain_y = generate_grain_y_c; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = generate_grain_uv_420_c; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = generate_grain_uv_422_c; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = generate_grain_uv_444_c; + + c->fgy_32x32xn = fgy_32x32xn_c; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = fguv_32x32xn_420_c; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = fguv_32x32xn_422_c; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = fguv_32x32xn_444_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + film_grain_dsp_init_arm(c); +#elif ARCH_X86 + film_grain_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/getbits.c b/third_party/dav1d/src/getbits.c new file mode 100644 index 0000000000..673070be3d --- /dev/null +++ b/third_party/dav1d/src/getbits.c @@ -0,0 +1,178 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/getbits.h" + +void dav1d_init_get_bits(GetBits *const c, const uint8_t *const data, + const size_t sz) +{ + assert(sz); + c->ptr = c->ptr_start = data; + c->ptr_end = &c->ptr_start[sz]; + c->state = 0; + c->bits_left = 0; + c->error = 0; +} + +unsigned dav1d_get_bit(GetBits *const c) { + if (!c->bits_left) { + if (c->ptr >= c->ptr_end) { + c->error = 1; + } else { + const unsigned state = *c->ptr++; + c->bits_left = 7; + c->state = (uint64_t) state << 57; + return state >> 7; + } + } + + const uint64_t state = c->state; + c->bits_left--; + c->state = state << 1; + return (unsigned) (state >> 63); +} + +static inline void refill(GetBits *const c, const int n) { + assert(c->bits_left >= 0 && c->bits_left < 32); + unsigned state = 0; + do { + if (c->ptr >= c->ptr_end) { + c->error = 1; + if (state) break; + return; + } + state = (state << 8) | *c->ptr++; + c->bits_left += 8; + } while (n > c->bits_left); + c->state |= (uint64_t) state << (64 - c->bits_left); +} + +#define GET_BITS(name, type, type64) \ +type name(GetBits *const c, const int n) { \ + assert(n > 0 && n <= 32); \ + /* Unsigned cast avoids refill after eob */ \ + if ((unsigned) n > (unsigned) c->bits_left) \ + refill(c, n); \ + const uint64_t state = c->state; \ + c->bits_left -= n; \ + c->state = state << n; \ + return (type) ((type64) state >> (64 - n)); \ +} + +GET_BITS(dav1d_get_bits, unsigned, uint64_t) +GET_BITS(dav1d_get_sbits, int, int64_t) + +unsigned dav1d_get_uleb128(GetBits *const c) { + uint64_t val = 0; + unsigned i = 0, more; + + do { + const int v = dav1d_get_bits(c, 8); + more = v & 0x80; + val |= ((uint64_t) (v & 0x7F)) << i; + i += 7; + } while (more && i < 56); + + if (val > UINT_MAX || more) { + c->error = 1; + return 0; + } + + return (unsigned) val; +} + +unsigned dav1d_get_uniform(GetBits *const c, const unsigned max) { + // Output in range [0..max-1] + // max must be > 1, or else nothing is read from the bitstream + assert(max > 1); + const int l = ulog2(max) + 1; + assert(l > 1); + const unsigned m = (1U << l) - max; + const unsigned v = dav1d_get_bits(c, l - 1); + return v < m ? v : (v << 1) - m + dav1d_get_bit(c); +} + +unsigned dav1d_get_vlc(GetBits *const c) { + if (dav1d_get_bit(c)) + return 0; + + int n_bits = 0; + do { + if (++n_bits == 32) + return 0xFFFFFFFFU; + } while (!dav1d_get_bit(c)); + + return ((1U << n_bits) - 1) + dav1d_get_bits(c, n_bits); +} + +static unsigned get_bits_subexp_u(GetBits *const c, const unsigned ref, + const unsigned n) +{ + unsigned v = 0; + + for (int i = 0;; i++) { + const int b = i ? 3 + i - 1 : 3; + + if (n < v + 3 * (1 << b)) { + v += dav1d_get_uniform(c, n - v + 1); + break; + } + + if (!dav1d_get_bit(c)) { + v += dav1d_get_bits(c, b); + break; + } + + v += 1 << b; + } + + return ref * 2 <= n ? inv_recenter(ref, v) : n - inv_recenter(n - ref, v); +} + +int dav1d_get_bits_subexp(GetBits *const c, const int ref, const unsigned n) { + return (int) get_bits_subexp_u(c, ref + (1 << n), 2 << n) - (1 << n); +} + +void dav1d_bytealign_get_bits(GetBits *c) { + // bits_left is never more than 7, because it is only incremented + // by refill(), called by dav1d_get_bits and that never reads more + // than 7 bits more than it needs. + // + // If this wasn't true, we would need to work out how many bits to + // discard (bits_left % 8), subtract that from bits_left and then + // shift state right by that amount. + assert(c->bits_left <= 7); + + c->bits_left = 0; + c->state = 0; +} diff --git a/third_party/dav1d/src/getbits.h b/third_party/dav1d/src/getbits.h new file mode 100644 index 0000000000..57b80dc714 --- /dev/null +++ b/third_party/dav1d/src/getbits.h @@ -0,0 +1,59 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_GETBITS_H +#define DAV1D_SRC_GETBITS_H + +#include +#include + +typedef struct GetBits { + uint64_t state; + int bits_left, error; + const uint8_t *ptr, *ptr_start, *ptr_end; +} GetBits; + +void dav1d_init_get_bits(GetBits *c, const uint8_t *data, size_t sz); +unsigned dav1d_get_bit(GetBits *c); +unsigned dav1d_get_bits(GetBits *c, int n); +int dav1d_get_sbits(GetBits *c, int n); +unsigned dav1d_get_uleb128(GetBits *c); + +// Output in range 0..max-1 +unsigned dav1d_get_uniform(GetBits *c, unsigned max); +unsigned dav1d_get_vlc(GetBits *c); +int dav1d_get_bits_subexp(GetBits *c, int ref, unsigned n); + +// Discard bits from the buffer until we're next byte-aligned. +void dav1d_bytealign_get_bits(GetBits *c); + +// Return the current bit position relative to the start of the buffer. +static inline unsigned dav1d_get_bits_pos(const GetBits *c) { + return (unsigned) (c->ptr - c->ptr_start) * 8 - c->bits_left; +} + +#endif /* DAV1D_SRC_GETBITS_H */ diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h new file mode 100644 index 0000000000..b5fd1e18ef --- /dev/null +++ b/third_party/dav1d/src/internal.h @@ -0,0 +1,467 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_INTERNAL_H +#define DAV1D_SRC_INTERNAL_H + +#include + +#include "dav1d/data.h" + +typedef struct Dav1dFrameContext Dav1dFrameContext; +typedef struct Dav1dTileState Dav1dTileState; +typedef struct Dav1dTaskContext Dav1dTaskContext; +typedef struct Dav1dTask Dav1dTask; + +#include "common/attributes.h" + +#include "src/cdef.h" +#include "src/cdf.h" +#include "src/data.h" +#include "src/env.h" +#include "src/filmgrain.h" +#include "src/intra_edge.h" +#include "src/ipred.h" +#include "src/itx.h" +#include "src/levels.h" +#include "src/lf_mask.h" +#include "src/loopfilter.h" +#include "src/looprestoration.h" +#include "src/mc.h" +#include "src/msac.h" +#include "src/picture.h" +#include "src/recon.h" +#include "src/refmvs.h" +#include "src/thread.h" + +typedef struct Dav1dDSPContext { + Dav1dFilmGrainDSPContext fg; + Dav1dIntraPredDSPContext ipred; + Dav1dMCDSPContext mc; + Dav1dInvTxfmDSPContext itx; + Dav1dLoopFilterDSPContext lf; + Dav1dCdefDSPContext cdef; + Dav1dLoopRestorationDSPContext lr; +} Dav1dDSPContext; + +struct Dav1dTileGroup { + Dav1dData data; + int start, end; +}; + +enum TaskType { + DAV1D_TASK_TYPE_INIT, + DAV1D_TASK_TYPE_INIT_CDF, + DAV1D_TASK_TYPE_TILE_ENTROPY, + DAV1D_TASK_TYPE_ENTROPY_PROGRESS, + DAV1D_TASK_TYPE_TILE_RECONSTRUCTION, + DAV1D_TASK_TYPE_DEBLOCK_COLS, + DAV1D_TASK_TYPE_DEBLOCK_ROWS, + DAV1D_TASK_TYPE_CDEF, + DAV1D_TASK_TYPE_SUPER_RESOLUTION, + DAV1D_TASK_TYPE_LOOP_RESTORATION, + DAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS, + DAV1D_TASK_TYPE_FG_PREP, + DAV1D_TASK_TYPE_FG_APPLY, +}; + +struct Dav1dContext { + Dav1dFrameContext *fc; + unsigned n_fc; + + Dav1dTaskContext *tc; + unsigned n_tc; + + // cache of OBUs that make up a single frame before we submit them + // to a frame worker to be decoded + struct Dav1dTileGroup *tile; + int n_tile_data_alloc; + int n_tile_data; + int n_tiles; + Dav1dMemPool *seq_hdr_pool; + Dav1dRef *seq_hdr_ref; + Dav1dSequenceHeader *seq_hdr; + Dav1dMemPool *frame_hdr_pool; + Dav1dRef *frame_hdr_ref; + Dav1dFrameHeader *frame_hdr; + + Dav1dRef *content_light_ref; + Dav1dContentLightLevel *content_light; + Dav1dRef *mastering_display_ref; + Dav1dMasteringDisplay *mastering_display; + Dav1dRef *itut_t35_ref; + Dav1dITUTT35 *itut_t35; + + // decoded output picture queue + Dav1dData in; + Dav1dThreadPicture out, cache; + // dummy is a pointer to prevent compiler errors about atomic_load() + // not taking const arguments + atomic_int flush_mem, *flush; + struct { + Dav1dThreadPicture *out_delayed; + unsigned next; + } frame_thread; + + // task threading (refer to tc[] for per_thread thingies) + struct TaskThreadData { + pthread_mutex_t lock; + pthread_cond_t cond; + atomic_uint first; + unsigned cur; + // This is used for delayed reset of the task cur pointer when + // such operation is needed but the thread doesn't enter a critical + // section (typically when executing the next sbrow task locklessly). + // See src/thread_task.c:reset_task_cur(). + atomic_uint reset_task_cur; + atomic_int cond_signaled; + struct { + int exec; + pthread_cond_t cond; + const Dav1dPicture *in; + Dav1dPicture *out; + enum TaskType type; + atomic_int progress[2]; /* [0]=started, [1]=completed */ + union { + struct { + ALIGN(int8_t grain_lut_8bpc[3][GRAIN_HEIGHT + 1][GRAIN_WIDTH], 16); + ALIGN(uint8_t scaling_8bpc[3][256], 64); + }; + struct { + ALIGN(int16_t grain_lut_16bpc[3][GRAIN_HEIGHT + 1][GRAIN_WIDTH], 16); + ALIGN(uint8_t scaling_16bpc[3][4096], 64); + }; + }; + } delayed_fg; + int inited; + } task_thread; + + // reference/entropy state + Dav1dMemPool *segmap_pool; + Dav1dMemPool *refmvs_pool; + struct { + Dav1dThreadPicture p; + Dav1dRef *segmap; + Dav1dRef *refmvs; + unsigned refpoc[7]; + } refs[8]; + Dav1dMemPool *cdf_pool; + CdfThreadContext cdf[8]; + + Dav1dDSPContext dsp[3 /* 8, 10, 12 bits/component */]; + Dav1dRefmvsDSPContext refmvs_dsp; + + // tree to keep track of which edges are available + struct { + EdgeNode *root[2 /* BL_128X128 vs. BL_64X64 */]; + EdgeBranch branch_sb128[1 + 4 + 16 + 64]; + EdgeBranch branch_sb64[1 + 4 + 16]; + EdgeTip tip_sb128[256]; + EdgeTip tip_sb64[64]; + } intra_edge; + + Dav1dPicAllocator allocator; + int apply_grain; + int operating_point; + unsigned operating_point_idc; + int all_layers; + int max_spatial_id; + unsigned frame_size_limit; + int strict_std_compliance; + int output_invisible_frames; + enum Dav1dInloopFilterType inloop_filters; + enum Dav1dDecodeFrameType decode_frame_type; + int drain; + enum PictureFlags frame_flags; + enum Dav1dEventFlags event_flags; + Dav1dDataProps cached_error_props; + int cached_error; + + Dav1dLogger logger; + + Dav1dMemPool *picture_pool; +}; + +struct Dav1dTask { + unsigned frame_idx; // frame thread id + enum TaskType type; // task work + int sby; // sbrow + + // task dependencies + int recon_progress, deblock_progress; + int deps_skip; + struct Dav1dTask *next; // only used in task queue +}; + +struct Dav1dFrameContext { + Dav1dRef *seq_hdr_ref; + Dav1dSequenceHeader *seq_hdr; + Dav1dRef *frame_hdr_ref; + Dav1dFrameHeader *frame_hdr; + Dav1dThreadPicture refp[7]; + Dav1dPicture cur; // during block coding / reconstruction + Dav1dThreadPicture sr_cur; // after super-resolution upscaling + Dav1dRef *mvs_ref; + refmvs_temporal_block *mvs, *ref_mvs[7]; + Dav1dRef *ref_mvs_ref[7]; + Dav1dRef *cur_segmap_ref, *prev_segmap_ref; + uint8_t *cur_segmap; + const uint8_t *prev_segmap; + unsigned refpoc[7], refrefpoc[7][7]; + uint8_t gmv_warp_allowed[7]; + CdfThreadContext in_cdf, out_cdf; + struct Dav1dTileGroup *tile; + int n_tile_data_alloc; + int n_tile_data; + + // for scalable references + struct ScalableMotionParams { + int scale; // if no scaling, this is 0 + int step; + } svc[7][2 /* x, y */]; + int resize_step[2 /* y, uv */], resize_start[2 /* y, uv */]; + + const Dav1dContext *c; + Dav1dTileState *ts; + int n_ts; + const Dav1dDSPContext *dsp; + struct { + recon_b_intra_fn recon_b_intra; + recon_b_inter_fn recon_b_inter; + filter_sbrow_fn filter_sbrow; + filter_sbrow_fn filter_sbrow_deblock_cols; + filter_sbrow_fn filter_sbrow_deblock_rows; + void (*filter_sbrow_cdef)(Dav1dTaskContext *tc, int sby); + filter_sbrow_fn filter_sbrow_resize; + filter_sbrow_fn filter_sbrow_lr; + backup_ipred_edge_fn backup_ipred_edge; + read_coef_blocks_fn read_coef_blocks; + } bd_fn; + + int ipred_edge_sz; + pixel *ipred_edge[3]; + ptrdiff_t b4_stride; + int w4, h4, bw, bh, sb128w, sb128h, sbh, sb_shift, sb_step, sr_sb128w; + uint16_t dq[DAV1D_MAX_SEGMENTS][3 /* plane */][2 /* dc/ac */]; + const uint8_t *qm[N_RECT_TX_SIZES][3 /* plane */]; + BlockContext *a; + int a_sz /* w*tile_rows */; + refmvs_frame rf; + uint8_t jnt_weights[7][7]; + int bitdepth_max; + + struct { + int next_tile_row[2 /* 0: reconstruction, 1: entropy */]; + atomic_int entropy_progress; + atomic_int deblock_progress; // in sby units + atomic_uint *frame_progress, *copy_lpf_progress; + // indexed using t->by * f->b4_stride + t->bx + Av1Block *b; + struct CodedBlockInfo { + int16_t eob[3 /* plane */]; + uint8_t txtp[3 /* plane */]; + } *cbi; + // indexed using (t->by >> 1) * (f->b4_stride >> 1) + (t->bx >> 1) + uint16_t (*pal)[3 /* plane */][8 /* idx */]; + // iterated over inside tile state + uint8_t *pal_idx; + coef *cf; + int prog_sz; + int pal_sz, pal_idx_sz, cf_sz; + // start offsets per tile + int *tile_start_off; + } frame_thread; + + // loopfilter + struct { + uint8_t (*level)[4]; + Av1Filter *mask; + Av1Restoration *lr_mask; + int mask_sz /* w*h */, lr_mask_sz; + int cdef_buf_plane_sz[2]; /* stride*sbh*4 */ + int cdef_buf_sbh; + int lr_buf_plane_sz[2]; /* (stride*sbh*4) << sb128 if n_tc > 1, else stride*4 */ + int re_sz /* h */; + ALIGN(Av1FilterLUT lim_lut, 16); + int last_sharpness; + uint8_t lvl[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; + uint8_t *tx_lpf_right_edge[2]; + uint8_t *cdef_line_buf, *lr_line_buf; + pixel *cdef_line[2 /* pre, post */][3 /* plane */]; + pixel *cdef_lpf_line[3 /* plane */]; + pixel *lr_lpf_line[3 /* plane */]; + + // in-loop filter per-frame state keeping + uint8_t *start_of_tile_row; + int start_of_tile_row_sz; + int need_cdef_lpf_copy; + pixel *p[3], *sr_p[3]; + Av1Filter *mask_ptr, *prev_mask_ptr; + int restore_planes; // enum LrRestorePlanes + } lf; + + struct { + pthread_mutex_t lock; + pthread_cond_t cond; + struct TaskThreadData *ttd; + struct Dav1dTask *tasks, *tile_tasks[2], init_task; + int num_tasks, num_tile_tasks; + atomic_int init_done; + atomic_int done[2]; + int retval; + int update_set; // whether we need to update CDF reference + atomic_int error; + atomic_int task_counter; + struct Dav1dTask *task_head, *task_tail; + // Points to the task directly before the cur pointer in the queue. + // This cur pointer is theoretical here, we actually keep track of the + // "prev_t" variable. This is needed to not loose the tasks in + // [head;cur-1] when picking one for execution. + struct Dav1dTask *task_cur_prev; + struct { // async task insertion + atomic_int merge; + pthread_mutex_t lock; + Dav1dTask *head, *tail; + } pending_tasks; + } task_thread; + + // threading (refer to tc[] for per-thread things) + struct FrameTileThreadData { + int (*lowest_pixel_mem)[7][2]; + int lowest_pixel_mem_sz; + } tile_thread; +}; + +struct Dav1dTileState { + CdfContext cdf; + MsacContext msac; + + struct { + int col_start, col_end, row_start, row_end; // in 4px units + int col, row; // in tile units + } tiling; + + // in sby units, TILE_ERROR after a decoding error + atomic_int progress[2 /* 0: reconstruction, 1: entropy */]; + struct { + uint8_t *pal_idx; + coef *cf; + } frame_thread[2 /* 0: reconstruction, 1: entropy */]; + + // in fullpel units, [0] = Y, [1] = UV, used for progress requirements + // each entry is one tile-sbrow; middle index is refidx + int (*lowest_pixel)[7][2]; + + uint16_t dqmem[DAV1D_MAX_SEGMENTS][3 /* plane */][2 /* dc/ac */]; + const uint16_t (*dq)[3][2]; + int last_qidx; + + int8_t last_delta_lf[4]; + uint8_t lflvlmem[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; + const uint8_t (*lflvl)[4][8][2]; + + Av1RestorationUnit *lr_ref[3]; +}; + +struct Dav1dTaskContext { + const Dav1dContext *c; + const Dav1dFrameContext *f; + Dav1dTileState *ts; + int bx, by; + BlockContext l, *a; + refmvs_tile rt; + ALIGN(union, 64) { + int16_t cf_8bpc [32 * 32]; + int32_t cf_16bpc[32 * 32]; + }; + // FIXME types can be changed to pixel (and dynamically allocated) + // which would make copy/assign operations slightly faster? + uint16_t al_pal[2 /* a/l */][32 /* bx/y4 */][3 /* plane */][8 /* palette_idx */]; + uint8_t pal_sz_uv[2 /* a/l */][32 /* bx4/by4 */]; + uint8_t txtp_map[32 * 32]; // inter-only + ALIGN(union, 64) { + struct { + union { + uint8_t lap_8bpc [128 * 32]; + uint16_t lap_16bpc[128 * 32]; + struct { + int16_t compinter[2][128 * 128]; + uint8_t seg_mask[128 * 128]; + }; + }; + union { + // stride=192 for non-SVC, or 320 for SVC + uint8_t emu_edge_8bpc [320 * (256 + 7)]; + uint16_t emu_edge_16bpc[320 * (256 + 7)]; + }; + }; + struct { + union { + uint8_t levels[32 * 34]; + struct { + uint8_t pal_order[64][8]; + uint8_t pal_ctx[64]; + }; + }; + int16_t ac[32 * 32]; + uint8_t pal_idx[2 * 64 * 64]; + uint16_t pal[3 /* plane */][8 /* palette_idx */]; + ALIGN(union, 64) { + struct { + uint8_t interintra_8bpc[64 * 64]; + uint8_t edge_8bpc[257]; + }; + struct { + uint16_t interintra_16bpc[64 * 64]; + uint16_t edge_16bpc[257]; + }; + }; + }; + } scratch; + + Dav1dWarpedMotionParams warpmv; + Av1Filter *lf_mask; + int top_pre_cdef_toggle; + int8_t *cur_sb_cdef_idx_ptr; + // for chroma sub8x8, we need to know the filter for all 4 subblocks in + // a 4x4 area, but the top/left one can go out of cache already, so this + // keeps it accessible + enum Filter2d tl_4x4_filter; + + struct { + int pass; + } frame_thread; + struct { + struct thread_data td; + struct TaskThreadData *ttd; + struct FrameTileThreadData *fttd; + int flushed; + int die; + } task_thread; +}; + +#endif /* DAV1D_SRC_INTERNAL_H */ diff --git a/third_party/dav1d/src/intra_edge.c b/third_party/dav1d/src/intra_edge.c new file mode 100644 index 0000000000..684d113fa9 --- /dev/null +++ b/third_party/dav1d/src/intra_edge.c @@ -0,0 +1,165 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" + +#include "src/intra_edge.h" +#include "src/levels.h" + +struct ModeSelMem { + EdgeBranch *nwc[3 /* 64x64, 32x32, 16x16 */]; + EdgeTip *nt; +}; + +static void init_edges(EdgeNode *const node, + const enum BlockLevel bl, + const enum EdgeFlags edge_flags) +{ + node->o = edge_flags; + +#define ALL_FL(t) (EDGE_I444_##t | EDGE_I422_##t | EDGE_I420_##t) + if (bl == BL_8X8) { + EdgeTip *const nt = (EdgeTip *) node; + + node->h[0] = edge_flags | ALL_FL(LEFT_HAS_BOTTOM); + node->h[1] = edge_flags & (ALL_FL(LEFT_HAS_BOTTOM) | + EDGE_I420_TOP_HAS_RIGHT); + + node->v[0] = edge_flags | ALL_FL(TOP_HAS_RIGHT); + node->v[1] = edge_flags & (ALL_FL(TOP_HAS_RIGHT) | + EDGE_I420_LEFT_HAS_BOTTOM | + EDGE_I422_LEFT_HAS_BOTTOM); + + nt->split[0] = ALL_FL(TOP_HAS_RIGHT) | ALL_FL(LEFT_HAS_BOTTOM); + nt->split[1] = (edge_flags & ALL_FL(TOP_HAS_RIGHT)) | + EDGE_I422_LEFT_HAS_BOTTOM; + nt->split[2] = edge_flags | EDGE_I444_TOP_HAS_RIGHT; + nt->split[3] = edge_flags & (EDGE_I420_TOP_HAS_RIGHT | + EDGE_I420_LEFT_HAS_BOTTOM | + EDGE_I422_LEFT_HAS_BOTTOM); + } else { + EdgeBranch *const nwc = (EdgeBranch *) node; + + node->h[0] = edge_flags | ALL_FL(LEFT_HAS_BOTTOM); + node->h[1] = edge_flags & ALL_FL(LEFT_HAS_BOTTOM); + + node->v[0] = edge_flags | ALL_FL(TOP_HAS_RIGHT); + node->v[1] = edge_flags & ALL_FL(TOP_HAS_RIGHT); + + nwc->h4[0] = edge_flags | ALL_FL(LEFT_HAS_BOTTOM); + nwc->h4[1] = + nwc->h4[2] = ALL_FL(LEFT_HAS_BOTTOM); + nwc->h4[3] = edge_flags & ALL_FL(LEFT_HAS_BOTTOM); + if (bl == BL_16X16) + nwc->h4[1] |= edge_flags & EDGE_I420_TOP_HAS_RIGHT; + + nwc->v4[0] = edge_flags | ALL_FL(TOP_HAS_RIGHT); + nwc->v4[1] = + nwc->v4[2] = ALL_FL(TOP_HAS_RIGHT); + nwc->v4[3] = edge_flags & ALL_FL(TOP_HAS_RIGHT); + if (bl == BL_16X16) + nwc->v4[1] |= edge_flags & (EDGE_I420_LEFT_HAS_BOTTOM | + EDGE_I422_LEFT_HAS_BOTTOM); + + nwc->tls[0] = ALL_FL(TOP_HAS_RIGHT) | ALL_FL(LEFT_HAS_BOTTOM); + nwc->tls[1] = edge_flags & ALL_FL(LEFT_HAS_BOTTOM); + nwc->tls[2] = edge_flags & ALL_FL(TOP_HAS_RIGHT); + + nwc->trs[0] = edge_flags | ALL_FL(TOP_HAS_RIGHT); + nwc->trs[1] = edge_flags | ALL_FL(LEFT_HAS_BOTTOM); + nwc->trs[2] = 0; + + nwc->tts[0] = ALL_FL(TOP_HAS_RIGHT) | ALL_FL(LEFT_HAS_BOTTOM); + nwc->tts[1] = edge_flags & ALL_FL(TOP_HAS_RIGHT); + nwc->tts[2] = edge_flags & ALL_FL(LEFT_HAS_BOTTOM); + + nwc->tbs[0] = edge_flags | ALL_FL(LEFT_HAS_BOTTOM); + nwc->tbs[1] = edge_flags | ALL_FL(TOP_HAS_RIGHT); + nwc->tbs[2] = 0; + } +} + +static void init_mode_node(EdgeBranch *const nwc, + const enum BlockLevel bl, + struct ModeSelMem *const mem, + const int top_has_right, + const int left_has_bottom) +{ + init_edges(&nwc->node, bl, + (top_has_right ? ALL_FL(TOP_HAS_RIGHT) : 0) | + (left_has_bottom ? ALL_FL(LEFT_HAS_BOTTOM) : 0)); + if (bl == BL_16X16) { + for (int n = 0; n < 4; n++) { + EdgeTip *const nt = mem->nt++; + nwc->split[n] = &nt->node; + init_edges(&nt->node, bl + 1, + ((n == 3 || (n == 1 && !top_has_right)) ? 0 : + ALL_FL(TOP_HAS_RIGHT)) | + (!(n == 0 || (n == 2 && left_has_bottom)) ? 0 : + ALL_FL(LEFT_HAS_BOTTOM))); + } + } else { + for (int n = 0; n < 4; n++) { + EdgeBranch *const nwc_child = mem->nwc[bl]++; + nwc->split[n] = &nwc_child->node; + init_mode_node(nwc_child, bl + 1, mem, + !(n == 3 || (n == 1 && !top_has_right)), + n == 0 || (n == 2 && left_has_bottom)); + } + } +} + +void dav1d_init_mode_tree(EdgeNode *const root_node, EdgeTip *const nt, + const int allow_sb128) +{ + EdgeBranch *const root = (EdgeBranch *) root_node; + struct ModeSelMem mem; + mem.nt = nt; + + if (allow_sb128) { + mem.nwc[BL_128X128] = &root[1]; + mem.nwc[BL_64X64] = &root[1 + 4]; + mem.nwc[BL_32X32] = &root[1 + 4 + 16]; + init_mode_node(root, BL_128X128, &mem, 1, 0); + assert(mem.nwc[BL_128X128] == &root[1 + 4]); + assert(mem.nwc[BL_64X64] == &root[1 + 4 + 16]); + assert(mem.nwc[BL_32X32] == &root[1 + 4 + 16 + 64]); + assert(mem.nt == &nt[256]); + } else { + mem.nwc[BL_128X128] = NULL; + mem.nwc[BL_64X64] = &root[1]; + mem.nwc[BL_32X32] = &root[1 + 4]; + init_mode_node(root, BL_64X64, &mem, 1, 0); + assert(mem.nwc[BL_64X64] == &root[1 + 4]); + assert(mem.nwc[BL_32X32] == &root[1 + 4 + 16]); + assert(mem.nt == &nt[64]); + } +} diff --git a/third_party/dav1d/src/intra_edge.h b/third_party/dav1d/src/intra_edge.h new file mode 100644 index 0000000000..8b4e150181 --- /dev/null +++ b/third_party/dav1d/src/intra_edge.h @@ -0,0 +1,57 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_INTRA_EDGE_H +#define DAV1D_SRC_INTRA_EDGE_H + +enum EdgeFlags { + EDGE_I444_TOP_HAS_RIGHT = 1 << 0, + EDGE_I422_TOP_HAS_RIGHT = 1 << 1, + EDGE_I420_TOP_HAS_RIGHT = 1 << 2, + EDGE_I444_LEFT_HAS_BOTTOM = 1 << 3, + EDGE_I422_LEFT_HAS_BOTTOM = 1 << 4, + EDGE_I420_LEFT_HAS_BOTTOM = 1 << 5, +}; + +typedef struct EdgeNode EdgeNode; +struct EdgeNode { + enum EdgeFlags o, h[2], v[2]; +}; +typedef struct EdgeTip { + EdgeNode node; + enum EdgeFlags split[4]; +} EdgeTip; +typedef struct EdgeBranch { + EdgeNode node; + enum EdgeFlags tts[3], tbs[3], tls[3], trs[3], h4[4], v4[4]; + EdgeNode *split[4]; +} EdgeBranch; + +void dav1d_init_mode_tree(EdgeNode *const root, EdgeTip *const nt, + const int allow_sb128); + +#endif /* DAV1D_SRC_INTRA_EDGE_H */ diff --git a/third_party/dav1d/src/ipred.h b/third_party/dav1d/src/ipred.h new file mode 100644 index 0000000000..739ef1a266 --- /dev/null +++ b/third_party/dav1d/src/ipred.h @@ -0,0 +1,94 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_IPRED_H +#define DAV1D_SRC_IPRED_H + +#include + +#include "common/bitdepth.h" + +#include "src/levels.h" + +/* + * Intra prediction. + * - a is the angle (in degrees) for directional intra predictors. For other + * modes, it is ignored; + * - topleft is the same as the argument given to dav1d_prepare_intra_edges(), + * see ipred_prepare.h for more detailed documentation. + */ +#define decl_angular_ipred_fn(name) \ +void (name)(pixel *dst, ptrdiff_t stride, const pixel *topleft, \ + int width, int height, int angle, int max_width, int max_height \ + HIGHBD_DECL_SUFFIX) +typedef decl_angular_ipred_fn(*angular_ipred_fn); + +/* + * Create a subsampled Y plane with the DC subtracted. + * - w/h_pad is the edge of the width/height that extends outside the visible + * portion of the frame in 4px units; + * - ac has a stride of 16. + */ +#define decl_cfl_ac_fn(name) \ +void (name)(int16_t *ac, const pixel *y, ptrdiff_t stride, \ + int w_pad, int h_pad, int cw, int ch) +typedef decl_cfl_ac_fn(*cfl_ac_fn); + +/* + * dst[x,y] += alpha * ac[x,y] + * - alpha contains a q3 scalar in [-16,16] range; + */ +#define decl_cfl_pred_fn(name) \ +void (name)(pixel *dst, ptrdiff_t stride, const pixel *topleft, \ + int width, int height, const int16_t *ac, int alpha \ + HIGHBD_DECL_SUFFIX) +typedef decl_cfl_pred_fn(*cfl_pred_fn); + +/* + * dst[x,y] = pal[idx[x,y]] + * - palette indices are [0-7] + * - only 16-byte alignment is guaranteed for idx. + */ +#define decl_pal_pred_fn(name) \ +void (name)(pixel *dst, ptrdiff_t stride, const uint16_t *pal, \ + const uint8_t *idx, int w, int h) +typedef decl_pal_pred_fn(*pal_pred_fn); + +typedef struct Dav1dIntraPredDSPContext { + angular_ipred_fn intra_pred[N_IMPL_INTRA_PRED_MODES]; + + // chroma-from-luma + cfl_ac_fn cfl_ac[3 /* 420, 422, 444 */]; + cfl_pred_fn cfl_pred[DC_128_PRED + 1]; + + // palette + pal_pred_fn pal_pred; +} Dav1dIntraPredDSPContext; + +bitfn_decls(void dav1d_intra_pred_dsp_init, Dav1dIntraPredDSPContext *c); + +#endif /* DAV1D_SRC_IPRED_H */ diff --git a/third_party/dav1d/src/ipred_prepare.h b/third_party/dav1d/src/ipred_prepare.h new file mode 100644 index 0000000000..6a7efeb3d7 --- /dev/null +++ b/third_party/dav1d/src/ipred_prepare.h @@ -0,0 +1,108 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_IPRED_PREPARE_H +#define DAV1D_SRC_IPRED_PREPARE_H + +#include +#include + +#include "common/bitdepth.h" + +#include "src/env.h" +#include "src/intra_edge.h" +#include "src/levels.h" + +/* + * Luma intra edge preparation. + * + * x/y/start/w/h are in luma block (4px) units: + * - x and y are the absolute block positions in the image; + * - start/w/h are the *dependent tile* boundary positions. In practice, start + * is the horizontal tile start, w is the horizontal tile end, the vertical + * tile start is assumed to be 0 and h is the vertical image end. + * + * edge_flags signals which edges are available for this transform-block inside + * the given partition, as well as for the partition inside the superblock + * structure. + * + * dst and stride are pointers to the top/left position of the current block, + * and can be used to locate the top, left, top/left, top/right and bottom/left + * edge pointers also. + * + * angle is the angle_delta [-3..3] on input, and the absolute angle on output. + * + * mode is the intra prediction mode as coded in the bitstream. The return value + * is this same mode, converted to an index in the DSP functions. + * + * tw/th are the size of the transform block in block (4px) units. + * + * topleft_out is a pointer to scratch memory that will be filled with the edge + * pixels. The memory array should have space to be indexed in the [-2*w,2*w] + * range, in the following order: + * - [0] will be the top/left edge pixel; + * - [1..w] will be the top edge pixels (1 being left-most, w being right-most); + * - [w+1..2*w] will be the top/right edge pixels; + * - [-1..-w] will be the left edge pixels (-1 being top-most, -w being bottom- + * most); + * - [-w-1..-2*w] will be the bottom/left edge pixels. + * Each edge may remain uninitialized if it is not used by the returned mode + * index. If edges are not available (because the edge position is outside the + * tile dimensions or because edge_flags indicates lack of edge availability), + * they will be extended from nearby edges as defined by the av1 spec. + */ +enum IntraPredMode + bytefn(dav1d_prepare_intra_edges)(int x, int have_left, int y, int have_top, + int w, int h, enum EdgeFlags edge_flags, + const pixel *dst, ptrdiff_t stride, + const pixel *prefilter_toplevel_sb_edge, + enum IntraPredMode mode, int *angle, + int tw, int th, int filter_edge, + pixel *topleft_out HIGHBD_DECL_SUFFIX); + +// These flags are OR'd with the angle argument into intra predictors. +// ANGLE_USE_EDGE_FILTER_FLAG signals that edges should be convolved +// with a filter before using them to predict values in a block. +// ANGLE_SMOOTH_EDGE_FLAG means that edges are smooth and should use +// reduced filter strength. +#define ANGLE_USE_EDGE_FILTER_FLAG 1024 +#define ANGLE_SMOOTH_EDGE_FLAG 512 + +static inline int sm_flag(const BlockContext *const b, const int idx) { + if (!b->intra[idx]) return 0; + const enum IntraPredMode m = b->mode[idx]; + return (m == SMOOTH_PRED || m == SMOOTH_H_PRED || + m == SMOOTH_V_PRED) ? ANGLE_SMOOTH_EDGE_FLAG : 0; +} + +static inline int sm_uv_flag(const BlockContext *const b, const int idx) { + const enum IntraPredMode m = b->uvmode[idx]; + return (m == SMOOTH_PRED || m == SMOOTH_H_PRED || + m == SMOOTH_V_PRED) ? ANGLE_SMOOTH_EDGE_FLAG : 0; +} + +#endif /* DAV1D_SRC_IPRED_PREPARE_H */ diff --git a/third_party/dav1d/src/ipred_prepare_tmpl.c b/third_party/dav1d/src/ipred_prepare_tmpl.c new file mode 100644 index 0000000000..0bf9de9418 --- /dev/null +++ b/third_party/dav1d/src/ipred_prepare_tmpl.c @@ -0,0 +1,204 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/intops.h" + +#include "src/ipred_prepare.h" + +static const uint8_t av1_mode_conv[N_INTRA_PRED_MODES] + [2 /* have_left */][2 /* have_top */] = +{ + [DC_PRED] = { { DC_128_PRED, TOP_DC_PRED }, + { LEFT_DC_PRED, DC_PRED } }, + [PAETH_PRED] = { { DC_128_PRED, VERT_PRED }, + { HOR_PRED, PAETH_PRED } }, +}; + +static const uint8_t av1_mode_to_angle_map[8] = { + 90, 180, 45, 135, 113, 157, 203, 67 +}; + +static const struct { + uint8_t needs_left:1; + uint8_t needs_top:1; + uint8_t needs_topleft:1; + uint8_t needs_topright:1; + uint8_t needs_bottomleft:1; +} av1_intra_prediction_edges[N_IMPL_INTRA_PRED_MODES] = { + [DC_PRED] = { .needs_top = 1, .needs_left = 1 }, + [VERT_PRED] = { .needs_top = 1 }, + [HOR_PRED] = { .needs_left = 1 }, + [LEFT_DC_PRED] = { .needs_left = 1 }, + [TOP_DC_PRED] = { .needs_top = 1 }, + [DC_128_PRED] = { 0 }, + [Z1_PRED] = { .needs_top = 1, .needs_topright = 1, + .needs_topleft = 1 }, + [Z2_PRED] = { .needs_left = 1, .needs_top = 1, .needs_topleft = 1 }, + [Z3_PRED] = { .needs_left = 1, .needs_bottomleft = 1, + .needs_topleft = 1 }, + [SMOOTH_PRED] = { .needs_left = 1, .needs_top = 1 }, + [SMOOTH_V_PRED] = { .needs_left = 1, .needs_top = 1 }, + [SMOOTH_H_PRED] = { .needs_left = 1, .needs_top = 1 }, + [PAETH_PRED] = { .needs_left = 1, .needs_top = 1, .needs_topleft = 1 }, + [FILTER_PRED] = { .needs_left = 1, .needs_top = 1, .needs_topleft = 1 }, +}; + +enum IntraPredMode +bytefn(dav1d_prepare_intra_edges)(const int x, const int have_left, + const int y, const int have_top, + const int w, const int h, + const enum EdgeFlags edge_flags, + const pixel *const dst, + const ptrdiff_t stride, + const pixel *prefilter_toplevel_sb_edge, + enum IntraPredMode mode, int *const angle, + const int tw, const int th, const int filter_edge, + pixel *const topleft_out HIGHBD_DECL_SUFFIX) +{ + const int bitdepth = bitdepth_from_max(bitdepth_max); + assert(y < h && x < w); + + switch (mode) { + case VERT_PRED: + case HOR_PRED: + case DIAG_DOWN_LEFT_PRED: + case DIAG_DOWN_RIGHT_PRED: + case VERT_RIGHT_PRED: + case HOR_DOWN_PRED: + case HOR_UP_PRED: + case VERT_LEFT_PRED: { + *angle = av1_mode_to_angle_map[mode - VERT_PRED] + 3 * *angle; + + if (*angle <= 90) + mode = *angle < 90 && have_top ? Z1_PRED : VERT_PRED; + else if (*angle < 180) + mode = Z2_PRED; + else + mode = *angle > 180 && have_left ? Z3_PRED : HOR_PRED; + break; + } + case DC_PRED: + case PAETH_PRED: + mode = av1_mode_conv[mode][have_left][have_top]; + break; + default: + break; + } + + const pixel *dst_top; + if (have_top && + (av1_intra_prediction_edges[mode].needs_top || + av1_intra_prediction_edges[mode].needs_topleft || + (av1_intra_prediction_edges[mode].needs_left && !have_left))) + { + if (prefilter_toplevel_sb_edge) { + dst_top = &prefilter_toplevel_sb_edge[x * 4]; + } else { + dst_top = &dst[-PXSTRIDE(stride)]; + } + } + + if (av1_intra_prediction_edges[mode].needs_left) { + const int sz = th << 2; + pixel *const left = &topleft_out[-sz]; + + if (have_left) { + const int px_have = imin(sz, (h - y) << 2); + + for (int i = 0; i < px_have; i++) + left[sz - 1 - i] = dst[PXSTRIDE(stride) * i - 1]; + if (px_have < sz) + pixel_set(left, left[sz - px_have], sz - px_have); + } else { + pixel_set(left, have_top ? *dst_top : ((1 << bitdepth) >> 1) + 1, sz); + } + + if (av1_intra_prediction_edges[mode].needs_bottomleft) { + const int have_bottomleft = (!have_left || y + th >= h) ? 0 : + (edge_flags & EDGE_I444_LEFT_HAS_BOTTOM); + + if (have_bottomleft) { + const int px_have = imin(sz, (h - y - th) << 2); + + for (int i = 0; i < px_have; i++) + left[-(i + 1)] = dst[(sz + i) * PXSTRIDE(stride) - 1]; + if (px_have < sz) + pixel_set(left - sz, left[-px_have], sz - px_have); + } else { + pixel_set(left - sz, left[0], sz); + } + } + } + + if (av1_intra_prediction_edges[mode].needs_top) { + const int sz = tw << 2; + pixel *const top = &topleft_out[1]; + + if (have_top) { + const int px_have = imin(sz, (w - x) << 2); + pixel_copy(top, dst_top, px_have); + if (px_have < sz) + pixel_set(top + px_have, top[px_have - 1], sz - px_have); + } else { + pixel_set(top, have_left ? dst[-1] : ((1 << bitdepth) >> 1) - 1, sz); + } + + if (av1_intra_prediction_edges[mode].needs_topright) { + const int have_topright = (!have_top || x + tw >= w) ? 0 : + (edge_flags & EDGE_I444_TOP_HAS_RIGHT); + + if (have_topright) { + const int px_have = imin(sz, (w - x - tw) << 2); + + pixel_copy(top + sz, &dst_top[sz], px_have); + if (px_have < sz) + pixel_set(top + sz + px_have, top[sz + px_have - 1], + sz - px_have); + } else { + pixel_set(top + sz, top[sz - 1], sz); + } + } + } + + if (av1_intra_prediction_edges[mode].needs_topleft) { + if (have_left) + *topleft_out = have_top ? dst_top[-1] : dst[-1]; + else + *topleft_out = have_top ? *dst_top : (1 << bitdepth) >> 1; + + if (mode == Z2_PRED && tw + th >= 6 && filter_edge) + *topleft_out = ((topleft_out[-1] + topleft_out[1]) * 5 + + topleft_out[0] * 6 + 8) >> 4; + } + + return mode; +} diff --git a/third_party/dav1d/src/ipred_tmpl.c b/third_party/dav1d/src/ipred_tmpl.c new file mode 100644 index 0000000000..151d4842a0 --- /dev/null +++ b/third_party/dav1d/src/ipred_tmpl.c @@ -0,0 +1,771 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/ipred.h" +#include "src/tables.h" + +static NOINLINE void +splat_dc(pixel *dst, const ptrdiff_t stride, + const int width, const int height, const int dc HIGHBD_DECL_SUFFIX) +{ +#if BITDEPTH == 8 + assert(dc <= 0xff); + if (width > 4) { + const uint64_t dcN = dc * 0x0101010101010101ULL; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x += sizeof(dcN)) + *((uint64_t *) &dst[x]) = dcN; + dst += PXSTRIDE(stride); + } + } else { + const unsigned dcN = dc * 0x01010101U; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x += sizeof(dcN)) + *((unsigned *) &dst[x]) = dcN; + dst += PXSTRIDE(stride); + } + } +#else + assert(dc <= bitdepth_max); + const uint64_t dcN = dc * 0x0001000100010001ULL; + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x += sizeof(dcN) >> 1) + *((uint64_t *) &dst[x]) = dcN; + dst += PXSTRIDE(stride); + } +#endif +} + +static NOINLINE void +cfl_pred(pixel *dst, const ptrdiff_t stride, + const int width, const int height, const int dc, + const int16_t *ac, const int alpha HIGHBD_DECL_SUFFIX) +{ + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + const int diff = alpha * ac[x]; + dst[x] = iclip_pixel(dc + apply_sign((abs(diff) + 32) >> 6, diff)); + } + ac += width; + dst += PXSTRIDE(stride); + } +} + +static unsigned dc_gen_top(const pixel *const topleft, const int width) { + unsigned dc = width >> 1; + for (int i = 0; i < width; i++) + dc += topleft[1 + i]; + return dc >> ctz(width); +} + +static void ipred_dc_top_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + splat_dc(dst, stride, width, height, dc_gen_top(topleft, width) + HIGHBD_TAIL_SUFFIX); +} + +static void ipred_cfl_top_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, + const int16_t *ac, const int alpha + HIGHBD_DECL_SUFFIX) +{ + cfl_pred(dst, stride, width, height, dc_gen_top(topleft, width), ac, alpha + HIGHBD_TAIL_SUFFIX); +} + +static unsigned dc_gen_left(const pixel *const topleft, const int height) { + unsigned dc = height >> 1; + for (int i = 0; i < height; i++) + dc += topleft[-(1 + i)]; + return dc >> ctz(height); +} + +static void ipred_dc_left_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + splat_dc(dst, stride, width, height, dc_gen_left(topleft, height) + HIGHBD_TAIL_SUFFIX); +} + +static void ipred_cfl_left_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, + const int16_t *ac, const int alpha + HIGHBD_DECL_SUFFIX) +{ + const unsigned dc = dc_gen_left(topleft, height); + cfl_pred(dst, stride, width, height, dc, ac, alpha HIGHBD_TAIL_SUFFIX); +} + +#if BITDEPTH == 8 +#define MULTIPLIER_1x2 0x5556 +#define MULTIPLIER_1x4 0x3334 +#define BASE_SHIFT 16 +#else +#define MULTIPLIER_1x2 0xAAAB +#define MULTIPLIER_1x4 0x6667 +#define BASE_SHIFT 17 +#endif + +static unsigned dc_gen(const pixel *const topleft, + const int width, const int height) +{ + unsigned dc = (width + height) >> 1; + for (int i = 0; i < width; i++) + dc += topleft[i + 1]; + for (int i = 0; i < height; i++) + dc += topleft[-(i + 1)]; + dc >>= ctz(width + height); + + if (width != height) { + dc *= (width > height * 2 || height > width * 2) ? MULTIPLIER_1x4 : + MULTIPLIER_1x2; + dc >>= BASE_SHIFT; + } + return dc; +} + +static void ipred_dc_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + splat_dc(dst, stride, width, height, dc_gen(topleft, width, height) + HIGHBD_TAIL_SUFFIX); +} + +static void ipred_cfl_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, + const int16_t *ac, const int alpha + HIGHBD_DECL_SUFFIX) +{ + unsigned dc = dc_gen(topleft, width, height); + cfl_pred(dst, stride, width, height, dc, ac, alpha HIGHBD_TAIL_SUFFIX); +} + +#undef MULTIPLIER_1x2 +#undef MULTIPLIER_1x4 +#undef BASE_SHIFT + +static void ipred_dc_128_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ +#if BITDEPTH == 16 + const int dc = (bitdepth_max + 1) >> 1; +#else + const int dc = 128; +#endif + splat_dc(dst, stride, width, height, dc HIGHBD_TAIL_SUFFIX); +} + +static void ipred_cfl_128_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, + const int16_t *ac, const int alpha + HIGHBD_DECL_SUFFIX) +{ +#if BITDEPTH == 16 + const int dc = (bitdepth_max + 1) >> 1; +#else + const int dc = 128; +#endif + cfl_pred(dst, stride, width, height, dc, ac, alpha HIGHBD_TAIL_SUFFIX); +} + +static void ipred_v_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + for (int y = 0; y < height; y++) { + pixel_copy(dst, topleft + 1, width); + dst += PXSTRIDE(stride); + } +} + +static void ipred_h_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + for (int y = 0; y < height; y++) { + pixel_set(dst, topleft[-(1 + y)], width); + dst += PXSTRIDE(stride); + } +} + +static void ipred_paeth_c(pixel *dst, const ptrdiff_t stride, + const pixel *const tl_ptr, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int topleft = tl_ptr[0]; + for (int y = 0; y < height; y++) { + const int left = tl_ptr[-(y + 1)]; + for (int x = 0; x < width; x++) { + const int top = tl_ptr[1 + x]; + const int base = left + top - topleft; + const int ldiff = abs(left - base); + const int tdiff = abs(top - base); + const int tldiff = abs(topleft - base); + + dst[x] = ldiff <= tdiff && ldiff <= tldiff ? left : + tdiff <= tldiff ? top : topleft; + } + dst += PXSTRIDE(stride); + } +} + +static void ipred_smooth_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const uint8_t *const weights_hor = &dav1d_sm_weights[width]; + const uint8_t *const weights_ver = &dav1d_sm_weights[height]; + const int right = topleft[width], bottom = topleft[-height]; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + const int pred = weights_ver[y] * topleft[1 + x] + + (256 - weights_ver[y]) * bottom + + weights_hor[x] * topleft[-(1 + y)] + + (256 - weights_hor[x]) * right; + dst[x] = (pred + 256) >> 9; + } + dst += PXSTRIDE(stride); + } +} + +static void ipred_smooth_v_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const uint8_t *const weights_ver = &dav1d_sm_weights[height]; + const int bottom = topleft[-height]; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + const int pred = weights_ver[y] * topleft[1 + x] + + (256 - weights_ver[y]) * bottom; + dst[x] = (pred + 128) >> 8; + } + dst += PXSTRIDE(stride); + } +} + +static void ipred_smooth_h_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft, + const int width, const int height, const int a, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const uint8_t *const weights_hor = &dav1d_sm_weights[width]; + const int right = topleft[width]; + + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + const int pred = weights_hor[x] * topleft[-(y + 1)] + + (256 - weights_hor[x]) * right; + dst[x] = (pred + 128) >> 8; + } + dst += PXSTRIDE(stride); + } +} + +static NOINLINE int get_filter_strength(const int wh, const int angle, + const int is_sm) +{ + if (is_sm) { + if (wh <= 8) { + if (angle >= 64) return 2; + if (angle >= 40) return 1; + } else if (wh <= 16) { + if (angle >= 48) return 2; + if (angle >= 20) return 1; + } else if (wh <= 24) { + if (angle >= 4) return 3; + } else { + return 3; + } + } else { + if (wh <= 8) { + if (angle >= 56) return 1; + } else if (wh <= 16) { + if (angle >= 40) return 1; + } else if (wh <= 24) { + if (angle >= 32) return 3; + if (angle >= 16) return 2; + if (angle >= 8) return 1; + } else if (wh <= 32) { + if (angle >= 32) return 3; + if (angle >= 4) return 2; + return 1; + } else { + return 3; + } + } + return 0; +} + +static NOINLINE void filter_edge(pixel *const out, const int sz, + const int lim_from, const int lim_to, + const pixel *const in, const int from, + const int to, const int strength) +{ + static const uint8_t kernel[3][5] = { + { 0, 4, 8, 4, 0 }, + { 0, 5, 6, 5, 0 }, + { 2, 4, 4, 4, 2 } + }; + + assert(strength > 0); + int i = 0; + for (; i < imin(sz, lim_from); i++) + out[i] = in[iclip(i, from, to - 1)]; + for (; i < imin(lim_to, sz); i++) { + int s = 0; + for (int j = 0; j < 5; j++) + s += in[iclip(i - 2 + j, from, to - 1)] * kernel[strength - 1][j]; + out[i] = (s + 8) >> 4; + } + for (; i < sz; i++) + out[i] = in[iclip(i, from, to - 1)]; +} + +static inline int get_upsample(const int wh, const int angle, const int is_sm) { + return angle < 40 && wh <= 16 >> is_sm; +} + +static NOINLINE void upsample_edge(pixel *const out, const int hsz, + const pixel *const in, const int from, + const int to HIGHBD_DECL_SUFFIX) +{ + static const int8_t kernel[4] = { -1, 9, 9, -1 }; + int i; + for (i = 0; i < hsz - 1; i++) { + out[i * 2] = in[iclip(i, from, to - 1)]; + + int s = 0; + for (int j = 0; j < 4; j++) + s += in[iclip(i + j - 1, from, to - 1)] * kernel[j]; + out[i * 2 + 1] = iclip_pixel((s + 8) >> 4); + } + out[i * 2] = in[iclip(i, from, to - 1)]; +} + +static void ipred_z1_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + assert(angle < 90); + int dx = dav1d_dr_intra_derivative[angle >> 1]; + pixel top_out[64 + 64]; + const pixel *top; + int max_base_x; + const int upsample_above = enable_intra_edge_filter ? + get_upsample(width + height, 90 - angle, is_sm) : 0; + if (upsample_above) { + upsample_edge(top_out, width + height, &topleft_in[1], -1, + width + imin(width, height) HIGHBD_TAIL_SUFFIX); + top = top_out; + max_base_x = 2 * (width + height) - 2; + dx <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, 90 - angle, is_sm) : 0; + if (filter_strength) { + filter_edge(top_out, width + height, 0, width + height, + &topleft_in[1], -1, width + imin(width, height), + filter_strength); + top = top_out; + max_base_x = width + height - 1; + } else { + top = &topleft_in[1]; + max_base_x = width + imin(width, height) - 1; + } + } + const int base_inc = 1 + upsample_above; + for (int y = 0, xpos = dx; y < height; + y++, dst += PXSTRIDE(stride), xpos += dx) + { + const int frac = xpos & 0x3E; + + for (int x = 0, base = xpos >> 6; x < width; x++, base += base_inc) { + if (base < max_base_x) { + const int v = top[base] * (64 - frac) + top[base + 1] * frac; + dst[x] = (v + 32) >> 6; + } else { + pixel_set(&dst[x], top[max_base_x], width - x); + break; + } + } + } +} + +static void ipred_z2_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + assert(angle > 90 && angle < 180); + int dy = dav1d_dr_intra_derivative[(angle - 90) >> 1]; + int dx = dav1d_dr_intra_derivative[(180 - angle) >> 1]; + const int upsample_left = enable_intra_edge_filter ? + get_upsample(width + height, 180 - angle, is_sm) : 0; + const int upsample_above = enable_intra_edge_filter ? + get_upsample(width + height, angle - 90, is_sm) : 0; + pixel edge[64 + 64 + 1]; + pixel *const topleft = &edge[64]; + + if (upsample_above) { + upsample_edge(topleft, width + 1, topleft_in, 0, width + 1 + HIGHBD_TAIL_SUFFIX); + dx <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, angle - 90, is_sm) : 0; + + if (filter_strength) { + filter_edge(&topleft[1], width, 0, max_width, + &topleft_in[1], -1, width, + filter_strength); + } else { + pixel_copy(&topleft[1], &topleft_in[1], width); + } + } + if (upsample_left) { + upsample_edge(&topleft[-height * 2], height + 1, &topleft_in[-height], + 0, height + 1 HIGHBD_TAIL_SUFFIX); + dy <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, 180 - angle, is_sm) : 0; + + if (filter_strength) { + filter_edge(&topleft[-height], height, height - max_height, height, + &topleft_in[-height], + 0, height + 1, filter_strength); + } else { + pixel_copy(&topleft[-height], &topleft_in[-height], height); + } + } + *topleft = *topleft_in; + + const int base_inc_x = 1 + upsample_above; + const pixel *const left = &topleft[-(1 + upsample_left)]; + for (int y = 0, xpos = ((1 + upsample_above) << 6) - dx; y < height; + y++, xpos -= dx, dst += PXSTRIDE(stride)) + { + int base_x = xpos >> 6; + const int frac_x = xpos & 0x3E; + + for (int x = 0, ypos = (y << (6 + upsample_left)) - dy; x < width; + x++, base_x += base_inc_x, ypos -= dy) + { + int v; + if (base_x >= 0) { + v = topleft[base_x] * (64 - frac_x) + + topleft[base_x + 1] * frac_x; + } else { + const int base_y = ypos >> 6; + assert(base_y >= -(1 + upsample_left)); + const int frac_y = ypos & 0x3E; + v = left[-base_y] * (64 - frac_y) + + left[-(base_y + 1)] * frac_y; + } + dst[x] = (v + 32) >> 6; + } + } +} + +static void ipred_z3_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + assert(angle > 180); + int dy = dav1d_dr_intra_derivative[(270 - angle) >> 1]; + pixel left_out[64 + 64]; + const pixel *left; + int max_base_y; + const int upsample_left = enable_intra_edge_filter ? + get_upsample(width + height, angle - 180, is_sm) : 0; + if (upsample_left) { + upsample_edge(left_out, width + height, + &topleft_in[-(width + height)], + imax(width - height, 0), width + height + 1 + HIGHBD_TAIL_SUFFIX); + left = &left_out[2 * (width + height) - 2]; + max_base_y = 2 * (width + height) - 2; + dy <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, angle - 180, is_sm) : 0; + + if (filter_strength) { + filter_edge(left_out, width + height, 0, width + height, + &topleft_in[-(width + height)], + imax(width - height, 0), width + height + 1, + filter_strength); + left = &left_out[width + height - 1]; + max_base_y = width + height - 1; + } else { + left = &topleft_in[-1]; + max_base_y = height + imin(width, height) - 1; + } + } + const int base_inc = 1 + upsample_left; + for (int x = 0, ypos = dy; x < width; x++, ypos += dy) { + const int frac = ypos & 0x3E; + + for (int y = 0, base = ypos >> 6; y < height; y++, base += base_inc) { + if (base < max_base_y) { + const int v = left[-base] * (64 - frac) + + left[-(base + 1)] * frac; + dst[y * PXSTRIDE(stride) + x] = (v + 32) >> 6; + } else { + do { + dst[y * PXSTRIDE(stride) + x] = left[-max_base_y]; + } while (++y < height); + break; + } + } + } +} + +#if ARCH_X86 +#define FILTER(flt_ptr, p0, p1, p2, p3, p4, p5, p6) \ + flt_ptr[ 0] * p0 + flt_ptr[ 1] * p1 + \ + flt_ptr[16] * p2 + flt_ptr[17] * p3 + \ + flt_ptr[32] * p4 + flt_ptr[33] * p5 + \ + flt_ptr[48] * p6 +#define FLT_INCR 2 +#else +#define FILTER(flt_ptr, p0, p1, p2, p3, p4, p5, p6) \ + flt_ptr[ 0] * p0 + flt_ptr[ 8] * p1 + \ + flt_ptr[16] * p2 + flt_ptr[24] * p3 + \ + flt_ptr[32] * p4 + flt_ptr[40] * p5 + \ + flt_ptr[48] * p6 +#define FLT_INCR 1 +#endif + +/* Up to 32x32 only */ +static void ipred_filter_c(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int filt_idx, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + filt_idx &= 511; + assert(filt_idx < 5); + + const int8_t *const filter = dav1d_filter_intra_taps[filt_idx]; + const pixel *top = &topleft_in[1]; + for (int y = 0; y < height; y += 2) { + const pixel *topleft = &topleft_in[-y]; + const pixel *left = &topleft[-1]; + ptrdiff_t left_stride = -1; + for (int x = 0; x < width; x += 4) { + const int p0 = *topleft; + const int p1 = top[0], p2 = top[1], p3 = top[2], p4 = top[3]; + const int p5 = left[0 * left_stride], p6 = left[1 * left_stride]; + pixel *ptr = &dst[x]; + const int8_t *flt_ptr = filter; + + for (int yy = 0; yy < 2; yy++) { + for (int xx = 0; xx < 4; xx++, flt_ptr += FLT_INCR) { + const int acc = FILTER(flt_ptr, p0, p1, p2, p3, p4, p5, p6); + ptr[xx] = iclip_pixel((acc + 8) >> 4); + } + ptr += PXSTRIDE(stride); + } + left = &dst[x + 4 - 1]; + left_stride = PXSTRIDE(stride); + top += 4; + topleft = &top[-1]; + } + top = &dst[PXSTRIDE(stride)]; + dst = &dst[PXSTRIDE(stride) * 2]; + } +} + +static NOINLINE void +cfl_ac_c(int16_t *ac, const pixel *ypx, const ptrdiff_t stride, + const int w_pad, const int h_pad, const int width, const int height, + const int ss_hor, const int ss_ver) +{ + int y, x; + int16_t *const ac_orig = ac; + + assert(w_pad >= 0 && w_pad * 4 < width); + assert(h_pad >= 0 && h_pad * 4 < height); + + for (y = 0; y < height - 4 * h_pad; y++) { + for (x = 0; x < width - 4 * w_pad; x++) { + int ac_sum = ypx[x << ss_hor]; + if (ss_hor) ac_sum += ypx[x * 2 + 1]; + if (ss_ver) { + ac_sum += ypx[(x << ss_hor) + PXSTRIDE(stride)]; + if (ss_hor) ac_sum += ypx[x * 2 + 1 + PXSTRIDE(stride)]; + } + ac[x] = ac_sum << (1 + !ss_ver + !ss_hor); + } + for (; x < width; x++) + ac[x] = ac[x - 1]; + ac += width; + ypx += PXSTRIDE(stride) << ss_ver; + } + for (; y < height; y++) { + memcpy(ac, &ac[-width], width * sizeof(*ac)); + ac += width; + } + + const int log2sz = ctz(width) + ctz(height); + int sum = (1 << log2sz) >> 1; + for (ac = ac_orig, y = 0; y < height; y++) { + for (x = 0; x < width; x++) + sum += ac[x]; + ac += width; + } + sum >>= log2sz; + + // subtract DC + for (ac = ac_orig, y = 0; y < height; y++) { + for (x = 0; x < width; x++) + ac[x] -= sum; + ac += width; + } +} + +#define cfl_ac_fn(fmt, ss_hor, ss_ver) \ +static void cfl_ac_##fmt##_c(int16_t *const ac, const pixel *const ypx, \ + const ptrdiff_t stride, const int w_pad, \ + const int h_pad, const int cw, const int ch) \ +{ \ + cfl_ac_c(ac, ypx, stride, w_pad, h_pad, cw, ch, ss_hor, ss_ver); \ +} + +cfl_ac_fn(420, 1, 1) +cfl_ac_fn(422, 1, 0) +cfl_ac_fn(444, 0, 0) + +static void pal_pred_c(pixel *dst, const ptrdiff_t stride, + const uint16_t *const pal, const uint8_t *idx, + const int w, const int h) +{ + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) + dst[x] = (pixel) pal[idx[x]]; + idx += w; + dst += PXSTRIDE(stride); + } +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/ipred.h" +#elif ARCH_X86 +#include "src/x86/ipred.h" +#endif +#endif + +COLD void bitfn(dav1d_intra_pred_dsp_init)(Dav1dIntraPredDSPContext *const c) { + c->intra_pred[DC_PRED ] = ipred_dc_c; + c->intra_pred[DC_128_PRED ] = ipred_dc_128_c; + c->intra_pred[TOP_DC_PRED ] = ipred_dc_top_c; + c->intra_pred[LEFT_DC_PRED ] = ipred_dc_left_c; + c->intra_pred[HOR_PRED ] = ipred_h_c; + c->intra_pred[VERT_PRED ] = ipred_v_c; + c->intra_pred[PAETH_PRED ] = ipred_paeth_c; + c->intra_pred[SMOOTH_PRED ] = ipred_smooth_c; + c->intra_pred[SMOOTH_V_PRED] = ipred_smooth_v_c; + c->intra_pred[SMOOTH_H_PRED] = ipred_smooth_h_c; + c->intra_pred[Z1_PRED ] = ipred_z1_c; + c->intra_pred[Z2_PRED ] = ipred_z2_c; + c->intra_pred[Z3_PRED ] = ipred_z3_c; + c->intra_pred[FILTER_PRED ] = ipred_filter_c; + + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = cfl_ac_420_c; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = cfl_ac_422_c; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I444 - 1] = cfl_ac_444_c; + + c->cfl_pred[DC_PRED ] = ipred_cfl_c; + c->cfl_pred[DC_128_PRED ] = ipred_cfl_128_c; + c->cfl_pred[TOP_DC_PRED ] = ipred_cfl_top_c; + c->cfl_pred[LEFT_DC_PRED] = ipred_cfl_left_c; + + c->pal_pred = pal_pred_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + intra_pred_dsp_init_arm(c); +#elif ARCH_X86 + intra_pred_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/itx.h b/third_party/dav1d/src/itx.h new file mode 100644 index 0000000000..d522079907 --- /dev/null +++ b/third_party/dav1d/src/itx.h @@ -0,0 +1,48 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_ITX_H +#define DAV1D_SRC_ITX_H + +#include + +#include "common/bitdepth.h" + +#include "src/levels.h" + +#define decl_itx_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob \ + HIGHBD_DECL_SUFFIX) +typedef decl_itx_fn(*itxfm_fn); + +typedef struct Dav1dInvTxfmDSPContext { + itxfm_fn itxfm_add[N_RECT_TX_SIZES][N_TX_TYPES_PLUS_LL]; +} Dav1dInvTxfmDSPContext; + +bitfn_decls(void dav1d_itx_dsp_init, Dav1dInvTxfmDSPContext *c, int bpc); + +#endif /* DAV1D_SRC_ITX_H */ diff --git a/third_party/dav1d/src/itx_1d.c b/third_party/dav1d/src/itx_1d.c new file mode 100644 index 0000000000..ca14fc8c41 --- /dev/null +++ b/third_party/dav1d/src/itx_1d.c @@ -0,0 +1,1034 @@ +/* + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/intops.h" + +#include "src/itx_1d.h" + +#define CLIP(a) iclip(a, min, max) + +/* + * In some places, we use the pattern like this: + * t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; + * even though the reference code might use something like: + * t2 = (in1 * 1567 - in3 * 3784 + 2048) >> 12; + * + * The reason for this is that for 12 bits/component bitstreams (corrupt/ + * invalid ones, but they are codable nonetheless), each coefficient or + * input can be 19(+sign) bits, and therefore if the combination of the + * two multipliers (each 12 bits) is >= 4096, the result of the add/sub + * after the pair of multiplies will exceed the 31+sign bit range. Signed + * integer overflows are UB in C, and we'd like to prevent that. + * + * To workaround this, we invert one of the two coefficients (or, if both are + * multiples of 2, we reduce their magnitude by one bit). It should be noted + * that SIMD implementations do not have to follow this exact behaviour. The + * AV1 spec clearly states that the result of the multiply/add pairs should + * fit in 31+sign bit intermediates, and that streams violating this convention + * are not AV1-compliant. So, as long as we don't trigger UB (which some people + * would consider a security vulnerability), we're fine. So, SIMD can simply + * use the faster implementation, even if that might in some cases result in + * integer overflows, since these are not considered valid AV1 anyway, and in + * e.g. x86 assembly, integer overflows are not considered UB, but they merely + * wrap around. + */ + +static NOINLINE void +inv_dct4_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) +{ + assert(stride > 0); + const int in0 = c[0 * stride], in1 = c[1 * stride]; + + int t0, t1, t2, t3; + if (tx64) { + t0 = t1 = (in0 * 181 + 128) >> 8; + t2 = (in1 * 1567 + 2048) >> 12; + t3 = (in1 * 3784 + 2048) >> 12; + } else { + const int in2 = c[2 * stride], in3 = c[3 * stride]; + + t0 = ((in0 + in2) * 181 + 128) >> 8; + t1 = ((in0 - in2) * 181 + 128) >> 8; + t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; + t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1; + } + + c[0 * stride] = CLIP(t0 + t3); + c[1 * stride] = CLIP(t1 + t2); + c[2 * stride] = CLIP(t1 - t2); + c[3 * stride] = CLIP(t0 - t3); +} + +void dav1d_inv_dct4_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct4_1d_internal_c(c, stride, min, max, 0); +} + +static NOINLINE void +inv_dct8_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) +{ + assert(stride > 0); + inv_dct4_1d_internal_c(c, stride << 1, min, max, tx64); + + const int in1 = c[1 * stride], in3 = c[3 * stride]; + + int t4a, t5a, t6a, t7a; + if (tx64) { + t4a = (in1 * 799 + 2048) >> 12; + t5a = (in3 * -2276 + 2048) >> 12; + t6a = (in3 * 3406 + 2048) >> 12; + t7a = (in1 * 4017 + 2048) >> 12; + } else { + const int in5 = c[5 * stride], in7 = c[7 * stride]; + + t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7; + t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; + t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; + t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; + } + + const int t4 = CLIP(t4a + t5a); + t5a = CLIP(t4a - t5a); + const int t7 = CLIP(t7a + t6a); + t6a = CLIP(t7a - t6a); + + const int t5 = ((t6a - t5a) * 181 + 128) >> 8; + const int t6 = ((t6a + t5a) * 181 + 128) >> 8; + + const int t0 = c[0 * stride]; + const int t1 = c[2 * stride]; + const int t2 = c[4 * stride]; + const int t3 = c[6 * stride]; + + c[0 * stride] = CLIP(t0 + t7); + c[1 * stride] = CLIP(t1 + t6); + c[2 * stride] = CLIP(t2 + t5); + c[3 * stride] = CLIP(t3 + t4); + c[4 * stride] = CLIP(t3 - t4); + c[5 * stride] = CLIP(t2 - t5); + c[6 * stride] = CLIP(t1 - t6); + c[7 * stride] = CLIP(t0 - t7); +} + +void dav1d_inv_dct8_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct8_1d_internal_c(c, stride, min, max, 0); +} + +static NOINLINE void +inv_dct16_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, int tx64) +{ + assert(stride > 0); + inv_dct8_1d_internal_c(c, stride << 1, min, max, tx64); + + const int in1 = c[1 * stride], in3 = c[3 * stride]; + const int in5 = c[5 * stride], in7 = c[7 * stride]; + + int t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a; + if (tx64) { + t8a = (in1 * 401 + 2048) >> 12; + t9a = (in7 * -2598 + 2048) >> 12; + t10a = (in5 * 1931 + 2048) >> 12; + t11a = (in3 * -1189 + 2048) >> 12; + t12a = (in3 * 3920 + 2048) >> 12; + t13a = (in5 * 3612 + 2048) >> 12; + t14a = (in7 * 3166 + 2048) >> 12; + t15a = (in1 * 4076 + 2048) >> 12; + } else { + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; + + t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15; + t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11; + t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11; + t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13; + t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3; + t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5; + t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11; + t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1; + } + + int t8 = CLIP(t8a + t9a); + int t9 = CLIP(t8a - t9a); + int t10 = CLIP(t11a - t10a); + int t11 = CLIP(t11a + t10a); + int t12 = CLIP(t12a + t13a); + int t13 = CLIP(t12a - t13a); + int t14 = CLIP(t15a - t14a); + int t15 = CLIP(t15a + t14a); + + t9a = (( t14 * 1567 - t9 * (3784 - 4096) + 2048) >> 12) - t9; + t14a = (( t14 * (3784 - 4096) + t9 * 1567 + 2048) >> 12) + t14; + t10a = ((-(t13 * (3784 - 4096) + t10 * 1567) + 2048) >> 12) - t13; + t13a = (( t13 * 1567 - t10 * (3784 - 4096) + 2048) >> 12) - t10; + + t8a = CLIP(t8 + t11); + t9 = CLIP(t9a + t10a); + t10 = CLIP(t9a - t10a); + t11a = CLIP(t8 - t11); + t12a = CLIP(t15 - t12); + t13 = CLIP(t14a - t13a); + t14 = CLIP(t14a + t13a); + t15a = CLIP(t15 + t12); + + t10a = ((t13 - t10) * 181 + 128) >> 8; + t13a = ((t13 + t10) * 181 + 128) >> 8; + t11 = ((t12a - t11a) * 181 + 128) >> 8; + t12 = ((t12a + t11a) * 181 + 128) >> 8; + + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + + c[ 0 * stride] = CLIP(t0 + t15a); + c[ 1 * stride] = CLIP(t1 + t14); + c[ 2 * stride] = CLIP(t2 + t13a); + c[ 3 * stride] = CLIP(t3 + t12); + c[ 4 * stride] = CLIP(t4 + t11); + c[ 5 * stride] = CLIP(t5 + t10a); + c[ 6 * stride] = CLIP(t6 + t9); + c[ 7 * stride] = CLIP(t7 + t8a); + c[ 8 * stride] = CLIP(t7 - t8a); + c[ 9 * stride] = CLIP(t6 - t9); + c[10 * stride] = CLIP(t5 - t10a); + c[11 * stride] = CLIP(t4 - t11); + c[12 * stride] = CLIP(t3 - t12); + c[13 * stride] = CLIP(t2 - t13a); + c[14 * stride] = CLIP(t1 - t14); + c[15 * stride] = CLIP(t0 - t15a); +} + +void dav1d_inv_dct16_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct16_1d_internal_c(c, stride, min, max, 0); +} + +static NOINLINE void +inv_dct32_1d_internal_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max, const int tx64) +{ + assert(stride > 0); + inv_dct16_1d_internal_c(c, stride << 1, min, max, tx64); + + const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; + const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; + + int t16a, t17a, t18a, t19a, t20a, t21a, t22a, t23a; + int t24a, t25a, t26a, t27a, t28a, t29a, t30a, t31a; + if (tx64) { + t16a = (in1 * 201 + 2048) >> 12; + t17a = (in15 * -2751 + 2048) >> 12; + t18a = (in9 * 1751 + 2048) >> 12; + t19a = (in7 * -1380 + 2048) >> 12; + t20a = (in5 * 995 + 2048) >> 12; + t21a = (in11 * -2106 + 2048) >> 12; + t22a = (in13 * 2440 + 2048) >> 12; + t23a = (in3 * -601 + 2048) >> 12; + t24a = (in3 * 4052 + 2048) >> 12; + t25a = (in13 * 3290 + 2048) >> 12; + t26a = (in11 * 3513 + 2048) >> 12; + t27a = (in5 * 3973 + 2048) >> 12; + t28a = (in7 * 3857 + 2048) >> 12; + t29a = (in9 * 3703 + 2048) >> 12; + t30a = (in15 * 3035 + 2048) >> 12; + t31a = (in1 * 4091 + 2048) >> 12; + } else { + const int in17 = c[17 * stride], in19 = c[19 * stride]; + const int in21 = c[21 * stride], in23 = c[23 * stride]; + const int in25 = c[25 * stride], in27 = c[27 * stride]; + const int in29 = c[29 * stride], in31 = c[31 * stride]; + + t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31; + t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17; + t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23; + t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25; + t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27; + t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21; + t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11; + t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29; + t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3; + t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11; + t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11; + t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5; + t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7; + t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9; + t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15; + t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1; + } + + int t16 = CLIP(t16a + t17a); + int t17 = CLIP(t16a - t17a); + int t18 = CLIP(t19a - t18a); + int t19 = CLIP(t19a + t18a); + int t20 = CLIP(t20a + t21a); + int t21 = CLIP(t20a - t21a); + int t22 = CLIP(t23a - t22a); + int t23 = CLIP(t23a + t22a); + int t24 = CLIP(t24a + t25a); + int t25 = CLIP(t24a - t25a); + int t26 = CLIP(t27a - t26a); + int t27 = CLIP(t27a + t26a); + int t28 = CLIP(t28a + t29a); + int t29 = CLIP(t28a - t29a); + int t30 = CLIP(t31a - t30a); + int t31 = CLIP(t31a + t30a); + + t17a = (( t30 * 799 - t17 * (4017 - 4096) + 2048) >> 12) - t17; + t30a = (( t30 * (4017 - 4096) + t17 * 799 + 2048) >> 12) + t30; + t18a = ((-(t29 * (4017 - 4096) + t18 * 799) + 2048) >> 12) - t29; + t29a = (( t29 * 799 - t18 * (4017 - 4096) + 2048) >> 12) - t18; + t21a = ( t26 * 1703 - t21 * 1138 + 1024) >> 11; + t26a = ( t26 * 1138 + t21 * 1703 + 1024) >> 11; + t22a = (-(t25 * 1138 + t22 * 1703 ) + 1024) >> 11; + t25a = ( t25 * 1703 - t22 * 1138 + 1024) >> 11; + + t16a = CLIP(t16 + t19); + t17 = CLIP(t17a + t18a); + t18 = CLIP(t17a - t18a); + t19a = CLIP(t16 - t19); + t20a = CLIP(t23 - t20); + t21 = CLIP(t22a - t21a); + t22 = CLIP(t22a + t21a); + t23a = CLIP(t23 + t20); + t24a = CLIP(t24 + t27); + t25 = CLIP(t25a + t26a); + t26 = CLIP(t25a - t26a); + t27a = CLIP(t24 - t27); + t28a = CLIP(t31 - t28); + t29 = CLIP(t30a - t29a); + t30 = CLIP(t30a + t29a); + t31a = CLIP(t31 + t28); + + t18a = (( t29 * 1567 - t18 * (3784 - 4096) + 2048) >> 12) - t18; + t29a = (( t29 * (3784 - 4096) + t18 * 1567 + 2048) >> 12) + t29; + t19 = (( t28a * 1567 - t19a * (3784 - 4096) + 2048) >> 12) - t19a; + t28 = (( t28a * (3784 - 4096) + t19a * 1567 + 2048) >> 12) + t28a; + t20 = ((-(t27a * (3784 - 4096) + t20a * 1567) + 2048) >> 12) - t27a; + t27 = (( t27a * 1567 - t20a * (3784 - 4096) + 2048) >> 12) - t20a; + t21a = ((-(t26 * (3784 - 4096) + t21 * 1567) + 2048) >> 12) - t26; + t26a = (( t26 * 1567 - t21 * (3784 - 4096) + 2048) >> 12) - t21; + + t16 = CLIP(t16a + t23a); + t17a = CLIP(t17 + t22); + t18 = CLIP(t18a + t21a); + t19a = CLIP(t19 + t20); + t20a = CLIP(t19 - t20); + t21 = CLIP(t18a - t21a); + t22a = CLIP(t17 - t22); + t23 = CLIP(t16a - t23a); + t24 = CLIP(t31a - t24a); + t25a = CLIP(t30 - t25); + t26 = CLIP(t29a - t26a); + t27a = CLIP(t28 - t27); + t28a = CLIP(t28 + t27); + t29 = CLIP(t29a + t26a); + t30a = CLIP(t30 + t25); + t31 = CLIP(t31a + t24a); + + t20 = ((t27a - t20a) * 181 + 128) >> 8; + t27 = ((t27a + t20a) * 181 + 128) >> 8; + t21a = ((t26 - t21 ) * 181 + 128) >> 8; + t26a = ((t26 + t21 ) * 181 + 128) >> 8; + t22 = ((t25a - t22a) * 181 + 128) >> 8; + t25 = ((t25a + t22a) * 181 + 128) >> 8; + t23a = ((t24 - t23 ) * 181 + 128) >> 8; + t24a = ((t24 + t23 ) * 181 + 128) >> 8; + + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + const int t8 = c[16 * stride]; + const int t9 = c[18 * stride]; + const int t10 = c[20 * stride]; + const int t11 = c[22 * stride]; + const int t12 = c[24 * stride]; + const int t13 = c[26 * stride]; + const int t14 = c[28 * stride]; + const int t15 = c[30 * stride]; + + c[ 0 * stride] = CLIP(t0 + t31); + c[ 1 * stride] = CLIP(t1 + t30a); + c[ 2 * stride] = CLIP(t2 + t29); + c[ 3 * stride] = CLIP(t3 + t28a); + c[ 4 * stride] = CLIP(t4 + t27); + c[ 5 * stride] = CLIP(t5 + t26a); + c[ 6 * stride] = CLIP(t6 + t25); + c[ 7 * stride] = CLIP(t7 + t24a); + c[ 8 * stride] = CLIP(t8 + t23a); + c[ 9 * stride] = CLIP(t9 + t22); + c[10 * stride] = CLIP(t10 + t21a); + c[11 * stride] = CLIP(t11 + t20); + c[12 * stride] = CLIP(t12 + t19a); + c[13 * stride] = CLIP(t13 + t18); + c[14 * stride] = CLIP(t14 + t17a); + c[15 * stride] = CLIP(t15 + t16); + c[16 * stride] = CLIP(t15 - t16); + c[17 * stride] = CLIP(t14 - t17a); + c[18 * stride] = CLIP(t13 - t18); + c[19 * stride] = CLIP(t12 - t19a); + c[20 * stride] = CLIP(t11 - t20); + c[21 * stride] = CLIP(t10 - t21a); + c[22 * stride] = CLIP(t9 - t22); + c[23 * stride] = CLIP(t8 - t23a); + c[24 * stride] = CLIP(t7 - t24a); + c[25 * stride] = CLIP(t6 - t25); + c[26 * stride] = CLIP(t5 - t26a); + c[27 * stride] = CLIP(t4 - t27); + c[28 * stride] = CLIP(t3 - t28a); + c[29 * stride] = CLIP(t2 - t29); + c[30 * stride] = CLIP(t1 - t30a); + c[31 * stride] = CLIP(t0 - t31); +} + +void dav1d_inv_dct32_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + inv_dct32_1d_internal_c(c, stride, min, max, 0); +} + +void dav1d_inv_dct64_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + inv_dct32_1d_internal_c(c, stride << 1, min, max, 1); + + const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; + const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; + const int in9 = c[ 9 * stride], in11 = c[11 * stride]; + const int in13 = c[13 * stride], in15 = c[15 * stride]; + const int in17 = c[17 * stride], in19 = c[19 * stride]; + const int in21 = c[21 * stride], in23 = c[23 * stride]; + const int in25 = c[25 * stride], in27 = c[27 * stride]; + const int in29 = c[29 * stride], in31 = c[31 * stride]; + + int t32a = (in1 * 101 + 2048) >> 12; + int t33a = (in31 * -2824 + 2048) >> 12; + int t34a = (in17 * 1660 + 2048) >> 12; + int t35a = (in15 * -1474 + 2048) >> 12; + int t36a = (in9 * 897 + 2048) >> 12; + int t37a = (in23 * -2191 + 2048) >> 12; + int t38a = (in25 * 2359 + 2048) >> 12; + int t39a = (in7 * -700 + 2048) >> 12; + int t40a = (in5 * 501 + 2048) >> 12; + int t41a = (in27 * -2520 + 2048) >> 12; + int t42a = (in21 * 2019 + 2048) >> 12; + int t43a = (in11 * -1092 + 2048) >> 12; + int t44a = (in13 * 1285 + 2048) >> 12; + int t45a = (in19 * -1842 + 2048) >> 12; + int t46a = (in29 * 2675 + 2048) >> 12; + int t47a = (in3 * -301 + 2048) >> 12; + int t48a = (in3 * 4085 + 2048) >> 12; + int t49a = (in29 * 3102 + 2048) >> 12; + int t50a = (in19 * 3659 + 2048) >> 12; + int t51a = (in13 * 3889 + 2048) >> 12; + int t52a = (in11 * 3948 + 2048) >> 12; + int t53a = (in21 * 3564 + 2048) >> 12; + int t54a = (in27 * 3229 + 2048) >> 12; + int t55a = (in5 * 4065 + 2048) >> 12; + int t56a = (in7 * 4036 + 2048) >> 12; + int t57a = (in25 * 3349 + 2048) >> 12; + int t58a = (in23 * 3461 + 2048) >> 12; + int t59a = (in9 * 3996 + 2048) >> 12; + int t60a = (in15 * 3822 + 2048) >> 12; + int t61a = (in17 * 3745 + 2048) >> 12; + int t62a = (in31 * 2967 + 2048) >> 12; + int t63a = (in1 * 4095 + 2048) >> 12; + + int t32 = CLIP(t32a + t33a); + int t33 = CLIP(t32a - t33a); + int t34 = CLIP(t35a - t34a); + int t35 = CLIP(t35a + t34a); + int t36 = CLIP(t36a + t37a); + int t37 = CLIP(t36a - t37a); + int t38 = CLIP(t39a - t38a); + int t39 = CLIP(t39a + t38a); + int t40 = CLIP(t40a + t41a); + int t41 = CLIP(t40a - t41a); + int t42 = CLIP(t43a - t42a); + int t43 = CLIP(t43a + t42a); + int t44 = CLIP(t44a + t45a); + int t45 = CLIP(t44a - t45a); + int t46 = CLIP(t47a - t46a); + int t47 = CLIP(t47a + t46a); + int t48 = CLIP(t48a + t49a); + int t49 = CLIP(t48a - t49a); + int t50 = CLIP(t51a - t50a); + int t51 = CLIP(t51a + t50a); + int t52 = CLIP(t52a + t53a); + int t53 = CLIP(t52a - t53a); + int t54 = CLIP(t55a - t54a); + int t55 = CLIP(t55a + t54a); + int t56 = CLIP(t56a + t57a); + int t57 = CLIP(t56a - t57a); + int t58 = CLIP(t59a - t58a); + int t59 = CLIP(t59a + t58a); + int t60 = CLIP(t60a + t61a); + int t61 = CLIP(t60a - t61a); + int t62 = CLIP(t63a - t62a); + int t63 = CLIP(t63a + t62a); + + t33a = ((t33 * (4096 - 4076) + t62 * 401 + 2048) >> 12) - t33; + t34a = ((t34 * -401 + t61 * (4096 - 4076) + 2048) >> 12) - t61; + t37a = (t37 * -1299 + t58 * 1583 + 1024) >> 11; + t38a = (t38 * -1583 + t57 * -1299 + 1024) >> 11; + t41a = ((t41 * (4096 - 3612) + t54 * 1931 + 2048) >> 12) - t41; + t42a = ((t42 * -1931 + t53 * (4096 - 3612) + 2048) >> 12) - t53; + t45a = ((t45 * -1189 + t50 * (3920 - 4096) + 2048) >> 12) + t50; + t46a = ((t46 * (4096 - 3920) + t49 * -1189 + 2048) >> 12) - t46; + t49a = ((t46 * -1189 + t49 * (3920 - 4096) + 2048) >> 12) + t49; + t50a = ((t45 * (3920 - 4096) + t50 * 1189 + 2048) >> 12) + t45; + t53a = ((t42 * (4096 - 3612) + t53 * 1931 + 2048) >> 12) - t42; + t54a = ((t41 * 1931 + t54 * (3612 - 4096) + 2048) >> 12) + t54; + t57a = (t38 * -1299 + t57 * 1583 + 1024) >> 11; + t58a = (t37 * 1583 + t58 * 1299 + 1024) >> 11; + t61a = ((t34 * (4096 - 4076) + t61 * 401 + 2048) >> 12) - t34; + t62a = ((t33 * 401 + t62 * (4076 - 4096) + 2048) >> 12) + t62; + + t32a = CLIP(t32 + t35); + t33 = CLIP(t33a + t34a); + t34 = CLIP(t33a - t34a); + t35a = CLIP(t32 - t35); + t36a = CLIP(t39 - t36); + t37 = CLIP(t38a - t37a); + t38 = CLIP(t38a + t37a); + t39a = CLIP(t39 + t36); + t40a = CLIP(t40 + t43); + t41 = CLIP(t41a + t42a); + t42 = CLIP(t41a - t42a); + t43a = CLIP(t40 - t43); + t44a = CLIP(t47 - t44); + t45 = CLIP(t46a - t45a); + t46 = CLIP(t46a + t45a); + t47a = CLIP(t47 + t44); + t48a = CLIP(t48 + t51); + t49 = CLIP(t49a + t50a); + t50 = CLIP(t49a - t50a); + t51a = CLIP(t48 - t51); + t52a = CLIP(t55 - t52); + t53 = CLIP(t54a - t53a); + t54 = CLIP(t54a + t53a); + t55a = CLIP(t55 + t52); + t56a = CLIP(t56 + t59); + t57 = CLIP(t57a + t58a); + t58 = CLIP(t57a - t58a); + t59a = CLIP(t56 - t59); + t60a = CLIP(t63 - t60); + t61 = CLIP(t62a - t61a); + t62 = CLIP(t62a + t61a); + t63a = CLIP(t63 + t60); + + t34a = ((t34 * (4096 - 4017) + t61 * 799 + 2048) >> 12) - t34; + t35 = ((t35a * (4096 - 4017) + t60a * 799 + 2048) >> 12) - t35a; + t36 = ((t36a * -799 + t59a * (4096 - 4017) + 2048) >> 12) - t59a; + t37a = ((t37 * -799 + t58 * (4096 - 4017) + 2048) >> 12) - t58; + t42a = (t42 * -1138 + t53 * 1703 + 1024) >> 11; + t43 = (t43a * -1138 + t52a * 1703 + 1024) >> 11; + t44 = (t44a * -1703 + t51a * -1138 + 1024) >> 11; + t45a = (t45 * -1703 + t50 * -1138 + 1024) >> 11; + t50a = (t45 * -1138 + t50 * 1703 + 1024) >> 11; + t51 = (t44a * -1138 + t51a * 1703 + 1024) >> 11; + t52 = (t43a * 1703 + t52a * 1138 + 1024) >> 11; + t53a = (t42 * 1703 + t53 * 1138 + 1024) >> 11; + t58a = ((t37 * (4096 - 4017) + t58 * 799 + 2048) >> 12) - t37; + t59 = ((t36a * (4096 - 4017) + t59a * 799 + 2048) >> 12) - t36a; + t60 = ((t35a * 799 + t60a * (4017 - 4096) + 2048) >> 12) + t60a; + t61a = ((t34 * 799 + t61 * (4017 - 4096) + 2048) >> 12) + t61; + + t32 = CLIP(t32a + t39a); + t33a = CLIP(t33 + t38); + t34 = CLIP(t34a + t37a); + t35a = CLIP(t35 + t36); + t36a = CLIP(t35 - t36); + t37 = CLIP(t34a - t37a); + t38a = CLIP(t33 - t38); + t39 = CLIP(t32a - t39a); + t40 = CLIP(t47a - t40a); + t41a = CLIP(t46 - t41); + t42 = CLIP(t45a - t42a); + t43a = CLIP(t44 - t43); + t44a = CLIP(t44 + t43); + t45 = CLIP(t45a + t42a); + t46a = CLIP(t46 + t41); + t47 = CLIP(t47a + t40a); + t48 = CLIP(t48a + t55a); + t49a = CLIP(t49 + t54); + t50 = CLIP(t50a + t53a); + t51a = CLIP(t51 + t52); + t52a = CLIP(t51 - t52); + t53 = CLIP(t50a - t53a); + t54a = CLIP(t49 - t54); + t55 = CLIP(t48a - t55a); + t56 = CLIP(t63a - t56a); + t57a = CLIP(t62 - t57); + t58 = CLIP(t61a - t58a); + t59a = CLIP(t60 - t59); + t60a = CLIP(t60 + t59); + t61 = CLIP(t61a + t58a); + t62a = CLIP(t62 + t57); + t63 = CLIP(t63a + t56a); + + t36 = ((t36a * (4096 - 3784) + t59a * 1567 + 2048) >> 12) - t36a; + t37a = ((t37 * (4096 - 3784) + t58 * 1567 + 2048) >> 12) - t37; + t38 = ((t38a * (4096 - 3784) + t57a * 1567 + 2048) >> 12) - t38a; + t39a = ((t39 * (4096 - 3784) + t56 * 1567 + 2048) >> 12) - t39; + t40a = ((t40 * -1567 + t55 * (4096 - 3784) + 2048) >> 12) - t55; + t41 = ((t41a * -1567 + t54a * (4096 - 3784) + 2048) >> 12) - t54a; + t42a = ((t42 * -1567 + t53 * (4096 - 3784) + 2048) >> 12) - t53; + t43 = ((t43a * -1567 + t52a * (4096 - 3784) + 2048) >> 12) - t52a; + t52 = ((t43a * (4096 - 3784) + t52a * 1567 + 2048) >> 12) - t43a; + t53a = ((t42 * (4096 - 3784) + t53 * 1567 + 2048) >> 12) - t42; + t54 = ((t41a * (4096 - 3784) + t54a * 1567 + 2048) >> 12) - t41a; + t55a = ((t40 * (4096 - 3784) + t55 * 1567 + 2048) >> 12) - t40; + t56a = ((t39 * 1567 + t56 * (3784 - 4096) + 2048) >> 12) + t56; + t57 = ((t38a * 1567 + t57a * (3784 - 4096) + 2048) >> 12) + t57a; + t58a = ((t37 * 1567 + t58 * (3784 - 4096) + 2048) >> 12) + t58; + t59 = ((t36a * 1567 + t59a * (3784 - 4096) + 2048) >> 12) + t59a; + + t32a = CLIP(t32 + t47); + t33 = CLIP(t33a + t46a); + t34a = CLIP(t34 + t45); + t35 = CLIP(t35a + t44a); + t36a = CLIP(t36 + t43); + t37 = CLIP(t37a + t42a); + t38a = CLIP(t38 + t41); + t39 = CLIP(t39a + t40a); + t40 = CLIP(t39a - t40a); + t41a = CLIP(t38 - t41); + t42 = CLIP(t37a - t42a); + t43a = CLIP(t36 - t43); + t44 = CLIP(t35a - t44a); + t45a = CLIP(t34 - t45); + t46 = CLIP(t33a - t46a); + t47a = CLIP(t32 - t47); + t48a = CLIP(t63 - t48); + t49 = CLIP(t62a - t49a); + t50a = CLIP(t61 - t50); + t51 = CLIP(t60a - t51a); + t52a = CLIP(t59 - t52); + t53 = CLIP(t58a - t53a); + t54a = CLIP(t57 - t54); + t55 = CLIP(t56a - t55a); + t56 = CLIP(t56a + t55a); + t57a = CLIP(t57 + t54); + t58 = CLIP(t58a + t53a); + t59a = CLIP(t59 + t52); + t60 = CLIP(t60a + t51a); + t61a = CLIP(t61 + t50); + t62 = CLIP(t62a + t49a); + t63a = CLIP(t63 + t48); + + t40a = ((t55 - t40 ) * 181 + 128) >> 8; + t41 = ((t54a - t41a) * 181 + 128) >> 8; + t42a = ((t53 - t42 ) * 181 + 128) >> 8; + t43 = ((t52a - t43a) * 181 + 128) >> 8; + t44a = ((t51 - t44 ) * 181 + 128) >> 8; + t45 = ((t50a - t45a) * 181 + 128) >> 8; + t46a = ((t49 - t46 ) * 181 + 128) >> 8; + t47 = ((t48a - t47a) * 181 + 128) >> 8; + t48 = ((t47a + t48a) * 181 + 128) >> 8; + t49a = ((t46 + t49 ) * 181 + 128) >> 8; + t50 = ((t45a + t50a) * 181 + 128) >> 8; + t51a = ((t44 + t51 ) * 181 + 128) >> 8; + t52 = ((t43a + t52a) * 181 + 128) >> 8; + t53a = ((t42 + t53 ) * 181 + 128) >> 8; + t54 = ((t41a + t54a) * 181 + 128) >> 8; + t55a = ((t40 + t55 ) * 181 + 128) >> 8; + + const int t0 = c[ 0 * stride]; + const int t1 = c[ 2 * stride]; + const int t2 = c[ 4 * stride]; + const int t3 = c[ 6 * stride]; + const int t4 = c[ 8 * stride]; + const int t5 = c[10 * stride]; + const int t6 = c[12 * stride]; + const int t7 = c[14 * stride]; + const int t8 = c[16 * stride]; + const int t9 = c[18 * stride]; + const int t10 = c[20 * stride]; + const int t11 = c[22 * stride]; + const int t12 = c[24 * stride]; + const int t13 = c[26 * stride]; + const int t14 = c[28 * stride]; + const int t15 = c[30 * stride]; + const int t16 = c[32 * stride]; + const int t17 = c[34 * stride]; + const int t18 = c[36 * stride]; + const int t19 = c[38 * stride]; + const int t20 = c[40 * stride]; + const int t21 = c[42 * stride]; + const int t22 = c[44 * stride]; + const int t23 = c[46 * stride]; + const int t24 = c[48 * stride]; + const int t25 = c[50 * stride]; + const int t26 = c[52 * stride]; + const int t27 = c[54 * stride]; + const int t28 = c[56 * stride]; + const int t29 = c[58 * stride]; + const int t30 = c[60 * stride]; + const int t31 = c[62 * stride]; + + c[ 0 * stride] = CLIP(t0 + t63a); + c[ 1 * stride] = CLIP(t1 + t62); + c[ 2 * stride] = CLIP(t2 + t61a); + c[ 3 * stride] = CLIP(t3 + t60); + c[ 4 * stride] = CLIP(t4 + t59a); + c[ 5 * stride] = CLIP(t5 + t58); + c[ 6 * stride] = CLIP(t6 + t57a); + c[ 7 * stride] = CLIP(t7 + t56); + c[ 8 * stride] = CLIP(t8 + t55a); + c[ 9 * stride] = CLIP(t9 + t54); + c[10 * stride] = CLIP(t10 + t53a); + c[11 * stride] = CLIP(t11 + t52); + c[12 * stride] = CLIP(t12 + t51a); + c[13 * stride] = CLIP(t13 + t50); + c[14 * stride] = CLIP(t14 + t49a); + c[15 * stride] = CLIP(t15 + t48); + c[16 * stride] = CLIP(t16 + t47); + c[17 * stride] = CLIP(t17 + t46a); + c[18 * stride] = CLIP(t18 + t45); + c[19 * stride] = CLIP(t19 + t44a); + c[20 * stride] = CLIP(t20 + t43); + c[21 * stride] = CLIP(t21 + t42a); + c[22 * stride] = CLIP(t22 + t41); + c[23 * stride] = CLIP(t23 + t40a); + c[24 * stride] = CLIP(t24 + t39); + c[25 * stride] = CLIP(t25 + t38a); + c[26 * stride] = CLIP(t26 + t37); + c[27 * stride] = CLIP(t27 + t36a); + c[28 * stride] = CLIP(t28 + t35); + c[29 * stride] = CLIP(t29 + t34a); + c[30 * stride] = CLIP(t30 + t33); + c[31 * stride] = CLIP(t31 + t32a); + c[32 * stride] = CLIP(t31 - t32a); + c[33 * stride] = CLIP(t30 - t33); + c[34 * stride] = CLIP(t29 - t34a); + c[35 * stride] = CLIP(t28 - t35); + c[36 * stride] = CLIP(t27 - t36a); + c[37 * stride] = CLIP(t26 - t37); + c[38 * stride] = CLIP(t25 - t38a); + c[39 * stride] = CLIP(t24 - t39); + c[40 * stride] = CLIP(t23 - t40a); + c[41 * stride] = CLIP(t22 - t41); + c[42 * stride] = CLIP(t21 - t42a); + c[43 * stride] = CLIP(t20 - t43); + c[44 * stride] = CLIP(t19 - t44a); + c[45 * stride] = CLIP(t18 - t45); + c[46 * stride] = CLIP(t17 - t46a); + c[47 * stride] = CLIP(t16 - t47); + c[48 * stride] = CLIP(t15 - t48); + c[49 * stride] = CLIP(t14 - t49a); + c[50 * stride] = CLIP(t13 - t50); + c[51 * stride] = CLIP(t12 - t51a); + c[52 * stride] = CLIP(t11 - t52); + c[53 * stride] = CLIP(t10 - t53a); + c[54 * stride] = CLIP(t9 - t54); + c[55 * stride] = CLIP(t8 - t55a); + c[56 * stride] = CLIP(t7 - t56); + c[57 * stride] = CLIP(t6 - t57a); + c[58 * stride] = CLIP(t5 - t58); + c[59 * stride] = CLIP(t4 - t59a); + c[60 * stride] = CLIP(t3 - t60); + c[61 * stride] = CLIP(t2 - t61a); + c[62 * stride] = CLIP(t1 - t62); + c[63 * stride] = CLIP(t0 - t63a); +} + +static NOINLINE void +inv_adst4_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) +{ + assert(in_s > 0 && out_s != 0); + const int in0 = in[0 * in_s], in1 = in[1 * in_s]; + const int in2 = in[2 * in_s], in3 = in[3 * in_s]; + + out[0 * out_s] = (( 1321 * in0 + (3803 - 4096) * in2 + + (2482 - 4096) * in3 + (3344 - 4096) * in1 + 2048) >> 12) + + in2 + in3 + in1; + out[1 * out_s] = (((2482 - 4096) * in0 - 1321 * in2 - + (3803 - 4096) * in3 + (3344 - 4096) * in1 + 2048) >> 12) + + in0 - in3 + in1; + out[2 * out_s] = (209 * (in0 - in2 + in3) + 128) >> 8; + out[3 * out_s] = (((3803 - 4096) * in0 + (2482 - 4096) * in2 - + 1321 * in3 - (3344 - 4096) * in1 + 2048) >> 12) + + in0 + in2 - in1; +} + +static NOINLINE void +inv_adst8_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) +{ + assert(in_s > 0 && out_s != 0); + const int in0 = in[0 * in_s], in1 = in[1 * in_s]; + const int in2 = in[2 * in_s], in3 = in[3 * in_s]; + const int in4 = in[4 * in_s], in5 = in[5 * in_s]; + const int in6 = in[6 * in_s], in7 = in[7 * in_s]; + + const int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7; + const int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0; + const int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5; + const int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2; + int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11; + int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11; + int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6; + int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1; + + const int t0 = CLIP(t0a + t4a); + const int t1 = CLIP(t1a + t5a); + int t2 = CLIP(t2a + t6a); + int t3 = CLIP(t3a + t7a); + const int t4 = CLIP(t0a - t4a); + const int t5 = CLIP(t1a - t5a); + int t6 = CLIP(t2a - t6a); + int t7 = CLIP(t3a - t7a); + + t4a = (((3784 - 4096) * t4 + 1567 * t5 + 2048) >> 12) + t4; + t5a = (( 1567 * t4 - (3784 - 4096) * t5 + 2048) >> 12) - t5; + t6a = (((3784 - 4096) * t7 - 1567 * t6 + 2048) >> 12) + t7; + t7a = (( 1567 * t7 + (3784 - 4096) * t6 + 2048) >> 12) + t6; + + out[0 * out_s] = CLIP(t0 + t2 ); + out[7 * out_s] = -CLIP(t1 + t3 ); + t2 = CLIP(t0 - t2 ); + t3 = CLIP(t1 - t3 ); + out[1 * out_s] = -CLIP(t4a + t6a); + out[6 * out_s] = CLIP(t5a + t7a); + t6 = CLIP(t4a - t6a); + t7 = CLIP(t5a - t7a); + + out[3 * out_s] = -(((t2 + t3) * 181 + 128) >> 8); + out[4 * out_s] = ((t2 - t3) * 181 + 128) >> 8; + out[2 * out_s] = ((t6 + t7) * 181 + 128) >> 8; + out[5 * out_s] = -(((t6 - t7) * 181 + 128) >> 8); +} + +static NOINLINE void +inv_adst16_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s, + const int min, const int max, + int32_t *const out, const ptrdiff_t out_s) +{ + assert(in_s > 0 && out_s != 0); + const int in0 = in[ 0 * in_s], in1 = in[ 1 * in_s]; + const int in2 = in[ 2 * in_s], in3 = in[ 3 * in_s]; + const int in4 = in[ 4 * in_s], in5 = in[ 5 * in_s]; + const int in6 = in[ 6 * in_s], in7 = in[ 7 * in_s]; + const int in8 = in[ 8 * in_s], in9 = in[ 9 * in_s]; + const int in10 = in[10 * in_s], in11 = in[11 * in_s]; + const int in12 = in[12 * in_s], in13 = in[13 * in_s]; + const int in14 = in[14 * in_s], in15 = in[15 * in_s]; + + int t0 = ((in15 * (4091 - 4096) + in0 * 201 + 2048) >> 12) + in15; + int t1 = ((in15 * 201 - in0 * (4091 - 4096) + 2048) >> 12) - in0; + int t2 = ((in13 * (3973 - 4096) + in2 * 995 + 2048) >> 12) + in13; + int t3 = ((in13 * 995 - in2 * (3973 - 4096) + 2048) >> 12) - in2; + int t4 = ((in11 * (3703 - 4096) + in4 * 1751 + 2048) >> 12) + in11; + int t5 = ((in11 * 1751 - in4 * (3703 - 4096) + 2048) >> 12) - in4; + int t6 = (in9 * 1645 + in6 * 1220 + 1024) >> 11; + int t7 = (in9 * 1220 - in6 * 1645 + 1024) >> 11; + int t8 = ((in7 * 2751 + in8 * (3035 - 4096) + 2048) >> 12) + in8; + int t9 = ((in7 * (3035 - 4096) - in8 * 2751 + 2048) >> 12) + in7; + int t10 = ((in5 * 2106 + in10 * (3513 - 4096) + 2048) >> 12) + in10; + int t11 = ((in5 * (3513 - 4096) - in10 * 2106 + 2048) >> 12) + in5; + int t12 = ((in3 * 1380 + in12 * (3857 - 4096) + 2048) >> 12) + in12; + int t13 = ((in3 * (3857 - 4096) - in12 * 1380 + 2048) >> 12) + in3; + int t14 = ((in1 * 601 + in14 * (4052 - 4096) + 2048) >> 12) + in14; + int t15 = ((in1 * (4052 - 4096) - in14 * 601 + 2048) >> 12) + in1; + + int t0a = CLIP(t0 + t8 ); + int t1a = CLIP(t1 + t9 ); + int t2a = CLIP(t2 + t10); + int t3a = CLIP(t3 + t11); + int t4a = CLIP(t4 + t12); + int t5a = CLIP(t5 + t13); + int t6a = CLIP(t6 + t14); + int t7a = CLIP(t7 + t15); + int t8a = CLIP(t0 - t8 ); + int t9a = CLIP(t1 - t9 ); + int t10a = CLIP(t2 - t10); + int t11a = CLIP(t3 - t11); + int t12a = CLIP(t4 - t12); + int t13a = CLIP(t5 - t13); + int t14a = CLIP(t6 - t14); + int t15a = CLIP(t7 - t15); + + t8 = ((t8a * (4017 - 4096) + t9a * 799 + 2048) >> 12) + t8a; + t9 = ((t8a * 799 - t9a * (4017 - 4096) + 2048) >> 12) - t9a; + t10 = ((t10a * 2276 + t11a * (3406 - 4096) + 2048) >> 12) + t11a; + t11 = ((t10a * (3406 - 4096) - t11a * 2276 + 2048) >> 12) + t10a; + t12 = ((t13a * (4017 - 4096) - t12a * 799 + 2048) >> 12) + t13a; + t13 = ((t13a * 799 + t12a * (4017 - 4096) + 2048) >> 12) + t12a; + t14 = ((t15a * 2276 - t14a * (3406 - 4096) + 2048) >> 12) - t14a; + t15 = ((t15a * (3406 - 4096) + t14a * 2276 + 2048) >> 12) + t15a; + + t0 = CLIP(t0a + t4a); + t1 = CLIP(t1a + t5a); + t2 = CLIP(t2a + t6a); + t3 = CLIP(t3a + t7a); + t4 = CLIP(t0a - t4a); + t5 = CLIP(t1a - t5a); + t6 = CLIP(t2a - t6a); + t7 = CLIP(t3a - t7a); + t8a = CLIP(t8 + t12); + t9a = CLIP(t9 + t13); + t10a = CLIP(t10 + t14); + t11a = CLIP(t11 + t15); + t12a = CLIP(t8 - t12); + t13a = CLIP(t9 - t13); + t14a = CLIP(t10 - t14); + t15a = CLIP(t11 - t15); + + t4a = ((t4 * (3784 - 4096) + t5 * 1567 + 2048) >> 12) + t4; + t5a = ((t4 * 1567 - t5 * (3784 - 4096) + 2048) >> 12) - t5; + t6a = ((t7 * (3784 - 4096) - t6 * 1567 + 2048) >> 12) + t7; + t7a = ((t7 * 1567 + t6 * (3784 - 4096) + 2048) >> 12) + t6; + t12 = ((t12a * (3784 - 4096) + t13a * 1567 + 2048) >> 12) + t12a; + t13 = ((t12a * 1567 - t13a * (3784 - 4096) + 2048) >> 12) - t13a; + t14 = ((t15a * (3784 - 4096) - t14a * 1567 + 2048) >> 12) + t15a; + t15 = ((t15a * 1567 + t14a * (3784 - 4096) + 2048) >> 12) + t14a; + + out[ 0 * out_s] = CLIP(t0 + t2 ); + out[15 * out_s] = -CLIP(t1 + t3 ); + t2a = CLIP(t0 - t2 ); + t3a = CLIP(t1 - t3 ); + out[ 3 * out_s] = -CLIP(t4a + t6a ); + out[12 * out_s] = CLIP(t5a + t7a ); + t6 = CLIP(t4a - t6a ); + t7 = CLIP(t5a - t7a ); + out[ 1 * out_s] = -CLIP(t8a + t10a); + out[14 * out_s] = CLIP(t9a + t11a); + t10 = CLIP(t8a - t10a); + t11 = CLIP(t9a - t11a); + out[ 2 * out_s] = CLIP(t12 + t14 ); + out[13 * out_s] = -CLIP(t13 + t15 ); + t14a = CLIP(t12 - t14 ); + t15a = CLIP(t13 - t15 ); + + out[ 7 * out_s] = -(((t2a + t3a) * 181 + 128) >> 8); + out[ 8 * out_s] = ((t2a - t3a) * 181 + 128) >> 8; + out[ 4 * out_s] = ((t6 + t7) * 181 + 128) >> 8; + out[11 * out_s] = -(((t6 - t7) * 181 + 128) >> 8); + out[ 6 * out_s] = ((t10 + t11) * 181 + 128) >> 8; + out[ 9 * out_s] = -(((t10 - t11) * 181 + 128) >> 8); + out[ 5 * out_s] = -(((t14a + t15a) * 181 + 128) >> 8); + out[10 * out_s] = ((t14a - t15a) * 181 + 128) >> 8; +} + +#define inv_adst_1d(sz) \ +void dav1d_inv_adst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \ + const int min, const int max) \ +{ \ + inv_adst##sz##_1d_internal_c(c, stride, min, max, c, stride); \ +} \ +void dav1d_inv_flipadst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \ + const int min, const int max) \ +{ \ + inv_adst##sz##_1d_internal_c(c, stride, min, max, \ + &c[(sz - 1) * stride], -stride); \ +} + +inv_adst_1d( 4) +inv_adst_1d( 8) +inv_adst_1d(16) + +#undef inv_adst_1d + +void dav1d_inv_identity4_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + for (int i = 0; i < 4; i++) { + const int in = c[stride * i]; + c[stride * i] = in + ((in * 1697 + 2048) >> 12); + } +} + +void dav1d_inv_identity8_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + for (int i = 0; i < 8; i++) + c[stride * i] *= 2; +} + +void dav1d_inv_identity16_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + for (int i = 0; i < 16; i++) { + const int in = c[stride * i]; + c[stride * i] = 2 * in + ((in * 1697 + 1024) >> 11); + } +} + +void dav1d_inv_identity32_1d_c(int32_t *const c, const ptrdiff_t stride, + const int min, const int max) +{ + assert(stride > 0); + for (int i = 0; i < 32; i++) + c[stride * i] *= 4; +} + +void dav1d_inv_wht4_1d_c(int32_t *const c, const ptrdiff_t stride) { + assert(stride > 0); + const int in0 = c[0 * stride], in1 = c[1 * stride]; + const int in2 = c[2 * stride], in3 = c[3 * stride]; + + const int t0 = in0 + in1; + const int t2 = in2 - in3; + const int t4 = (t0 - t2) >> 1; + const int t3 = t4 - in3; + const int t1 = t4 - in1; + + c[0 * stride] = t0 - t3; + c[1 * stride] = t3; + c[2 * stride] = t1; + c[3 * stride] = t2 + t1; +} diff --git a/third_party/dav1d/src/itx_1d.h b/third_party/dav1d/src/itx_1d.h new file mode 100644 index 0000000000..b63d71b020 --- /dev/null +++ b/third_party/dav1d/src/itx_1d.h @@ -0,0 +1,59 @@ +/* + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifndef DAV1D_SRC_ITX_1D_H +#define DAV1D_SRC_ITX_1D_H + +#define decl_itx_1d_fn(name) \ +void (name)(int32_t *c, ptrdiff_t stride, int min, int max) +typedef decl_itx_1d_fn(*itx_1d_fn); + +decl_itx_1d_fn(dav1d_inv_dct4_1d_c); +decl_itx_1d_fn(dav1d_inv_dct8_1d_c); +decl_itx_1d_fn(dav1d_inv_dct16_1d_c); +decl_itx_1d_fn(dav1d_inv_dct32_1d_c); +decl_itx_1d_fn(dav1d_inv_dct64_1d_c); + +decl_itx_1d_fn(dav1d_inv_adst4_1d_c); +decl_itx_1d_fn(dav1d_inv_adst8_1d_c); +decl_itx_1d_fn(dav1d_inv_adst16_1d_c); + +decl_itx_1d_fn(dav1d_inv_flipadst4_1d_c); +decl_itx_1d_fn(dav1d_inv_flipadst8_1d_c); +decl_itx_1d_fn(dav1d_inv_flipadst16_1d_c); + +decl_itx_1d_fn(dav1d_inv_identity4_1d_c); +decl_itx_1d_fn(dav1d_inv_identity8_1d_c); +decl_itx_1d_fn(dav1d_inv_identity16_1d_c); +decl_itx_1d_fn(dav1d_inv_identity32_1d_c); + +void dav1d_inv_wht4_1d_c(int32_t *c, ptrdiff_t stride); + +#endif /* DAV1D_SRC_ITX_1D_H */ diff --git a/third_party/dav1d/src/itx_tmpl.c b/third_party/dav1d/src/itx_tmpl.c new file mode 100644 index 0000000000..d3859892d8 --- /dev/null +++ b/third_party/dav1d/src/itx_tmpl.c @@ -0,0 +1,264 @@ +/* + * Copyright © 2018-2019, VideoLAN and dav1d authors + * Copyright © 2018-2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/itx.h" +#include "src/itx_1d.h" + +static NOINLINE void +inv_txfm_add_c(pixel *dst, const ptrdiff_t stride, coef *const coeff, + const int eob, const int w, const int h, const int shift, + const itx_1d_fn first_1d_fn, const itx_1d_fn second_1d_fn, + const int has_dconly HIGHBD_DECL_SUFFIX) +{ + assert(w >= 4 && w <= 64); + assert(h >= 4 && h <= 64); + assert(eob >= 0); + + const int is_rect2 = w * 2 == h || h * 2 == w; + const int rnd = (1 << shift) >> 1; + + if (eob < has_dconly) { + int dc = coeff[0]; + coeff[0] = 0; + if (is_rect2) + dc = (dc * 181 + 128) >> 8; + dc = (dc * 181 + 128) >> 8; + dc = (dc + rnd) >> shift; + dc = (dc * 181 + 128 + 2048) >> 12; + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel(dst[x] + dc); + return; + } + + const int sh = imin(h, 32), sw = imin(w, 32); +#if BITDEPTH == 8 + const int row_clip_min = INT16_MIN; + const int col_clip_min = INT16_MIN; +#else + const int row_clip_min = (int) ((unsigned) ~bitdepth_max << 7); + const int col_clip_min = (int) ((unsigned) ~bitdepth_max << 5); +#endif + const int row_clip_max = ~row_clip_min; + const int col_clip_max = ~col_clip_min; + + int32_t tmp[64 * 64], *c = tmp; + for (int y = 0; y < sh; y++, c += w) { + if (is_rect2) + for (int x = 0; x < sw; x++) + c[x] = (coeff[y + x * sh] * 181 + 128) >> 8; + else + for (int x = 0; x < sw; x++) + c[x] = coeff[y + x * sh]; + first_1d_fn(c, 1, row_clip_min, row_clip_max); + } + + memset(coeff, 0, sizeof(*coeff) * sw * sh); + for (int i = 0; i < w * sh; i++) + tmp[i] = iclip((tmp[i] + rnd) >> shift, col_clip_min, col_clip_max); + + for (int x = 0; x < w; x++) + second_1d_fn(&tmp[x], w, col_clip_min, col_clip_max); + + c = tmp; + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel(dst[x] + ((*c++ + 8) >> 4)); +} + +#define inv_txfm_fn(type1, type2, w, h, shift, has_dconly) \ +static void \ +inv_txfm_add_##type1##_##type2##_##w##x##h##_c(pixel *dst, \ + const ptrdiff_t stride, \ + coef *const coeff, \ + const int eob \ + HIGHBD_DECL_SUFFIX) \ +{ \ + inv_txfm_add_c(dst, stride, coeff, eob, w, h, shift, \ + dav1d_inv_##type1##w##_1d_c, dav1d_inv_##type2##h##_1d_c, \ + has_dconly HIGHBD_TAIL_SUFFIX); \ +} + +#define inv_txfm_fn64(w, h, shift) \ +inv_txfm_fn(dct, dct, w, h, shift, 1) + +#define inv_txfm_fn32(w, h, shift) \ +inv_txfm_fn64(w, h, shift) \ +inv_txfm_fn(identity, identity, w, h, shift, 0) + +#define inv_txfm_fn16(w, h, shift) \ +inv_txfm_fn32(w, h, shift) \ +inv_txfm_fn(adst, dct, w, h, shift, 0) \ +inv_txfm_fn(dct, adst, w, h, shift, 0) \ +inv_txfm_fn(adst, adst, w, h, shift, 0) \ +inv_txfm_fn(dct, flipadst, w, h, shift, 0) \ +inv_txfm_fn(flipadst, dct, w, h, shift, 0) \ +inv_txfm_fn(adst, flipadst, w, h, shift, 0) \ +inv_txfm_fn(flipadst, adst, w, h, shift, 0) \ +inv_txfm_fn(flipadst, flipadst, w, h, shift, 0) \ +inv_txfm_fn(identity, dct, w, h, shift, 0) \ +inv_txfm_fn(dct, identity, w, h, shift, 0) \ + +#define inv_txfm_fn84(w, h, shift) \ +inv_txfm_fn16(w, h, shift) \ +inv_txfm_fn(identity, flipadst, w, h, shift, 0) \ +inv_txfm_fn(flipadst, identity, w, h, shift, 0) \ +inv_txfm_fn(identity, adst, w, h, shift, 0) \ +inv_txfm_fn(adst, identity, w, h, shift, 0) \ + +inv_txfm_fn84( 4, 4, 0) +inv_txfm_fn84( 4, 8, 0) +inv_txfm_fn84( 4, 16, 1) +inv_txfm_fn84( 8, 4, 0) +inv_txfm_fn84( 8, 8, 1) +inv_txfm_fn84( 8, 16, 1) +inv_txfm_fn32( 8, 32, 2) +inv_txfm_fn84(16, 4, 1) +inv_txfm_fn84(16, 8, 1) +inv_txfm_fn16(16, 16, 2) +inv_txfm_fn32(16, 32, 1) +inv_txfm_fn64(16, 64, 2) +inv_txfm_fn32(32, 8, 2) +inv_txfm_fn32(32, 16, 1) +inv_txfm_fn32(32, 32, 2) +inv_txfm_fn64(32, 64, 1) +inv_txfm_fn64(64, 16, 2) +inv_txfm_fn64(64, 32, 1) +inv_txfm_fn64(64, 64, 2) + +static void inv_txfm_add_wht_wht_4x4_c(pixel *dst, const ptrdiff_t stride, + coef *const coeff, const int eob + HIGHBD_DECL_SUFFIX) +{ + int32_t tmp[4 * 4], *c = tmp; + for (int y = 0; y < 4; y++, c += 4) { + for (int x = 0; x < 4; x++) + c[x] = coeff[y + x * 4] >> 2; + dav1d_inv_wht4_1d_c(c, 1); + } + memset(coeff, 0, sizeof(*coeff) * 4 * 4); + + for (int x = 0; x < 4; x++) + dav1d_inv_wht4_1d_c(&tmp[x], 4); + + c = tmp; + for (int y = 0; y < 4; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < 4; x++) + dst[x] = iclip_pixel(dst[x] + *c++); +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/itx.h" +#elif ARCH_X86 +#include "src/x86/itx.h" +#endif +#endif + +COLD void bitfn(dav1d_itx_dsp_init)(Dav1dInvTxfmDSPContext *const c, int bpc) { +#define assign_itx_all_fn64(w, h, pfx) \ + c->itxfm_add[pfx##TX_##w##X##h][DCT_DCT ] = \ + inv_txfm_add_dct_dct_##w##x##h##_c + +#define assign_itx_all_fn32(w, h, pfx) \ + assign_itx_all_fn64(w, h, pfx); \ + c->itxfm_add[pfx##TX_##w##X##h][IDTX] = \ + inv_txfm_add_identity_identity_##w##x##h##_c + +#define assign_itx_all_fn16(w, h, pfx) \ + assign_itx_all_fn32(w, h, pfx); \ + c->itxfm_add[pfx##TX_##w##X##h][DCT_ADST ] = \ + inv_txfm_add_adst_dct_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][ADST_DCT ] = \ + inv_txfm_add_dct_adst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][ADST_ADST] = \ + inv_txfm_add_adst_adst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][ADST_FLIPADST] = \ + inv_txfm_add_flipadst_adst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_ADST] = \ + inv_txfm_add_adst_flipadst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][DCT_FLIPADST] = \ + inv_txfm_add_flipadst_dct_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_DCT] = \ + inv_txfm_add_dct_flipadst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][FLIPADST_FLIPADST] = \ + inv_txfm_add_flipadst_flipadst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][H_DCT] = \ + inv_txfm_add_dct_identity_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][V_DCT] = \ + inv_txfm_add_identity_dct_##w##x##h##_c + +#define assign_itx_all_fn84(w, h, pfx) \ + assign_itx_all_fn16(w, h, pfx); \ + c->itxfm_add[pfx##TX_##w##X##h][H_FLIPADST] = \ + inv_txfm_add_flipadst_identity_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][V_FLIPADST] = \ + inv_txfm_add_identity_flipadst_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][H_ADST] = \ + inv_txfm_add_adst_identity_##w##x##h##_c; \ + c->itxfm_add[pfx##TX_##w##X##h][V_ADST] = \ + inv_txfm_add_identity_adst_##w##x##h##_c; \ + + c->itxfm_add[TX_4X4][WHT_WHT] = inv_txfm_add_wht_wht_4x4_c; + assign_itx_all_fn84( 4, 4, ); + assign_itx_all_fn84( 4, 8, R); + assign_itx_all_fn84( 4, 16, R); + assign_itx_all_fn84( 8, 4, R); + assign_itx_all_fn84( 8, 8, ); + assign_itx_all_fn84( 8, 16, R); + assign_itx_all_fn32( 8, 32, R); + assign_itx_all_fn84(16, 4, R); + assign_itx_all_fn84(16, 8, R); + assign_itx_all_fn16(16, 16, ); + assign_itx_all_fn32(16, 32, R); + assign_itx_all_fn64(16, 64, R); + assign_itx_all_fn32(32, 8, R); + assign_itx_all_fn32(32, 16, R); + assign_itx_all_fn32(32, 32, ); + assign_itx_all_fn64(32, 64, R); + assign_itx_all_fn64(64, 16, R); + assign_itx_all_fn64(64, 32, R); + assign_itx_all_fn64(64, 64, ); + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + itx_dsp_init_arm(c, bpc); +#endif +#if ARCH_X86 + itx_dsp_init_x86(c, bpc); +#endif +#endif +} diff --git a/third_party/dav1d/src/levels.h b/third_party/dav1d/src/levels.h new file mode 100644 index 0000000000..0f510e9f30 --- /dev/null +++ b/third_party/dav1d/src/levels.h @@ -0,0 +1,289 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LEVELS_H +#define DAV1D_SRC_LEVELS_H + +#include + +#include "dav1d/headers.h" +#include "common/attributes.h" + +enum ObuMetaType { + OBU_META_HDR_CLL = 1, + OBU_META_HDR_MDCV = 2, + OBU_META_SCALABILITY = 3, + OBU_META_ITUT_T35 = 4, + OBU_META_TIMECODE = 5, +}; + +enum TxfmSize { + TX_4X4, + TX_8X8, + TX_16X16, + TX_32X32, + TX_64X64, + N_TX_SIZES, +}; + +enum BlockLevel { + BL_128X128, + BL_64X64, + BL_32X32, + BL_16X16, + BL_8X8, + N_BL_LEVELS, +}; + +enum RectTxfmSize { + RTX_4X8 = N_TX_SIZES, + RTX_8X4, + RTX_8X16, + RTX_16X8, + RTX_16X32, + RTX_32X16, + RTX_32X64, + RTX_64X32, + RTX_4X16, + RTX_16X4, + RTX_8X32, + RTX_32X8, + RTX_16X64, + RTX_64X16, + N_RECT_TX_SIZES +}; + +enum TxfmType { + DCT_DCT, // DCT in both horizontal and vertical + ADST_DCT, // ADST in vertical, DCT in horizontal + DCT_ADST, // DCT in vertical, ADST in horizontal + ADST_ADST, // ADST in both directions + FLIPADST_DCT, + DCT_FLIPADST, + FLIPADST_FLIPADST, + ADST_FLIPADST, + FLIPADST_ADST, + IDTX, + V_DCT, + H_DCT, + V_ADST, + H_ADST, + V_FLIPADST, + H_FLIPADST, + N_TX_TYPES, + WHT_WHT = N_TX_TYPES, + N_TX_TYPES_PLUS_LL, +}; + +enum TxClass { + TX_CLASS_2D, + TX_CLASS_H, + TX_CLASS_V, +}; + +enum IntraPredMode { + DC_PRED, + VERT_PRED, + HOR_PRED, + DIAG_DOWN_LEFT_PRED, + DIAG_DOWN_RIGHT_PRED, + VERT_RIGHT_PRED, + HOR_DOWN_PRED, + HOR_UP_PRED, + VERT_LEFT_PRED, + SMOOTH_PRED, + SMOOTH_V_PRED, + SMOOTH_H_PRED, + PAETH_PRED, + N_INTRA_PRED_MODES, + CFL_PRED = N_INTRA_PRED_MODES, + N_UV_INTRA_PRED_MODES, + N_IMPL_INTRA_PRED_MODES = N_UV_INTRA_PRED_MODES, + LEFT_DC_PRED = DIAG_DOWN_LEFT_PRED, + TOP_DC_PRED, + DC_128_PRED, + Z1_PRED, + Z2_PRED, + Z3_PRED, + FILTER_PRED = N_INTRA_PRED_MODES, +}; + +enum InterIntraPredMode { + II_DC_PRED, + II_VERT_PRED, + II_HOR_PRED, + II_SMOOTH_PRED, + N_INTER_INTRA_PRED_MODES, +}; + +enum BlockPartition { + PARTITION_NONE, // [ ] <-. + PARTITION_H, // [-] | + PARTITION_V, // [|] | + PARTITION_SPLIT, // [+] --' + PARTITION_T_TOP_SPLIT, // [⊥] i.e. split top, H bottom + PARTITION_T_BOTTOM_SPLIT, // [т] i.e. H top, split bottom + PARTITION_T_LEFT_SPLIT, // [-|] i.e. split left, V right + PARTITION_T_RIGHT_SPLIT, // [|-] i.e. V left, split right + PARTITION_H4, // [Ⲷ] + PARTITION_V4, // [Ⲽ] + N_PARTITIONS, + N_SUB8X8_PARTITIONS = PARTITION_T_TOP_SPLIT, +}; + +enum BlockSize { + BS_128x128, + BS_128x64, + BS_64x128, + BS_64x64, + BS_64x32, + BS_64x16, + BS_32x64, + BS_32x32, + BS_32x16, + BS_32x8, + BS_16x64, + BS_16x32, + BS_16x16, + BS_16x8, + BS_16x4, + BS_8x32, + BS_8x16, + BS_8x8, + BS_8x4, + BS_4x16, + BS_4x8, + BS_4x4, + N_BS_SIZES, +}; + +enum Filter2d { // order is horizontal, vertical + FILTER_2D_8TAP_REGULAR, + FILTER_2D_8TAP_REGULAR_SMOOTH, + FILTER_2D_8TAP_REGULAR_SHARP, + FILTER_2D_8TAP_SHARP_REGULAR, + FILTER_2D_8TAP_SHARP_SMOOTH, + FILTER_2D_8TAP_SHARP, + FILTER_2D_8TAP_SMOOTH_REGULAR, + FILTER_2D_8TAP_SMOOTH, + FILTER_2D_8TAP_SMOOTH_SHARP, + FILTER_2D_BILINEAR, + N_2D_FILTERS, +}; + +enum MVJoint { + MV_JOINT_ZERO, + MV_JOINT_H, + MV_JOINT_V, + MV_JOINT_HV, + N_MV_JOINTS, +}; + +enum InterPredMode { + NEARESTMV, + NEARMV, + GLOBALMV, + NEWMV, + N_INTER_PRED_MODES, +}; + +enum DRL_PROXIMITY { + NEAREST_DRL, + NEARER_DRL, + NEAR_DRL, + NEARISH_DRL +}; + +enum CompInterPredMode { + NEARESTMV_NEARESTMV, + NEARMV_NEARMV, + NEARESTMV_NEWMV, + NEWMV_NEARESTMV, + NEARMV_NEWMV, + NEWMV_NEARMV, + GLOBALMV_GLOBALMV, + NEWMV_NEWMV, + N_COMP_INTER_PRED_MODES, +}; + +enum CompInterType { + COMP_INTER_NONE, + COMP_INTER_WEIGHTED_AVG, + COMP_INTER_AVG, + COMP_INTER_SEG, + COMP_INTER_WEDGE, +}; + +enum InterIntraType { + INTER_INTRA_NONE, + INTER_INTRA_BLEND, + INTER_INTRA_WEDGE, +}; + +typedef union mv { + struct { + int16_t y, x; + }; + uint32_t n; +} mv; + +enum MotionMode { + MM_TRANSLATION, + MM_OBMC, + MM_WARP, +}; + +#define QINDEX_RANGE 256 + +typedef struct Av1Block { + uint8_t bl, bs, bp; + uint8_t intra, seg_id, skip_mode, skip, uvtx; + union { + struct { + uint8_t y_mode, uv_mode, tx, pal_sz[2]; + int8_t y_angle, uv_angle, cfl_alpha[2]; + }; // intra + struct { + union { + struct { + union mv mv[2]; + uint8_t wedge_idx, mask_sign, interintra_mode; + }; + struct { + union mv mv2d; + int16_t matrix[4]; + }; + }; + uint8_t comp_type, inter_mode, motion_mode, drl_idx; + int8_t ref[2]; + uint8_t max_ytx, filter2d, interintra_type, tx_split0; + uint16_t tx_split1; + }; // inter + }; +} Av1Block; + +#endif /* DAV1D_SRC_LEVELS_H */ diff --git a/third_party/dav1d/src/lf_apply.h b/third_party/dav1d/src/lf_apply.h new file mode 100644 index 0000000000..cf4c898550 --- /dev/null +++ b/third_party/dav1d/src/lf_apply.h @@ -0,0 +1,48 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LF_APPLY_H +#define DAV1D_SRC_LF_APPLY_H + +#include + +#include "common/bitdepth.h" + +#include "src/internal.h" +#include "src/levels.h" + +void bytefn(dav1d_loopfilter_sbrow_cols)(const Dav1dFrameContext *f, + pixel *const p[3], Av1Filter *lflvl, + int sby, int start_of_tile_row); +void bytefn(dav1d_loopfilter_sbrow_rows)(const Dav1dFrameContext *f, + pixel *const p[3], Av1Filter *lflvl, + int sby); + +void bytefn(dav1d_copy_lpf)(Dav1dFrameContext *const f, + /*const*/ pixel *const src[3], int sby); + +#endif /* DAV1D_SRC_LF_APPLY_H */ diff --git a/third_party/dav1d/src/lf_apply_tmpl.c b/third_party/dav1d/src/lf_apply_tmpl.c new file mode 100644 index 0000000000..4ef3becd82 --- /dev/null +++ b/third_party/dav1d/src/lf_apply_tmpl.c @@ -0,0 +1,466 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/lf_apply.h" +#include "src/lr_apply.h" + +// The loop filter buffer stores 12 rows of pixels. A superblock block will +// contain at most 2 stripes. Each stripe requires 4 rows pixels (2 above +// and 2 below) the final 4 rows are used to swap the bottom of the last +// stripe with the top of the next super block row. +static void backup_lpf(const Dav1dFrameContext *const f, + pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int ss_ver, const int sb128, + int row, const int row_h, const int src_w, + const int h, const int ss_hor, const int lr_backup) +{ + const int cdef_backup = !lr_backup; + const int dst_w = f->frame_hdr->super_res.enabled ? + (f->frame_hdr->width[1] + ss_hor) >> ss_hor : src_w; + + // The first stripe of the frame is shorter by 8 luma pixel rows. + int stripe_h = ((64 << (cdef_backup & sb128)) - 8 * !row) >> ss_ver; + src += (stripe_h - 2) * PXSTRIDE(src_stride); + + if (f->c->n_tc == 1) { + if (row) { + const int top = 4 << sb128; + // Copy the top part of the stored loop filtered pixels from the + // previous sb row needed above the first stripe of this sb row. + pixel_copy(&dst[PXSTRIDE(dst_stride) * 0], + &dst[PXSTRIDE(dst_stride) * top], dst_w); + pixel_copy(&dst[PXSTRIDE(dst_stride) * 1], + &dst[PXSTRIDE(dst_stride) * (top + 1)], dst_w); + pixel_copy(&dst[PXSTRIDE(dst_stride) * 2], + &dst[PXSTRIDE(dst_stride) * (top + 2)], dst_w); + pixel_copy(&dst[PXSTRIDE(dst_stride) * 3], + &dst[PXSTRIDE(dst_stride) * (top + 3)], dst_w); + } + dst += 4 * PXSTRIDE(dst_stride); + } + + if (lr_backup && (f->frame_hdr->width[0] != f->frame_hdr->width[1])) { + while (row + stripe_h <= row_h) { + const int n_lines = 4 - (row + stripe_h + 1 == h); + f->dsp->mc.resize(dst, dst_stride, src, src_stride, + dst_w, n_lines, src_w, f->resize_step[ss_hor], + f->resize_start[ss_hor] HIGHBD_CALL_SUFFIX); + row += stripe_h; // unmodified stripe_h for the 1st stripe + stripe_h = 64 >> ss_ver; + src += stripe_h * PXSTRIDE(src_stride); + dst += n_lines * PXSTRIDE(dst_stride); + if (n_lines == 3) { + pixel_copy(dst, &dst[-PXSTRIDE(dst_stride)], dst_w); + dst += PXSTRIDE(dst_stride); + } + } + } else { + while (row + stripe_h <= row_h) { + const int n_lines = 4 - (row + stripe_h + 1 == h); + for (int i = 0; i < 4; i++) { + pixel_copy(dst, i == n_lines ? &dst[-PXSTRIDE(dst_stride)] : + src, src_w); + dst += PXSTRIDE(dst_stride); + src += PXSTRIDE(src_stride); + } + row += stripe_h; // unmodified stripe_h for the 1st stripe + stripe_h = 64 >> ss_ver; + src += (stripe_h - 4) * PXSTRIDE(src_stride); + } + } +} + +void bytefn(dav1d_copy_lpf)(Dav1dFrameContext *const f, + /*const*/ pixel *const src[3], const int sby) +{ + const int have_tt = f->c->n_tc > 1; + const int resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; + const int offset = 8 * !!sby; + const ptrdiff_t *const src_stride = f->cur.stride; + const ptrdiff_t *const lr_stride = f->sr_cur.p.stride; + const int tt_off = have_tt * sby * (4 << f->seq_hdr->sb128); + pixel *const dst[3] = { + f->lf.lr_lpf_line[0] + tt_off * PXSTRIDE(lr_stride[0]), + f->lf.lr_lpf_line[1] + tt_off * PXSTRIDE(lr_stride[1]), + f->lf.lr_lpf_line[2] + tt_off * PXSTRIDE(lr_stride[1]) + }; + + // TODO Also check block level restore type to reduce copying. + const int restore_planes = f->lf.restore_planes; + + if (f->seq_hdr->cdef || restore_planes & LR_RESTORE_Y) { + const int h = f->cur.p.h; + const int w = f->bw << 2; + const int row_h = imin((sby + 1) << (6 + f->seq_hdr->sb128), h - 1); + const int y_stripe = (sby << (6 + f->seq_hdr->sb128)) - offset; + if (restore_planes & LR_RESTORE_Y || !resize) + backup_lpf(f, dst[0], lr_stride[0], + src[0] - offset * PXSTRIDE(src_stride[0]), src_stride[0], + 0, f->seq_hdr->sb128, y_stripe, row_h, w, h, 0, 1); + if (have_tt && resize) { + const ptrdiff_t cdef_off_y = sby * 4 * PXSTRIDE(src_stride[0]); + backup_lpf(f, f->lf.cdef_lpf_line[0] + cdef_off_y, src_stride[0], + src[0] - offset * PXSTRIDE(src_stride[0]), src_stride[0], + 0, f->seq_hdr->sb128, y_stripe, row_h, w, h, 0, 0); + } + } + if ((f->seq_hdr->cdef || restore_planes & (LR_RESTORE_U | LR_RESTORE_V)) && + f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400) + { + const int ss_ver = f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->sr_cur.p.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h = (f->cur.p.h + ss_ver) >> ss_ver; + const int w = f->bw << (2 - ss_hor); + const int row_h = imin((sby + 1) << ((6 - ss_ver) + f->seq_hdr->sb128), h - 1); + const int offset_uv = offset >> ss_ver; + const int y_stripe = (sby << ((6 - ss_ver) + f->seq_hdr->sb128)) - offset_uv; + const ptrdiff_t cdef_off_uv = sby * 4 * PXSTRIDE(src_stride[1]); + if (f->seq_hdr->cdef || restore_planes & LR_RESTORE_U) { + if (restore_planes & LR_RESTORE_U || !resize) + backup_lpf(f, dst[1], lr_stride[1], + src[1] - offset_uv * PXSTRIDE(src_stride[1]), + src_stride[1], ss_ver, f->seq_hdr->sb128, y_stripe, + row_h, w, h, ss_hor, 1); + if (have_tt && resize) + backup_lpf(f, f->lf.cdef_lpf_line[1] + cdef_off_uv, src_stride[1], + src[1] - offset_uv * PXSTRIDE(src_stride[1]), + src_stride[1], ss_ver, f->seq_hdr->sb128, y_stripe, + row_h, w, h, ss_hor, 0); + } + if (f->seq_hdr->cdef || restore_planes & LR_RESTORE_V) { + if (restore_planes & LR_RESTORE_V || !resize) + backup_lpf(f, dst[2], lr_stride[1], + src[2] - offset_uv * PXSTRIDE(src_stride[1]), + src_stride[1], ss_ver, f->seq_hdr->sb128, y_stripe, + row_h, w, h, ss_hor, 1); + if (have_tt && resize) + backup_lpf(f, f->lf.cdef_lpf_line[2] + cdef_off_uv, src_stride[1], + src[2] - offset_uv * PXSTRIDE(src_stride[1]), + src_stride[1], ss_ver, f->seq_hdr->sb128, y_stripe, + row_h, w, h, ss_hor, 0); + } + } +} + +static inline void filter_plane_cols_y(const Dav1dFrameContext *const f, + const int have_left, + const uint8_t (*lvl)[4], + const ptrdiff_t b4_stride, + const uint16_t (*const mask)[3][2], + pixel *dst, const ptrdiff_t ls, + const int w, + const int starty4, const int endy4) +{ + const Dav1dDSPContext *const dsp = f->dsp; + + // filter edges between columns (e.g. block1 | block2) + for (int x = 0; x < w; x++) { + if (!have_left && !x) continue; + uint32_t hmask[4]; + if (!starty4) { + hmask[0] = mask[x][0][0]; + hmask[1] = mask[x][1][0]; + hmask[2] = mask[x][2][0]; + if (endy4 > 16) { + hmask[0] |= (unsigned) mask[x][0][1] << 16; + hmask[1] |= (unsigned) mask[x][1][1] << 16; + hmask[2] |= (unsigned) mask[x][2][1] << 16; + } + } else { + hmask[0] = mask[x][0][1]; + hmask[1] = mask[x][1][1]; + hmask[2] = mask[x][2][1]; + } + hmask[3] = 0; + dsp->lf.loop_filter_sb[0][0](&dst[x * 4], ls, hmask, + (const uint8_t(*)[4]) &lvl[x][0], b4_stride, + &f->lf.lim_lut, endy4 - starty4 HIGHBD_CALL_SUFFIX); + } +} + +static inline void filter_plane_rows_y(const Dav1dFrameContext *const f, + const int have_top, + const uint8_t (*lvl)[4], + const ptrdiff_t b4_stride, + const uint16_t (*const mask)[3][2], + pixel *dst, const ptrdiff_t ls, + const int w, + const int starty4, const int endy4) +{ + const Dav1dDSPContext *const dsp = f->dsp; + + // block1 + // filter edges between rows (e.g. ------) + // block2 + for (int y = starty4; y < endy4; + y++, dst += 4 * PXSTRIDE(ls), lvl += b4_stride) + { + if (!have_top && !y) continue; + const uint32_t vmask[4] = { + mask[y][0][0] | ((unsigned) mask[y][0][1] << 16), + mask[y][1][0] | ((unsigned) mask[y][1][1] << 16), + mask[y][2][0] | ((unsigned) mask[y][2][1] << 16), + 0, + }; + dsp->lf.loop_filter_sb[0][1](dst, ls, vmask, + (const uint8_t(*)[4]) &lvl[0][1], b4_stride, + &f->lf.lim_lut, w HIGHBD_CALL_SUFFIX); + } +} + +static inline void filter_plane_cols_uv(const Dav1dFrameContext *const f, + const int have_left, + const uint8_t (*lvl)[4], + const ptrdiff_t b4_stride, + const uint16_t (*const mask)[2][2], + pixel *const u, pixel *const v, + const ptrdiff_t ls, const int w, + const int starty4, const int endy4, + const int ss_ver) +{ + const Dav1dDSPContext *const dsp = f->dsp; + + // filter edges between columns (e.g. block1 | block2) + for (int x = 0; x < w; x++) { + if (!have_left && !x) continue; + uint32_t hmask[3]; + if (!starty4) { + hmask[0] = mask[x][0][0]; + hmask[1] = mask[x][1][0]; + if (endy4 > (16 >> ss_ver)) { + hmask[0] |= (unsigned) mask[x][0][1] << (16 >> ss_ver); + hmask[1] |= (unsigned) mask[x][1][1] << (16 >> ss_ver); + } + } else { + hmask[0] = mask[x][0][1]; + hmask[1] = mask[x][1][1]; + } + hmask[2] = 0; + dsp->lf.loop_filter_sb[1][0](&u[x * 4], ls, hmask, + (const uint8_t(*)[4]) &lvl[x][2], b4_stride, + &f->lf.lim_lut, endy4 - starty4 HIGHBD_CALL_SUFFIX); + dsp->lf.loop_filter_sb[1][0](&v[x * 4], ls, hmask, + (const uint8_t(*)[4]) &lvl[x][3], b4_stride, + &f->lf.lim_lut, endy4 - starty4 HIGHBD_CALL_SUFFIX); + } +} + +static inline void filter_plane_rows_uv(const Dav1dFrameContext *const f, + const int have_top, + const uint8_t (*lvl)[4], + const ptrdiff_t b4_stride, + const uint16_t (*const mask)[2][2], + pixel *const u, pixel *const v, + const ptrdiff_t ls, const int w, + const int starty4, const int endy4, + const int ss_hor) +{ + const Dav1dDSPContext *const dsp = f->dsp; + ptrdiff_t off_l = 0; + + // block1 + // filter edges between rows (e.g. ------) + // block2 + for (int y = starty4; y < endy4; + y++, off_l += 4 * PXSTRIDE(ls), lvl += b4_stride) + { + if (!have_top && !y) continue; + const uint32_t vmask[3] = { + mask[y][0][0] | ((unsigned) mask[y][0][1] << (16 >> ss_hor)), + mask[y][1][0] | ((unsigned) mask[y][1][1] << (16 >> ss_hor)), + 0, + }; + dsp->lf.loop_filter_sb[1][1](&u[off_l], ls, vmask, + (const uint8_t(*)[4]) &lvl[0][2], b4_stride, + &f->lf.lim_lut, w HIGHBD_CALL_SUFFIX); + dsp->lf.loop_filter_sb[1][1](&v[off_l], ls, vmask, + (const uint8_t(*)[4]) &lvl[0][3], b4_stride, + &f->lf.lim_lut, w HIGHBD_CALL_SUFFIX); + } +} + +void bytefn(dav1d_loopfilter_sbrow_cols)(const Dav1dFrameContext *const f, + pixel *const p[3], Av1Filter *const lflvl, + int sby, const int start_of_tile_row) +{ + int x, have_left; + // Don't filter outside the frame + const int is_sb64 = !f->seq_hdr->sb128; + const int starty4 = (sby & is_sb64) << 4; + const int sbsz = 32 >> is_sb64; + const int sbl2 = 5 - is_sb64; + const int halign = (f->bh + 31) & ~31; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int vmask = 16 >> ss_ver, hmask = 16 >> ss_hor; + const unsigned vmax = 1U << vmask, hmax = 1U << hmask; + const unsigned endy4 = starty4 + imin(f->h4 - sby * sbsz, sbsz); + const unsigned uv_endy4 = (endy4 + ss_ver) >> ss_ver; + + // fix lpf strength at tile col boundaries + const uint8_t *lpf_y = &f->lf.tx_lpf_right_edge[0][sby << sbl2]; + const uint8_t *lpf_uv = &f->lf.tx_lpf_right_edge[1][sby << (sbl2 - ss_ver)]; + for (int tile_col = 1;; tile_col++) { + x = f->frame_hdr->tiling.col_start_sb[tile_col]; + if ((x << sbl2) >= f->bw) break; + const int bx4 = x & is_sb64 ? 16 : 0, cbx4 = bx4 >> ss_hor; + x >>= is_sb64; + + uint16_t (*const y_hmask)[2] = lflvl[x].filter_y[0][bx4]; + for (unsigned y = starty4, mask = 1 << y; y < endy4; y++, mask <<= 1) { + const int sidx = mask >= 0x10000U; + const unsigned smask = mask >> (sidx << 4); + const int idx = 2 * !!(y_hmask[2][sidx] & smask) + + !!(y_hmask[1][sidx] & smask); + y_hmask[2][sidx] &= ~smask; + y_hmask[1][sidx] &= ~smask; + y_hmask[0][sidx] &= ~smask; + y_hmask[imin(idx, lpf_y[y - starty4])][sidx] |= smask; + } + + if (f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400) { + uint16_t (*const uv_hmask)[2] = lflvl[x].filter_uv[0][cbx4]; + for (unsigned y = starty4 >> ss_ver, uv_mask = 1 << y; y < uv_endy4; + y++, uv_mask <<= 1) + { + const int sidx = uv_mask >= vmax; + const unsigned smask = uv_mask >> (sidx << (4 - ss_ver)); + const int idx = !!(uv_hmask[1][sidx] & smask); + uv_hmask[1][sidx] &= ~smask; + uv_hmask[0][sidx] &= ~smask; + uv_hmask[imin(idx, lpf_uv[y - (starty4 >> ss_ver)])][sidx] |= smask; + } + } + lpf_y += halign; + lpf_uv += halign >> ss_ver; + } + + // fix lpf strength at tile row boundaries + if (start_of_tile_row) { + const BlockContext *a; + for (x = 0, a = &f->a[f->sb128w * (start_of_tile_row - 1)]; + x < f->sb128w; x++, a++) + { + uint16_t (*const y_vmask)[2] = lflvl[x].filter_y[1][starty4]; + const unsigned w = imin(32, f->w4 - (x << 5)); + for (unsigned mask = 1, i = 0; i < w; mask <<= 1, i++) { + const int sidx = mask >= 0x10000U; + const unsigned smask = mask >> (sidx << 4); + const int idx = 2 * !!(y_vmask[2][sidx] & smask) + + !!(y_vmask[1][sidx] & smask); + y_vmask[2][sidx] &= ~smask; + y_vmask[1][sidx] &= ~smask; + y_vmask[0][sidx] &= ~smask; + y_vmask[imin(idx, a->tx_lpf_y[i])][sidx] |= smask; + } + + if (f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400) { + const unsigned cw = (w + ss_hor) >> ss_hor; + uint16_t (*const uv_vmask)[2] = lflvl[x].filter_uv[1][starty4 >> ss_ver]; + for (unsigned uv_mask = 1, i = 0; i < cw; uv_mask <<= 1, i++) { + const int sidx = uv_mask >= hmax; + const unsigned smask = uv_mask >> (sidx << (4 - ss_hor)); + const int idx = !!(uv_vmask[1][sidx] & smask); + uv_vmask[1][sidx] &= ~smask; + uv_vmask[0][sidx] &= ~smask; + uv_vmask[imin(idx, a->tx_lpf_uv[i])][sidx] |= smask; + } + } + } + } + + pixel *ptr; + uint8_t (*level_ptr)[4] = f->lf.level + f->b4_stride * sby * sbsz; + for (ptr = p[0], have_left = 0, x = 0; x < f->sb128w; + x++, have_left = 1, ptr += 128, level_ptr += 32) + { + filter_plane_cols_y(f, have_left, level_ptr, f->b4_stride, + lflvl[x].filter_y[0], ptr, f->cur.stride[0], + imin(32, f->w4 - x * 32), starty4, endy4); + } + + if (!f->frame_hdr->loopfilter.level_u && !f->frame_hdr->loopfilter.level_v) + return; + + ptrdiff_t uv_off; + level_ptr = f->lf.level + f->b4_stride * (sby * sbsz >> ss_ver); + for (uv_off = 0, have_left = 0, x = 0; x < f->sb128w; + x++, have_left = 1, uv_off += 128 >> ss_hor, level_ptr += 32 >> ss_hor) + { + filter_plane_cols_uv(f, have_left, level_ptr, f->b4_stride, + lflvl[x].filter_uv[0], + &p[1][uv_off], &p[2][uv_off], f->cur.stride[1], + (imin(32, f->w4 - x * 32) + ss_hor) >> ss_hor, + starty4 >> ss_ver, uv_endy4, ss_ver); + } +} + +void bytefn(dav1d_loopfilter_sbrow_rows)(const Dav1dFrameContext *const f, + pixel *const p[3], Av1Filter *const lflvl, + int sby) +{ + int x; + // Don't filter outside the frame + const int have_top = sby > 0; + const int is_sb64 = !f->seq_hdr->sb128; + const int starty4 = (sby & is_sb64) << 4; + const int sbsz = 32 >> is_sb64; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const unsigned endy4 = starty4 + imin(f->h4 - sby * sbsz, sbsz); + const unsigned uv_endy4 = (endy4 + ss_ver) >> ss_ver; + + pixel *ptr; + uint8_t (*level_ptr)[4] = f->lf.level + f->b4_stride * sby * sbsz; + for (ptr = p[0], x = 0; x < f->sb128w; x++, ptr += 128, level_ptr += 32) { + filter_plane_rows_y(f, have_top, level_ptr, f->b4_stride, + lflvl[x].filter_y[1], ptr, f->cur.stride[0], + imin(32, f->w4 - x * 32), starty4, endy4); + } + + if (!f->frame_hdr->loopfilter.level_u && !f->frame_hdr->loopfilter.level_v) + return; + + ptrdiff_t uv_off; + level_ptr = f->lf.level + f->b4_stride * (sby * sbsz >> ss_ver); + for (uv_off = 0, x = 0; x < f->sb128w; + x++, uv_off += 128 >> ss_hor, level_ptr += 32 >> ss_hor) + { + filter_plane_rows_uv(f, have_top, level_ptr, f->b4_stride, + lflvl[x].filter_uv[1], + &p[1][uv_off], &p[2][uv_off], f->cur.stride[1], + (imin(32, f->w4 - x * 32) + ss_hor) >> ss_hor, + starty4 >> ss_ver, uv_endy4, ss_hor); + } +} diff --git a/third_party/dav1d/src/lf_mask.c b/third_party/dav1d/src/lf_mask.c new file mode 100644 index 0000000000..91fe4a02c8 --- /dev/null +++ b/third_party/dav1d/src/lf_mask.c @@ -0,0 +1,491 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/ctx.h" +#include "src/levels.h" +#include "src/lf_mask.h" +#include "src/tables.h" + +static void decomp_tx(uint8_t (*const txa)[2 /* txsz, step */][32 /* y */][32 /* x */], + const enum RectTxfmSize from, + const int depth, + const int y_off, const int x_off, + const uint16_t *const tx_masks) +{ + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[from]; + const int is_split = (from == (int) TX_4X4 || depth > 1) ? 0 : + (tx_masks[depth] >> (y_off * 4 + x_off)) & 1; + + if (is_split) { + const enum RectTxfmSize sub = t_dim->sub; + const int htw4 = t_dim->w >> 1, hth4 = t_dim->h >> 1; + + decomp_tx(txa, sub, depth + 1, y_off * 2 + 0, x_off * 2 + 0, tx_masks); + if (t_dim->w >= t_dim->h) + decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][0][htw4], + sub, depth + 1, y_off * 2 + 0, x_off * 2 + 1, tx_masks); + if (t_dim->h >= t_dim->w) { + decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][hth4][0], + sub, depth + 1, y_off * 2 + 1, x_off * 2 + 0, tx_masks); + if (t_dim->w >= t_dim->h) + decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][hth4][htw4], + sub, depth + 1, y_off * 2 + 1, x_off * 2 + 1, tx_masks); + } + } else { + const int lw = imin(2, t_dim->lw), lh = imin(2, t_dim->lh); + +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + for (int y = 0; y < t_dim->h; y++) { \ + rep_macro(type, txa[0][0][y], off, mul * lw); \ + rep_macro(type, txa[1][0][y], off, mul * lh); \ + txa[0][1][y][0] = t_dim->w; \ + } + case_set_upto16(t_dim->w,,, 0); +#undef set_ctx +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, txa[1][1][0], off, mul * t_dim->h) + case_set_upto16(t_dim->w,,, 0); +#undef set_ctx + } +} + +static inline void mask_edges_inter(uint16_t (*const masks)[32][3][2], + const int by4, const int bx4, + const int w4, const int h4, const int skip, + const enum RectTxfmSize max_tx, + const uint16_t *const tx_masks, + uint8_t *const a, uint8_t *const l) +{ + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[max_tx]; + int y, x; + + ALIGN_STK_16(uint8_t, txa, 2 /* edge */, [2 /* txsz, step */][32 /* y */][32 /* x */]); + for (int y_off = 0, y = 0; y < h4; y += t_dim->h, y_off++) + for (int x_off = 0, x = 0; x < w4; x += t_dim->w, x_off++) + decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][y][x], + max_tx, 0, y_off, x_off, tx_masks); + + // left block edge + unsigned mask = 1U << by4; + for (y = 0; y < h4; y++, mask <<= 1) { + const int sidx = mask >= 0x10000; + const unsigned smask = mask >> (sidx << 4); + masks[0][bx4][imin(txa[0][0][y][0], l[y])][sidx] |= smask; + } + + // top block edge + for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) { + const int sidx = mask >= 0x10000; + const unsigned smask = mask >> (sidx << 4); + masks[1][by4][imin(txa[1][0][0][x], a[x])][sidx] |= smask; + } + + if (!skip) { + // inner (tx) left|right edges + for (y = 0, mask = 1U << by4; y < h4; y++, mask <<= 1) { + const int sidx = mask >= 0x10000U; + const unsigned smask = mask >> (sidx << 4); + int ltx = txa[0][0][y][0]; + int step = txa[0][1][y][0]; + for (x = step; x < w4; x += step) { + const int rtx = txa[0][0][y][x]; + masks[0][bx4 + x][imin(rtx, ltx)][sidx] |= smask; + ltx = rtx; + step = txa[0][1][y][x]; + } + } + + // top + // inner (tx) --- edges + // bottom + for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) { + const int sidx = mask >= 0x10000U; + const unsigned smask = mask >> (sidx << 4); + int ttx = txa[1][0][0][x]; + int step = txa[1][1][0][x]; + for (y = step; y < h4; y += step) { + const int btx = txa[1][0][y][x]; + masks[1][by4 + y][imin(ttx, btx)][sidx] |= smask; + ttx = btx; + step = txa[1][1][y][x]; + } + } + } + + for (y = 0; y < h4; y++) + l[y] = txa[0][0][y][w4 - 1]; + memcpy(a, txa[1][0][h4 - 1], w4); +} + +static inline void mask_edges_intra(uint16_t (*const masks)[32][3][2], + const int by4, const int bx4, + const int w4, const int h4, + const enum RectTxfmSize tx, + uint8_t *const a, uint8_t *const l) +{ + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx]; + const int twl4 = t_dim->lw, thl4 = t_dim->lh; + const int twl4c = imin(2, twl4), thl4c = imin(2, thl4); + int y, x; + + // left block edge + unsigned mask = 1U << by4; + for (y = 0; y < h4; y++, mask <<= 1) { + const int sidx = mask >= 0x10000; + const unsigned smask = mask >> (sidx << 4); + masks[0][bx4][imin(twl4c, l[y])][sidx] |= smask; + } + + // top block edge + for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) { + const int sidx = mask >= 0x10000; + const unsigned smask = mask >> (sidx << 4); + masks[1][by4][imin(thl4c, a[x])][sidx] |= smask; + } + + // inner (tx) left|right edges + const int hstep = t_dim->w; + unsigned t = 1U << by4; + unsigned inner = (unsigned) ((((uint64_t) t) << h4) - t); + unsigned inner1 = inner & 0xffff, inner2 = inner >> 16; + for (x = hstep; x < w4; x += hstep) { + if (inner1) masks[0][bx4 + x][twl4c][0] |= inner1; + if (inner2) masks[0][bx4 + x][twl4c][1] |= inner2; + } + + // top + // inner (tx) --- edges + // bottom + const int vstep = t_dim->h; + t = 1U << bx4; + inner = (unsigned) ((((uint64_t) t) << w4) - t); + inner1 = inner & 0xffff; + inner2 = inner >> 16; + for (y = vstep; y < h4; y += vstep) { + if (inner1) masks[1][by4 + y][thl4c][0] |= inner1; + if (inner2) masks[1][by4 + y][thl4c][1] |= inner2; + } + +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, a, off, mul * thl4c) +#define default_memset(dir, diridx, off, var) \ + memset(a, thl4c, var) + case_set_upto32_with_default(w4,,, 0); +#undef default_memset +#undef set_ctx +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, l, off, mul * twl4c) +#define default_memset(dir, diridx, off, var) \ + memset(l, twl4c, var) + case_set_upto32_with_default(h4,,, 0); +#undef default_memset +#undef set_ctx +} + +static void mask_edges_chroma(uint16_t (*const masks)[32][2][2], + const int cby4, const int cbx4, + const int cw4, const int ch4, + const int skip_inter, + const enum RectTxfmSize tx, + uint8_t *const a, uint8_t *const l, + const int ss_hor, const int ss_ver) +{ + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx]; + const int twl4 = t_dim->lw, thl4 = t_dim->lh; + const int twl4c = !!twl4, thl4c = !!thl4; + int y, x; + const int vbits = 4 - ss_ver, hbits = 4 - ss_hor; + const int vmask = 16 >> ss_ver, hmask = 16 >> ss_hor; + const unsigned vmax = 1 << vmask, hmax = 1 << hmask; + + // left block edge + unsigned mask = 1U << cby4; + for (y = 0; y < ch4; y++, mask <<= 1) { + const int sidx = mask >= vmax; + const unsigned smask = mask >> (sidx << vbits); + masks[0][cbx4][imin(twl4c, l[y])][sidx] |= smask; + } + + // top block edge + for (x = 0, mask = 1U << cbx4; x < cw4; x++, mask <<= 1) { + const int sidx = mask >= hmax; + const unsigned smask = mask >> (sidx << hbits); + masks[1][cby4][imin(thl4c, a[x])][sidx] |= smask; + } + + if (!skip_inter) { + // inner (tx) left|right edges + const int hstep = t_dim->w; + unsigned t = 1U << cby4; + unsigned inner = (unsigned) ((((uint64_t) t) << ch4) - t); + unsigned inner1 = inner & ((1 << vmask) - 1), inner2 = inner >> vmask; + for (x = hstep; x < cw4; x += hstep) { + if (inner1) masks[0][cbx4 + x][twl4c][0] |= inner1; + if (inner2) masks[0][cbx4 + x][twl4c][1] |= inner2; + } + + // top + // inner (tx) --- edges + // bottom + const int vstep = t_dim->h; + t = 1U << cbx4; + inner = (unsigned) ((((uint64_t) t) << cw4) - t); + inner1 = inner & ((1 << hmask) - 1), inner2 = inner >> hmask; + for (y = vstep; y < ch4; y += vstep) { + if (inner1) masks[1][cby4 + y][thl4c][0] |= inner1; + if (inner2) masks[1][cby4 + y][thl4c][1] |= inner2; + } + } + +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, a, off, mul * thl4c) +#define default_memset(dir, diridx, off, var) \ + memset(a, thl4c, var) + case_set_upto32_with_default(cw4,,, 0); +#undef default_memset +#undef set_ctx +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, l, off, mul * twl4c) +#define default_memset(dir, diridx, off, var) \ + memset(l, twl4c, var) + case_set_upto32_with_default(ch4,,, 0); +#undef default_memset +#undef set_ctx +} + +void dav1d_create_lf_mask_intra(Av1Filter *const lflvl, + uint8_t (*const level_cache)[4], + const ptrdiff_t b4_stride, + const uint8_t (*filter_level)[8][2], + const int bx, const int by, + const int iw, const int ih, + const enum BlockSize bs, + const enum RectTxfmSize ytx, + const enum RectTxfmSize uvtx, + const enum Dav1dPixelLayout layout, + uint8_t *const ay, uint8_t *const ly, + uint8_t *const auv, uint8_t *const luv) +{ + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = imin(iw - bx, b_dim[0]); + const int bh4 = imin(ih - by, b_dim[1]); + const int bx4 = bx & 31; + const int by4 = by & 31; + + if (bw4 && bh4) { + uint8_t (*level_cache_ptr)[4] = level_cache + by * b4_stride + bx; + for (int y = 0; y < bh4; y++) { + for (int x = 0; x < bw4; x++) { + level_cache_ptr[x][0] = filter_level[0][0][0]; + level_cache_ptr[x][1] = filter_level[1][0][0]; + } + level_cache_ptr += b4_stride; + } + + mask_edges_intra(lflvl->filter_y, by4, bx4, bw4, bh4, ytx, ay, ly); + } + + if (!auv) return; + + const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const int cbw4 = imin(((iw + ss_hor) >> ss_hor) - (bx >> ss_hor), + (b_dim[0] + ss_hor) >> ss_hor); + const int cbh4 = imin(((ih + ss_ver) >> ss_ver) - (by >> ss_ver), + (b_dim[1] + ss_ver) >> ss_ver); + + if (!cbw4 || !cbh4) return; + + const int cbx4 = bx4 >> ss_hor; + const int cby4 = by4 >> ss_ver; + + uint8_t (*level_cache_ptr)[4] = + level_cache + (by >> ss_ver) * b4_stride + (bx >> ss_hor); + for (int y = 0; y < cbh4; y++) { + for (int x = 0; x < cbw4; x++) { + level_cache_ptr[x][2] = filter_level[2][0][0]; + level_cache_ptr[x][3] = filter_level[3][0][0]; + } + level_cache_ptr += b4_stride; + } + + mask_edges_chroma(lflvl->filter_uv, cby4, cbx4, cbw4, cbh4, 0, uvtx, + auv, luv, ss_hor, ss_ver); +} + +void dav1d_create_lf_mask_inter(Av1Filter *const lflvl, + uint8_t (*const level_cache)[4], + const ptrdiff_t b4_stride, + const uint8_t (*filter_level)[8][2], + const int bx, const int by, + const int iw, const int ih, + const int skip, const enum BlockSize bs, + const enum RectTxfmSize max_ytx, + const uint16_t *const tx_masks, + const enum RectTxfmSize uvtx, + const enum Dav1dPixelLayout layout, + uint8_t *const ay, uint8_t *const ly, + uint8_t *const auv, uint8_t *const luv) +{ + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = imin(iw - bx, b_dim[0]); + const int bh4 = imin(ih - by, b_dim[1]); + const int bx4 = bx & 31; + const int by4 = by & 31; + + if (bw4 && bh4) { + uint8_t (*level_cache_ptr)[4] = level_cache + by * b4_stride + bx; + for (int y = 0; y < bh4; y++) { + for (int x = 0; x < bw4; x++) { + level_cache_ptr[x][0] = filter_level[0][0][0]; + level_cache_ptr[x][1] = filter_level[1][0][0]; + } + level_cache_ptr += b4_stride; + } + + mask_edges_inter(lflvl->filter_y, by4, bx4, bw4, bh4, skip, + max_ytx, tx_masks, ay, ly); + } + + if (!auv) return; + + const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const int cbw4 = imin(((iw + ss_hor) >> ss_hor) - (bx >> ss_hor), + (b_dim[0] + ss_hor) >> ss_hor); + const int cbh4 = imin(((ih + ss_ver) >> ss_ver) - (by >> ss_ver), + (b_dim[1] + ss_ver) >> ss_ver); + + if (!cbw4 || !cbh4) return; + + const int cbx4 = bx4 >> ss_hor; + const int cby4 = by4 >> ss_ver; + + uint8_t (*level_cache_ptr)[4] = + level_cache + (by >> ss_ver) * b4_stride + (bx >> ss_hor); + for (int y = 0; y < cbh4; y++) { + for (int x = 0; x < cbw4; x++) { + level_cache_ptr[x][2] = filter_level[2][0][0]; + level_cache_ptr[x][3] = filter_level[3][0][0]; + } + level_cache_ptr += b4_stride; + } + + mask_edges_chroma(lflvl->filter_uv, cby4, cbx4, cbw4, cbh4, skip, uvtx, + auv, luv, ss_hor, ss_ver); +} + +void dav1d_calc_eih(Av1FilterLUT *const lim_lut, const int filter_sharpness) { + // set E/I/H values from loopfilter level + const int sharp = filter_sharpness; + for (int level = 0; level < 64; level++) { + int limit = level; + + if (sharp > 0) { + limit >>= (sharp + 3) >> 2; + limit = imin(limit, 9 - sharp); + } + limit = imax(limit, 1); + + lim_lut->i[level] = limit; + lim_lut->e[level] = 2 * (level + 2) + limit; + } + lim_lut->sharp[0] = (sharp + 3) >> 2; + lim_lut->sharp[1] = sharp ? 9 - sharp : 0xff; +} + +static void calc_lf_value(uint8_t (*const lflvl_values)[2], + const int base_lvl, const int lf_delta, + const int seg_delta, + const Dav1dLoopfilterModeRefDeltas *const mr_delta) +{ + const int base = iclip(iclip(base_lvl + lf_delta, 0, 63) + seg_delta, 0, 63); + + if (!mr_delta) { + memset(lflvl_values, base, 8 * 2); + } else { + const int sh = base >= 32; + lflvl_values[0][0] = lflvl_values[0][1] = + iclip(base + (mr_delta->ref_delta[0] * (1 << sh)), 0, 63); + for (int r = 1; r < 8; r++) { + for (int m = 0; m < 2; m++) { + const int delta = + mr_delta->mode_delta[m] + mr_delta->ref_delta[r]; + lflvl_values[r][m] = iclip(base + (delta * (1 << sh)), 0, 63); + } + } + } +} + +static inline void calc_lf_value_chroma(uint8_t (*const lflvl_values)[2], + const int base_lvl, const int lf_delta, + const int seg_delta, + const Dav1dLoopfilterModeRefDeltas *const mr_delta) +{ + if (!base_lvl) + memset(lflvl_values, 0, 8 * 2); + else + calc_lf_value(lflvl_values, base_lvl, lf_delta, seg_delta, mr_delta); +} + +void dav1d_calc_lf_values(uint8_t (*const lflvl_values)[4][8][2], + const Dav1dFrameHeader *const hdr, + const int8_t lf_delta[4]) +{ + const int n_seg = hdr->segmentation.enabled ? 8 : 1; + + if (!hdr->loopfilter.level_y[0] && !hdr->loopfilter.level_y[1]) { + memset(lflvl_values, 0, 8 * 4 * 2 * n_seg); + return; + } + + const Dav1dLoopfilterModeRefDeltas *const mr_deltas = + hdr->loopfilter.mode_ref_delta_enabled ? + &hdr->loopfilter.mode_ref_deltas : NULL; + for (int s = 0; s < n_seg; s++) { + const Dav1dSegmentationData *const segd = + hdr->segmentation.enabled ? &hdr->segmentation.seg_data.d[s] : NULL; + + calc_lf_value(lflvl_values[s][0], hdr->loopfilter.level_y[0], + lf_delta[0], segd ? segd->delta_lf_y_v : 0, mr_deltas); + calc_lf_value(lflvl_values[s][1], hdr->loopfilter.level_y[1], + lf_delta[hdr->delta.lf.multi ? 1 : 0], + segd ? segd->delta_lf_y_h : 0, mr_deltas); + calc_lf_value_chroma(lflvl_values[s][2], hdr->loopfilter.level_u, + lf_delta[hdr->delta.lf.multi ? 2 : 0], + segd ? segd->delta_lf_u : 0, mr_deltas); + calc_lf_value_chroma(lflvl_values[s][3], hdr->loopfilter.level_v, + lf_delta[hdr->delta.lf.multi ? 3 : 0], + segd ? segd->delta_lf_v : 0, mr_deltas); + } +} diff --git a/third_party/dav1d/src/lf_mask.h b/third_party/dav1d/src/lf_mask.h new file mode 100644 index 0000000000..5edf4a0932 --- /dev/null +++ b/third_party/dav1d/src/lf_mask.h @@ -0,0 +1,83 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LF_MASK_H +#define DAV1D_SRC_LF_MASK_H + +#include +#include + +#include "src/levels.h" + +typedef struct Av1FilterLUT { + uint8_t e[64]; + uint8_t i[64]; + uint64_t sharp[2]; +} Av1FilterLUT; + +typedef struct Av1RestorationUnit { + uint8_t /* enum Dav1dRestorationType */ type; + int8_t filter_h[3]; + int8_t filter_v[3]; + uint8_t sgr_idx; + int8_t sgr_weights[2]; +} Av1RestorationUnit; + +// each struct describes one 128x128 area (1 or 4 SBs), pre-superres-scaling +typedef struct Av1Filter { + // each bit is 1 col + uint16_t filter_y[2 /* 0=col, 1=row */][32][3][2]; + uint16_t filter_uv[2 /* 0=col, 1=row */][32][2][2]; + int8_t cdef_idx[4]; // -1 means "unset" + uint16_t noskip_mask[16][2]; // for 8x8 blocks, but stored on a 4x8 basis +} Av1Filter; + +// each struct describes one 128x128 area (1 or 4 SBs), post-superres-scaling +typedef struct Av1Restoration { + Av1RestorationUnit lr[3][4]; +} Av1Restoration; + +void dav1d_create_lf_mask_intra(Av1Filter *lflvl, uint8_t (*level_cache)[4], + const ptrdiff_t b4_stride, + const uint8_t (*level)[8][2], int bx, int by, + int iw, int ih, enum BlockSize bs, + enum RectTxfmSize ytx, enum RectTxfmSize uvtx, + enum Dav1dPixelLayout layout, uint8_t *ay, + uint8_t *ly, uint8_t *auv, uint8_t *luv); +void dav1d_create_lf_mask_inter(Av1Filter *lflvl, uint8_t (*level_cache)[4], + const ptrdiff_t b4_stride, + const uint8_t (*level)[8][2], int bx, int by, + int iw, int ih, int skip_inter, + enum BlockSize bs, enum RectTxfmSize max_ytx, + const uint16_t *tx_mask, enum RectTxfmSize uvtx, + enum Dav1dPixelLayout layout, uint8_t *ay, + uint8_t *ly, uint8_t *auv, uint8_t *luv); +void dav1d_calc_eih(Av1FilterLUT *lim_lut, int filter_sharpness); +void dav1d_calc_lf_values(uint8_t (*values)[4][8][2], const Dav1dFrameHeader *hdr, + const int8_t lf_delta[4]); + +#endif /* DAV1D_SRC_LF_MASK_H */ diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c new file mode 100644 index 0000000000..eca22ebe03 --- /dev/null +++ b/third_party/dav1d/src/lib.c @@ -0,0 +1,802 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "vcs_version.h" + +#include +#include + +#if defined(__linux__) && defined(HAVE_DLSYM) +#include +#endif + +#include "dav1d/dav1d.h" +#include "dav1d/data.h" + +#include "common/validate.h" + +#include "src/cpu.h" +#include "src/fg_apply.h" +#include "src/internal.h" +#include "src/log.h" +#include "src/obu.h" +#include "src/qm.h" +#include "src/ref.h" +#include "src/thread_task.h" +#include "src/wedge.h" + +static COLD void init_internal(void) { + dav1d_init_cpu(); + dav1d_init_interintra_masks(); + dav1d_init_qm_tables(); + dav1d_init_thread(); + dav1d_init_wedge_masks(); +} + +COLD const char *dav1d_version(void) { + return DAV1D_VERSION; +} + +COLD void dav1d_default_settings(Dav1dSettings *const s) { + s->n_threads = 0; + s->max_frame_delay = 0; + s->apply_grain = 1; + s->allocator.cookie = NULL; + s->allocator.alloc_picture_callback = dav1d_default_picture_alloc; + s->allocator.release_picture_callback = dav1d_default_picture_release; + s->logger.cookie = NULL; + s->logger.callback = dav1d_log_default_callback; + s->operating_point = 0; + s->all_layers = 1; // just until the tests are adjusted + s->frame_size_limit = 0; + s->strict_std_compliance = 0; + s->output_invisible_frames = 0; + s->inloop_filters = DAV1D_INLOOPFILTER_ALL; + s->decode_frame_type = DAV1D_DECODEFRAMETYPE_ALL; +} + +static void close_internal(Dav1dContext **const c_out, int flush); + +NO_SANITIZE("cfi-icall") // CFI is broken with dlsym() +static COLD size_t get_stack_size_internal(const pthread_attr_t *const thread_attr) { +#if defined(__linux__) && defined(HAVE_DLSYM) && defined(__GLIBC__) + /* glibc has an issue where the size of the TLS is subtracted from the stack + * size instead of allocated separately. As a result the specified stack + * size may be insufficient when used in an application with large amounts + * of TLS data. The following is a workaround to compensate for that. + * See https://sourceware.org/bugzilla/show_bug.cgi?id=11787 */ + size_t (*const get_minstack)(const pthread_attr_t*) = + dlsym(RTLD_DEFAULT, "__pthread_get_minstack"); + if (get_minstack) + return get_minstack(thread_attr) - PTHREAD_STACK_MIN; +#endif + return 0; +} + +static COLD void get_num_threads(Dav1dContext *const c, const Dav1dSettings *const s, + unsigned *n_tc, unsigned *n_fc) +{ + /* ceil(sqrt(n)) */ + static const uint8_t fc_lut[49] = { + 1, /* 1 */ + 2, 2, 2, /* 2- 4 */ + 3, 3, 3, 3, 3, /* 5- 9 */ + 4, 4, 4, 4, 4, 4, 4, /* 10-16 */ + 5, 5, 5, 5, 5, 5, 5, 5, 5, /* 17-25 */ + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 26-36 */ + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* 37-49 */ + }; + *n_tc = s->n_threads ? s->n_threads : + iclip(dav1d_num_logical_processors(c), 1, DAV1D_MAX_THREADS); + *n_fc = s->max_frame_delay ? umin(s->max_frame_delay, *n_tc) : + *n_tc < 50 ? fc_lut[*n_tc - 1] : 8; // min(8, ceil(sqrt(n))) +} + +COLD int dav1d_get_frame_delay(const Dav1dSettings *const s) { + unsigned n_tc, n_fc; + validate_input_or_ret(s != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->n_threads >= 0 && + s->n_threads <= DAV1D_MAX_THREADS, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->max_frame_delay >= 0 && + s->max_frame_delay <= DAV1D_MAX_FRAME_DELAY, DAV1D_ERR(EINVAL)); + + get_num_threads(NULL, s, &n_tc, &n_fc); + return n_fc; +} + +COLD int dav1d_open(Dav1dContext **const c_out, const Dav1dSettings *const s) { + static pthread_once_t initted = PTHREAD_ONCE_INIT; + pthread_once(&initted, init_internal); + + validate_input_or_ret(c_out != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->n_threads >= 0 && + s->n_threads <= DAV1D_MAX_THREADS, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->max_frame_delay >= 0 && + s->max_frame_delay <= DAV1D_MAX_FRAME_DELAY, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->allocator.alloc_picture_callback != NULL, + DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->allocator.release_picture_callback != NULL, + DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->operating_point >= 0 && + s->operating_point <= 31, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->decode_frame_type >= DAV1D_DECODEFRAMETYPE_ALL && + s->decode_frame_type <= DAV1D_DECODEFRAMETYPE_KEY, DAV1D_ERR(EINVAL)); + + pthread_attr_t thread_attr; + if (pthread_attr_init(&thread_attr)) return DAV1D_ERR(ENOMEM); + size_t stack_size = 1024 * 1024 + get_stack_size_internal(&thread_attr); + + pthread_attr_setstacksize(&thread_attr, stack_size); + + Dav1dContext *const c = *c_out = dav1d_alloc_aligned(sizeof(*c), 64); + if (!c) goto error; + memset(c, 0, sizeof(*c)); + + c->allocator = s->allocator; + c->logger = s->logger; + c->apply_grain = s->apply_grain; + c->operating_point = s->operating_point; + c->all_layers = s->all_layers; + c->frame_size_limit = s->frame_size_limit; + c->strict_std_compliance = s->strict_std_compliance; + c->output_invisible_frames = s->output_invisible_frames; + c->inloop_filters = s->inloop_filters; + c->decode_frame_type = s->decode_frame_type; + + dav1d_data_props_set_defaults(&c->cached_error_props); + + if (dav1d_mem_pool_init(&c->seq_hdr_pool) || + dav1d_mem_pool_init(&c->frame_hdr_pool) || + dav1d_mem_pool_init(&c->segmap_pool) || + dav1d_mem_pool_init(&c->refmvs_pool) || + dav1d_mem_pool_init(&c->cdf_pool)) + { + goto error; + } + + if (c->allocator.alloc_picture_callback == dav1d_default_picture_alloc && + c->allocator.release_picture_callback == dav1d_default_picture_release) + { + if (c->allocator.cookie) goto error; + if (dav1d_mem_pool_init(&c->picture_pool)) goto error; + c->allocator.cookie = c->picture_pool; + } else if (c->allocator.alloc_picture_callback == dav1d_default_picture_alloc || + c->allocator.release_picture_callback == dav1d_default_picture_release) + { + goto error; + } + + /* On 32-bit systems extremely large frame sizes can cause overflows in + * dav1d_decode_frame() malloc size calculations. Prevent that from occuring + * by enforcing a maximum frame size limit, chosen to roughly correspond to + * the largest size possible to decode without exhausting virtual memory. */ + if (sizeof(size_t) < 8 && s->frame_size_limit - 1 >= 8192 * 8192) { + c->frame_size_limit = 8192 * 8192; + if (s->frame_size_limit) + dav1d_log(c, "Frame size limit reduced from %u to %u.\n", + s->frame_size_limit, c->frame_size_limit); + } + + c->flush = &c->flush_mem; + atomic_init(c->flush, 0); + + get_num_threads(c, s, &c->n_tc, &c->n_fc); + + c->fc = dav1d_alloc_aligned(sizeof(*c->fc) * c->n_fc, 32); + if (!c->fc) goto error; + memset(c->fc, 0, sizeof(*c->fc) * c->n_fc); + + c->tc = dav1d_alloc_aligned(sizeof(*c->tc) * c->n_tc, 64); + if (!c->tc) goto error; + memset(c->tc, 0, sizeof(*c->tc) * c->n_tc); + if (c->n_tc > 1) { + if (pthread_mutex_init(&c->task_thread.lock, NULL)) goto error; + if (pthread_cond_init(&c->task_thread.cond, NULL)) { + pthread_mutex_destroy(&c->task_thread.lock); + goto error; + } + if (pthread_cond_init(&c->task_thread.delayed_fg.cond, NULL)) { + pthread_cond_destroy(&c->task_thread.cond); + pthread_mutex_destroy(&c->task_thread.lock); + goto error; + } + c->task_thread.cur = c->n_fc; + atomic_init(&c->task_thread.reset_task_cur, UINT_MAX); + atomic_init(&c->task_thread.cond_signaled, 0); + c->task_thread.inited = 1; + } + + if (c->n_fc > 1) { + c->frame_thread.out_delayed = + calloc(c->n_fc, sizeof(*c->frame_thread.out_delayed)); + if (!c->frame_thread.out_delayed) goto error; + } + for (unsigned n = 0; n < c->n_fc; n++) { + Dav1dFrameContext *const f = &c->fc[n]; + if (c->n_tc > 1) { + if (pthread_mutex_init(&f->task_thread.lock, NULL)) goto error; + if (pthread_cond_init(&f->task_thread.cond, NULL)) { + pthread_mutex_destroy(&f->task_thread.lock); + goto error; + } + if (pthread_mutex_init(&f->task_thread.pending_tasks.lock, NULL)) { + pthread_cond_destroy(&f->task_thread.cond); + pthread_mutex_destroy(&f->task_thread.lock); + goto error; + } + } + f->c = c; + f->task_thread.ttd = &c->task_thread; + f->lf.last_sharpness = -1; + dav1d_refmvs_init(&f->rf); + } + + for (unsigned m = 0; m < c->n_tc; m++) { + Dav1dTaskContext *const t = &c->tc[m]; + t->f = &c->fc[0]; + t->task_thread.ttd = &c->task_thread; + t->c = c; + memset(t->cf_16bpc, 0, sizeof(t->cf_16bpc)); + if (c->n_tc > 1) { + if (pthread_mutex_init(&t->task_thread.td.lock, NULL)) goto error; + if (pthread_cond_init(&t->task_thread.td.cond, NULL)) { + pthread_mutex_destroy(&t->task_thread.td.lock); + goto error; + } + if (pthread_create(&t->task_thread.td.thread, &thread_attr, dav1d_worker_task, t)) { + pthread_cond_destroy(&t->task_thread.td.cond); + pthread_mutex_destroy(&t->task_thread.td.lock); + goto error; + } + t->task_thread.td.inited = 1; + } + } + dav1d_refmvs_dsp_init(&c->refmvs_dsp); + + // intra edge tree + c->intra_edge.root[BL_128X128] = &c->intra_edge.branch_sb128[0].node; + dav1d_init_mode_tree(c->intra_edge.root[BL_128X128], c->intra_edge.tip_sb128, 1); + c->intra_edge.root[BL_64X64] = &c->intra_edge.branch_sb64[0].node; + dav1d_init_mode_tree(c->intra_edge.root[BL_64X64], c->intra_edge.tip_sb64, 0); + + pthread_attr_destroy(&thread_attr); + + return 0; + +error: + if (c) close_internal(c_out, 0); + pthread_attr_destroy(&thread_attr); + return DAV1D_ERR(ENOMEM); +} + +static void dummy_free(const uint8_t *const data, void *const user_data) { + assert(data && !user_data); +} + +int dav1d_parse_sequence_header(Dav1dSequenceHeader *const out, + const uint8_t *const ptr, const size_t sz) +{ + Dav1dData buf = { 0 }; + int res; + + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + + Dav1dSettings s; + dav1d_default_settings(&s); + s.n_threads = 1; + s.logger.callback = NULL; + + Dav1dContext *c; + res = dav1d_open(&c, &s); + if (res < 0) return res; + + if (ptr) { + res = dav1d_data_wrap_internal(&buf, ptr, sz, dummy_free, NULL); + if (res < 0) goto error; + } + + while (buf.sz > 0) { + res = dav1d_parse_obus(c, &buf, 1); + if (res < 0) goto error; + + assert((size_t)res <= buf.sz); + buf.sz -= res; + buf.data += res; + } + + if (!c->seq_hdr) { + res = DAV1D_ERR(ENOENT); + goto error; + } + + memcpy(out, c->seq_hdr, sizeof(*out)); + + res = 0; +error: + dav1d_data_unref_internal(&buf); + dav1d_close(&c); + + return res; +} + +static int has_grain(const Dav1dPicture *const pic) +{ + const Dav1dFilmGrainData *fgdata = &pic->frame_hdr->film_grain.data; + return fgdata->num_y_points || fgdata->num_uv_points[0] || + fgdata->num_uv_points[1] || (fgdata->clip_to_restricted_range && + fgdata->chroma_scaling_from_luma); +} + +static int output_image(Dav1dContext *const c, Dav1dPicture *const out) +{ + int res = 0; + + Dav1dThreadPicture *const in = (c->all_layers || !c->max_spatial_id) + ? &c->out : &c->cache; + if (!c->apply_grain || !has_grain(&in->p)) { + dav1d_picture_move_ref(out, &in->p); + dav1d_thread_picture_unref(in); + goto end; + } + + res = dav1d_apply_grain(c, out, &in->p); + dav1d_thread_picture_unref(in); +end: + if (!c->all_layers && c->max_spatial_id && c->out.p.data[0]) { + dav1d_thread_picture_move_ref(in, &c->out); + } + return res; +} + +static int output_picture_ready(Dav1dContext *const c, const int drain) { + if (c->cached_error) return 1; + if (!c->all_layers && c->max_spatial_id) { + if (c->out.p.data[0] && c->cache.p.data[0]) { + if (c->max_spatial_id == c->cache.p.frame_hdr->spatial_id || + c->out.flags & PICTURE_FLAG_NEW_TEMPORAL_UNIT) + return 1; + dav1d_thread_picture_unref(&c->cache); + dav1d_thread_picture_move_ref(&c->cache, &c->out); + return 0; + } else if (c->cache.p.data[0] && drain) { + return 1; + } else if (c->out.p.data[0]) { + dav1d_thread_picture_move_ref(&c->cache, &c->out); + return 0; + } + } + + return !!c->out.p.data[0]; +} + +static int drain_picture(Dav1dContext *const c, Dav1dPicture *const out) { + unsigned drain_count = 0; + int drained = 0; + do { + const unsigned next = c->frame_thread.next; + Dav1dFrameContext *const f = &c->fc[next]; + pthread_mutex_lock(&c->task_thread.lock); + while (f->n_tile_data > 0) + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); + Dav1dThreadPicture *const out_delayed = + &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + unsigned first = atomic_load(&c->task_thread.first); + if (first + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + atomic_compare_exchange_strong(&c->task_thread.reset_task_cur, + &first, UINT_MAX); + if (c->task_thread.cur && c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + drained = 1; + } else if (drained) { + pthread_mutex_unlock(&c->task_thread.lock); + break; + } + if (++c->frame_thread.next == c->n_fc) + c->frame_thread.next = 0; + pthread_mutex_unlock(&c->task_thread.lock); + const int error = f->task_thread.retval; + if (error) { + f->task_thread.retval = 0; + dav1d_data_props_copy(&c->cached_error_props, &out_delayed->p.m); + dav1d_thread_picture_unref(out_delayed); + return error; + } + if (out_delayed->p.data[0]) { + const unsigned progress = + atomic_load_explicit(&out_delayed->progress[1], + memory_order_relaxed); + if ((out_delayed->visible || c->output_invisible_frames) && + progress != FRAME_ERROR) + { + dav1d_thread_picture_ref(&c->out, out_delayed); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } + dav1d_thread_picture_unref(out_delayed); + if (output_picture_ready(c, 0)) + return output_image(c, out); + } + } while (++drain_count < c->n_fc); + + if (output_picture_ready(c, 1)) + return output_image(c, out); + + return DAV1D_ERR(EAGAIN); +} + +static int gen_picture(Dav1dContext *const c) +{ + int res; + Dav1dData *const in = &c->in; + + if (output_picture_ready(c, 0)) + return 0; + + while (in->sz > 0) { + res = dav1d_parse_obus(c, in, 0); + if (res < 0) { + dav1d_data_unref_internal(in); + } else { + assert((size_t)res <= in->sz); + in->sz -= res; + in->data += res; + if (!in->sz) dav1d_data_unref_internal(in); + } + if (output_picture_ready(c, 0)) + break; + if (res < 0) + return res; + } + + return 0; +} + +int dav1d_send_data(Dav1dContext *const c, Dav1dData *const in) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in->data == NULL || in->sz, DAV1D_ERR(EINVAL)); + + if (in->data) + c->drain = 0; + if (c->in.data) + return DAV1D_ERR(EAGAIN); + dav1d_data_ref(&c->in, in); + + int res = gen_picture(c); + if (!res) + dav1d_data_unref_internal(in); + + return res; +} + +int dav1d_get_picture(Dav1dContext *const c, Dav1dPicture *const out) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + + const int drain = c->drain; + c->drain = 1; + + int res = gen_picture(c); + if (res < 0) + return res; + + if (c->cached_error) { + const int res = c->cached_error; + c->cached_error = 0; + return res; + } + + if (output_picture_ready(c, c->n_fc == 1)) + return output_image(c, out); + + if (c->n_fc > 1 && drain) + return drain_picture(c, out); + + return DAV1D_ERR(EAGAIN); +} + +int dav1d_apply_grain(Dav1dContext *const c, Dav1dPicture *const out, + const Dav1dPicture *const in) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); + + if (!has_grain(in)) { + dav1d_picture_ref(out, in); + return 0; + } + + int res = dav1d_picture_alloc_copy(c, out, in->p.w, in); + if (res < 0) goto error; + + if (c->n_tc > 1) { + dav1d_task_delayed_fg(c, out, in); + } else { + switch (out->p.bpc) { +#if CONFIG_8BPC + case 8: + dav1d_apply_grain_8bpc(&c->dsp[0].fg, out, in); + break; +#endif +#if CONFIG_16BPC + case 10: + case 12: + dav1d_apply_grain_16bpc(&c->dsp[(out->p.bpc >> 1) - 4].fg, out, in); + break; +#endif + default: abort(); + } + } + + return 0; + +error: + dav1d_picture_unref_internal(out); + return res; +} + +void dav1d_flush(Dav1dContext *const c) { + dav1d_data_unref_internal(&c->in); + if (c->out.p.frame_hdr) + dav1d_thread_picture_unref(&c->out); + if (c->cache.p.frame_hdr) + dav1d_thread_picture_unref(&c->cache); + + c->drain = 0; + c->cached_error = 0; + + for (int i = 0; i < 8; i++) { + if (c->refs[i].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[i].p); + dav1d_ref_dec(&c->refs[i].segmap); + dav1d_ref_dec(&c->refs[i].refmvs); + dav1d_cdf_thread_unref(&c->cdf[i]); + } + c->frame_hdr = NULL; + c->seq_hdr = NULL; + dav1d_ref_dec(&c->seq_hdr_ref); + + c->mastering_display = NULL; + c->content_light = NULL; + c->itut_t35 = NULL; + dav1d_ref_dec(&c->mastering_display_ref); + dav1d_ref_dec(&c->content_light_ref); + dav1d_ref_dec(&c->itut_t35_ref); + + dav1d_data_props_unref_internal(&c->cached_error_props); + + if (c->n_fc == 1 && c->n_tc == 1) return; + atomic_store(c->flush, 1); + + // stop running tasks in worker threads + if (c->n_tc > 1) { + pthread_mutex_lock(&c->task_thread.lock); + for (unsigned i = 0; i < c->n_tc; i++) { + Dav1dTaskContext *const tc = &c->tc[i]; + while (!tc->task_thread.flushed) { + pthread_cond_wait(&tc->task_thread.td.cond, &c->task_thread.lock); + } + } + for (unsigned i = 0; i < c->n_fc; i++) { + c->fc[i].task_thread.task_head = NULL; + c->fc[i].task_thread.task_tail = NULL; + c->fc[i].task_thread.task_cur_prev = NULL; + c->fc[i].task_thread.pending_tasks.head = NULL; + c->fc[i].task_thread.pending_tasks.tail = NULL; + atomic_init(&c->fc[i].task_thread.pending_tasks.merge, 0); + } + atomic_init(&c->task_thread.first, 0); + c->task_thread.cur = c->n_fc; + atomic_store(&c->task_thread.reset_task_cur, UINT_MAX); + atomic_store(&c->task_thread.cond_signaled, 0); + pthread_mutex_unlock(&c->task_thread.lock); + } + + // wait for threads to complete flushing + if (c->n_fc > 1) { + for (unsigned n = 0, next = c->frame_thread.next; n < c->n_fc; n++, next++) { + if (next == c->n_fc) next = 0; + Dav1dFrameContext *const f = &c->fc[next]; + dav1d_decode_frame_exit(f, -1); + f->n_tile_data = 0; + f->task_thread.retval = 0; + Dav1dThreadPicture *out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.frame_hdr) { + dav1d_thread_picture_unref(out_delayed); + } + } + c->frame_thread.next = 0; + } + atomic_store(c->flush, 0); +} + +COLD void dav1d_close(Dav1dContext **const c_out) { + validate_input(c_out != NULL); + close_internal(c_out, 1); +} + +static COLD void close_internal(Dav1dContext **const c_out, int flush) { + Dav1dContext *const c = *c_out; + if (!c) return; + + if (flush) dav1d_flush(c); + + if (c->tc) { + struct TaskThreadData *ttd = &c->task_thread; + if (ttd->inited) { + pthread_mutex_lock(&ttd->lock); + for (unsigned n = 0; n < c->n_tc && c->tc[n].task_thread.td.inited; n++) + c->tc[n].task_thread.die = 1; + pthread_cond_broadcast(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + for (unsigned n = 0; n < c->n_tc; n++) { + Dav1dTaskContext *const pf = &c->tc[n]; + if (!pf->task_thread.td.inited) break; + pthread_join(pf->task_thread.td.thread, NULL); + pthread_cond_destroy(&pf->task_thread.td.cond); + pthread_mutex_destroy(&pf->task_thread.td.lock); + } + pthread_cond_destroy(&ttd->delayed_fg.cond); + pthread_cond_destroy(&ttd->cond); + pthread_mutex_destroy(&ttd->lock); + } + dav1d_free_aligned(c->tc); + } + + for (unsigned n = 0; c->fc && n < c->n_fc; n++) { + Dav1dFrameContext *const f = &c->fc[n]; + + // clean-up threading stuff + if (c->n_fc > 1) { + freep(&f->tile_thread.lowest_pixel_mem); + freep(&f->frame_thread.b); + dav1d_freep_aligned(&f->frame_thread.pal_idx); + dav1d_freep_aligned(&f->frame_thread.cf); + freep(&f->frame_thread.tile_start_off); + dav1d_freep_aligned(&f->frame_thread.pal); + freep(&f->frame_thread.cbi); + } + if (c->n_tc > 1) { + pthread_mutex_destroy(&f->task_thread.pending_tasks.lock); + pthread_cond_destroy(&f->task_thread.cond); + pthread_mutex_destroy(&f->task_thread.lock); + } + freep(&f->frame_thread.frame_progress); + freep(&f->task_thread.tasks); + freep(&f->task_thread.tile_tasks[0]); + dav1d_free_aligned(f->ts); + dav1d_free_aligned(f->ipred_edge[0]); + free(f->a); + free(f->tile); + free(f->lf.mask); + free(f->lf.lr_mask); + free(f->lf.level); + free(f->lf.tx_lpf_right_edge[0]); + free(f->lf.start_of_tile_row); + dav1d_refmvs_clear(&f->rf); + dav1d_free_aligned(f->lf.cdef_line_buf); + dav1d_free_aligned(f->lf.lr_line_buf); + } + dav1d_free_aligned(c->fc); + if (c->n_fc > 1 && c->frame_thread.out_delayed) { + for (unsigned n = 0; n < c->n_fc; n++) + if (c->frame_thread.out_delayed[n].p.frame_hdr) + dav1d_thread_picture_unref(&c->frame_thread.out_delayed[n]); + free(c->frame_thread.out_delayed); + } + for (int n = 0; n < c->n_tile_data; n++) + dav1d_data_unref_internal(&c->tile[n].data); + free(c->tile); + for (int n = 0; n < 8; n++) { + dav1d_cdf_thread_unref(&c->cdf[n]); + if (c->refs[n].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[n].p); + dav1d_ref_dec(&c->refs[n].refmvs); + dav1d_ref_dec(&c->refs[n].segmap); + } + dav1d_ref_dec(&c->seq_hdr_ref); + dav1d_ref_dec(&c->frame_hdr_ref); + + dav1d_ref_dec(&c->mastering_display_ref); + dav1d_ref_dec(&c->content_light_ref); + dav1d_ref_dec(&c->itut_t35_ref); + + dav1d_mem_pool_end(c->seq_hdr_pool); + dav1d_mem_pool_end(c->frame_hdr_pool); + dav1d_mem_pool_end(c->segmap_pool); + dav1d_mem_pool_end(c->refmvs_pool); + dav1d_mem_pool_end(c->cdf_pool); + dav1d_mem_pool_end(c->picture_pool); + + dav1d_freep_aligned(c_out); +} + +int dav1d_get_event_flags(Dav1dContext *const c, enum Dav1dEventFlags *const flags) { + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(flags != NULL, DAV1D_ERR(EINVAL)); + + *flags = c->event_flags; + c->event_flags = 0; + return 0; +} + +int dav1d_get_decode_error_data_props(Dav1dContext *const c, Dav1dDataProps *const out) { + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + + dav1d_data_props_unref_internal(out); + *out = c->cached_error_props; + dav1d_data_props_set_defaults(&c->cached_error_props); + + return 0; +} + +void dav1d_picture_unref(Dav1dPicture *const p) { + dav1d_picture_unref_internal(p); +} + +uint8_t *dav1d_data_create(Dav1dData *const buf, const size_t sz) { + return dav1d_data_create_internal(buf, sz); +} + +int dav1d_data_wrap(Dav1dData *const buf, const uint8_t *const ptr, + const size_t sz, + void (*const free_callback)(const uint8_t *data, + void *user_data), + void *const user_data) +{ + return dav1d_data_wrap_internal(buf, ptr, sz, free_callback, user_data); +} + +int dav1d_data_wrap_user_data(Dav1dData *const buf, + const uint8_t *const user_data, + void (*const free_callback)(const uint8_t *user_data, + void *cookie), + void *const cookie) +{ + return dav1d_data_wrap_user_data_internal(buf, + user_data, + free_callback, + cookie); +} + +void dav1d_data_unref(Dav1dData *const buf) { + dav1d_data_unref_internal(buf); +} + +void dav1d_data_props_unref(Dav1dDataProps *const props) { + dav1d_data_props_unref_internal(props); +} diff --git a/third_party/dav1d/src/log.c b/third_party/dav1d/src/log.c new file mode 100644 index 0000000000..de6776a617 --- /dev/null +++ b/third_party/dav1d/src/log.c @@ -0,0 +1,57 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "dav1d/dav1d.h" + +#include "common/validate.h" + +#include "src/internal.h" +#include "src/log.h" + +#if CONFIG_LOG +COLD void dav1d_log_default_callback(void *const cookie, + const char *const format, va_list ap) +{ + vfprintf(stderr, format, ap); +} + +COLD void dav1d_log(Dav1dContext *const c, const char *const format, ...) { + validate_input(c != NULL); + + if (!c->logger.callback) + return; + + va_list ap; + va_start(ap, format); + c->logger.callback(c->logger.cookie, format, ap); + va_end(ap); +} +#endif diff --git a/third_party/dav1d/src/log.h b/third_party/dav1d/src/log.h new file mode 100644 index 0000000000..df32de7f25 --- /dev/null +++ b/third_party/dav1d/src/log.h @@ -0,0 +1,47 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LOG_H +#define DAV1D_SRC_LOG_H + +#include "config.h" + +#include + +#include "dav1d/dav1d.h" + +#include "common/attributes.h" + +#if CONFIG_LOG +#define dav1d_log dav1d_log +void dav1d_log_default_callback(void *cookie, const char *format, va_list ap); +void dav1d_log(Dav1dContext *c, const char *format, ...) ATTR_FORMAT_PRINTF(2, 3); +#else +#define dav1d_log_default_callback NULL +#define dav1d_log(...) do { } while(0) +#endif + +#endif /* DAV1D_SRC_LOG_H */ diff --git a/third_party/dav1d/src/loopfilter.h b/third_party/dav1d/src/loopfilter.h new file mode 100644 index 0000000000..a0f78c9657 --- /dev/null +++ b/third_party/dav1d/src/loopfilter.h @@ -0,0 +1,57 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LOOPFILTER_H +#define DAV1D_SRC_LOOPFILTER_H + +#include +#include + +#include "common/bitdepth.h" + +#include "src/levels.h" +#include "src/lf_mask.h" + +#define decl_loopfilter_sb_fn(name) \ +void (name)(pixel *dst, ptrdiff_t stride, const uint32_t *mask, \ + const uint8_t (*lvl)[4], ptrdiff_t lvl_stride, \ + const Av1FilterLUT *lut, int w HIGHBD_DECL_SUFFIX) +typedef decl_loopfilter_sb_fn(*loopfilter_sb_fn); + +typedef struct Dav1dLoopFilterDSPContext { + /* + * dimension 1: plane (0=luma, 1=chroma) + * dimension 2: 0=col-edge filter (h), 1=row-edge filter (v) + * + * dst/stride are aligned by 32 + */ + loopfilter_sb_fn loop_filter_sb[2][2]; +} Dav1dLoopFilterDSPContext; + +bitfn_decls(void dav1d_loop_filter_dsp_init, Dav1dLoopFilterDSPContext *c); + +#endif /* DAV1D_SRC_LOOPFILTER_H */ diff --git a/third_party/dav1d/src/loopfilter_tmpl.c b/third_party/dav1d/src/loopfilter_tmpl.c new file mode 100644 index 0000000000..cacf258756 --- /dev/null +++ b/third_party/dav1d/src/loopfilter_tmpl.c @@ -0,0 +1,268 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/loopfilter.h" + +static NOINLINE void +loop_filter(pixel *dst, int E, int I, int H, + const ptrdiff_t stridea, const ptrdiff_t strideb, const int wd + HIGHBD_DECL_SUFFIX) +{ + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const int F = 1 << bitdepth_min_8; + E <<= bitdepth_min_8; + I <<= bitdepth_min_8; + H <<= bitdepth_min_8; + + for (int i = 0; i < 4; i++, dst += stridea) { + int p6, p5, p4, p3, p2; + int p1 = dst[strideb * -2], p0 = dst[strideb * -1]; + int q0 = dst[strideb * +0], q1 = dst[strideb * +1]; + int q2, q3, q4, q5, q6; + int fm, flat8out, flat8in; + + fm = abs(p1 - p0) <= I && abs(q1 - q0) <= I && + abs(p0 - q0) * 2 + (abs(p1 - q1) >> 1) <= E; + + if (wd > 4) { + p2 = dst[strideb * -3]; + q2 = dst[strideb * +2]; + + fm &= abs(p2 - p1) <= I && abs(q2 - q1) <= I; + + if (wd > 6) { + p3 = dst[strideb * -4]; + q3 = dst[strideb * +3]; + + fm &= abs(p3 - p2) <= I && abs(q3 - q2) <= I; + } + } + if (!fm) continue; + + if (wd >= 16) { + p6 = dst[strideb * -7]; + p5 = dst[strideb * -6]; + p4 = dst[strideb * -5]; + q4 = dst[strideb * +4]; + q5 = dst[strideb * +5]; + q6 = dst[strideb * +6]; + + flat8out = abs(p6 - p0) <= F && abs(p5 - p0) <= F && + abs(p4 - p0) <= F && abs(q4 - q0) <= F && + abs(q5 - q0) <= F && abs(q6 - q0) <= F; + } + + if (wd >= 6) + flat8in = abs(p2 - p0) <= F && abs(p1 - p0) <= F && + abs(q1 - q0) <= F && abs(q2 - q0) <= F; + + if (wd >= 8) + flat8in &= abs(p3 - p0) <= F && abs(q3 - q0) <= F; + + if (wd >= 16 && (flat8out & flat8in)) { + dst[strideb * -6] = (p6 + p6 + p6 + p6 + p6 + p6 * 2 + p5 * 2 + + p4 * 2 + p3 + p2 + p1 + p0 + q0 + 8) >> 4; + dst[strideb * -5] = (p6 + p6 + p6 + p6 + p6 + p5 * 2 + p4 * 2 + + p3 * 2 + p2 + p1 + p0 + q0 + q1 + 8) >> 4; + dst[strideb * -4] = (p6 + p6 + p6 + p6 + p5 + p4 * 2 + p3 * 2 + + p2 * 2 + p1 + p0 + q0 + q1 + q2 + 8) >> 4; + dst[strideb * -3] = (p6 + p6 + p6 + p5 + p4 + p3 * 2 + p2 * 2 + + p1 * 2 + p0 + q0 + q1 + q2 + q3 + 8) >> 4; + dst[strideb * -2] = (p6 + p6 + p5 + p4 + p3 + p2 * 2 + p1 * 2 + + p0 * 2 + q0 + q1 + q2 + q3 + q4 + 8) >> 4; + dst[strideb * -1] = (p6 + p5 + p4 + p3 + p2 + p1 * 2 + p0 * 2 + + q0 * 2 + q1 + q2 + q3 + q4 + q5 + 8) >> 4; + dst[strideb * +0] = (p5 + p4 + p3 + p2 + p1 + p0 * 2 + q0 * 2 + + q1 * 2 + q2 + q3 + q4 + q5 + q6 + 8) >> 4; + dst[strideb * +1] = (p4 + p3 + p2 + p1 + p0 + q0 * 2 + q1 * 2 + + q2 * 2 + q3 + q4 + q5 + q6 + q6 + 8) >> 4; + dst[strideb * +2] = (p3 + p2 + p1 + p0 + q0 + q1 * 2 + q2 * 2 + + q3 * 2 + q4 + q5 + q6 + q6 + q6 + 8) >> 4; + dst[strideb * +3] = (p2 + p1 + p0 + q0 + q1 + q2 * 2 + q3 * 2 + + q4 * 2 + q5 + q6 + q6 + q6 + q6 + 8) >> 4; + dst[strideb * +4] = (p1 + p0 + q0 + q1 + q2 + q3 * 2 + q4 * 2 + + q5 * 2 + q6 + q6 + q6 + q6 + q6 + 8) >> 4; + dst[strideb * +5] = (p0 + q0 + q1 + q2 + q3 + q4 * 2 + q5 * 2 + + q6 * 2 + q6 + q6 + q6 + q6 + q6 + 8) >> 4; + } else if (wd >= 8 && flat8in) { + dst[strideb * -3] = (p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0 + 4) >> 3; + dst[strideb * -2] = (p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1 + 4) >> 3; + dst[strideb * -1] = (p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + 4) >> 3; + dst[strideb * +0] = (p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + 4) >> 3; + dst[strideb * +1] = (p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3 + 4) >> 3; + dst[strideb * +2] = (p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3 + 4) >> 3; + } else if (wd == 6 && flat8in) { + dst[strideb * -2] = (p2 + 2 * p2 + 2 * p1 + 2 * p0 + q0 + 4) >> 3; + dst[strideb * -1] = (p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3; + dst[strideb * +0] = (p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3; + dst[strideb * +1] = (p0 + 2 * q0 + 2 * q1 + 2 * q2 + q2 + 4) >> 3; + } else { + const int hev = abs(p1 - p0) > H || abs(q1 - q0) > H; + +#define iclip_diff(v) iclip(v, -128 * (1 << bitdepth_min_8), \ + 128 * (1 << bitdepth_min_8) - 1) + + if (hev) { + int f = iclip_diff(p1 - q1), f1, f2; + f = iclip_diff(3 * (q0 - p0) + f); + + f1 = imin(f + 4, (128 << bitdepth_min_8) - 1) >> 3; + f2 = imin(f + 3, (128 << bitdepth_min_8) - 1) >> 3; + + dst[strideb * -1] = iclip_pixel(p0 + f2); + dst[strideb * +0] = iclip_pixel(q0 - f1); + } else { + int f = iclip_diff(3 * (q0 - p0)), f1, f2; + + f1 = imin(f + 4, (128 << bitdepth_min_8) - 1) >> 3; + f2 = imin(f + 3, (128 << bitdepth_min_8) - 1) >> 3; + + dst[strideb * -1] = iclip_pixel(p0 + f2); + dst[strideb * +0] = iclip_pixel(q0 - f1); + + f = (f1 + 1) >> 1; + dst[strideb * -2] = iclip_pixel(p1 + f); + dst[strideb * +1] = iclip_pixel(q1 - f); + } +#undef iclip_diff + } + } +} + +static void loop_filter_h_sb128y_c(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int h + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1] | vmask[2]; + for (unsigned y = 1; vm & ~(y - 1); + y <<= 1, dst += 4 * PXSTRIDE(stride), l += b4_stride) + { + if (vm & y) { + const int L = l[0][0] ? l[0][0] : l[-1][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = (vmask[2] & y) ? 2 : !!(vmask[1] & y); + loop_filter(dst, E, I, H, PXSTRIDE(stride), 1, 4 << idx + HIGHBD_TAIL_SUFFIX); + } + } +} + +static void loop_filter_v_sb128y_c(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int w + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1] | vmask[2]; + for (unsigned x = 1; vm & ~(x - 1); x <<= 1, dst += 4, l++) { + if (vm & x) { + const int L = l[0][0] ? l[0][0] : l[-b4_stride][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = (vmask[2] & x) ? 2 : !!(vmask[1] & x); + loop_filter(dst, E, I, H, 1, PXSTRIDE(stride), 4 << idx + HIGHBD_TAIL_SUFFIX); + } + } +} + +static void loop_filter_h_sb128uv_c(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int h + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1]; + for (unsigned y = 1; vm & ~(y - 1); + y <<= 1, dst += 4 * PXSTRIDE(stride), l += b4_stride) + { + if (vm & y) { + const int L = l[0][0] ? l[0][0] : l[-1][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = !!(vmask[1] & y); + loop_filter(dst, E, I, H, PXSTRIDE(stride), 1, 4 + 2 * idx + HIGHBD_TAIL_SUFFIX); + } + } +} + +static void loop_filter_v_sb128uv_c(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int w + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1]; + for (unsigned x = 1; vm & ~(x - 1); x <<= 1, dst += 4, l++) { + if (vm & x) { + const int L = l[0][0] ? l[0][0] : l[-b4_stride][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = !!(vmask[1] & x); + loop_filter(dst, E, I, H, 1, PXSTRIDE(stride), 4 + 2 * idx + HIGHBD_TAIL_SUFFIX); + } + } +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/loopfilter.h" +#elif ARCH_X86 +#include "src/x86/loopfilter.h" +#endif +#endif + +COLD void bitfn(dav1d_loop_filter_dsp_init)(Dav1dLoopFilterDSPContext *const c) { + c->loop_filter_sb[0][0] = loop_filter_h_sb128y_c; + c->loop_filter_sb[0][1] = loop_filter_v_sb128y_c; + c->loop_filter_sb[1][0] = loop_filter_h_sb128uv_c; + c->loop_filter_sb[1][1] = loop_filter_v_sb128uv_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + loop_filter_dsp_init_arm(c); +#elif ARCH_X86 + loop_filter_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/looprestoration.h b/third_party/dav1d/src/looprestoration.h new file mode 100644 index 0000000000..f55dd31947 --- /dev/null +++ b/third_party/dav1d/src/looprestoration.h @@ -0,0 +1,79 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LOOPRESTORATION_H +#define DAV1D_SRC_LOOPRESTORATION_H + +#include +#include + +#include "common/bitdepth.h" + +enum LrEdgeFlags { + LR_HAVE_LEFT = 1 << 0, + LR_HAVE_RIGHT = 1 << 1, + LR_HAVE_TOP = 1 << 2, + LR_HAVE_BOTTOM = 1 << 3, +}; + +#ifdef BITDEPTH +typedef const pixel (*const_left_pixel_row)[4]; +#else +typedef const void *const_left_pixel_row; +#endif + +typedef union LooprestorationParams { + ALIGN(int16_t filter[2][8], 16); + struct { + uint32_t s0, s1; + int16_t w0, w1; + } sgr; +} LooprestorationParams; + +// Although the spec applies restoration filters over 4x4 blocks, +// they can be applied to a bigger surface. +// * w is constrained by the restoration unit size (w <= 256) +// * h is constrained by the stripe height (h <= 64) +// The filter functions are allowed to do aligned writes past the right +// edge of the buffer, aligned up to the minimum loop restoration unit size +// (which is 32 pixels for subsampled chroma and 64 pixels for luma). +#define decl_lr_filter_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const_left_pixel_row left, \ + const pixel *lpf, int w, int h, \ + const LooprestorationParams *params, \ + enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +typedef decl_lr_filter_fn(*looprestorationfilter_fn); + +typedef struct Dav1dLoopRestorationDSPContext { + looprestorationfilter_fn wiener[2]; /* 7-tap, 5-tap */ + looprestorationfilter_fn sgr[3]; /* 5x5, 3x3, mix */ +} Dav1dLoopRestorationDSPContext; + +bitfn_decls(void dav1d_loop_restoration_dsp_init, Dav1dLoopRestorationDSPContext *c, int bpc); + +#endif /* DAV1D_SRC_LOOPRESTORATION_H */ diff --git a/third_party/dav1d/src/looprestoration_tmpl.c b/third_party/dav1d/src/looprestoration_tmpl.c new file mode 100644 index 0000000000..d4d7867dba --- /dev/null +++ b/third_party/dav1d/src/looprestoration_tmpl.c @@ -0,0 +1,554 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/looprestoration.h" +#include "src/tables.h" + +// 256 * 1.5 + 3 + 3 = 390 +#define REST_UNIT_STRIDE (390) + +// TODO Reuse p when no padding is needed (add and remove lpf pixels in p) +// TODO Chroma only requires 2 rows of padding. +static NOINLINE void +padding(pixel *dst, const pixel *p, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, int unit_w, + const int stripe_h, const enum LrEdgeFlags edges) +{ + const int have_left = !!(edges & LR_HAVE_LEFT); + const int have_right = !!(edges & LR_HAVE_RIGHT); + + // Copy more pixels if we don't have to pad them + unit_w += 3 * have_left + 3 * have_right; + pixel *dst_l = dst + 3 * !have_left; + p -= 3 * have_left; + lpf -= 3 * have_left; + + if (edges & LR_HAVE_TOP) { + // Copy previous loop filtered rows + const pixel *const above_1 = lpf; + const pixel *const above_2 = above_1 + PXSTRIDE(stride); + pixel_copy(dst_l, above_1, unit_w); + pixel_copy(dst_l + REST_UNIT_STRIDE, above_1, unit_w); + pixel_copy(dst_l + 2 * REST_UNIT_STRIDE, above_2, unit_w); + } else { + // Pad with first row + pixel_copy(dst_l, p, unit_w); + pixel_copy(dst_l + REST_UNIT_STRIDE, p, unit_w); + pixel_copy(dst_l + 2 * REST_UNIT_STRIDE, p, unit_w); + if (have_left) { + pixel_copy(dst_l, &left[0][1], 3); + pixel_copy(dst_l + REST_UNIT_STRIDE, &left[0][1], 3); + pixel_copy(dst_l + 2 * REST_UNIT_STRIDE, &left[0][1], 3); + } + } + + pixel *dst_tl = dst_l + 3 * REST_UNIT_STRIDE; + if (edges & LR_HAVE_BOTTOM) { + // Copy next loop filtered rows + const pixel *const below_1 = lpf + 6 * PXSTRIDE(stride); + const pixel *const below_2 = below_1 + PXSTRIDE(stride); + pixel_copy(dst_tl + stripe_h * REST_UNIT_STRIDE, below_1, unit_w); + pixel_copy(dst_tl + (stripe_h + 1) * REST_UNIT_STRIDE, below_2, unit_w); + pixel_copy(dst_tl + (stripe_h + 2) * REST_UNIT_STRIDE, below_2, unit_w); + } else { + // Pad with last row + const pixel *const src = p + (stripe_h - 1) * PXSTRIDE(stride); + pixel_copy(dst_tl + stripe_h * REST_UNIT_STRIDE, src, unit_w); + pixel_copy(dst_tl + (stripe_h + 1) * REST_UNIT_STRIDE, src, unit_w); + pixel_copy(dst_tl + (stripe_h + 2) * REST_UNIT_STRIDE, src, unit_w); + if (have_left) { + pixel_copy(dst_tl + stripe_h * REST_UNIT_STRIDE, &left[stripe_h - 1][1], 3); + pixel_copy(dst_tl + (stripe_h + 1) * REST_UNIT_STRIDE, &left[stripe_h - 1][1], 3); + pixel_copy(dst_tl + (stripe_h + 2) * REST_UNIT_STRIDE, &left[stripe_h - 1][1], 3); + } + } + + // Inner UNIT_WxSTRIPE_H + for (int j = 0; j < stripe_h; j++) { + pixel_copy(dst_tl + 3 * have_left, p + 3 * have_left, unit_w - 3 * have_left); + dst_tl += REST_UNIT_STRIDE; + p += PXSTRIDE(stride); + } + + if (!have_right) { + pixel *pad = dst_l + unit_w; + pixel *row_last = &dst_l[unit_w - 1]; + // Pad 3x(STRIPE_H+6) with last column + for (int j = 0; j < stripe_h + 6; j++) { + pixel_set(pad, *row_last, 3); + pad += REST_UNIT_STRIDE; + row_last += REST_UNIT_STRIDE; + } + } + + if (!have_left) { + // Pad 3x(STRIPE_H+6) with first column + for (int j = 0; j < stripe_h + 6; j++) { + pixel_set(dst, *dst_l, 3); + dst += REST_UNIT_STRIDE; + dst_l += REST_UNIT_STRIDE; + } + } else { + dst += 3 * REST_UNIT_STRIDE; + for (int j = 0; j < stripe_h; j++) { + pixel_copy(dst, &left[j][1], 3); + dst += REST_UNIT_STRIDE; + } + } +} + +// FIXME Could split into luma and chroma specific functions, +// (since first and last tops are always 0 for chroma) +// FIXME Could implement a version that requires less temporary memory +// (should be possible to implement with only 6 rows of temp storage) +static void wiener_c(pixel *p, const ptrdiff_t stride, + const pixel (*const left)[4], + const pixel *lpf, const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + // Wiener filtering is applied to a maximum stripe height of 64 + 3 pixels + // of padding above and below + pixel tmp[70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE]; + pixel *tmp_ptr = tmp; + + padding(tmp, p, stride, left, lpf, w, h, edges); + + // Values stored between horizontal and vertical filtering don't + // fit in a uint8_t. + uint16_t hor[70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE]; + uint16_t *hor_ptr = hor; + + const int16_t (*const filter)[8] = params->filter; + const int bitdepth = bitdepth_from_max(bitdepth_max); + const int round_bits_h = 3 + (bitdepth == 12) * 2; + const int rounding_off_h = 1 << (round_bits_h - 1); + const int clip_limit = 1 << (bitdepth + 1 + 7 - round_bits_h); + for (int j = 0; j < h + 6; j++) { + for (int i = 0; i < w; i++) { + int sum = (1 << (bitdepth + 6)); +#if BITDEPTH == 8 + sum += tmp_ptr[i + 3] * 128; +#endif + + for (int k = 0; k < 7; k++) { + sum += tmp_ptr[i + k] * filter[0][k]; + } + + hor_ptr[i] = + iclip((sum + rounding_off_h) >> round_bits_h, 0, clip_limit - 1); + } + tmp_ptr += REST_UNIT_STRIDE; + hor_ptr += REST_UNIT_STRIDE; + } + + const int round_bits_v = 11 - (bitdepth == 12) * 2; + const int rounding_off_v = 1 << (round_bits_v - 1); + const int round_offset = 1 << (bitdepth + (round_bits_v - 1)); + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + int sum = -round_offset; + + for (int k = 0; k < 7; k++) { + sum += hor[(j + k) * REST_UNIT_STRIDE + i] * filter[1][k]; + } + + p[j * PXSTRIDE(stride) + i] = + iclip_pixel((sum + rounding_off_v) >> round_bits_v); + } + } +} + +// Sum over a 3x3 area +// The dst and src pointers are positioned 3 pixels above and 3 pixels to the +// left of the top left corner. However, the self guided filter only needs 1 +// pixel above and one pixel to the left. As for the pixels below and to the +// right they must be computed in the sums, but don't need to be stored. +// +// Example for a 4x4 block: +// x x x x x x x x x x +// x c c c c c c c c x +// x i s s s s s s i x +// x i s s s s s s i x +// x i s s s s s s i x +// x i s s s s s s i x +// x i s s s s s s i x +// x i s s s s s s i x +// x c c c c c c c c x +// x x x x x x x x x x +// +// s: Pixel summed and stored +// i: Pixel summed and stored (between loops) +// c: Pixel summed not stored +// x: Pixel not summed not stored +static void boxsum3(int32_t *sumsq, coef *sum, const pixel *src, + const int w, const int h) +{ + // We skip the first row, as it is never used + src += REST_UNIT_STRIDE; + + // We skip the first and last columns, as they are never used + for (int x = 1; x < w - 1; x++) { + coef *sum_v = sum + x; + int32_t *sumsq_v = sumsq + x; + const pixel *s = src + x; + int a = s[0], a2 = a * a; + int b = s[REST_UNIT_STRIDE], b2 = b * b; + + // We skip the first 2 rows, as they are skipped in the next loop and + // we don't need the last 2 row as it is skipped in the next loop + for (int y = 2; y < h - 2; y++) { + s += REST_UNIT_STRIDE; + const int c = s[REST_UNIT_STRIDE]; + const int c2 = c * c; + sum_v += REST_UNIT_STRIDE; + sumsq_v += REST_UNIT_STRIDE; + *sum_v = a + b + c; + *sumsq_v = a2 + b2 + c2; + a = b; + a2 = b2; + b = c; + b2 = c2; + } + } + + // We skip the first row as it is never read + sum += REST_UNIT_STRIDE; + sumsq += REST_UNIT_STRIDE; + // We skip the last 2 rows as it is never read + for (int y = 2; y < h - 2; y++) { + int a = sum[1], a2 = sumsq[1]; + int b = sum[2], b2 = sumsq[2]; + + // We don't store the first column as it is never read and + // we don't store the last 2 columns as they are never read + for (int x = 2; x < w - 2; x++) { + const int c = sum[x + 1], c2 = sumsq[x + 1]; + sum[x] = a + b + c; + sumsq[x] = a2 + b2 + c2; + a = b; + a2 = b2; + b = c; + b2 = c2; + } + sum += REST_UNIT_STRIDE; + sumsq += REST_UNIT_STRIDE; + } +} + +// Sum over a 5x5 area +// The dst and src pointers are positioned 3 pixels above and 3 pixels to the +// left of the top left corner. However, the self guided filter only needs 1 +// pixel above and one pixel to the left. As for the pixels below and to the +// right they must be computed in the sums, but don't need to be stored. +// +// Example for a 4x4 block: +// c c c c c c c c c c +// c c c c c c c c c c +// i i s s s s s s i i +// i i s s s s s s i i +// i i s s s s s s i i +// i i s s s s s s i i +// i i s s s s s s i i +// i i s s s s s s i i +// c c c c c c c c c c +// c c c c c c c c c c +// +// s: Pixel summed and stored +// i: Pixel summed and stored (between loops) +// c: Pixel summed not stored +// x: Pixel not summed not stored +static void boxsum5(int32_t *sumsq, coef *sum, const pixel *const src, + const int w, const int h) +{ + for (int x = 0; x < w; x++) { + coef *sum_v = sum + x; + int32_t *sumsq_v = sumsq + x; + const pixel *s = src + 3 * REST_UNIT_STRIDE + x; + int a = s[-3 * REST_UNIT_STRIDE], a2 = a * a; + int b = s[-2 * REST_UNIT_STRIDE], b2 = b * b; + int c = s[-1 * REST_UNIT_STRIDE], c2 = c * c; + int d = s[0], d2 = d * d; + + // We skip the first 2 rows, as they are skipped in the next loop and + // we don't need the last 2 row as it is skipped in the next loop + for (int y = 2; y < h - 2; y++) { + s += REST_UNIT_STRIDE; + const int e = *s, e2 = e * e; + sum_v += REST_UNIT_STRIDE; + sumsq_v += REST_UNIT_STRIDE; + *sum_v = a + b + c + d + e; + *sumsq_v = a2 + b2 + c2 + d2 + e2; + a = b; + b = c; + c = d; + d = e; + a2 = b2; + b2 = c2; + c2 = d2; + d2 = e2; + } + } + + // We skip the first row as it is never read + sum += REST_UNIT_STRIDE; + sumsq += REST_UNIT_STRIDE; + for (int y = 2; y < h - 2; y++) { + int a = sum[0], a2 = sumsq[0]; + int b = sum[1], b2 = sumsq[1]; + int c = sum[2], c2 = sumsq[2]; + int d = sum[3], d2 = sumsq[3]; + + for (int x = 2; x < w - 2; x++) { + const int e = sum[x + 2], e2 = sumsq[x + 2]; + sum[x] = a + b + c + d + e; + sumsq[x] = a2 + b2 + c2 + d2 + e2; + a = b; + b = c; + c = d; + d = e; + a2 = b2; + b2 = c2; + c2 = d2; + d2 = e2; + } + sum += REST_UNIT_STRIDE; + sumsq += REST_UNIT_STRIDE; + } +} + +static NOINLINE void +selfguided_filter(coef *dst, const pixel *src, const ptrdiff_t src_stride, + const int w, const int h, const int n, const unsigned s + HIGHBD_DECL_SUFFIX) +{ + const unsigned sgr_one_by_x = n == 25 ? 164 : 455; + + // Selfguided filter is applied to a maximum stripe height of 64 + 3 pixels + // of padding above and below + int32_t sumsq[68 /*(64 + 2 + 2)*/ * REST_UNIT_STRIDE]; + int32_t *A = sumsq + 2 * REST_UNIT_STRIDE + 3; + // By inverting A and B after the boxsums, B can be of size coef instead + // of int32_t + coef sum[68 /*(64 + 2 + 2)*/ * REST_UNIT_STRIDE]; + coef *B = sum + 2 * REST_UNIT_STRIDE + 3; + + const int step = (n == 25) + 1; + if (n == 25) + boxsum5(sumsq, sum, src, w + 6, h + 6); + else + boxsum3(sumsq, sum, src, w + 6, h + 6); + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + + int32_t *AA = A - REST_UNIT_STRIDE; + coef *BB = B - REST_UNIT_STRIDE; + for (int j = -1; j < h + 1; j+= step) { + for (int i = -1; i < w + 1; i++) { + const int a = + (AA[i] + ((1 << (2 * bitdepth_min_8)) >> 1)) >> (2 * bitdepth_min_8); + const int b = + (BB[i] + ((1 << bitdepth_min_8) >> 1)) >> bitdepth_min_8; + + const unsigned p = imax(a * n - b * b, 0); + const unsigned z = (p * s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + + // This is where we invert A and B, so that B is of size coef. + AA[i] = (x * BB[i] * sgr_one_by_x + (1 << 11)) >> 12; + BB[i] = x; + } + AA += step * REST_UNIT_STRIDE; + BB += step * REST_UNIT_STRIDE; + } + + src += 3 * REST_UNIT_STRIDE + 3; + if (n == 25) { + int j = 0; +#define SIX_NEIGHBORS(P, i)\ + ((P[i - REST_UNIT_STRIDE] + P[i + REST_UNIT_STRIDE]) * 6 + \ + (P[i - 1 - REST_UNIT_STRIDE] + P[i - 1 + REST_UNIT_STRIDE] + \ + P[i + 1 - REST_UNIT_STRIDE] + P[i + 1 + REST_UNIT_STRIDE]) * 5) + for (; j < h - 1; j+=2) { + for (int i = 0; i < w; i++) { + const int a = SIX_NEIGHBORS(B, i); + const int b = SIX_NEIGHBORS(A, i); + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; + } + dst += 384 /* Maximum restoration width is 384 (256 * 1.5) */; + src += REST_UNIT_STRIDE; + B += REST_UNIT_STRIDE; + A += REST_UNIT_STRIDE; + for (int i = 0; i < w; i++) { + const int a = B[i] * 6 + (B[i - 1] + B[i + 1]) * 5; + const int b = A[i] * 6 + (A[i - 1] + A[i + 1]) * 5; + dst[i] = (b - a * src[i] + (1 << 7)) >> 8; + } + dst += 384 /* Maximum restoration width is 384 (256 * 1.5) */; + src += REST_UNIT_STRIDE; + B += REST_UNIT_STRIDE; + A += REST_UNIT_STRIDE; + } + if (j + 1 == h) { // Last row, when number of rows is odd + for (int i = 0; i < w; i++) { + const int a = SIX_NEIGHBORS(B, i); + const int b = SIX_NEIGHBORS(A, i); + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; + } + } +#undef SIX_NEIGHBORS + } else { +#define EIGHT_NEIGHBORS(P, i)\ + ((P[i] + P[i - 1] + P[i + 1] + P[i - REST_UNIT_STRIDE] + P[i + REST_UNIT_STRIDE]) * 4 + \ + (P[i - 1 - REST_UNIT_STRIDE] + P[i - 1 + REST_UNIT_STRIDE] + \ + P[i + 1 - REST_UNIT_STRIDE] + P[i + 1 + REST_UNIT_STRIDE]) * 3) + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + const int a = EIGHT_NEIGHBORS(B, i); + const int b = EIGHT_NEIGHBORS(A, i); + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; + } + dst += 384; + src += REST_UNIT_STRIDE; + B += REST_UNIT_STRIDE; + A += REST_UNIT_STRIDE; + } + } +#undef EIGHT_NEIGHBORS +} + +static void sgr_5x5_c(pixel *p, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + // Selfguided filter is applied to a maximum stripe height of 64 + 3 pixels + // of padding above and below + pixel tmp[70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE]; + + // Selfguided filter outputs to a maximum stripe height of 64 and a + // maximum restoration width of 384 (256 * 1.5) + coef dst[64 * 384]; + + padding(tmp, p, stride, left, lpf, w, h, edges); + selfguided_filter(dst, tmp, REST_UNIT_STRIDE, w, h, 25, + params->sgr.s0 HIGHBD_TAIL_SUFFIX); + + const int w0 = params->sgr.w0; + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + const int v = w0 * dst[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); + } + p += PXSTRIDE(stride); + } +} + +static void sgr_3x3_c(pixel *p, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + pixel tmp[70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE]; + coef dst[64 * 384]; + + padding(tmp, p, stride, left, lpf, w, h, edges); + selfguided_filter(dst, tmp, REST_UNIT_STRIDE, w, h, 9, + params->sgr.s1 HIGHBD_TAIL_SUFFIX); + + const int w1 = params->sgr.w1; + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + const int v = w1 * dst[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); + } + p += PXSTRIDE(stride); + } +} + +static void sgr_mix_c(pixel *p, const ptrdiff_t stride, + const pixel (*const left)[4], const pixel *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + pixel tmp[70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE]; + coef dst0[64 * 384]; + coef dst1[64 * 384]; + + padding(tmp, p, stride, left, lpf, w, h, edges); + selfguided_filter(dst0, tmp, REST_UNIT_STRIDE, w, h, 25, + params->sgr.s0 HIGHBD_TAIL_SUFFIX); + selfguided_filter(dst1, tmp, REST_UNIT_STRIDE, w, h, 9, + params->sgr.s1 HIGHBD_TAIL_SUFFIX); + + const int w0 = params->sgr.w0; + const int w1 = params->sgr.w1; + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + const int v = w0 * dst0[j * 384 + i] + w1 * dst1[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); + } + p += PXSTRIDE(stride); + } +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/looprestoration.h" +#elif ARCH_PPC64LE +#include "src/ppc/looprestoration.h" +#elif ARCH_X86 +#include "src/x86/looprestoration.h" +#endif +#endif + +COLD void bitfn(dav1d_loop_restoration_dsp_init)(Dav1dLoopRestorationDSPContext *const c, + const int bpc) +{ + c->wiener[0] = c->wiener[1] = wiener_c; + c->sgr[0] = sgr_5x5_c; + c->sgr[1] = sgr_3x3_c; + c->sgr[2] = sgr_mix_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + loop_restoration_dsp_init_arm(c, bpc); +#elif ARCH_PPC64LE + loop_restoration_dsp_init_ppc(c, bpc); +#elif ARCH_X86 + loop_restoration_dsp_init_x86(c, bpc); +#endif +#endif +} diff --git a/third_party/dav1d/src/lr_apply.h b/third_party/dav1d/src/lr_apply.h new file mode 100644 index 0000000000..2815367534 --- /dev/null +++ b/third_party/dav1d/src/lr_apply.h @@ -0,0 +1,47 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_LR_APPLY_H +#define DAV1D_SRC_LR_APPLY_H + +#include +#include + +#include "common/bitdepth.h" + +#include "src/internal.h" + +enum LrRestorePlanes { + LR_RESTORE_Y = 1 << 0, + LR_RESTORE_U = 1 << 1, + LR_RESTORE_V = 1 << 2, +}; + +void bytefn(dav1d_lr_sbrow)(Dav1dFrameContext *const f, pixel *const dst[3], + int sby); + +#endif /* DAV1D_SRC_LR_APPLY_H */ diff --git a/third_party/dav1d/src/lr_apply_tmpl.c b/third_party/dav1d/src/lr_apply_tmpl.c new file mode 100644 index 0000000000..c517f89820 --- /dev/null +++ b/third_party/dav1d/src/lr_apply_tmpl.c @@ -0,0 +1,201 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/lr_apply.h" + +static void lr_stripe(const Dav1dFrameContext *const f, pixel *p, + const pixel (*left)[4], int x, int y, + const int plane, const int unit_w, const int row_h, + const Av1RestorationUnit *const lr, enum LrEdgeFlags edges) +{ + const Dav1dDSPContext *const dsp = f->dsp; + const int chroma = !!plane; + const int ss_ver = chroma & (f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420); + const ptrdiff_t stride = f->sr_cur.p.stride[chroma]; + const int sby = (y + (y ? 8 << ss_ver : 0)) >> (6 - ss_ver + f->seq_hdr->sb128); + const int have_tt = f->c->n_tc > 1; + const pixel *lpf = f->lf.lr_lpf_line[plane] + + have_tt * (sby * (4 << f->seq_hdr->sb128) - 4) * PXSTRIDE(stride) + x; + + // The first stripe of the frame is shorter by 8 luma pixel rows. + int stripe_h = imin((64 - 8 * !y) >> ss_ver, row_h - y); + + looprestorationfilter_fn lr_fn; + LooprestorationParams params; + if (lr->type == DAV1D_RESTORATION_WIENER) { + int16_t (*const filter)[8] = params.filter; + filter[0][0] = filter[0][6] = lr->filter_h[0]; + filter[0][1] = filter[0][5] = lr->filter_h[1]; + filter[0][2] = filter[0][4] = lr->filter_h[2]; + filter[0][3] = -(filter[0][0] + filter[0][1] + filter[0][2]) * 2; +#if BITDEPTH != 8 + /* For 8-bit SIMD it's beneficial to handle the +128 separately + * in order to avoid overflows. */ + filter[0][3] += 128; +#endif + + filter[1][0] = filter[1][6] = lr->filter_v[0]; + filter[1][1] = filter[1][5] = lr->filter_v[1]; + filter[1][2] = filter[1][4] = lr->filter_v[2]; + filter[1][3] = 128 - (filter[1][0] + filter[1][1] + filter[1][2]) * 2; + + lr_fn = dsp->lr.wiener[!(filter[0][0] | filter[1][0])]; + } else { + assert(lr->type == DAV1D_RESTORATION_SGRPROJ); + const uint16_t *const sgr_params = dav1d_sgr_params[lr->sgr_idx]; + params.sgr.s0 = sgr_params[0]; + params.sgr.s1 = sgr_params[1]; + params.sgr.w0 = lr->sgr_weights[0]; + params.sgr.w1 = 128 - (lr->sgr_weights[0] + lr->sgr_weights[1]); + + lr_fn = dsp->lr.sgr[!!sgr_params[0] + !!sgr_params[1] * 2 - 1]; + } + + while (y + stripe_h <= row_h) { + // Change the HAVE_BOTTOM bit in edges to (sby + 1 != f->sbh || y + stripe_h != row_h) + edges ^= (-(sby + 1 != f->sbh || y + stripe_h != row_h) ^ edges) & LR_HAVE_BOTTOM; + lr_fn(p, stride, left, lpf, unit_w, stripe_h, ¶ms, edges HIGHBD_CALL_SUFFIX); + + left += stripe_h; + y += stripe_h; + p += stripe_h * PXSTRIDE(stride); + edges |= LR_HAVE_TOP; + stripe_h = imin(64 >> ss_ver, row_h - y); + if (stripe_h == 0) break; + lpf += 4 * PXSTRIDE(stride); + } +} + +static void backup4xU(pixel (*dst)[4], const pixel *src, const ptrdiff_t src_stride, + int u) +{ + for (; u > 0; u--, dst++, src += PXSTRIDE(src_stride)) + pixel_copy(dst, src, 4); +} + +static void lr_sbrow(const Dav1dFrameContext *const f, pixel *p, const int y, + const int w, const int h, const int row_h, const int plane) +{ + const int chroma = !!plane; + const int ss_ver = chroma & (f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420); + const int ss_hor = chroma & (f->sr_cur.p.p.layout != DAV1D_PIXEL_LAYOUT_I444); + const ptrdiff_t p_stride = f->sr_cur.p.stride[chroma]; + + const int unit_size_log2 = f->frame_hdr->restoration.unit_size[!!plane]; + const int unit_size = 1 << unit_size_log2; + const int half_unit_size = unit_size >> 1; + const int max_unit_size = unit_size + half_unit_size; + + // Y coordinate of the sbrow (y is 8 luma pixel rows above row_y) + const int row_y = y + ((8 >> ss_ver) * !!y); + + // FIXME This is an ugly hack to lookup the proper AV1Filter unit for + // chroma planes. Question: For Multithreaded decoding, is it better + // to store the chroma LR information with collocated Luma information? + // In other words. For a chroma restoration unit locate at 128,128 and + // with a 4:2:0 chroma subsampling, do we store the filter information at + // the AV1Filter unit located at (128,128) or (256,256) + // TODO Support chroma subsampling. + const int shift_hor = 7 - ss_hor; + + /* maximum sbrow height is 128 + 8 rows offset */ + ALIGN_STK_16(pixel, pre_lr_border, 2, [128 + 8][4]); + const Av1RestorationUnit *lr[2]; + + enum LrEdgeFlags edges = (y > 0 ? LR_HAVE_TOP : 0) | LR_HAVE_RIGHT; + + int aligned_unit_pos = row_y & ~(unit_size - 1); + if (aligned_unit_pos && aligned_unit_pos + half_unit_size > h) + aligned_unit_pos -= unit_size; + aligned_unit_pos <<= ss_ver; + const int sb_idx = (aligned_unit_pos >> 7) * f->sr_sb128w; + const int unit_idx = ((aligned_unit_pos >> 6) & 1) << 1; + lr[0] = &f->lf.lr_mask[sb_idx].lr[plane][unit_idx]; + int restore = lr[0]->type != DAV1D_RESTORATION_NONE; + int x = 0, bit = 0; + for (; x + max_unit_size <= w; p += unit_size, edges |= LR_HAVE_LEFT, bit ^= 1) { + const int next_x = x + unit_size; + const int next_u_idx = unit_idx + ((next_x >> (shift_hor - 1)) & 1); + lr[!bit] = + &f->lf.lr_mask[sb_idx + (next_x >> shift_hor)].lr[plane][next_u_idx]; + const int restore_next = lr[!bit]->type != DAV1D_RESTORATION_NONE; + if (restore_next) + backup4xU(pre_lr_border[bit], p + unit_size - 4, p_stride, row_h - y); + if (restore) + lr_stripe(f, p, pre_lr_border[!bit], x, y, plane, unit_size, row_h, + lr[bit], edges); + x = next_x; + restore = restore_next; + } + if (restore) { + edges &= ~LR_HAVE_RIGHT; + const int unit_w = w - x; + lr_stripe(f, p, pre_lr_border[!bit], x, y, plane, unit_w, row_h, lr[bit], edges); + } +} + +void bytefn(dav1d_lr_sbrow)(Dav1dFrameContext *const f, pixel *const dst[3], + const int sby) +{ + const int offset_y = 8 * !!sby; + const ptrdiff_t *const dst_stride = f->sr_cur.p.stride; + const int restore_planes = f->lf.restore_planes; + const int not_last = sby + 1 < f->sbh; + + if (restore_planes & LR_RESTORE_Y) { + const int h = f->sr_cur.p.p.h; + const int w = f->sr_cur.p.p.w; + const int next_row_y = (sby + 1) << (6 + f->seq_hdr->sb128); + const int row_h = imin(next_row_y - 8 * not_last, h); + const int y_stripe = (sby << (6 + f->seq_hdr->sb128)) - offset_y; + lr_sbrow(f, dst[0] - offset_y * PXSTRIDE(dst_stride[0]), y_stripe, w, + h, row_h, 0); + } + if (restore_planes & (LR_RESTORE_U | LR_RESTORE_V)) { + const int ss_ver = f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->sr_cur.p.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h = (f->sr_cur.p.p.h + ss_ver) >> ss_ver; + const int w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; + const int next_row_y = (sby + 1) << ((6 - ss_ver) + f->seq_hdr->sb128); + const int row_h = imin(next_row_y - (8 >> ss_ver) * not_last, h); + const int offset_uv = offset_y >> ss_ver; + const int y_stripe = (sby << ((6 - ss_ver) + f->seq_hdr->sb128)) - offset_uv; + if (restore_planes & LR_RESTORE_U) + lr_sbrow(f, dst[1] - offset_uv * PXSTRIDE(dst_stride[1]), y_stripe, + w, h, row_h, 1); + + if (restore_planes & LR_RESTORE_V) + lr_sbrow(f, dst[2] - offset_uv * PXSTRIDE(dst_stride[1]), y_stripe, + w, h, row_h, 2); + } +} diff --git a/third_party/dav1d/src/mc.h b/third_party/dav1d/src/mc.h new file mode 100644 index 0000000000..59ba2d9a5a --- /dev/null +++ b/third_party/dav1d/src/mc.h @@ -0,0 +1,136 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_MC_H +#define DAV1D_SRC_MC_H + +#include +#include + +#include "common/bitdepth.h" + +#include "src/levels.h" + +#define decl_mc_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const pixel *src, ptrdiff_t src_stride, \ + int w, int h, int mx, int my HIGHBD_DECL_SUFFIX) +typedef decl_mc_fn(*mc_fn); + +#define decl_mc_scaled_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const pixel *src, ptrdiff_t src_stride, \ + int w, int h, int mx, int my, int dx, int dy HIGHBD_DECL_SUFFIX) +typedef decl_mc_scaled_fn(*mc_scaled_fn); + +#define decl_warp8x8_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const pixel *src, ptrdiff_t src_stride, \ + const int16_t *abcd, int mx, int my HIGHBD_DECL_SUFFIX) +typedef decl_warp8x8_fn(*warp8x8_fn); + +#define decl_mct_fn(name) \ +void (name)(int16_t *tmp, const pixel *src, ptrdiff_t src_stride, \ + int w, int h, int mx, int my HIGHBD_DECL_SUFFIX) +typedef decl_mct_fn(*mct_fn); + +#define decl_mct_scaled_fn(name) \ +void (name)(int16_t *tmp, const pixel *src, ptrdiff_t src_stride, \ + int w, int h, int mx, int my, int dx, int dy HIGHBD_DECL_SUFFIX) +typedef decl_mct_scaled_fn(*mct_scaled_fn); + +#define decl_warp8x8t_fn(name) \ +void (name)(int16_t *tmp, const ptrdiff_t tmp_stride, \ + const pixel *src, ptrdiff_t src_stride, \ + const int16_t *abcd, int mx, int my HIGHBD_DECL_SUFFIX) +typedef decl_warp8x8t_fn(*warp8x8t_fn); + +#define decl_avg_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const int16_t *tmp1, const int16_t *tmp2, int w, int h \ + HIGHBD_DECL_SUFFIX) +typedef decl_avg_fn(*avg_fn); + +#define decl_w_avg_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const int16_t *tmp1, const int16_t *tmp2, int w, int h, int weight \ + HIGHBD_DECL_SUFFIX) +typedef decl_w_avg_fn(*w_avg_fn); + +#define decl_mask_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const int16_t *tmp1, const int16_t *tmp2, int w, int h, \ + const uint8_t *mask HIGHBD_DECL_SUFFIX) +typedef decl_mask_fn(*mask_fn); + +#define decl_w_mask_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const int16_t *tmp1, const int16_t *tmp2, int w, int h, \ + uint8_t *mask, int sign HIGHBD_DECL_SUFFIX) +typedef decl_w_mask_fn(*w_mask_fn); + +#define decl_blend_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, \ + int w, int h, const uint8_t *mask) +typedef decl_blend_fn(*blend_fn); + +#define decl_blend_dir_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, const pixel *tmp, int w, int h) +typedef decl_blend_dir_fn(*blend_dir_fn); + +#define decl_emu_edge_fn(name) \ +void (name)(intptr_t bw, intptr_t bh, intptr_t iw, intptr_t ih, intptr_t x, intptr_t y, \ + pixel *dst, ptrdiff_t dst_stride, const pixel *src, ptrdiff_t src_stride) +typedef decl_emu_edge_fn(*emu_edge_fn); + +#define decl_resize_fn(name) \ +void (name)(pixel *dst, ptrdiff_t dst_stride, \ + const pixel *src, ptrdiff_t src_stride, \ + int dst_w, int h, int src_w, int dx, int mx HIGHBD_DECL_SUFFIX) +typedef decl_resize_fn(*resize_fn); + +typedef struct Dav1dMCDSPContext { + mc_fn mc[N_2D_FILTERS]; + mc_scaled_fn mc_scaled[N_2D_FILTERS]; + mct_fn mct[N_2D_FILTERS]; + mct_scaled_fn mct_scaled[N_2D_FILTERS]; + avg_fn avg; + w_avg_fn w_avg; + mask_fn mask; + w_mask_fn w_mask[3 /* 444, 422, 420 */]; + blend_fn blend; + blend_dir_fn blend_v; + blend_dir_fn blend_h; + warp8x8_fn warp8x8; + warp8x8t_fn warp8x8t; + emu_edge_fn emu_edge; + resize_fn resize; +} Dav1dMCDSPContext; + +bitfn_decls(void dav1d_mc_dsp_init, Dav1dMCDSPContext *c); + +#endif /* DAV1D_SRC_MC_H */ diff --git a/third_party/dav1d/src/mc_tmpl.c b/third_party/dav1d/src/mc_tmpl.c new file mode 100644 index 0000000000..20226d8a39 --- /dev/null +++ b/third_party/dav1d/src/mc_tmpl.c @@ -0,0 +1,953 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/mc.h" +#include "src/tables.h" + +#if BITDEPTH == 8 +#define get_intermediate_bits(bitdepth_max) 4 +// Output in interval [-5132, 9212], fits in int16_t as is +#define PREP_BIAS 0 +#else +// 4 for 10 bits/component, 2 for 12 bits/component +#define get_intermediate_bits(bitdepth_max) (14 - bitdepth_from_max(bitdepth_max)) +// Output in interval [-20588, 36956] (10-bit), [-20602, 36983] (12-bit) +// Subtract a bias to ensure the output fits in int16_t +#define PREP_BIAS 8192 +#endif + +static NOINLINE void +put_c(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, const int w, int h) +{ + do { + pixel_copy(dst, src, w); + + dst += dst_stride; + src += src_stride; + } while (--h); +} + +static NOINLINE void +prep_c(int16_t *tmp, const pixel *src, const ptrdiff_t src_stride, + const int w, int h HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + do { + for (int x = 0; x < w; x++) + tmp[x] = (src[x] << intermediate_bits) - PREP_BIAS; + + tmp += w; + src += src_stride; + } while (--h); +} + +#define FILTER_8TAP(src, x, F, stride) \ + (F[0] * src[x + -3 * stride] + \ + F[1] * src[x + -2 * stride] + \ + F[2] * src[x + -1 * stride] + \ + F[3] * src[x + +0 * stride] + \ + F[4] * src[x + +1 * stride] + \ + F[5] * src[x + +2 * stride] + \ + F[6] * src[x + +3 * stride] + \ + F[7] * src[x + +4 * stride]) + +#define DAV1D_FILTER_8TAP_RND(src, x, F, stride, sh) \ + ((FILTER_8TAP(src, x, F, stride) + ((1 << (sh)) >> 1)) >> (sh)) + +#define DAV1D_FILTER_8TAP_RND2(src, x, F, stride, rnd, sh) \ + ((FILTER_8TAP(src, x, F, stride) + (rnd)) >> (sh)) + +#define DAV1D_FILTER_8TAP_CLIP(src, x, F, stride, sh) \ + iclip_pixel(DAV1D_FILTER_8TAP_RND(src, x, F, stride, sh)) + +#define DAV1D_FILTER_8TAP_CLIP2(src, x, F, stride, rnd, sh) \ + iclip_pixel(DAV1D_FILTER_8TAP_RND2(src, x, F, stride, rnd, sh)) + +#define GET_H_FILTER(mx) \ + const int8_t *const fh = !(mx) ? NULL : w > 4 ? \ + dav1d_mc_subpel_filters[filter_type & 3][(mx) - 1] : \ + dav1d_mc_subpel_filters[3 + (filter_type & 1)][(mx) - 1] + +#define GET_V_FILTER(my) \ + const int8_t *const fv = !(my) ? NULL : h > 4 ? \ + dav1d_mc_subpel_filters[filter_type >> 2][(my) - 1] : \ + dav1d_mc_subpel_filters[3 + ((filter_type >> 2) & 1)][(my) - 1] + +#define GET_FILTERS() \ + GET_H_FILTER(mx); \ + GET_V_FILTER(my) + +static NOINLINE void +put_8tap_c(pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my, + const int filter_type HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int intermediate_rnd = 32 + ((1 << (6 - intermediate_bits)) >> 1); + + GET_FILTERS(); + dst_stride = PXSTRIDE(dst_stride); + src_stride = PXSTRIDE(src_stride); + + if (fh) { + if (fv) { + int tmp_h = h + 7; + int16_t mid[128 * 135], *mid_ptr = mid; + + src -= src_stride * 3; + do { + for (int x = 0; x < w; x++) + mid_ptr[x] = DAV1D_FILTER_8TAP_RND(src, x, fh, 1, + 6 - intermediate_bits); + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid + 128 * 3; + do { + for (int x = 0; x < w; x++) + dst[x] = DAV1D_FILTER_8TAP_CLIP(mid_ptr, x, fv, 128, + 6 + intermediate_bits); + + mid_ptr += 128; + dst += dst_stride; + } while (--h); + } else { + do { + for (int x = 0; x < w; x++) { + dst[x] = DAV1D_FILTER_8TAP_CLIP2(src, x, fh, 1, + intermediate_rnd, 6); + } + + dst += dst_stride; + src += src_stride; + } while (--h); + } + } else if (fv) { + do { + for (int x = 0; x < w; x++) + dst[x] = DAV1D_FILTER_8TAP_CLIP(src, x, fv, src_stride, 6); + + dst += dst_stride; + src += src_stride; + } while (--h); + } else + put_c(dst, dst_stride, src, src_stride, w, h); +} + +static NOINLINE void +put_8tap_scaled_c(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, int my, + const int dx, const int dy, const int filter_type + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int intermediate_rnd = (1 << intermediate_bits) >> 1; + int tmp_h = (((h - 1) * dy + my) >> 10) + 8; + int16_t mid[128 * (256 + 7)], *mid_ptr = mid; + src_stride = PXSTRIDE(src_stride); + + src -= src_stride * 3; + do { + int x; + int imx = mx, ioff = 0; + + for (x = 0; x < w; x++) { + GET_H_FILTER(imx >> 6); + mid_ptr[x] = fh ? DAV1D_FILTER_8TAP_RND(src, ioff, fh, 1, + 6 - intermediate_bits) : + src[ioff] << intermediate_bits; + imx += dx; + ioff += imx >> 10; + imx &= 0x3ff; + } + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid + 128 * 3; + for (int y = 0; y < h; y++) { + int x; + GET_V_FILTER(my >> 6); + + for (x = 0; x < w; x++) + dst[x] = fv ? DAV1D_FILTER_8TAP_CLIP(mid_ptr, x, fv, 128, + 6 + intermediate_bits) : + iclip_pixel((mid_ptr[x] + intermediate_rnd) >> + intermediate_bits); + + my += dy; + mid_ptr += (my >> 10) * 128; + my &= 0x3ff; + dst += PXSTRIDE(dst_stride); + } +} + +static NOINLINE void +prep_8tap_c(int16_t *tmp, const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my, + const int filter_type HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + GET_FILTERS(); + src_stride = PXSTRIDE(src_stride); + + if (fh) { + if (fv) { + int tmp_h = h + 7; + int16_t mid[128 * 135], *mid_ptr = mid; + + src -= src_stride * 3; + do { + for (int x = 0; x < w; x++) + mid_ptr[x] = DAV1D_FILTER_8TAP_RND(src, x, fh, 1, + 6 - intermediate_bits); + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid + 128 * 3; + do { + for (int x = 0; x < w; x++) { + int t = DAV1D_FILTER_8TAP_RND(mid_ptr, x, fv, 128, 6) - + PREP_BIAS; + assert(t >= INT16_MIN && t <= INT16_MAX); + tmp[x] = t; + } + + mid_ptr += 128; + tmp += w; + } while (--h); + } else { + do { + for (int x = 0; x < w; x++) + tmp[x] = DAV1D_FILTER_8TAP_RND(src, x, fh, 1, + 6 - intermediate_bits) - + PREP_BIAS; + + tmp += w; + src += src_stride; + } while (--h); + } + } else if (fv) { + do { + for (int x = 0; x < w; x++) + tmp[x] = DAV1D_FILTER_8TAP_RND(src, x, fv, src_stride, + 6 - intermediate_bits) - + PREP_BIAS; + + tmp += w; + src += src_stride; + } while (--h); + } else + prep_c(tmp, src, src_stride, w, h HIGHBD_TAIL_SUFFIX); +} + +static NOINLINE void +prep_8tap_scaled_c(int16_t *tmp, const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, int my, + const int dx, const int dy, const int filter_type + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + int tmp_h = (((h - 1) * dy + my) >> 10) + 8; + int16_t mid[128 * (256 + 7)], *mid_ptr = mid; + src_stride = PXSTRIDE(src_stride); + + src -= src_stride * 3; + do { + int x; + int imx = mx, ioff = 0; + + for (x = 0; x < w; x++) { + GET_H_FILTER(imx >> 6); + mid_ptr[x] = fh ? DAV1D_FILTER_8TAP_RND(src, ioff, fh, 1, + 6 - intermediate_bits) : + src[ioff] << intermediate_bits; + imx += dx; + ioff += imx >> 10; + imx &= 0x3ff; + } + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid + 128 * 3; + for (int y = 0; y < h; y++) { + int x; + GET_V_FILTER(my >> 6); + + for (x = 0; x < w; x++) + tmp[x] = (fv ? DAV1D_FILTER_8TAP_RND(mid_ptr, x, fv, 128, 6) + : mid_ptr[x]) - PREP_BIAS; + + my += dy; + mid_ptr += (my >> 10) * 128; + my &= 0x3ff; + tmp += w; + } +} + +#define filter_fns(type, type_h, type_v) \ +static void put_8tap_##type##_c(pixel *const dst, \ + const ptrdiff_t dst_stride, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my \ + HIGHBD_DECL_SUFFIX) \ +{ \ + put_8tap_c(dst, dst_stride, src, src_stride, w, h, mx, my, \ + type_h | (type_v << 2) HIGHBD_TAIL_SUFFIX); \ +} \ +static void put_8tap_##type##_scaled_c(pixel *const dst, \ + const ptrdiff_t dst_stride, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my, \ + const int dx, const int dy \ + HIGHBD_DECL_SUFFIX) \ +{ \ + put_8tap_scaled_c(dst, dst_stride, src, src_stride, w, h, mx, my, dx, dy, \ + type_h | (type_v << 2) HIGHBD_TAIL_SUFFIX); \ +} \ +static void prep_8tap_##type##_c(int16_t *const tmp, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my \ + HIGHBD_DECL_SUFFIX) \ +{ \ + prep_8tap_c(tmp, src, src_stride, w, h, mx, my, \ + type_h | (type_v << 2) HIGHBD_TAIL_SUFFIX); \ +} \ +static void prep_8tap_##type##_scaled_c(int16_t *const tmp, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my, \ + const int dx, const int dy \ + HIGHBD_DECL_SUFFIX) \ +{ \ + prep_8tap_scaled_c(tmp, src, src_stride, w, h, mx, my, dx, dy, \ + type_h | (type_v << 2) HIGHBD_TAIL_SUFFIX); \ +} + +filter_fns(regular, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_REGULAR) +filter_fns(regular_sharp, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SHARP) +filter_fns(regular_smooth, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SMOOTH) +filter_fns(smooth, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SMOOTH) +filter_fns(smooth_regular, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_REGULAR) +filter_fns(smooth_sharp, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SHARP) +filter_fns(sharp, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SHARP) +filter_fns(sharp_regular, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_REGULAR) +filter_fns(sharp_smooth, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SMOOTH) + +#define FILTER_BILIN(src, x, mxy, stride) \ + (16 * src[x] + ((mxy) * (src[x + stride] - src[x]))) + +#define FILTER_BILIN_RND(src, x, mxy, stride, sh) \ + ((FILTER_BILIN(src, x, mxy, stride) + ((1 << (sh)) >> 1)) >> (sh)) + +#define FILTER_BILIN_CLIP(src, x, mxy, stride, sh) \ + iclip_pixel(FILTER_BILIN_RND(src, x, mxy, stride, sh)) + +static void put_bilin_c(pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int intermediate_rnd = (1 << intermediate_bits) >> 1; + dst_stride = PXSTRIDE(dst_stride); + src_stride = PXSTRIDE(src_stride); + + if (mx) { + if (my) { + int16_t mid[128 * 129], *mid_ptr = mid; + int tmp_h = h + 1; + + do { + for (int x = 0; x < w; x++) + mid_ptr[x] = FILTER_BILIN_RND(src, x, mx, 1, + 4 - intermediate_bits); + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid; + do { + for (int x = 0; x < w; x++) + dst[x] = FILTER_BILIN_CLIP(mid_ptr, x, my, 128, + 4 + intermediate_bits); + + mid_ptr += 128; + dst += dst_stride; + } while (--h); + } else { + do { + for (int x = 0; x < w; x++) { + const int px = FILTER_BILIN_RND(src, x, mx, 1, + 4 - intermediate_bits); + dst[x] = iclip_pixel((px + intermediate_rnd) >> intermediate_bits); + } + + dst += dst_stride; + src += src_stride; + } while (--h); + } + } else if (my) { + do { + for (int x = 0; x < w; x++) + dst[x] = FILTER_BILIN_CLIP(src, x, my, src_stride, 4); + + dst += dst_stride; + src += src_stride; + } while (--h); + } else + put_c(dst, dst_stride, src, src_stride, w, h); +} + +static void put_bilin_scaled_c(pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, int my, + const int dx, const int dy + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + int tmp_h = (((h - 1) * dy + my) >> 10) + 2; + int16_t mid[128 * (256 + 1)], *mid_ptr = mid; + + do { + int x; + int imx = mx, ioff = 0; + + for (x = 0; x < w; x++) { + mid_ptr[x] = FILTER_BILIN_RND(src, ioff, imx >> 6, 1, + 4 - intermediate_bits); + imx += dx; + ioff += imx >> 10; + imx &= 0x3ff; + } + + mid_ptr += 128; + src += PXSTRIDE(src_stride); + } while (--tmp_h); + + mid_ptr = mid; + do { + int x; + + for (x = 0; x < w; x++) + dst[x] = FILTER_BILIN_CLIP(mid_ptr, x, my >> 6, 128, + 4 + intermediate_bits); + + my += dy; + mid_ptr += (my >> 10) * 128; + my &= 0x3ff; + dst += PXSTRIDE(dst_stride); + } while (--h); +} + +static void prep_bilin_c(int16_t *tmp, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + src_stride = PXSTRIDE(src_stride); + + if (mx) { + if (my) { + int16_t mid[128 * 129], *mid_ptr = mid; + int tmp_h = h + 1; + + do { + for (int x = 0; x < w; x++) + mid_ptr[x] = FILTER_BILIN_RND(src, x, mx, 1, + 4 - intermediate_bits); + + mid_ptr += 128; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid; + do { + for (int x = 0; x < w; x++) + tmp[x] = FILTER_BILIN_RND(mid_ptr, x, my, 128, 4) - + PREP_BIAS; + + mid_ptr += 128; + tmp += w; + } while (--h); + } else { + do { + for (int x = 0; x < w; x++) + tmp[x] = FILTER_BILIN_RND(src, x, mx, 1, + 4 - intermediate_bits) - + PREP_BIAS; + + tmp += w; + src += src_stride; + } while (--h); + } + } else if (my) { + do { + for (int x = 0; x < w; x++) + tmp[x] = FILTER_BILIN_RND(src, x, my, src_stride, + 4 - intermediate_bits) - PREP_BIAS; + + tmp += w; + src += src_stride; + } while (--h); + } else + prep_c(tmp, src, src_stride, w, h HIGHBD_TAIL_SUFFIX); +} + +static void prep_bilin_scaled_c(int16_t *tmp, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, int my, + const int dx, const int dy HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + int tmp_h = (((h - 1) * dy + my) >> 10) + 2; + int16_t mid[128 * (256 + 1)], *mid_ptr = mid; + + do { + int x; + int imx = mx, ioff = 0; + + for (x = 0; x < w; x++) { + mid_ptr[x] = FILTER_BILIN_RND(src, ioff, imx >> 6, 1, + 4 - intermediate_bits); + imx += dx; + ioff += imx >> 10; + imx &= 0x3ff; + } + + mid_ptr += 128; + src += PXSTRIDE(src_stride); + } while (--tmp_h); + + mid_ptr = mid; + do { + int x; + + for (x = 0; x < w; x++) + tmp[x] = FILTER_BILIN_RND(mid_ptr, x, my >> 6, 128, 4) - PREP_BIAS; + + my += dy; + mid_ptr += (my >> 10) * 128; + my &= 0x3ff; + tmp += w; + } while (--h); +} + +static void avg_c(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, const int w, int h + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int sh = intermediate_bits + 1; + const int rnd = (1 << intermediate_bits) + PREP_BIAS * 2; + do { + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel((tmp1[x] + tmp2[x] + rnd) >> sh); + + tmp1 += w; + tmp2 += w; + dst += PXSTRIDE(dst_stride); + } while (--h); +} + +static void w_avg_c(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, const int w, int h, + const int weight HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int sh = intermediate_bits + 4; + const int rnd = (8 << intermediate_bits) + PREP_BIAS * 16; + do { + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel((tmp1[x] * weight + + tmp2[x] * (16 - weight) + rnd) >> sh); + + tmp1 += w; + tmp2 += w; + dst += PXSTRIDE(dst_stride); + } while (--h); +} + +static void mask_c(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, const int w, int h, + const uint8_t *mask HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int sh = intermediate_bits + 6; + const int rnd = (32 << intermediate_bits) + PREP_BIAS * 64; + do { + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel((tmp1[x] * mask[x] + + tmp2[x] * (64 - mask[x]) + rnd) >> sh); + + tmp1 += w; + tmp2 += w; + mask += w; + dst += PXSTRIDE(dst_stride); + } while (--h); +} + +#define blend_px(a, b, m) (((a * (64 - m) + b * m) + 32) >> 6) +static void blend_c(pixel *dst, const ptrdiff_t dst_stride, const pixel *tmp, + const int w, int h, const uint8_t *mask) +{ + do { + for (int x = 0; x < w; x++) { + dst[x] = blend_px(dst[x], tmp[x], mask[x]); + } + dst += PXSTRIDE(dst_stride); + tmp += w; + mask += w; + } while (--h); +} + +static void blend_v_c(pixel *dst, const ptrdiff_t dst_stride, const pixel *tmp, + const int w, int h) +{ + const uint8_t *const mask = &dav1d_obmc_masks[w]; + do { + for (int x = 0; x < (w * 3) >> 2; x++) { + dst[x] = blend_px(dst[x], tmp[x], mask[x]); + } + dst += PXSTRIDE(dst_stride); + tmp += w; + } while (--h); +} + +static void blend_h_c(pixel *dst, const ptrdiff_t dst_stride, const pixel *tmp, + const int w, int h) +{ + const uint8_t *mask = &dav1d_obmc_masks[h]; + h = (h * 3) >> 2; + do { + const int m = *mask++; + for (int x = 0; x < w; x++) { + dst[x] = blend_px(dst[x], tmp[x], m); + } + dst += PXSTRIDE(dst_stride); + tmp += w; + } while (--h); +} + +static void w_mask_c(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, const int w, int h, + uint8_t *mask, const int sign, + const int ss_hor, const int ss_ver HIGHBD_DECL_SUFFIX) +{ + // store mask at 2x2 resolution, i.e. store 2x1 sum for even rows, + // and then load this intermediate to calculate final value for odd rows + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + const int bitdepth = bitdepth_from_max(bitdepth_max); + const int sh = intermediate_bits + 6; + const int rnd = (32 << intermediate_bits) + PREP_BIAS * 64; + const int mask_sh = bitdepth + intermediate_bits - 4; + const int mask_rnd = 1 << (mask_sh - 5); + do { + for (int x = 0; x < w; x++) { + const int m = imin(38 + ((abs(tmp1[x] - tmp2[x]) + mask_rnd) >> mask_sh), 64); + dst[x] = iclip_pixel((tmp1[x] * m + + tmp2[x] * (64 - m) + rnd) >> sh); + + if (ss_hor) { + x++; + + const int n = imin(38 + ((abs(tmp1[x] - tmp2[x]) + mask_rnd) >> mask_sh), 64); + dst[x] = iclip_pixel((tmp1[x] * n + + tmp2[x] * (64 - n) + rnd) >> sh); + + if (h & ss_ver) { + mask[x >> 1] = (m + n + mask[x >> 1] + 2 - sign) >> 2; + } else if (ss_ver) { + mask[x >> 1] = m + n; + } else { + mask[x >> 1] = (m + n + 1 - sign) >> 1; + } + } else { + mask[x] = m; + } + } + + tmp1 += w; + tmp2 += w; + dst += PXSTRIDE(dst_stride); + if (!ss_ver || (h & 1)) mask += w >> ss_hor; + } while (--h); +} + +#define w_mask_fns(ssn, ss_hor, ss_ver) \ +static void w_mask_##ssn##_c(pixel *const dst, const ptrdiff_t dst_stride, \ + const int16_t *const tmp1, const int16_t *const tmp2, \ + const int w, const int h, uint8_t *mask, \ + const int sign HIGHBD_DECL_SUFFIX) \ +{ \ + w_mask_c(dst, dst_stride, tmp1, tmp2, w, h, mask, sign, ss_hor, ss_ver \ + HIGHBD_TAIL_SUFFIX); \ +} + +w_mask_fns(444, 0, 0); +w_mask_fns(422, 1, 0); +w_mask_fns(420, 1, 1); + +#undef w_mask_fns + +#define FILTER_WARP_RND(src, x, F, stride, sh) \ + ((F[0] * src[x - 3 * stride] + \ + F[1] * src[x - 2 * stride] + \ + F[2] * src[x - 1 * stride] + \ + F[3] * src[x + 0 * stride] + \ + F[4] * src[x + 1 * stride] + \ + F[5] * src[x + 2 * stride] + \ + F[6] * src[x + 3 * stride] + \ + F[7] * src[x + 4 * stride] + \ + ((1 << (sh)) >> 1)) >> (sh)) + +#define FILTER_WARP_CLIP(src, x, F, stride, sh) \ + iclip_pixel(FILTER_WARP_RND(src, x, F, stride, sh)) + +static void warp_affine_8x8_c(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *const abcd, int mx, int my + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + int16_t mid[15 * 8], *mid_ptr = mid; + + src -= 3 * PXSTRIDE(src_stride); + for (int y = 0; y < 15; y++, mx += abcd[1]) { + for (int x = 0, tmx = mx; x < 8; x++, tmx += abcd[0]) { + const int8_t *const filter = + dav1d_mc_warp_filter[64 + ((tmx + 512) >> 10)]; + + mid_ptr[x] = FILTER_WARP_RND(src, x, filter, 1, + 7 - intermediate_bits); + } + src += PXSTRIDE(src_stride); + mid_ptr += 8; + } + + mid_ptr = &mid[3 * 8]; + for (int y = 0; y < 8; y++, my += abcd[3]) { + for (int x = 0, tmy = my; x < 8; x++, tmy += abcd[2]) { + const int8_t *const filter = + dav1d_mc_warp_filter[64 + ((tmy + 512) >> 10)]; + + dst[x] = FILTER_WARP_CLIP(mid_ptr, x, filter, 8, + 7 + intermediate_bits); + } + mid_ptr += 8; + dst += PXSTRIDE(dst_stride); + } +} + +static void warp_affine_8x8t_c(int16_t *tmp, const ptrdiff_t tmp_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *const abcd, int mx, int my + HIGHBD_DECL_SUFFIX) +{ + const int intermediate_bits = get_intermediate_bits(bitdepth_max); + int16_t mid[15 * 8], *mid_ptr = mid; + + src -= 3 * PXSTRIDE(src_stride); + for (int y = 0; y < 15; y++, mx += abcd[1]) { + for (int x = 0, tmx = mx; x < 8; x++, tmx += abcd[0]) { + const int8_t *const filter = + dav1d_mc_warp_filter[64 + ((tmx + 512) >> 10)]; + + mid_ptr[x] = FILTER_WARP_RND(src, x, filter, 1, + 7 - intermediate_bits); + } + src += PXSTRIDE(src_stride); + mid_ptr += 8; + } + + mid_ptr = &mid[3 * 8]; + for (int y = 0; y < 8; y++, my += abcd[3]) { + for (int x = 0, tmy = my; x < 8; x++, tmy += abcd[2]) { + const int8_t *const filter = + dav1d_mc_warp_filter[64 + ((tmy + 512) >> 10)]; + + tmp[x] = FILTER_WARP_RND(mid_ptr, x, filter, 8, 7) - PREP_BIAS; + } + mid_ptr += 8; + tmp += tmp_stride; + } +} + +static void emu_edge_c(const intptr_t bw, const intptr_t bh, + const intptr_t iw, const intptr_t ih, + const intptr_t x, const intptr_t y, + pixel *dst, const ptrdiff_t dst_stride, + const pixel *ref, const ptrdiff_t ref_stride) +{ + // find offset in reference of visible block to copy + ref += iclip((int) y, 0, (int) ih - 1) * PXSTRIDE(ref_stride) + + iclip((int) x, 0, (int) iw - 1); + + // number of pixels to extend (left, right, top, bottom) + const int left_ext = iclip((int) -x, 0, (int) bw - 1); + const int right_ext = iclip((int) (x + bw - iw), 0, (int) bw - 1); + assert(left_ext + right_ext < bw); + const int top_ext = iclip((int) -y, 0, (int) bh - 1); + const int bottom_ext = iclip((int) (y + bh - ih), 0, (int) bh - 1); + assert(top_ext + bottom_ext < bh); + + // copy visible portion first + pixel *blk = dst + top_ext * PXSTRIDE(dst_stride); + const int center_w = (int) (bw - left_ext - right_ext); + const int center_h = (int) (bh - top_ext - bottom_ext); + for (int y = 0; y < center_h; y++) { + pixel_copy(blk + left_ext, ref, center_w); + // extend left edge for this line + if (left_ext) + pixel_set(blk, blk[left_ext], left_ext); + // extend right edge for this line + if (right_ext) + pixel_set(blk + left_ext + center_w, blk[left_ext + center_w - 1], + right_ext); + ref += PXSTRIDE(ref_stride); + blk += PXSTRIDE(dst_stride); + } + + // copy top + blk = dst + top_ext * PXSTRIDE(dst_stride); + for (int y = 0; y < top_ext; y++) { + pixel_copy(dst, blk, bw); + dst += PXSTRIDE(dst_stride); + } + + // copy bottom + dst += center_h * PXSTRIDE(dst_stride); + for (int y = 0; y < bottom_ext; y++) { + pixel_copy(dst, &dst[-PXSTRIDE(dst_stride)], bw); + dst += PXSTRIDE(dst_stride); + } +} + +static void resize_c(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int dst_w, int h, const int src_w, + const int dx, const int mx0 HIGHBD_DECL_SUFFIX) +{ + do { + int mx = mx0, src_x = -1; + for (int x = 0; x < dst_w; x++) { + const int8_t *const F = dav1d_resize_filter[mx >> 8]; + dst[x] = iclip_pixel((-(F[0] * src[iclip(src_x - 3, 0, src_w - 1)] + + F[1] * src[iclip(src_x - 2, 0, src_w - 1)] + + F[2] * src[iclip(src_x - 1, 0, src_w - 1)] + + F[3] * src[iclip(src_x + 0, 0, src_w - 1)] + + F[4] * src[iclip(src_x + 1, 0, src_w - 1)] + + F[5] * src[iclip(src_x + 2, 0, src_w - 1)] + + F[6] * src[iclip(src_x + 3, 0, src_w - 1)] + + F[7] * src[iclip(src_x + 4, 0, src_w - 1)]) + + 64) >> 7); + mx += dx; + src_x += mx >> 14; + mx &= 0x3fff; + } + + dst += PXSTRIDE(dst_stride); + src += PXSTRIDE(src_stride); + } while (--h); +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/mc.h" +#elif ARCH_X86 +#include "src/x86/mc.h" +#endif +#endif + +COLD void bitfn(dav1d_mc_dsp_init)(Dav1dMCDSPContext *const c) { +#define init_mc_fns(type, name) do { \ + c->mc [type] = put_##name##_c; \ + c->mc_scaled [type] = put_##name##_scaled_c; \ + c->mct [type] = prep_##name##_c; \ + c->mct_scaled[type] = prep_##name##_scaled_c; \ +} while (0) + + init_mc_fns(FILTER_2D_8TAP_REGULAR, 8tap_regular); + init_mc_fns(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth); + init_mc_fns(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp); + init_mc_fns(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular); + init_mc_fns(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth); + init_mc_fns(FILTER_2D_8TAP_SHARP, 8tap_sharp); + init_mc_fns(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular); + init_mc_fns(FILTER_2D_8TAP_SMOOTH, 8tap_smooth); + init_mc_fns(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp); + init_mc_fns(FILTER_2D_BILINEAR, bilin); + + c->avg = avg_c; + c->w_avg = w_avg_c; + c->mask = mask_c; + c->blend = blend_c; + c->blend_v = blend_v_c; + c->blend_h = blend_h_c; + c->w_mask[0] = w_mask_444_c; + c->w_mask[1] = w_mask_422_c; + c->w_mask[2] = w_mask_420_c; + c->warp8x8 = warp_affine_8x8_c; + c->warp8x8t = warp_affine_8x8t_c; + c->emu_edge = emu_edge_c; + c->resize = resize_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + mc_dsp_init_arm(c); +#elif ARCH_X86 + mc_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/mem.c b/third_party/dav1d/src/mem.c new file mode 100644 index 0000000000..558bc01cae --- /dev/null +++ b/third_party/dav1d/src/mem.c @@ -0,0 +1,119 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "src/internal.h" + +static COLD void mem_pool_destroy(Dav1dMemPool *const pool) { + pthread_mutex_destroy(&pool->lock); + free(pool); +} + +void dav1d_mem_pool_push(Dav1dMemPool *const pool, Dav1dMemPoolBuffer *const buf) { + pthread_mutex_lock(&pool->lock); + const int ref_cnt = --pool->ref_cnt; + if (!pool->end) { + buf->next = pool->buf; + pool->buf = buf; + pthread_mutex_unlock(&pool->lock); + assert(ref_cnt > 0); + } else { + pthread_mutex_unlock(&pool->lock); + dav1d_free_aligned(buf->data); + if (!ref_cnt) mem_pool_destroy(pool); + } +} + +Dav1dMemPoolBuffer *dav1d_mem_pool_pop(Dav1dMemPool *const pool, const size_t size) { + assert(!(size & (sizeof(void*) - 1))); + pthread_mutex_lock(&pool->lock); + Dav1dMemPoolBuffer *buf = pool->buf; + pool->ref_cnt++; + uint8_t *data; + if (buf) { + pool->buf = buf->next; + pthread_mutex_unlock(&pool->lock); + data = buf->data; + if ((uintptr_t)buf - (uintptr_t)data != size) { + /* Reallocate if the size has changed */ + dav1d_free_aligned(data); + goto alloc; + } + } else { + pthread_mutex_unlock(&pool->lock); +alloc: + data = dav1d_alloc_aligned(size + sizeof(Dav1dMemPoolBuffer), 64); + if (!data) { + pthread_mutex_lock(&pool->lock); + const int ref_cnt = --pool->ref_cnt; + pthread_mutex_unlock(&pool->lock); + if (!ref_cnt) mem_pool_destroy(pool); + return NULL; + } + buf = (Dav1dMemPoolBuffer*)(data + size); + buf->data = data; + } + + return buf; +} + +COLD int dav1d_mem_pool_init(Dav1dMemPool **const ppool) { + Dav1dMemPool *const pool = malloc(sizeof(Dav1dMemPool)); + if (pool) { + if (!pthread_mutex_init(&pool->lock, NULL)) { + pool->buf = NULL; + pool->ref_cnt = 1; + pool->end = 0; + *ppool = pool; + return 0; + } + free(pool); + } + *ppool = NULL; + return DAV1D_ERR(ENOMEM); +} + +COLD void dav1d_mem_pool_end(Dav1dMemPool *const pool) { + if (pool) { + pthread_mutex_lock(&pool->lock); + Dav1dMemPoolBuffer *buf = pool->buf; + const int ref_cnt = --pool->ref_cnt; + pool->buf = NULL; + pool->end = 1; + pthread_mutex_unlock(&pool->lock); + + while (buf) { + void *const data = buf->data; + buf = buf->next; + dav1d_free_aligned(data); + } + if (!ref_cnt) mem_pool_destroy(pool); + } +} diff --git a/third_party/dav1d/src/mem.h b/third_party/dav1d/src/mem.h new file mode 100644 index 0000000000..41ae47a2fd --- /dev/null +++ b/third_party/dav1d/src/mem.h @@ -0,0 +1,103 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_MEM_H +#define DAV1D_SRC_MEM_H + +#include + +#if defined(HAVE_ALIGNED_MALLOC) || defined(HAVE_MEMALIGN) +#include +#endif + +#include "common/attributes.h" + +#include "src/thread.h" + +typedef struct Dav1dMemPoolBuffer { + void *data; + struct Dav1dMemPoolBuffer *next; +} Dav1dMemPoolBuffer; + +typedef struct Dav1dMemPool { + pthread_mutex_t lock; + Dav1dMemPoolBuffer *buf; + int ref_cnt; + int end; +} Dav1dMemPool; + +void dav1d_mem_pool_push(Dav1dMemPool *pool, Dav1dMemPoolBuffer *buf); +Dav1dMemPoolBuffer *dav1d_mem_pool_pop(Dav1dMemPool *pool, size_t size); +int dav1d_mem_pool_init(Dav1dMemPool **pool); +void dav1d_mem_pool_end(Dav1dMemPool *pool); + +/* + * Allocate align-byte aligned memory. The return value can be released + * by calling the dav1d_free_aligned() function. + */ +static inline void *dav1d_alloc_aligned(size_t sz, size_t align) { + assert(!(align & (align - 1))); +#ifdef HAVE_POSIX_MEMALIGN + void *ptr; + if (posix_memalign(&ptr, align, sz)) return NULL; + return ptr; +#elif defined(HAVE_ALIGNED_MALLOC) + return _aligned_malloc(sz, align); +#elif defined(HAVE_MEMALIGN) + return memalign(align, sz); +#else +#error Missing aligned alloc implementation +#endif +} + +static inline void dav1d_free_aligned(void* ptr) { +#ifdef HAVE_POSIX_MEMALIGN + free(ptr); +#elif defined(HAVE_ALIGNED_MALLOC) + _aligned_free(ptr); +#elif defined(HAVE_MEMALIGN) + free(ptr); +#endif +} + +static inline void dav1d_freep_aligned(void* ptr) { + void **mem = (void **) ptr; + if (*mem) { + dav1d_free_aligned(*mem); + *mem = NULL; + } +} + +static inline void freep(void *ptr) { + void **mem = (void **) ptr; + if (*mem) { + free(*mem); + *mem = NULL; + } +} + +#endif /* DAV1D_SRC_MEM_H */ diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build new file mode 100644 index 0000000000..2ace55e1a8 --- /dev/null +++ b/third_party/dav1d/src/meson.build @@ -0,0 +1,348 @@ +# Copyright © 2018-2019, VideoLAN and dav1d authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# +# Build definition for the dav1d library +# + +# libdav1d source files +libdav1d_sources = files( + 'cdf.c', + 'cpu.c', + 'data.c', + 'decode.c', + 'dequant_tables.c', + 'getbits.c', + 'intra_edge.c', + 'itx_1d.c', + 'lf_mask.c', + 'lib.c', + 'log.c', + 'mem.c', + 'msac.c', + 'obu.c', + 'picture.c', + 'qm.c', + 'ref.c', + 'refmvs.c', + 'scan.c', + 'tables.c', + 'thread_task.c', + 'warpmv.c', + 'wedge.c', +) + +# libdav1d bitdepth source files +# These files are compiled for each bitdepth with +# `BITDEPTH` defined to the currently built bitdepth. +libdav1d_tmpl_sources = files( + 'cdef_apply_tmpl.c', + 'cdef_tmpl.c', + 'fg_apply_tmpl.c', + 'filmgrain_tmpl.c', + 'ipred_prepare_tmpl.c', + 'ipred_tmpl.c', + 'itx_tmpl.c', + 'lf_apply_tmpl.c', + 'loopfilter_tmpl.c', + 'looprestoration_tmpl.c', + 'lr_apply_tmpl.c', + 'mc_tmpl.c', + 'recon_tmpl.c', +) + +libdav1d_arch_tmpl_sources = [] + +libdav1d_bitdepth_objs = [] + +# ASM specific sources +libdav1d_asm_objs = [] +# Arch-specific flags +arch_flags = [] +if is_asm_enabled + if (host_machine.cpu_family() == 'aarch64' or + host_machine.cpu_family().startswith('arm')) + + libdav1d_sources += files( + 'arm/cpu.c', + ) + if (host_machine.cpu_family() == 'aarch64' or + host_machine.cpu() == 'arm64') + libdav1d_sources_asm = files( + # itx.S is used for both 8 and 16 bpc. + 'arm/64/itx.S', + 'arm/64/looprestoration_common.S', + 'arm/64/msac.S', + 'arm/64/refmvs.S', + ) + + if dav1d_bitdepths.contains('8') + libdav1d_sources_asm += files( + 'arm/64/cdef.S', + 'arm/64/filmgrain.S', + 'arm/64/ipred.S', + 'arm/64/loopfilter.S', + 'arm/64/looprestoration.S', + 'arm/64/mc.S', + ) + endif + + if dav1d_bitdepths.contains('16') + libdav1d_sources_asm += files( + 'arm/64/cdef16.S', + 'arm/64/filmgrain16.S', + 'arm/64/ipred16.S', + 'arm/64/itx16.S', + 'arm/64/loopfilter16.S', + 'arm/64/looprestoration16.S', + 'arm/64/mc16.S', + ) + endif + elif host_machine.cpu_family().startswith('arm') + libdav1d_sources_asm = files( + # itx.S is used for both 8 and 16 bpc. + 'arm/32/itx.S', + 'arm/32/looprestoration_common.S', + 'arm/32/msac.S', + 'arm/32/refmvs.S', + ) + + if dav1d_bitdepths.contains('8') + libdav1d_sources_asm += files( + 'arm/32/cdef.S', + 'arm/32/filmgrain.S', + 'arm/32/ipred.S', + 'arm/32/loopfilter.S', + 'arm/32/looprestoration.S', + 'arm/32/mc.S', + ) + endif + + if dav1d_bitdepths.contains('16') + libdav1d_sources_asm += files( + 'arm/32/cdef16.S', + 'arm/32/filmgrain16.S', + 'arm/32/ipred16.S', + 'arm/32/itx16.S', + 'arm/32/loopfilter16.S', + 'arm/32/looprestoration16.S', + 'arm/32/mc16.S', + ) + endif + endif + + if use_gaspp + libdav1d_asm_objs = gaspp_gen.process(libdav1d_sources_asm) + else + libdav1d_sources += libdav1d_sources_asm + endif + elif host_machine.cpu_family().startswith('x86') + + libdav1d_sources += files( + 'x86/cpu.c', + ) + + # NASM source files + libdav1d_sources_asm = files( + 'x86/cpuid.asm', + 'x86/msac.asm', + 'x86/refmvs.asm', + 'x86/itx_avx512.asm', + 'x86/cdef_avx2.asm', + 'x86/itx_avx2.asm', + 'x86/looprestoration_avx2.asm', + 'x86/cdef_sse.asm', + 'x86/itx_sse.asm', + ) + + if dav1d_bitdepths.contains('8') + libdav1d_sources_asm += files( + 'x86/cdef_avx512.asm', + 'x86/filmgrain_avx512.asm', + 'x86/ipred_avx512.asm', + 'x86/loopfilter_avx512.asm', + 'x86/looprestoration_avx512.asm', + 'x86/mc_avx512.asm', + 'x86/filmgrain_avx2.asm', + 'x86/ipred_avx2.asm', + 'x86/loopfilter_avx2.asm', + 'x86/mc_avx2.asm', + 'x86/filmgrain_sse.asm', + 'x86/ipred_sse.asm', + 'x86/loopfilter_sse.asm', + 'x86/looprestoration_sse.asm', + 'x86/mc_sse.asm', + ) + endif + + if dav1d_bitdepths.contains('16') + libdav1d_sources_asm += files( + 'x86/cdef16_avx512.asm', + 'x86/filmgrain16_avx512.asm', + 'x86/ipred16_avx512.asm', + 'x86/itx16_avx512.asm', + 'x86/loopfilter16_avx512.asm', + 'x86/looprestoration16_avx512.asm', + 'x86/mc16_avx512.asm', + 'x86/cdef16_avx2.asm', + 'x86/filmgrain16_avx2.asm', + 'x86/ipred16_avx2.asm', + 'x86/itx16_avx2.asm', + 'x86/loopfilter16_avx2.asm', + 'x86/looprestoration16_avx2.asm', + 'x86/mc16_avx2.asm', + 'x86/cdef16_sse.asm', + 'x86/filmgrain16_sse.asm', + 'x86/ipred16_sse.asm', + 'x86/itx16_sse.asm', + 'x86/loopfilter16_sse.asm', + 'x86/looprestoration16_sse.asm', + 'x86/mc16_sse.asm', + ) + endif + + # Compile the ASM sources with NASM + libdav1d_asm_objs = nasm_gen.process(libdav1d_sources_asm) + elif host_machine.cpu() == 'ppc64le' + arch_flags = ['-maltivec', '-mvsx'] + libdav1d_sources += files( + 'ppc/cpu.c', + ) + libdav1d_arch_tmpl_sources += files( + 'ppc/cdef_tmpl.c', + 'ppc/looprestoration_tmpl.c', + ) + endif +endif + + + +libdav1d_rc_obj = [] +libdav1d_flags = [] +api_export_flags = [] + +# +# Windows .rc file and API export flags +# + +if host_machine.system() == 'windows' + if get_option('default_library') != 'static' + rc_file = configure_file( + input : 'dav1d.rc.in', + output : 'dav1d.rc', + configuration : rc_data + ) + + libdav1d_rc_obj = winmod.compile_resources(rc_file) + + api_export_flags = ['-DDAV1D_BUILDING_DLL'] + endif + + if (host_machine.cpu_family() == 'x86_64' and cc.get_id() == 'gcc') + # We don't expect to reference data members from other DLLs without + # dllimport attributes. Set the -mcmodel=small flag, which avoids + # generating indirection via .refptr. for all potentially + # dllimported variable references. + libdav1d_flags += '-mcmodel=small' + endif +endif + + + +# +# Library definitions +# + +# Helper library for each bitdepth +libdav1d_bitdepth_objs = [] +foreach bitdepth : dav1d_bitdepths + libdav1d_bitdepth_objs += static_library( + 'dav1d_bitdepth_@0@'.format(bitdepth), + libdav1d_tmpl_sources, config_h_target, + include_directories: dav1d_inc_dirs, + dependencies : [stdatomic_dependencies], + c_args : ['-DBITDEPTH=@0@'.format(bitdepth)] + libdav1d_flags, + install : false, + build_by_default : false, + ).extract_all_objects(recursive: true) +endforeach + +# Helper library for each bitdepth and architecture-specific flags +foreach bitdepth : dav1d_bitdepths + libdav1d_bitdepth_objs += static_library( + 'dav1d_arch_bitdepth_@0@'.format(bitdepth), + libdav1d_arch_tmpl_sources, config_h_target, + include_directories: dav1d_inc_dirs, + dependencies : [stdatomic_dependencies], + c_args : ['-DBITDEPTH=@0@'.format(bitdepth)] + libdav1d_flags + arch_flags, + install : false, + build_by_default : false, + ).extract_all_objects(recursive: true) +endforeach + +# The final dav1d library +if host_machine.system() == 'windows' + dav1d_soversion = '' +else + dav1d_soversion = dav1d_api_version_major +endif + +libdav1d = library('dav1d', + libdav1d_sources, + libdav1d_asm_objs, + libdav1d_rc_obj, + rev_target, + config_h_target, + + objects : [ + libdav1d_bitdepth_objs, + ], + + include_directories : dav1d_inc_dirs, + dependencies : [ + stdatomic_dependencies, + thread_dependency, + thread_compat_dep, + libdl_dependency, + ], + c_args : [libdav1d_flags, api_export_flags], + version : dav1d_soname_version, + soversion : dav1d_soversion, + install : true, +) + +dav1d_dep = declare_dependency(link_with: libdav1d, + include_directories : include_directories('../include/dav1d') +) + +# +# Generate pkg-config .pc file +# +pkg_mod = import('pkgconfig') +pkg_mod.generate(libraries: libdav1d, + version: meson.project_version(), + name: 'libdav1d', + filebase: 'dav1d', + description: 'AV1 decoding library' +) diff --git a/third_party/dav1d/src/msac.c b/third_party/dav1d/src/msac.c new file mode 100644 index 0000000000..43d8ae5d07 --- /dev/null +++ b/third_party/dav1d/src/msac.c @@ -0,0 +1,208 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/msac.h" + +#define EC_PROB_SHIFT 6 +#define EC_MIN_PROB 4 // must be <= (1<buf_pos; + const uint8_t *buf_end = s->buf_end; + int c = EC_WIN_SIZE - s->cnt - 24; + ec_win dif = s->dif; + while (c >= 0 && buf_pos < buf_end) { + dif ^= ((ec_win)*buf_pos++) << c; + c -= 8; + } + s->dif = dif; + s->cnt = EC_WIN_SIZE - c - 24; + s->buf_pos = buf_pos; +} + +/* Takes updated dif and range values, renormalizes them so that + * 32768 <= rng < 65536 (reading more bytes from the stream into dif if + * necessary), and stores them back in the decoder context. + * dif: The new value of dif. + * rng: The new value of the range. */ +static inline void ctx_norm(MsacContext *const s, const ec_win dif, + const unsigned rng) +{ + const int d = 15 ^ (31 ^ clz(rng)); + assert(rng <= 65535U); + s->cnt -= d; + s->dif = ((dif + 1) << d) - 1; /* Shift in 1s in the LSBs */ + s->rng = rng << d; + if (s->cnt < 0) + ctx_refill(s); +} + +unsigned dav1d_msac_decode_bool_equi_c(MsacContext *const s) { + const unsigned r = s->rng; + ec_win dif = s->dif; + assert((dif >> (EC_WIN_SIZE - 16)) < r); + // When the probability is 1/2, f = 16384 >> EC_PROB_SHIFT = 256 and we can + // replace the multiply with a simple shift. + unsigned v = ((r >> 8) << 7) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); + ctx_norm(s, dif, v); + return !ret; +} + +/* Decode a single binary value. + * f: The probability that the bit is one + * Return: The value decoded (0 or 1). */ +unsigned dav1d_msac_decode_bool_c(MsacContext *const s, const unsigned f) { + const unsigned r = s->rng; + ec_win dif = s->dif; + assert((dif >> (EC_WIN_SIZE - 16)) < r); + unsigned v = ((r >> 8) * (f >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); + ctx_norm(s, dif, v); + return !ret; +} + +int dav1d_msac_decode_subexp(MsacContext *const s, const int ref, + const int n, unsigned k) +{ + assert(n >> k == 8); + + unsigned a = 0; + if (dav1d_msac_decode_bool_equi(s)) { + if (dav1d_msac_decode_bool_equi(s)) + k += dav1d_msac_decode_bool_equi(s) + 1; + a = 1 << k; + } + const unsigned v = dav1d_msac_decode_bools(s, k) + a; + return ref * 2 <= n ? inv_recenter(ref, v) : + n - 1 - inv_recenter(n - 1 - ref, v); +} + +/* Decodes a symbol given an inverse cumulative distribution function (CDF) + * table in Q15. */ +unsigned dav1d_msac_decode_symbol_adapt_c(MsacContext *const s, + uint16_t *const cdf, + const size_t n_symbols) +{ + const unsigned c = s->dif >> (EC_WIN_SIZE - 16), r = s->rng >> 8; + unsigned u, v = s->rng, val = -1; + + assert(n_symbols <= 15); + assert(cdf[n_symbols] <= 32); + + do { + val++; + u = v; + v = r * (cdf[val] >> EC_PROB_SHIFT); + v >>= 7 - EC_PROB_SHIFT; + v += EC_MIN_PROB * ((unsigned)n_symbols - val); + } while (c < v); + + assert(u <= s->rng); + + ctx_norm(s, s->dif - ((ec_win)v << (EC_WIN_SIZE - 16)), u - v); + + if (s->allow_update_cdf) { + const unsigned count = cdf[n_symbols]; + const unsigned rate = 4 + (count >> 4) + (n_symbols > 2); + unsigned i; + for (i = 0; i < val; i++) + cdf[i] += (32768 - cdf[i]) >> rate; + for (; i < n_symbols; i++) + cdf[i] -= cdf[i] >> rate; + cdf[n_symbols] = count + (count < 32); + } + + return val; +} + +unsigned dav1d_msac_decode_bool_adapt_c(MsacContext *const s, + uint16_t *const cdf) +{ + const unsigned bit = dav1d_msac_decode_bool(s, *cdf); + + if (s->allow_update_cdf) { + // update_cdf() specialized for boolean CDFs + const unsigned count = cdf[1]; + const int rate = 4 + (count >> 4); + if (bit) + cdf[0] += (32768 - cdf[0]) >> rate; + else + cdf[0] -= cdf[0] >> rate; + cdf[1] = count + (count < 32); + } + + return bit; +} + +unsigned dav1d_msac_decode_hi_tok_c(MsacContext *const s, uint16_t *const cdf) { + unsigned tok_br = dav1d_msac_decode_symbol_adapt4(s, cdf, 3); + unsigned tok = 3 + tok_br; + if (tok_br == 3) { + tok_br = dav1d_msac_decode_symbol_adapt4(s, cdf, 3); + tok = 6 + tok_br; + if (tok_br == 3) { + tok_br = dav1d_msac_decode_symbol_adapt4(s, cdf, 3); + tok = 9 + tok_br; + if (tok_br == 3) + tok = 12 + dav1d_msac_decode_symbol_adapt4(s, cdf, 3); + } + } + return tok; +} + +void dav1d_msac_init(MsacContext *const s, const uint8_t *const data, + const size_t sz, const int disable_cdf_update_flag) +{ + s->buf_pos = data; + s->buf_end = data + sz; + s->dif = ((ec_win)1 << (EC_WIN_SIZE - 1)) - 1; + s->rng = 0x8000; + s->cnt = -15; + s->allow_update_cdf = !disable_cdf_update_flag; + ctx_refill(s); + +#if ARCH_X86_64 && HAVE_ASM + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt_c; + + msac_init_x86(s); +#endif +} diff --git a/third_party/dav1d/src/msac.h b/third_party/dav1d/src/msac.h new file mode 100644 index 0000000000..eb04f58f81 --- /dev/null +++ b/third_party/dav1d/src/msac.h @@ -0,0 +1,108 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_MSAC_H +#define DAV1D_SRC_MSAC_H + +#include +#include + +#include "common/intops.h" + +typedef size_t ec_win; + +typedef struct MsacContext { + const uint8_t *buf_pos; + const uint8_t *buf_end; + ec_win dif; + unsigned rng; + int cnt; + int allow_update_cdf; + +#if ARCH_X86_64 && HAVE_ASM + unsigned (*symbol_adapt16)(struct MsacContext *s, uint16_t *cdf, size_t n_symbols); +#endif +} MsacContext; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/msac.h" +#elif ARCH_X86 +#include "src/x86/msac.h" +#endif +#endif + +void dav1d_msac_init(MsacContext *s, const uint8_t *data, size_t sz, + int disable_cdf_update_flag); +unsigned dav1d_msac_decode_symbol_adapt_c(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_bool_adapt_c(MsacContext *s, uint16_t *cdf); +unsigned dav1d_msac_decode_bool_equi_c(MsacContext *s); +unsigned dav1d_msac_decode_bool_c(MsacContext *s, unsigned f); +unsigned dav1d_msac_decode_hi_tok_c(MsacContext *s, uint16_t *cdf); +int dav1d_msac_decode_subexp(MsacContext *s, int ref, int n, unsigned k); + +/* Supported n_symbols ranges: adapt4: 1-4, adapt8: 1-7, adapt16: 3-15 */ +#ifndef dav1d_msac_decode_symbol_adapt4 +#define dav1d_msac_decode_symbol_adapt4 dav1d_msac_decode_symbol_adapt_c +#endif +#ifndef dav1d_msac_decode_symbol_adapt8 +#define dav1d_msac_decode_symbol_adapt8 dav1d_msac_decode_symbol_adapt_c +#endif +#ifndef dav1d_msac_decode_symbol_adapt16 +#define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt_c +#endif +#ifndef dav1d_msac_decode_bool_adapt +#define dav1d_msac_decode_bool_adapt dav1d_msac_decode_bool_adapt_c +#endif +#ifndef dav1d_msac_decode_bool_equi +#define dav1d_msac_decode_bool_equi dav1d_msac_decode_bool_equi_c +#endif +#ifndef dav1d_msac_decode_bool +#define dav1d_msac_decode_bool dav1d_msac_decode_bool_c +#endif +#ifndef dav1d_msac_decode_hi_tok +#define dav1d_msac_decode_hi_tok dav1d_msac_decode_hi_tok_c +#endif + +static inline unsigned dav1d_msac_decode_bools(MsacContext *const s, unsigned n) { + unsigned v = 0; + while (n--) + v = (v << 1) | dav1d_msac_decode_bool_equi(s); + return v; +} + +static inline int dav1d_msac_decode_uniform(MsacContext *const s, const unsigned n) { + assert(n > 0); + const int l = ulog2(n) + 1; + assert(l > 1); + const unsigned m = (1 << l) - n; + const unsigned v = dav1d_msac_decode_bools(s, l - 1); + return v < m ? v : (v << 1) - m + dav1d_msac_decode_bool_equi(s); +} + +#endif /* DAV1D_SRC_MSAC_H */ diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c new file mode 100644 index 0000000000..e08129aba5 --- /dev/null +++ b/third_party/dav1d/src/obu.c @@ -0,0 +1,1702 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "dav1d/data.h" + +#include "common/frame.h" +#include "common/intops.h" + +#include "src/decode.h" +#include "src/getbits.h" +#include "src/levels.h" +#include "src/log.h" +#include "src/obu.h" +#include "src/ref.h" +#include "src/thread_task.h" + +static int parse_seq_hdr(Dav1dContext *const c, GetBits *const gb, + Dav1dSequenceHeader *const hdr) +{ +#define DEBUG_SEQ_HDR 0 + +#if DEBUG_SEQ_HDR + const unsigned init_bit_pos = dav1d_get_bits_pos(gb); +#endif + + memset(hdr, 0, sizeof(*hdr)); + hdr->profile = dav1d_get_bits(gb, 3); + if (hdr->profile > 2) goto error; +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-profile: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + hdr->still_picture = dav1d_get_bit(gb); + hdr->reduced_still_picture_header = dav1d_get_bit(gb); + if (hdr->reduced_still_picture_header && !hdr->still_picture) goto error; +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-stillpicture_flags: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + if (hdr->reduced_still_picture_header) { + hdr->num_operating_points = 1; + hdr->operating_points[0].major_level = dav1d_get_bits(gb, 3); + hdr->operating_points[0].minor_level = dav1d_get_bits(gb, 2); + hdr->operating_points[0].initial_display_delay = 10; + } else { + hdr->timing_info_present = dav1d_get_bit(gb); + if (hdr->timing_info_present) { + hdr->num_units_in_tick = dav1d_get_bits(gb, 32); + hdr->time_scale = dav1d_get_bits(gb, 32); + hdr->equal_picture_interval = dav1d_get_bit(gb); + if (hdr->equal_picture_interval) { + const unsigned num_ticks_per_picture = dav1d_get_vlc(gb); + if (num_ticks_per_picture == 0xFFFFFFFFU) + goto error; + hdr->num_ticks_per_picture = num_ticks_per_picture + 1; + } + + hdr->decoder_model_info_present = dav1d_get_bit(gb); + if (hdr->decoder_model_info_present) { + hdr->encoder_decoder_buffer_delay_length = dav1d_get_bits(gb, 5) + 1; + hdr->num_units_in_decoding_tick = dav1d_get_bits(gb, 32); + hdr->buffer_removal_delay_length = dav1d_get_bits(gb, 5) + 1; + hdr->frame_presentation_delay_length = dav1d_get_bits(gb, 5) + 1; + } + } +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-timinginfo: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + hdr->display_model_info_present = dav1d_get_bit(gb); + hdr->num_operating_points = dav1d_get_bits(gb, 5) + 1; + for (int i = 0; i < hdr->num_operating_points; i++) { + struct Dav1dSequenceHeaderOperatingPoint *const op = + &hdr->operating_points[i]; + op->idc = dav1d_get_bits(gb, 12); + if (op->idc && (!(op->idc & 0xff) || !(op->idc & 0xf00))) + goto error; + op->major_level = 2 + dav1d_get_bits(gb, 3); + op->minor_level = dav1d_get_bits(gb, 2); + if (op->major_level > 3) + op->tier = dav1d_get_bit(gb); + if (hdr->decoder_model_info_present) { + op->decoder_model_param_present = dav1d_get_bit(gb); + if (op->decoder_model_param_present) { + struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = + &hdr->operating_parameter_info[i]; + opi->decoder_buffer_delay = + dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); + opi->encoder_buffer_delay = + dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); + opi->low_delay_mode = dav1d_get_bit(gb); + } + } + if (hdr->display_model_info_present) + op->display_model_param_present = dav1d_get_bit(gb); + op->initial_display_delay = + op->display_model_param_present ? dav1d_get_bits(gb, 4) + 1 : 10; + } +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-operating-points: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + } + + const int op_idx = + c->operating_point < hdr->num_operating_points ? c->operating_point : 0; + c->operating_point_idc = hdr->operating_points[op_idx].idc; + const unsigned spatial_mask = c->operating_point_idc >> 8; + c->max_spatial_id = spatial_mask ? ulog2(spatial_mask) : 0; + + hdr->width_n_bits = dav1d_get_bits(gb, 4) + 1; + hdr->height_n_bits = dav1d_get_bits(gb, 4) + 1; + hdr->max_width = dav1d_get_bits(gb, hdr->width_n_bits) + 1; + hdr->max_height = dav1d_get_bits(gb, hdr->height_n_bits) + 1; +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-size: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + if (!hdr->reduced_still_picture_header) { + hdr->frame_id_numbers_present = dav1d_get_bit(gb); + if (hdr->frame_id_numbers_present) { + hdr->delta_frame_id_n_bits = dav1d_get_bits(gb, 4) + 2; + hdr->frame_id_n_bits = dav1d_get_bits(gb, 3) + hdr->delta_frame_id_n_bits + 1; + } + } +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-frame-id-numbers-present: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + hdr->sb128 = dav1d_get_bit(gb); + hdr->filter_intra = dav1d_get_bit(gb); + hdr->intra_edge_filter = dav1d_get_bit(gb); + if (hdr->reduced_still_picture_header) { + hdr->screen_content_tools = DAV1D_ADAPTIVE; + hdr->force_integer_mv = DAV1D_ADAPTIVE; + } else { + hdr->inter_intra = dav1d_get_bit(gb); + hdr->masked_compound = dav1d_get_bit(gb); + hdr->warped_motion = dav1d_get_bit(gb); + hdr->dual_filter = dav1d_get_bit(gb); + hdr->order_hint = dav1d_get_bit(gb); + if (hdr->order_hint) { + hdr->jnt_comp = dav1d_get_bit(gb); + hdr->ref_frame_mvs = dav1d_get_bit(gb); + } + hdr->screen_content_tools = dav1d_get_bit(gb) ? DAV1D_ADAPTIVE : dav1d_get_bit(gb); + #if DEBUG_SEQ_HDR + printf("SEQHDR: post-screentools: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); + #endif + hdr->force_integer_mv = hdr->screen_content_tools ? + dav1d_get_bit(gb) ? DAV1D_ADAPTIVE : dav1d_get_bit(gb) : 2; + if (hdr->order_hint) + hdr->order_hint_n_bits = dav1d_get_bits(gb, 3) + 1; + } + hdr->super_res = dav1d_get_bit(gb); + hdr->cdef = dav1d_get_bit(gb); + hdr->restoration = dav1d_get_bit(gb); +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-featurebits: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + hdr->hbd = dav1d_get_bit(gb); + if (hdr->profile == 2 && hdr->hbd) + hdr->hbd += dav1d_get_bit(gb); + if (hdr->profile != 1) + hdr->monochrome = dav1d_get_bit(gb); + hdr->color_description_present = dav1d_get_bit(gb); + if (hdr->color_description_present) { + hdr->pri = dav1d_get_bits(gb, 8); + hdr->trc = dav1d_get_bits(gb, 8); + hdr->mtrx = dav1d_get_bits(gb, 8); + } else { + hdr->pri = DAV1D_COLOR_PRI_UNKNOWN; + hdr->trc = DAV1D_TRC_UNKNOWN; + hdr->mtrx = DAV1D_MC_UNKNOWN; + } + if (hdr->monochrome) { + hdr->color_range = dav1d_get_bit(gb); + hdr->layout = DAV1D_PIXEL_LAYOUT_I400; + hdr->ss_hor = hdr->ss_ver = 1; + hdr->chr = DAV1D_CHR_UNKNOWN; + } else if (hdr->pri == DAV1D_COLOR_PRI_BT709 && + hdr->trc == DAV1D_TRC_SRGB && + hdr->mtrx == DAV1D_MC_IDENTITY) + { + hdr->layout = DAV1D_PIXEL_LAYOUT_I444; + hdr->color_range = 1; + if (hdr->profile != 1 && !(hdr->profile == 2 && hdr->hbd == 2)) + goto error; + } else { + hdr->color_range = dav1d_get_bit(gb); + switch (hdr->profile) { + case 0: hdr->layout = DAV1D_PIXEL_LAYOUT_I420; + hdr->ss_hor = hdr->ss_ver = 1; + break; + case 1: hdr->layout = DAV1D_PIXEL_LAYOUT_I444; + break; + case 2: + if (hdr->hbd == 2) { + hdr->ss_hor = dav1d_get_bit(gb); + if (hdr->ss_hor) + hdr->ss_ver = dav1d_get_bit(gb); + } else + hdr->ss_hor = 1; + hdr->layout = hdr->ss_hor ? + hdr->ss_ver ? DAV1D_PIXEL_LAYOUT_I420 : + DAV1D_PIXEL_LAYOUT_I422 : + DAV1D_PIXEL_LAYOUT_I444; + break; + } + hdr->chr = (hdr->ss_hor & hdr->ss_ver) ? + dav1d_get_bits(gb, 2) : DAV1D_CHR_UNKNOWN; + } + if (c->strict_std_compliance && + hdr->mtrx == DAV1D_MC_IDENTITY && hdr->layout != DAV1D_PIXEL_LAYOUT_I444) + { + goto error; + } + if (!hdr->monochrome) + hdr->separate_uv_delta_q = dav1d_get_bit(gb); +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-colorinfo: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + hdr->film_grain_present = dav1d_get_bit(gb); +#if DEBUG_SEQ_HDR + printf("SEQHDR: post-filmgrain: off=%u\n", + dav1d_get_bits_pos(gb) - init_bit_pos); +#endif + + dav1d_get_bit(gb); // dummy bit + + // We needn't bother flushing the OBU here: we'll check we didn't + // overrun in the caller and will then discard gb, so there's no + // point in setting its position properly. + + return 0; + +error: + dav1d_log(c, "Error parsing sequence header\n"); + return DAV1D_ERR(EINVAL); +} + +static int read_frame_size(Dav1dContext *const c, GetBits *const gb, + const int use_ref) +{ + const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; + Dav1dFrameHeader *const hdr = c->frame_hdr; + + if (use_ref) { + for (int i = 0; i < 7; i++) { + if (dav1d_get_bit(gb)) { + const Dav1dThreadPicture *const ref = + &c->refs[c->frame_hdr->refidx[i]].p; + if (!ref->p.frame_hdr) return -1; + hdr->width[1] = ref->p.frame_hdr->width[1]; + hdr->height = ref->p.frame_hdr->height; + hdr->render_width = ref->p.frame_hdr->render_width; + hdr->render_height = ref->p.frame_hdr->render_height; + hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bit(gb); + if (hdr->super_res.enabled) { + const int d = hdr->super_res.width_scale_denominator = + 9 + dav1d_get_bits(gb, 3); + hdr->width[0] = imax((hdr->width[1] * 8 + (d >> 1)) / d, + imin(16, hdr->width[1])); + } else { + hdr->super_res.width_scale_denominator = 8; + hdr->width[0] = hdr->width[1]; + } + return 0; + } + } + } + + if (hdr->frame_size_override) { + hdr->width[1] = dav1d_get_bits(gb, seqhdr->width_n_bits) + 1; + hdr->height = dav1d_get_bits(gb, seqhdr->height_n_bits) + 1; + } else { + hdr->width[1] = seqhdr->max_width; + hdr->height = seqhdr->max_height; + } + hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bit(gb); + if (hdr->super_res.enabled) { + const int d = hdr->super_res.width_scale_denominator = 9 + dav1d_get_bits(gb, 3); + hdr->width[0] = imax((hdr->width[1] * 8 + (d >> 1)) / d, imin(16, hdr->width[1])); + } else { + hdr->super_res.width_scale_denominator = 8; + hdr->width[0] = hdr->width[1]; + } + hdr->have_render_size = dav1d_get_bit(gb); + if (hdr->have_render_size) { + hdr->render_width = dav1d_get_bits(gb, 16) + 1; + hdr->render_height = dav1d_get_bits(gb, 16) + 1; + } else { + hdr->render_width = hdr->width[1]; + hdr->render_height = hdr->height; + } + return 0; +} + +static inline int tile_log2(const int sz, const int tgt) { + int k; + for (k = 0; (sz << k) < tgt; k++) ; + return k; +} + +static const Dav1dLoopfilterModeRefDeltas default_mode_ref_deltas = { + .mode_delta = { 0, 0 }, + .ref_delta = { 1, 0, 0, 0, -1, 0, -1, -1 }, +}; + +static int parse_frame_hdr(Dav1dContext *const c, GetBits *const gb) { +#define DEBUG_FRAME_HDR 0 + +#if DEBUG_FRAME_HDR + const uint8_t *const init_ptr = gb->ptr; +#endif + const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; + Dav1dFrameHeader *const hdr = c->frame_hdr; + + hdr->show_existing_frame = + !seqhdr->reduced_still_picture_header && dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-show_existing_frame: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + if (hdr->show_existing_frame) { + hdr->existing_frame_idx = dav1d_get_bits(gb, 3); + if (seqhdr->decoder_model_info_present && !seqhdr->equal_picture_interval) + hdr->frame_presentation_delay = dav1d_get_bits(gb, seqhdr->frame_presentation_delay_length); + if (seqhdr->frame_id_numbers_present) { + hdr->frame_id = dav1d_get_bits(gb, seqhdr->frame_id_n_bits); + Dav1dFrameHeader *const ref_frame_hdr = c->refs[hdr->existing_frame_idx].p.p.frame_hdr; + if (!ref_frame_hdr || ref_frame_hdr->frame_id != hdr->frame_id) goto error; + } + return 0; + } + + hdr->frame_type = seqhdr->reduced_still_picture_header ? DAV1D_FRAME_TYPE_KEY : dav1d_get_bits(gb, 2); + hdr->show_frame = seqhdr->reduced_still_picture_header || dav1d_get_bit(gb); + if (hdr->show_frame) { + if (seqhdr->decoder_model_info_present && !seqhdr->equal_picture_interval) + hdr->frame_presentation_delay = dav1d_get_bits(gb, seqhdr->frame_presentation_delay_length); + hdr->showable_frame = hdr->frame_type != DAV1D_FRAME_TYPE_KEY; + } else + hdr->showable_frame = dav1d_get_bit(gb); + hdr->error_resilient_mode = + (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) || + hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH || + seqhdr->reduced_still_picture_header || dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-frametype_bits: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->disable_cdf_update = dav1d_get_bit(gb); + hdr->allow_screen_content_tools = seqhdr->screen_content_tools == DAV1D_ADAPTIVE ? + dav1d_get_bit(gb) : seqhdr->screen_content_tools; + if (hdr->allow_screen_content_tools) + hdr->force_integer_mv = seqhdr->force_integer_mv == DAV1D_ADAPTIVE ? + dav1d_get_bit(gb) : seqhdr->force_integer_mv; + else + hdr->force_integer_mv = 0; + + if (IS_KEY_OR_INTRA(hdr)) + hdr->force_integer_mv = 1; + + if (seqhdr->frame_id_numbers_present) + hdr->frame_id = dav1d_get_bits(gb, seqhdr->frame_id_n_bits); + + hdr->frame_size_override = seqhdr->reduced_still_picture_header ? 0 : + hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH ? 1 : dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-frame_size_override_flag: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->frame_offset = seqhdr->order_hint ? + dav1d_get_bits(gb, seqhdr->order_hint_n_bits) : 0; + hdr->primary_ref_frame = !hdr->error_resilient_mode && IS_INTER_OR_SWITCH(hdr) ? + dav1d_get_bits(gb, 3) : DAV1D_PRIMARY_REF_NONE; + + if (seqhdr->decoder_model_info_present) { + hdr->buffer_removal_time_present = dav1d_get_bit(gb); + if (hdr->buffer_removal_time_present) { + for (int i = 0; i < c->seq_hdr->num_operating_points; i++) { + const struct Dav1dSequenceHeaderOperatingPoint *const seqop = &seqhdr->operating_points[i]; + struct Dav1dFrameHeaderOperatingPoint *const op = &hdr->operating_points[i]; + if (seqop->decoder_model_param_present) { + int in_temporal_layer = (seqop->idc >> hdr->temporal_id) & 1; + int in_spatial_layer = (seqop->idc >> (hdr->spatial_id + 8)) & 1; + if (!seqop->idc || (in_temporal_layer && in_spatial_layer)) + op->buffer_removal_time = dav1d_get_bits(gb, seqhdr->buffer_removal_delay_length); + } + } + } + } + + if (IS_KEY_OR_INTRA(hdr)) { + hdr->refresh_frame_flags = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && + hdr->show_frame) ? 0xff : dav1d_get_bits(gb, 8); + if (hdr->refresh_frame_flags != 0xff && hdr->error_resilient_mode && seqhdr->order_hint) + for (int i = 0; i < 8; i++) + dav1d_get_bits(gb, seqhdr->order_hint_n_bits); + if (c->strict_std_compliance && + hdr->frame_type == DAV1D_FRAME_TYPE_INTRA && hdr->refresh_frame_flags == 0xff) + { + goto error; + } + if (read_frame_size(c, gb, 0) < 0) goto error; + hdr->allow_intrabc = hdr->allow_screen_content_tools && + !hdr->super_res.enabled && dav1d_get_bit(gb); + hdr->use_ref_frame_mvs = 0; + } else { + hdr->allow_intrabc = 0; + hdr->refresh_frame_flags = hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH ? 0xff : + dav1d_get_bits(gb, 8); + if (hdr->error_resilient_mode && seqhdr->order_hint) + for (int i = 0; i < 8; i++) + dav1d_get_bits(gb, seqhdr->order_hint_n_bits); + hdr->frame_ref_short_signaling = + seqhdr->order_hint && dav1d_get_bit(gb); + if (hdr->frame_ref_short_signaling) { // FIXME: Nearly verbatim copy from section 7.8 + hdr->refidx[0] = dav1d_get_bits(gb, 3); + hdr->refidx[1] = hdr->refidx[2] = -1; + hdr->refidx[3] = dav1d_get_bits(gb, 3); + hdr->refidx[4] = hdr->refidx[5] = hdr->refidx[6] = -1; + + int shifted_frame_offset[8]; + const int current_frame_offset = 1 << (seqhdr->order_hint_n_bits - 1); + for (int i = 0; i < 8; i++) { + if (!c->refs[i].p.p.frame_hdr) goto error; + shifted_frame_offset[i] = current_frame_offset + + get_poc_diff(seqhdr->order_hint_n_bits, + c->refs[i].p.p.frame_hdr->frame_offset, + hdr->frame_offset); + } + + int used_frame[8] = { 0 }; + used_frame[hdr->refidx[0]] = 1; + used_frame[hdr->refidx[3]] = 1; + + int latest_frame_offset = -1; + for (int i = 0; i < 8; i++) { + const int hint = shifted_frame_offset[i]; + if (!used_frame[i] && hint >= current_frame_offset && + hint >= latest_frame_offset) + { + hdr->refidx[6] = i; + latest_frame_offset = hint; + } + } + if (latest_frame_offset != -1) + used_frame[hdr->refidx[6]] = 1; + + int earliest_frame_offset = INT_MAX; + for (int i = 0; i < 8; i++) { + const int hint = shifted_frame_offset[i]; + if (!used_frame[i] && hint >= current_frame_offset && + hint < earliest_frame_offset) + { + hdr->refidx[4] = i; + earliest_frame_offset = hint; + } + } + if (earliest_frame_offset != INT_MAX) + used_frame[hdr->refidx[4]] = 1; + + earliest_frame_offset = INT_MAX; + for (int i = 0; i < 8; i++) { + const int hint = shifted_frame_offset[i]; + if (!used_frame[i] && hint >= current_frame_offset && + (hint < earliest_frame_offset)) + { + hdr->refidx[5] = i; + earliest_frame_offset = hint; + } + } + if (earliest_frame_offset != INT_MAX) + used_frame[hdr->refidx[5]] = 1; + + for (int i = 1; i < 7; i++) { + if (hdr->refidx[i] < 0) { + latest_frame_offset = -1; + for (int j = 0; j < 8; j++) { + const int hint = shifted_frame_offset[j]; + if (!used_frame[j] && hint < current_frame_offset && + hint >= latest_frame_offset) + { + hdr->refidx[i] = j; + latest_frame_offset = hint; + } + } + if (latest_frame_offset != -1) + used_frame[hdr->refidx[i]] = 1; + } + } + + earliest_frame_offset = INT_MAX; + int ref = -1; + for (int i = 0; i < 8; i++) { + const int hint = shifted_frame_offset[i]; + if (hint < earliest_frame_offset) { + ref = i; + earliest_frame_offset = hint; + } + } + for (int i = 0; i < 7; i++) { + if (hdr->refidx[i] < 0) + hdr->refidx[i] = ref; + } + } + for (int i = 0; i < 7; i++) { + if (!hdr->frame_ref_short_signaling) + hdr->refidx[i] = dav1d_get_bits(gb, 3); + if (seqhdr->frame_id_numbers_present) { + const int delta_ref_frame_id_minus_1 = dav1d_get_bits(gb, seqhdr->delta_frame_id_n_bits); + const int ref_frame_id = (hdr->frame_id + (1 << seqhdr->frame_id_n_bits) - delta_ref_frame_id_minus_1 - 1) & ((1 << seqhdr->frame_id_n_bits) - 1); + Dav1dFrameHeader *const ref_frame_hdr = c->refs[hdr->refidx[i]].p.p.frame_hdr; + if (!ref_frame_hdr || ref_frame_hdr->frame_id != ref_frame_id) goto error; + } + } + const int use_ref = !hdr->error_resilient_mode && + hdr->frame_size_override; + if (read_frame_size(c, gb, use_ref) < 0) goto error; + hdr->hp = !hdr->force_integer_mv && dav1d_get_bit(gb); + hdr->subpel_filter_mode = dav1d_get_bit(gb) ? DAV1D_FILTER_SWITCHABLE : + dav1d_get_bits(gb, 2); + hdr->switchable_motion_mode = dav1d_get_bit(gb); + hdr->use_ref_frame_mvs = !hdr->error_resilient_mode && + seqhdr->ref_frame_mvs && seqhdr->order_hint && + IS_INTER_OR_SWITCH(hdr) && dav1d_get_bit(gb); + } +#if DEBUG_FRAME_HDR + printf("HDR: post-frametype-specific-bits: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + hdr->refresh_context = !seqhdr->reduced_still_picture_header && + !hdr->disable_cdf_update && !dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-refresh_context: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // tile data + hdr->tiling.uniform = dav1d_get_bit(gb); + const int sbsz_min1 = (64 << seqhdr->sb128) - 1; + const int sbsz_log2 = 6 + seqhdr->sb128; + const int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; + const int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; + const int max_tile_width_sb = 4096 >> sbsz_log2; + const int max_tile_area_sb = 4096 * 2304 >> (2 * sbsz_log2); + hdr->tiling.min_log2_cols = tile_log2(max_tile_width_sb, sbw); + hdr->tiling.max_log2_cols = tile_log2(1, imin(sbw, DAV1D_MAX_TILE_COLS)); + hdr->tiling.max_log2_rows = tile_log2(1, imin(sbh, DAV1D_MAX_TILE_ROWS)); + const int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), + hdr->tiling.min_log2_cols); + if (hdr->tiling.uniform) { + for (hdr->tiling.log2_cols = hdr->tiling.min_log2_cols; + hdr->tiling.log2_cols < hdr->tiling.max_log2_cols && dav1d_get_bit(gb); + hdr->tiling.log2_cols++) ; + const int tile_w = 1 + ((sbw - 1) >> hdr->tiling.log2_cols); + hdr->tiling.cols = 0; + for (int sbx = 0; sbx < sbw; sbx += tile_w, hdr->tiling.cols++) + hdr->tiling.col_start_sb[hdr->tiling.cols] = sbx; + hdr->tiling.min_log2_rows = + imax(min_log2_tiles - hdr->tiling.log2_cols, 0); + + for (hdr->tiling.log2_rows = hdr->tiling.min_log2_rows; + hdr->tiling.log2_rows < hdr->tiling.max_log2_rows && dav1d_get_bit(gb); + hdr->tiling.log2_rows++) ; + const int tile_h = 1 + ((sbh - 1) >> hdr->tiling.log2_rows); + hdr->tiling.rows = 0; + for (int sby = 0; sby < sbh; sby += tile_h, hdr->tiling.rows++) + hdr->tiling.row_start_sb[hdr->tiling.rows] = sby; + } else { + hdr->tiling.cols = 0; + int widest_tile = 0, max_tile_area_sb = sbw * sbh; + for (int sbx = 0; sbx < sbw && hdr->tiling.cols < DAV1D_MAX_TILE_COLS; hdr->tiling.cols++) { + const int tile_width_sb = imin(sbw - sbx, max_tile_width_sb); + const int tile_w = (tile_width_sb > 1) ? + 1 + dav1d_get_uniform(gb, tile_width_sb) : + 1; + hdr->tiling.col_start_sb[hdr->tiling.cols] = sbx; + sbx += tile_w; + widest_tile = imax(widest_tile, tile_w); + } + hdr->tiling.log2_cols = tile_log2(1, hdr->tiling.cols); + if (min_log2_tiles) max_tile_area_sb >>= min_log2_tiles + 1; + const int max_tile_height_sb = imax(max_tile_area_sb / widest_tile, 1); + + hdr->tiling.rows = 0; + for (int sby = 0; sby < sbh && hdr->tiling.rows < DAV1D_MAX_TILE_ROWS; hdr->tiling.rows++) { + const int tile_height_sb = imin(sbh - sby, max_tile_height_sb); + const int tile_h = (tile_height_sb > 1) ? + 1 + dav1d_get_uniform(gb, tile_height_sb) : + 1; + hdr->tiling.row_start_sb[hdr->tiling.rows] = sby; + sby += tile_h; + } + hdr->tiling.log2_rows = tile_log2(1, hdr->tiling.rows); + } + hdr->tiling.col_start_sb[hdr->tiling.cols] = sbw; + hdr->tiling.row_start_sb[hdr->tiling.rows] = sbh; + if (hdr->tiling.log2_cols || hdr->tiling.log2_rows) { + hdr->tiling.update = dav1d_get_bits(gb, hdr->tiling.log2_cols + + hdr->tiling.log2_rows); + if (hdr->tiling.update >= hdr->tiling.cols * hdr->tiling.rows) + goto error; + hdr->tiling.n_bytes = dav1d_get_bits(gb, 2) + 1; + } else { + hdr->tiling.n_bytes = hdr->tiling.update = 0; + } +#if DEBUG_FRAME_HDR + printf("HDR: post-tiling: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // quant data + hdr->quant.yac = dav1d_get_bits(gb, 8); + hdr->quant.ydc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + if (!seqhdr->monochrome) { + // If the sequence header says that delta_q might be different + // for U, V, we must check whether it actually is for this + // frame. + const int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bit(gb) : 0; + hdr->quant.udc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + hdr->quant.uac_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + if (diff_uv_delta) { + hdr->quant.vdc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + hdr->quant.vac_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + } else { + hdr->quant.vdc_delta = hdr->quant.udc_delta; + hdr->quant.vac_delta = hdr->quant.uac_delta; + } + } +#if DEBUG_FRAME_HDR + printf("HDR: post-quant: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->quant.qm = dav1d_get_bit(gb); + if (hdr->quant.qm) { + hdr->quant.qm_y = dav1d_get_bits(gb, 4); + hdr->quant.qm_u = dav1d_get_bits(gb, 4); + hdr->quant.qm_v = + seqhdr->separate_uv_delta_q ? (int)dav1d_get_bits(gb, 4) : + hdr->quant.qm_u; + } +#if DEBUG_FRAME_HDR + printf("HDR: post-qm: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // segmentation data + hdr->segmentation.enabled = dav1d_get_bit(gb); + if (hdr->segmentation.enabled) { + if (hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { + hdr->segmentation.update_map = 1; + hdr->segmentation.temporal = 0; + hdr->segmentation.update_data = 1; + } else { + hdr->segmentation.update_map = dav1d_get_bit(gb); + hdr->segmentation.temporal = + hdr->segmentation.update_map ? dav1d_get_bit(gb) : 0; + hdr->segmentation.update_data = dav1d_get_bit(gb); + } + + if (hdr->segmentation.update_data) { + hdr->segmentation.seg_data.preskip = 0; + hdr->segmentation.seg_data.last_active_segid = -1; + for (int i = 0; i < DAV1D_MAX_SEGMENTS; i++) { + Dav1dSegmentationData *const seg = + &hdr->segmentation.seg_data.d[i]; + if (dav1d_get_bit(gb)) { + seg->delta_q = dav1d_get_sbits(gb, 9); + hdr->segmentation.seg_data.last_active_segid = i; + } else { + seg->delta_q = 0; + } + if (dav1d_get_bit(gb)) { + seg->delta_lf_y_v = dav1d_get_sbits(gb, 7); + hdr->segmentation.seg_data.last_active_segid = i; + } else { + seg->delta_lf_y_v = 0; + } + if (dav1d_get_bit(gb)) { + seg->delta_lf_y_h = dav1d_get_sbits(gb, 7); + hdr->segmentation.seg_data.last_active_segid = i; + } else { + seg->delta_lf_y_h = 0; + } + if (dav1d_get_bit(gb)) { + seg->delta_lf_u = dav1d_get_sbits(gb, 7); + hdr->segmentation.seg_data.last_active_segid = i; + } else { + seg->delta_lf_u = 0; + } + if (dav1d_get_bit(gb)) { + seg->delta_lf_v = dav1d_get_sbits(gb, 7); + hdr->segmentation.seg_data.last_active_segid = i; + } else { + seg->delta_lf_v = 0; + } + if (dav1d_get_bit(gb)) { + seg->ref = dav1d_get_bits(gb, 3); + hdr->segmentation.seg_data.last_active_segid = i; + hdr->segmentation.seg_data.preskip = 1; + } else { + seg->ref = -1; + } + if ((seg->skip = dav1d_get_bit(gb))) { + hdr->segmentation.seg_data.last_active_segid = i; + hdr->segmentation.seg_data.preskip = 1; + } + if ((seg->globalmv = dav1d_get_bit(gb))) { + hdr->segmentation.seg_data.last_active_segid = i; + hdr->segmentation.seg_data.preskip = 1; + } + } + } else { + // segmentation.update_data was false so we should copy + // segmentation data from the reference frame. + assert(hdr->primary_ref_frame != DAV1D_PRIMARY_REF_NONE); + const int pri_ref = hdr->refidx[hdr->primary_ref_frame]; + if (!c->refs[pri_ref].p.p.frame_hdr) goto error; + hdr->segmentation.seg_data = + c->refs[pri_ref].p.p.frame_hdr->segmentation.seg_data; + } + } else { + memset(&hdr->segmentation.seg_data, 0, sizeof(Dav1dSegmentationDataSet)); + for (int i = 0; i < DAV1D_MAX_SEGMENTS; i++) + hdr->segmentation.seg_data.d[i].ref = -1; + } +#if DEBUG_FRAME_HDR + printf("HDR: post-segmentation: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // delta q + hdr->delta.q.present = hdr->quant.yac ? dav1d_get_bit(gb) : 0; + hdr->delta.q.res_log2 = hdr->delta.q.present ? dav1d_get_bits(gb, 2) : 0; + hdr->delta.lf.present = hdr->delta.q.present && !hdr->allow_intrabc && + dav1d_get_bit(gb); + hdr->delta.lf.res_log2 = hdr->delta.lf.present ? dav1d_get_bits(gb, 2) : 0; + hdr->delta.lf.multi = hdr->delta.lf.present ? dav1d_get_bit(gb) : 0; +#if DEBUG_FRAME_HDR + printf("HDR: post-delta_q_lf_flags: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // derive lossless flags + const int delta_lossless = !hdr->quant.ydc_delta && !hdr->quant.udc_delta && + !hdr->quant.uac_delta && !hdr->quant.vdc_delta && !hdr->quant.vac_delta; + hdr->all_lossless = 1; + for (int i = 0; i < DAV1D_MAX_SEGMENTS; i++) { + hdr->segmentation.qidx[i] = hdr->segmentation.enabled ? + iclip_u8(hdr->quant.yac + hdr->segmentation.seg_data.d[i].delta_q) : + hdr->quant.yac; + hdr->segmentation.lossless[i] = + !hdr->segmentation.qidx[i] && delta_lossless; + hdr->all_lossless &= hdr->segmentation.lossless[i]; + } + + // loopfilter + if (hdr->all_lossless || hdr->allow_intrabc) { + hdr->loopfilter.level_y[0] = hdr->loopfilter.level_y[1] = 0; + hdr->loopfilter.level_u = hdr->loopfilter.level_v = 0; + hdr->loopfilter.sharpness = 0; + hdr->loopfilter.mode_ref_delta_enabled = 1; + hdr->loopfilter.mode_ref_delta_update = 1; + hdr->loopfilter.mode_ref_deltas = default_mode_ref_deltas; + } else { + hdr->loopfilter.level_y[0] = dav1d_get_bits(gb, 6); + hdr->loopfilter.level_y[1] = dav1d_get_bits(gb, 6); + if (!seqhdr->monochrome && + (hdr->loopfilter.level_y[0] || hdr->loopfilter.level_y[1])) + { + hdr->loopfilter.level_u = dav1d_get_bits(gb, 6); + hdr->loopfilter.level_v = dav1d_get_bits(gb, 6); + } + hdr->loopfilter.sharpness = dav1d_get_bits(gb, 3); + + if (hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { + hdr->loopfilter.mode_ref_deltas = default_mode_ref_deltas; + } else { + const int ref = hdr->refidx[hdr->primary_ref_frame]; + if (!c->refs[ref].p.p.frame_hdr) goto error; + hdr->loopfilter.mode_ref_deltas = + c->refs[ref].p.p.frame_hdr->loopfilter.mode_ref_deltas; + } + hdr->loopfilter.mode_ref_delta_enabled = dav1d_get_bit(gb); + if (hdr->loopfilter.mode_ref_delta_enabled) { + hdr->loopfilter.mode_ref_delta_update = dav1d_get_bit(gb); + if (hdr->loopfilter.mode_ref_delta_update) { + for (int i = 0; i < 8; i++) + if (dav1d_get_bit(gb)) + hdr->loopfilter.mode_ref_deltas.ref_delta[i] = + dav1d_get_sbits(gb, 7); + for (int i = 0; i < 2; i++) + if (dav1d_get_bit(gb)) + hdr->loopfilter.mode_ref_deltas.mode_delta[i] = + dav1d_get_sbits(gb, 7); + } + } + } +#if DEBUG_FRAME_HDR + printf("HDR: post-lpf: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // cdef + if (!hdr->all_lossless && seqhdr->cdef && !hdr->allow_intrabc) { + hdr->cdef.damping = dav1d_get_bits(gb, 2) + 3; + hdr->cdef.n_bits = dav1d_get_bits(gb, 2); + for (int i = 0; i < (1 << hdr->cdef.n_bits); i++) { + hdr->cdef.y_strength[i] = dav1d_get_bits(gb, 6); + if (!seqhdr->monochrome) + hdr->cdef.uv_strength[i] = dav1d_get_bits(gb, 6); + } + } else { + hdr->cdef.n_bits = 0; + hdr->cdef.y_strength[0] = 0; + hdr->cdef.uv_strength[0] = 0; + } +#if DEBUG_FRAME_HDR + printf("HDR: post-cdef: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + // restoration + if ((!hdr->all_lossless || hdr->super_res.enabled) && + seqhdr->restoration && !hdr->allow_intrabc) + { + hdr->restoration.type[0] = dav1d_get_bits(gb, 2); + if (!seqhdr->monochrome) { + hdr->restoration.type[1] = dav1d_get_bits(gb, 2); + hdr->restoration.type[2] = dav1d_get_bits(gb, 2); + } else { + hdr->restoration.type[1] = + hdr->restoration.type[2] = DAV1D_RESTORATION_NONE; + } + + if (hdr->restoration.type[0] || hdr->restoration.type[1] || + hdr->restoration.type[2]) + { + // Log2 of the restoration unit size. + hdr->restoration.unit_size[0] = 6 + seqhdr->sb128; + if (dav1d_get_bit(gb)) { + hdr->restoration.unit_size[0]++; + if (!seqhdr->sb128) + hdr->restoration.unit_size[0] += dav1d_get_bit(gb); + } + hdr->restoration.unit_size[1] = hdr->restoration.unit_size[0]; + if ((hdr->restoration.type[1] || hdr->restoration.type[2]) && + seqhdr->ss_hor == 1 && seqhdr->ss_ver == 1) + { + hdr->restoration.unit_size[1] -= dav1d_get_bit(gb); + } + } else { + hdr->restoration.unit_size[0] = 8; + } + } else { + hdr->restoration.type[0] = DAV1D_RESTORATION_NONE; + hdr->restoration.type[1] = DAV1D_RESTORATION_NONE; + hdr->restoration.type[2] = DAV1D_RESTORATION_NONE; + } +#if DEBUG_FRAME_HDR + printf("HDR: post-restoration: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + hdr->txfm_mode = hdr->all_lossless ? DAV1D_TX_4X4_ONLY : + dav1d_get_bit(gb) ? DAV1D_TX_SWITCHABLE : DAV1D_TX_LARGEST; +#if DEBUG_FRAME_HDR + printf("HDR: post-txfmmode: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->switchable_comp_refs = IS_INTER_OR_SWITCH(hdr) ? dav1d_get_bit(gb) : 0; +#if DEBUG_FRAME_HDR + printf("HDR: post-refmode: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->skip_mode_allowed = 0; + if (hdr->switchable_comp_refs && IS_INTER_OR_SWITCH(hdr) && seqhdr->order_hint) { + const unsigned poc = hdr->frame_offset; + unsigned off_before = 0xFFFFFFFFU; + int off_after = -1; + int off_before_idx, off_after_idx; + for (int i = 0; i < 7; i++) { + if (!c->refs[hdr->refidx[i]].p.p.frame_hdr) goto error; + const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; + + const int diff = get_poc_diff(seqhdr->order_hint_n_bits, refpoc, poc); + if (diff > 0) { + if (off_after == -1 || get_poc_diff(seqhdr->order_hint_n_bits, + off_after, refpoc) > 0) + { + off_after = refpoc; + off_after_idx = i; + } + } else if (diff < 0 && (off_before == 0xFFFFFFFFU || + get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before) > 0)) + { + off_before = refpoc; + off_before_idx = i; + } + } + + if (off_before != 0xFFFFFFFFU && off_after != -1) { + hdr->skip_mode_refs[0] = imin(off_before_idx, off_after_idx); + hdr->skip_mode_refs[1] = imax(off_before_idx, off_after_idx); + hdr->skip_mode_allowed = 1; + } else if (off_before != 0xFFFFFFFFU) { + unsigned off_before2 = 0xFFFFFFFFU; + int off_before2_idx; + for (int i = 0; i < 7; i++) { + if (!c->refs[hdr->refidx[i]].p.p.frame_hdr) goto error; + const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; + if (get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before) < 0) { + if (off_before2 == 0xFFFFFFFFU || + get_poc_diff(seqhdr->order_hint_n_bits, + refpoc, off_before2) > 0) + { + off_before2 = refpoc; + off_before2_idx = i; + } + } + } + + if (off_before2 != 0xFFFFFFFFU) { + hdr->skip_mode_refs[0] = imin(off_before_idx, off_before2_idx); + hdr->skip_mode_refs[1] = imax(off_before_idx, off_before2_idx); + hdr->skip_mode_allowed = 1; + } + } + } + hdr->skip_mode_enabled = hdr->skip_mode_allowed ? dav1d_get_bit(gb) : 0; +#if DEBUG_FRAME_HDR + printf("HDR: post-extskip: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->warp_motion = !hdr->error_resilient_mode && IS_INTER_OR_SWITCH(hdr) && + seqhdr->warped_motion && dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-warpmotionbit: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + hdr->reduced_txtp_set = dav1d_get_bit(gb); +#if DEBUG_FRAME_HDR + printf("HDR: post-reducedtxtpset: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + for (int i = 0; i < 7; i++) + hdr->gmv[i] = dav1d_default_wm_params; + + if (IS_INTER_OR_SWITCH(hdr)) { + for (int i = 0; i < 7; i++) { + hdr->gmv[i].type = !dav1d_get_bit(gb) ? DAV1D_WM_TYPE_IDENTITY : + dav1d_get_bit(gb) ? DAV1D_WM_TYPE_ROT_ZOOM : + dav1d_get_bit(gb) ? DAV1D_WM_TYPE_TRANSLATION : + DAV1D_WM_TYPE_AFFINE; + + if (hdr->gmv[i].type == DAV1D_WM_TYPE_IDENTITY) continue; + + const Dav1dWarpedMotionParams *ref_gmv; + if (hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { + ref_gmv = &dav1d_default_wm_params; + } else { + const int pri_ref = hdr->refidx[hdr->primary_ref_frame]; + if (!c->refs[pri_ref].p.p.frame_hdr) goto error; + ref_gmv = &c->refs[pri_ref].p.p.frame_hdr->gmv[i]; + } + int32_t *const mat = hdr->gmv[i].matrix; + const int32_t *const ref_mat = ref_gmv->matrix; + int bits, shift; + + if (hdr->gmv[i].type >= DAV1D_WM_TYPE_ROT_ZOOM) { + mat[2] = (1 << 16) + 2 * + dav1d_get_bits_subexp(gb, (ref_mat[2] - (1 << 16)) >> 1, 12); + mat[3] = 2 * dav1d_get_bits_subexp(gb, ref_mat[3] >> 1, 12); + + bits = 12; + shift = 10; + } else { + bits = 9 - !hdr->hp; + shift = 13 + !hdr->hp; + } + + if (hdr->gmv[i].type == DAV1D_WM_TYPE_AFFINE) { + mat[4] = 2 * dav1d_get_bits_subexp(gb, ref_mat[4] >> 1, 12); + mat[5] = (1 << 16) + 2 * + dav1d_get_bits_subexp(gb, (ref_mat[5] - (1 << 16)) >> 1, 12); + } else { + mat[4] = -mat[3]; + mat[5] = mat[2]; + } + + mat[0] = dav1d_get_bits_subexp(gb, ref_mat[0] >> shift, bits) * (1 << shift); + mat[1] = dav1d_get_bits_subexp(gb, ref_mat[1] >> shift, bits) * (1 << shift); + } + } +#if DEBUG_FRAME_HDR + printf("HDR: post-gmv: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + hdr->film_grain.present = seqhdr->film_grain_present && + (hdr->show_frame || hdr->showable_frame) && + dav1d_get_bit(gb); + if (hdr->film_grain.present) { + const unsigned seed = dav1d_get_bits(gb, 16); + hdr->film_grain.update = hdr->frame_type != DAV1D_FRAME_TYPE_INTER || dav1d_get_bit(gb); + if (!hdr->film_grain.update) { + const int refidx = dav1d_get_bits(gb, 3); + int i; + for (i = 0; i < 7; i++) + if (hdr->refidx[i] == refidx) + break; + if (i == 7 || !c->refs[refidx].p.p.frame_hdr) goto error; + hdr->film_grain.data = c->refs[refidx].p.p.frame_hdr->film_grain.data; + hdr->film_grain.data.seed = seed; + } else { + Dav1dFilmGrainData *const fgd = &hdr->film_grain.data; + fgd->seed = seed; + + fgd->num_y_points = dav1d_get_bits(gb, 4); + if (fgd->num_y_points > 14) goto error; + for (int i = 0; i < fgd->num_y_points; i++) { + fgd->y_points[i][0] = dav1d_get_bits(gb, 8); + if (i && fgd->y_points[i - 1][0] >= fgd->y_points[i][0]) + goto error; + fgd->y_points[i][1] = dav1d_get_bits(gb, 8); + } + + fgd->chroma_scaling_from_luma = + !seqhdr->monochrome && dav1d_get_bit(gb); + if (seqhdr->monochrome || fgd->chroma_scaling_from_luma || + (seqhdr->ss_ver == 1 && seqhdr->ss_hor == 1 && !fgd->num_y_points)) + { + fgd->num_uv_points[0] = fgd->num_uv_points[1] = 0; + } else for (int pl = 0; pl < 2; pl++) { + fgd->num_uv_points[pl] = dav1d_get_bits(gb, 4); + if (fgd->num_uv_points[pl] > 10) goto error; + for (int i = 0; i < fgd->num_uv_points[pl]; i++) { + fgd->uv_points[pl][i][0] = dav1d_get_bits(gb, 8); + if (i && fgd->uv_points[pl][i - 1][0] >= fgd->uv_points[pl][i][0]) + goto error; + fgd->uv_points[pl][i][1] = dav1d_get_bits(gb, 8); + } + } + + if (seqhdr->ss_hor == 1 && seqhdr->ss_ver == 1 && + !!fgd->num_uv_points[0] != !!fgd->num_uv_points[1]) + { + goto error; + } + + fgd->scaling_shift = dav1d_get_bits(gb, 2) + 8; + fgd->ar_coeff_lag = dav1d_get_bits(gb, 2); + const int num_y_pos = 2 * fgd->ar_coeff_lag * (fgd->ar_coeff_lag + 1); + if (fgd->num_y_points) + for (int i = 0; i < num_y_pos; i++) + fgd->ar_coeffs_y[i] = dav1d_get_bits(gb, 8) - 128; + for (int pl = 0; pl < 2; pl++) + if (fgd->num_uv_points[pl] || fgd->chroma_scaling_from_luma) { + const int num_uv_pos = num_y_pos + !!fgd->num_y_points; + for (int i = 0; i < num_uv_pos; i++) + fgd->ar_coeffs_uv[pl][i] = dav1d_get_bits(gb, 8) - 128; + if (!fgd->num_y_points) + fgd->ar_coeffs_uv[pl][num_uv_pos] = 0; + } + fgd->ar_coeff_shift = dav1d_get_bits(gb, 2) + 6; + fgd->grain_scale_shift = dav1d_get_bits(gb, 2); + for (int pl = 0; pl < 2; pl++) + if (fgd->num_uv_points[pl]) { + fgd->uv_mult[pl] = dav1d_get_bits(gb, 8) - 128; + fgd->uv_luma_mult[pl] = dav1d_get_bits(gb, 8) - 128; + fgd->uv_offset[pl] = dav1d_get_bits(gb, 9) - 256; + } + fgd->overlap_flag = dav1d_get_bit(gb); + fgd->clip_to_restricted_range = dav1d_get_bit(gb); + } + } else { + memset(&hdr->film_grain.data, 0, sizeof(hdr->film_grain.data)); + } +#if DEBUG_FRAME_HDR + printf("HDR: post-filmgrain: off=%td\n", + (gb->ptr - init_ptr) * 8 - gb->bits_left); +#endif + + return 0; + +error: + dav1d_log(c, "Error parsing frame header\n"); + return DAV1D_ERR(EINVAL); +} + +static void parse_tile_hdr(Dav1dContext *const c, GetBits *const gb) { + const int n_tiles = c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows; + const int have_tile_pos = n_tiles > 1 ? dav1d_get_bit(gb) : 0; + + if (have_tile_pos) { + const int n_bits = c->frame_hdr->tiling.log2_cols + + c->frame_hdr->tiling.log2_rows; + c->tile[c->n_tile_data].start = dav1d_get_bits(gb, n_bits); + c->tile[c->n_tile_data].end = dav1d_get_bits(gb, n_bits); + } else { + c->tile[c->n_tile_data].start = 0; + c->tile[c->n_tile_data].end = n_tiles - 1; + } +} + +// Check that we haven't read more than obu_len bytes from the buffer +// since init_bit_pos. +static int check_for_overrun(Dav1dContext *const c, GetBits *const gb, + const unsigned init_bit_pos, + const unsigned obu_len) +{ + // Make sure we haven't actually read past the end of the gb buffer + if (gb->error) { + dav1d_log(c, "Overrun in OBU bit buffer\n"); + return 1; + } + + const unsigned pos = dav1d_get_bits_pos(gb); + + // We assume that init_bit_pos was the bit position of the buffer + // at some point in the past, so cannot be smaller than pos. + assert (init_bit_pos <= pos); + + if (pos - init_bit_pos > 8 * obu_len) { + dav1d_log(c, "Overrun in OBU bit buffer into next OBU\n"); + return 1; + } + + return 0; +} + +int dav1d_parse_obus(Dav1dContext *const c, Dav1dData *const in, const int global) { + GetBits gb; + int res; + + dav1d_init_get_bits(&gb, in->data, in->sz); + + // obu header + dav1d_get_bit(&gb); // obu_forbidden_bit + const enum Dav1dObuType type = dav1d_get_bits(&gb, 4); + const int has_extension = dav1d_get_bit(&gb); + const int has_length_field = dav1d_get_bit(&gb); + dav1d_get_bit(&gb); // reserved + + int temporal_id = 0, spatial_id = 0; + if (has_extension) { + temporal_id = dav1d_get_bits(&gb, 3); + spatial_id = dav1d_get_bits(&gb, 2); + dav1d_get_bits(&gb, 3); // reserved + } + + // obu length field + const unsigned len = has_length_field ? + dav1d_get_uleb128(&gb) : (unsigned) in->sz - 1 - has_extension; + if (gb.error) goto error; + + const unsigned init_bit_pos = dav1d_get_bits_pos(&gb); + const unsigned init_byte_pos = init_bit_pos >> 3; + + // We must have read a whole number of bytes at this point (1 byte + // for the header and whole bytes at a time when reading the + // leb128 length field). + assert((init_bit_pos & 7) == 0); + + // We also know that we haven't tried to read more than in->sz + // bytes yet (otherwise the error flag would have been set by the + // code in getbits.c) + assert(in->sz >= init_byte_pos); + + // Make sure that there are enough bits left in the buffer for the + // rest of the OBU. + if (len > in->sz - init_byte_pos) goto error; + + // skip obu not belonging to the selected temporal/spatial layer + if (type != DAV1D_OBU_SEQ_HDR && type != DAV1D_OBU_TD && + has_extension && c->operating_point_idc != 0) + { + const int in_temporal_layer = (c->operating_point_idc >> temporal_id) & 1; + const int in_spatial_layer = (c->operating_point_idc >> (spatial_id + 8)) & 1; + if (!in_temporal_layer || !in_spatial_layer) + return len + init_byte_pos; + } + + switch (type) { + case DAV1D_OBU_SEQ_HDR: { + Dav1dRef *ref = dav1d_ref_create_using_pool(c->seq_hdr_pool, + sizeof(Dav1dSequenceHeader)); + if (!ref) return DAV1D_ERR(ENOMEM); + Dav1dSequenceHeader *seq_hdr = ref->data; + if ((res = parse_seq_hdr(c, &gb, seq_hdr)) < 0) { + dav1d_ref_dec(&ref); + goto error; + } + if (check_for_overrun(c, &gb, init_bit_pos, len)) { + dav1d_ref_dec(&ref); + goto error; + } + // If we have read a sequence header which is different from + // the old one, this is a new video sequence and can't use any + // previous state. Free that state. + + if (!c->seq_hdr) { + c->frame_hdr = NULL; + c->frame_flags |= PICTURE_FLAG_NEW_SEQUENCE; + // see 7.5, operating_parameter_info is allowed to change in + // sequence headers of a single sequence + } else if (memcmp(seq_hdr, c->seq_hdr, offsetof(Dav1dSequenceHeader, operating_parameter_info))) { + c->frame_hdr = NULL; + c->mastering_display = NULL; + c->content_light = NULL; + dav1d_ref_dec(&c->mastering_display_ref); + dav1d_ref_dec(&c->content_light_ref); + for (int i = 0; i < 8; i++) { + if (c->refs[i].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[i].p); + dav1d_ref_dec(&c->refs[i].segmap); + dav1d_ref_dec(&c->refs[i].refmvs); + dav1d_cdf_thread_unref(&c->cdf[i]); + } + c->frame_flags |= PICTURE_FLAG_NEW_SEQUENCE; + // If operating_parameter_info changed, signal it + } else if (memcmp(seq_hdr->operating_parameter_info, c->seq_hdr->operating_parameter_info, + sizeof(seq_hdr->operating_parameter_info))) + { + c->frame_flags |= PICTURE_FLAG_NEW_OP_PARAMS_INFO; + } + dav1d_ref_dec(&c->seq_hdr_ref); + c->seq_hdr_ref = ref; + c->seq_hdr = seq_hdr; + break; + } + case DAV1D_OBU_REDUNDANT_FRAME_HDR: + if (c->frame_hdr) break; + // fall-through + case DAV1D_OBU_FRAME: + case DAV1D_OBU_FRAME_HDR: + if (global) break; + if (!c->seq_hdr) goto error; + if (!c->frame_hdr_ref) { + c->frame_hdr_ref = dav1d_ref_create_using_pool(c->frame_hdr_pool, + sizeof(Dav1dFrameHeader)); + if (!c->frame_hdr_ref) return DAV1D_ERR(ENOMEM); + } +#ifndef NDEBUG + // ensure that the reference is writable + assert(dav1d_ref_is_writable(c->frame_hdr_ref)); +#endif + c->frame_hdr = c->frame_hdr_ref->data; + memset(c->frame_hdr, 0, sizeof(*c->frame_hdr)); + c->frame_hdr->temporal_id = temporal_id; + c->frame_hdr->spatial_id = spatial_id; + if ((res = parse_frame_hdr(c, &gb)) < 0) { + c->frame_hdr = NULL; + goto error; + } + for (int n = 0; n < c->n_tile_data; n++) + dav1d_data_unref_internal(&c->tile[n].data); + c->n_tile_data = 0; + c->n_tiles = 0; + if (type != DAV1D_OBU_FRAME) { + // This is actually a frame header OBU so read the + // trailing bit and check for overrun. + dav1d_get_bit(&gb); + if (check_for_overrun(c, &gb, init_bit_pos, len)) { + c->frame_hdr = NULL; + goto error; + } + } + + if (c->frame_size_limit && (int64_t)c->frame_hdr->width[1] * + c->frame_hdr->height > c->frame_size_limit) + { + dav1d_log(c, "Frame size %dx%d exceeds limit %u\n", c->frame_hdr->width[1], + c->frame_hdr->height, c->frame_size_limit); + c->frame_hdr = NULL; + return DAV1D_ERR(ERANGE); + } + + if (type != DAV1D_OBU_FRAME) + break; + // OBU_FRAMEs shouldn't be signaled with show_existing_frame + if (c->frame_hdr->show_existing_frame) { + c->frame_hdr = NULL; + goto error; + } + + // This is the frame header at the start of a frame OBU. + // There's no trailing bit at the end to skip, but we do need + // to align to the next byte. + dav1d_bytealign_get_bits(&gb); + // fall-through + case DAV1D_OBU_TILE_GRP: { + if (global) break; + if (!c->frame_hdr) goto error; + if (c->n_tile_data_alloc < c->n_tile_data + 1) { + if ((c->n_tile_data + 1) > INT_MAX / (int)sizeof(*c->tile)) goto error; + struct Dav1dTileGroup *tile = realloc(c->tile, (c->n_tile_data + 1) * sizeof(*c->tile)); + if (!tile) goto error; + c->tile = tile; + memset(c->tile + c->n_tile_data, 0, sizeof(*c->tile)); + c->n_tile_data_alloc = c->n_tile_data + 1; + } + parse_tile_hdr(c, &gb); + // Align to the next byte boundary and check for overrun. + dav1d_bytealign_get_bits(&gb); + if (check_for_overrun(c, &gb, init_bit_pos, len)) + goto error; + // The current bit position is a multiple of 8 (because we + // just aligned it) and less than 8*pkt_bytelen because + // otherwise the overrun check would have fired. + const unsigned pkt_bytelen = init_byte_pos + len; + const unsigned bit_pos = dav1d_get_bits_pos(&gb); + assert((bit_pos & 7) == 0); + assert(pkt_bytelen >= (bit_pos >> 3)); + dav1d_data_ref(&c->tile[c->n_tile_data].data, in); + c->tile[c->n_tile_data].data.data += bit_pos >> 3; + c->tile[c->n_tile_data].data.sz = pkt_bytelen - (bit_pos >> 3); + // ensure tile groups are in order and sane, see 6.10.1 + if (c->tile[c->n_tile_data].start > c->tile[c->n_tile_data].end || + c->tile[c->n_tile_data].start != c->n_tiles) + { + for (int i = 0; i <= c->n_tile_data; i++) + dav1d_data_unref_internal(&c->tile[i].data); + c->n_tile_data = 0; + c->n_tiles = 0; + goto error; + } + c->n_tiles += 1 + c->tile[c->n_tile_data].end - + c->tile[c->n_tile_data].start; + c->n_tile_data++; + break; + } + case DAV1D_OBU_METADATA: { +#define DEBUG_OBU_METADATA 0 +#if DEBUG_OBU_METADATA + const uint8_t *const init_ptr = gb.ptr; +#endif + // obu metadta type field + const enum ObuMetaType meta_type = dav1d_get_uleb128(&gb); + const int meta_type_len = (dav1d_get_bits_pos(&gb) - init_bit_pos) >> 3; + if (gb.error) goto error; + + switch (meta_type) { + case OBU_META_HDR_CLL: { + Dav1dRef *ref = dav1d_ref_create(sizeof(Dav1dContentLightLevel)); + if (!ref) return DAV1D_ERR(ENOMEM); + Dav1dContentLightLevel *const content_light = ref->data; + + content_light->max_content_light_level = dav1d_get_bits(&gb, 16); +#if DEBUG_OBU_METADATA + printf("CLLOBU: max-content-light-level: %d [off=%td]\n", + content_light->max_content_light_level, + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + content_light->max_frame_average_light_level = dav1d_get_bits(&gb, 16); +#if DEBUG_OBU_METADATA + printf("CLLOBU: max-frame-average-light-level: %d [off=%td]\n", + content_light->max_frame_average_light_level, + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + + // Skip the trailing bit, align to the next byte boundary and check for overrun. + dav1d_get_bit(&gb); + dav1d_bytealign_get_bits(&gb); + if (check_for_overrun(c, &gb, init_bit_pos, len)) { + dav1d_ref_dec(&ref); + goto error; + } + + dav1d_ref_dec(&c->content_light_ref); + c->content_light = content_light; + c->content_light_ref = ref; + break; + } + case OBU_META_HDR_MDCV: { + Dav1dRef *ref = dav1d_ref_create(sizeof(Dav1dMasteringDisplay)); + if (!ref) return DAV1D_ERR(ENOMEM); + Dav1dMasteringDisplay *const mastering_display = ref->data; + + for (int i = 0; i < 3; i++) { + mastering_display->primaries[i][0] = dav1d_get_bits(&gb, 16); + mastering_display->primaries[i][1] = dav1d_get_bits(&gb, 16); +#if DEBUG_OBU_METADATA + printf("MDCVOBU: primaries[%d]: (%d, %d) [off=%td]\n", i, + mastering_display->primaries[i][0], + mastering_display->primaries[i][1], + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + } + mastering_display->white_point[0] = dav1d_get_bits(&gb, 16); +#if DEBUG_OBU_METADATA + printf("MDCVOBU: white-point-x: %d [off=%td]\n", + mastering_display->white_point[0], + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + mastering_display->white_point[1] = dav1d_get_bits(&gb, 16); +#if DEBUG_OBU_METADATA + printf("MDCVOBU: white-point-y: %d [off=%td]\n", + mastering_display->white_point[1], + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + mastering_display->max_luminance = dav1d_get_bits(&gb, 32); +#if DEBUG_OBU_METADATA + printf("MDCVOBU: max-luminance: %d [off=%td]\n", + mastering_display->max_luminance, + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + mastering_display->min_luminance = dav1d_get_bits(&gb, 32); +#if DEBUG_OBU_METADATA + printf("MDCVOBU: min-luminance: %d [off=%td]\n", + mastering_display->min_luminance, + (gb.ptr - init_ptr) * 8 - gb.bits_left); +#endif + // Skip the trailing bit, align to the next byte boundary and check for overrun. + dav1d_get_bit(&gb); + dav1d_bytealign_get_bits(&gb); + if (check_for_overrun(c, &gb, init_bit_pos, len)) { + dav1d_ref_dec(&ref); + goto error; + } + + dav1d_ref_dec(&c->mastering_display_ref); + c->mastering_display = mastering_display; + c->mastering_display_ref = ref; + break; + } + case OBU_META_ITUT_T35: { + int payload_size = len; + // Don't take into account all the trailing bits for payload_size + while (payload_size > 0 && !in->data[init_byte_pos + payload_size - 1]) + payload_size--; // trailing_zero_bit x 8 + payload_size--; // trailing_one_bit + trailing_zero_bit x 7 + + // Don't take into account meta_type bytes + payload_size -= meta_type_len; + + int country_code_extension_byte = 0; + const int country_code = dav1d_get_bits(&gb, 8); + payload_size--; + if (country_code == 0xFF) { + country_code_extension_byte = dav1d_get_bits(&gb, 8); + payload_size--; + } + + if (payload_size <= 0) { + dav1d_log(c, "Malformed ITU-T T.35 metadata message format\n"); + break; + } + + Dav1dRef *ref = dav1d_ref_create(sizeof(Dav1dITUTT35) + payload_size * sizeof(uint8_t)); + if (!ref) return DAV1D_ERR(ENOMEM); + Dav1dITUTT35 *const itut_t35_metadata = ref->data; + + // We need our public headers to be C++ compatible, so payload can't be + // a flexible array member + itut_t35_metadata->payload = (uint8_t *) &itut_t35_metadata[1]; + itut_t35_metadata->country_code = country_code; + itut_t35_metadata->country_code_extension_byte = country_code_extension_byte; + for (int i = 0; i < payload_size; i++) + itut_t35_metadata->payload[i] = dav1d_get_bits(&gb, 8); + itut_t35_metadata->payload_size = payload_size; + + dav1d_ref_dec(&c->itut_t35_ref); + c->itut_t35 = itut_t35_metadata; + c->itut_t35_ref = ref; + break; + } + case OBU_META_SCALABILITY: + case OBU_META_TIMECODE: + // ignore metadata OBUs we don't care about + break; + default: + // print a warning but don't fail for unknown types + dav1d_log(c, "Unknown Metadata OBU type %d\n", meta_type); + break; + } + + break; + } + case DAV1D_OBU_TD: + c->frame_flags |= PICTURE_FLAG_NEW_TEMPORAL_UNIT; + break; + case DAV1D_OBU_PADDING: + // ignore OBUs we don't care about + break; + default: + // print a warning but don't fail for unknown types + dav1d_log(c, "Unknown OBU type %d of size %u\n", type, len); + break; + } + + if (c->seq_hdr && c->frame_hdr) { + if (c->frame_hdr->show_existing_frame) { + if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr) goto error; + switch (c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr->frame_type) { + case DAV1D_FRAME_TYPE_INTER: + case DAV1D_FRAME_TYPE_SWITCH: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_REFERENCE) + goto skip; + break; + case DAV1D_FRAME_TYPE_INTRA: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_INTRA) + goto skip; + // fall-through + default: + break; + } + if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.data[0]) goto error; + if (c->strict_std_compliance && + !c->refs[c->frame_hdr->existing_frame_idx].p.showable) + { + goto error; + } + if (c->n_fc == 1) { + dav1d_thread_picture_ref(&c->out, + &c->refs[c->frame_hdr->existing_frame_idx].p); + dav1d_picture_copy_props(&c->out.p, + c->content_light, c->content_light_ref, + c->mastering_display, c->mastering_display_ref, + c->itut_t35, c->itut_t35_ref, + &in->m); + // Must be removed from the context after being attached to the frame + dav1d_ref_dec(&c->itut_t35_ref); + c->itut_t35 = NULL; + c->event_flags |= dav1d_picture_get_event_flags(&c->refs[c->frame_hdr->existing_frame_idx].p); + } else { + pthread_mutex_lock(&c->task_thread.lock); + // need to append this to the frame output queue + const unsigned next = c->frame_thread.next++; + if (c->frame_thread.next == c->n_fc) + c->frame_thread.next = 0; + + Dav1dFrameContext *const f = &c->fc[next]; + while (f->n_tile_data > 0) + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); + Dav1dThreadPicture *const out_delayed = + &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + unsigned first = atomic_load(&c->task_thread.first); + if (first + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + atomic_compare_exchange_strong(&c->task_thread.reset_task_cur, + &first, UINT_MAX); + if (c->task_thread.cur && c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + } + const int error = f->task_thread.retval; + if (error) { + c->cached_error = error; + f->task_thread.retval = 0; + dav1d_data_props_copy(&c->cached_error_props, &out_delayed->p.m); + dav1d_thread_picture_unref(out_delayed); + } else if (out_delayed->p.data[0]) { + const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], + memory_order_relaxed); + if ((out_delayed->visible || c->output_invisible_frames) && + progress != FRAME_ERROR) + { + dav1d_thread_picture_ref(&c->out, out_delayed); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } + dav1d_thread_picture_unref(out_delayed); + } + dav1d_thread_picture_ref(out_delayed, + &c->refs[c->frame_hdr->existing_frame_idx].p); + out_delayed->visible = 1; + dav1d_picture_copy_props(&out_delayed->p, + c->content_light, c->content_light_ref, + c->mastering_display, c->mastering_display_ref, + c->itut_t35, c->itut_t35_ref, + &in->m); + // Must be removed from the context after being attached to the frame + dav1d_ref_dec(&c->itut_t35_ref); + c->itut_t35 = NULL; + + pthread_mutex_unlock(&c->task_thread.lock); + } + if (c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY) { + const int r = c->frame_hdr->existing_frame_idx; + c->refs[r].p.showable = 0; + for (int i = 0; i < 8; i++) { + if (i == r) continue; + + if (c->refs[i].p.p.frame_hdr) + dav1d_thread_picture_unref(&c->refs[i].p); + dav1d_thread_picture_ref(&c->refs[i].p, &c->refs[r].p); + + dav1d_cdf_thread_unref(&c->cdf[i]); + dav1d_cdf_thread_ref(&c->cdf[i], &c->cdf[r]); + + dav1d_ref_dec(&c->refs[i].segmap); + c->refs[i].segmap = c->refs[r].segmap; + if (c->refs[r].segmap) + dav1d_ref_inc(c->refs[r].segmap); + dav1d_ref_dec(&c->refs[i].refmvs); + } + } + c->frame_hdr = NULL; + } else if (c->n_tiles == c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows) { + switch (c->frame_hdr->frame_type) { + case DAV1D_FRAME_TYPE_INTER: + case DAV1D_FRAME_TYPE_SWITCH: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_REFERENCE || + (c->decode_frame_type == DAV1D_DECODEFRAMETYPE_REFERENCE && + !c->frame_hdr->refresh_frame_flags)) + goto skip; + break; + case DAV1D_FRAME_TYPE_INTRA: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_INTRA || + (c->decode_frame_type == DAV1D_DECODEFRAMETYPE_REFERENCE && + !c->frame_hdr->refresh_frame_flags)) + goto skip; + // fall-through + default: + break; + } + if (!c->n_tile_data) + goto error; + if ((res = dav1d_submit_frame(c)) < 0) + return res; + assert(!c->n_tile_data); + c->frame_hdr = NULL; + c->n_tiles = 0; + } + } + + return len + init_byte_pos; + +skip: + // update refs with only the headers in case we skip the frame + for (int i = 0; i < 8; i++) { + if (c->frame_hdr->refresh_frame_flags & (1 << i)) { + dav1d_thread_picture_unref(&c->refs[i].p); + c->refs[i].p.p.frame_hdr = c->frame_hdr; + c->refs[i].p.p.seq_hdr = c->seq_hdr; + c->refs[i].p.p.frame_hdr_ref = c->frame_hdr_ref; + c->refs[i].p.p.seq_hdr_ref = c->seq_hdr_ref; + dav1d_ref_inc(c->frame_hdr_ref); + dav1d_ref_inc(c->seq_hdr_ref); + } + } + + dav1d_ref_dec(&c->frame_hdr_ref); + c->frame_hdr = NULL; + c->n_tiles = 0; + + return len + init_byte_pos; + +error: + dav1d_data_props_copy(&c->cached_error_props, &in->m); + dav1d_log(c, "Error parsing OBU data\n"); + return DAV1D_ERR(EINVAL); +} diff --git a/third_party/dav1d/src/obu.h b/third_party/dav1d/src/obu.h new file mode 100644 index 0000000000..aa79b5277a --- /dev/null +++ b/third_party/dav1d/src/obu.h @@ -0,0 +1,36 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_OBU_H +#define DAV1D_SRC_OBU_H + +#include "dav1d/data.h" +#include "src/internal.h" + +int dav1d_parse_obus(Dav1dContext *c, Dav1dData *in, int global); + +#endif /* DAV1D_SRC_OBU_H */ diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c new file mode 100644 index 0000000000..3e55d3f273 --- /dev/null +++ b/third_party/dav1d/src/picture.c @@ -0,0 +1,367 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "common/intops.h" +#include "common/validate.h" + +#include "src/internal.h" +#include "src/log.h" +#include "src/picture.h" +#include "src/ref.h" +#include "src/thread.h" +#include "src/thread_task.h" + +int dav1d_default_picture_alloc(Dav1dPicture *const p, void *const cookie) { + assert(sizeof(Dav1dMemPoolBuffer) <= DAV1D_PICTURE_ALIGNMENT); + const int hbd = p->p.bpc > 8; + const int aligned_w = (p->p.w + 127) & ~127; + const int aligned_h = (p->p.h + 127) & ~127; + const int has_chroma = p->p.layout != DAV1D_PIXEL_LAYOUT_I400; + const int ss_ver = p->p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = p->p.layout != DAV1D_PIXEL_LAYOUT_I444; + ptrdiff_t y_stride = aligned_w << hbd; + ptrdiff_t uv_stride = has_chroma ? y_stride >> ss_hor : 0; + /* Due to how mapping of addresses to sets works in most L1 and L2 cache + * implementations, strides of multiples of certain power-of-two numbers + * may cause multiple rows of the same superblock to map to the same set, + * causing evictions of previous rows resulting in a reduction in cache + * hit rate. Avoid that by slightly padding the stride when necessary. */ + if (!(y_stride & 1023)) + y_stride += DAV1D_PICTURE_ALIGNMENT; + if (!(uv_stride & 1023) && has_chroma) + uv_stride += DAV1D_PICTURE_ALIGNMENT; + p->stride[0] = y_stride; + p->stride[1] = uv_stride; + const size_t y_sz = y_stride * aligned_h; + const size_t uv_sz = uv_stride * (aligned_h >> ss_ver); + const size_t pic_size = y_sz + 2 * uv_sz; + + Dav1dMemPoolBuffer *const buf = dav1d_mem_pool_pop(cookie, pic_size + + DAV1D_PICTURE_ALIGNMENT - + sizeof(Dav1dMemPoolBuffer)); + if (!buf) return DAV1D_ERR(ENOMEM); + p->allocator_data = buf; + + uint8_t *const data = buf->data; + p->data[0] = data; + p->data[1] = has_chroma ? data + y_sz : NULL; + p->data[2] = has_chroma ? data + y_sz + uv_sz : NULL; + + return 0; +} + +void dav1d_default_picture_release(Dav1dPicture *const p, void *const cookie) { + dav1d_mem_pool_push(cookie, p->allocator_data); +} + +struct pic_ctx_context { + struct Dav1dRef *plane_ref[3]; /* MUST BE FIRST */ + enum Dav1dPixelLayout layout; + void *extra_ptr; /* MUST BE AT THE END */ +}; + +struct plane_ctx_context { + Dav1dPicAllocator allocator; + Dav1dPicture pic; +}; + +static void free_buffer(const uint8_t *const data, void *const user_data) { + struct pic_ctx_context *pic_ctx = user_data; + const int planes = pic_ctx->layout != DAV1D_PIXEL_LAYOUT_I400 ? 3 : 1; + + for (int i = 0; i < planes; i++) + dav1d_ref_dec(&pic_ctx->plane_ref[i]); + free(pic_ctx); +} + +static void free_plane_buffer(const uint8_t *const data, void *const user_data) { + struct plane_ctx_context *plane_ctx = user_data; + + plane_ctx->allocator.release_picture_callback(&plane_ctx->pic, + plane_ctx->allocator.cookie); + free(plane_ctx); +} + +static int picture_alloc_with_edges(Dav1dContext *const c, + Dav1dPicture *const p, + const int w, const int h, + Dav1dSequenceHeader *const seq_hdr, Dav1dRef *const seq_hdr_ref, + Dav1dFrameHeader *const frame_hdr, Dav1dRef *const frame_hdr_ref, + Dav1dContentLightLevel *const content_light, Dav1dRef *const content_light_ref, + Dav1dMasteringDisplay *const mastering_display, Dav1dRef *const mastering_display_ref, + Dav1dITUTT35 *const itut_t35, Dav1dRef *const itut_t35_ref, + const int bpc, + const Dav1dDataProps *const props, + Dav1dPicAllocator *const p_allocator, + const size_t extra, void **const extra_ptr) +{ + if (p->data[0]) { + dav1d_log(c, "Picture already allocated!\n"); + return -1; + } + assert(bpc > 0 && bpc <= 16); + + struct pic_ctx_context *pic_ctx = malloc(extra + sizeof(struct pic_ctx_context)); + if (pic_ctx == NULL) + return DAV1D_ERR(ENOMEM); + memset(pic_ctx, 0, sizeof(struct pic_ctx_context)); + + p->p.w = w; + p->p.h = h; + p->seq_hdr = seq_hdr; + p->frame_hdr = frame_hdr; + p->p.layout = seq_hdr->layout; + p->p.bpc = bpc; + dav1d_data_props_set_defaults(&p->m); + const int res = p_allocator->alloc_picture_callback(p, p_allocator->cookie); + if (res < 0) { + free(pic_ctx); + return res; + } + + pic_ctx->layout = p->p.layout; + + if (!(p->ref = dav1d_ref_wrap(p->data[0], free_buffer, pic_ctx))) { + p_allocator->release_picture_callback(p, p_allocator->cookie); + free(pic_ctx); + dav1d_log(c, "Failed to wrap picture: %s\n", strerror(errno)); + return DAV1D_ERR(ENOMEM); + } + + struct plane_ctx_context *plane_ctx = malloc(sizeof(struct plane_ctx_context)); + if (plane_ctx == NULL){ + dav1d_ref_dec(&p->ref); + p_allocator->release_picture_callback(p, p_allocator->cookie); + return DAV1D_ERR(ENOMEM); + } + + plane_ctx->allocator = *p_allocator; + plane_ctx->pic = *p; + + pic_ctx->plane_ref[0] = dav1d_ref_wrap(p->data[0], free_plane_buffer, plane_ctx); + if (!pic_ctx->plane_ref[0]) { + dav1d_ref_dec(&p->ref); + p_allocator->release_picture_callback(p, p_allocator->cookie); + free(plane_ctx); + dav1d_log(c, "Failed to wrap picture plane: %s\n", strerror(errno)); + return DAV1D_ERR(ENOMEM); + } + + const int planes = p->p.layout != DAV1D_PIXEL_LAYOUT_I400 ? 3 : 1; + for (int i = 1; i < planes; i++) { + pic_ctx->plane_ref[i] = pic_ctx->plane_ref[0]; + dav1d_ref_inc(pic_ctx->plane_ref[i]); + } + + p->seq_hdr_ref = seq_hdr_ref; + if (seq_hdr_ref) dav1d_ref_inc(seq_hdr_ref); + + p->frame_hdr_ref = frame_hdr_ref; + if (frame_hdr_ref) dav1d_ref_inc(frame_hdr_ref); + + dav1d_picture_copy_props(p, content_light, content_light_ref, + mastering_display, mastering_display_ref, + itut_t35, itut_t35_ref, props); + + if (extra && extra_ptr) + *extra_ptr = &pic_ctx->extra_ptr; + + return 0; +} + +void dav1d_picture_copy_props(Dav1dPicture *const p, + Dav1dContentLightLevel *const content_light, Dav1dRef *const content_light_ref, + Dav1dMasteringDisplay *const mastering_display, Dav1dRef *const mastering_display_ref, + Dav1dITUTT35 *const itut_t35, Dav1dRef *const itut_t35_ref, + const Dav1dDataProps *const props) +{ + dav1d_data_props_copy(&p->m, props); + + dav1d_ref_dec(&p->content_light_ref); + p->content_light_ref = content_light_ref; + p->content_light = content_light; + if (content_light_ref) dav1d_ref_inc(content_light_ref); + + dav1d_ref_dec(&p->mastering_display_ref); + p->mastering_display_ref = mastering_display_ref; + p->mastering_display = mastering_display; + if (mastering_display_ref) dav1d_ref_inc(mastering_display_ref); + + dav1d_ref_dec(&p->itut_t35_ref); + p->itut_t35_ref = itut_t35_ref; + p->itut_t35 = itut_t35; + if (itut_t35_ref) dav1d_ref_inc(itut_t35_ref); +} + +int dav1d_thread_picture_alloc(Dav1dContext *const c, Dav1dFrameContext *const f, + const int bpc) +{ + Dav1dThreadPicture *const p = &f->sr_cur; + const int have_frame_mt = c->n_fc > 1; + + const int res = + picture_alloc_with_edges(c, &p->p, f->frame_hdr->width[1], f->frame_hdr->height, + f->seq_hdr, f->seq_hdr_ref, + f->frame_hdr, f->frame_hdr_ref, + c->content_light, c->content_light_ref, + c->mastering_display, c->mastering_display_ref, + c->itut_t35, c->itut_t35_ref, + bpc, &f->tile[0].data.m, &c->allocator, + have_frame_mt ? sizeof(atomic_int) * 2 : 0, + (void **) &p->progress); + if (res) return res; + + // Must be removed from the context after being attached to the frame + dav1d_ref_dec(&c->itut_t35_ref); + c->itut_t35 = NULL; + + // Don't clear these flags from c->frame_flags if the frame is not visible. + // This way they will be added to the next visible frame too. + const int flags_mask = (f->frame_hdr->show_frame || c->output_invisible_frames) + ? 0 : (PICTURE_FLAG_NEW_SEQUENCE | PICTURE_FLAG_NEW_OP_PARAMS_INFO); + p->flags = c->frame_flags; + c->frame_flags &= flags_mask; + + p->visible = f->frame_hdr->show_frame; + p->showable = f->frame_hdr->showable_frame; + if (have_frame_mt) { + atomic_init(&p->progress[0], 0); + atomic_init(&p->progress[1], 0); + } + return res; +} + +int dav1d_picture_alloc_copy(Dav1dContext *const c, Dav1dPicture *const dst, const int w, + const Dav1dPicture *const src) +{ + struct pic_ctx_context *const pic_ctx = src->ref->user_data; + struct plane_ctx_context *const plane_ctx = pic_ctx->plane_ref[0]->user_data; + const int res = picture_alloc_with_edges(c, dst, w, src->p.h, + src->seq_hdr, src->seq_hdr_ref, + src->frame_hdr, src->frame_hdr_ref, + src->content_light, src->content_light_ref, + src->mastering_display, src->mastering_display_ref, + src->itut_t35, src->itut_t35_ref, + src->p.bpc, &src->m, &plane_ctx->allocator, + 0, NULL); + return res; +} + +void dav1d_picture_ref(Dav1dPicture *const dst, const Dav1dPicture *const src) { + validate_input(dst != NULL); + validate_input(dst->data[0] == NULL); + validate_input(src != NULL); + + if (src->ref) { + validate_input(src->data[0] != NULL); + dav1d_ref_inc(src->ref); + } + if (src->frame_hdr_ref) dav1d_ref_inc(src->frame_hdr_ref); + if (src->seq_hdr_ref) dav1d_ref_inc(src->seq_hdr_ref); + if (src->m.user_data.ref) dav1d_ref_inc(src->m.user_data.ref); + if (src->content_light_ref) dav1d_ref_inc(src->content_light_ref); + if (src->mastering_display_ref) dav1d_ref_inc(src->mastering_display_ref); + if (src->itut_t35_ref) dav1d_ref_inc(src->itut_t35_ref); + *dst = *src; +} + +void dav1d_picture_move_ref(Dav1dPicture *const dst, Dav1dPicture *const src) { + validate_input(dst != NULL); + validate_input(dst->data[0] == NULL); + validate_input(src != NULL); + + if (src->ref) + validate_input(src->data[0] != NULL); + + *dst = *src; + memset(src, 0, sizeof(*src)); +} + +void dav1d_thread_picture_ref(Dav1dThreadPicture *const dst, + const Dav1dThreadPicture *const src) +{ + dav1d_picture_ref(&dst->p, &src->p); + dst->visible = src->visible; + dst->showable = src->showable; + dst->progress = src->progress; + dst->flags = src->flags; +} + +void dav1d_thread_picture_move_ref(Dav1dThreadPicture *const dst, + Dav1dThreadPicture *const src) +{ + dav1d_picture_move_ref(&dst->p, &src->p); + dst->visible = src->visible; + dst->showable = src->showable; + dst->progress = src->progress; + dst->flags = src->flags; + memset(src, 0, sizeof(*src)); +} + +void dav1d_picture_unref_internal(Dav1dPicture *const p) { + validate_input(p != NULL); + + if (p->ref) { + validate_input(p->data[0] != NULL); + dav1d_ref_dec(&p->ref); + } + dav1d_ref_dec(&p->seq_hdr_ref); + dav1d_ref_dec(&p->frame_hdr_ref); + dav1d_ref_dec(&p->m.user_data.ref); + dav1d_ref_dec(&p->content_light_ref); + dav1d_ref_dec(&p->mastering_display_ref); + dav1d_ref_dec(&p->itut_t35_ref); + memset(p, 0, sizeof(*p)); + dav1d_data_props_set_defaults(&p->m); +} + +void dav1d_thread_picture_unref(Dav1dThreadPicture *const p) { + dav1d_picture_unref_internal(&p->p); + + p->progress = NULL; +} + +enum Dav1dEventFlags dav1d_picture_get_event_flags(const Dav1dThreadPicture *const p) { + if (!p->flags) + return 0; + + enum Dav1dEventFlags flags = 0; + if (p->flags & PICTURE_FLAG_NEW_SEQUENCE) + flags |= DAV1D_EVENT_FLAG_NEW_SEQUENCE; + if (p->flags & PICTURE_FLAG_NEW_OP_PARAMS_INFO) + flags |= DAV1D_EVENT_FLAG_NEW_OP_PARAMS_INFO; + + return flags; +} diff --git a/third_party/dav1d/src/picture.h b/third_party/dav1d/src/picture.h new file mode 100644 index 0000000000..0c3a0ec562 --- /dev/null +++ b/third_party/dav1d/src/picture.h @@ -0,0 +1,115 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_PICTURE_H +#define DAV1D_SRC_PICTURE_H + +#include + +#include "src/thread.h" +#include "dav1d/picture.h" + +#include "src/thread_data.h" +#include "src/ref.h" + +enum PlaneType { + PLANE_TYPE_Y, + PLANE_TYPE_UV, + PLANE_TYPE_BLOCK, + PLANE_TYPE_ALL, +}; + +enum PictureFlags { + PICTURE_FLAG_NEW_SEQUENCE = 1 << 0, + PICTURE_FLAG_NEW_OP_PARAMS_INFO = 1 << 1, + PICTURE_FLAG_NEW_TEMPORAL_UNIT = 1 << 2, +}; + +typedef struct Dav1dThreadPicture { + Dav1dPicture p; + int visible; + // This can be set for inter frames, non-key intra frames, or for invisible + // keyframes that have not yet been made visible using the show-existing-frame + // mechanism. + int showable; + enum PictureFlags flags; + // [0] block data (including segmentation map and motion vectors) + // [1] pixel data + atomic_uint *progress; +} Dav1dThreadPicture; + +typedef struct Dav1dPictureBuffer { + void *data; + struct Dav1dPictureBuffer *next; +} Dav1dPictureBuffer; + +/* + * Allocate a picture with custom border size. + */ +int dav1d_thread_picture_alloc(Dav1dContext *c, Dav1dFrameContext *f, const int bpc); + +/** + * Allocate a picture with identical metadata to an existing picture. + * The width is a separate argument so this function can be used for + * super-res, where the width changes, but everything else is the same. + * For the more typical use case of allocating a new image of the same + * dimensions, use src->p.w as width. + */ +int dav1d_picture_alloc_copy(Dav1dContext *c, Dav1dPicture *dst, const int w, + const Dav1dPicture *src); + +/** + * Create a copy of a picture. + */ +void dav1d_picture_ref(Dav1dPicture *dst, const Dav1dPicture *src); +void dav1d_thread_picture_ref(Dav1dThreadPicture *dst, + const Dav1dThreadPicture *src); +void dav1d_thread_picture_move_ref(Dav1dThreadPicture *dst, + Dav1dThreadPicture *src); +void dav1d_thread_picture_unref(Dav1dThreadPicture *p); + +/** + * Move a picture reference. + */ +void dav1d_picture_move_ref(Dav1dPicture *dst, Dav1dPicture *src); + +int dav1d_default_picture_alloc(Dav1dPicture *p, void *cookie); +void dav1d_default_picture_release(Dav1dPicture *p, void *cookie); +void dav1d_picture_unref_internal(Dav1dPicture *p); + +void dav1d_picture_copy_props(Dav1dPicture *p, + Dav1dContentLightLevel *content_light, Dav1dRef *content_light_ref, + Dav1dMasteringDisplay *mastering_display, Dav1dRef *mastering_display_ref, + Dav1dITUTT35 *itut_t35, Dav1dRef *itut_t35_ref, + const Dav1dDataProps *props); + +/** + * Get event flags from picture flags. + */ +enum Dav1dEventFlags dav1d_picture_get_event_flags(const Dav1dThreadPicture *p); + +#endif /* DAV1D_SRC_PICTURE_H */ diff --git a/third_party/dav1d/src/ppc/cdef.h b/third_party/dav1d/src/ppc/cdef.h new file mode 100644 index 0000000000..b794ba53be --- /dev/null +++ b/third_party/dav1d/src/ppc/cdef.h @@ -0,0 +1,61 @@ +/* + * Copyright © 2019, Luca Barbato + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "common/bitdepth.h" +#include "common/intops.h" + +#include "src/cdef.h" +#include "src/cpu.h" + +#define cdef_vsx_fn(w, h) \ +void dav1d_cdef_filter_##w##x##h##_vsx(pixel *const dst, \ + const ptrdiff_t dst_stride, \ + const pixel (*left)[2], \ + const pixel *const top, \ + const pixel *const bottom, \ + const int pri_strength, \ + const int sec_strength, \ + const int dir, \ + const int damping, \ + const enum CdefEdgeFlags edges) + +cdef_vsx_fn(4, 4); +cdef_vsx_fn(4, 8); +cdef_vsx_fn(8, 8); + +static ALWAYS_INLINE void cdef_dsp_init_ppc(Dav1dCdefDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_PPC_CPU_FLAG_VSX)) return; + +#if BITDEPTH == 8 + c->fb[0] = dav1d_cdef_filter_8x8_vsx; + c->fb[1] = dav1d_cdef_filter_4x8_vsx; + c->fb[2] = dav1d_cdef_filter_4x4_vsx; +#endif +} diff --git a/third_party/dav1d/src/ppc/cdef_tmpl.c b/third_party/dav1d/src/ppc/cdef_tmpl.c new file mode 100644 index 0000000000..e2e759810f --- /dev/null +++ b/third_party/dav1d/src/ppc/cdef_tmpl.c @@ -0,0 +1,487 @@ +/* + * Copyright © 2019, Luca Barbato + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/ppc/dav1d_types.h" +#include "src/ppc/cdef.h" + +#if BITDEPTH == 8 +static inline i16x8 vconstrain(const i16x8 diff, const int16_t threshold, + const int damping) +{ + const i16x8 zero = vec_splat_s16(0); + if (!threshold) return zero; + const uint16_t shift = imax(0, damping - ulog2(threshold)); + const i16x8 abs_diff = vec_abs(diff); + const b16x8 mask = vec_cmplt(diff, zero); + const i16x8 thr = vec_splats(threshold); + const i16x8 sub = vec_sub(thr, vec_sra(abs_diff, vec_splats(shift))); + const i16x8 max = vec_max(zero, sub); + const i16x8 min = vec_min(abs_diff, max); + const i16x8 neg = vec_sub(zero, min); + return vec_sel(min, neg, mask); +} + +static inline void copy4xN(uint16_t *tmp, const ptrdiff_t tmp_stride, + const uint8_t *src, const ptrdiff_t src_stride, + const uint8_t (*left)[2], const uint8_t *const top, + const uint8_t *const bottom, const int w, const int h, + const enum CdefEdgeFlags edges) +{ + const u16x8 fill = vec_splats((uint16_t)INT16_MAX); + + u16x8 l0; + u16x8 l1; + + int y_start = -2, y_end = h + 2; + + // Copy top and bottom first + if (!(edges & CDEF_HAVE_TOP)) { + l0 = fill; + l1 = fill; + y_start = 0; + } else { + l0 = u8h_to_u16(vec_vsx_ld(0, top + 0 * src_stride - 2)); + l1 = u8h_to_u16(vec_vsx_ld(0, top + 1 * src_stride - 2)); + } + + vec_st(l0, 0, tmp - 2 * 8); + vec_st(l1, 0, tmp - 1 * 8); + + if (!(edges & CDEF_HAVE_BOTTOM)) { + l0 = fill; + l1 = fill; + y_end -= 2; + } else { + l0 = u8h_to_u16(vec_vsx_ld(0, bottom + 0 * src_stride - 2)); + l1 = u8h_to_u16(vec_vsx_ld(0, bottom + 1 * src_stride - 2)); + } + + vec_st(l0, 0, tmp + (h + 0) * 8); + vec_st(l1, 0, tmp + (h + 1) * 8); + + int y_with_left_edge = 0; + if (!(edges & CDEF_HAVE_LEFT)) { + u16x8 l = u8h_to_u16(vec_vsx_ld(0, src)); + vec_vsx_st(l, 0, tmp + 2); + + y_with_left_edge = 1; + } + + for (int y = y_with_left_edge; y < h; y++) { + u16x8 l = u8h_to_u16(vec_vsx_ld(0, src - 2 + y * src_stride)); + vec_st(l, 0, tmp + y * 8); + } + + if (!(edges & CDEF_HAVE_LEFT)) { + for (int y = y_start; y < y_end; y++) { + tmp[y * 8] = INT16_MAX; + tmp[1 + y * 8] = INT16_MAX; + } + } else { + for (int y = 0; y < h; y++) { + tmp[y * 8] = left[y][0]; + tmp[1 + y * 8] = left[y][1]; + } + } + if (!(edges & CDEF_HAVE_RIGHT)) { + for (int y = y_start; y < y_end; y++) { + tmp[- 2 + (y + 1) * 8] = INT16_MAX; + tmp[- 1 + (y + 1) * 8] = INT16_MAX; + } + } +} + +static inline void copy8xN(uint16_t *tmp, const ptrdiff_t tmp_stride, + const uint8_t *src, const ptrdiff_t src_stride, + const uint8_t (*left)[2], const uint8_t *const top, + const uint8_t *const bottom, const int w, const int h, + const enum CdefEdgeFlags edges) +{ + const u16x8 fill = vec_splats((uint16_t)INT16_MAX); + + u16x8 l0h, l0l; + u16x8 l1h, l1l; + + int y_start = -2, y_end = h + 2; + + // Copy top and bottom first + if (!(edges & CDEF_HAVE_TOP)) { + l0h = fill; + l0l = fill; + l1h = fill; + l1l = fill; + y_start = 0; + } else { + u8x16 l0 = vec_vsx_ld(0, top + 0 * src_stride - 2); + u8x16 l1 = vec_vsx_ld(0, top + 1 * src_stride - 2); + l0h = u8h_to_u16(l0); + l0l = u8l_to_u16(l0); + l1h = u8h_to_u16(l1); + l1l = u8l_to_u16(l1); + } + + vec_st(l0h, 0, tmp - 4 * 8); + vec_st(l0l, 0, tmp - 3 * 8); + vec_st(l1h, 0, tmp - 2 * 8); + vec_st(l1l, 0, tmp - 1 * 8); + + if (!(edges & CDEF_HAVE_BOTTOM)) { + l0h = fill; + l0l = fill; + l1h = fill; + l1l = fill; + y_end -= 2; + } else { + u8x16 l0 = vec_vsx_ld(0, bottom + 0 * src_stride - 2); + u8x16 l1 = vec_vsx_ld(0, bottom + 1 * src_stride - 2); + l0h = u8h_to_u16(l0); + l0l = u8l_to_u16(l0); + l1h = u8h_to_u16(l1); + l1l = u8l_to_u16(l1); + } + + vec_st(l0h, 0, tmp + (h + 0) * 16); + vec_st(l0l, 0, tmp + (h + 0) * 16 + 8); + vec_st(l1h, 0, tmp + (h + 1) * 16); + vec_st(l1l, 0, tmp + (h + 1) * 16 + 8); + + int y_with_left_edge = 0; + if (!(edges & CDEF_HAVE_LEFT)) { + u8x16 l = vec_vsx_ld(0, src); + u16x8 lh = u8h_to_u16(l); + u16x8 ll = u8l_to_u16(l); + vec_vsx_st(lh, 0, tmp + 2); + vec_vsx_st(ll, 0, tmp + 8 + 2); + + y_with_left_edge = 1; + } + + for (int y = y_with_left_edge; y < h; y++) { + u8x16 l = vec_vsx_ld(0, src - 2 + y * src_stride); + u16x8 lh = u8h_to_u16(l); + u16x8 ll = u8l_to_u16(l); + vec_st(lh, 0, tmp + y * 16); + vec_st(ll, 0, tmp + 8 + y * 16); + } + + if (!(edges & CDEF_HAVE_LEFT)) { + for (int y = y_start; y < y_end; y++) { + tmp[y * 16] = INT16_MAX; + tmp[1 + y * 16] = INT16_MAX; + } + } else { + for (int y = 0; y < h; y++) { + tmp[y * 16] = left[y][0]; + tmp[1 + y * 16] = left[y][1]; + } + } + if (!(edges & CDEF_HAVE_RIGHT)) { + for (int y = y_start; y < y_end; y++) { + tmp[- 6 + (y + 1) * 16] = INT16_MAX; + tmp[- 5 + (y + 1) * 16] = INT16_MAX; + } + } +} + +static inline i16x8 max_mask(i16x8 a, i16x8 b) { + const i16x8 I16X8_INT16_MAX = vec_splats((int16_t)INT16_MAX); + + const b16x8 mask = vec_cmpeq(a, I16X8_INT16_MAX); + + const i16x8 val = vec_sel(a, b, mask); + + return vec_max(val, b); +} + +#define LOAD_PIX(addr) \ + const i16x8 px = (i16x8)vec_vsx_ld(0, addr); \ + i16x8 max = px; \ + i16x8 min = px; \ + i16x8 sum = vec_splat_s16(0); + +#define LOAD_PIX4(addr) \ + const i16x8 a = (i16x8)vec_vsx_ld(0, addr); \ + const i16x8 b = (i16x8)vec_vsx_ld(0, addr + tmp_stride); \ + const i16x8 px = vec_xxpermdi(a, b, 0); \ + i16x8 max = px; \ + i16x8 min = px; \ + i16x8 sum = vec_splat_s16(0); + +#define LOAD_DIR(p, addr, o0, o1) \ + const i16x8 p ## 0 = (i16x8)vec_vsx_ld(0, addr + o0); \ + const i16x8 p ## 1 = (i16x8)vec_vsx_ld(0, addr - o0); \ + const i16x8 p ## 2 = (i16x8)vec_vsx_ld(0, addr + o1); \ + const i16x8 p ## 3 = (i16x8)vec_vsx_ld(0, addr - o1); + +#define LOAD_DIR4(p, addr, o0, o1) \ + LOAD_DIR(p ## a, addr, o0, o1) \ + LOAD_DIR(p ## b, addr + tmp_stride, o0, o1) \ + const i16x8 p ## 0 = vec_xxpermdi(p ## a ## 0, p ## b ## 0, 0); \ + const i16x8 p ## 1 = vec_xxpermdi(p ## a ## 1, p ## b ## 1, 0); \ + const i16x8 p ## 2 = vec_xxpermdi(p ## a ## 2, p ## b ## 2, 0); \ + const i16x8 p ## 3 = vec_xxpermdi(p ## a ## 3, p ## b ## 3, 0); + +#define CONSTRAIN(p, strength) \ + const i16x8 p ## _d0 = vec_sub(p ## 0, px); \ + const i16x8 p ## _d1 = vec_sub(p ## 1, px); \ + const i16x8 p ## _d2 = vec_sub(p ## 2, px); \ + const i16x8 p ## _d3 = vec_sub(p ## 3, px); \ +\ + i16x8 p ## _c0 = vconstrain(p ## _d0, strength, damping); \ + i16x8 p ## _c1 = vconstrain(p ## _d1, strength, damping); \ + i16x8 p ## _c2 = vconstrain(p ## _d2, strength, damping); \ + i16x8 p ## _c3 = vconstrain(p ## _d3, strength, damping); + +#define MIN_MAX(p) \ + max = max_mask(p ## 0, max); \ + min = vec_min(p ## 0, min); \ + max = max_mask(p ## 1, max); \ + min = vec_min(p ## 1, min); \ + max = max_mask(p ## 2, max); \ + min = vec_min(p ## 2, min); \ + max = max_mask(p ## 3, max); \ + min = vec_min(p ## 3, min); + +#define PRI_0(p) \ + p ## _c0 = vec_add(vec_sl(p ## _c0, vec_splat_u16(1)), vec_sl(p ## _c0, vec_splats(tap_even))); \ + p ## _c1 = vec_add(vec_sl(p ## _c1, vec_splat_u16(1)), vec_sl(p ## _c1, vec_splats(tap_even))); + +#define PRI_1(p) \ + p ## _c2 = vec_sub(vec_sl(p ## _c2, vec_splat_u16(2)), vec_sl(p ## _c2, vec_splats(tap_even))); \ + p ## _c3 = vec_sub(vec_sl(p ## _c3, vec_splat_u16(2)), vec_sl(p ## _c3, vec_splats(tap_even))); + +#define SEC_0(p) \ + p ## _c0 = vec_sl(p ## _c0, vec_splat_u16(1)); \ + p ## _c1 = vec_sl(p ## _c1, vec_splat_u16(1)); \ + p ## _c2 = vec_sl(p ## _c2, vec_splat_u16(1)); \ + p ## _c3 = vec_sl(p ## _c3, vec_splat_u16(1)); + +#define UPDATE_SUM(p) \ + const i16x8 p ## sum0 = vec_add(p ## _c0, p ## _c1); \ + const i16x8 p ## sum1 = vec_add(p ## _c2, p ## _c3); \ + sum = vec_add(sum, p ## sum0); \ + sum = vec_add(sum, p ## sum1); + +static inline void +filter_4xN(pixel *dst, const ptrdiff_t dst_stride, + const pixel (*left)[2], const pixel *const top, + const pixel *const bottom, const int w, const int h, + const int pri_strength, const int sec_strength, const int dir, + const int damping, const enum CdefEdgeFlags edges, + const ptrdiff_t tmp_stride, uint16_t *tmp) +{ + const int8_t cdef_directions[8 /* dir */][2 /* pass */] = { + { -1 * tmp_stride + 1, -2 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, -1 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, 0 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, 1 * tmp_stride + 2 }, + { 1 * tmp_stride + 1, 2 * tmp_stride + 2 }, + { 1 * tmp_stride + 0, 2 * tmp_stride + 1 }, + { 1 * tmp_stride + 0, 2 * tmp_stride + 0 }, + { 1 * tmp_stride + 0, 2 * tmp_stride - 1 } + }; + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + const uint16_t tap_even = !((pri_strength >> bitdepth_min_8) & 1); + const int off1 = cdef_directions[dir][0]; + const int off1_1 = cdef_directions[dir][1]; + + const int off2 = cdef_directions[(dir + 2) & 7][0]; + const int off3 = cdef_directions[(dir + 6) & 7][0]; + + const int off2_1 = cdef_directions[(dir + 2) & 7][1]; + const int off3_1 = cdef_directions[(dir + 6) & 7][1]; + + copy4xN(tmp - 2, tmp_stride, dst, dst_stride, left, top, bottom, w, h, edges); + + for (int y = 0; y < h / 2; y++) { + LOAD_PIX4(tmp) + + // Primary pass + LOAD_DIR4(p, tmp, off1, off1_1) + + CONSTRAIN(p, pri_strength) + + MIN_MAX(p) + + PRI_0(p) + PRI_1(p) + + UPDATE_SUM(p) + + // Secondary pass 1 + LOAD_DIR4(s, tmp, off2, off3) + + CONSTRAIN(s, sec_strength) + + MIN_MAX(s) + + SEC_0(s) + + UPDATE_SUM(s) + + // Secondary pass 2 + LOAD_DIR4(s2, tmp, off2_1, off3_1) + + CONSTRAIN(s2, sec_strength) + + MIN_MAX(s2) + + UPDATE_SUM(s2) + + // Store + i16x8 bias = vec_and((i16x8)vec_cmplt(sum, vec_splat_s16(0)), vec_splat_s16(1)); + bias = vec_sub(vec_splat_s16(8), bias); + i16x8 unclamped = vec_add(px, vec_sra(vec_add(sum, bias), vec_splat_u16(4))); + i16x8 vdst = vec_max(vec_min(unclamped, max), min); + + dst[0] = vdst[0]; + dst[1] = vdst[1]; + dst[2] = vdst[2]; + dst[3] = vdst[3]; + + tmp += tmp_stride; + dst += PXSTRIDE(dst_stride); + dst[0] = vdst[4]; + dst[1] = vdst[5]; + dst[2] = vdst[6]; + dst[3] = vdst[7]; + + tmp += tmp_stride; + dst += PXSTRIDE(dst_stride); + } +} + +static inline void +filter_8xN(pixel *dst, const ptrdiff_t dst_stride, + const pixel (*left)[2], const pixel *const top, + const pixel *const bottom, const int w, const int h, + const int pri_strength, const int sec_strength, const int dir, + const int damping, const enum CdefEdgeFlags edges, + const ptrdiff_t tmp_stride, uint16_t *tmp) +{ + const int8_t cdef_directions[8 /* dir */][2 /* pass */] = { + { -1 * tmp_stride + 1, -2 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, -1 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, 0 * tmp_stride + 2 }, + { 0 * tmp_stride + 1, 1 * tmp_stride + 2 }, + { 1 * tmp_stride + 1, 2 * tmp_stride + 2 }, + { 1 * tmp_stride + 0, 2 * tmp_stride + 1 }, + { 1 * tmp_stride + 0, 2 * tmp_stride + 0 }, + { 1 * tmp_stride + 0, 2 * tmp_stride - 1 } + }; + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + + + const uint16_t tap_even = !((pri_strength >> bitdepth_min_8) & 1); + const int off1 = cdef_directions[dir][0]; + const int off1_1 = cdef_directions[dir][1]; + + const int off2 = cdef_directions[(dir + 2) & 7][0]; + const int off3 = cdef_directions[(dir + 6) & 7][0]; + + const int off2_1 = cdef_directions[(dir + 2) & 7][1]; + const int off3_1 = cdef_directions[(dir + 6) & 7][1]; + + copy8xN(tmp - 2, tmp_stride, dst, dst_stride, left, top, bottom, w, h, edges); + + for (int y = 0; y < h; y++) { + LOAD_PIX(tmp) + + // Primary pass + LOAD_DIR(p, tmp, off1, off1_1) + + CONSTRAIN(p, pri_strength) + + MIN_MAX(p) + + PRI_0(p) + PRI_1(p) + + UPDATE_SUM(p) + + // Secondary pass 1 + LOAD_DIR(s, tmp, off2, off3) + + CONSTRAIN(s, sec_strength) + + MIN_MAX(s) + + SEC_0(s) + + UPDATE_SUM(s) + + // Secondary pass 2 + LOAD_DIR(s2, tmp, off2_1, off3_1) + + CONSTRAIN(s2, sec_strength) + + MIN_MAX(s2) + + UPDATE_SUM(s2) + + // Store + i16x8 bias = vec_and((i16x8)vec_cmplt(sum, vec_splat_s16(0)), vec_splat_s16(1)); + bias = vec_sub(vec_splat_s16(8), bias); + i16x8 unclamped = vec_add(px, vec_sra(vec_add(sum, bias), vec_splat_u16(4))); + i16x8 vdst = vec_max(vec_min(unclamped, max), min); + + dst[0] = vdst[0]; + dst[1] = vdst[1]; + dst[2] = vdst[2]; + dst[3] = vdst[3]; + dst[4] = vdst[4]; + dst[5] = vdst[5]; + dst[6] = vdst[6]; + dst[7] = vdst[7]; + + tmp += tmp_stride; + dst += PXSTRIDE(dst_stride); + } + +} + +#define cdef_fn(w, h, tmp_stride) \ +void dav1d_cdef_filter_##w##x##h##_vsx(pixel *const dst, \ + const ptrdiff_t dst_stride, \ + const pixel (*left)[2], \ + const pixel *const top, \ + const pixel *const bottom, \ + const int pri_strength, \ + const int sec_strength, \ + const int dir, \ + const int damping, \ + const enum CdefEdgeFlags edges) \ +{ \ + ALIGN_STK_16(uint16_t, tmp_buf, 12 * tmp_stride + 8,); \ + uint16_t *tmp = tmp_buf + 2 * tmp_stride + 2; \ + filter_##w##xN(dst, dst_stride, left, top, bottom, w, h, pri_strength, \ + sec_strength, dir, damping, edges, tmp_stride, tmp); \ +} + +cdef_fn(4, 4, 8); +cdef_fn(4, 8, 8); +cdef_fn(8, 8, 16); +#endif diff --git a/third_party/dav1d/src/ppc/cpu.c b/third_party/dav1d/src/ppc/cpu.c new file mode 100644 index 0000000000..fe77057c57 --- /dev/null +++ b/third_party/dav1d/src/ppc/cpu.c @@ -0,0 +1,51 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "common/attributes.h" + +#include "src/ppc/cpu.h" + +#if (defined(HAVE_GETAUXVAL) || defined(HAVE_ELF_AUX_INFO)) && ARCH_PPC64LE +#include +#define HAVE_AUX +#endif + +COLD unsigned dav1d_get_cpu_flags_ppc(void) { + unsigned flags = 0; +#if defined(HAVE_GETAUXVAL) && ARCH_PPC64LE + unsigned long hw_cap = getauxval(AT_HWCAP); +#elif defined(HAVE_ELF_AUX_INFO) && ARCH_PPC64LE + unsigned long hw_cap = 0; + elf_aux_info(AT_HWCAP, &hw_cap, sizeof(hw_cap)); +#endif +#ifdef HAVE_AUX + flags |= (hw_cap & PPC_FEATURE_HAS_VSX) ? DAV1D_PPC_CPU_FLAG_VSX : 0; +#endif + return flags; +} diff --git a/third_party/dav1d/src/ppc/cpu.h b/third_party/dav1d/src/ppc/cpu.h new file mode 100644 index 0000000000..cfd2ff4ff5 --- /dev/null +++ b/third_party/dav1d/src/ppc/cpu.h @@ -0,0 +1,37 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Janne Grunau + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_PPC_CPU_H +#define DAV1D_SRC_PPC_CPU_H + +enum CpuFlags { + DAV1D_PPC_CPU_FLAG_VSX = 1 << 0, +}; + +unsigned dav1d_get_cpu_flags_ppc(void); + +#endif /* DAV1D_SRC_PPC_CPU_H */ diff --git a/third_party/dav1d/src/ppc/dav1d_types.h b/third_party/dav1d/src/ppc/dav1d_types.h new file mode 100644 index 0000000000..0b4bd72f0e --- /dev/null +++ b/third_party/dav1d/src/ppc/dav1d_types.h @@ -0,0 +1,54 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Luca Barbato + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_PPC_TYPES_H +#define DAV1D_SRC_PPC_TYPES_H + +#include +#undef pixel + +#define u8x16 vector unsigned char +#define i8x16 vector signed char +#define b8x16 vector bool char +#define u16x8 vector unsigned short +#define i16x8 vector signed short +#define b16x8 vector bool short +#define u32x4 vector unsigned int +#define i32x4 vector signed int +#define b32x4 vector bool int +#define u64x2 vector unsigned long long +#define i64x2 vector signed long long +#define b64x2 vector bool long long + +#define u8h_to_u16(v) ((u16x8) vec_mergeh((u8x16) v, vec_splat_u8(0))) +#define u8l_to_u16(v) ((u16x8) vec_mergel((u8x16) v, vec_splat_u8(0))) +#define u16h_to_i32(v) ((i32x4) vec_mergeh((u16x8) v, vec_splat_u16(0))) +#define i16h_to_i32(v) ((i32x4) vec_unpackh((i16x8)v)) +#define u16l_to_i32(v) ((i32x4) vec_mergel((u16x8) v, vec_splat_u16(0))) +#define i16l_to_i32(v) ((i32x4) vec_unpackl((i16x8)v)) + +#endif /* DAV1D_SRC_PPC_TYPES_H */ diff --git a/third_party/dav1d/src/ppc/looprestoration.h b/third_party/dav1d/src/ppc/looprestoration.h new file mode 100644 index 0000000000..3fe16318bd --- /dev/null +++ b/third_party/dav1d/src/ppc/looprestoration.h @@ -0,0 +1,48 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Michail Alvanos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "common/intops.h" + +#include "src/cpu.h" +#include "src/looprestoration.h" + +void dav1d_wiener_filter_vsx(uint8_t *p, const ptrdiff_t stride, + const uint8_t (*const left)[4], + const uint8_t *lpf, + const int w, const int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX); + +static ALWAYS_INLINE void loop_restoration_dsp_init_ppc(Dav1dLoopRestorationDSPContext *const c, const int bpc) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_PPC_CPU_FLAG_VSX)) return; + +#if BITDEPTH == 8 + c->wiener[0] = c->wiener[1] = dav1d_wiener_filter_vsx; +#endif +} diff --git a/third_party/dav1d/src/ppc/looprestoration_tmpl.c b/third_party/dav1d/src/ppc/looprestoration_tmpl.c new file mode 100644 index 0000000000..c0c64e1800 --- /dev/null +++ b/third_party/dav1d/src/ppc/looprestoration_tmpl.c @@ -0,0 +1,321 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Michail Alvanos + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/ppc/dav1d_types.h" +#include "src/ppc/looprestoration.h" + +#if BITDEPTH == 8 + +#define REST_UNIT_STRIDE (400) + +static inline i32x4 iclip_vec(i32x4 v, const i32x4 minv, const i32x4 maxv) { + v = vec_max(minv, v); + v = vec_min(maxv, v); + return v; +} + +#define APPLY_FILTER_H(v, f, ssum1, ssum2) do { \ + i16x8 ktmp_u16_high = (i16x8) u8h_to_u16(v); \ + i16x8 ktmp_u16_low = (i16x8) u8l_to_u16(v); \ + ssum1 = vec_madd(ktmp_u16_high, f, ssum1); \ + ssum2 = vec_madd(ktmp_u16_low, f, ssum2); \ +} while (0) + +static void wiener_filter_h_vsx(int32_t *hor_ptr, + uint8_t *tmp_ptr, + const int16_t filterh[8], + const int w, const int h) +{ + const i32x4 zerov = vec_splats(0); + const i32x4 seven_vec = vec_splats(7); + const i32x4 bitdepth_added_vec = vec_splats(1 << 14); + const i32x4 round_bits_vec = vec_splats(3); + const i32x4 rounding_off_vec = vec_splats(1<<2); + const i32x4 clip_limit_v = vec_splats((1 << 13) - 1); + + i16x8 filterhvall = vec_vsx_ld(0, filterh); + i16x8 filterhv0 = vec_splat( filterhvall, 0); + i16x8 filterhv1 = vec_splat( filterhvall, 1); + i16x8 filterhv2 = vec_splat( filterhvall, 2); + i16x8 filterhv3 = vec_splat( filterhvall, 3); + i16x8 filterhv4 = vec_splat( filterhvall, 4); + i16x8 filterhv5 = vec_splat( filterhvall, 5); + i16x8 filterhv6 = vec_splat( filterhvall, 6); + + for (int j = 0; j < h + 6; j++) { + for (int i = 0; i < w; i+=16) { + i32x4 sum1 = bitdepth_added_vec; + i32x4 sum2 = bitdepth_added_vec; + i32x4 sum3 = bitdepth_added_vec; + i32x4 sum4 = bitdepth_added_vec; + + u8x16 tmp_v0 = vec_ld(0, &tmp_ptr[i]); + u8x16 tmp_v7 = vec_ld(0, &tmp_ptr[i+16]); + + u8x16 tmp_v1 = vec_sld( tmp_v7, tmp_v0, 15); + u8x16 tmp_v2 = vec_sld( tmp_v7, tmp_v0, 14); + u8x16 tmp_v3 = vec_sld( tmp_v7, tmp_v0, 13); + u8x16 tmp_v4 = vec_sld( tmp_v7, tmp_v0, 12); + u8x16 tmp_v5 = vec_sld( tmp_v7, tmp_v0, 11); + u8x16 tmp_v6 = vec_sld( tmp_v7, tmp_v0, 10); + + u16x8 tmp_u16_high = u8h_to_u16(tmp_v3); + u16x8 tmp_u16_low = u8l_to_u16(tmp_v3); + + i32x4 tmp_expanded1 = i16h_to_i32(tmp_u16_high); + i32x4 tmp_expanded2 = i16l_to_i32(tmp_u16_high); + i32x4 tmp_expanded3 = i16h_to_i32(tmp_u16_low); + i32x4 tmp_expanded4 = i16l_to_i32(tmp_u16_low); + + i16x8 ssum1 = (i16x8) zerov; + i16x8 ssum2 = (i16x8) zerov; + + APPLY_FILTER_H(tmp_v0, filterhv0, ssum1, ssum2); + APPLY_FILTER_H(tmp_v1, filterhv1, ssum1, ssum2); + APPLY_FILTER_H(tmp_v2, filterhv2, ssum1, ssum2); + APPLY_FILTER_H(tmp_v3, filterhv3, ssum1, ssum2); + APPLY_FILTER_H(tmp_v4, filterhv4, ssum1, ssum2); + APPLY_FILTER_H(tmp_v5, filterhv5, ssum1, ssum2); + APPLY_FILTER_H(tmp_v6, filterhv6, ssum1, ssum2); + + sum1 += i16h_to_i32(ssum1) + (tmp_expanded1 << seven_vec); + sum2 += i16l_to_i32(ssum1) + (tmp_expanded2 << seven_vec); + sum3 += i16h_to_i32(ssum2) + (tmp_expanded3 << seven_vec); + sum4 += i16l_to_i32(ssum2) + (tmp_expanded4 << seven_vec); + + sum1 = (sum1 + rounding_off_vec) >> round_bits_vec; + sum2 = (sum2 + rounding_off_vec) >> round_bits_vec; + sum3 = (sum3 + rounding_off_vec) >> round_bits_vec; + sum4 = (sum4 + rounding_off_vec) >> round_bits_vec; + + sum1 = iclip_vec(sum1, zerov, clip_limit_v); + sum2 = iclip_vec(sum2, zerov, clip_limit_v); + sum3 = iclip_vec(sum3, zerov, clip_limit_v); + sum4 = iclip_vec(sum4, zerov, clip_limit_v); + + vec_st(sum1, 0, &hor_ptr[i]); + vec_st(sum2, 16, &hor_ptr[i]); + vec_st(sum3, 32, &hor_ptr[i]); + vec_st(sum4, 48, &hor_ptr[i]); + } + tmp_ptr += REST_UNIT_STRIDE; + hor_ptr += REST_UNIT_STRIDE; + } +} + +static inline i16x8 iclip_u8_vec(i16x8 v) { + const i16x8 zerov = vec_splats((int16_t)0); + const i16x8 maxv = vec_splats((int16_t)255); + v = vec_max(zerov, v); + v = vec_min(maxv, v); + return v; +} + +#define APPLY_FILTER_V(index, f) do { \ + i32x4 v1 = vec_ld( 0, &hor[(j + index) * REST_UNIT_STRIDE + i]); \ + i32x4 v2 = vec_ld(16, &hor[(j + index) * REST_UNIT_STRIDE + i]); \ + i32x4 v3 = vec_ld(32, &hor[(j + index) * REST_UNIT_STRIDE + i]); \ + i32x4 v4 = vec_ld(48, &hor[(j + index) * REST_UNIT_STRIDE + i]); \ + sum1 = sum1 + v1 * f; \ + sum2 = sum2 + v2 * f; \ + sum3 = sum3 + v3 * f; \ + sum4 = sum4 + v4 * f; \ +} while (0) + +#define LOAD_AND_APPLY_FILTER_V(sumpixelv, hor) do { \ + i32x4 sum1 = round_vec; \ + i32x4 sum2 = round_vec; \ + i32x4 sum3 = round_vec; \ + i32x4 sum4 = round_vec; \ + APPLY_FILTER_V(0, filterv0); \ + APPLY_FILTER_V(1, filterv1); \ + APPLY_FILTER_V(2, filterv2); \ + APPLY_FILTER_V(3, filterv3); \ + APPLY_FILTER_V(4, filterv4); \ + APPLY_FILTER_V(5, filterv5); \ + APPLY_FILTER_V(6, filterv6); \ + sum1 = sum1 >> round_bits_vec; \ + sum2 = sum2 >> round_bits_vec; \ + sum3 = sum3 >> round_bits_vec; \ + sum4 = sum4 >> round_bits_vec; \ + i16x8 sum_short_packed_1 = (i16x8) vec_pack(sum1, sum2); \ + i16x8 sum_short_packed_2 = (i16x8) vec_pack(sum3, sum4); \ + sum_short_packed_1 = iclip_u8_vec(sum_short_packed_1); \ + sum_short_packed_2 = iclip_u8_vec(sum_short_packed_2); \ + sum_pixel = (u8x16) vec_pack(sum_short_packed_1, sum_short_packed_2); \ +} while (0) + +static inline void wiener_filter_v_vsx(uint8_t *p, + const ptrdiff_t stride, + const int32_t *hor, + const int16_t filterv[8], + const int w, const int h) +{ + const i32x4 round_bits_vec = vec_splats(11); + const i32x4 round_vec = vec_splats((1 << 10) - (1 << 18)); + + i32x4 filterv0 = vec_splats((int32_t) filterv[0]); + i32x4 filterv1 = vec_splats((int32_t) filterv[1]); + i32x4 filterv2 = vec_splats((int32_t) filterv[2]); + i32x4 filterv3 = vec_splats((int32_t) filterv[3]); + i32x4 filterv4 = vec_splats((int32_t) filterv[4]); + i32x4 filterv5 = vec_splats((int32_t) filterv[5]); + i32x4 filterv6 = vec_splats((int32_t) filterv[6]); + + for (int j = 0; j < h; j++) { + for (int i = 0; i <(w-w%16); i += 16) { + u8x16 sum_pixel; + LOAD_AND_APPLY_FILTER_V(sum_pixel, hor); + vec_vsx_st(sum_pixel, 0, &p[j * PXSTRIDE(stride) + i]); + } + // remaining loop + if (w & 0xf){ + int i=w-w%16; + ALIGN_STK_16(uint8_t, tmp_out, 16,); + u8x16 sum_pixel; + + LOAD_AND_APPLY_FILTER_V(sum_pixel, hor); + vec_vsx_st(sum_pixel, 0, tmp_out); + + for (int k=0; ifilter; + + // Wiener filtering is applied to a maximum stripe height of 64 + 3 pixels + // of padding above and below + ALIGN_STK_16(uint8_t, tmp, 70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE,); + padding(tmp, p, stride, left, lpf, w, h, edges); + ALIGN_STK_16(int32_t, hor, 70 /*(64 + 3 + 3)*/ * REST_UNIT_STRIDE + 64,); + + wiener_filter_h_vsx(hor, tmp, filter[0], w, h); + wiener_filter_v_vsx(p, stride, hor, filter[1], w, h); +} +#endif diff --git a/third_party/dav1d/src/qm.c b/third_party/dav1d/src/qm.c new file mode 100644 index 0000000000..e2e0d61874 --- /dev/null +++ b/third_party/dav1d/src/qm.c @@ -0,0 +1,3148 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" + +#include "src/qm.h" + +static const uint8_t qm_tbl_4x4_t[][2][10] = { + { + { + 32, + 43, 67, + 73, 94, 137, + 97, 110, 150, 200, + }, { + 35, + 46, 60, + 57, 69, 90, + 66, 71, 90, 109, + }, + }, { + { + 32, + 41, 63, + 69, 88, 127, + 92, 103, 140, 184, + }, { + 33, + 45, 58, + 56, 66, 86, + 64, 69, 87, 105, + }, + }, { + { + 32, + 38, 56, + 63, 78, 113, + 86, 97, 130, 169, + }, { + 32, + 45, 55, + 53, 62, 80, + 63, 67, 84, 101, + }, + }, { + { + 32, + 37, 54, + 58, 72, 102, + 81, 91, 121, 156, + }, { + 32, + 45, 54, + 51, 59, 75, + 61, 65, 81, 97, + }, + }, { + { + 32, + 34, 49, + 53, 64, 91, + 75, 81, 112, 140, + }, { + 32, + 46, 53, + 49, 55, 70, + 58, 62, 78, 91, + }, + }, { + { + 32, + 34, 48, + 49, 60, 82, + 72, 79, 104, 134, + }, { + 32, + 46, 53, + 47, 54, 66, + 57, 60, 75, 89, + }, + }, { + { + 32, + 33, 39, + 45, 51, 71, + 62, 64, 87, 108, + }, { + 31, + 42, 48, + 47, 50, 61, + 53, 54, 67, 78, + }, + }, { + { + 32, + 33, 38, + 42, 46, 63, + 55, 57, 75, 92, + }, { + 31, + 41, 48, + 46, 48, 58, + 51, 51, 62, 71, + }, + }, { + { + 32, + 32, 35, + 38, 40, 54, + 51, 49, 64, 81, + }, { + 31, + 38, 47, + 47, 46, 54, + 49, 46, 57, 66, + }, + }, { + { + 32, + 32, 34, + 35, 37, 48, + 43, 43, 54, 65, + }, { + 31, + 37, 44, + 47, 47, 53, + 47, 45, 53, 59, + }, + }, { + { + 32, + 32, 33, + 34, 35, 39, + 38, 39, 45, 54, + }, { + 31, + 34, 39, + 42, 45, 48, + 47, 46, 49, 54, + }, + }, { + { + 32, + 32, 32, + 32, 33, 35, + 35, 35, 38, 46, + }, { + 31, + 32, 34, + 38, 41, 47, + 46, 46, 47, 52, + }, + }, { + { + 31, + 32, 32, + 32, 32, 33, + 32, 33, 34, 35, + }, { + 31, + 31, 32, + 34, 35, 39, + 38, 40, 43, 47, + }, + }, { + { + 31, + 31, 32, + 31, 32, 32, + 32, 32, 32, 33, + }, { + 31, + 31, 31, + 31, 31, 32, + 34, 35, 35, 39, + }, + }, { + { + 31, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + }, { + 31, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + }, + }, +}; + +static const uint8_t qm_tbl_8x4[][2][32] = { + { + { + 32, 33, 37, 49, 65, 80, 91, 104, + 42, 42, 58, 71, 84, 97, 100, 112, + 75, 69, 84, 103, 125, 142, 145, 146, + 91, 86, 91, 110, 128, 152, 178, 190, + }, { + 31, 40, 46, 48, 54, 61, 64, 68, + 47, 45, 56, 61, 65, 69, 68, 71, + 60, 54, 64, 75, 85, 92, 90, 87, + 66, 61, 64, 73, 82, 92, 102, 105, + }, + }, { + { + 32, 33, 36, 46, 60, 75, 86, 98, + 42, 42, 56, 67, 79, 92, 95, 105, + 69, 64, 77, 93, 112, 130, 136, 136, + 88, 83, 88, 105, 122, 144, 167, 177, + }, { + 31, 40, 46, 47, 52, 59, 63, 66, + 47, 45, 55, 60, 64, 68, 66, 69, + 57, 52, 61, 70, 79, 87, 88, 85, + 65, 61, 63, 72, 81, 90, 99, 102, + }, + }, { + { + 32, 32, 34, 44, 54, 72, 82, 92, + 38, 40, 51, 61, 69, 84, 89, 98, + 62, 58, 68, 85, 98, 118, 129, 127, + 86, 80, 85, 101, 117, 136, 157, 165, + }, { + 31, 38, 46, 46, 50, 57, 61, 65, + 47, 46, 53, 56, 59, 64, 65, 67, + 54, 50, 57, 66, 74, 82, 85, 82, + 64, 60, 62, 71, 79, 88, 97, 99, + }, + }, { + { + 32, 32, 34, 41, 51, 65, 75, 86, + 35, 36, 47, 53, 61, 73, 81, 92, + 59, 57, 65, 78, 92, 108, 117, 119, + 83, 78, 82, 97, 111, 129, 148, 154, + }, { + 31, 36, 46, 45, 49, 54, 59, 63, + 47, 47, 52, 53, 55, 58, 61, 65, + 53, 50, 55, 63, 71, 77, 81, 80, + 63, 59, 61, 70, 77, 86, 94, 95, + }, + }, { + { + 32, 32, 34, 38, 48, 60, 72, 81, + 35, 36, 42, 51, 59, 68, 79, 86, + 51, 50, 54, 67, 80, 92, 104, 112, + 77, 72, 75, 87, 103, 119, 135, 144, + }, { + 31, 36, 43, 45, 47, 52, 57, 61, + 47, 47, 50, 53, 54, 56, 60, 63, + 50, 47, 50, 58, 66, 70, 75, 77, + 61, 57, 58, 65, 74, 82, 90, 93, + }, + }, { + { + 32, 32, 34, 37, 45, 54, 65, 75, + 35, 36, 42, 50, 56, 63, 73, 81, + 51, 50, 54, 65, 76, 87, 97, 106, + 75, 71, 73, 84, 96, 110, 125, 136, + }, { + 31, 36, 43, 46, 46, 50, 54, 59, + 47, 47, 50, 53, 54, 55, 58, 61, + 50, 47, 50, 57, 64, 68, 72, 75, + 60, 56, 57, 64, 71, 78, 85, 90, + }, + }, { + { + 32, 32, 33, 35, 41, 49, 57, 66, + 34, 34, 37, 43, 48, 54, 60, 68, + 43, 42, 44, 54, 64, 71, 78, 86, + 62, 59, 58, 68, 79, 91, 101, 111, + }, { + 31, 33, 40, 47, 45, 48, 51, 55, + 42, 44, 47, 50, 49, 50, 52, 55, + 47, 45, 46, 54, 59, 61, 63, 66, + 54, 51, 50, 57, 64, 70, 75, 79, + }, + }, { + { + 32, 32, 32, 34, 38, 44, 50, 61, + 32, 33, 35, 37, 40, 45, 50, 58, + 42, 41, 42, 50, 58, 66, 71, 79, + 56, 53, 52, 59, 68, 78, 86, 97, + }, { + 31, 32, 39, 44, 46, 47, 48, 53, + 38, 40, 47, 47, 47, 46, 47, 50, + 47, 45, 45, 51, 56, 59, 61, 64, + 52, 49, 48, 53, 58, 64, 68, 73, + }, + }, { + { + 32, 32, 32, 34, 35, 40, 46, 52, + 32, 33, 34, 37, 38, 42, 46, 51, + 37, 36, 38, 44, 49, 55, 59, 64, + 52, 49, 49, 54, 60, 69, 76, 83, + }, { + 31, 31, 36, 42, 47, 46, 48, 50, + 38, 40, 44, 47, 48, 46, 46, 48, + 47, 46, 47, 50, 53, 54, 55, 56, + 50, 48, 47, 50, 54, 60, 64, 67, + }, + }, { + { + 31, 32, 32, 32, 34, 37, 42, 46, + 32, 33, 34, 35, 37, 40, 43, 46, + 35, 34, 36, 38, 43, 49, 53, 56, + 43, 41, 42, 42, 49, 56, 63, 67, + }, { + 31, 31, 35, 39, 43, 47, 46, 48, + 38, 40, 43, 47, 47, 47, 46, 46, + 47, 46, 47, 47, 50, 53, 53, 54, + 48, 45, 46, 45, 50, 55, 58, 59, + }, + }, { + { + 31, 32, 32, 32, 33, 34, 37, 40, + 32, 32, 33, 33, 34, 36, 38, 40, + 34, 34, 34, 36, 38, 41, 44, 46, + 39, 38, 38, 40, 42, 47, 52, 56, + }, { + 31, 31, 33, 36, 40, 45, 47, 47, + 34, 35, 37, 41, 44, 46, 47, 46, + 42, 42, 44, 46, 48, 49, 50, 49, + 48, 46, 46, 46, 48, 51, 54, 55, + }, + }, { + { + 31, 32, 32, 32, 32, 33, 34, 35, + 31, 32, 32, 32, 33, 33, 34, 34, + 32, 32, 33, 34, 35, 36, 37, 38, + 35, 35, 34, 36, 38, 40, 42, 48, + }, { + 31, 31, 31, 34, 37, 39, 42, 48, + 31, 31, 32, 36, 39, 41, 43, 46, + 37, 38, 40, 43, 46, 47, 47, 48, + 48, 47, 46, 47, 47, 48, 50, 53, + }, + }, { + { + 31, 31, 32, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 33, 34, 34, 35, + 32, 32, 32, 33, 34, 34, 35, 36, + }, { + 31, 31, 31, 31, 34, 35, 38, 41, + 31, 31, 32, 32, 36, 37, 40, 42, + 35, 36, 37, 37, 40, 42, 45, 45, + 37, 38, 39, 40, 43, 44, 47, 47, + }, + }, { + { + 31, 31, 31, 31, 31, 31, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 33, 33, 33, + }, { + 31, 31, 31, 31, 31, 31, 34, 34, + 31, 31, 31, 32, 32, 33, 36, 36, + 31, 31, 31, 32, 32, 33, 36, 36, + 34, 35, 35, 36, 36, 37, 40, 40, + }, + }, { + { + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, + }, { + 31, 31, 31, 31, 31, 31, 31, 30, + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, + }, + }, +}; + +static const uint8_t qm_tbl_8x8_t[][2][36] = { + { + { + 32, + 32, 35, + 38, 40, 54, + 51, 49, 65, 82, + 68, 63, 78, 97, 117, + 84, 76, 91, 111, 134, 152, + 95, 89, 98, 113, 138, 159, 183, + 109, 102, 106, 121, 142, 168, 199, 220, + }, { + 31, + 38, 47, + 47, 46, 54, + 50, 47, 57, 66, + 57, 52, 61, 72, 82, + 63, 57, 66, 77, 88, 96, + 67, 62, 67, 75, 86, 95, 104, + 71, 67, 68, 75, 84, 95, 107, 113, + }, + }, { + { + 32, + 32, 35, + 37, 39, 51, + 47, 46, 60, 73, + 62, 58, 71, 87, 105, + 78, 72, 84, 100, 121, 140, + 90, 84, 93, 106, 129, 148, 169, + 102, 96, 100, 113, 132, 155, 183, 201, + }, { + 31, + 38, 47, + 47, 47, 53, + 48, 46, 55, 62, + 54, 50, 58, 67, 76, + 61, 55, 63, 72, 83, 91, + 66, 61, 65, 73, 84, 92, 101, + 69, 65, 66, 73, 82, 92, 103, 109, + }, + }, { + { + 32, + 32, 34, + 35, 37, 48, + 46, 45, 56, 70, + 57, 54, 64, 80, 93, + 76, 70, 79, 96, 111, 134, + 85, 79, 87, 100, 121, 138, 156, + 96, 90, 93, 105, 122, 144, 168, 184, + }, { + 31, + 36, 43, + 47, 47, 53, + 48, 46, 54, 61, + 52, 49, 55, 65, 71, + 60, 55, 60, 70, 78, 89, + 64, 59, 63, 71, 81, 89, 97, + 67, 63, 64, 71, 79, 89, 99, 104, + }, + }, { + { + 32, + 32, 33, + 35, 36, 46, + 42, 42, 52, 63, + 53, 51, 60, 73, 86, + 68, 64, 72, 84, 100, 117, + 78, 74, 80, 92, 109, 128, 140, + 90, 84, 87, 98, 114, 133, 155, 168, + }, { + 31, + 34, 39, + 46, 47, 52, + 47, 45, 52, 58, + 50, 48, 54, 62, 68, + 57, 53, 58, 65, 73, 82, + 61, 57, 61, 68, 77, 86, 91, + 65, 61, 62, 68, 76, 86, 95, 100, + }, + }, { + { + 32, + 32, 33, + 34, 35, 39, + 39, 40, 46, 56, + 50, 48, 53, 65, 78, + 62, 59, 63, 75, 90, 105, + 76, 71, 74, 86, 101, 118, 134, + 84, 79, 81, 92, 106, 123, 142, 153, + }, { + 31, + 34, 39, + 42, 45, 48, + 47, 46, 49, 55, + 49, 47, 50, 58, 65, + 54, 51, 53, 61, 69, 76, + 60, 56, 57, 65, 73, 82, 89, + 64, 59, 60, 66, 74, 83, 92, 96, + }, + }, { + { + 32, + 32, 33, + 34, 35, 39, + 38, 39, 45, 54, + 46, 45, 51, 61, 71, + 56, 54, 58, 69, 80, 92, + 68, 64, 68, 78, 90, 103, 117, + 78, 74, 76, 86, 99, 113, 128, 140, + }, { + 31, + 34, 39, + 42, 45, 48, + 47, 46, 49, 54, + 48, 46, 50, 56, 61, + 52, 49, 52, 58, 65, 71, + 57, 53, 55, 61, 68, 75, 82, + 61, 57, 58, 64, 71, 79, 86, 91, + }, + }, { + { + 31, + 32, 32, + 32, 33, 35, + 35, 35, 38, 48, + 42, 41, 43, 54, 63, + 51, 49, 49, 59, 71, 81, + 59, 56, 56, 66, 77, 89, 98, + 69, 65, 64, 73, 85, 97, 108, 119, + }, { + 31, + 32, 35, + 38, 42, 47, + 48, 47, 48, 53, + 47, 45, 45, 53, 58, + 50, 47, 47, 54, 61, 66, + 53, 50, 49, 56, 63, 69, 73, + 57, 54, 52, 58, 65, 72, 77, 82, + }, + }, { + { + 31, + 32, 32, + 32, 32, 35, + 34, 34, 37, 42, + 38, 37, 40, 47, 54, + 46, 44, 45, 52, 60, 69, + 52, 49, 49, 56, 65, 75, 82, + 63, 59, 58, 65, 73, 84, 92, 105, + }, { + 31, + 31, 32, + 38, 40, 47, + 44, 44, 47, 50, + 47, 45, 46, 51, 54, + 48, 46, 46, 51, 56, 61, + 50, 47, 47, 52, 57, 63, 66, + 55, 52, 50, 54, 60, 66, 70, 76, + }, + }, { + { + 31, + 32, 32, + 32, 32, 34, + 34, 33, 35, 39, + 35, 34, 37, 42, 48, + 41, 40, 41, 47, 53, 60, + 47, 44, 45, 51, 57, 65, 71, + 53, 50, 51, 55, 61, 70, 77, 85, + }, { + 31, + 31, 32, + 35, 36, 41, + 42, 42, 45, 48, + 48, 46, 47, 50, 53, + 47, 45, 45, 49, 53, 57, + 49, 46, 46, 50, 54, 59, 61, + 51, 48, 48, 51, 54, 60, 64, 68, + }, + }, { + { + 31, + 31, 32, + 32, 32, 33, + 32, 32, 34, 35, + 34, 34, 35, 37, 41, + 37, 36, 38, 39, 45, 51, + 43, 41, 42, 42, 49, 56, 63, + 47, 44, 45, 46, 52, 59, 67, 71, + }, { + 31, + 31, 32, + 34, 35, 39, + 37, 40, 43, 47, + 43, 43, 45, 47, 49, + 48, 46, 46, 47, 50, 53, + 47, 45, 45, 45, 50, 55, 58, + 49, 46, 46, 46, 50, 55, 60, 61, + }, + }, { + { + 31, + 31, 32, + 32, 32, 32, + 32, 32, 33, 34, + 33, 33, 34, 35, 37, + 34, 34, 35, 36, 39, 43, + 37, 36, 37, 38, 41, 46, 51, + 41, 39, 40, 41, 44, 49, 54, 58, + }, { + 31, + 31, 31, + 32, 33, 35, + 35, 37, 39, 43, + 39, 41, 42, 45, 47, + 45, 44, 45, 47, 48, 50, + 48, 46, 46, 47, 48, 51, 53, + 48, 46, 45, 46, 47, 51, 54, 56, + }, + }, { + { + 31, + 31, 32, + 31, 32, 32, + 32, 32, 32, 33, + 32, 32, 32, 34, 35, + 32, 33, 33, 34, 35, 36, + 34, 34, 33, 35, 36, 38, 39, + 35, 35, 34, 36, 38, 40, 42, 48, + }, { + 31, + 31, 31, + 30, 31, 32, + 34, 34, 35, 39, + 36, 37, 39, 42, 46, + 39, 40, 41, 44, 47, 47, + 42, 42, 42, 45, 47, 48, 48, + 48, 47, 46, 47, 47, 49, 50, 53, + }, + }, { + { + 31, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 32, 32, 32, 32, 33, + 32, 32, 32, 32, 33, 34, + 32, 32, 32, 32, 34, 34, 35, + 33, 33, 33, 33, 35, 35, 36, 38, + }, { + 31, + 31, 31, + 31, 31, 31, + 30, 31, 31, 32, + 34, 34, 35, 35, 39, + 35, 35, 36, 36, 40, 41, + 37, 38, 39, 40, 43, 44, 47, + 40, 41, 41, 42, 44, 45, 47, 48, + }, + }, { + { + 31, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 33, + 32, 32, 32, 32, 32, 32, 33, 33, + }, { + 31, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 30, 31, 31, 31, 32, + 31, 32, 32, 32, 32, 33, + 33, 34, 34, 35, 35, 36, 39, + 33, 34, 34, 35, 35, 36, 39, 39, + }, + }, { + { + 31, + 31, 31, + 31, 31, 31, + 31, 31, 32, 32, + 31, 31, 32, 32, 32, + 31, 31, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, + }, { + 31, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 31, 31, 31, + }, + }, +}; + +static const uint8_t qm_tbl_16x4[][2][64] = { + { + { + 31, 32, 32, 34, 34, 41, 45, 54, 60, 72, 75, 83, 88, 94, 101, 108, + 44, 41, 42, 48, 54, 63, 67, 75, 79, 90, 92, 100, 100, 101, 108, 115, + 79, 72, 71, 73, 78, 90, 96, 110, 118, 133, 136, 142, 140, 144, 141, 151, + 96, 90, 86, 83, 89, 95, 102, 111, 123, 135, 149, 160, 173, 180, 188, 197, + }, { + 31, 32, 36, 43, 46, 45, 46, 50, 52, 57, 59, 62, 63, 65, 67, 69, + 49, 45, 46, 49, 53, 58, 59, 62, 64, 67, 68, 71, 69, 68, 70, 72, + 63, 57, 56, 57, 60, 67, 71, 78, 82, 89, 90, 91, 89, 89, 86, 88, + 69, 65, 62, 60, 63, 66, 70, 74, 80, 85, 91, 96, 101, 103, 105, 107, + }, + }, { + { + 31, 32, 32, 33, 34, 37, 44, 49, 56, 65, 72, 78, 84, 89, 95, 101, + 44, 41, 42, 44, 54, 58, 66, 71, 77, 84, 90, 95, 95, 95, 101, 108, + 73, 67, 65, 66, 74, 79, 90, 99, 107, 119, 127, 133, 132, 136, 132, 141, + 93, 87, 83, 81, 86, 92, 98, 107, 117, 129, 141, 151, 163, 169, 175, 183, + }, { + 31, 32, 36, 41, 46, 46, 46, 48, 51, 54, 57, 60, 62, 64, 65, 67, + 49, 45, 46, 47, 53, 56, 59, 61, 63, 65, 67, 69, 67, 66, 68, 70, + 61, 55, 54, 54, 59, 62, 68, 73, 77, 82, 86, 88, 86, 87, 83, 86, + 69, 64, 61, 59, 62, 65, 68, 73, 78, 84, 89, 93, 98, 100, 102, 103, + }, + }, { + { + 31, 32, 32, 33, 34, 37, 41, 46, 53, 60, 65, 74, 79, 84, 89, 94, + 39, 38, 39, 40, 47, 54, 58, 62, 68, 73, 78, 85, 90, 90, 95, 101, + 65, 60, 59, 58, 65, 73, 79, 86, 97, 105, 111, 120, 125, 128, 124, 131, + 90, 84, 81, 78, 83, 89, 94, 102, 112, 123, 134, 143, 154, 158, 164, 170, + }, { + 31, 32, 36, 40, 44, 46, 45, 47, 49, 52, 54, 58, 60, 62, 64, 65, + 48, 46, 46, 46, 51, 54, 56, 57, 58, 60, 62, 64, 66, 64, 66, 68, + 57, 53, 51, 50, 54, 60, 64, 68, 73, 76, 79, 82, 84, 84, 81, 83, + 68, 63, 60, 58, 61, 64, 67, 71, 77, 82, 87, 91, 95, 97, 99, 100, + }, + }, { + { + 31, 32, 32, 33, 34, 34, 39, 44, 49, 54, 60, 68, 75, 79, 84, 88, + 36, 35, 36, 38, 42, 48, 51, 56, 60, 63, 68, 75, 81, 85, 89, 94, + 62, 58, 57, 56, 61, 66, 74, 82, 90, 95, 102, 110, 117, 120, 116, 123, + 88, 82, 79, 76, 81, 85, 91, 98, 107, 117, 127, 135, 145, 148, 153, 159, + }, { + 31, 32, 35, 40, 43, 46, 45, 46, 48, 50, 52, 56, 58, 60, 62, 63, + 48, 46, 47, 47, 50, 53, 53, 54, 54, 55, 56, 59, 61, 63, 64, 66, + 56, 52, 50, 49, 53, 56, 61, 65, 70, 72, 75, 79, 81, 82, 79, 81, + 67, 62, 60, 57, 60, 63, 66, 70, 75, 80, 85, 89, 93, 94, 96, 97, + }, + }, { + { + 31, 32, 32, 32, 33, 34, 37, 41, 45, 49, 54, 61, 68, 74, 78, 83, + 36, 35, 35, 37, 41, 48, 50, 53, 56, 60, 63, 69, 75, 80, 84, 88, + 53, 51, 49, 49, 53, 60, 65, 71, 76, 82, 87, 93, 100, 105, 109, 114, + 81, 76, 73, 71, 74, 80, 85, 91, 98, 105, 112, 121, 130, 137, 142, 148, + }, { + 31, 31, 33, 38, 42, 46, 46, 45, 46, 48, 50, 52, 56, 58, 60, 62, + 48, 47, 46, 47, 49, 53, 53, 53, 54, 54, 55, 57, 59, 61, 62, 64, + 52, 49, 48, 47, 50, 54, 57, 61, 64, 66, 68, 71, 73, 75, 76, 78, + 64, 60, 57, 56, 57, 61, 64, 68, 71, 75, 78, 83, 87, 90, 92, 94, + }, + }, { + { + 31, 32, 32, 32, 33, 34, 34, 37, 41, 45, 49, 54, 60, 65, 72, 75, + 36, 35, 34, 36, 38, 42, 48, 50, 53, 56, 60, 63, 68, 73, 79, 81, + 53, 51, 49, 50, 49, 54, 60, 65, 71, 76, 82, 87, 92, 97, 104, 106, + 79, 75, 72, 71, 69, 73, 78, 84, 90, 96, 103, 110, 118, 125, 133, 136, + }, { + 31, 31, 32, 36, 40, 43, 46, 46, 45, 46, 48, 50, 52, 54, 57, 59, + 48, 47, 46, 47, 47, 50, 53, 53, 53, 54, 54, 55, 56, 58, 60, 61, + 52, 50, 48, 47, 47, 50, 54, 57, 61, 64, 66, 68, 70, 72, 75, 75, + 63, 60, 57, 56, 54, 57, 60, 64, 67, 71, 75, 78, 82, 85, 89, 90, + }, + }, { + { + 31, 32, 32, 32, 32, 34, 34, 36, 39, 42, 45, 50, 54, 60, 66, 73, + 34, 34, 33, 35, 37, 39, 42, 44, 46, 48, 51, 54, 58, 63, 68, 74, + 44, 43, 41, 43, 43, 48, 53, 57, 60, 64, 67, 72, 76, 80, 85, 91, + 65, 62, 59, 59, 58, 63, 67, 71, 76, 81, 85, 92, 98, 105, 111, 118, + }, { + 31, 31, 32, 35, 40, 43, 46, 46, 46, 46, 47, 48, 50, 52, 55, 58, + 42, 42, 42, 45, 47, 48, 50, 50, 49, 49, 50, 50, 52, 53, 55, 58, + 49, 47, 45, 46, 46, 49, 53, 55, 57, 59, 60, 61, 63, 64, 66, 68, + 57, 54, 52, 51, 50, 53, 56, 58, 61, 64, 67, 71, 73, 76, 79, 82, + }, + }, { + { + 31, 32, 32, 32, 32, 32, 34, 35, 37, 39, 41, 45, 50, 54, 57, 61, + 32, 32, 33, 34, 34, 35, 37, 38, 40, 41, 43, 46, 50, 53, 56, 58, + 44, 42, 41, 42, 42, 42, 48, 54, 57, 60, 63, 67, 71, 74, 77, 79, + 58, 55, 53, 53, 53, 52, 57, 63, 67, 70, 74, 79, 86, 90, 93, 97, + }, { + 31, 31, 32, 34, 37, 39, 42, 47, 46, 46, 46, 47, 48, 50, 51, 53, + 37, 38, 40, 42, 44, 47, 47, 48, 47, 46, 46, 46, 47, 48, 49, 50, + 49, 47, 45, 45, 45, 45, 49, 53, 55, 57, 58, 59, 61, 62, 63, 64, + 54, 51, 49, 49, 48, 48, 51, 55, 58, 60, 62, 65, 68, 70, 71, 73, + }, + }, { + { + 31, 32, 32, 32, 32, 32, 33, 34, 35, 37, 38, 42, 45, 47, 51, 55, + 32, 32, 32, 33, 34, 35, 36, 37, 38, 40, 40, 43, 45, 47, 50, 54, + 38, 37, 36, 36, 38, 39, 41, 44, 49, 51, 52, 56, 58, 60, 63, 67, + 53, 51, 49, 49, 50, 49, 51, 54, 60, 63, 65, 71, 75, 77, 82, 87, + }, { + 31, 31, 31, 32, 35, 39, 40, 42, 47, 47, 46, 46, 47, 48, 49, 51, + 37, 38, 39, 40, 43, 47, 47, 47, 48, 47, 47, 46, 46, 47, 47, 49, + 48, 47, 46, 46, 46, 47, 48, 50, 53, 53, 54, 55, 55, 55, 56, 57, + 52, 50, 48, 48, 47, 47, 48, 50, 54, 56, 57, 61, 63, 64, 66, 68, + }, + }, { + { + 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 35, 38, 40, 42, 45, 46, + 32, 32, 32, 33, 34, 34, 35, 36, 37, 38, 38, 40, 41, 43, 45, 46, + 36, 35, 35, 34, 36, 36, 38, 40, 42, 47, 48, 50, 51, 53, 56, 56, + 44, 42, 41, 41, 42, 42, 42, 44, 48, 52, 54, 58, 60, 63, 66, 67, + }, { + 31, 31, 31, 31, 34, 35, 39, 40, 42, 46, 47, 47, 47, 46, 48, 48, + 37, 38, 39, 40, 42, 43, 47, 47, 47, 48, 48, 47, 46, 46, 46, 46, + 48, 47, 46, 46, 47, 47, 47, 48, 50, 52, 53, 53, 53, 53, 54, 54, + 49, 47, 46, 45, 45, 46, 45, 47, 49, 53, 53, 56, 57, 58, 59, 59, + }, + }, { + { + 31, 32, 32, 32, 32, 32, 32, 32, 32, 34, 34, 35, 35, 38, 38, 42, + 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 39, 39, 42, + 34, 34, 34, 33, 33, 35, 35, 37, 37, 39, 39, 43, 43, 45, 45, 48, + 39, 38, 38, 37, 37, 39, 39, 40, 40, 45, 45, 51, 51, 54, 54, 58, + }, { + 31, 31, 31, 31, 31, 34, 34, 38, 38, 42, 42, 48, 48, 47, 47, 47, + 33, 34, 34, 35, 35, 39, 39, 43, 43, 45, 45, 47, 47, 46, 46, 45, + 42, 42, 42, 42, 42, 45, 45, 47, 47, 48, 48, 50, 50, 50, 50, 49, + 48, 47, 47, 45, 45, 46, 46, 46, 46, 50, 50, 53, 53, 54, 54, 56, + }, + }, { + { + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, + 36, 35, 35, 35, 34, 34, 36, 36, 37, 38, 38, 41, 42, 44, 48, 48, + }, { + 31, 31, 31, 31, 31, 31, 34, 34, 36, 38, 38, 41, 42, 44, 48, 48, + 31, 31, 31, 32, 32, 32, 35, 36, 37, 40, 40, 42, 43, 44, 46, 46, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, + 48, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 49, 50, 51, 53, 53, + }, + }, { + { + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 35, 36, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, + }, { + 31, 31, 31, 31, 31, 31, 31, 31, 33, 34, 34, 36, 38, 38, 39, 42, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, + 35, 35, 36, 36, 36, 37, 37, 38, 40, 40, 40, 43, 45, 45, 45, 46, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, + }, + }, { + { + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, + }, { + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 34, 34, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, + 34, 34, 35, 35, 35, 35, 36, 36, 36, 36, 36, 37, 39, 40, 40, 40, + }, + }, { + { + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, { + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + }, + }, +}; + +static const uint8_t qm_tbl_16x8[][2][128] = { + { + { + 32, 31, 32, 34, 36, 44, 48, 58, 65, 79, 82, 91, 97, 103, 110, 118, + 32, 33, 34, 37, 38, 43, 46, 54, 58, 70, 72, 80, 86, 93, 100, 107, + 36, 34, 36, 42, 48, 53, 56, 63, 68, 79, 81, 88, 94, 98, 101, 105, + 53, 49, 50, 54, 60, 71, 76, 87, 92, 104, 106, 106, 107, 114, 117, 118, + 65, 59, 59, 63, 68, 79, 85, 98, 105, 118, 121, 130, 128, 131, 138, 136, + 87, 78, 77, 79, 84, 95, 102, 116, 124, 141, 144, 148, 157, 150, 161, 157, + 93, 86, 82, 80, 86, 94, 105, 112, 122, 135, 149, 162, 167, 174, 183, 182, + 99, 93, 89, 88, 90, 97, 105, 115, 124, 135, 146, 159, 171, 186, 193, 203, + }, { + 32, 30, 33, 42, 49, 49, 50, 54, 57, 63, 64, 68, 70, 72, 74, 76, + 37, 40, 43, 47, 48, 46, 46, 49, 50, 55, 56, 59, 62, 64, 67, 69, + 48, 46, 47, 50, 53, 53, 54, 55, 56, 60, 61, 64, 66, 66, 66, 67, + 52, 48, 47, 50, 54, 61, 64, 68, 70, 75, 75, 74, 73, 75, 74, 73, + 57, 52, 51, 53, 57, 64, 67, 73, 76, 82, 83, 86, 83, 83, 84, 82, + 66, 60, 59, 60, 62, 69, 73, 80, 84, 92, 93, 94, 96, 92, 94, 91, + 68, 63, 60, 59, 62, 66, 72, 76, 80, 87, 93, 98, 99, 101, 103, 101, + 71, 66, 63, 62, 62, 66, 70, 75, 79, 84, 89, 94, 98, 104, 106, 109, + }, + }, { + { + 32, 31, 32, 32, 36, 39, 47, 53, 61, 71, 79, 86, 92, 98, 104, 110, + 32, 32, 34, 35, 37, 40, 45, 50, 56, 64, 70, 76, 82, 88, 94, 100, + 36, 35, 36, 40, 48, 50, 56, 60, 65, 73, 79, 84, 89, 93, 95, 98, + 47, 44, 45, 47, 56, 60, 69, 75, 81, 89, 95, 100, 101, 108, 110, 111, + 65, 60, 59, 60, 68, 73, 84, 92, 100, 111, 118, 124, 121, 124, 129, 127, + 79, 72, 71, 71, 78, 84, 95, 103, 113, 125, 133, 140, 148, 141, 151, 147, + 90, 84, 80, 78, 83, 91, 101, 108, 116, 129, 142, 153, 157, 163, 171, 169, + 96, 90, 87, 85, 87, 94, 101, 110, 118, 129, 138, 150, 161, 174, 181, 188, + }, { + 32, 30, 33, 39, 49, 48, 50, 52, 55, 60, 63, 66, 68, 70, 72, 74, + 35, 38, 41, 46, 48, 46, 46, 47, 49, 53, 55, 58, 60, 62, 65, 67, + 48, 46, 47, 48, 53, 53, 54, 54, 56, 58, 60, 62, 64, 65, 65, 65, + 50, 46, 46, 47, 54, 56, 61, 63, 65, 68, 70, 72, 71, 73, 72, 71, + 57, 52, 51, 51, 57, 60, 66, 71, 74, 79, 82, 84, 81, 81, 82, 79, + 63, 58, 56, 55, 60, 64, 70, 75, 79, 85, 89, 91, 94, 89, 92, 89, + 68, 63, 60, 58, 61, 65, 71, 75, 79, 85, 91, 95, 97, 98, 100, 98, + 70, 65, 63, 61, 61, 65, 69, 74, 78, 82, 87, 91, 96, 101, 103, 105, + }, + }, { + { + 32, 31, 32, 32, 34, 39, 44, 49, 57, 65, 71, 81, 87, 92, 98, 103, + 32, 32, 33, 34, 36, 39, 42, 46, 53, 59, 64, 72, 77, 83, 88, 94, + 36, 35, 36, 38, 44, 50, 53, 57, 63, 68, 73, 80, 85, 88, 89, 92, + 44, 41, 42, 42, 50, 58, 63, 67, 74, 79, 84, 91, 96, 102, 103, 103, + 58, 54, 53, 52, 59, 68, 74, 81, 90, 97, 102, 110, 114, 117, 121, 119, + 79, 73, 71, 69, 75, 84, 90, 97, 108, 118, 125, 135, 140, 133, 141, 137, + 88, 81, 78, 76, 81, 88, 97, 104, 111, 123, 135, 145, 148, 153, 160, 158, + 93, 88, 84, 82, 84, 90, 97, 105, 113, 122, 131, 141, 151, 163, 169, 175, + }, { + 32, 31, 33, 37, 44, 48, 49, 51, 54, 57, 60, 64, 66, 68, 70, 72, + 34, 36, 40, 44, 46, 46, 45, 47, 49, 51, 53, 57, 59, 61, 63, 65, + 48, 46, 47, 47, 51, 53, 53, 54, 55, 56, 58, 61, 63, 63, 63, 63, + 49, 46, 46, 45, 51, 56, 58, 60, 62, 64, 65, 68, 69, 71, 70, 69, + 54, 50, 49, 48, 53, 58, 62, 65, 70, 73, 75, 78, 79, 79, 80, 77, + 63, 58, 56, 54, 59, 64, 67, 71, 77, 82, 85, 89, 91, 87, 89, 86, + 67, 62, 59, 57, 60, 64, 70, 73, 77, 83, 89, 93, 94, 96, 97, 95, + 69, 65, 62, 60, 61, 64, 68, 72, 76, 81, 85, 89, 93, 98, 100, 102, + }, + }, { + { + 32, 31, 31, 32, 34, 36, 41, 47, 53, 58, 65, 74, 82, 87, 92, 97, + 31, 32, 33, 34, 35, 36, 40, 44, 50, 54, 59, 67, 73, 78, 83, 88, + 35, 34, 35, 37, 41, 46, 49, 53, 57, 61, 66, 73, 79, 83, 84, 86, + 44, 41, 42, 42, 48, 54, 60, 66, 71, 75, 79, 86, 92, 96, 97, 97, + 53, 50, 49, 49, 54, 60, 67, 75, 82, 87, 92, 100, 105, 110, 114, 111, + 65, 61, 59, 58, 63, 68, 76, 84, 92, 98, 105, 113, 120, 125, 132, 128, + 82, 76, 73, 71, 76, 80, 88, 97, 106, 112, 120, 131, 139, 144, 150, 147, + 90, 85, 81, 79, 81, 87, 93, 101, 108, 116, 124, 134, 142, 153, 157, 163, + }, { + 32, 31, 33, 37, 42, 49, 48, 50, 52, 54, 57, 61, 64, 66, 68, 70, + 33, 34, 37, 43, 44, 47, 46, 46, 47, 49, 51, 55, 57, 59, 61, 63, + 45, 45, 46, 47, 49, 52, 51, 52, 53, 54, 55, 58, 60, 61, 61, 61, + 49, 46, 45, 45, 49, 53, 57, 59, 61, 62, 64, 66, 68, 69, 68, 67, + 52, 49, 47, 47, 50, 54, 59, 63, 66, 68, 70, 73, 75, 77, 77, 75, + 57, 53, 51, 50, 53, 57, 61, 66, 71, 73, 76, 80, 83, 84, 86, 83, + 64, 60, 57, 55, 58, 61, 66, 71, 75, 79, 83, 87, 91, 93, 94, 92, + 68, 64, 61, 59, 60, 63, 67, 71, 74, 79, 83, 87, 91, 95, 97, 98, + }, + }, { + { + 32, 31, 31, 32, 33, 36, 39, 44, 48, 53, 58, 66, 74, 81, 86, 91, + 31, 32, 32, 33, 34, 35, 38, 41, 45, 49, 54, 60, 67, 73, 78, 82, + 33, 33, 34, 36, 38, 42, 44, 46, 50, 53, 57, 63, 69, 75, 78, 80, + 40, 39, 38, 40, 44, 51, 54, 59, 62, 66, 70, 75, 81, 86, 90, 90, + 51, 49, 47, 48, 52, 58, 63, 69, 74, 79, 84, 90, 97, 102, 106, 103, + 65, 61, 59, 58, 62, 68, 73, 79, 85, 92, 98, 106, 113, 120, 124, 119, + 79, 74, 71, 69, 72, 78, 84, 90, 96, 103, 110, 119, 128, 135, 140, 137, + 87, 82, 79, 77, 78, 84, 89, 96, 103, 111, 118, 126, 134, 143, 147, 151, + }, { + 32, 31, 31, 35, 41, 49, 48, 49, 50, 52, 54, 57, 61, 64, 66, 68, + 32, 33, 35, 39, 43, 47, 46, 45, 46, 48, 50, 52, 55, 58, 59, 61, + 40, 41, 43, 46, 48, 50, 49, 48, 49, 50, 51, 53, 56, 58, 59, 59, + 49, 47, 46, 46, 49, 53, 54, 56, 57, 58, 59, 61, 63, 65, 66, 65, + 51, 49, 47, 47, 49, 54, 57, 61, 63, 65, 67, 69, 72, 73, 75, 72, + 57, 54, 51, 50, 52, 57, 60, 64, 67, 71, 73, 77, 80, 82, 84, 81, + 63, 59, 57, 55, 57, 60, 64, 67, 71, 75, 78, 82, 86, 89, 91, 89, + 67, 63, 60, 58, 59, 62, 65, 69, 73, 77, 81, 85, 88, 92, 94, 95, + }, + }, { + { + 32, 31, 31, 32, 32, 34, 36, 39, 44, 48, 53, 58, 65, 71, 79, 82, + 31, 32, 32, 32, 33, 34, 34, 37, 41, 45, 49, 54, 60, 65, 72, 75, + 32, 32, 33, 34, 35, 37, 38, 40, 43, 46, 50, 54, 58, 63, 70, 72, + 36, 35, 34, 36, 38, 42, 48, 50, 53, 56, 60, 63, 68, 73, 79, 81, + 44, 42, 41, 42, 42, 48, 54, 58, 63, 67, 71, 75, 79, 84, 90, 92, + 53, 51, 49, 50, 49, 54, 60, 65, 71, 76, 82, 87, 92, 97, 104, 106, + 65, 62, 59, 59, 58, 63, 68, 73, 79, 85, 92, 98, 105, 111, 118, 121, + 79, 75, 72, 71, 69, 73, 78, 84, 90, 96, 103, 110, 118, 125, 133, 136, + }, { + 32, 31, 30, 33, 37, 42, 49, 48, 49, 50, 52, 54, 57, 60, 63, 64, + 31, 31, 32, 36, 40, 43, 46, 46, 45, 46, 48, 50, 52, 54, 57, 59, + 37, 38, 40, 43, 47, 47, 48, 47, 46, 46, 47, 49, 50, 52, 55, 56, + 48, 47, 46, 47, 47, 50, 53, 53, 53, 54, 54, 55, 56, 58, 60, 61, + 49, 47, 45, 46, 45, 49, 53, 56, 58, 59, 61, 62, 64, 65, 67, 68, + 52, 50, 48, 47, 47, 50, 54, 57, 61, 64, 66, 68, 70, 72, 75, 75, + 57, 54, 52, 51, 50, 53, 57, 60, 64, 67, 71, 73, 76, 79, 82, 83, + 63, 60, 57, 56, 54, 57, 60, 64, 67, 71, 75, 78, 82, 85, 89, 90, + }, + }, { + { + 32, 31, 31, 32, 32, 34, 35, 38, 41, 44, 48, 53, 58, 65, 71, 79, + 31, 32, 32, 32, 33, 34, 34, 36, 39, 42, 45, 49, 54, 60, 65, 72, + 32, 32, 33, 34, 35, 37, 38, 40, 41, 43, 46, 50, 54, 58, 63, 70, + 36, 35, 34, 36, 38, 42, 47, 49, 51, 54, 56, 60, 63, 68, 73, 79, + 44, 42, 41, 42, 42, 48, 52, 56, 60, 64, 67, 71, 75, 79, 84, 90, + 53, 51, 49, 50, 49, 54, 59, 63, 67, 72, 76, 82, 87, 92, 97, 104, + 62, 59, 57, 57, 56, 61, 65, 69, 74, 79, 83, 90, 95, 102, 108, 115, + 73, 69, 66, 65, 64, 69, 73, 77, 81, 86, 91, 99, 105, 112, 119, 127, + }, { + 32, 31, 30, 33, 37, 42, 47, 48, 48, 49, 50, 52, 54, 57, 60, 63, + 31, 31, 32, 36, 40, 43, 46, 46, 45, 45, 46, 48, 50, 52, 54, 57, + 37, 38, 40, 43, 47, 47, 48, 47, 46, 46, 46, 47, 49, 50, 52, 55, + 48, 47, 46, 47, 47, 50, 52, 53, 53, 53, 54, 54, 55, 56, 58, 60, + 49, 47, 45, 46, 45, 49, 53, 55, 57, 58, 59, 61, 62, 64, 65, 67, + 52, 50, 48, 47, 47, 50, 53, 56, 59, 62, 64, 66, 68, 70, 72, 75, + 56, 53, 51, 50, 49, 53, 55, 58, 61, 64, 66, 70, 72, 75, 77, 80, + 61, 57, 55, 54, 52, 56, 58, 61, 63, 66, 69, 73, 76, 79, 82, 86, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 34, 36, 38, 41, 44, 48, 53, 57, 61, 65, + 31, 32, 32, 32, 32, 33, 34, 34, 37, 39, 41, 45, 49, 53, 56, 60, + 32, 32, 33, 34, 34, 35, 37, 38, 40, 41, 43, 46, 50, 53, 56, 58, + 35, 35, 34, 35, 36, 37, 41, 46, 47, 49, 51, 54, 57, 60, 63, 66, + 39, 38, 37, 38, 39, 40, 44, 50, 52, 54, 57, 60, 64, 67, 69, 72, + 44, 42, 41, 42, 42, 42, 48, 54, 57, 60, 63, 67, 71, 74, 77, 79, + 53, 51, 49, 49, 49, 49, 54, 60, 64, 67, 71, 76, 82, 86, 89, 92, + 65, 62, 59, 59, 58, 58, 63, 68, 72, 76, 79, 85, 92, 97, 100, 105, + }, { + 32, 31, 30, 33, 35, 37, 42, 49, 48, 48, 49, 50, 52, 54, 55, 57, + 31, 31, 32, 35, 37, 40, 43, 46, 46, 45, 45, 46, 48, 49, 51, 52, + 37, 38, 40, 42, 44, 47, 47, 48, 47, 46, 46, 46, 47, 48, 49, 50, + 45, 45, 44, 46, 46, 47, 49, 52, 51, 51, 51, 52, 53, 54, 54, 55, + 48, 47, 45, 46, 46, 47, 50, 53, 54, 54, 55, 56, 57, 58, 58, 59, + 49, 47, 45, 45, 45, 45, 49, 53, 55, 57, 58, 59, 61, 62, 63, 64, + 52, 50, 48, 47, 47, 47, 50, 54, 57, 59, 61, 64, 66, 68, 69, 70, + 57, 54, 52, 51, 51, 50, 53, 57, 59, 61, 64, 67, 71, 73, 74, 76, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 32, 34, 36, 38, 39, 44, 47, 49, 53, 58, + 31, 32, 32, 32, 32, 33, 33, 34, 34, 36, 37, 41, 44, 46, 49, 54, + 32, 32, 32, 33, 34, 35, 35, 36, 37, 39, 40, 42, 45, 47, 50, 54, + 32, 33, 33, 33, 34, 36, 36, 38, 40, 41, 42, 45, 47, 48, 51, 55, + 36, 35, 35, 35, 36, 38, 40, 42, 48, 49, 50, 53, 56, 57, 60, 63, + 44, 42, 41, 41, 42, 42, 44, 48, 54, 56, 58, 63, 66, 67, 71, 75, + 47, 45, 44, 44, 45, 45, 47, 50, 56, 58, 60, 66, 69, 71, 75, 79, + 53, 51, 49, 49, 50, 49, 51, 54, 60, 63, 65, 71, 75, 77, 82, 87, + }, { + 32, 31, 30, 31, 33, 37, 39, 42, 49, 48, 48, 49, 50, 51, 52, 54, + 31, 31, 32, 33, 36, 40, 41, 43, 46, 46, 46, 45, 46, 47, 48, 50, + 35, 37, 38, 38, 41, 45, 46, 46, 48, 47, 46, 45, 46, 47, 47, 49, + 38, 40, 40, 41, 44, 47, 47, 48, 49, 48, 48, 47, 48, 48, 48, 50, + 48, 47, 46, 46, 47, 47, 48, 50, 53, 53, 53, 53, 54, 54, 54, 55, + 49, 47, 45, 45, 46, 45, 47, 49, 53, 55, 56, 58, 59, 60, 61, 62, + 50, 48, 46, 46, 46, 46, 47, 50, 54, 55, 56, 59, 61, 61, 63, 65, + 52, 50, 48, 48, 47, 47, 48, 50, 54, 56, 57, 61, 63, 64, 66, 68, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 34, 35, 36, 39, 41, 44, 47, 48, + 31, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 37, 39, 41, 44, 45, + 31, 32, 32, 32, 33, 33, 34, 34, 35, 36, 36, 39, 40, 42, 44, 45, + 32, 32, 32, 33, 34, 34, 35, 36, 37, 38, 38, 40, 41, 43, 45, 46, + 35, 35, 34, 34, 35, 36, 37, 39, 41, 45, 46, 48, 49, 51, 53, 54, + 36, 35, 35, 34, 36, 36, 38, 40, 42, 47, 48, 50, 51, 53, 56, 56, + 44, 42, 41, 41, 42, 42, 42, 44, 48, 52, 54, 58, 60, 63, 66, 67, + 47, 45, 45, 44, 44, 45, 45, 47, 50, 55, 56, 60, 62, 66, 69, 70, + }, { + 32, 31, 31, 30, 33, 33, 37, 39, 42, 47, 49, 48, 48, 49, 50, 50, + 31, 31, 32, 32, 35, 36, 40, 41, 43, 46, 46, 46, 45, 45, 46, 46, + 33, 34, 34, 35, 37, 38, 43, 43, 44, 46, 47, 46, 46, 45, 46, 46, + 37, 38, 39, 40, 42, 43, 47, 47, 47, 48, 48, 47, 46, 46, 46, 46, + 45, 45, 45, 44, 46, 46, 47, 48, 49, 51, 52, 51, 51, 51, 52, 52, + 48, 47, 46, 46, 47, 47, 47, 48, 50, 52, 53, 53, 53, 53, 54, 54, + 49, 47, 46, 45, 45, 46, 45, 47, 49, 53, 53, 56, 57, 58, 59, 59, + 50, 48, 47, 46, 46, 46, 46, 47, 50, 53, 54, 56, 57, 59, 61, 61, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 32, 34, 34, 36, 36, 39, 39, 44, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 37, 37, 41, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 37, 37, 41, + 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 43, + 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 43, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, 50, 50, 53, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, 50, 50, 53, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 48, 48, 54, 54, 58, 58, 63, + }, { + 32, 31, 31, 30, 30, 33, 33, 37, 37, 42, 42, 49, 49, 48, 48, 49, + 31, 31, 31, 32, 32, 36, 36, 40, 40, 43, 43, 46, 46, 46, 46, 45, + 31, 31, 31, 32, 32, 36, 36, 40, 40, 43, 43, 46, 46, 46, 46, 45, + 37, 38, 38, 40, 40, 43, 43, 47, 47, 47, 47, 48, 48, 47, 47, 46, + 37, 38, 38, 40, 40, 43, 43, 47, 47, 47, 47, 48, 48, 47, 47, 46, + 48, 47, 47, 46, 46, 47, 47, 47, 47, 50, 50, 53, 53, 53, 53, 53, + 48, 47, 47, 46, 46, 47, 47, 47, 47, 50, 50, 53, 53, 53, 53, 53, + 49, 47, 47, 45, 45, 46, 46, 45, 45, 49, 49, 53, 53, 56, 56, 58, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 33, 34, 34, 36, 36, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, + 31, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 35, 35, 35, 36, 36, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, + 35, 35, 35, 34, 34, 34, 35, 36, 36, 37, 37, 40, 41, 43, 46, 46, + 36, 35, 35, 35, 34, 34, 36, 36, 37, 38, 38, 41, 42, 44, 48, 48, + }, { + 32, 31, 31, 31, 30, 30, 33, 33, 35, 37, 37, 41, 42, 44, 49, 49, + 31, 31, 31, 31, 32, 32, 34, 35, 37, 39, 39, 42, 42, 44, 47, 47, + 31, 31, 31, 32, 32, 32, 35, 36, 37, 40, 40, 42, 43, 44, 46, 46, + 33, 34, 34, 34, 35, 35, 37, 38, 40, 43, 43, 44, 44, 45, 47, 47, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, + 45, 45, 45, 45, 44, 44, 46, 46, 46, 47, 47, 49, 49, 50, 52, 52, + 48, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 49, 50, 51, 53, 53, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 35, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, + 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 35, 36, 36, 36, 38, + }, { + 32, 31, 31, 31, 31, 30, 30, 31, 33, 33, 33, 35, 37, 37, 39, 42, + 31, 31, 31, 31, 31, 31, 31, 32, 34, 35, 35, 37, 39, 39, 40, 42, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, + 33, 33, 34, 34, 34, 35, 35, 35, 37, 38, 38, 41, 43, 43, 43, 44, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, + 38, 39, 40, 40, 40, 41, 41, 41, 43, 44, 44, 46, 47, 47, 47, 48, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, + }, { + 32, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 31, 33, 33, 33, 33, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 34, 34, 34, 34, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 33, 34, 35, 35, 35, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, + 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 36, 37, 38, 38, 38, + 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 40, 41, 41, 41, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + }, + }, +}; + +static const uint8_t qm_tbl_32x8[][2][256] = { + { + { + 32, 31, 31, 31, 32, 32, 34, 35, 36, 39, 44, 46, 48, 53, 58, 61, 65, 71, 79, 81, 82, 88, 91, 94, 97, 100, 103, 107, 110, 114, 118, 122, + 32, 32, 33, 33, 34, 35, 37, 37, 38, 40, 43, 44, 46, 50, 54, 56, 58, 63, 70, 71, 72, 77, 80, 83, 86, 89, 93, 96, 100, 104, 107, 111, + 36, 35, 34, 35, 36, 38, 42, 45, 48, 50, 53, 55, 56, 60, 63, 66, 68, 73, 79, 80, 81, 85, 88, 91, 94, 97, 98, 100, 101, 103, 105, 107, + 53, 51, 49, 49, 50, 49, 54, 57, 60, 65, 71, 73, 76, 82, 87, 89, 92, 97, 104, 105, 106, 108, 106, 105, 107, 111, 114, 117, 117, 117, 118, 119, + 65, 62, 59, 59, 59, 58, 63, 65, 68, 73, 79, 82, 85, 92, 98, 101, 105, 111, 118, 119, 121, 126, 130, 131, 128, 127, 131, 136, 138, 137, 136, 136, + 87, 82, 78, 78, 77, 75, 79, 82, 84, 89, 95, 98, 102, 109, 116, 120, 124, 132, 141, 142, 144, 149, 148, 153, 157, 152, 150, 155, 161, 159, 157, 156, + 93, 88, 86, 84, 82, 82, 80, 84, 86, 91, 94, 98, 105, 107, 112, 119, 122, 130, 135, 140, 149, 153, 162, 165, 167, 173, 174, 177, 183, 185, 182, 179, + 99, 94, 93, 90, 89, 89, 88, 87, 90, 93, 97, 99, 105, 107, 115, 116, 124, 127, 135, 139, 146, 152, 159, 166, 171, 182, 186, 191, 193, 201, 203, 204, + }, { + 32, 31, 30, 32, 33, 37, 42, 45, 49, 48, 49, 49, 50, 52, 54, 55, 57, 60, 63, 64, 64, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 37, 38, 40, 41, 43, 47, 47, 47, 48, 47, 46, 46, 46, 47, 49, 49, 50, 52, 55, 55, 56, 58, 59, 60, 62, 63, 64, 65, 67, 68, 69, 70, + 48, 47, 46, 46, 47, 47, 50, 51, 53, 53, 53, 53, 54, 54, 55, 56, 56, 58, 60, 61, 61, 63, 64, 65, 66, 67, 66, 66, 66, 66, 67, 67, + 52, 50, 48, 48, 47, 47, 50, 52, 54, 57, 61, 62, 64, 66, 68, 69, 70, 72, 75, 75, 75, 76, 74, 72, 73, 74, 75, 75, 74, 74, 73, 73, + 57, 54, 52, 51, 51, 50, 53, 55, 57, 60, 64, 65, 67, 71, 73, 75, 76, 79, 82, 82, 83, 85, 86, 85, 83, 82, 83, 84, 84, 83, 82, 81, + 66, 63, 60, 59, 59, 57, 60, 61, 62, 66, 69, 71, 73, 77, 80, 82, 84, 88, 92, 92, 93, 95, 94, 95, 96, 93, 92, 93, 94, 93, 91, 90, + 68, 65, 63, 62, 60, 60, 59, 61, 62, 65, 66, 68, 72, 73, 76, 79, 80, 84, 87, 89, 93, 94, 98, 99, 99, 102, 101, 102, 103, 103, 101, 99, + 71, 67, 66, 64, 63, 62, 62, 61, 62, 64, 66, 67, 70, 71, 75, 76, 79, 81, 84, 86, 89, 91, 94, 97, 98, 102, 104, 106, 106, 109, 109, 108, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 32, 34, 36, 38, 39, 44, 47, 49, 53, 58, 61, 65, 71, 76, 79, 82, 86, 89, 92, 95, 98, 101, 104, 107, 110, 114, + 32, 32, 32, 33, 34, 35, 35, 36, 37, 39, 40, 42, 45, 47, 50, 54, 56, 59, 64, 68, 70, 73, 76, 79, 82, 85, 88, 91, 94, 97, 100, 104, + 36, 35, 35, 35, 36, 38, 40, 42, 48, 49, 50, 53, 56, 57, 60, 63, 65, 68, 73, 76, 79, 81, 84, 87, 89, 92, 93, 94, 95, 96, 98, 100, + 47, 45, 44, 44, 45, 45, 47, 50, 56, 58, 60, 66, 69, 71, 75, 79, 81, 84, 89, 92, 95, 97, 100, 99, 101, 105, 108, 110, 110, 110, 111, 111, + 65, 62, 60, 59, 59, 58, 60, 63, 68, 71, 73, 79, 84, 86, 92, 98, 100, 105, 111, 115, 118, 121, 124, 124, 121, 120, 124, 128, 129, 128, 127, 127, + 79, 75, 72, 71, 71, 69, 71, 73, 78, 81, 84, 90, 95, 97, 103, 110, 113, 118, 125, 130, 133, 136, 140, 145, 148, 143, 141, 146, 151, 149, 147, 145, + 90, 86, 84, 82, 80, 80, 78, 82, 83, 88, 91, 94, 101, 103, 108, 114, 116, 124, 129, 134, 142, 145, 153, 156, 157, 163, 163, 166, 171, 173, 169, 166, + 96, 91, 90, 87, 87, 86, 85, 84, 87, 90, 94, 96, 101, 102, 110, 111, 118, 121, 129, 132, 138, 144, 150, 156, 161, 171, 174, 179, 181, 188, 188, 190, + }, { + 32, 31, 30, 31, 33, 37, 39, 42, 49, 48, 48, 49, 50, 51, 52, 54, 55, 57, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, + 35, 37, 38, 38, 41, 45, 46, 46, 48, 47, 46, 45, 46, 47, 47, 49, 49, 50, 53, 54, 55, 56, 58, 59, 60, 61, 62, 64, 65, 66, 67, 68, + 48, 47, 46, 46, 47, 47, 48, 50, 53, 53, 53, 53, 54, 54, 54, 55, 56, 56, 58, 60, 60, 61, 62, 63, 64, 65, 65, 65, 65, 65, 65, 65, + 50, 48, 46, 46, 46, 46, 47, 50, 54, 55, 56, 59, 61, 61, 63, 65, 65, 66, 68, 69, 70, 71, 72, 71, 71, 72, 73, 73, 72, 72, 71, 71, + 57, 54, 52, 52, 51, 50, 51, 53, 57, 58, 60, 64, 66, 68, 71, 73, 74, 76, 79, 81, 82, 83, 84, 83, 81, 80, 81, 82, 82, 81, 79, 78, + 63, 60, 58, 57, 56, 54, 55, 57, 60, 62, 64, 67, 70, 71, 75, 78, 79, 82, 85, 87, 89, 90, 91, 93, 94, 91, 89, 90, 92, 90, 89, 87, + 68, 64, 63, 61, 60, 59, 58, 60, 61, 64, 65, 67, 71, 72, 75, 78, 79, 83, 85, 87, 91, 92, 95, 96, 97, 99, 98, 99, 100, 100, 98, 96, + 70, 66, 65, 63, 63, 62, 61, 60, 61, 63, 65, 66, 69, 70, 74, 74, 78, 79, 82, 84, 87, 89, 91, 94, 96, 100, 101, 103, 103, 105, 105, 105, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 32, 34, 34, 36, 39, 40, 44, 47, 49, 53, 57, 59, 65, 69, 71, 79, 81, 82, 87, 90, 92, 95, 98, 100, 103, 106, + 32, 32, 32, 32, 33, 34, 34, 35, 36, 37, 39, 40, 42, 45, 46, 50, 53, 54, 59, 62, 64, 71, 72, 73, 77, 80, 83, 85, 88, 91, 94, 97, + 36, 35, 35, 34, 36, 37, 38, 42, 44, 48, 50, 51, 53, 56, 57, 60, 63, 64, 68, 71, 73, 79, 80, 81, 85, 87, 88, 88, 89, 90, 92, 93, + 44, 42, 41, 41, 42, 42, 42, 48, 50, 54, 58, 59, 63, 66, 67, 71, 74, 75, 79, 83, 84, 90, 91, 92, 96, 99, 102, 103, 103, 103, 103, 104, + 58, 55, 54, 53, 53, 53, 52, 57, 59, 63, 68, 70, 74, 79, 81, 86, 90, 91, 97, 100, 102, 109, 110, 111, 114, 113, 117, 120, 121, 120, 119, 118, + 79, 75, 73, 72, 71, 70, 69, 73, 75, 78, 84, 85, 90, 95, 97, 103, 108, 111, 118, 122, 125, 133, 135, 136, 140, 135, 133, 137, 141, 139, 137, 135, + 88, 83, 81, 79, 78, 77, 76, 79, 81, 85, 88, 91, 97, 99, 104, 109, 111, 119, 123, 127, 135, 137, 145, 147, 148, 153, 153, 155, 160, 161, 158, 155, + 93, 88, 88, 84, 84, 83, 82, 81, 84, 86, 90, 92, 97, 98, 105, 106, 113, 115, 122, 125, 131, 136, 141, 147, 151, 160, 163, 168, 169, 175, 175, 176, + }, { + 32, 31, 31, 30, 33, 35, 37, 42, 44, 49, 48, 48, 49, 50, 51, 52, 54, 54, 57, 59, 60, 63, 64, 64, 66, 67, 68, 69, 70, 71, 72, 73, + 34, 35, 36, 36, 40, 42, 44, 45, 46, 47, 46, 46, 45, 46, 47, 47, 49, 49, 51, 52, 53, 56, 57, 57, 59, 60, 61, 62, 63, 64, 65, 66, + 48, 47, 46, 46, 47, 47, 47, 50, 51, 53, 53, 53, 53, 54, 54, 54, 55, 55, 56, 58, 58, 60, 61, 61, 63, 63, 63, 63, 63, 63, 63, 63, + 49, 47, 46, 45, 46, 45, 45, 49, 51, 53, 56, 56, 58, 59, 60, 61, 62, 62, 64, 65, 65, 67, 68, 68, 69, 70, 71, 71, 70, 70, 69, 69, + 54, 51, 50, 49, 49, 48, 48, 51, 53, 55, 58, 59, 62, 65, 65, 68, 70, 70, 73, 74, 75, 77, 78, 78, 79, 78, 79, 80, 80, 78, 77, 76, + 63, 60, 58, 57, 56, 55, 54, 57, 59, 60, 64, 65, 67, 70, 71, 75, 77, 78, 82, 84, 85, 89, 89, 90, 91, 88, 87, 88, 89, 88, 86, 84, + 67, 63, 62, 60, 59, 58, 57, 59, 60, 63, 64, 66, 70, 70, 73, 76, 77, 81, 83, 85, 89, 90, 93, 94, 94, 96, 96, 96, 97, 97, 95, 93, + 69, 65, 65, 62, 62, 61, 60, 59, 61, 62, 64, 65, 68, 68, 72, 73, 76, 77, 81, 82, 85, 87, 89, 92, 93, 97, 98, 100, 100, 102, 102, 101, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 34, 35, 36, 39, 41, 44, 47, 48, 53, 55, 58, 63, 65, 71, 74, 79, 82, 82, 87, 89, 92, 94, 97, 99, + 31, 32, 32, 32, 33, 33, 34, 34, 35, 36, 36, 39, 40, 42, 44, 45, 50, 51, 54, 58, 59, 64, 67, 71, 73, 74, 78, 81, 83, 85, 88, 91, + 35, 35, 34, 34, 35, 36, 37, 39, 41, 45, 46, 48, 49, 51, 53, 54, 57, 59, 61, 65, 66, 71, 73, 77, 79, 79, 83, 83, 84, 85, 86, 87, + 44, 42, 41, 41, 42, 42, 42, 44, 48, 52, 54, 58, 60, 63, 66, 67, 71, 72, 75, 78, 79, 84, 86, 90, 92, 92, 96, 97, 97, 97, 97, 97, + 53, 51, 50, 49, 49, 50, 49, 51, 54, 59, 60, 65, 67, 71, 75, 76, 82, 84, 87, 91, 92, 97, 100, 104, 105, 106, 110, 113, 114, 112, 111, 110, + 65, 62, 61, 59, 59, 59, 58, 60, 63, 67, 68, 73, 76, 79, 84, 85, 92, 94, 98, 103, 105, 111, 113, 118, 120, 121, 125, 128, 132, 130, 128, 126, + 82, 78, 76, 74, 73, 73, 71, 73, 76, 79, 80, 86, 88, 92, 97, 98, 106, 108, 112, 118, 120, 127, 131, 136, 139, 139, 144, 145, 150, 151, 147, 144, + 90, 86, 85, 82, 81, 80, 79, 78, 81, 83, 87, 88, 93, 94, 101, 101, 108, 110, 116, 119, 124, 129, 134, 139, 142, 150, 153, 157, 157, 163, 163, 163, + }, { + 32, 31, 31, 30, 33, 33, 37, 39, 42, 47, 49, 48, 48, 49, 50, 50, 52, 53, 54, 56, 57, 60, 61, 63, 64, 64, 66, 67, 68, 69, 70, 70, + 33, 34, 34, 35, 37, 38, 43, 43, 44, 46, 47, 46, 46, 45, 46, 46, 47, 48, 49, 51, 51, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 45, 45, 45, 44, 46, 46, 47, 48, 49, 51, 52, 51, 51, 51, 52, 52, 53, 53, 54, 55, 55, 57, 58, 59, 60, 60, 61, 61, 61, 61, 61, 61, + 49, 47, 46, 45, 45, 46, 45, 47, 49, 53, 53, 56, 57, 58, 59, 59, 61, 61, 62, 63, 64, 65, 66, 67, 68, 68, 69, 69, 68, 68, 67, 67, + 52, 50, 49, 48, 47, 47, 47, 48, 50, 53, 54, 57, 59, 61, 63, 64, 66, 67, 68, 70, 70, 72, 73, 75, 75, 75, 77, 78, 77, 76, 75, 74, + 57, 54, 53, 52, 51, 51, 50, 51, 53, 56, 57, 60, 61, 64, 66, 67, 71, 72, 73, 76, 76, 79, 80, 82, 83, 83, 84, 85, 86, 85, 83, 82, + 64, 61, 60, 58, 57, 57, 55, 56, 58, 61, 61, 64, 66, 68, 71, 71, 75, 77, 79, 82, 83, 86, 87, 90, 91, 91, 93, 93, 94, 94, 92, 90, + 68, 64, 64, 61, 61, 60, 59, 58, 60, 61, 63, 64, 67, 67, 71, 71, 74, 75, 79, 80, 83, 85, 87, 89, 91, 94, 95, 97, 97, 99, 98, 98, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 33, 34, 36, 36, 39, 40, 44, 46, 48, 52, 53, 58, 58, 65, 66, 71, 74, 79, 81, 82, 86, 88, 91, 93, + 31, 32, 32, 32, 32, 33, 33, 33, 34, 34, 35, 35, 38, 39, 41, 43, 45, 48, 49, 53, 54, 59, 60, 65, 67, 72, 73, 74, 78, 80, 82, 85, + 33, 33, 33, 33, 34, 35, 36, 36, 38, 39, 42, 42, 44, 45, 46, 48, 50, 52, 53, 57, 57, 62, 63, 67, 69, 73, 75, 75, 78, 80, 80, 81, + 40, 39, 39, 38, 38, 39, 40, 41, 44, 45, 51, 51, 54, 56, 59, 60, 62, 65, 66, 69, 70, 74, 75, 79, 81, 85, 86, 87, 90, 90, 90, 90, + 51, 49, 49, 47, 47, 48, 48, 48, 52, 53, 58, 59, 63, 65, 69, 72, 74, 78, 79, 83, 84, 89, 90, 94, 97, 101, 102, 103, 106, 105, 103, 103, + 65, 62, 61, 59, 59, 59, 58, 58, 62, 63, 68, 68, 73, 75, 79, 82, 85, 90, 92, 97, 98, 105, 106, 111, 113, 118, 120, 121, 124, 122, 119, 117, + 79, 75, 74, 72, 71, 71, 69, 69, 72, 73, 78, 79, 84, 85, 90, 93, 96, 101, 103, 109, 110, 118, 119, 125, 128, 133, 135, 136, 140, 140, 137, 134, + 87, 83, 82, 79, 79, 78, 77, 75, 78, 80, 84, 85, 89, 90, 96, 97, 103, 105, 111, 113, 118, 122, 126, 131, 134, 141, 143, 147, 147, 152, 151, 152, + }, { + 32, 31, 31, 30, 31, 33, 35, 37, 41, 42, 49, 49, 48, 48, 49, 49, 50, 51, 52, 54, 54, 57, 57, 60, 61, 63, 64, 64, 66, 67, 68, 68, + 32, 33, 33, 33, 35, 37, 39, 41, 43, 43, 47, 47, 46, 46, 45, 46, 46, 47, 48, 49, 50, 52, 52, 54, 55, 57, 58, 58, 59, 60, 61, 62, + 40, 41, 41, 42, 43, 44, 46, 47, 48, 48, 50, 50, 49, 49, 48, 49, 49, 49, 50, 51, 51, 52, 53, 55, 56, 57, 58, 58, 59, 59, 59, 59, + 49, 47, 47, 45, 46, 46, 46, 46, 49, 49, 53, 53, 54, 55, 56, 57, 57, 58, 58, 59, 59, 60, 61, 62, 63, 64, 65, 65, 66, 66, 65, 65, + 51, 49, 49, 47, 47, 47, 47, 46, 49, 50, 54, 54, 57, 58, 61, 62, 63, 64, 65, 67, 67, 69, 69, 71, 72, 73, 73, 74, 75, 74, 72, 71, + 57, 54, 54, 52, 51, 51, 50, 50, 52, 53, 57, 57, 60, 61, 64, 65, 67, 69, 71, 73, 73, 76, 77, 79, 80, 82, 82, 83, 84, 82, 81, 79, + 63, 60, 59, 57, 57, 56, 55, 54, 57, 57, 60, 61, 64, 65, 67, 69, 71, 73, 75, 77, 78, 82, 82, 85, 86, 89, 89, 90, 91, 91, 89, 87, + 67, 63, 63, 60, 60, 59, 58, 57, 59, 60, 62, 63, 65, 66, 69, 70, 73, 74, 77, 78, 81, 83, 85, 87, 88, 92, 92, 94, 94, 96, 95, 95, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 32, 34, 34, 36, 36, 39, 39, 44, 44, 48, 48, 53, 53, 58, 58, 65, 65, 71, 71, 79, 79, 82, 82, 87, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 37, 37, 41, 41, 45, 45, 49, 49, 54, 54, 60, 60, 65, 65, 72, 72, 75, 75, 79, + 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 43, 43, 46, 46, 50, 50, 54, 54, 58, 58, 63, 63, 70, 70, 72, 72, 76, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, 50, 50, 53, 53, 56, 56, 60, 60, 63, 63, 68, 68, 73, 73, 79, 79, 81, 81, 84, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 48, 48, 54, 54, 58, 58, 63, 63, 67, 67, 71, 71, 75, 75, 79, 79, 84, 84, 90, 90, 92, 92, 96, + 53, 51, 51, 49, 49, 50, 50, 49, 49, 54, 54, 60, 60, 65, 65, 71, 71, 76, 76, 82, 82, 87, 87, 92, 92, 97, 97, 104, 104, 106, 106, 109, + 65, 62, 62, 59, 59, 59, 59, 58, 58, 63, 63, 68, 68, 73, 73, 79, 79, 85, 85, 92, 92, 98, 98, 105, 105, 111, 111, 118, 118, 121, 121, 124, + 79, 75, 75, 72, 72, 71, 71, 69, 69, 73, 73, 78, 78, 84, 84, 90, 90, 96, 96, 103, 103, 110, 110, 118, 118, 125, 125, 133, 133, 136, 136, 141, + }, { + 32, 31, 31, 30, 30, 33, 33, 37, 37, 42, 42, 49, 49, 48, 48, 49, 49, 50, 50, 52, 52, 54, 54, 57, 57, 60, 60, 63, 63, 64, 64, 66, + 31, 31, 31, 32, 32, 36, 36, 40, 40, 43, 43, 46, 46, 46, 46, 45, 45, 46, 46, 48, 48, 50, 50, 52, 52, 54, 54, 57, 57, 59, 59, 60, + 37, 38, 38, 40, 40, 43, 43, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 47, 47, 49, 49, 50, 50, 52, 52, 55, 55, 56, 56, 57, + 48, 47, 47, 46, 46, 47, 47, 47, 47, 50, 50, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 56, 56, 58, 58, 60, 60, 61, 61, 63, + 49, 47, 47, 45, 45, 46, 46, 45, 45, 49, 49, 53, 53, 56, 56, 58, 58, 59, 59, 61, 61, 62, 62, 64, 64, 65, 65, 67, 67, 68, 68, 69, + 52, 50, 50, 48, 48, 47, 47, 47, 47, 50, 50, 54, 54, 57, 57, 61, 61, 64, 64, 66, 66, 68, 68, 70, 70, 72, 72, 75, 75, 75, 75, 77, + 57, 54, 54, 52, 52, 51, 51, 50, 50, 53, 53, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 73, 73, 76, 76, 79, 79, 82, 82, 83, 83, 84, + 63, 60, 60, 57, 57, 56, 56, 54, 54, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 75, 75, 78, 78, 82, 82, 85, 85, 89, 89, 90, 90, 92, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 34, 35, 36, 38, 39, 41, 44, 44, 48, 48, 53, 53, 57, 58, 61, 65, 67, 71, 72, 79, 79, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 36, 37, 39, 41, 42, 45, 45, 49, 49, 52, 54, 57, 60, 61, 65, 66, 72, 72, + 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 41, 43, 43, 46, 46, 49, 50, 52, 54, 56, 58, 60, 63, 64, 70, 70, + 36, 35, 35, 35, 34, 35, 36, 37, 38, 39, 42, 42, 47, 48, 49, 50, 51, 53, 54, 56, 56, 59, 60, 62, 63, 66, 68, 69, 73, 73, 79, 79, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 43, 48, 48, 52, 54, 56, 58, 60, 63, 64, 67, 67, 71, 71, 74, 75, 77, 79, 81, 84, 85, 90, 90, + 53, 51, 51, 50, 49, 49, 50, 49, 49, 50, 54, 54, 59, 60, 63, 65, 67, 71, 72, 76, 76, 81, 82, 85, 87, 89, 92, 94, 97, 98, 104, 104, + 62, 60, 59, 58, 57, 57, 57, 56, 56, 56, 61, 61, 65, 66, 69, 71, 74, 78, 79, 83, 83, 89, 90, 94, 95, 98, 102, 103, 108, 108, 115, 115, + 73, 70, 69, 67, 66, 66, 65, 65, 64, 64, 69, 69, 73, 74, 77, 79, 81, 85, 86, 91, 91, 98, 99, 103, 105, 108, 112, 114, 119, 119, 127, 127, + }, { + 32, 31, 31, 30, 30, 32, 33, 34, 37, 37, 42, 42, 47, 49, 48, 48, 48, 49, 49, 50, 50, 52, 52, 53, 54, 55, 57, 58, 60, 60, 63, 63, + 31, 31, 31, 32, 32, 34, 36, 37, 40, 40, 43, 43, 46, 46, 46, 46, 45, 45, 45, 46, 46, 48, 48, 49, 50, 51, 52, 53, 54, 55, 57, 57, + 37, 38, 38, 39, 40, 41, 43, 44, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 46, 47, 47, 48, 49, 49, 50, 51, 52, 53, 55, 55, + 48, 47, 47, 46, 46, 46, 47, 47, 47, 48, 50, 50, 52, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 56, 56, 57, 58, 59, 60, 60, + 49, 47, 47, 46, 45, 45, 46, 45, 45, 46, 49, 49, 53, 53, 55, 56, 57, 58, 58, 59, 59, 61, 61, 62, 62, 63, 64, 64, 65, 65, 67, 67, + 52, 50, 50, 48, 48, 48, 47, 47, 47, 47, 50, 50, 53, 54, 56, 57, 59, 61, 62, 64, 64, 66, 66, 68, 68, 69, 70, 71, 72, 73, 75, 75, + 56, 54, 53, 52, 51, 51, 50, 50, 49, 49, 53, 53, 55, 56, 58, 59, 61, 63, 64, 66, 66, 69, 70, 71, 72, 74, 75, 76, 77, 78, 80, 80, + 61, 58, 57, 56, 55, 54, 54, 53, 52, 53, 56, 56, 58, 59, 61, 62, 63, 66, 66, 69, 69, 72, 73, 75, 76, 78, 79, 80, 82, 83, 86, 86, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 33, 34, 34, 36, 36, 38, 39, 41, 44, 44, 47, 48, 50, 53, 53, 57, 58, 61, 65, 65, 70, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 37, 37, 39, 41, 41, 44, 45, 46, 49, 49, 53, 54, 56, 60, 60, 64, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, 40, 40, 41, 43, 43, 45, 46, 47, 50, 50, 53, 54, 56, 58, 58, 62, + 35, 35, 35, 34, 34, 34, 35, 36, 36, 37, 37, 40, 41, 43, 46, 46, 47, 48, 49, 51, 51, 53, 54, 55, 57, 57, 60, 61, 63, 66, 66, 70, + 39, 38, 38, 37, 37, 37, 38, 38, 39, 40, 40, 43, 44, 46, 50, 50, 52, 53, 54, 57, 57, 59, 60, 61, 64, 64, 67, 68, 69, 72, 72, 76, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 46, 48, 50, 54, 54, 57, 58, 60, 63, 63, 66, 67, 68, 71, 71, 74, 75, 77, 79, 79, 83, + 53, 52, 51, 50, 49, 49, 49, 50, 49, 49, 49, 53, 54, 56, 60, 60, 64, 65, 67, 71, 71, 75, 76, 78, 82, 82, 86, 87, 89, 92, 92, 96, + 65, 63, 62, 61, 59, 59, 59, 59, 58, 58, 58, 62, 63, 65, 68, 68, 72, 73, 76, 79, 79, 84, 85, 88, 92, 92, 97, 98, 100, 105, 105, 109, + }, { + 32, 31, 31, 31, 30, 30, 33, 33, 35, 37, 37, 41, 42, 44, 49, 49, 48, 48, 48, 49, 49, 50, 50, 51, 52, 52, 54, 54, 55, 57, 57, 59, + 31, 31, 31, 32, 32, 32, 35, 36, 37, 40, 40, 42, 43, 44, 46, 46, 46, 46, 45, 45, 45, 46, 46, 47, 48, 48, 49, 50, 51, 52, 52, 54, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 49, 49, 50, 50, 52, + 45, 45, 45, 45, 44, 44, 46, 46, 46, 47, 47, 49, 49, 50, 52, 52, 51, 51, 51, 51, 51, 52, 52, 52, 53, 53, 54, 54, 54, 55, 55, 57, + 48, 47, 47, 46, 45, 45, 46, 46, 46, 47, 47, 49, 50, 51, 53, 53, 54, 54, 54, 55, 55, 56, 56, 56, 57, 57, 58, 58, 58, 59, 59, 61, + 49, 47, 47, 46, 45, 45, 45, 46, 45, 45, 45, 48, 49, 51, 53, 53, 55, 56, 57, 58, 58, 59, 59, 60, 61, 61, 62, 62, 63, 64, 64, 65, + 52, 50, 50, 49, 48, 48, 47, 47, 47, 47, 47, 50, 50, 52, 54, 54, 57, 57, 59, 61, 61, 63, 64, 65, 66, 66, 68, 68, 69, 70, 70, 72, + 57, 55, 54, 53, 52, 52, 51, 51, 51, 50, 50, 52, 53, 54, 57, 57, 59, 60, 61, 64, 64, 66, 67, 68, 71, 71, 73, 73, 74, 76, 76, 78, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 34, 34, 35, 36, 36, 38, 39, 39, 42, 44, 44, 47, 48, 49, 53, 53, 55, 58, 58, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 36, 37, 37, 40, 41, 41, 44, 45, 46, 49, 49, 51, 54, 54, + 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 35, 36, 36, 37, 37, 37, 39, 40, 40, 42, 42, 43, 45, 46, 47, 49, 50, 51, 54, 54, + 32, 33, 33, 33, 33, 33, 33, 34, 34, 35, 36, 36, 36, 38, 38, 39, 40, 40, 41, 42, 42, 44, 45, 45, 47, 48, 48, 51, 51, 53, 55, 55, + 36, 35, 35, 35, 35, 34, 35, 36, 36, 37, 38, 38, 40, 42, 42, 45, 48, 48, 49, 50, 50, 52, 53, 54, 56, 56, 57, 59, 60, 61, 63, 63, + 44, 43, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 44, 48, 48, 50, 54, 54, 56, 58, 58, 61, 63, 63, 66, 67, 67, 71, 71, 72, 75, 75, + 47, 46, 45, 45, 44, 44, 44, 45, 45, 45, 45, 45, 47, 50, 50, 53, 56, 56, 58, 60, 60, 64, 66, 66, 69, 70, 71, 74, 75, 76, 79, 79, + 53, 52, 51, 51, 49, 49, 49, 49, 50, 49, 49, 49, 51, 54, 54, 57, 60, 60, 63, 65, 65, 69, 71, 72, 75, 76, 77, 81, 82, 83, 87, 87, + }, { + 32, 31, 31, 31, 30, 30, 31, 33, 33, 34, 37, 37, 39, 42, 42, 45, 49, 49, 48, 48, 48, 49, 49, 49, 50, 50, 51, 52, 52, 53, 54, 54, + 31, 31, 31, 31, 32, 32, 33, 35, 36, 37, 40, 40, 41, 43, 43, 44, 46, 46, 46, 46, 46, 45, 45, 45, 46, 46, 47, 48, 48, 48, 50, 50, + 35, 36, 37, 37, 38, 38, 38, 41, 41, 42, 45, 45, 46, 46, 46, 47, 48, 48, 47, 46, 46, 46, 45, 46, 46, 46, 47, 47, 47, 48, 49, 49, + 38, 39, 40, 40, 40, 41, 41, 43, 44, 45, 47, 47, 47, 48, 48, 48, 49, 49, 48, 48, 48, 47, 47, 47, 48, 48, 48, 48, 48, 49, 50, 50, + 48, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 48, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 54, 54, 55, 55, + 49, 48, 47, 47, 45, 45, 45, 45, 46, 45, 45, 45, 47, 49, 49, 51, 53, 53, 55, 56, 56, 57, 58, 58, 59, 59, 60, 61, 61, 61, 62, 62, + 50, 49, 48, 48, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 52, 54, 54, 55, 56, 56, 58, 59, 60, 61, 61, 61, 63, 63, 63, 65, 65, + 52, 50, 50, 50, 48, 48, 48, 47, 47, 47, 47, 47, 48, 50, 50, 52, 54, 54, 56, 57, 57, 60, 61, 61, 63, 64, 64, 66, 66, 67, 68, 68, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 34, 34, 34, 35, 36, 36, 38, 39, 39, 41, 44, 44, 44, 47, 48, 48, 51, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 34, 36, 37, 37, 39, 41, 41, 42, 44, 45, 45, 47, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 36, 36, 37, 39, 39, 40, 42, 42, 42, 44, 45, 45, 48, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 38, 38, 38, 40, 40, 40, 41, 43, 43, 43, 45, 46, 46, 48, + 35, 35, 35, 35, 34, 34, 34, 34, 35, 36, 36, 37, 37, 37, 39, 41, 41, 42, 45, 46, 46, 47, 48, 48, 49, 51, 51, 51, 53, 54, 54, 56, + 36, 35, 35, 35, 35, 34, 34, 35, 36, 36, 36, 37, 38, 38, 40, 42, 42, 43, 47, 48, 48, 49, 50, 50, 51, 53, 53, 54, 56, 56, 56, 58, + 44, 43, 42, 42, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 44, 48, 48, 49, 52, 54, 54, 56, 58, 58, 60, 63, 63, 64, 66, 67, 67, 69, + 47, 46, 45, 45, 45, 44, 44, 44, 44, 45, 45, 45, 45, 45, 47, 50, 50, 51, 55, 56, 56, 58, 60, 60, 62, 66, 66, 67, 69, 70, 70, 73, + }, { + 32, 31, 31, 31, 31, 30, 30, 31, 33, 33, 33, 35, 37, 37, 39, 42, 42, 43, 47, 49, 49, 48, 48, 48, 48, 49, 49, 49, 50, 50, 50, 51, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, 43, 43, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 46, 46, 46, 47, + 33, 33, 34, 34, 34, 35, 35, 35, 37, 38, 38, 41, 43, 43, 43, 44, 44, 45, 46, 47, 47, 46, 46, 46, 46, 45, 45, 45, 46, 46, 46, 47, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 47, + 45, 45, 45, 45, 45, 44, 44, 45, 46, 46, 46, 47, 47, 47, 48, 49, 49, 50, 51, 52, 52, 52, 51, 51, 51, 51, 51, 52, 52, 52, 52, 52, + 48, 47, 47, 47, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 50, 50, 50, 52, 53, 53, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, + 49, 48, 47, 47, 46, 45, 45, 45, 45, 46, 46, 45, 45, 45, 47, 49, 49, 50, 53, 53, 53, 55, 56, 56, 57, 58, 58, 58, 59, 59, 59, 60, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 50, 53, 54, 54, 55, 56, 56, 57, 59, 59, 60, 61, 61, 61, 62, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 36, 36, 36, 37, 39, 39, 39, 41, 44, 44, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 36, 37, 37, 37, 39, 41, 41, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 36, 37, 37, 37, 39, 41, 41, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 49, 50, 50, 50, 52, 53, 53, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 49, 50, 50, 50, 52, 53, 53, + 44, 43, 42, 42, 42, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 45, 48, 48, 48, 50, 54, 54, 54, 56, 58, 58, 58, 60, 63, 63, + }, { + 32, 31, 31, 31, 31, 31, 30, 30, 30, 32, 33, 33, 33, 35, 37, 37, 37, 39, 42, 42, 42, 45, 49, 49, 49, 48, 48, 48, 48, 48, 49, 49, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 36, 36, 36, 38, 40, 40, 40, 41, 43, 43, 43, 44, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 36, 36, 36, 38, 40, 40, 40, 41, 43, 43, 43, 44, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, + 37, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, + 37, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 49, 50, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 49, 50, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 46, 46, 46, 45, 45, 45, 45, 47, 49, 49, 49, 51, 53, 53, 53, 54, 56, 56, 56, 57, 58, 58, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, 36, 36, 36, 37, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 36, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 36, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, 35, 36, 36, 36, 36, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, + 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, + 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 39, 40, 41, 41, 41, 43, 45, 46, 46, 46, 46, + 36, 35, 35, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 41, 42, 42, 42, 44, 47, 48, 48, 48, 49, + }, { + 32, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 31, 33, 33, 33, 33, 35, 36, 37, 37, 37, 39, 41, 42, 42, 42, 44, 47, 49, 49, 49, 49, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 33, 34, 35, 35, 35, 37, 38, 39, 39, 39, 40, 42, 42, 42, 42, 44, 46, 47, 47, 47, 47, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, 37, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 46, 46, 46, 46, + 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 36, 37, 38, 38, 38, 40, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, + 37, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 47, + 37, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 47, + 45, 45, 45, 45, 45, 45, 45, 44, 44, 44, 44, 45, 46, 46, 46, 46, 46, 47, 47, 47, 47, 48, 49, 49, 49, 49, 50, 51, 52, 52, 52, 52, + 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, 53, 53, 53, 53, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 36, 36, 36, 36, 36, 36, 37, 38, 38, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 31, 32, 33, 33, 33, 33, 33, 34, 35, 36, 37, 37, 37, 37, 39, 40, 42, 42, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 35, 35, 35, 35, 36, 37, 38, 39, 39, 39, 39, 40, 41, 42, 42, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 36, 37, 38, 38, 38, 38, 39, 41, 42, 43, 43, 43, 43, 43, 44, 44, 44, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 38, 39, 39, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 42, 43, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, 47, 47, 47, 48, 48, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, 31, 31, 32, 33, 33, 33, 33, 33, 33, 33, 34, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 35, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 38, 38, 38, 38, 38, 39, + 35, 35, 36, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 38, 38, 38, 38, 38, 38, 38, 38, 39, 40, 40, 41, 41, 41, 41, 41, 41, 42, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, + }, +}; + +static const uint8_t qm_tbl_32x16[][2][512] = { + { + { + 32, 31, 31, 31, 32, 32, 34, 35, 36, 39, 44, 46, 48, 53, 58, 61, 65, 71, 79, 81, 82, 88, 91, 94, 97, 100, 103, 107, 110, 114, 118, 122, + 31, 32, 32, 32, 32, 33, 34, 34, 34, 37, 41, 43, 45, 49, 54, 57, 60, 65, 72, 74, 75, 80, 83, 85, 88, 91, 94, 97, 101, 104, 108, 111, + 32, 32, 33, 33, 34, 35, 37, 37, 38, 40, 43, 44, 46, 50, 54, 56, 58, 63, 70, 71, 72, 77, 80, 83, 86, 89, 93, 96, 100, 104, 107, 111, + 34, 34, 33, 34, 35, 37, 39, 41, 43, 45, 48, 49, 51, 54, 58, 60, 63, 68, 74, 75, 76, 80, 81, 82, 85, 87, 90, 93, 97, 100, 103, 107, + 36, 35, 34, 35, 36, 38, 42, 45, 48, 50, 53, 55, 56, 60, 63, 66, 68, 73, 79, 80, 81, 85, 88, 91, 94, 97, 98, 100, 101, 103, 105, 107, + 44, 42, 41, 41, 42, 42, 48, 50, 54, 58, 63, 65, 67, 71, 75, 77, 79, 84, 90, 91, 92, 97, 100, 100, 100, 100, 101, 104, 108, 112, 115, 119, + 53, 51, 49, 49, 50, 49, 54, 57, 60, 65, 71, 73, 76, 82, 87, 89, 92, 97, 104, 105, 106, 108, 106, 105, 107, 111, 114, 117, 117, 117, 118, 119, + 59, 56, 54, 54, 54, 53, 58, 61, 64, 69, 75, 78, 80, 87, 92, 95, 98, 103, 110, 112, 113, 115, 114, 118, 123, 121, 120, 119, 123, 127, 131, 136, + 65, 62, 59, 59, 59, 58, 63, 65, 68, 73, 79, 82, 85, 92, 98, 101, 105, 111, 118, 119, 121, 126, 130, 131, 128, 127, 131, 136, 138, 137, 136, 136, + 79, 75, 72, 71, 71, 69, 73, 76, 78, 84, 90, 93, 96, 103, 110, 114, 118, 125, 133, 135, 136, 142, 142, 137, 140, 145, 144, 142, 141, 146, 151, 156, + 87, 82, 78, 78, 77, 75, 79, 82, 84, 89, 95, 98, 102, 109, 116, 120, 124, 132, 141, 142, 144, 149, 148, 153, 157, 152, 150, 155, 161, 159, 157, 156, + 90, 85, 82, 81, 80, 78, 78, 83, 87, 89, 93, 100, 102, 107, 115, 118, 123, 132, 136, 140, 151, 153, 155, 160, 161, 164, 170, 168, 165, 167, 172, 178, + 93, 88, 86, 84, 82, 82, 80, 84, 86, 91, 94, 98, 105, 107, 112, 119, 122, 130, 135, 140, 149, 153, 162, 165, 167, 173, 174, 177, 183, 185, 182, 179, + 96, 91, 90, 87, 86, 86, 83, 84, 89, 91, 95, 100, 102, 110, 111, 118, 123, 128, 135, 138, 149, 152, 160, 167, 173, 178, 180, 187, 188, 190, 197, 203, + 99, 94, 93, 90, 89, 89, 88, 87, 90, 93, 97, 99, 105, 107, 115, 116, 124, 127, 135, 139, 146, 152, 159, 166, 171, 182, 186, 191, 193, 201, 203, 204, + 102, 97, 97, 93, 93, 92, 92, 90, 90, 96, 97, 103, 104, 111, 112, 120, 121, 130, 131, 142, 143, 154, 155, 168, 169, 181, 183, 198, 200, 206, 208, 217, + }, { + 32, 31, 30, 32, 33, 37, 42, 45, 49, 48, 49, 49, 50, 52, 54, 55, 57, 60, 63, 64, 64, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 31, 31, 32, 34, 36, 40, 43, 44, 46, 46, 45, 46, 46, 48, 50, 51, 52, 54, 57, 58, 59, 61, 62, 62, 63, 64, 65, 66, 67, 68, 69, 70, + 37, 38, 40, 41, 43, 47, 47, 47, 48, 47, 46, 46, 46, 47, 49, 49, 50, 52, 55, 55, 56, 58, 59, 60, 62, 63, 64, 65, 67, 68, 69, 70, + 42, 42, 42, 44, 45, 47, 48, 49, 50, 50, 49, 49, 50, 50, 52, 52, 53, 55, 58, 58, 58, 60, 60, 60, 60, 61, 62, 63, 64, 65, 66, 67, + 48, 47, 46, 46, 47, 47, 50, 51, 53, 53, 53, 53, 54, 54, 55, 56, 56, 58, 60, 61, 61, 63, 64, 65, 66, 67, 66, 66, 66, 66, 67, 67, + 49, 47, 45, 45, 46, 45, 49, 51, 53, 56, 58, 59, 59, 61, 62, 63, 64, 65, 67, 68, 68, 69, 71, 70, 69, 68, 68, 69, 70, 71, 72, 73, + 52, 50, 48, 48, 47, 47, 50, 52, 54, 57, 61, 62, 64, 66, 68, 69, 70, 72, 75, 75, 75, 76, 74, 72, 73, 74, 75, 75, 74, 74, 73, 73, + 54, 52, 50, 49, 49, 48, 52, 54, 55, 59, 62, 64, 65, 68, 71, 72, 73, 75, 78, 78, 79, 79, 78, 79, 81, 79, 78, 76, 77, 78, 80, 81, + 57, 54, 52, 51, 51, 50, 53, 55, 57, 60, 64, 65, 67, 71, 73, 75, 76, 79, 82, 82, 83, 85, 86, 85, 83, 82, 83, 84, 84, 83, 82, 81, + 63, 60, 57, 57, 56, 54, 57, 59, 60, 64, 67, 69, 71, 75, 78, 80, 82, 85, 89, 89, 90, 92, 91, 88, 89, 90, 89, 87, 86, 87, 88, 90, + 66, 63, 60, 59, 59, 57, 60, 61, 62, 66, 69, 71, 73, 77, 80, 82, 84, 88, 92, 92, 93, 95, 94, 95, 96, 93, 92, 93, 94, 93, 91, 90, + 67, 64, 62, 61, 60, 58, 58, 61, 63, 65, 67, 70, 72, 74, 78, 80, 82, 86, 88, 90, 95, 96, 96, 98, 97, 98, 100, 98, 96, 96, 97, 99, + 68, 65, 63, 62, 60, 60, 59, 61, 62, 65, 66, 68, 72, 73, 76, 79, 80, 84, 87, 89, 93, 94, 98, 99, 99, 102, 101, 102, 103, 103, 101, 99, + 69, 66, 65, 63, 62, 61, 60, 60, 63, 64, 66, 68, 70, 73, 74, 78, 80, 82, 85, 87, 91, 92, 96, 98, 101, 102, 103, 105, 105, 105, 107, 108, + 71, 67, 66, 64, 63, 62, 62, 61, 62, 64, 66, 67, 70, 71, 75, 76, 79, 81, 84, 86, 89, 91, 94, 97, 98, 102, 104, 106, 106, 109, 109, 108, + 72, 68, 68, 65, 65, 63, 63, 61, 62, 65, 65, 68, 69, 72, 73, 77, 77, 81, 81, 86, 87, 91, 91, 96, 97, 101, 102, 107, 107, 109, 110, 113, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 32, 34, 36, 38, 39, 44, 47, 49, 53, 58, 61, 65, 71, 76, 79, 82, 86, 89, 92, 95, 98, 101, 104, 107, 110, 114, + 31, 32, 32, 32, 32, 33, 33, 34, 34, 36, 37, 41, 44, 46, 49, 54, 56, 60, 65, 69, 72, 75, 78, 81, 84, 86, 89, 92, 95, 98, 101, 104, + 32, 32, 32, 33, 34, 35, 35, 36, 37, 39, 40, 42, 45, 47, 50, 54, 56, 59, 64, 68, 70, 73, 76, 79, 82, 85, 88, 91, 94, 97, 100, 104, + 32, 33, 33, 33, 34, 36, 36, 38, 40, 41, 42, 45, 47, 48, 51, 55, 57, 60, 65, 69, 71, 74, 77, 78, 80, 83, 85, 88, 91, 94, 97, 100, + 36, 35, 35, 35, 36, 38, 40, 42, 48, 49, 50, 53, 56, 57, 60, 63, 65, 68, 73, 76, 79, 81, 84, 87, 89, 92, 93, 94, 95, 96, 98, 100, + 44, 42, 41, 41, 42, 42, 44, 48, 54, 56, 58, 63, 66, 67, 71, 75, 77, 79, 84, 88, 90, 92, 95, 95, 95, 95, 95, 98, 101, 105, 108, 111, + 47, 45, 44, 44, 45, 45, 47, 50, 56, 58, 60, 66, 69, 71, 75, 79, 81, 84, 89, 92, 95, 97, 100, 99, 101, 105, 108, 110, 110, 110, 111, 111, + 53, 51, 49, 49, 50, 49, 51, 54, 60, 63, 65, 71, 75, 77, 82, 87, 89, 92, 97, 101, 104, 106, 109, 112, 116, 114, 113, 112, 115, 119, 123, 126, + 65, 62, 60, 59, 59, 58, 60, 63, 68, 71, 73, 79, 84, 86, 92, 98, 100, 105, 111, 115, 118, 121, 124, 124, 121, 120, 124, 128, 129, 128, 127, 127, + 73, 69, 67, 66, 65, 64, 66, 69, 74, 77, 79, 85, 90, 93, 99, 105, 107, 112, 119, 123, 127, 130, 133, 130, 132, 136, 136, 133, 132, 136, 141, 145, + 79, 75, 72, 71, 71, 69, 71, 73, 78, 81, 84, 90, 95, 97, 103, 110, 113, 118, 125, 130, 133, 136, 140, 145, 148, 143, 141, 146, 151, 149, 147, 145, + 87, 83, 80, 79, 78, 76, 76, 80, 84, 86, 90, 96, 99, 103, 111, 114, 118, 126, 130, 134, 143, 146, 147, 152, 151, 155, 160, 158, 154, 156, 161, 166, + 90, 86, 84, 82, 80, 80, 78, 82, 83, 88, 91, 94, 101, 103, 108, 114, 116, 124, 129, 134, 142, 145, 153, 156, 157, 163, 163, 166, 171, 173, 169, 166, + 93, 88, 87, 84, 83, 83, 81, 81, 86, 88, 92, 96, 98, 105, 107, 113, 117, 122, 129, 131, 141, 144, 151, 157, 163, 167, 169, 175, 175, 177, 183, 189, + 96, 91, 90, 87, 87, 86, 85, 84, 87, 90, 94, 96, 101, 102, 110, 111, 118, 121, 129, 132, 138, 144, 150, 156, 161, 171, 174, 179, 181, 188, 188, 190, + 99, 94, 94, 90, 90, 88, 89, 86, 87, 93, 93, 99, 99, 106, 107, 115, 116, 124, 125, 135, 136, 145, 146, 158, 159, 170, 171, 185, 186, 192, 193, 201, + }, { + 32, 31, 30, 31, 33, 37, 39, 42, 49, 48, 48, 49, 50, 51, 52, 54, 55, 57, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, + 31, 31, 32, 33, 36, 40, 41, 43, 46, 46, 46, 45, 46, 47, 48, 50, 51, 52, 54, 56, 57, 59, 60, 61, 62, 63, 64, 65, 65, 66, 67, 68, + 35, 37, 38, 38, 41, 45, 46, 46, 48, 47, 46, 45, 46, 47, 47, 49, 49, 50, 53, 54, 55, 56, 58, 59, 60, 61, 62, 64, 65, 66, 67, 68, + 38, 40, 40, 41, 44, 47, 47, 48, 49, 48, 48, 47, 48, 48, 48, 50, 50, 51, 53, 55, 56, 57, 58, 58, 59, 60, 60, 61, 62, 63, 64, 65, + 48, 47, 46, 46, 47, 47, 48, 50, 53, 53, 53, 53, 54, 54, 54, 55, 56, 56, 58, 60, 60, 61, 62, 63, 64, 65, 65, 65, 65, 65, 65, 65, + 49, 47, 45, 45, 46, 45, 47, 49, 53, 55, 56, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 68, 67, 66, 66, 67, 68, 69, 70, 71, + 50, 48, 46, 46, 46, 46, 47, 50, 54, 55, 56, 59, 61, 61, 63, 65, 65, 66, 68, 69, 70, 71, 72, 71, 71, 72, 73, 73, 72, 72, 71, 71, + 52, 50, 48, 48, 47, 47, 48, 50, 54, 56, 57, 61, 63, 64, 66, 68, 69, 70, 72, 74, 75, 75, 76, 78, 79, 77, 76, 74, 75, 76, 77, 78, + 57, 54, 52, 52, 51, 50, 51, 53, 57, 58, 60, 64, 66, 68, 71, 73, 74, 76, 79, 81, 82, 83, 84, 83, 81, 80, 81, 82, 82, 81, 79, 78, + 61, 57, 55, 55, 54, 52, 54, 56, 59, 61, 62, 66, 68, 70, 73, 76, 77, 79, 82, 84, 86, 87, 88, 86, 86, 88, 87, 85, 83, 85, 86, 87, + 63, 60, 58, 57, 56, 54, 55, 57, 60, 62, 64, 67, 70, 71, 75, 78, 79, 82, 85, 87, 89, 90, 91, 93, 94, 91, 89, 90, 92, 90, 89, 87, + 67, 63, 61, 60, 59, 57, 57, 60, 63, 64, 66, 69, 71, 73, 77, 79, 81, 85, 87, 88, 92, 93, 94, 96, 95, 96, 97, 95, 93, 93, 94, 96, + 68, 64, 63, 61, 60, 59, 58, 60, 61, 64, 65, 67, 71, 72, 75, 78, 79, 83, 85, 87, 91, 92, 95, 96, 97, 99, 98, 99, 100, 100, 98, 96, + 69, 65, 64, 62, 61, 61, 59, 59, 62, 63, 65, 67, 68, 72, 73, 76, 78, 81, 84, 85, 89, 90, 93, 96, 98, 99, 100, 102, 102, 102, 103, 105, + 70, 66, 65, 63, 63, 62, 61, 60, 61, 63, 65, 66, 69, 70, 74, 74, 78, 79, 82, 84, 87, 89, 91, 94, 96, 100, 101, 103, 103, 105, 105, 105, + 71, 67, 67, 64, 64, 62, 62, 60, 61, 64, 64, 67, 67, 71, 71, 75, 75, 79, 80, 84, 84, 89, 89, 94, 94, 98, 99, 104, 104, 106, 106, 109, + }, + }, { + { + 32, 31, 31, 31, 32, 32, 32, 34, 34, 36, 39, 40, 44, 47, 49, 53, 57, 59, 65, 69, 71, 79, 81, 82, 87, 90, 92, 95, 98, 100, 103, 106, + 31, 32, 32, 32, 32, 32, 33, 34, 34, 34, 37, 38, 41, 44, 46, 49, 53, 54, 60, 63, 65, 72, 74, 75, 79, 82, 84, 87, 89, 92, 94, 97, + 32, 32, 32, 32, 33, 34, 34, 35, 36, 37, 39, 40, 42, 45, 46, 50, 53, 54, 59, 62, 64, 71, 72, 73, 77, 80, 83, 85, 88, 91, 94, 97, + 32, 32, 32, 33, 34, 34, 35, 37, 37, 38, 40, 41, 43, 46, 47, 50, 53, 54, 58, 62, 63, 70, 71, 72, 76, 78, 81, 83, 85, 88, 90, 93, + 36, 35, 35, 34, 36, 37, 38, 42, 44, 48, 50, 51, 53, 56, 57, 60, 63, 64, 68, 71, 73, 79, 80, 81, 85, 87, 88, 88, 89, 90, 92, 93, + 39, 38, 38, 37, 39, 40, 40, 45, 47, 51, 54, 55, 58, 61, 62, 65, 68, 69, 73, 76, 78, 84, 85, 86, 90, 89, 90, 92, 95, 98, 101, 104, + 44, 42, 41, 41, 42, 42, 42, 48, 50, 54, 58, 59, 63, 66, 67, 71, 74, 75, 79, 83, 84, 90, 91, 92, 96, 99, 102, 103, 103, 103, 103, 104, + 53, 51, 50, 49, 50, 49, 49, 54, 56, 60, 65, 67, 71, 75, 77, 82, 86, 87, 92, 96, 97, 104, 105, 106, 110, 108, 106, 105, 108, 111, 114, 118, + 58, 55, 54, 53, 53, 53, 52, 57, 59, 63, 68, 70, 74, 79, 81, 86, 90, 91, 97, 100, 102, 109, 110, 111, 114, 113, 117, 120, 121, 120, 119, 118, + 65, 62, 60, 59, 59, 58, 58, 63, 65, 68, 73, 75, 79, 85, 86, 92, 97, 98, 105, 109, 111, 118, 120, 121, 125, 129, 128, 125, 124, 127, 131, 135, + 79, 75, 73, 72, 71, 70, 69, 73, 75, 78, 84, 85, 90, 95, 97, 103, 108, 111, 118, 122, 125, 133, 135, 136, 140, 135, 133, 137, 141, 139, 137, 135, + 81, 77, 75, 74, 72, 71, 70, 75, 77, 80, 85, 87, 91, 97, 99, 105, 110, 112, 119, 124, 127, 135, 137, 139, 143, 146, 150, 148, 144, 146, 150, 154, + 88, 83, 81, 79, 78, 77, 76, 79, 81, 85, 88, 91, 97, 99, 104, 109, 111, 119, 123, 127, 135, 137, 145, 147, 148, 153, 153, 155, 160, 161, 158, 155, + 90, 86, 84, 82, 81, 80, 78, 79, 83, 85, 89, 92, 94, 101, 102, 108, 112, 117, 123, 125, 134, 136, 143, 148, 154, 157, 158, 164, 164, 165, 170, 175, + 93, 88, 88, 84, 84, 83, 82, 81, 84, 86, 90, 92, 97, 98, 105, 106, 113, 115, 122, 125, 131, 136, 141, 147, 151, 160, 163, 168, 169, 175, 175, 176, + 96, 91, 91, 87, 87, 85, 86, 83, 84, 89, 89, 95, 95, 102, 102, 110, 110, 118, 119, 128, 129, 137, 138, 149, 149, 159, 160, 173, 174, 179, 180, 187, + }, { + 32, 31, 31, 30, 33, 35, 37, 42, 44, 49, 48, 48, 49, 50, 51, 52, 54, 54, 57, 59, 60, 63, 64, 64, 66, 67, 68, 69, 70, 71, 72, 73, + 31, 31, 32, 32, 36, 38, 40, 43, 44, 46, 46, 45, 45, 46, 47, 48, 49, 50, 52, 54, 54, 57, 58, 59, 60, 61, 62, 63, 64, 65, 65, 66, + 34, 35, 36, 36, 40, 42, 44, 45, 46, 47, 46, 46, 45, 46, 47, 47, 49, 49, 51, 52, 53, 56, 57, 57, 59, 60, 61, 62, 63, 64, 65, 66, + 37, 38, 39, 40, 43, 45, 47, 47, 47, 48, 47, 46, 46, 46, 47, 47, 48, 49, 50, 52, 52, 55, 55, 56, 57, 58, 59, 60, 60, 61, 62, 63, + 48, 47, 46, 46, 47, 47, 47, 50, 51, 53, 53, 53, 53, 54, 54, 54, 55, 55, 56, 58, 58, 60, 61, 61, 63, 63, 63, 63, 63, 63, 63, 63, + 48, 47, 46, 45, 46, 46, 46, 50, 51, 53, 54, 55, 56, 56, 57, 57, 58, 59, 60, 61, 62, 64, 64, 65, 66, 65, 64, 65, 66, 67, 68, 69, + 49, 47, 46, 45, 46, 45, 45, 49, 51, 53, 56, 56, 58, 59, 60, 61, 62, 62, 64, 65, 65, 67, 68, 68, 69, 70, 71, 71, 70, 70, 69, 69, + 52, 50, 48, 48, 47, 47, 47, 50, 52, 54, 57, 58, 61, 63, 64, 66, 68, 68, 70, 72, 72, 75, 75, 75, 77, 75, 74, 72, 73, 74, 75, 76, + 54, 51, 50, 49, 49, 48, 48, 51, 53, 55, 58, 59, 62, 65, 65, 68, 70, 70, 73, 74, 75, 77, 78, 78, 79, 78, 79, 80, 80, 78, 77, 76, + 57, 54, 53, 52, 51, 50, 50, 53, 54, 57, 60, 61, 64, 66, 68, 71, 73, 74, 76, 78, 79, 82, 82, 83, 84, 85, 84, 82, 81, 82, 83, 84, + 63, 60, 58, 57, 56, 55, 54, 57, 59, 60, 64, 65, 67, 70, 71, 75, 77, 78, 82, 84, 85, 89, 89, 90, 91, 88, 87, 88, 89, 88, 86, 84, + 64, 61, 59, 58, 57, 56, 55, 58, 59, 61, 64, 65, 68, 71, 72, 75, 78, 79, 82, 85, 86, 90, 90, 91, 93, 93, 94, 93, 90, 90, 92, 93, + 67, 63, 62, 60, 59, 58, 57, 59, 60, 63, 64, 66, 70, 70, 73, 76, 77, 81, 83, 85, 89, 90, 93, 94, 94, 96, 96, 96, 97, 97, 95, 93, + 68, 64, 63, 61, 60, 60, 58, 58, 61, 62, 64, 66, 67, 71, 71, 75, 77, 79, 82, 83, 87, 88, 91, 93, 95, 97, 97, 99, 99, 99, 100, 101, + 69, 65, 65, 62, 62, 61, 60, 59, 61, 62, 64, 65, 68, 68, 72, 73, 76, 77, 81, 82, 85, 87, 89, 92, 93, 97, 98, 100, 100, 102, 102, 101, + 69, 66, 66, 63, 63, 61, 61, 59, 60, 63, 63, 66, 66, 70, 70, 73, 74, 78, 78, 82, 82, 86, 87, 91, 91, 95, 96, 101, 101, 103, 103, 105, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 34, 35, 36, 39, 41, 44, 47, 48, 53, 55, 58, 63, 65, 71, 74, 79, 82, 82, 87, 89, 92, 94, 97, 99, + 31, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 37, 39, 41, 44, 45, 49, 51, 54, 58, 60, 65, 68, 72, 75, 75, 79, 82, 84, 86, 88, 91, + 31, 32, 32, 32, 33, 33, 34, 34, 35, 36, 36, 39, 40, 42, 44, 45, 50, 51, 54, 58, 59, 64, 67, 71, 73, 74, 78, 81, 83, 85, 88, 91, + 32, 32, 32, 33, 34, 34, 35, 36, 37, 38, 38, 40, 41, 43, 45, 46, 50, 51, 54, 57, 58, 63, 66, 70, 72, 72, 76, 78, 80, 82, 85, 87, + 35, 35, 34, 34, 35, 36, 37, 39, 41, 45, 46, 48, 49, 51, 53, 54, 57, 59, 61, 65, 66, 71, 73, 77, 79, 79, 83, 83, 84, 85, 86, 87, + 36, 35, 35, 34, 36, 36, 38, 40, 42, 47, 48, 50, 51, 53, 56, 56, 60, 61, 63, 67, 68, 73, 75, 79, 81, 81, 85, 87, 89, 92, 94, 97, + 44, 42, 41, 41, 42, 42, 42, 44, 48, 52, 54, 58, 60, 63, 66, 67, 71, 72, 75, 78, 79, 84, 86, 90, 92, 92, 96, 97, 97, 97, 97, 97, + 47, 45, 45, 44, 44, 45, 45, 47, 50, 55, 56, 60, 62, 66, 69, 70, 75, 77, 79, 83, 84, 89, 91, 95, 97, 97, 100, 99, 101, 104, 107, 110, + 53, 51, 50, 49, 49, 50, 49, 51, 54, 59, 60, 65, 67, 71, 75, 76, 82, 84, 87, 91, 92, 97, 100, 104, 105, 106, 110, 113, 114, 112, 111, 110, + 62, 59, 58, 57, 57, 57, 56, 58, 61, 65, 66, 71, 74, 78, 82, 83, 90, 92, 95, 100, 102, 108, 110, 115, 117, 117, 120, 118, 116, 119, 123, 126, + 65, 62, 61, 59, 59, 59, 58, 60, 63, 67, 68, 73, 76, 79, 84, 85, 92, 94, 98, 103, 105, 111, 113, 118, 120, 121, 125, 128, 132, 130, 128, 126, + 79, 75, 74, 72, 71, 71, 69, 71, 73, 77, 78, 84, 86, 90, 95, 96, 103, 106, 110, 116, 118, 125, 128, 133, 136, 136, 141, 139, 135, 136, 140, 144, + 82, 78, 76, 74, 73, 73, 71, 73, 76, 79, 80, 86, 88, 92, 97, 98, 106, 108, 112, 118, 120, 127, 131, 136, 139, 139, 144, 145, 150, 151, 147, 144, + 88, 83, 82, 79, 79, 78, 76, 76, 81, 82, 85, 89, 91, 97, 98, 104, 107, 111, 117, 119, 127, 129, 135, 140, 145, 148, 148, 153, 153, 154, 159, 163, + 90, 86, 85, 82, 81, 80, 79, 78, 81, 83, 87, 88, 93, 94, 101, 101, 108, 110, 116, 119, 124, 129, 134, 139, 142, 150, 153, 157, 157, 163, 163, 163, + 93, 88, 88, 84, 84, 82, 83, 80, 80, 86, 86, 91, 91, 97, 98, 105, 105, 112, 113, 121, 122, 130, 130, 140, 140, 149, 150, 161, 162, 166, 167, 173, + }, { + 32, 31, 31, 30, 33, 33, 37, 39, 42, 47, 49, 48, 48, 49, 50, 50, 52, 53, 54, 56, 57, 60, 61, 63, 64, 64, 66, 67, 68, 69, 70, 70, + 31, 31, 32, 32, 35, 36, 40, 41, 43, 46, 46, 46, 45, 45, 46, 46, 48, 49, 50, 51, 52, 54, 56, 57, 58, 59, 60, 61, 62, 63, 63, 64, + 33, 34, 34, 35, 37, 38, 43, 43, 44, 46, 47, 46, 46, 45, 46, 46, 47, 48, 49, 51, 51, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 37, 38, 39, 40, 42, 43, 47, 47, 47, 48, 48, 47, 46, 46, 46, 46, 47, 48, 49, 50, 50, 52, 53, 55, 56, 56, 57, 58, 59, 59, 60, 61, + 45, 45, 45, 44, 46, 46, 47, 48, 49, 51, 52, 51, 51, 51, 52, 52, 53, 53, 54, 55, 55, 57, 58, 59, 60, 60, 61, 61, 61, 61, 61, 61, + 48, 47, 46, 46, 47, 47, 47, 48, 50, 52, 53, 53, 53, 53, 54, 54, 54, 55, 55, 56, 56, 58, 59, 60, 61, 61, 63, 63, 64, 65, 66, 67, + 49, 47, 46, 45, 45, 46, 45, 47, 49, 53, 53, 56, 57, 58, 59, 59, 61, 61, 62, 63, 64, 65, 66, 67, 68, 68, 69, 69, 68, 68, 67, 67, + 50, 48, 47, 46, 46, 46, 46, 47, 50, 53, 54, 56, 57, 59, 61, 61, 63, 64, 65, 66, 66, 68, 69, 70, 71, 71, 72, 70, 71, 72, 73, 74, + 52, 50, 49, 48, 47, 47, 47, 48, 50, 53, 54, 57, 59, 61, 63, 64, 66, 67, 68, 70, 70, 72, 73, 75, 75, 75, 77, 78, 77, 76, 75, 74, + 56, 53, 52, 51, 50, 50, 49, 50, 53, 55, 56, 59, 61, 63, 65, 66, 70, 71, 72, 74, 75, 77, 79, 80, 81, 81, 82, 80, 79, 80, 81, 82, + 57, 54, 53, 52, 51, 51, 50, 51, 53, 56, 57, 60, 61, 64, 66, 67, 71, 72, 73, 76, 76, 79, 80, 82, 83, 83, 84, 85, 86, 85, 83, 82, + 63, 60, 59, 57, 56, 56, 54, 55, 57, 60, 60, 64, 65, 67, 70, 71, 75, 76, 78, 81, 82, 85, 86, 89, 90, 90, 92, 90, 88, 88, 89, 90, + 64, 61, 60, 58, 57, 57, 55, 56, 58, 61, 61, 64, 66, 68, 71, 71, 75, 77, 79, 82, 83, 86, 87, 90, 91, 91, 93, 93, 94, 94, 92, 90, + 67, 63, 62, 60, 60, 59, 57, 57, 60, 61, 63, 65, 66, 70, 70, 73, 75, 77, 80, 81, 85, 86, 89, 91, 93, 94, 94, 96, 96, 95, 97, 98, + 68, 64, 64, 61, 61, 60, 59, 58, 60, 61, 63, 64, 67, 67, 71, 71, 74, 75, 79, 80, 83, 85, 87, 89, 91, 94, 95, 97, 97, 99, 98, 98, + 68, 65, 65, 62, 62, 60, 61, 59, 59, 62, 62, 65, 65, 68, 68, 72, 72, 76, 76, 80, 80, 84, 84, 89, 89, 93, 93, 97, 98, 99, 99, 102, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 33, 34, 36, 36, 39, 40, 44, 46, 48, 52, 53, 58, 58, 65, 66, 71, 74, 79, 81, 82, 86, 88, 91, 93, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 37, 38, 41, 43, 45, 48, 49, 53, 54, 60, 61, 65, 68, 72, 74, 75, 78, 81, 83, 85, + 31, 32, 32, 32, 32, 33, 33, 33, 34, 34, 35, 35, 38, 39, 41, 43, 45, 48, 49, 53, 54, 59, 60, 65, 67, 72, 73, 74, 78, 80, 82, 85, + 32, 32, 32, 33, 33, 34, 35, 35, 36, 37, 38, 38, 40, 41, 43, 44, 46, 49, 50, 53, 54, 58, 59, 63, 66, 70, 71, 72, 75, 77, 79, 81, + 33, 33, 33, 33, 34, 35, 36, 36, 38, 39, 42, 42, 44, 45, 46, 48, 50, 52, 53, 57, 57, 62, 63, 67, 69, 73, 75, 75, 78, 80, 80, 81, + 36, 35, 35, 34, 35, 36, 37, 38, 41, 42, 48, 48, 50, 51, 53, 55, 56, 59, 60, 63, 63, 68, 69, 73, 75, 79, 80, 81, 84, 86, 88, 90, + 40, 39, 39, 38, 38, 39, 40, 41, 44, 45, 51, 51, 54, 56, 59, 60, 62, 65, 66, 69, 70, 74, 75, 79, 81, 85, 86, 87, 90, 90, 90, 90, + 44, 42, 42, 41, 41, 42, 42, 42, 46, 48, 54, 54, 58, 59, 63, 65, 67, 70, 71, 74, 75, 79, 80, 84, 86, 90, 91, 92, 95, 98, 100, 102, + 51, 49, 49, 47, 47, 48, 48, 48, 52, 53, 58, 59, 63, 65, 69, 72, 74, 78, 79, 83, 84, 89, 90, 94, 97, 101, 102, 103, 106, 105, 103, 103, + 53, 51, 51, 49, 49, 50, 49, 49, 53, 54, 60, 60, 65, 67, 71, 73, 76, 80, 82, 86, 87, 92, 93, 97, 100, 104, 105, 106, 109, 112, 114, 117, + 65, 62, 61, 59, 59, 59, 58, 58, 62, 63, 68, 68, 73, 75, 79, 82, 85, 90, 92, 97, 98, 105, 106, 111, 113, 118, 120, 121, 124, 122, 119, 117, + 66, 63, 62, 60, 60, 60, 59, 59, 63, 64, 69, 69, 74, 76, 80, 83, 86, 91, 93, 98, 99, 106, 107, 112, 115, 119, 121, 122, 125, 127, 130, 134, + 79, 75, 74, 72, 71, 71, 69, 69, 72, 73, 78, 79, 84, 85, 90, 93, 96, 101, 103, 109, 110, 118, 119, 125, 128, 133, 135, 136, 140, 140, 137, 134, + 81, 77, 76, 74, 73, 72, 71, 70, 74, 75, 80, 80, 85, 87, 91, 94, 98, 103, 105, 111, 112, 119, 121, 127, 130, 135, 137, 139, 142, 144, 148, 151, + 87, 83, 82, 79, 79, 78, 77, 75, 78, 80, 84, 85, 89, 90, 96, 97, 103, 105, 111, 113, 118, 122, 126, 131, 134, 141, 143, 147, 147, 152, 151, 152, + 90, 85, 85, 81, 81, 80, 80, 77, 78, 83, 83, 87, 88, 93, 93, 100, 100, 107, 107, 115, 115, 123, 123, 132, 132, 140, 140, 151, 151, 155, 155, 160, + }, { + 32, 31, 31, 30, 31, 33, 35, 37, 41, 42, 49, 49, 48, 48, 49, 49, 50, 51, 52, 54, 54, 57, 57, 60, 61, 63, 64, 64, 66, 67, 68, 68, + 31, 31, 31, 32, 33, 36, 38, 40, 42, 43, 46, 46, 46, 45, 45, 46, 46, 47, 48, 50, 50, 52, 52, 54, 56, 57, 58, 59, 60, 61, 62, 62, + 32, 33, 33, 33, 35, 37, 39, 41, 43, 43, 47, 47, 46, 46, 45, 46, 46, 47, 48, 49, 50, 52, 52, 54, 55, 57, 58, 58, 59, 60, 61, 62, + 37, 38, 38, 40, 41, 43, 45, 47, 47, 47, 48, 48, 47, 46, 46, 46, 46, 47, 47, 48, 49, 50, 51, 52, 53, 55, 55, 56, 57, 58, 58, 59, + 40, 41, 41, 42, 43, 44, 46, 47, 48, 48, 50, 50, 49, 49, 48, 49, 49, 49, 50, 51, 51, 52, 53, 55, 56, 57, 58, 58, 59, 59, 59, 59, + 48, 47, 47, 46, 46, 47, 47, 47, 49, 50, 53, 53, 53, 53, 53, 53, 54, 54, 54, 55, 55, 56, 57, 58, 59, 60, 61, 61, 62, 63, 64, 65, + 49, 47, 47, 45, 46, 46, 46, 46, 49, 49, 53, 53, 54, 55, 56, 57, 57, 58, 58, 59, 59, 60, 61, 62, 63, 64, 65, 65, 66, 66, 65, 65, + 49, 47, 47, 45, 45, 46, 45, 45, 48, 49, 53, 54, 56, 56, 58, 59, 59, 61, 61, 62, 62, 64, 64, 65, 66, 67, 68, 68, 69, 70, 71, 71, + 51, 49, 49, 47, 47, 47, 47, 46, 49, 50, 54, 54, 57, 58, 61, 62, 63, 64, 65, 67, 67, 69, 69, 71, 72, 73, 73, 74, 75, 74, 72, 71, + 52, 50, 49, 48, 48, 47, 47, 47, 50, 50, 54, 55, 57, 58, 61, 62, 64, 66, 66, 68, 68, 70, 71, 72, 73, 75, 75, 75, 76, 77, 78, 79, + 57, 54, 54, 52, 51, 51, 50, 50, 52, 53, 57, 57, 60, 61, 64, 65, 67, 69, 71, 73, 73, 76, 77, 79, 80, 82, 82, 83, 84, 82, 81, 79, + 58, 55, 54, 52, 52, 52, 51, 50, 53, 54, 57, 57, 60, 61, 64, 66, 67, 70, 71, 73, 74, 77, 77, 79, 81, 82, 83, 83, 85, 85, 86, 87, + 63, 60, 59, 57, 57, 56, 55, 54, 57, 57, 60, 61, 64, 65, 67, 69, 71, 73, 75, 77, 78, 82, 82, 85, 86, 89, 89, 90, 91, 91, 89, 87, + 64, 61, 60, 58, 57, 57, 56, 55, 57, 58, 61, 61, 64, 65, 68, 69, 71, 74, 75, 78, 78, 82, 83, 86, 87, 90, 90, 91, 92, 93, 94, 95, + 67, 63, 63, 60, 60, 59, 58, 57, 59, 60, 62, 63, 65, 66, 69, 70, 73, 74, 77, 78, 81, 83, 85, 87, 88, 92, 92, 94, 94, 96, 95, 95, + 67, 64, 64, 61, 61, 60, 60, 58, 58, 61, 61, 64, 64, 67, 67, 70, 71, 74, 74, 78, 78, 82, 82, 86, 86, 90, 90, 95, 95, 96, 96, 98, + }, + }, { + { + 32, 31, 31, 31, 31, 32, 32, 32, 32, 34, 34, 36, 36, 39, 39, 44, 44, 48, 48, 53, 53, 58, 58, 65, 65, 71, 71, 79, 79, 82, 82, 87, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 37, 37, 41, 41, 45, 45, 49, 49, 54, 54, 60, 60, 65, 65, 72, 72, 75, 75, 79, + 31, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 37, 37, 41, 41, 45, 45, 49, 49, 54, 54, 60, 60, 65, 65, 72, 72, 75, 75, 79, + 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 43, 43, 46, 46, 50, 50, 54, 54, 58, 58, 63, 63, 70, 70, 72, 72, 76, + 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 43, 43, 46, 46, 50, 50, 54, 54, 58, 58, 63, 63, 70, 70, 72, 72, 76, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, 50, 50, 53, 53, 56, 56, 60, 60, 63, 63, 68, 68, 73, 73, 79, 79, 81, 81, 84, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, 50, 50, 53, 53, 56, 56, 60, 60, 63, 63, 68, 68, 73, 73, 79, 79, 81, 81, 84, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 48, 48, 54, 54, 58, 58, 63, 63, 67, 67, 71, 71, 75, 75, 79, 79, 84, 84, 90, 90, 92, 92, 96, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 48, 48, 54, 54, 58, 58, 63, 63, 67, 67, 71, 71, 75, 75, 79, 79, 84, 84, 90, 90, 92, 92, 96, + 53, 51, 51, 49, 49, 50, 50, 49, 49, 54, 54, 60, 60, 65, 65, 71, 71, 76, 76, 82, 82, 87, 87, 92, 92, 97, 97, 104, 104, 106, 106, 109, + 53, 51, 51, 49, 49, 50, 50, 49, 49, 54, 54, 60, 60, 65, 65, 71, 71, 76, 76, 82, 82, 87, 87, 92, 92, 97, 97, 104, 104, 106, 106, 109, + 65, 62, 62, 59, 59, 59, 59, 58, 58, 63, 63, 68, 68, 73, 73, 79, 79, 85, 85, 92, 92, 98, 98, 105, 105, 111, 111, 118, 118, 121, 121, 124, + 65, 62, 62, 59, 59, 59, 59, 58, 58, 63, 63, 68, 68, 73, 73, 79, 79, 85, 85, 92, 92, 98, 98, 105, 105, 111, 111, 118, 118, 121, 121, 124, + 79, 75, 75, 72, 72, 71, 71, 69, 69, 73, 73, 78, 78, 84, 84, 90, 90, 96, 96, 103, 103, 110, 110, 118, 118, 125, 125, 133, 133, 136, 136, 141, + 79, 75, 75, 72, 72, 71, 71, 69, 69, 73, 73, 78, 78, 84, 84, 90, 90, 96, 96, 103, 103, 110, 110, 118, 118, 125, 125, 133, 133, 136, 136, 141, + 87, 82, 82, 78, 78, 77, 77, 75, 75, 79, 79, 84, 84, 89, 89, 95, 95, 102, 102, 109, 109, 116, 116, 124, 124, 132, 132, 141, 141, 144, 144, 149, + }, { + 32, 31, 31, 30, 30, 33, 33, 37, 37, 42, 42, 49, 49, 48, 48, 49, 49, 50, 50, 52, 52, 54, 54, 57, 57, 60, 60, 63, 63, 64, 64, 66, + 31, 31, 31, 32, 32, 36, 36, 40, 40, 43, 43, 46, 46, 46, 46, 45, 45, 46, 46, 48, 48, 50, 50, 52, 52, 54, 54, 57, 57, 59, 59, 60, + 31, 31, 31, 32, 32, 36, 36, 40, 40, 43, 43, 46, 46, 46, 46, 45, 45, 46, 46, 48, 48, 50, 50, 52, 52, 54, 54, 57, 57, 59, 59, 60, + 37, 38, 38, 40, 40, 43, 43, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 47, 47, 49, 49, 50, 50, 52, 52, 55, 55, 56, 56, 57, + 37, 38, 38, 40, 40, 43, 43, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 47, 47, 49, 49, 50, 50, 52, 52, 55, 55, 56, 56, 57, + 48, 47, 47, 46, 46, 47, 47, 47, 47, 50, 50, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 56, 56, 58, 58, 60, 60, 61, 61, 63, + 48, 47, 47, 46, 46, 47, 47, 47, 47, 50, 50, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 56, 56, 58, 58, 60, 60, 61, 61, 63, + 49, 47, 47, 45, 45, 46, 46, 45, 45, 49, 49, 53, 53, 56, 56, 58, 58, 59, 59, 61, 61, 62, 62, 64, 64, 65, 65, 67, 67, 68, 68, 69, + 49, 47, 47, 45, 45, 46, 46, 45, 45, 49, 49, 53, 53, 56, 56, 58, 58, 59, 59, 61, 61, 62, 62, 64, 64, 65, 65, 67, 67, 68, 68, 69, + 52, 50, 50, 48, 48, 47, 47, 47, 47, 50, 50, 54, 54, 57, 57, 61, 61, 64, 64, 66, 66, 68, 68, 70, 70, 72, 72, 75, 75, 75, 75, 77, + 52, 50, 50, 48, 48, 47, 47, 47, 47, 50, 50, 54, 54, 57, 57, 61, 61, 64, 64, 66, 66, 68, 68, 70, 70, 72, 72, 75, 75, 75, 75, 77, + 57, 54, 54, 52, 52, 51, 51, 50, 50, 53, 53, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 73, 73, 76, 76, 79, 79, 82, 82, 83, 83, 84, + 57, 54, 54, 52, 52, 51, 51, 50, 50, 53, 53, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 73, 73, 76, 76, 79, 79, 82, 82, 83, 83, 84, + 63, 60, 60, 57, 57, 56, 56, 54, 54, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 75, 75, 78, 78, 82, 82, 85, 85, 89, 89, 90, 90, 92, + 63, 60, 60, 57, 57, 56, 56, 54, 54, 57, 57, 60, 60, 64, 64, 67, 67, 71, 71, 75, 75, 78, 78, 82, 82, 85, 85, 89, 89, 90, 90, 92, + 66, 63, 63, 60, 60, 59, 59, 57, 57, 60, 60, 62, 62, 66, 66, 69, 69, 73, 73, 77, 77, 80, 80, 84, 84, 88, 88, 92, 92, 93, 93, 95, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 34, 35, 36, 38, 39, 41, 44, 44, 48, 48, 53, 53, 57, 58, 61, 65, 67, 71, 72, 79, 79, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 36, 38, 39, 41, 42, 45, 45, 49, 50, 53, 54, 57, 60, 62, 66, 66, 73, 73, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 36, 37, 39, 41, 42, 45, 45, 49, 49, 52, 54, 57, 60, 61, 65, 66, 72, 72, + 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 36, 36, 37, 37, 38, 40, 41, 42, 43, 46, 46, 49, 50, 52, 54, 56, 59, 60, 64, 64, 71, 71, + 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 37, 37, 38, 38, 40, 40, 41, 43, 43, 46, 46, 49, 50, 52, 54, 56, 58, 60, 63, 64, 70, 70, + 34, 34, 34, 33, 33, 34, 35, 35, 37, 37, 39, 39, 42, 43, 44, 45, 46, 48, 48, 51, 51, 54, 54, 57, 58, 60, 63, 64, 68, 68, 74, 74, + 36, 35, 35, 35, 34, 35, 36, 37, 38, 39, 42, 42, 47, 48, 49, 50, 51, 53, 54, 56, 56, 59, 60, 62, 63, 66, 68, 69, 73, 73, 79, 79, + 38, 37, 37, 36, 36, 37, 38, 38, 39, 40, 44, 44, 48, 49, 51, 52, 54, 56, 56, 59, 59, 62, 63, 65, 67, 69, 71, 72, 76, 76, 82, 82, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 43, 48, 48, 52, 54, 56, 58, 60, 63, 64, 67, 67, 71, 71, 74, 75, 77, 79, 81, 84, 85, 90, 90, + 44, 43, 43, 42, 41, 42, 43, 43, 43, 44, 48, 48, 53, 54, 57, 58, 60, 64, 64, 67, 67, 71, 72, 75, 76, 78, 80, 82, 85, 86, 91, 91, + 53, 51, 51, 50, 49, 49, 50, 49, 49, 50, 54, 54, 59, 60, 63, 65, 67, 71, 72, 76, 76, 81, 82, 85, 87, 89, 92, 94, 97, 98, 104, 104, + 53, 51, 51, 50, 49, 49, 50, 49, 49, 50, 54, 54, 59, 60, 63, 65, 67, 71, 72, 76, 76, 81, 82, 85, 87, 89, 92, 94, 97, 98, 104, 104, + 62, 60, 59, 58, 57, 57, 57, 56, 56, 56, 61, 61, 65, 66, 69, 71, 74, 78, 79, 83, 83, 89, 90, 94, 95, 98, 102, 103, 108, 108, 115, 115, + 65, 62, 62, 60, 59, 59, 59, 59, 58, 58, 63, 63, 67, 68, 71, 73, 76, 79, 81, 85, 85, 91, 92, 96, 98, 101, 105, 106, 111, 111, 118, 118, + 73, 70, 69, 67, 66, 66, 65, 65, 64, 64, 69, 69, 73, 74, 77, 79, 81, 85, 86, 91, 91, 98, 99, 103, 105, 108, 112, 114, 119, 119, 127, 127, + 79, 75, 75, 73, 72, 71, 71, 70, 69, 69, 73, 73, 77, 78, 81, 84, 86, 90, 91, 96, 96, 103, 103, 108, 110, 114, 118, 120, 125, 125, 133, 133, + }, { + 32, 31, 31, 30, 30, 32, 33, 34, 37, 37, 42, 42, 47, 49, 48, 48, 48, 49, 49, 50, 50, 52, 52, 53, 54, 55, 57, 58, 60, 60, 63, 63, + 31, 31, 31, 32, 32, 33, 35, 37, 40, 40, 43, 43, 46, 47, 46, 46, 46, 45, 46, 47, 47, 48, 48, 50, 50, 51, 52, 53, 55, 55, 58, 58, + 31, 31, 31, 32, 32, 34, 36, 37, 40, 40, 43, 43, 46, 46, 46, 46, 45, 45, 45, 46, 46, 48, 48, 49, 50, 51, 52, 53, 54, 55, 57, 57, + 35, 36, 36, 37, 37, 39, 40, 42, 45, 45, 46, 46, 47, 47, 47, 46, 46, 45, 46, 46, 46, 47, 47, 48, 49, 50, 51, 51, 53, 53, 56, 56, + 37, 38, 38, 39, 40, 41, 43, 44, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 46, 47, 47, 48, 49, 49, 50, 51, 52, 53, 55, 55, + 42, 42, 42, 42, 42, 44, 45, 45, 47, 47, 48, 48, 50, 50, 50, 50, 49, 49, 49, 50, 50, 50, 50, 51, 52, 52, 53, 54, 55, 55, 58, 58, + 48, 47, 47, 46, 46, 46, 47, 47, 47, 48, 50, 50, 52, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 56, 56, 57, 58, 59, 60, 60, + 48, 47, 47, 46, 46, 46, 46, 47, 47, 47, 50, 50, 52, 53, 53, 54, 54, 55, 55, 55, 55, 56, 56, 57, 57, 58, 58, 59, 60, 60, 62, 62, + 49, 47, 47, 46, 45, 45, 46, 45, 45, 46, 49, 49, 53, 53, 55, 56, 57, 58, 58, 59, 59, 61, 61, 62, 62, 63, 64, 64, 65, 65, 67, 67, + 49, 47, 47, 46, 45, 45, 46, 46, 46, 46, 49, 49, 53, 54, 55, 56, 57, 59, 59, 60, 60, 61, 61, 62, 63, 63, 64, 65, 66, 66, 68, 68, + 52, 50, 50, 48, 48, 48, 47, 47, 47, 47, 50, 50, 53, 54, 56, 57, 59, 61, 62, 64, 64, 66, 66, 68, 68, 69, 70, 71, 72, 73, 75, 75, + 52, 50, 50, 48, 48, 48, 47, 47, 47, 47, 50, 50, 53, 54, 56, 57, 59, 61, 62, 64, 64, 66, 66, 68, 68, 69, 70, 71, 72, 73, 75, 75, + 56, 54, 53, 52, 51, 51, 50, 50, 49, 49, 53, 53, 55, 56, 58, 59, 61, 63, 64, 66, 66, 69, 70, 71, 72, 74, 75, 76, 77, 78, 80, 80, + 57, 54, 54, 52, 52, 51, 51, 51, 50, 50, 53, 53, 56, 57, 58, 60, 61, 64, 64, 67, 67, 70, 71, 72, 73, 75, 76, 77, 79, 79, 82, 82, + 61, 58, 57, 56, 55, 54, 54, 53, 52, 53, 56, 56, 58, 59, 61, 62, 63, 66, 66, 69, 69, 72, 73, 75, 76, 78, 79, 80, 82, 83, 86, 86, + 63, 60, 60, 58, 57, 57, 56, 55, 54, 55, 57, 57, 60, 60, 62, 64, 65, 67, 68, 71, 71, 74, 75, 77, 78, 80, 82, 83, 85, 85, 89, 89, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 33, 34, 34, 36, 36, 38, 39, 41, 44, 44, 47, 48, 50, 53, 53, 57, 58, 61, 65, 65, 70, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, 37, 38, 39, 41, 41, 44, 45, 47, 50, 50, 54, 55, 57, 61, 61, 65, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 37, 37, 39, 41, 41, 44, 45, 46, 49, 49, 53, 54, 56, 60, 60, 64, + 31, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 35, 35, 35, 36, 36, 38, 39, 40, 42, 42, 44, 45, 47, 50, 50, 53, 54, 56, 59, 59, 63, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, 40, 40, 41, 43, 43, 45, 46, 47, 50, 50, 53, 54, 56, 58, 58, 62, + 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 38, 38, 40, 40, 41, 43, 43, 45, 46, 47, 50, 50, 53, 54, 56, 58, 58, 62, + 35, 35, 35, 34, 34, 34, 35, 36, 36, 37, 37, 40, 41, 43, 46, 46, 47, 48, 49, 51, 51, 53, 54, 55, 57, 57, 60, 61, 63, 66, 66, 70, + 36, 35, 35, 35, 34, 34, 36, 36, 37, 38, 38, 41, 42, 44, 48, 48, 50, 50, 51, 53, 53, 56, 56, 58, 60, 60, 63, 63, 65, 68, 68, 72, + 39, 38, 38, 37, 37, 37, 38, 38, 39, 40, 40, 43, 44, 46, 50, 50, 52, 53, 54, 57, 57, 59, 60, 61, 64, 64, 67, 68, 69, 72, 72, 76, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 46, 48, 50, 54, 54, 57, 58, 60, 63, 63, 66, 67, 68, 71, 71, 74, 75, 77, 79, 79, 83, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 46, 48, 50, 54, 54, 57, 58, 60, 63, 63, 66, 67, 68, 71, 71, 74, 75, 77, 79, 79, 83, + 51, 49, 49, 48, 47, 47, 48, 48, 48, 48, 48, 52, 53, 55, 58, 58, 62, 63, 66, 69, 69, 73, 74, 76, 79, 79, 83, 84, 86, 89, 89, 93, + 53, 52, 51, 50, 49, 49, 49, 50, 49, 49, 49, 53, 54, 56, 60, 60, 64, 65, 67, 71, 71, 75, 76, 78, 82, 82, 86, 87, 89, 92, 92, 96, + 58, 56, 55, 54, 53, 53, 53, 53, 53, 52, 52, 56, 57, 59, 63, 63, 67, 68, 70, 74, 74, 78, 79, 82, 86, 86, 90, 91, 93, 97, 97, 101, + 65, 63, 62, 61, 59, 59, 59, 59, 58, 58, 58, 62, 63, 65, 68, 68, 72, 73, 76, 79, 79, 84, 85, 88, 92, 92, 97, 98, 100, 105, 105, 109, + 65, 63, 62, 61, 59, 59, 59, 59, 58, 58, 58, 62, 63, 65, 68, 68, 72, 73, 76, 79, 79, 84, 85, 88, 92, 92, 97, 98, 100, 105, 105, 109, + }, { + 32, 31, 31, 31, 30, 30, 33, 33, 35, 37, 37, 41, 42, 44, 49, 49, 48, 48, 48, 49, 49, 50, 50, 51, 52, 52, 54, 54, 55, 57, 57, 59, + 31, 31, 31, 31, 32, 32, 34, 35, 37, 39, 39, 42, 42, 44, 47, 47, 46, 46, 46, 46, 46, 47, 47, 48, 48, 48, 50, 51, 51, 53, 53, 55, + 31, 31, 31, 32, 32, 32, 35, 36, 37, 40, 40, 42, 43, 44, 46, 46, 46, 46, 45, 45, 45, 46, 46, 47, 48, 48, 49, 50, 51, 52, 52, 54, + 33, 34, 34, 34, 35, 35, 37, 38, 40, 43, 43, 44, 44, 45, 47, 47, 46, 46, 46, 45, 45, 46, 46, 47, 47, 47, 49, 49, 50, 51, 51, 53, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 49, 49, 50, 50, 52, + 37, 38, 38, 39, 40, 40, 42, 43, 44, 47, 47, 47, 47, 47, 48, 48, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 49, 49, 50, 50, 52, + 45, 45, 45, 45, 44, 44, 46, 46, 46, 47, 47, 49, 49, 50, 52, 52, 51, 51, 51, 51, 51, 52, 52, 52, 53, 53, 54, 54, 54, 55, 55, 57, + 48, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 49, 50, 51, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 54, 55, 55, 56, 56, 56, 58, + 48, 47, 47, 46, 45, 45, 46, 46, 46, 47, 47, 49, 50, 51, 53, 53, 54, 54, 54, 55, 55, 56, 56, 56, 57, 57, 58, 58, 58, 59, 59, 61, + 49, 47, 47, 46, 45, 45, 45, 46, 45, 45, 45, 48, 49, 51, 53, 53, 55, 56, 57, 58, 58, 59, 59, 60, 61, 61, 62, 62, 63, 64, 64, 65, + 49, 47, 47, 46, 45, 45, 45, 46, 45, 45, 45, 48, 49, 51, 53, 53, 55, 56, 57, 58, 58, 59, 59, 60, 61, 61, 62, 62, 63, 64, 64, 65, + 51, 50, 49, 48, 47, 47, 47, 47, 47, 46, 46, 49, 50, 52, 54, 54, 56, 57, 58, 61, 61, 62, 63, 64, 65, 65, 67, 67, 68, 69, 69, 70, + 52, 50, 50, 49, 48, 48, 47, 47, 47, 47, 47, 50, 50, 52, 54, 54, 57, 57, 59, 61, 61, 63, 64, 65, 66, 66, 68, 68, 69, 70, 70, 72, + 54, 52, 51, 51, 49, 49, 49, 49, 48, 48, 48, 51, 51, 53, 55, 55, 58, 58, 60, 62, 62, 64, 65, 66, 68, 68, 70, 70, 71, 73, 73, 74, + 57, 55, 54, 53, 52, 52, 51, 51, 51, 50, 50, 52, 53, 54, 57, 57, 59, 60, 61, 64, 64, 66, 67, 68, 71, 71, 73, 73, 74, 76, 76, 78, + 57, 55, 54, 53, 52, 52, 51, 51, 51, 50, 50, 52, 53, 54, 57, 57, 59, 60, 61, 64, 64, 66, 67, 68, 71, 71, 73, 73, 74, 76, 76, 78, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 34, 34, 35, 36, 36, 38, 39, 39, 42, 44, 44, 47, 48, 49, 53, 53, 55, 58, 58, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 35, 37, 38, 38, 40, 42, 42, 45, 46, 47, 50, 51, 52, 55, 55, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 36, 37, 37, 40, 41, 41, 44, 45, 46, 49, 49, 51, 54, 54, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 34, 34, 34, 35, 35, 37, 38, 38, 40, 41, 41, 44, 45, 46, 49, 49, 51, 54, 54, + 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 35, 36, 36, 37, 37, 37, 39, 40, 40, 42, 42, 43, 45, 46, 47, 49, 50, 51, 54, 54, + 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 35, 35, 36, 37, 37, 37, 38, 38, 40, 40, 40, 42, 43, 43, 45, 46, 47, 49, 50, 51, 54, 54, + 32, 33, 33, 33, 33, 33, 33, 34, 34, 35, 36, 36, 36, 38, 38, 39, 40, 40, 41, 42, 42, 44, 45, 45, 47, 48, 48, 51, 51, 53, 55, 55, + 35, 35, 35, 35, 34, 34, 35, 36, 36, 37, 38, 38, 39, 42, 42, 44, 47, 47, 48, 49, 49, 51, 52, 52, 54, 55, 56, 58, 59, 60, 62, 62, + 36, 35, 35, 35, 35, 34, 35, 36, 36, 37, 38, 38, 40, 42, 42, 45, 48, 48, 49, 50, 50, 52, 53, 54, 56, 56, 57, 59, 60, 61, 63, 63, + 38, 37, 37, 37, 36, 36, 36, 38, 38, 38, 39, 39, 41, 44, 44, 46, 49, 49, 51, 52, 52, 55, 56, 56, 58, 59, 60, 62, 63, 64, 67, 67, + 44, 43, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 44, 48, 48, 50, 54, 54, 56, 58, 58, 61, 63, 63, 66, 67, 67, 71, 71, 72, 75, 75, + 44, 43, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 44, 48, 48, 50, 54, 54, 56, 58, 58, 61, 63, 63, 66, 67, 67, 71, 71, 72, 75, 75, + 47, 46, 45, 45, 44, 44, 44, 45, 45, 45, 45, 45, 47, 50, 50, 53, 56, 56, 58, 60, 60, 64, 66, 66, 69, 70, 71, 74, 75, 76, 79, 79, + 53, 52, 51, 51, 49, 49, 49, 49, 50, 49, 49, 49, 51, 54, 54, 57, 60, 60, 63, 65, 65, 69, 71, 72, 75, 76, 77, 81, 82, 83, 87, 87, + 53, 52, 51, 51, 49, 49, 49, 49, 50, 49, 49, 49, 51, 54, 54, 57, 60, 60, 63, 65, 65, 69, 71, 72, 75, 76, 77, 81, 82, 83, 87, 87, + 59, 57, 56, 56, 54, 54, 54, 54, 54, 54, 53, 53, 55, 58, 58, 61, 64, 64, 67, 69, 69, 73, 75, 76, 79, 80, 81, 86, 87, 88, 92, 92, + }, { + 32, 31, 31, 31, 30, 30, 31, 33, 33, 34, 37, 37, 39, 42, 42, 45, 49, 49, 48, 48, 48, 49, 49, 49, 50, 50, 51, 52, 52, 53, 54, 54, + 31, 31, 31, 31, 31, 31, 32, 35, 35, 36, 39, 39, 40, 42, 42, 45, 47, 47, 47, 46, 46, 46, 46, 46, 47, 48, 48, 49, 49, 50, 51, 51, + 31, 31, 31, 31, 32, 32, 33, 35, 36, 37, 40, 40, 41, 43, 43, 44, 46, 46, 46, 46, 46, 45, 45, 45, 46, 46, 47, 48, 48, 48, 50, 50, + 31, 32, 32, 32, 32, 33, 33, 36, 36, 37, 41, 41, 42, 43, 43, 45, 47, 47, 46, 46, 46, 45, 45, 45, 46, 46, 47, 48, 48, 48, 50, 50, + 35, 36, 37, 37, 38, 38, 38, 41, 41, 42, 45, 45, 46, 46, 46, 47, 48, 48, 47, 46, 46, 46, 45, 46, 46, 46, 47, 47, 47, 48, 49, 49, + 37, 38, 38, 38, 39, 40, 40, 43, 43, 44, 47, 47, 47, 47, 47, 47, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 49, 49, + 38, 39, 40, 40, 40, 41, 41, 43, 44, 45, 47, 47, 47, 48, 48, 48, 49, 49, 48, 48, 48, 47, 47, 47, 48, 48, 48, 48, 48, 49, 50, 50, + 47, 46, 46, 46, 45, 45, 45, 46, 46, 47, 47, 47, 48, 50, 50, 51, 52, 52, 52, 52, 52, 52, 52, 52, 53, 53, 53, 53, 53, 54, 55, 55, + 48, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 48, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, 54, 54, 55, 55, + 48, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 50, 50, 51, 53, 53, 53, 54, 54, 54, 55, 55, 55, 55, 55, 56, 56, 56, 57, 57, + 49, 48, 47, 47, 45, 45, 45, 45, 46, 45, 45, 45, 47, 49, 49, 51, 53, 53, 55, 56, 56, 57, 58, 58, 59, 59, 60, 61, 61, 61, 62, 62, + 49, 48, 47, 47, 45, 45, 45, 45, 46, 45, 45, 45, 47, 49, 49, 51, 53, 53, 55, 56, 56, 57, 58, 58, 59, 59, 60, 61, 61, 61, 62, 62, + 50, 49, 48, 48, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 52, 54, 54, 55, 56, 56, 58, 59, 60, 61, 61, 61, 63, 63, 63, 65, 65, + 52, 50, 50, 50, 48, 48, 48, 47, 47, 47, 47, 47, 48, 50, 50, 52, 54, 54, 56, 57, 57, 60, 61, 61, 63, 64, 64, 66, 66, 67, 68, 68, + 52, 50, 50, 50, 48, 48, 48, 47, 47, 47, 47, 47, 48, 50, 50, 52, 54, 54, 56, 57, 57, 60, 61, 61, 63, 64, 64, 66, 66, 67, 68, 68, + 54, 53, 52, 52, 50, 50, 50, 49, 49, 49, 48, 48, 50, 52, 52, 54, 55, 55, 57, 59, 59, 61, 62, 63, 65, 65, 66, 68, 68, 69, 71, 71, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 34, 34, 34, 35, 36, 36, 38, 39, 39, 41, 44, 44, 44, 47, 48, 48, 51, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 35, 35, 37, 38, 38, 40, 42, 42, 43, 45, 46, 46, 49, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 34, 36, 37, 37, 39, 41, 41, 42, 44, 45, 45, 47, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 34, 36, 37, 37, 39, 41, 41, 42, 44, 45, 45, 47, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 36, 36, 37, 39, 39, 40, 42, 42, 42, 44, 45, 45, 48, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 38, 38, 38, 40, 40, 40, 41, 43, 43, 43, 45, 46, 46, 48, + 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 38, 38, 38, 40, 40, 40, 41, 43, 43, 43, 45, 46, 46, 48, + 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 35, 36, 36, 36, 38, 38, 38, 39, 40, 40, 41, 42, 42, 43, 45, 45, 45, 47, 48, 48, 50, + 35, 35, 35, 35, 34, 34, 34, 34, 35, 36, 36, 37, 37, 37, 39, 41, 41, 42, 45, 46, 46, 47, 48, 48, 49, 51, 51, 51, 53, 54, 54, 56, + 36, 35, 35, 35, 35, 34, 34, 35, 36, 36, 36, 37, 38, 38, 40, 42, 42, 43, 47, 48, 48, 49, 50, 50, 51, 53, 53, 54, 56, 56, 56, 58, + 36, 35, 35, 35, 35, 34, 34, 35, 36, 36, 36, 37, 38, 38, 40, 42, 42, 43, 47, 48, 48, 49, 50, 50, 51, 53, 53, 54, 56, 56, 56, 58, + 40, 39, 39, 39, 39, 38, 38, 38, 39, 39, 39, 40, 41, 41, 42, 45, 45, 46, 50, 51, 51, 53, 54, 54, 56, 59, 59, 59, 61, 62, 62, 64, + 44, 43, 42, 42, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 44, 48, 48, 49, 52, 54, 54, 56, 58, 58, 60, 63, 63, 64, 66, 67, 67, 69, + 44, 43, 42, 42, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 44, 48, 48, 49, 52, 54, 54, 56, 58, 58, 60, 63, 63, 64, 66, 67, 67, 69, + 47, 46, 45, 45, 45, 44, 44, 44, 44, 45, 45, 45, 45, 45, 47, 50, 50, 51, 55, 56, 56, 58, 60, 60, 62, 66, 66, 67, 69, 70, 70, 73, + 53, 52, 51, 51, 50, 49, 49, 49, 49, 50, 50, 49, 49, 49, 51, 54, 54, 55, 59, 60, 60, 63, 65, 65, 67, 71, 71, 72, 75, 76, 76, 79, + }, { + 32, 31, 31, 31, 31, 30, 30, 31, 33, 33, 33, 35, 37, 37, 39, 42, 42, 43, 47, 49, 49, 48, 48, 48, 48, 49, 49, 49, 50, 50, 50, 51, + 31, 31, 31, 31, 31, 31, 31, 32, 34, 35, 35, 37, 39, 39, 40, 42, 42, 43, 46, 47, 47, 47, 47, 47, 47, 46, 46, 47, 48, 48, 48, 49, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, 43, 43, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 46, 46, 46, 47, + 31, 31, 31, 31, 32, 32, 32, 33, 35, 36, 36, 38, 40, 40, 41, 43, 43, 43, 46, 46, 46, 46, 46, 46, 45, 45, 45, 45, 46, 46, 46, 47, + 33, 33, 34, 34, 34, 35, 35, 35, 37, 38, 38, 41, 43, 43, 43, 44, 44, 45, 46, 47, 47, 46, 46, 46, 46, 45, 45, 45, 46, 46, 46, 47, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 47, + 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, 45, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 47, + 38, 39, 40, 40, 40, 41, 41, 41, 43, 44, 44, 46, 47, 47, 47, 48, 48, 48, 48, 49, 49, 48, 48, 48, 47, 47, 47, 47, 48, 48, 48, 48, + 45, 45, 45, 45, 45, 44, 44, 45, 46, 46, 46, 47, 47, 47, 48, 49, 49, 50, 51, 52, 52, 52, 51, 51, 51, 51, 51, 52, 52, 52, 52, 52, + 48, 47, 47, 47, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 50, 50, 50, 52, 53, 53, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, + 48, 47, 47, 47, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 50, 50, 50, 52, 53, 53, 53, 53, 53, 53, 53, 53, 53, 54, 54, 54, 54, + 49, 48, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 47, 49, 49, 50, 52, 53, 53, 54, 54, 54, 55, 56, 56, 56, 57, 57, 57, 58, + 49, 48, 47, 47, 46, 45, 45, 45, 45, 46, 46, 45, 45, 45, 47, 49, 49, 50, 53, 53, 53, 55, 56, 56, 57, 58, 58, 58, 59, 59, 59, 60, + 49, 48, 47, 47, 46, 45, 45, 45, 45, 46, 46, 45, 45, 45, 47, 49, 49, 50, 53, 53, 53, 55, 56, 56, 57, 58, 58, 58, 59, 59, 59, 60, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 50, 53, 54, 54, 55, 56, 56, 57, 59, 59, 60, 61, 61, 61, 62, + 52, 51, 50, 50, 49, 48, 48, 48, 47, 47, 47, 47, 47, 47, 48, 50, 50, 51, 53, 54, 54, 56, 57, 57, 59, 61, 61, 62, 63, 64, 64, 65, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 36, 36, 36, 37, 39, 39, 39, 41, 44, 44, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, 35, 35, 37, 38, 38, 38, 40, 42, 42, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 36, 37, 37, 37, 39, 41, 41, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 36, 37, 37, 37, 39, 41, 41, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 36, 37, 37, 37, 39, 41, 41, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 36, 36, 36, 38, 39, 39, 39, 40, 42, 42, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 37, 37, 38, 38, 38, 39, 40, 40, 40, 42, 43, 43, + 34, 34, 34, 34, 34, 34, 33, 33, 33, 34, 35, 35, 35, 36, 37, 37, 37, 38, 39, 39, 39, 41, 43, 43, 43, 44, 45, 45, 45, 46, 48, 48, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 49, 50, 50, 50, 52, 53, 53, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 49, 50, 50, 50, 52, 53, 53, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 49, 50, 50, 50, 52, 53, 53, + 39, 39, 38, 38, 38, 38, 37, 37, 37, 38, 39, 39, 39, 40, 40, 40, 40, 42, 45, 45, 45, 47, 51, 51, 51, 52, 54, 54, 54, 56, 58, 58, + 44, 43, 42, 42, 42, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 45, 48, 48, 48, 50, 54, 54, 54, 56, 58, 58, 58, 60, 63, 63, + 44, 43, 42, 42, 42, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 45, 48, 48, 48, 50, 54, 54, 54, 56, 58, 58, 58, 60, 63, 63, + }, { + 32, 31, 31, 31, 31, 31, 30, 30, 30, 32, 33, 33, 33, 35, 37, 37, 37, 39, 42, 42, 42, 45, 49, 49, 49, 48, 48, 48, 48, 48, 49, 49, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 33, 34, 34, 34, 36, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, 47, 47, 47, 47, 47, 47, 47, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 36, 36, 36, 38, 40, 40, 40, 41, 43, 43, 43, 44, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 36, 36, 36, 38, 40, 40, 40, 41, 43, 43, 43, 44, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 34, 36, 36, 36, 38, 40, 40, 40, 41, 43, 43, 43, 44, 46, 46, 46, 46, 46, 46, 46, 45, 45, 45, + 33, 34, 34, 34, 34, 35, 35, 35, 35, 37, 39, 39, 39, 41, 43, 43, 43, 44, 45, 45, 45, 46, 47, 47, 47, 47, 46, 46, 46, 46, 45, 45, + 37, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, + 37, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, + 37, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 47, 47, 47, 47, 46, 46, 46, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 44, 45, 45, 45, 46, 47, 47, 47, 48, 48, 48, 48, 49, 50, 50, 50, 50, 50, 50, 50, 49, 49, 49, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 49, 50, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 49, 50, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 49, 50, 50, 50, 51, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 48, 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 48, 50, 50, 50, 51, 53, 53, 53, 54, 54, 54, 54, 55, 56, 56, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 46, 46, 46, 45, 45, 45, 45, 47, 49, 49, 49, 51, 53, 53, 53, 54, 56, 56, 56, 57, 58, 58, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 46, 46, 46, 45, 45, 45, 45, 47, 49, 49, 49, 51, 53, 53, 53, 54, 56, 56, 56, 57, 58, 58, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, 36, 36, 36, 37, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, 36, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 36, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 36, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 36, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 36, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, 35, 36, 36, 36, 36, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 37, 38, + 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, + 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, + 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 36, 36, 36, 36, 37, 38, 39, 39, 39, 40, 41, 42, 42, 42, 42, + 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 39, 40, 41, 41, 41, 43, 45, 46, 46, 46, 46, + 36, 35, 35, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 41, 42, 42, 42, 44, 47, 48, 48, 48, 49, + 36, 35, 35, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 41, 42, 42, 42, 44, 47, 48, 48, 48, 49, + 36, 35, 35, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 41, 42, 42, 42, 44, 47, 48, 48, 48, 49, + }, { + 32, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 31, 33, 33, 33, 33, 35, 36, 37, 37, 37, 39, 41, 42, 42, 42, 44, 47, 49, 49, 49, 49, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 34, 34, 34, 34, 36, 37, 38, 38, 38, 39, 41, 42, 42, 42, 44, 46, 48, 48, 48, 48, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 33, 34, 35, 35, 35, 37, 38, 39, 39, 39, 40, 42, 42, 42, 42, 44, 46, 47, 47, 47, 47, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, 37, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 46, 46, 46, 46, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, 37, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 46, 46, 46, 46, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 33, 35, 36, 36, 36, 37, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 46, 46, 46, 46, + 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 36, 37, 38, 38, 38, 40, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, + 35, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 40, 41, 41, 41, 43, 44, 45, 45, 45, 46, 46, 46, 46, 46, 47, 47, 48, 48, 48, 47, + 37, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 47, + 37, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 47, + 37, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 47, + 40, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 43, 44, 44, 44, 44, 45, 47, 47, 47, 47, 48, 48, 48, 48, 48, 49, 49, 50, 50, 50, 49, + 45, 45, 45, 45, 45, 45, 45, 44, 44, 44, 44, 45, 46, 46, 46, 46, 46, 47, 47, 47, 47, 48, 49, 49, 49, 49, 50, 51, 52, 52, 52, 52, + 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, 53, 53, 53, 53, + 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, 53, 53, 53, 53, + 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, 53, 53, 53, 53, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 36, 36, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, + 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 36, 36, 36, 36, 36, 36, 37, 38, 38, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 39, 39, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 31, 32, 33, 33, 33, 33, 33, 34, 35, 36, 37, 37, 37, 37, 39, 40, 42, 42, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 34, 34, 34, 35, 36, 37, 38, 38, 38, 38, 39, 41, 42, 42, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 35, 35, 35, 35, 36, 37, 38, 39, 39, 39, 39, 40, 41, 42, 42, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 35, 35, 35, 35, 35, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 33, 34, 35, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, 42, 43, 43, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 34, 35, 36, 36, 36, 36, 37, 39, 40, 41, 41, 41, 41, 42, 42, 43, 43, + 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 36, 37, 38, 38, 38, 38, 39, 41, 42, 43, 43, 43, 43, 43, 44, 44, 44, + 35, 35, 35, 36, 36, 36, 36, 36, 36, 37, 37, 37, 37, 37, 38, 39, 40, 40, 40, 40, 40, 42, 43, 44, 45, 45, 45, 45, 45, 45, 46, 46, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 45, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 38, 39, 39, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 42, 43, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, 47, 47, 47, 48, 48, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 44, 44, 45, 45, 45, 45, 45, 46, 47, 47, 47, 47, 47, 48, 48, 48, 48, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, 31, 31, 32, 33, 33, 33, 33, 33, 33, 33, 34, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 33, 34, 34, 34, 34, 34, 34, 34, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 35, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 33, 33, 34, 35, 35, 35, 35, 35, 35, 35, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 35, 35, 36, 37, 37, 37, 37, 37, 37, 38, + 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 38, 38, 38, 38, 38, 39, + 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 36, 36, 36, 36, 36, 36, 36, 36, 37, 37, 38, 39, 40, 40, 40, 40, 40, 40, 40, + 35, 35, 36, 36, 36, 37, 37, 37, 37, 37, 37, 37, 37, 37, 38, 38, 38, 38, 38, 38, 38, 38, 39, 40, 40, 41, 41, 41, 41, 41, 41, 42, + 37, 37, 37, 38, 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 41, 41, 42, 43, 43, 43, 43, 43, 43, 44, + }, + }, { + { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, { + 32, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, 30, 30, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, + }, +}; + +static const uint8_t qm_tbl_32x32_t[][2][528] = { + { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 33, 33, + 32, 32, 32, 33, 34, 35, + 34, 34, 33, 34, 35, 37, 39, + 35, 34, 34, 35, 36, 37, 41, 43, + 36, 35, 34, 35, 36, 38, 42, 45, 48, + 39, 38, 37, 38, 39, 40, 45, 47, 50, 54, + 44, 42, 41, 41, 42, 42, 47, 50, 54, 58, 63, + 46, 44, 42, 43, 44, 44, 49, 52, 55, 59, 65, 67, + 48, 46, 44, 45, 45, 46, 51, 53, 57, 61, 67, 69, 71, + 54, 51, 49, 49, 50, 49, 54, 57, 60, 65, 71, 74, 76, 82, + 59, 56, 54, 54, 54, 53, 58, 61, 64, 69, 75, 78, 80, 87, 92, + 62, 59, 56, 56, 56, 55, 60, 63, 66, 71, 77, 80, 83, 89, 95, 98, + 65, 62, 59, 59, 59, 58, 63, 65, 68, 73, 79, 82, 85, 92, 98, 101, 105, + 71, 68, 65, 64, 64, 63, 68, 70, 73, 78, 84, 87, 90, 97, 103, 107, 111, 117, + 80, 76, 72, 72, 71, 69, 74, 76, 79, 84, 90, 93, 96, 104, 110, 114, 118, 125, 134, + 81, 77, 73, 73, 72, 70, 75, 77, 80, 85, 91, 94, 97, 105, 111, 115, 119, 126, 135, 137, + 83, 78, 75, 74, 74, 72, 76, 79, 81, 86, 92, 95, 99, 106, 113, 117, 121, 128, 137, 138, 140, + 88, 84, 80, 79, 78, 76, 80, 82, 85, 91, 95, 98, 103, 111, 115, 119, 126, 134, 139, 144, 147, 152, + 91, 86, 83, 82, 81, 79, 81, 84, 88, 92, 95, 100, 107, 110, 115, 123, 127, 132, 140, 147, 151, 154, 159, + 94, 89, 86, 85, 84, 82, 82, 86, 90, 92, 97, 103, 105, 111, 119, 121, 128, 136, 139, 146, 156, 158, 161, 166, + 97, 92, 90, 88, 86, 85, 84, 89, 91, 95, 100, 102, 108, 114, 116, 125, 130, 133, 143, 148, 152, 163, 166, 168, 174, + 101, 95, 93, 91, 89, 89, 87, 91, 93, 98, 101, 105, 111, 113, 120, 126, 130, 138, 142, 149, 157, 159, 171, 174, 176, 183, + 104, 99, 97, 94, 93, 93, 90, 92, 96, 100, 102, 108, 111, 116, 122, 125, 134, 137, 144, 151, 155, 165, 169, 179, 182, 184, 191, + 107, 102, 101, 97, 96, 96, 93, 93, 99, 101, 105, 110, 113, 120, 122, 129, 133, 140, 146, 150, 161, 163, 173, 178, 187, 191, 193, 200, + 111, 105, 104, 101, 100, 99, 97, 96, 102, 103, 109, 111, 117, 120, 125, 131, 135, 143, 146, 156, 158, 168, 173, 180, 189, 195, 200, 202, 210, + 115, 109, 108, 104, 104, 102, 101, 100, 103, 106, 111, 113, 119, 121, 129, 131, 140, 142, 151, 155, 162, 168, 176, 183, 188, 199, 204, 210, 212, 220, + 119, 113, 112, 107, 107, 106, 105, 103, 105, 110, 112, 117, 120, 125, 130, 135, 140, 145, 152, 157, 165, 169, 179, 183, 193, 197, 210, 214, 220, 222, 231, + 123, 116, 116, 111, 111, 109, 110, 107, 107, 114, 114, 121, 122, 130, 130, 140, 140, 150, 151, 163, 164, 176, 177, 190, 191, 204, 206, 222, 224, 230, 232, 242, + }, { + 32, + 31, 31, + 30, 31, 32, + 32, 33, 33, 35, + 33, 34, 35, 37, 39, + 36, 38, 40, 41, 43, 47, + 41, 42, 42, 43, 45, 47, 48, + 45, 45, 44, 45, 46, 47, 49, 50, + 49, 47, 46, 47, 47, 48, 50, 51, 53, + 48, 47, 45, 46, 46, 46, 49, 51, 53, 54, + 49, 47, 45, 45, 45, 45, 49, 51, 53, 55, 58, + 50, 47, 45, 46, 46, 46, 49, 51, 54, 56, 59, 60, + 50, 48, 46, 46, 46, 46, 50, 52, 54, 56, 60, 60, 61, + 52, 50, 47, 47, 47, 47, 50, 52, 54, 57, 61, 62, 63, 66, + 54, 52, 49, 49, 49, 48, 52, 53, 55, 58, 62, 64, 65, 68, 71, + 56, 53, 51, 50, 50, 49, 52, 54, 56, 59, 63, 64, 66, 69, 72, 73, + 57, 54, 52, 51, 51, 50, 53, 55, 56, 60, 63, 65, 67, 70, 73, 75, 76, + 60, 57, 54, 54, 53, 52, 55, 57, 58, 61, 65, 67, 68, 72, 75, 77, 79, 82, + 63, 60, 57, 57, 56, 54, 57, 59, 60, 63, 67, 69, 71, 75, 78, 80, 82, 85, 89, + 64, 61, 58, 57, 57, 55, 58, 59, 61, 64, 67, 69, 71, 75, 78, 80, 82, 85, 89, 90, + 65, 61, 58, 58, 57, 55, 58, 60, 61, 64, 68, 70, 71, 75, 79, 81, 83, 86, 90, 91, 91, + 67, 63, 61, 60, 59, 57, 60, 61, 63, 66, 69, 70, 73, 77, 79, 81, 85, 88, 90, 92, 94, 96, + 68, 64, 62, 61, 60, 58, 59, 61, 64, 66, 67, 71, 74, 75, 78, 82, 84, 86, 90, 93, 94, 96, 98, + 69, 65, 63, 62, 61, 59, 59, 62, 64, 65, 68, 71, 72, 75, 79, 80, 83, 87, 89, 92, 96, 97, 98, 100, + 70, 66, 64, 63, 62, 61, 60, 63, 64, 66, 69, 70, 73, 76, 77, 81, 84, 85, 89, 92, 93, 98, 99, 100, 102, + 71, 67, 66, 64, 63, 62, 61, 63, 64, 67, 68, 70, 74, 75, 78, 81, 83, 86, 88, 91, 94, 95, 100, 101, 102, 104, + 72, 68, 67, 65, 64, 64, 61, 63, 65, 67, 68, 71, 73, 75, 78, 79, 84, 85, 88, 91, 93, 97, 98, 102, 103, 104, 106, + 73, 69, 68, 66, 65, 65, 63, 63, 66, 67, 69, 71, 73, 76, 77, 81, 82, 85, 88, 90, 94, 95, 99, 101, 104, 105, 106, 109, + 74, 70, 70, 67, 66, 66, 64, 63, 66, 67, 70, 71, 74, 75, 78, 80, 82, 86, 87, 91, 92, 96, 98, 101, 104, 106, 108, 108, 111, + 75, 71, 71, 68, 68, 67, 66, 64, 66, 68, 70, 71, 74, 75, 79, 79, 84, 84, 88, 90, 93, 95, 98, 101, 103, 107, 108, 110, 111, 113, + 76, 72, 72, 69, 69, 68, 67, 65, 66, 69, 70, 72, 74, 76, 78, 81, 83, 85, 88, 90, 93, 95, 98, 100, 104, 105, 109, 111, 112, 113, 116, + 78, 74, 74, 70, 70, 69, 69, 66, 66, 70, 70, 74, 74, 77, 78, 82, 82, 86, 87, 92, 92, 96, 97, 102, 102, 107, 107, 112, 113, 115, 115, 118, + }, + }, { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 33, + 32, 32, 32, 33, 34, 35, + 32, 33, 33, 33, 34, 36, 36, + 34, 34, 33, 34, 35, 37, 38, 39, + 36, 35, 34, 35, 36, 38, 40, 42, 48, + 38, 37, 36, 36, 38, 39, 41, 44, 50, 51, + 39, 38, 37, 38, 39, 40, 42, 45, 50, 52, 54, + 44, 42, 41, 41, 42, 42, 44, 47, 54, 56, 58, 63, + 47, 45, 44, 44, 45, 45, 47, 50, 56, 58, 60, 66, 69, + 49, 47, 46, 45, 46, 46, 48, 51, 57, 60, 62, 68, 71, 73, + 54, 51, 50, 49, 50, 49, 51, 54, 60, 63, 65, 71, 75, 77, 82, + 59, 56, 54, 54, 54, 53, 55, 58, 64, 67, 69, 75, 79, 81, 87, 92, + 61, 58, 56, 56, 56, 55, 57, 60, 65, 68, 70, 77, 81, 83, 89, 94, 97, + 65, 62, 60, 59, 59, 58, 60, 63, 68, 71, 73, 79, 84, 87, 92, 98, 101, 105, + 71, 68, 65, 65, 64, 63, 65, 68, 73, 76, 78, 84, 89, 92, 97, 103, 106, 111, 117, + 76, 72, 70, 69, 68, 66, 68, 71, 76, 79, 81, 88, 92, 95, 101, 107, 110, 115, 122, 127, + 80, 76, 73, 72, 71, 69, 71, 74, 79, 82, 84, 90, 95, 98, 104, 110, 113, 118, 125, 130, 134, + 83, 78, 76, 75, 74, 72, 73, 76, 81, 84, 86, 92, 97, 100, 106, 113, 116, 121, 128, 133, 137, 140, + 86, 82, 79, 78, 77, 74, 76, 79, 84, 87, 89, 95, 100, 103, 109, 116, 119, 124, 131, 136, 140, 144, 147, + 89, 85, 82, 81, 79, 78, 78, 82, 86, 87, 92, 97, 100, 105, 112, 114, 120, 128, 131, 136, 146, 147, 150, 155, + 92, 88, 85, 84, 82, 81, 80, 85, 86, 90, 95, 97, 102, 107, 110, 117, 122, 125, 134, 138, 142, 152, 154, 156, 162, + 95, 90, 88, 86, 85, 84, 82, 86, 88, 93, 95, 99, 105, 106, 113, 118, 121, 129, 132, 139, 146, 148, 159, 161, 163, 169, + 98, 93, 91, 89, 88, 87, 85, 87, 90, 94, 96, 102, 104, 109, 114, 117, 126, 128, 134, 141, 145, 154, 157, 166, 168, 170, 176, + 101, 96, 95, 92, 91, 90, 88, 88, 93, 95, 99, 103, 106, 112, 114, 121, 124, 131, 136, 140, 149, 151, 160, 165, 173, 176, 178, 184, + 104, 99, 98, 95, 94, 93, 91, 90, 95, 96, 102, 103, 109, 112, 117, 122, 125, 133, 136, 145, 146, 156, 160, 167, 174, 180, 184, 186, 193, + 108, 102, 101, 98, 97, 96, 95, 93, 97, 100, 104, 106, 111, 113, 121, 122, 130, 132, 140, 143, 150, 155, 162, 169, 174, 183, 188, 192, 194, 201, + 111, 105, 105, 101, 100, 99, 98, 96, 98, 103, 105, 109, 112, 117, 121, 125, 130, 135, 141, 146, 152, 156, 165, 169, 178, 181, 193, 196, 201, 202, 210, + 114, 109, 109, 104, 104, 102, 102, 99, 100, 106, 106, 113, 113, 120, 121, 129, 130, 139, 140, 151, 151, 162, 162, 175, 176, 187, 188, 203, 204, 210, 211, 219, + }, { + 32, + 31, 31, + 30, 31, 31, + 31, 32, 32, 33, + 33, 34, 35, 36, 39, + 36, 38, 39, 40, 43, 47, + 38, 40, 41, 41, 44, 47, 47, + 41, 42, 42, 43, 45, 47, 48, 48, + 49, 47, 46, 46, 47, 48, 49, 50, 53, + 49, 47, 46, 46, 46, 47, 48, 50, 53, 53, + 48, 47, 46, 45, 46, 46, 48, 49, 53, 54, 54, + 49, 47, 45, 45, 45, 45, 47, 49, 53, 55, 55, 58, + 50, 48, 46, 46, 46, 46, 47, 50, 54, 55, 56, 59, 61, + 51, 48, 47, 46, 47, 46, 47, 50, 54, 55, 56, 60, 61, 62, + 52, 50, 48, 47, 47, 47, 48, 50, 54, 56, 57, 61, 63, 64, 66, + 54, 52, 50, 49, 49, 48, 49, 52, 55, 57, 58, 62, 64, 66, 68, 71, + 55, 53, 51, 50, 50, 49, 50, 52, 56, 58, 59, 63, 65, 66, 69, 72, 73, + 57, 54, 52, 51, 51, 50, 51, 53, 56, 58, 60, 63, 66, 67, 70, 73, 74, 76, + 60, 57, 55, 54, 53, 52, 53, 55, 58, 60, 61, 65, 68, 69, 72, 75, 77, 79, 82, + 62, 59, 57, 56, 55, 53, 54, 56, 59, 61, 63, 66, 69, 70, 74, 77, 78, 80, 84, 86, + 63, 60, 58, 57, 56, 54, 55, 57, 60, 62, 63, 67, 70, 71, 75, 78, 79, 82, 85, 87, 89, + 65, 61, 59, 58, 57, 55, 56, 58, 61, 63, 64, 68, 71, 72, 75, 79, 80, 83, 86, 88, 90, 91, + 66, 63, 60, 59, 58, 56, 58, 59, 62, 64, 65, 69, 72, 73, 76, 80, 81, 84, 87, 90, 91, 93, 94, + 67, 64, 62, 61, 59, 58, 58, 60, 63, 64, 66, 69, 71, 73, 77, 78, 81, 85, 86, 89, 93, 94, 95, 97, + 68, 65, 63, 62, 60, 59, 58, 61, 62, 64, 67, 68, 71, 74, 75, 79, 81, 83, 87, 89, 91, 95, 96, 97, 99, + 69, 66, 64, 63, 61, 61, 59, 61, 62, 65, 66, 68, 72, 73, 76, 78, 80, 84, 85, 88, 91, 92, 97, 98, 98, 101, + 70, 67, 65, 63, 62, 62, 60, 61, 63, 65, 66, 69, 71, 73, 76, 77, 81, 83, 85, 88, 90, 94, 95, 99, 100, 100, 103, + 71, 67, 67, 64, 63, 63, 61, 61, 64, 65, 67, 69, 71, 74, 75, 78, 80, 83, 85, 87, 91, 92, 95, 97, 100, 102, 102, 105, + 72, 68, 68, 65, 65, 64, 62, 62, 64, 65, 68, 69, 72, 73, 76, 78, 80, 83, 84, 88, 89, 93, 95, 97, 100, 102, 104, 104, 107, + 73, 69, 69, 66, 66, 65, 64, 63, 64, 66, 68, 69, 72, 73, 77, 77, 81, 82, 86, 87, 90, 92, 95, 97, 99, 103, 104, 106, 106, 109, + 74, 70, 70, 67, 67, 66, 65, 63, 64, 67, 68, 70, 72, 74, 76, 78, 80, 82, 85, 87, 90, 91, 95, 96, 100, 101, 105, 106, 108, 108, 111, + 75, 71, 71, 68, 68, 66, 66, 64, 64, 68, 68, 71, 71, 75, 75, 79, 79, 83, 84, 88, 89, 93, 93, 98, 98, 102, 103, 108, 108, 110, 110, 113, + }, + }, { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 33, + 32, 32, 32, 32, 33, 34, + 32, 32, 32, 32, 34, 34, 35, + 34, 34, 33, 33, 35, 36, 37, 39, + 34, 34, 34, 34, 36, 36, 37, 41, 42, + 36, 35, 34, 34, 36, 37, 38, 42, 45, 48, + 39, 38, 38, 37, 39, 40, 40, 45, 47, 50, 54, + 41, 39, 39, 38, 40, 40, 41, 46, 48, 51, 55, 56, + 44, 42, 41, 41, 42, 42, 42, 47, 50, 54, 58, 59, 63, + 48, 46, 45, 44, 45, 45, 45, 50, 53, 56, 61, 62, 66, 70, + 49, 47, 46, 45, 46, 46, 46, 51, 53, 57, 62, 63, 68, 71, 73, + 54, 51, 50, 49, 50, 49, 49, 54, 56, 60, 65, 67, 71, 76, 77, 82, + 58, 55, 54, 53, 53, 53, 52, 57, 59, 63, 68, 70, 74, 79, 81, 86, 90, + 59, 57, 55, 54, 54, 54, 54, 59, 61, 64, 69, 71, 75, 80, 82, 87, 91, 93, + 65, 62, 60, 59, 59, 58, 58, 63, 65, 68, 73, 75, 79, 85, 87, 92, 97, 99, 105, + 69, 66, 64, 63, 63, 62, 61, 66, 68, 71, 76, 78, 83, 88, 90, 96, 100, 102, 109, 113, + 71, 68, 66, 65, 64, 63, 63, 68, 70, 73, 78, 80, 84, 90, 92, 97, 102, 104, 111, 115, 117, + 80, 76, 73, 72, 71, 70, 69, 74, 76, 79, 84, 86, 90, 96, 98, 104, 109, 111, 118, 123, 125, 134, + 81, 77, 75, 74, 73, 72, 71, 75, 77, 80, 85, 87, 91, 97, 99, 105, 110, 112, 120, 125, 127, 136, 137, + 83, 78, 76, 75, 74, 73, 72, 76, 78, 81, 86, 88, 92, 98, 100, 106, 111, 113, 121, 126, 128, 137, 139, 140, + 87, 83, 81, 79, 78, 77, 75, 80, 82, 85, 90, 91, 96, 101, 103, 110, 114, 117, 125, 129, 133, 142, 143, 145, 150, + 90, 85, 83, 81, 80, 79, 78, 81, 83, 87, 89, 93, 98, 100, 106, 110, 114, 121, 124, 130, 136, 138, 148, 149, 151, 156, + 93, 88, 86, 84, 83, 82, 80, 82, 85, 89, 90, 96, 98, 102, 107, 109, 118, 120, 125, 131, 134, 143, 145, 153, 156, 157, 163, + 95, 90, 89, 86, 85, 85, 83, 83, 88, 89, 93, 97, 99, 105, 106, 113, 116, 122, 127, 130, 139, 140, 148, 153, 159, 162, 164, 169, + 98, 93, 92, 89, 88, 87, 86, 85, 89, 90, 96, 97, 102, 105, 109, 114, 117, 124, 126, 134, 136, 144, 148, 154, 160, 166, 169, 170, 176, + 101, 96, 95, 91, 91, 90, 89, 87, 90, 93, 97, 99, 104, 105, 112, 113, 121, 122, 130, 133, 139, 144, 150, 155, 160, 168, 172, 176, 177, 184, + 104, 99, 98, 94, 94, 92, 92, 90, 92, 96, 98, 102, 104, 109, 112, 116, 121, 125, 130, 135, 141, 144, 152, 155, 163, 166, 177, 179, 184, 185, 191, + 107, 101, 101, 97, 97, 95, 95, 93, 93, 99, 99, 105, 105, 112, 112, 120, 120, 129, 129, 139, 140, 149, 149, 161, 161, 172, 172, 185, 186, 191, 192, 199, + }, { + 32, + 31, 31, + 30, 31, 31, + 30, 31, 31, 32, + 33, 34, 35, 35, 39, + 35, 36, 37, 37, 41, 43, + 36, 38, 39, 40, 43, 45, 47, + 41, 42, 42, 42, 45, 46, 47, 48, + 44, 44, 44, 44, 46, 46, 47, 49, 50, + 49, 47, 47, 46, 47, 47, 48, 50, 51, 53, + 48, 47, 46, 45, 46, 46, 46, 49, 51, 53, 54, + 48, 47, 46, 45, 46, 46, 46, 49, 51, 53, 54, 55, + 49, 47, 46, 45, 45, 45, 45, 49, 51, 53, 55, 56, 58, + 50, 48, 47, 46, 46, 46, 46, 50, 51, 54, 56, 57, 59, 61, + 51, 48, 47, 46, 47, 46, 46, 50, 51, 54, 56, 57, 60, 62, 62, + 52, 50, 48, 47, 47, 47, 47, 50, 52, 54, 57, 58, 61, 63, 64, 66, + 54, 51, 50, 49, 49, 48, 48, 51, 53, 55, 58, 59, 62, 64, 65, 68, 70, + 55, 52, 51, 50, 49, 49, 48, 52, 53, 55, 59, 60, 62, 65, 66, 68, 70, 71, + 57, 54, 53, 52, 51, 50, 50, 53, 54, 56, 60, 61, 63, 66, 67, 70, 73, 73, 76, + 59, 56, 54, 53, 53, 52, 51, 54, 56, 58, 61, 62, 65, 68, 69, 72, 74, 75, 78, 80, + 60, 57, 55, 54, 53, 53, 52, 55, 56, 58, 61, 63, 65, 68, 69, 72, 75, 76, 79, 81, 82, + 63, 60, 58, 57, 56, 55, 54, 57, 59, 60, 63, 65, 67, 70, 71, 75, 77, 78, 82, 84, 85, 89, + 64, 61, 59, 58, 57, 56, 55, 58, 59, 61, 64, 65, 68, 71, 72, 75, 78, 79, 82, 85, 86, 89, 90, + 65, 61, 60, 58, 57, 56, 55, 58, 59, 61, 64, 65, 68, 71, 72, 75, 78, 79, 83, 85, 86, 90, 91, 91, + 67, 63, 61, 60, 59, 58, 57, 60, 61, 63, 65, 66, 69, 72, 73, 77, 79, 80, 84, 86, 88, 92, 93, 93, 95, + 68, 64, 63, 61, 60, 59, 58, 60, 61, 63, 65, 67, 70, 71, 74, 76, 78, 81, 83, 86, 88, 89, 94, 94, 95, 97, + 68, 65, 64, 62, 61, 60, 58, 59, 61, 64, 64, 68, 69, 71, 74, 75, 79, 80, 83, 86, 87, 91, 92, 95, 96, 97, 99, + 69, 66, 65, 63, 62, 61, 59, 59, 62, 63, 65, 67, 69, 72, 72, 76, 78, 80, 83, 84, 88, 89, 92, 94, 97, 98, 99, 101, + 70, 67, 66, 63, 63, 62, 61, 60, 63, 63, 66, 67, 69, 71, 73, 76, 77, 81, 82, 85, 86, 90, 91, 94, 96, 99, 100, 100, 103, + 71, 67, 67, 64, 64, 63, 62, 61, 62, 64, 66, 67, 70, 71, 74, 74, 78, 79, 83, 84, 87, 89, 91, 94, 95, 99, 100, 102, 102, 104, + 72, 68, 68, 65, 65, 64, 63, 61, 62, 65, 66, 68, 69, 71, 73, 75, 77, 79, 82, 84, 87, 88, 92, 93, 96, 97, 101, 102, 104, 104, 106, + 73, 69, 69, 66, 66, 64, 64, 62, 62, 66, 66, 69, 69, 72, 73, 76, 77, 81, 81, 85, 85, 89, 90, 94, 94, 99, 99, 104, 104, 106, 106, 108, + }, + }, { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 33, + 31, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 33, 34, 35, + 32, 33, 33, 33, 34, 34, 36, 36, + 34, 34, 34, 33, 35, 35, 37, 38, 39, + 35, 35, 34, 34, 36, 36, 38, 39, 42, 46, + 36, 35, 35, 34, 36, 36, 38, 40, 42, 47, 48, + 39, 38, 38, 37, 39, 39, 40, 42, 45, 49, 50, 54, + 41, 40, 39, 38, 40, 40, 41, 43, 46, 50, 52, 55, 57, + 44, 42, 42, 41, 42, 42, 42, 44, 47, 52, 54, 58, 60, 63, + 47, 45, 45, 44, 44, 45, 45, 47, 50, 55, 56, 60, 62, 66, 69, + 48, 46, 45, 44, 45, 45, 46, 47, 51, 55, 57, 61, 63, 67, 70, 71, + 54, 51, 50, 49, 49, 50, 49, 51, 54, 59, 60, 65, 67, 71, 75, 76, 82, + 56, 53, 52, 51, 51, 51, 51, 53, 56, 60, 61, 66, 69, 73, 77, 78, 84, 86, + 59, 56, 55, 54, 54, 54, 53, 55, 58, 62, 64, 69, 71, 75, 79, 80, 87, 89, 92, + 64, 61, 60, 58, 58, 58, 57, 59, 62, 66, 67, 72, 75, 79, 83, 84, 91, 93, 97, 102, + 65, 62, 61, 59, 59, 59, 58, 60, 63, 67, 68, 73, 75, 79, 84, 85, 92, 94, 98, 103, 105, + 71, 68, 67, 65, 64, 64, 63, 65, 68, 72, 73, 78, 80, 84, 89, 90, 97, 100, 103, 109, 111, 117, + 74, 71, 69, 68, 67, 67, 65, 67, 70, 74, 75, 80, 83, 86, 91, 93, 100, 102, 106, 112, 114, 120, 123, + 80, 76, 74, 72, 71, 71, 69, 71, 74, 78, 79, 84, 86, 90, 95, 96, 104, 106, 110, 116, 118, 125, 128, 134, + 82, 78, 76, 74, 73, 73, 71, 73, 76, 79, 80, 86, 88, 92, 97, 98, 106, 108, 112, 118, 120, 127, 131, 136, 139, + 83, 78, 77, 75, 74, 74, 72, 73, 76, 80, 81, 86, 89, 92, 97, 99, 106, 109, 113, 119, 121, 128, 131, 137, 139, 140, + 87, 83, 81, 79, 78, 78, 75, 77, 80, 83, 85, 90, 92, 96, 100, 102, 110, 112, 117, 122, 125, 133, 135, 142, 144, 145, 150, + 90, 85, 84, 81, 80, 80, 78, 78, 82, 84, 87, 91, 93, 98, 99, 106, 108, 113, 118, 121, 129, 130, 137, 141, 147, 150, 151, 156, + 92, 88, 87, 84, 83, 82, 80, 80, 84, 85, 90, 91, 95, 98, 102, 106, 109, 115, 117, 125, 126, 134, 137, 142, 148, 152, 155, 156, 162, + 95, 90, 89, 86, 85, 84, 83, 82, 85, 87, 91, 92, 97, 98, 105, 105, 112, 114, 121, 123, 129, 133, 138, 143, 147, 155, 158, 161, 162, 168, + 97, 92, 92, 88, 88, 86, 86, 84, 85, 90, 91, 95, 97, 101, 104, 108, 112, 116, 121, 125, 130, 133, 140, 143, 150, 152, 162, 164, 168, 168, 174, + 100, 95, 95, 90, 90, 89, 89, 86, 86, 92, 92, 97, 98, 104, 104, 111, 111, 119, 119, 128, 129, 137, 137, 147, 148, 157, 158, 169, 170, 174, 175, 181, + }, { + 32, + 31, 31, + 31, 31, 31, + 30, 31, 31, 32, + 33, 34, 34, 34, 37, + 33, 34, 35, 35, 38, 39, + 36, 38, 39, 40, 42, 43, 47, + 38, 40, 40, 41, 43, 44, 47, 47, + 41, 42, 42, 42, 44, 45, 47, 48, 48, + 47, 46, 46, 45, 46, 47, 47, 48, 50, 52, + 49, 47, 47, 46, 47, 47, 48, 49, 50, 52, 53, + 48, 47, 46, 45, 46, 46, 46, 48, 49, 52, 53, 54, + 49, 47, 46, 45, 46, 46, 46, 47, 49, 52, 53, 55, 55, + 49, 47, 46, 45, 45, 45, 45, 47, 49, 52, 53, 55, 57, 58, + 50, 48, 47, 46, 46, 46, 46, 47, 50, 53, 54, 56, 57, 59, 61, + 50, 48, 47, 46, 46, 46, 46, 47, 50, 53, 54, 56, 58, 60, 61, 61, + 52, 50, 49, 47, 47, 47, 47, 48, 50, 53, 54, 57, 59, 61, 63, 63, 66, + 53, 50, 50, 48, 48, 48, 47, 49, 51, 54, 55, 58, 59, 62, 64, 64, 67, 68, + 54, 52, 51, 49, 49, 49, 48, 49, 52, 55, 55, 58, 60, 62, 64, 65, 68, 69, 71, + 56, 54, 53, 51, 51, 51, 49, 51, 53, 55, 56, 59, 61, 63, 66, 66, 70, 71, 73, 75, + 57, 54, 53, 52, 51, 51, 50, 51, 53, 56, 56, 60, 61, 63, 66, 67, 70, 71, 73, 76, 76, + 60, 57, 56, 54, 53, 53, 52, 53, 55, 58, 58, 61, 63, 65, 68, 68, 72, 73, 75, 78, 79, 82, + 61, 58, 57, 55, 55, 54, 53, 54, 56, 58, 59, 62, 64, 66, 69, 69, 73, 74, 76, 79, 80, 83, 84, + 63, 60, 59, 57, 56, 56, 54, 55, 57, 60, 60, 63, 65, 67, 70, 71, 75, 76, 78, 81, 82, 85, 86, 89, + 64, 61, 60, 58, 57, 57, 55, 56, 58, 60, 61, 64, 66, 68, 70, 71, 75, 77, 79, 82, 82, 86, 87, 90, 91, + 65, 61, 60, 58, 57, 57, 55, 56, 58, 61, 61, 64, 66, 68, 71, 71, 75, 77, 79, 82, 83, 86, 88, 90, 91, 91, + 67, 63, 62, 60, 59, 59, 57, 58, 60, 62, 63, 66, 67, 69, 72, 73, 77, 78, 80, 83, 84, 88, 89, 92, 93, 93, 95, + 67, 64, 63, 61, 60, 60, 58, 58, 61, 61, 63, 65, 67, 70, 70, 74, 75, 78, 80, 81, 85, 86, 89, 91, 93, 94, 95, 97, + 68, 65, 64, 62, 61, 60, 59, 58, 61, 61, 64, 65, 67, 69, 71, 73, 75, 78, 79, 83, 83, 87, 88, 91, 93, 95, 96, 97, 99, + 69, 65, 65, 62, 62, 61, 60, 59, 61, 62, 64, 65, 68, 68, 72, 72, 76, 76, 80, 81, 84, 86, 88, 90, 92, 95, 96, 98, 98, 100, + 70, 66, 66, 63, 63, 62, 61, 60, 60, 63, 64, 66, 67, 69, 71, 73, 75, 77, 79, 81, 84, 85, 88, 89, 93, 93, 97, 98, 100, 100, 102, + 71, 67, 67, 64, 64, 62, 62, 60, 60, 64, 64, 67, 67, 70, 70, 74, 74, 78, 78, 82, 82, 86, 86, 91, 91, 95, 95, 100, 100, 101, 101, 104, + }, + }, { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 33, 33, 34, + 32, 32, 32, 32, 33, 34, 35, 35, + 33, 33, 33, 33, 34, 35, 36, 36, 38, + 34, 34, 34, 33, 34, 35, 36, 37, 39, 39, + 36, 35, 35, 34, 35, 36, 37, 38, 42, 42, 48, + 36, 35, 35, 34, 35, 36, 38, 38, 42, 43, 48, 49, + 39, 38, 38, 37, 38, 39, 40, 40, 44, 45, 50, 51, 54, + 41, 39, 39, 38, 39, 40, 40, 41, 45, 46, 51, 52, 55, 56, + 44, 42, 42, 41, 41, 42, 42, 42, 46, 47, 54, 54, 58, 59, 63, + 46, 44, 44, 42, 43, 44, 44, 44, 48, 49, 55, 55, 59, 61, 65, 67, + 48, 46, 46, 44, 45, 45, 45, 46, 50, 51, 57, 57, 61, 63, 67, 69, 71, + 52, 50, 49, 48, 48, 48, 48, 48, 52, 53, 59, 59, 64, 65, 70, 72, 74, 78, + 54, 51, 51, 49, 49, 50, 49, 49, 53, 54, 60, 60, 65, 67, 71, 74, 76, 80, 82, + 58, 56, 55, 53, 53, 53, 53, 53, 57, 58, 63, 64, 68, 70, 75, 77, 80, 84, 86, 91, + 59, 56, 56, 54, 54, 54, 53, 53, 57, 58, 64, 64, 69, 70, 75, 78, 80, 85, 87, 91, 92, + 65, 62, 61, 59, 59, 59, 58, 58, 62, 63, 68, 68, 73, 75, 79, 82, 85, 90, 92, 97, 98, 105, + 66, 63, 63, 60, 60, 60, 59, 59, 63, 64, 69, 69, 74, 76, 80, 83, 86, 91, 93, 98, 99, 106, 107, + 71, 68, 67, 65, 65, 64, 63, 63, 67, 68, 73, 73, 78, 80, 84, 87, 90, 95, 97, 103, 103, 111, 112, 117, + 74, 71, 70, 68, 67, 67, 66, 65, 69, 70, 75, 75, 80, 82, 86, 89, 93, 97, 100, 105, 106, 114, 115, 120, 123, + 80, 76, 75, 72, 72, 71, 70, 69, 73, 74, 79, 79, 84, 86, 90, 93, 96, 101, 104, 110, 110, 118, 119, 125, 128, 134, + 81, 77, 77, 74, 73, 73, 71, 71, 74, 75, 80, 80, 85, 87, 91, 94, 98, 103, 105, 111, 112, 120, 121, 127, 130, 136, 137, + 83, 78, 78, 75, 74, 74, 72, 72, 75, 76, 81, 81, 86, 88, 92, 95, 99, 104, 106, 112, 113, 121, 122, 128, 131, 137, 139, 140, + 86, 82, 81, 78, 77, 77, 75, 74, 78, 79, 84, 84, 89, 91, 95, 98, 101, 106, 109, 115, 116, 124, 125, 131, 135, 140, 142, 144, 147, + 89, 84, 84, 80, 80, 79, 78, 77, 79, 81, 85, 86, 91, 92, 97, 98, 104, 106, 112, 114, 119, 123, 128, 132, 135, 142, 145, 148, 149, 153, + 91, 86, 86, 82, 82, 81, 80, 79, 80, 84, 85, 88, 91, 94, 97, 100, 104, 107, 112, 115, 120, 123, 129, 132, 138, 140, 148, 150, 153, 154, 159, + 93, 88, 88, 84, 84, 83, 83, 80, 81, 86, 86, 91, 91, 96, 97, 103, 103, 110, 110, 118, 119, 126, 126, 135, 136, 144, 144, 155, 155, 159, 159, 164, + }, { + 32, + 31, 31, + 31, 31, 31, + 30, 31, 31, 32, + 31, 32, 32, 33, 34, + 33, 34, 35, 35, 37, 39, + 35, 37, 37, 38, 39, 41, 44, + 36, 38, 39, 40, 41, 43, 46, 47, + 40, 41, 41, 42, 43, 44, 46, 47, 48, + 41, 42, 42, 42, 43, 45, 46, 47, 48, 48, + 49, 47, 47, 46, 46, 47, 47, 48, 50, 50, 53, + 49, 47, 47, 46, 46, 47, 47, 47, 49, 50, 53, 53, + 48, 47, 47, 45, 46, 46, 46, 46, 49, 49, 53, 53, 54, + 48, 47, 46, 45, 45, 46, 46, 46, 49, 49, 53, 53, 54, 55, + 49, 47, 46, 45, 45, 45, 45, 45, 48, 49, 53, 54, 55, 56, 58, + 50, 47, 47, 45, 46, 46, 46, 46, 49, 49, 54, 54, 56, 57, 59, 60, + 50, 48, 48, 46, 46, 46, 46, 46, 49, 50, 54, 54, 56, 57, 60, 60, 61, + 52, 49, 49, 47, 47, 47, 47, 46, 49, 50, 54, 54, 57, 58, 61, 62, 63, 65, + 52, 50, 49, 47, 47, 47, 47, 47, 49, 50, 54, 54, 57, 58, 61, 62, 63, 65, 66, + 54, 52, 51, 49, 49, 49, 48, 48, 51, 52, 55, 55, 58, 59, 62, 63, 65, 67, 68, 70, + 54, 52, 51, 49, 49, 49, 48, 48, 51, 52, 55, 56, 58, 60, 62, 64, 65, 67, 68, 70, 71, + 57, 54, 54, 52, 51, 51, 50, 50, 52, 53, 56, 57, 60, 61, 63, 65, 67, 69, 70, 73, 73, 76, + 57, 55, 54, 52, 52, 51, 51, 50, 53, 53, 57, 57, 60, 61, 64, 65, 67, 70, 71, 73, 74, 77, 77, + 60, 57, 56, 54, 54, 53, 52, 52, 54, 55, 58, 59, 61, 63, 65, 67, 68, 71, 72, 75, 75, 79, 79, 82, + 61, 58, 57, 55, 55, 54, 53, 53, 55, 56, 59, 59, 62, 63, 66, 68, 69, 72, 73, 76, 76, 80, 80, 83, 84, + 63, 60, 59, 57, 57, 56, 55, 54, 57, 57, 60, 61, 63, 65, 67, 69, 71, 73, 75, 78, 78, 82, 82, 85, 86, 89, + 64, 61, 60, 58, 57, 57, 56, 55, 57, 58, 61, 61, 64, 65, 68, 69, 71, 74, 75, 78, 78, 82, 83, 86, 87, 89, 90, + 65, 61, 61, 58, 58, 57, 56, 55, 58, 58, 61, 62, 64, 65, 68, 70, 71, 74, 75, 78, 79, 83, 83, 86, 88, 90, 91, 91, + 66, 63, 62, 60, 59, 58, 57, 56, 59, 59, 62, 63, 65, 66, 69, 70, 72, 75, 76, 79, 80, 84, 84, 87, 89, 91, 92, 93, 94, + 67, 64, 63, 61, 60, 59, 58, 57, 59, 60, 62, 63, 66, 66, 70, 70, 73, 74, 77, 78, 81, 83, 85, 87, 89, 92, 93, 94, 94, 96, + 68, 64, 64, 61, 61, 60, 59, 58, 59, 61, 62, 64, 65, 67, 69, 71, 72, 74, 77, 78, 81, 82, 85, 86, 89, 90, 94, 94, 96, 96, 98, + 69, 65, 65, 62, 62, 61, 61, 58, 59, 62, 62, 65, 65, 68, 68, 71, 71, 75, 75, 79, 79, 83, 83, 87, 87, 91, 91, 96, 96, 97, 97, 99, + }, + }, { + { + 32, + 31, 32, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 32, 34, 34, 35, + 32, 32, 32, 32, 32, 34, 34, 35, 35, + 34, 34, 34, 33, 33, 35, 35, 37, 37, 39, + 34, 34, 34, 33, 33, 35, 35, 37, 37, 39, 39, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, + 36, 35, 35, 34, 34, 36, 36, 38, 38, 42, 42, 48, 48, + 39, 38, 38, 37, 37, 39, 39, 40, 40, 45, 45, 50, 50, 54, + 39, 38, 38, 37, 37, 39, 39, 40, 40, 45, 45, 50, 50, 54, 54, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 47, 47, 54, 54, 58, 58, 63, + 44, 42, 42, 41, 41, 42, 42, 42, 42, 47, 47, 54, 54, 58, 58, 63, 63, + 48, 46, 46, 44, 44, 45, 45, 46, 46, 51, 51, 57, 57, 61, 61, 67, 67, 71, + 48, 46, 46, 44, 44, 45, 45, 46, 46, 51, 51, 57, 57, 61, 61, 67, 67, 71, 71, + 54, 51, 51, 49, 49, 50, 50, 49, 49, 54, 54, 60, 60, 65, 65, 71, 71, 76, 76, 82, + 54, 51, 51, 49, 49, 50, 50, 49, 49, 54, 54, 60, 60, 65, 65, 71, 71, 76, 76, 82, 82, + 59, 56, 56, 54, 54, 54, 54, 53, 53, 58, 58, 64, 64, 69, 69, 75, 75, 80, 80, 87, 87, 92, + 59, 56, 56, 54, 54, 54, 54, 53, 53, 58, 58, 64, 64, 69, 69, 75, 75, 80, 80, 87, 87, 92, 92, + 65, 62, 62, 59, 59, 59, 59, 58, 58, 63, 63, 68, 68, 73, 73, 79, 79, 85, 85, 92, 92, 98, 98, 105, + 65, 62, 62, 59, 59, 59, 59, 58, 58, 63, 63, 68, 68, 73, 73, 79, 79, 85, 85, 92, 92, 98, 98, 105, 105, + 71, 68, 68, 65, 65, 64, 64, 63, 63, 68, 68, 73, 73, 78, 78, 84, 84, 90, 90, 97, 97, 103, 103, 111, 111, 117, + 71, 68, 68, 65, 65, 64, 64, 63, 63, 68, 68, 73, 73, 78, 78, 84, 84, 90, 90, 97, 97, 103, 103, 111, 111, 117, 117, + 80, 76, 76, 72, 72, 71, 71, 69, 69, 74, 74, 79, 79, 84, 84, 90, 90, 96, 96, 104, 104, 110, 110, 118, 118, 125, 125, 134, + 80, 76, 76, 72, 72, 71, 71, 69, 69, 74, 74, 79, 79, 84, 84, 90, 90, 96, 96, 104, 104, 110, 110, 118, 118, 125, 125, 134, 134, + 83, 78, 78, 75, 75, 74, 74, 72, 72, 76, 76, 81, 81, 86, 86, 92, 92, 99, 99, 106, 106, 113, 113, 121, 121, 128, 128, 137, 137, 140, + 83, 78, 78, 75, 75, 74, 74, 72, 72, 76, 76, 81, 81, 86, 86, 92, 92, 99, 99, 106, 106, 113, 113, 121, 121, 128, 128, 137, 137, 140, 140, + 87, 83, 83, 79, 79, 77, 77, 75, 75, 80, 80, 84, 84, 90, 90, 96, 96, 102, 102, 109, 109, 116, 116, 124, 124, 132, 132, 141, 141, 144, 144, 149, + }, { + 32, + 31, 31, + 31, 31, 31, + 30, 31, 31, 32, + 30, 31, 31, 32, 32, + 33, 34, 34, 35, 35, 39, + 33, 34, 34, 35, 35, 39, 39, + 36, 38, 38, 40, 40, 43, 43, 47, + 36, 38, 38, 40, 40, 43, 43, 47, 47, + 41, 42, 42, 42, 42, 45, 45, 47, 47, 48, + 41, 42, 42, 42, 42, 45, 45, 47, 47, 48, 48, + 49, 47, 47, 46, 46, 47, 47, 48, 48, 50, 50, 53, + 49, 47, 47, 46, 46, 47, 47, 48, 48, 50, 50, 53, 53, + 48, 47, 47, 45, 45, 46, 46, 46, 46, 49, 49, 53, 53, 54, + 48, 47, 47, 45, 45, 46, 46, 46, 46, 49, 49, 53, 53, 54, 54, + 49, 47, 47, 45, 45, 45, 45, 45, 45, 49, 49, 53, 53, 55, 55, 58, + 49, 47, 47, 45, 45, 45, 45, 45, 45, 49, 49, 53, 53, 55, 55, 58, 58, + 50, 48, 48, 46, 46, 46, 46, 46, 46, 50, 50, 54, 54, 56, 56, 60, 60, 61, + 50, 48, 48, 46, 46, 46, 46, 46, 46, 50, 50, 54, 54, 56, 56, 60, 60, 61, 61, + 52, 50, 50, 47, 47, 47, 47, 47, 47, 50, 50, 54, 54, 57, 57, 61, 61, 63, 63, 66, + 52, 50, 50, 47, 47, 47, 47, 47, 47, 50, 50, 54, 54, 57, 57, 61, 61, 63, 63, 66, 66, + 54, 52, 52, 49, 49, 49, 49, 48, 48, 52, 52, 55, 55, 58, 58, 62, 62, 65, 65, 68, 68, 71, + 54, 52, 52, 49, 49, 49, 49, 48, 48, 52, 52, 55, 55, 58, 58, 62, 62, 65, 65, 68, 68, 71, 71, + 57, 54, 54, 52, 52, 51, 51, 50, 50, 53, 53, 56, 56, 60, 60, 63, 63, 67, 67, 70, 70, 73, 73, 76, + 57, 54, 54, 52, 52, 51, 51, 50, 50, 53, 53, 56, 56, 60, 60, 63, 63, 67, 67, 70, 70, 73, 73, 76, 76, + 60, 57, 57, 54, 54, 53, 53, 52, 52, 55, 55, 58, 58, 61, 61, 65, 65, 68, 68, 72, 72, 75, 75, 79, 79, 82, + 60, 57, 57, 54, 54, 53, 53, 52, 52, 55, 55, 58, 58, 61, 61, 65, 65, 68, 68, 72, 72, 75, 75, 79, 79, 82, 82, + 63, 60, 60, 57, 57, 56, 56, 54, 54, 57, 57, 60, 60, 63, 63, 67, 67, 71, 71, 75, 75, 78, 78, 82, 82, 85, 85, 89, + 63, 60, 60, 57, 57, 56, 56, 54, 54, 57, 57, 60, 60, 63, 63, 67, 67, 71, 71, 75, 75, 78, 78, 82, 82, 85, 85, 89, 89, + 65, 61, 61, 58, 58, 57, 57, 55, 55, 58, 58, 61, 61, 64, 64, 68, 68, 71, 71, 75, 75, 79, 79, 83, 83, 86, 86, 90, 90, 91, + 65, 61, 61, 58, 58, 57, 57, 55, 55, 58, 58, 61, 61, 64, 64, 68, 68, 71, 71, 75, 75, 79, 79, 83, 83, 86, 86, 90, 90, 91, 91, + 67, 63, 63, 60, 60, 59, 59, 57, 57, 60, 60, 62, 62, 66, 66, 69, 69, 72, 72, 76, 76, 80, 80, 84, 84, 88, 88, 92, 92, 93, 93, 95, + }, + }, { + { + 32, + 31, 31, + 31, 32, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 32, 33, 33, 34, + 32, 32, 32, 32, 32, 33, 34, 34, 35, + 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, + 34, 34, 34, 33, 33, 34, 35, 35, 37, 37, 39, + 34, 34, 34, 33, 33, 34, 35, 35, 37, 37, 39, 39, + 35, 35, 35, 34, 34, 35, 36, 36, 38, 38, 42, 42, 46, + 36, 35, 35, 34, 34, 35, 36, 37, 38, 38, 42, 42, 47, 48, + 38, 37, 37, 36, 36, 37, 38, 38, 39, 40, 44, 44, 48, 50, 51, + 39, 38, 38, 38, 37, 38, 39, 39, 40, 41, 45, 45, 49, 50, 52, 54, + 41, 40, 40, 39, 38, 39, 40, 40, 41, 41, 46, 46, 50, 52, 54, 55, 57, + 44, 42, 42, 41, 41, 41, 42, 42, 42, 43, 47, 47, 52, 54, 56, 58, 60, 63, + 45, 43, 43, 42, 41, 42, 42, 43, 43, 43, 48, 48, 53, 54, 57, 58, 60, 64, 65, + 48, 46, 46, 45, 44, 45, 45, 45, 46, 46, 51, 51, 55, 57, 59, 61, 63, 67, 68, 71, + 48, 46, 46, 45, 44, 45, 45, 45, 46, 46, 51, 51, 55, 57, 59, 61, 63, 67, 68, 71, 71, + 53, 51, 51, 49, 49, 49, 49, 49, 49, 49, 54, 54, 58, 59, 62, 64, 67, 71, 72, 75, 75, 81, + 54, 52, 51, 50, 49, 49, 50, 49, 49, 50, 54, 54, 59, 60, 63, 65, 67, 71, 72, 76, 76, 81, 82, + 57, 55, 55, 53, 52, 52, 52, 52, 52, 52, 57, 57, 61, 62, 65, 67, 70, 74, 75, 79, 79, 85, 85, 89, + 59, 56, 56, 54, 54, 54, 54, 54, 53, 54, 58, 58, 62, 64, 67, 69, 71, 75, 76, 80, 80, 86, 87, 90, 92, + 62, 59, 59, 57, 56, 56, 56, 56, 55, 56, 60, 60, 64, 66, 69, 71, 73, 77, 78, 83, 83, 89, 89, 93, 95, 98, + 65, 62, 62, 60, 59, 59, 59, 59, 58, 58, 63, 63, 67, 68, 71, 73, 75, 79, 81, 85, 85, 91, 92, 96, 98, 101, 105, + 67, 64, 64, 62, 61, 61, 60, 60, 59, 60, 64, 64, 68, 69, 72, 74, 77, 81, 82, 87, 87, 93, 94, 98, 99, 103, 106, 108, + 71, 68, 68, 66, 65, 64, 64, 64, 63, 63, 68, 68, 72, 73, 76, 78, 80, 84, 85, 90, 90, 97, 97, 102, 103, 107, 111, 113, 117, + 72, 69, 69, 66, 65, 65, 65, 64, 63, 64, 68, 68, 72, 73, 76, 78, 81, 85, 86, 91, 91, 97, 98, 102, 104, 108, 111, 113, 118, 119, + 80, 76, 76, 73, 72, 72, 71, 70, 69, 70, 74, 74, 78, 79, 82, 84, 86, 90, 91, 96, 96, 103, 104, 108, 110, 114, 118, 120, 125, 126, 134, + 80, 76, 76, 73, 72, 72, 71, 70, 69, 70, 74, 74, 78, 79, 82, 84, 86, 90, 91, 96, 96, 103, 104, 108, 110, 114, 118, 120, 125, 126, 134, 134, + }, { + 32, + 31, 31, + 31, 31, 31, + 30, 31, 31, 31, + 30, 31, 31, 31, 32, + 32, 32, 33, 33, 33, 35, + 33, 34, 34, 35, 35, 37, 39, + 34, 35, 35, 36, 36, 38, 40, 41, + 36, 38, 38, 39, 40, 41, 43, 44, 47, + 37, 38, 39, 40, 40, 42, 43, 44, 47, 47, + 41, 42, 42, 42, 42, 43, 45, 45, 47, 47, 48, + 41, 42, 42, 42, 42, 43, 45, 45, 47, 47, 48, 48, + 47, 46, 46, 46, 45, 46, 47, 47, 47, 48, 50, 50, 52, + 49, 48, 47, 47, 46, 47, 47, 47, 48, 48, 50, 50, 52, 53, + 49, 47, 47, 46, 46, 46, 46, 47, 47, 47, 50, 50, 52, 53, 53, + 48, 47, 47, 46, 45, 46, 46, 46, 46, 47, 49, 49, 52, 53, 54, 54, + 49, 47, 47, 46, 45, 45, 46, 46, 46, 46, 49, 49, 52, 53, 54, 55, 55, + 49, 47, 47, 45, 45, 45, 45, 45, 45, 45, 49, 49, 52, 53, 55, 55, 57, 58, + 49, 47, 47, 46, 45, 45, 45, 45, 45, 46, 49, 49, 52, 53, 55, 56, 57, 59, 59, + 50, 48, 48, 47, 46, 46, 46, 46, 46, 46, 50, 50, 53, 54, 55, 56, 58, 60, 60, 61, + 50, 48, 48, 47, 46, 46, 46, 46, 46, 46, 50, 50, 53, 54, 55, 56, 58, 60, 60, 61, 61, + 52, 50, 49, 48, 47, 47, 47, 47, 46, 47, 50, 50, 53, 54, 56, 57, 59, 61, 61, 63, 63, 66, + 52, 50, 50, 48, 47, 47, 47, 47, 47, 47, 50, 50, 53, 54, 56, 57, 59, 61, 61, 63, 63, 66, 66, + 54, 51, 51, 50, 49, 49, 49, 48, 48, 48, 51, 51, 54, 55, 57, 58, 60, 62, 62, 65, 65, 67, 68, 69, + 54, 52, 52, 50, 49, 49, 49, 49, 48, 48, 52, 52, 55, 55, 57, 58, 60, 62, 63, 65, 65, 68, 68, 70, 71, + 56, 53, 53, 51, 51, 50, 50, 50, 49, 49, 52, 52, 55, 56, 58, 59, 61, 63, 63, 66, 66, 69, 69, 71, 72, 73, + 57, 54, 54, 52, 52, 51, 51, 51, 50, 50, 53, 53, 56, 56, 58, 60, 61, 63, 64, 67, 67, 70, 70, 72, 73, 75, 76, + 58, 55, 55, 53, 52, 52, 52, 51, 50, 51, 54, 54, 56, 57, 59, 60, 62, 64, 65, 67, 67, 71, 71, 73, 74, 75, 77, 78, + 60, 57, 57, 55, 54, 54, 53, 53, 52, 52, 55, 55, 58, 58, 60, 61, 63, 65, 66, 68, 68, 72, 72, 74, 75, 77, 79, 80, 82, + 60, 57, 57, 55, 54, 54, 54, 53, 52, 52, 55, 55, 58, 58, 60, 62, 63, 65, 66, 69, 69, 72, 73, 75, 76, 77, 79, 80, 82, 82, + 63, 60, 60, 58, 57, 57, 56, 55, 54, 55, 57, 57, 60, 60, 62, 63, 65, 67, 68, 71, 71, 74, 75, 77, 78, 80, 82, 83, 85, 85, 89, + 63, 60, 60, 58, 57, 57, 56, 55, 54, 55, 57, 57, 60, 60, 62, 63, 65, 67, 68, 71, 71, 74, 75, 77, 78, 80, 82, 83, 85, 85, 89, 89, + }, + }, { + { + 32, + 31, 31, + 31, 31, 32, + 31, 32, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 32, 32, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, + 32, 32, 32, 32, 32, 32, 33, 34, 34, 35, 35, + 33, 33, 33, 33, 33, 33, 34, 35, 35, 36, 36, 38, + 34, 34, 34, 34, 33, 33, 35, 35, 36, 37, 37, 39, 39, + 34, 34, 34, 34, 34, 34, 35, 36, 36, 37, 37, 40, 41, 42, + 36, 35, 35, 35, 34, 34, 36, 36, 37, 38, 38, 42, 42, 45, 48, + 36, 35, 35, 35, 34, 34, 36, 36, 37, 38, 38, 42, 42, 45, 48, 48, + 38, 38, 38, 37, 37, 37, 38, 38, 39, 40, 40, 43, 44, 46, 50, 50, 52, + 39, 38, 38, 38, 37, 37, 39, 39, 39, 40, 40, 44, 45, 47, 50, 50, 53, 54, + 41, 40, 40, 39, 38, 38, 40, 40, 40, 41, 41, 45, 46, 48, 52, 52, 54, 55, 57, + 44, 42, 42, 42, 41, 41, 42, 42, 42, 42, 42, 46, 47, 50, 54, 54, 57, 58, 60, 63, + 44, 42, 42, 42, 41, 41, 42, 42, 42, 42, 42, 46, 47, 50, 54, 54, 57, 58, 60, 63, 63, + 47, 46, 45, 45, 44, 44, 44, 45, 45, 45, 45, 49, 50, 52, 56, 56, 59, 60, 62, 66, 66, 69, + 48, 47, 46, 45, 44, 44, 45, 45, 45, 46, 46, 50, 51, 53, 57, 57, 60, 61, 63, 67, 67, 70, 71, + 50, 49, 48, 47, 46, 46, 47, 47, 47, 47, 47, 51, 52, 54, 58, 58, 61, 62, 65, 68, 68, 72, 73, 75, + 54, 52, 51, 50, 49, 49, 49, 50, 49, 49, 49, 53, 54, 56, 60, 60, 64, 65, 67, 71, 71, 75, 76, 78, 82, + 54, 52, 51, 50, 49, 49, 49, 50, 49, 49, 49, 53, 54, 56, 60, 60, 64, 65, 67, 71, 71, 75, 76, 78, 82, 82, + 58, 56, 55, 54, 53, 53, 53, 53, 53, 52, 52, 56, 57, 59, 63, 63, 67, 68, 70, 74, 74, 78, 79, 82, 86, 86, 90, + 59, 57, 56, 55, 54, 54, 54, 54, 54, 53, 53, 57, 58, 60, 64, 64, 68, 69, 71, 75, 75, 79, 80, 83, 87, 87, 91, 92, + 61, 59, 58, 57, 56, 56, 56, 56, 55, 55, 55, 59, 60, 62, 65, 65, 69, 70, 73, 77, 77, 81, 82, 85, 89, 89, 93, 94, 97, + 65, 63, 62, 61, 59, 59, 59, 59, 59, 58, 58, 62, 63, 65, 68, 68, 72, 73, 75, 79, 79, 84, 85, 88, 92, 92, 97, 98, 101, 105, + 65, 63, 62, 61, 59, 59, 59, 59, 59, 58, 58, 62, 63, 65, 68, 68, 72, 73, 75, 79, 79, 84, 85, 88, 92, 92, 97, 98, 101, 105, 105, + 70, 67, 67, 65, 64, 64, 63, 63, 63, 62, 62, 66, 67, 69, 72, 72, 76, 77, 79, 83, 83, 88, 89, 92, 96, 96, 101, 102, 105, 109, 109, 114, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 30, 31, 31, 31, 32, + 30, 31, 31, 31, 32, 32, + 33, 33, 34, 34, 34, 34, 37, + 33, 34, 34, 35, 35, 35, 38, 39, + 34, 36, 36, 36, 37, 37, 40, 40, 42, + 36, 38, 38, 39, 40, 40, 42, 43, 45, 47, + 36, 38, 38, 39, 40, 40, 42, 43, 45, 47, 47, + 40, 41, 41, 41, 42, 42, 44, 44, 45, 47, 47, 48, + 41, 42, 42, 42, 42, 42, 44, 45, 46, 47, 47, 48, 48, + 44, 44, 44, 44, 44, 44, 45, 46, 46, 47, 47, 49, 49, 50, + 49, 48, 47, 47, 46, 46, 47, 47, 47, 48, 48, 50, 50, 51, 53, + 49, 48, 47, 47, 46, 46, 47, 47, 47, 48, 48, 50, 50, 51, 53, 53, + 48, 47, 47, 46, 45, 45, 46, 46, 46, 47, 47, 49, 50, 51, 53, 53, 54, + 48, 47, 47, 46, 45, 45, 46, 46, 46, 46, 46, 49, 49, 51, 53, 53, 54, 54, + 49, 47, 47, 46, 45, 45, 46, 46, 46, 46, 46, 49, 49, 51, 53, 53, 54, 55, 55, + 49, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 48, 49, 51, 53, 53, 55, 55, 57, 58, + 49, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 48, 49, 51, 53, 53, 55, 55, 57, 58, 58, + 50, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 49, 50, 51, 54, 54, 56, 56, 57, 59, 59, 61, + 50, 49, 48, 47, 46, 46, 46, 46, 46, 46, 46, 49, 50, 51, 54, 54, 56, 56, 58, 60, 60, 61, 61, + 51, 49, 49, 48, 47, 47, 47, 47, 47, 46, 46, 49, 50, 51, 54, 54, 56, 57, 58, 60, 60, 62, 62, 63, + 52, 50, 50, 49, 47, 47, 47, 47, 47, 47, 47, 49, 50, 52, 54, 54, 57, 57, 59, 61, 61, 63, 63, 65, 66, + 52, 50, 50, 49, 47, 47, 47, 47, 47, 47, 47, 49, 50, 52, 54, 54, 57, 57, 59, 61, 61, 63, 63, 65, 66, 66, + 54, 52, 51, 50, 49, 49, 49, 49, 48, 48, 48, 51, 51, 53, 55, 55, 58, 58, 60, 62, 62, 64, 65, 66, 68, 68, 70, + 54, 52, 52, 51, 49, 49, 49, 49, 49, 48, 48, 51, 52, 53, 55, 55, 58, 58, 60, 62, 62, 64, 65, 66, 68, 68, 70, 71, + 55, 53, 53, 52, 50, 50, 50, 50, 49, 49, 49, 51, 52, 54, 56, 56, 58, 59, 60, 63, 63, 65, 66, 67, 69, 69, 71, 72, 73, + 57, 55, 54, 53, 52, 52, 51, 51, 50, 50, 50, 52, 53, 54, 56, 56, 59, 60, 61, 63, 63, 66, 67, 68, 70, 70, 73, 73, 74, 76, + 57, 55, 54, 53, 52, 52, 51, 51, 50, 50, 50, 52, 53, 54, 56, 56, 59, 60, 61, 63, 63, 66, 67, 68, 70, 70, 73, 73, 74, 76, 76, + 59, 57, 56, 55, 54, 54, 53, 53, 52, 51, 51, 54, 55, 56, 58, 58, 60, 61, 63, 65, 65, 67, 68, 70, 72, 72, 74, 75, 76, 78, 78, 80, + }, + }, { + { + 32, + 31, 31, + 31, 31, 32, + 31, 31, 32, 32, + 31, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 32, 32, 33, 33, + 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, + 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 35, 35, + 32, 33, 33, 33, 33, 33, 33, 34, 34, 35, 36, 36, 36, + 34, 34, 34, 34, 33, 33, 34, 35, 35, 35, 37, 37, 38, 39, + 34, 34, 34, 34, 33, 33, 34, 35, 35, 35, 37, 37, 38, 39, 39, + 35, 34, 34, 34, 34, 34, 34, 35, 36, 36, 37, 37, 39, 41, 41, 43, + 36, 35, 35, 35, 34, 34, 35, 36, 36, 37, 38, 38, 40, 42, 42, 45, 48, + 36, 35, 35, 35, 34, 34, 35, 36, 36, 37, 38, 38, 40, 42, 42, 45, 48, 48, + 38, 37, 37, 37, 36, 36, 36, 38, 38, 38, 39, 39, 41, 44, 44, 47, 50, 50, 51, + 39, 39, 38, 38, 37, 37, 38, 39, 39, 39, 40, 40, 42, 45, 45, 47, 50, 50, 52, 54, + 39, 39, 38, 38, 37, 37, 38, 39, 39, 39, 40, 40, 42, 45, 45, 47, 50, 50, 52, 54, 54, + 42, 41, 41, 41, 40, 40, 40, 41, 41, 41, 42, 42, 44, 47, 47, 49, 53, 53, 55, 56, 56, 60, + 44, 43, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 44, 47, 47, 50, 54, 54, 56, 58, 58, 61, 63, + 44, 43, 43, 42, 41, 41, 41, 42, 42, 42, 43, 43, 45, 48, 48, 51, 54, 54, 56, 58, 58, 62, 64, 64, + 47, 46, 45, 45, 44, 44, 44, 44, 45, 45, 45, 45, 47, 50, 50, 53, 56, 56, 58, 60, 60, 64, 66, 66, 69, + 48, 47, 46, 46, 45, 44, 45, 45, 45, 45, 46, 46, 47, 51, 51, 53, 57, 57, 59, 61, 61, 65, 67, 67, 70, 71, + 49, 48, 47, 47, 46, 45, 45, 46, 46, 46, 46, 46, 48, 51, 51, 54, 57, 57, 60, 62, 62, 66, 68, 68, 71, 72, 73, + 53, 51, 51, 51, 49, 49, 49, 49, 49, 49, 49, 49, 51, 54, 54, 57, 59, 59, 62, 64, 64, 69, 71, 71, 74, 75, 77, 81, + 54, 52, 51, 51, 50, 49, 49, 50, 50, 49, 49, 49, 51, 54, 54, 57, 60, 60, 63, 65, 65, 69, 71, 72, 75, 76, 77, 81, 82, + 55, 53, 53, 52, 51, 50, 50, 51, 51, 51, 50, 50, 52, 55, 55, 58, 61, 61, 64, 66, 66, 70, 72, 73, 76, 77, 78, 83, 83, 85, + 59, 57, 56, 56, 54, 54, 54, 54, 54, 54, 53, 53, 55, 58, 58, 61, 64, 64, 67, 69, 69, 73, 75, 76, 79, 80, 81, 86, 87, 88, 92, + 59, 57, 56, 56, 54, 54, 54, 54, 54, 54, 53, 53, 55, 58, 58, 61, 64, 64, 67, 69, 69, 73, 75, 76, 79, 80, 81, 86, 87, 88, 92, 92, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 30, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 32, + 31, 31, 32, 32, 32, 32, 33, + 33, 34, 34, 34, 35, 35, 35, 38, + 33, 34, 34, 34, 35, 35, 36, 38, 39, + 34, 35, 35, 36, 36, 36, 37, 40, 40, 41, + 36, 38, 38, 38, 39, 40, 40, 43, 43, 44, 47, + 36, 38, 38, 38, 39, 40, 40, 43, 43, 44, 47, 47, + 38, 39, 40, 40, 41, 41, 41, 43, 44, 45, 47, 47, 47, + 41, 42, 42, 42, 42, 42, 43, 44, 45, 45, 47, 47, 48, 48, + 41, 42, 42, 42, 42, 42, 43, 44, 45, 45, 47, 47, 48, 48, 48, + 45, 45, 45, 45, 44, 44, 44, 46, 46, 46, 47, 47, 48, 49, 49, 50, + 49, 48, 47, 47, 46, 46, 46, 47, 47, 47, 48, 48, 49, 50, 50, 51, 53, + 49, 48, 47, 47, 46, 46, 46, 47, 47, 47, 48, 48, 49, 50, 50, 51, 53, 53, + 49, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 48, 50, 50, 51, 53, 53, 53, + 48, 47, 47, 47, 46, 45, 45, 46, 46, 46, 46, 46, 48, 49, 49, 51, 53, 53, 54, 54, + 48, 47, 47, 47, 46, 45, 45, 46, 46, 46, 46, 46, 48, 49, 49, 51, 53, 53, 54, 54, 54, + 49, 47, 47, 47, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 51, 53, 53, 54, 55, 55, 57, + 49, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 51, 53, 53, 55, 55, 55, 57, 58, + 49, 47, 47, 47, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 51, 53, 53, 55, 56, 56, 58, 58, 59, + 50, 49, 48, 48, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 52, 54, 54, 55, 56, 56, 58, 59, 59, 61, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 52, 54, 54, 55, 56, 56, 59, 60, 60, 61, 61, + 51, 49, 48, 48, 47, 46, 46, 47, 47, 46, 46, 46, 47, 50, 50, 52, 54, 54, 55, 56, 56, 59, 60, 60, 61, 62, 62, + 52, 50, 49, 49, 48, 47, 47, 47, 47, 47, 46, 46, 48, 50, 50, 52, 54, 54, 56, 57, 57, 60, 61, 61, 63, 63, 64, 66, + 52, 50, 50, 49, 48, 47, 47, 47, 47, 47, 47, 47, 48, 50, 50, 52, 54, 54, 56, 57, 57, 60, 61, 61, 63, 63, 64, 66, 66, + 53, 51, 50, 50, 48, 48, 48, 48, 48, 48, 47, 47, 48, 51, 51, 52, 54, 54, 56, 58, 58, 60, 61, 62, 63, 64, 64, 67, 67, 68, + 54, 53, 52, 52, 50, 49, 49, 49, 49, 49, 48, 48, 49, 52, 52, 53, 55, 55, 57, 58, 58, 61, 62, 63, 64, 65, 66, 68, 68, 69, 71, + 54, 53, 52, 52, 50, 49, 49, 49, 49, 49, 48, 48, 49, 52, 52, 53, 55, 55, 57, 58, 58, 61, 62, 63, 64, 65, 66, 68, 68, 69, 71, 71, + }, + }, { + { + 32, + 31, 31, + 31, 31, 32, + 31, 31, 32, 32, + 31, 31, 32, 32, 32, + 31, 31, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 35, 35, 35, + 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 35, 36, 36, 36, + 34, 34, 34, 34, 34, 33, 33, 34, 35, 35, 35, 36, 37, 37, 38, 39, + 34, 34, 34, 34, 34, 33, 33, 34, 35, 35, 35, 36, 37, 37, 38, 39, 39, + 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 36, 37, 37, 38, 40, 40, 41, + 35, 35, 35, 35, 34, 34, 34, 34, 36, 36, 36, 37, 38, 38, 39, 42, 42, 43, 46, + 36, 35, 35, 35, 35, 34, 34, 35, 36, 36, 36, 37, 38, 38, 40, 42, 42, 44, 47, 48, + 36, 35, 35, 35, 35, 34, 34, 35, 36, 36, 36, 37, 38, 38, 40, 42, 42, 44, 47, 48, 48, + 38, 37, 37, 37, 36, 36, 36, 36, 37, 38, 38, 39, 39, 39, 41, 44, 44, 45, 48, 50, 50, 51, + 39, 39, 38, 38, 38, 37, 37, 38, 39, 39, 39, 40, 40, 40, 42, 45, 45, 46, 49, 50, 50, 52, 54, + 39, 39, 38, 38, 38, 37, 37, 38, 39, 39, 39, 40, 40, 40, 42, 45, 45, 46, 49, 50, 50, 52, 54, 54, + 41, 40, 40, 40, 39, 38, 38, 39, 40, 40, 40, 41, 41, 41, 43, 46, 46, 47, 50, 52, 52, 54, 55, 55, 57, + 44, 43, 42, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 42, 44, 47, 47, 49, 52, 54, 54, 56, 58, 58, 60, 63, + 44, 43, 42, 42, 42, 41, 41, 41, 42, 42, 42, 42, 42, 42, 44, 47, 47, 49, 52, 54, 54, 56, 58, 58, 60, 63, 63, + 45, 44, 43, 43, 42, 41, 41, 42, 42, 42, 42, 43, 43, 43, 45, 48, 48, 49, 53, 54, 54, 57, 58, 58, 60, 64, 64, 65, + 47, 46, 45, 45, 45, 44, 44, 44, 44, 45, 45, 45, 45, 45, 47, 50, 50, 51, 55, 56, 56, 58, 60, 60, 62, 66, 66, 67, 69, + 48, 47, 46, 46, 45, 44, 44, 45, 45, 45, 45, 45, 46, 46, 47, 51, 51, 52, 55, 57, 57, 59, 61, 61, 63, 67, 67, 68, 70, 71, + 48, 47, 46, 46, 45, 44, 44, 45, 45, 45, 45, 45, 46, 46, 47, 51, 51, 52, 55, 57, 57, 59, 61, 61, 63, 67, 67, 68, 70, 71, 71, + 51, 50, 49, 49, 48, 47, 47, 47, 48, 48, 48, 48, 48, 48, 50, 53, 53, 54, 57, 58, 58, 61, 63, 63, 66, 69, 69, 70, 73, 74, 74, 77, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 32, + 30, 31, 31, 31, 31, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 33, + 33, 33, 34, 34, 34, 34, 34, 35, 37, + 33, 34, 34, 34, 35, 35, 35, 36, 38, 39, + 33, 34, 34, 34, 35, 35, 35, 36, 38, 39, 39, + 35, 36, 37, 37, 37, 38, 38, 38, 41, 41, 41, 44, + 36, 37, 38, 38, 39, 40, 40, 40, 42, 43, 43, 46, 47, + 36, 37, 38, 38, 39, 40, 40, 40, 42, 43, 43, 46, 47, 47, + 38, 39, 40, 40, 40, 41, 41, 41, 43, 44, 44, 46, 47, 47, 47, + 41, 42, 42, 42, 42, 42, 42, 43, 44, 45, 45, 46, 47, 47, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 43, 44, 45, 45, 46, 47, 47, 48, 48, 48, + 43, 43, 43, 43, 43, 43, 43, 43, 45, 45, 45, 46, 47, 47, 48, 49, 49, 49, + 47, 47, 46, 46, 46, 45, 45, 46, 46, 47, 47, 47, 47, 47, 48, 50, 50, 50, 52, + 49, 48, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 48, 48, 49, 50, 50, 51, 52, 53, + 49, 48, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 48, 48, 49, 50, 50, 51, 52, 53, 53, + 49, 48, 47, 47, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 48, 50, 50, 50, 52, 53, 53, 53, + 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 48, 49, 49, 50, 52, 53, 53, 54, 54, + 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 48, 49, 49, 50, 52, 53, 53, 54, 54, 54, + 49, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 47, 49, 49, 50, 52, 53, 53, 54, 55, 55, 55, + 49, 47, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 50, 52, 53, 53, 55, 55, 55, 57, 58, + 49, 47, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 50, 52, 53, 53, 55, 55, 55, 57, 58, 58, + 49, 48, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 50, 52, 53, 53, 55, 56, 56, 57, 59, 59, 59, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 50, 53, 54, 54, 55, 56, 56, 57, 59, 59, 60, 61, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 50, 53, 54, 54, 55, 56, 56, 58, 60, 60, 60, 61, 61, + 50, 49, 48, 48, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 50, 50, 50, 53, 54, 54, 55, 56, 56, 58, 60, 60, 60, 61, 61, 61, + 51, 50, 49, 49, 48, 47, 47, 47, 47, 47, 47, 47, 46, 46, 48, 50, 50, 51, 53, 54, 54, 56, 57, 57, 58, 60, 60, 61, 62, 63, 63, 64, + }, + }, { + { + 32, + 31, 31, + 31, 31, 32, + 31, 31, 32, 32, + 31, 31, 32, 32, 32, + 31, 31, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 34, 34, 34, 34, 35, 35, 35, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 35, 36, 36, 36, 37, + 34, 34, 34, 34, 34, 34, 33, 33, 33, 34, 35, 35, 35, 36, 37, 37, 37, 38, 39, + 34, 34, 34, 34, 34, 34, 33, 33, 33, 34, 35, 35, 35, 36, 37, 37, 37, 38, 39, 39, + 34, 34, 34, 34, 34, 34, 33, 33, 33, 34, 35, 35, 35, 36, 37, 37, 37, 38, 39, 39, 39, + 35, 34, 34, 34, 34, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 37, 37, 39, 41, 41, 41, 43, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, + 36, 35, 35, 35, 35, 35, 34, 34, 34, 35, 36, 36, 36, 37, 38, 38, 38, 40, 42, 42, 42, 45, 48, 48, 48, + 37, 37, 37, 37, 37, 36, 36, 36, 36, 37, 38, 38, 38, 38, 39, 39, 39, 41, 44, 44, 44, 46, 49, 49, 49, 51, + 39, 39, 38, 38, 38, 38, 37, 37, 37, 38, 39, 39, 39, 40, 40, 40, 40, 42, 45, 45, 45, 47, 50, 50, 50, 52, 54, + 39, 39, 38, 38, 38, 38, 37, 37, 37, 38, 39, 39, 39, 40, 40, 40, 40, 42, 45, 45, 45, 47, 50, 50, 50, 52, 54, 54, + 39, 39, 38, 38, 38, 38, 37, 37, 37, 38, 39, 39, 39, 40, 40, 40, 40, 42, 45, 45, 45, 47, 50, 50, 50, 52, 54, 54, 54, + 41, 41, 40, 40, 40, 39, 39, 39, 39, 40, 40, 40, 40, 41, 41, 41, 41, 44, 46, 46, 46, 49, 52, 52, 52, 54, 56, 56, 56, 58, + 44, 43, 42, 42, 42, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 45, 47, 47, 47, 50, 54, 54, 54, 56, 58, 58, 58, 60, 63, + 44, 43, 42, 42, 42, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 42, 42, 45, 47, 47, 47, 50, 54, 54, 54, 56, 58, 58, 58, 60, 63, 63, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 31, 32, + 30, 31, 31, 31, 31, 31, 32, 32, + 30, 31, 31, 31, 31, 31, 32, 32, 32, + 32, 32, 33, 33, 33, 33, 33, 33, 33, 35, + 33, 34, 34, 34, 34, 35, 35, 35, 35, 37, 39, + 33, 34, 34, 34, 34, 35, 35, 35, 35, 37, 39, 39, + 33, 34, 34, 34, 34, 35, 35, 35, 35, 37, 39, 39, 39, + 35, 35, 36, 36, 36, 37, 37, 37, 37, 39, 41, 41, 41, 43, + 36, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, + 36, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, + 36, 37, 38, 38, 38, 39, 40, 40, 40, 41, 43, 43, 43, 45, 47, 47, 47, + 39, 39, 40, 40, 40, 41, 41, 41, 41, 42, 44, 44, 44, 45, 47, 47, 47, 47, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 43, 45, 45, 45, 46, 47, 47, 47, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 43, 45, 45, 45, 46, 47, 47, 47, 48, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 43, 45, 45, 45, 46, 47, 47, 47, 48, 48, 48, 48, + 45, 45, 45, 45, 45, 44, 44, 44, 44, 45, 46, 46, 46, 47, 47, 47, 47, 48, 49, 49, 49, 50, + 49, 48, 47, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 51, 53, + 49, 48, 47, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 51, 53, 53, + 49, 48, 47, 47, 47, 47, 46, 46, 46, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 51, 53, 53, 53, + 49, 48, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 48, 50, 50, 50, 51, 53, 53, 53, 53, + 48, 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 48, 49, 49, 49, 51, 53, 53, 53, 53, 54, + 48, 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 48, 49, 49, 49, 51, 53, 53, 53, 53, 54, 54, + 48, 48, 47, 47, 47, 46, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 48, 49, 49, 49, 51, 53, 53, 53, 53, 54, 54, 54, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 47, 49, 49, 49, 51, 53, 53, 53, 54, 55, 55, 55, 56, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 49, 51, 53, 53, 53, 54, 55, 55, 55, 57, 58, + 49, 48, 47, 47, 47, 46, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 49, 49, 49, 51, 53, 53, 53, 54, 55, 55, 55, 57, 58, 58, + }, + }, { + { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 32, + 31, 31, 31, 32, 32, + 31, 31, 31, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 35, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, + 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 36, 36, 36, 36, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 36, 36, 36, 36, 37, 38, + 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 38, 39, 39, + 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 38, 39, 39, 39, + 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 34, 35, 35, 35, 35, 36, 36, 37, 37, 37, 38, 39, 39, 39, 39, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 40, 41, 41, 41, 42, + 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 37, 38, 38, 38, 39, 41, 42, 42, 42, 44, 46, + 36, 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 42, 42, 42, 42, 45, 47, 48, + 36, 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 42, 42, 42, 42, 45, 47, 48, 48, + 36, 35, 35, 35, 35, 35, 35, 34, 34, 34, 34, 35, 36, 36, 36, 36, 37, 38, 38, 38, 38, 40, 42, 42, 42, 42, 45, 47, 48, 48, 48, + 37, 37, 36, 36, 36, 36, 36, 35, 35, 35, 35, 36, 37, 37, 37, 37, 38, 39, 39, 39, 39, 41, 42, 43, 43, 43, 45, 48, 49, 49, 49, 50, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 31, 31, 31, 31, 31, 31, 32, + 30, 30, 31, 31, 31, 31, 31, 31, 32, 32, + 30, 30, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 34, + 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 36, 37, + 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 37, 38, 39, + 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 37, 38, 39, 39, + 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 37, 38, 39, 39, 39, + 34, 35, 36, 36, 36, 36, 36, 37, 37, 37, 37, 38, 40, 40, 40, 40, 42, + 36, 36, 37, 37, 37, 37, 38, 38, 39, 39, 39, 40, 41, 42, 42, 42, 44, 46, + 36, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 45, 46, 47, + 36, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 45, 46, 47, 47, + 36, 37, 38, 38, 38, 38, 39, 39, 40, 40, 40, 41, 42, 43, 43, 43, 45, 46, 47, 47, 47, + 38, 39, 39, 40, 40, 40, 40, 41, 41, 41, 41, 42, 43, 44, 44, 44, 45, 47, 47, 47, 47, 47, + 40, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 43, 44, 44, 44, 44, 45, 47, 47, 47, 47, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 44, 45, 45, 45, 46, 47, 47, 47, 47, 48, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 44, 45, 45, 45, 46, 47, 47, 47, 47, 48, 48, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 44, 45, 45, 45, 46, 47, 47, 47, 47, 48, 48, 48, 48, 48, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 45, 46, 46, 46, 46, 47, 47, 47, 47, 48, 49, 49, 49, 49, 50, + 47, 47, 46, 46, 46, 46, 46, 46, 45, 45, 45, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, + 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 50, 51, 52, 53, + 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 50, 51, 52, 53, 53, + 49, 48, 48, 47, 47, 47, 47, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 48, 48, 48, 49, 50, 50, 50, 50, 51, 52, 53, 53, 53, + 49, 48, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 47, 47, 47, 48, 49, 50, 50, 50, 51, 52, 53, 53, 53, 53, + }, + }, { + { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 32, 32, + 31, 31, 31, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, + 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 35, 35, 35, 36, 36, 36, 36, 36, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 36, 36, 36, 37, 38, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 39, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 33, 33, 33, 33, 33, 34, 34, 35, 35, 35, 35, 35, 35, 36, 36, 37, 37, 37, 37, 38, 38, 39, 39, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, + 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 35, + 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 36, 37, + 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 37, 38, 39, + 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 37, 38, 39, 39, + 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 37, 38, 39, 39, 39, + 33, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 36, 37, 38, 39, 39, 39, 39, + 34, 35, 35, 35, 35, 35, 35, 36, 36, 36, 36, 36, 36, 36, 37, 38, 39, 40, 40, 40, 40, 41, + 35, 36, 36, 36, 37, 37, 37, 37, 37, 37, 38, 38, 38, 38, 38, 39, 41, 41, 41, 41, 41, 42, 44, + 36, 37, 37, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 41, 42, 43, 43, 43, 43, 44, 45, 46, + 36, 37, 37, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 46, 47, 47, + 36, 37, 37, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 46, 47, 47, 47, + 36, 37, 37, 38, 38, 38, 38, 38, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 44, 46, 47, 47, 47, 47, + 37, 37, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 41, 42, 43, 43, 43, 43, 43, 44, 46, 47, 47, 47, 47, 47, + 38, 39, 39, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 42, 43, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, 47, 47, + 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 42, 42, 42, 42, 42, 43, 44, 44, 44, 44, 44, 45, 46, 47, 47, 47, 47, 47, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 43, 44, 45, 45, 45, 45, 45, 46, 47, 47, 47, 47, 47, 48, 48, 48, + 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 43, 43, 44, 45, 45, 45, 45, 45, 46, 47, 47, 47, 47, 47, 48, 48, 48, 48, + }, + }, { + { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 32, + 31, 31, 31, 31, 31, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, + 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, + 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, + 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 35, 36, + 33, 33, 33, 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 36, 37, 37, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, 39, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, 39, 39, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, 39, 39, 39, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, 39, 39, 39, 39, + 33, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 37, 37, 38, 39, 39, 39, 39, 39, 39, + 34, 34, 34, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 36, 36, 36, 36, 36, 36, 36, 36, 37, 37, 38, 39, 40, 40, 40, 40, 40, 40, 40, + }, + }, { + { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 31, 31, 31, 31, 31, 31, 31, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + }, { + 32, + 31, 31, + 31, 31, 31, + 31, 31, 31, 31, + 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, + 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, + }, + }, +}; + +const uint8_t *dav1d_qm_tbl[16][2][N_RECT_TX_SIZES]; +static uint8_t qm_tbl_4x4[15][2][16]; +static uint8_t qm_tbl_4x8[15][2][32]; +static uint8_t qm_tbl_4x16[15][2][64]; +static uint8_t qm_tbl_8x8[15][2][64]; +static uint8_t qm_tbl_8x16[15][2][128]; +static uint8_t qm_tbl_8x32[15][2][256]; +static uint8_t qm_tbl_16x16[15][2][256]; +static uint8_t qm_tbl_16x32[15][2][512]; +static uint8_t qm_tbl_32x32[15][2][1024]; + +static void subsample(uint8_t *const dst, const uint8_t *const src, + const int sz, const int step) +{ + for (int y = 0; y < sz; y++) + for (int x = 0; x < sz; x++) + dst[y * sz + x] = src[y * sz * step * step + x * step]; +} + +static void transpose(uint8_t *const dst, const uint8_t *const src, + const int w, const int h) +{ + for (int y = 0, y_off = 0; y < h; y++, y_off += w) + for (int x = 0, x_off = 0; x < w; x++, x_off += h) + dst[x_off + y] = src[y_off + x]; +} + +static void untriangle(uint8_t *dst, const uint8_t *src, const int sz) { + for (int y = 0; y < sz; y++) { + memcpy(dst, src, y + 1); + const uint8_t *src_ptr = &src[y]; + for (int x = y + 1; x < sz; x++) { + src_ptr += x; + dst[x] = *src_ptr; + } + dst += sz; + src += y + 1; + } +} + +COLD void dav1d_init_qm_tables(void) { + // This function is guaranteed to be called only once + + for (int i = 0; i < 15; i++) + for (int j = 0; j < 2; j++) { + // note that the w/h in the assignment is inverted, this is on purpose + // because we store coefficients transposed + dav1d_qm_tbl[i][j][RTX_4X8 ] = qm_tbl_8x4[i][j]; + dav1d_qm_tbl[i][j][RTX_8X4 ] = qm_tbl_4x8[i][j]; + transpose(qm_tbl_4x8[i][j], qm_tbl_8x4[i][j], 8, 4); + dav1d_qm_tbl[i][j][RTX_4X16 ] = qm_tbl_16x4[i][j]; + dav1d_qm_tbl[i][j][RTX_16X4 ] = qm_tbl_4x16[i][j]; + transpose(qm_tbl_4x16[i][j], qm_tbl_16x4[i][j], 16, 4); + dav1d_qm_tbl[i][j][RTX_8X16 ] = qm_tbl_16x8[i][j]; + dav1d_qm_tbl[i][j][RTX_16X8 ] = qm_tbl_8x16[i][j]; + transpose(qm_tbl_8x16[i][j], qm_tbl_16x8[i][j], 16, 8); + dav1d_qm_tbl[i][j][RTX_8X32 ] = qm_tbl_32x8[i][j]; + dav1d_qm_tbl[i][j][RTX_32X8 ] = qm_tbl_8x32[i][j]; + transpose(qm_tbl_8x32[i][j], qm_tbl_32x8[i][j], 32, 8); + dav1d_qm_tbl[i][j][RTX_16X32] = qm_tbl_32x16[i][j]; + dav1d_qm_tbl[i][j][RTX_32X16] = qm_tbl_16x32[i][j]; + transpose(qm_tbl_16x32[i][j], qm_tbl_32x16[i][j], 32, 16); + + dav1d_qm_tbl[i][j][ TX_4X4 ] = qm_tbl_4x4[i][j]; + dav1d_qm_tbl[i][j][ TX_8X8 ] = qm_tbl_8x8[i][j]; + dav1d_qm_tbl[i][j][ TX_16X16] = qm_tbl_16x16[i][j]; + dav1d_qm_tbl[i][j][ TX_32X32] = qm_tbl_32x32[i][j]; + untriangle(qm_tbl_4x4[i][j], qm_tbl_4x4_t[i][j], 4); + untriangle(qm_tbl_8x8[i][j], qm_tbl_8x8_t[i][j], 8); + untriangle(qm_tbl_32x32[i][j], qm_tbl_32x32_t[i][j], 32); + subsample(qm_tbl_16x16[i][j], qm_tbl_32x32[i][j], 16, 2); + + dav1d_qm_tbl[i][j][ TX_64X64] = dav1d_qm_tbl[i][j][ TX_32X32]; + dav1d_qm_tbl[i][j][RTX_64X32] = dav1d_qm_tbl[i][j][ TX_32X32]; + dav1d_qm_tbl[i][j][RTX_64X16] = dav1d_qm_tbl[i][j][RTX_32X16]; + dav1d_qm_tbl[i][j][RTX_32X64] = dav1d_qm_tbl[i][j][ TX_32X32]; + dav1d_qm_tbl[i][j][RTX_16X64] = dav1d_qm_tbl[i][j][RTX_16X32]; + } + + // dav1d_qm_tbl[15][*][*] == NULL +} diff --git a/third_party/dav1d/src/qm.h b/third_party/dav1d/src/qm.h new file mode 100644 index 0000000000..8191c8afa7 --- /dev/null +++ b/third_party/dav1d/src/qm.h @@ -0,0 +1,37 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_QM_H +#define DAV1D_SRC_QM_H + +#include "src/levels.h" + +EXTERN const uint8_t *dav1d_qm_tbl[16][2][N_RECT_TX_SIZES]; + +void dav1d_init_qm_tables(void); + +#endif /* DAV1D_SRC_QM_H */ diff --git a/third_party/dav1d/src/recon.h b/third_party/dav1d/src/recon.h new file mode 100644 index 0000000000..e97ac31ffb --- /dev/null +++ b/third_party/dav1d/src/recon.h @@ -0,0 +1,85 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_RECON_H +#define DAV1D_SRC_RECON_H + +#include "src/internal.h" +#include "src/levels.h" + +#define DEBUG_BLOCK_INFO 0 && \ + f->frame_hdr->frame_offset == 2 && t->by >= 0 && t->by < 4 && \ + t->bx >= 8 && t->bx < 12 +#define DEBUG_B_PIXELS 0 + +#define decl_recon_b_intra_fn(name) \ +void (name)(Dav1dTaskContext *t, enum BlockSize bs, \ + enum EdgeFlags intra_edge_flags, const Av1Block *b) +typedef decl_recon_b_intra_fn(*recon_b_intra_fn); + +#define decl_recon_b_inter_fn(name) \ +int (name)(Dav1dTaskContext *t, enum BlockSize bs, const Av1Block *b) +typedef decl_recon_b_inter_fn(*recon_b_inter_fn); + +#define decl_filter_sbrow_fn(name) \ +void (name)(Dav1dFrameContext *f, int sby) +typedef decl_filter_sbrow_fn(*filter_sbrow_fn); + +#define decl_backup_ipred_edge_fn(name) \ +void (name)(Dav1dTaskContext *t) +typedef decl_backup_ipred_edge_fn(*backup_ipred_edge_fn); + +#define decl_read_coef_blocks_fn(name) \ +void (name)(Dav1dTaskContext *t, enum BlockSize bs, const Av1Block *b) +typedef decl_read_coef_blocks_fn(*read_coef_blocks_fn); + +decl_recon_b_intra_fn(dav1d_recon_b_intra_8bpc); +decl_recon_b_intra_fn(dav1d_recon_b_intra_16bpc); + +decl_recon_b_inter_fn(dav1d_recon_b_inter_8bpc); +decl_recon_b_inter_fn(dav1d_recon_b_inter_16bpc); + +decl_filter_sbrow_fn(dav1d_filter_sbrow_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_16bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_cols_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_cols_16bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_rows_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_rows_16bpc); +void dav1d_filter_sbrow_cdef_8bpc(Dav1dTaskContext *tc, int sby); +void dav1d_filter_sbrow_cdef_16bpc(Dav1dTaskContext *tc, int sby); +decl_filter_sbrow_fn(dav1d_filter_sbrow_resize_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_resize_16bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_lr_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_lr_16bpc); + +decl_backup_ipred_edge_fn(dav1d_backup_ipred_edge_8bpc); +decl_backup_ipred_edge_fn(dav1d_backup_ipred_edge_16bpc); + +decl_read_coef_blocks_fn(dav1d_read_coef_blocks_8bpc); +decl_read_coef_blocks_fn(dav1d_read_coef_blocks_16bpc); + +#endif /* DAV1D_SRC_RECON_H */ diff --git a/third_party/dav1d/src/recon_tmpl.c b/third_party/dav1d/src/recon_tmpl.c new file mode 100644 index 0000000000..3158ef5b02 --- /dev/null +++ b/third_party/dav1d/src/recon_tmpl.c @@ -0,0 +1,2202 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/attributes.h" +#include "common/bitdepth.h" +#include "common/dump.h" +#include "common/frame.h" +#include "common/intops.h" + +#include "src/cdef_apply.h" +#include "src/ctx.h" +#include "src/ipred_prepare.h" +#include "src/lf_apply.h" +#include "src/lr_apply.h" +#include "src/recon.h" +#include "src/scan.h" +#include "src/tables.h" +#include "src/wedge.h" + +static inline unsigned read_golomb(MsacContext *const msac) { + int len = 0; + unsigned val = 1; + + while (!dav1d_msac_decode_bool_equi(msac) && len < 32) len++; + while (len--) val = (val << 1) + dav1d_msac_decode_bool_equi(msac); + + return val - 1; +} + +static inline unsigned get_skip_ctx(const TxfmInfo *const t_dim, + const enum BlockSize bs, + const uint8_t *const a, + const uint8_t *const l, + const int chroma, + const enum Dav1dPixelLayout layout) +{ + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + + if (chroma) { + const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const int not_one_blk = b_dim[2] - (!!b_dim[2] && ss_hor) > t_dim->lw || + b_dim[3] - (!!b_dim[3] && ss_ver) > t_dim->lh; + unsigned ca, cl; + +#define MERGE_CTX(dir, type, no_val) \ + c##dir = *(const type *) dir != no_val; \ + break + + switch (t_dim->lw) { + /* For some reason the MSVC CRT _wassert() function is not flagged as + * __declspec(noreturn), so when using those headers the compiler will + * expect execution to continue after an assertion has been triggered + * and will therefore complain about the use of uninitialized variables + * when compiled in debug mode if we put the default case at the end. */ + default: assert(0); /* fall-through */ + case TX_4X4: MERGE_CTX(a, uint8_t, 0x40); + case TX_8X8: MERGE_CTX(a, uint16_t, 0x4040); + case TX_16X16: MERGE_CTX(a, uint32_t, 0x40404040U); + case TX_32X32: MERGE_CTX(a, uint64_t, 0x4040404040404040ULL); + } + switch (t_dim->lh) { + default: assert(0); /* fall-through */ + case TX_4X4: MERGE_CTX(l, uint8_t, 0x40); + case TX_8X8: MERGE_CTX(l, uint16_t, 0x4040); + case TX_16X16: MERGE_CTX(l, uint32_t, 0x40404040U); + case TX_32X32: MERGE_CTX(l, uint64_t, 0x4040404040404040ULL); + } +#undef MERGE_CTX + + return 7 + not_one_blk * 3 + ca + cl; + } else if (b_dim[2] == t_dim->lw && b_dim[3] == t_dim->lh) { + return 0; + } else { + unsigned la, ll; + +#define MERGE_CTX(dir, type, tx) \ + if (tx == TX_64X64) { \ + uint64_t tmp = *(const uint64_t *) dir; \ + tmp |= *(const uint64_t *) &dir[8]; \ + l##dir = (unsigned) (tmp >> 32) | (unsigned) tmp; \ + } else \ + l##dir = *(const type *) dir; \ + if (tx == TX_32X32) l##dir |= *(const type *) &dir[sizeof(type)]; \ + if (tx >= TX_16X16) l##dir |= l##dir >> 16; \ + if (tx >= TX_8X8) l##dir |= l##dir >> 8; \ + break + + switch (t_dim->lw) { + default: assert(0); /* fall-through */ + case TX_4X4: MERGE_CTX(a, uint8_t, TX_4X4); + case TX_8X8: MERGE_CTX(a, uint16_t, TX_8X8); + case TX_16X16: MERGE_CTX(a, uint32_t, TX_16X16); + case TX_32X32: MERGE_CTX(a, uint32_t, TX_32X32); + case TX_64X64: MERGE_CTX(a, uint32_t, TX_64X64); + } + switch (t_dim->lh) { + default: assert(0); /* fall-through */ + case TX_4X4: MERGE_CTX(l, uint8_t, TX_4X4); + case TX_8X8: MERGE_CTX(l, uint16_t, TX_8X8); + case TX_16X16: MERGE_CTX(l, uint32_t, TX_16X16); + case TX_32X32: MERGE_CTX(l, uint32_t, TX_32X32); + case TX_64X64: MERGE_CTX(l, uint32_t, TX_64X64); + } +#undef MERGE_CTX + + return dav1d_skip_ctx[umin(la & 0x3F, 4)][umin(ll & 0x3F, 4)]; + } +} + +static inline unsigned get_dc_sign_ctx(const int /*enum RectTxfmSize*/ tx, + const uint8_t *const a, + const uint8_t *const l) +{ + uint64_t mask = 0xC0C0C0C0C0C0C0C0ULL, mul = 0x0101010101010101ULL; + int s; + +#if ARCH_X86_64 && defined(__GNUC__) + /* Coerce compilers into producing better code. For some reason + * every x86-64 compiler is awful at handling 64-bit constants. */ + __asm__("" : "+r"(mask), "+r"(mul)); +#endif + + switch(tx) { + default: assert(0); /* fall-through */ + case TX_4X4: { + int t = *(const uint8_t *) a >> 6; + t += *(const uint8_t *) l >> 6; + s = t - 1 - 1; + break; + } + case TX_8X8: { + uint32_t t = *(const uint16_t *) a & (uint32_t) mask; + t += *(const uint16_t *) l & (uint32_t) mask; + t *= 0x04040404U; + s = (int) (t >> 24) - 2 - 2; + break; + } + case TX_16X16: { + uint32_t t = (*(const uint32_t *) a & (uint32_t) mask) >> 6; + t += (*(const uint32_t *) l & (uint32_t) mask) >> 6; + t *= (uint32_t) mul; + s = (int) (t >> 24) - 4 - 4; + break; + } + case TX_32X32: { + uint64_t t = (*(const uint64_t *) a & mask) >> 6; + t += (*(const uint64_t *) l & mask) >> 6; + t *= mul; + s = (int) (t >> 56) - 8 - 8; + break; + } + case TX_64X64: { + uint64_t t = (*(const uint64_t *) &a[0] & mask) >> 6; + t += (*(const uint64_t *) &a[8] & mask) >> 6; + t += (*(const uint64_t *) &l[0] & mask) >> 6; + t += (*(const uint64_t *) &l[8] & mask) >> 6; + t *= mul; + s = (int) (t >> 56) - 16 - 16; + break; + } + case RTX_4X8: { + uint32_t t = *(const uint8_t *) a & (uint32_t) mask; + t += *(const uint16_t *) l & (uint32_t) mask; + t *= 0x04040404U; + s = (int) (t >> 24) - 1 - 2; + break; + } + case RTX_8X4: { + uint32_t t = *(const uint16_t *) a & (uint32_t) mask; + t += *(const uint8_t *) l & (uint32_t) mask; + t *= 0x04040404U; + s = (int) (t >> 24) - 2 - 1; + break; + } + case RTX_8X16: { + uint32_t t = *(const uint16_t *) a & (uint32_t) mask; + t += *(const uint32_t *) l & (uint32_t) mask; + t = (t >> 6) * (uint32_t) mul; + s = (int) (t >> 24) - 2 - 4; + break; + } + case RTX_16X8: { + uint32_t t = *(const uint32_t *) a & (uint32_t) mask; + t += *(const uint16_t *) l & (uint32_t) mask; + t = (t >> 6) * (uint32_t) mul; + s = (int) (t >> 24) - 4 - 2; + break; + } + case RTX_16X32: { + uint64_t t = *(const uint32_t *) a & (uint32_t) mask; + t += *(const uint64_t *) l & mask; + t = (t >> 6) * mul; + s = (int) (t >> 56) - 4 - 8; + break; + } + case RTX_32X16: { + uint64_t t = *(const uint64_t *) a & mask; + t += *(const uint32_t *) l & (uint32_t) mask; + t = (t >> 6) * mul; + s = (int) (t >> 56) - 8 - 4; + break; + } + case RTX_32X64: { + uint64_t t = (*(const uint64_t *) &a[0] & mask) >> 6; + t += (*(const uint64_t *) &l[0] & mask) >> 6; + t += (*(const uint64_t *) &l[8] & mask) >> 6; + t *= mul; + s = (int) (t >> 56) - 8 - 16; + break; + } + case RTX_64X32: { + uint64_t t = (*(const uint64_t *) &a[0] & mask) >> 6; + t += (*(const uint64_t *) &a[8] & mask) >> 6; + t += (*(const uint64_t *) &l[0] & mask) >> 6; + t *= mul; + s = (int) (t >> 56) - 16 - 8; + break; + } + case RTX_4X16: { + uint32_t t = *(const uint8_t *) a & (uint32_t) mask; + t += *(const uint32_t *) l & (uint32_t) mask; + t = (t >> 6) * (uint32_t) mul; + s = (int) (t >> 24) - 1 - 4; + break; + } + case RTX_16X4: { + uint32_t t = *(const uint32_t *) a & (uint32_t) mask; + t += *(const uint8_t *) l & (uint32_t) mask; + t = (t >> 6) * (uint32_t) mul; + s = (int) (t >> 24) - 4 - 1; + break; + } + case RTX_8X32: { + uint64_t t = *(const uint16_t *) a & (uint32_t) mask; + t += *(const uint64_t *) l & mask; + t = (t >> 6) * mul; + s = (int) (t >> 56) - 2 - 8; + break; + } + case RTX_32X8: { + uint64_t t = *(const uint64_t *) a & mask; + t += *(const uint16_t *) l & (uint32_t) mask; + t = (t >> 6) * mul; + s = (int) (t >> 56) - 8 - 2; + break; + } + case RTX_16X64: { + uint64_t t = *(const uint32_t *) a & (uint32_t) mask; + t += *(const uint64_t *) &l[0] & mask; + t = (t >> 6) + ((*(const uint64_t *) &l[8] & mask) >> 6); + t *= mul; + s = (int) (t >> 56) - 4 - 16; + break; + } + case RTX_64X16: { + uint64_t t = *(const uint64_t *) &a[0] & mask; + t += *(const uint32_t *) l & (uint32_t) mask; + t = (t >> 6) + ((*(const uint64_t *) &a[8] & mask) >> 6); + t *= mul; + s = (int) (t >> 56) - 16 - 4; + break; + } + } + + return (s != 0) + (s > 0); +} + +static inline unsigned get_lo_ctx(const uint8_t *const levels, + const enum TxClass tx_class, + unsigned *const hi_mag, + const uint8_t (*const ctx_offsets)[5], + const unsigned x, const unsigned y, + const ptrdiff_t stride) +{ + unsigned mag = levels[0 * stride + 1] + levels[1 * stride + 0]; + unsigned offset; + if (tx_class == TX_CLASS_2D) { + mag += levels[1 * stride + 1]; + *hi_mag = mag; + mag += levels[0 * stride + 2] + levels[2 * stride + 0]; + offset = ctx_offsets[umin(y, 4)][umin(x, 4)]; + } else { + mag += levels[0 * stride + 2]; + *hi_mag = mag; + mag += levels[0 * stride + 3] + levels[0 * stride + 4]; + offset = 26 + (y > 1 ? 10 : y * 5); + } + return offset + (mag > 512 ? 4 : (mag + 64) >> 7); +} + +static int decode_coefs(Dav1dTaskContext *const t, + uint8_t *const a, uint8_t *const l, + const enum RectTxfmSize tx, const enum BlockSize bs, + const Av1Block *const b, const int intra, + const int plane, coef *cf, + enum TxfmType *const txtp, uint8_t *res_ctx) +{ + Dav1dTileState *const ts = t->ts; + const int chroma = !!plane; + const Dav1dFrameContext *const f = t->f; + const int lossless = f->frame_hdr->segmentation.lossless[b->seg_id]; + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx]; + const int dbg = DEBUG_BLOCK_INFO && plane && 0; + + if (dbg) + printf("Start: r=%d\n", ts->msac.rng); + + // does this block have any non-zero coefficients + const int sctx = get_skip_ctx(t_dim, bs, a, l, chroma, f->cur.p.layout); + const int all_skip = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.coef.skip[t_dim->ctx][sctx]); + if (dbg) + printf("Post-non-zero[%d][%d][%d]: r=%d\n", + t_dim->ctx, sctx, all_skip, ts->msac.rng); + if (all_skip) { + *res_ctx = 0x40; + *txtp = lossless * WHT_WHT; /* lossless ? WHT_WHT : DCT_DCT */ + return -1; + } + + // transform type (chroma: derived, luma: explicitly coded) + if (lossless) { + assert(t_dim->max == TX_4X4); + *txtp = WHT_WHT; + } else if (t_dim->max + intra >= TX_64X64) { + *txtp = DCT_DCT; + } else if (chroma) { + // inferred from either the luma txtp (inter) or a LUT (intra) + *txtp = intra ? dav1d_txtp_from_uvmode[b->uv_mode] : + get_uv_inter_txtp(t_dim, *txtp); + } else if (!f->frame_hdr->segmentation.qidx[b->seg_id]) { + // In libaom, lossless is checked by a literal qidx == 0, but not all + // such blocks are actually lossless. The remainder gets an implicit + // transform type (for luma) + *txtp = DCT_DCT; + } else { + unsigned idx; + if (intra) { + const enum IntraPredMode y_mode_nofilt = b->y_mode == FILTER_PRED ? + dav1d_filter_mode_to_y_mode[b->y_angle] : b->y_mode; + if (f->frame_hdr->reduced_txtp_set || t_dim->min == TX_16X16) { + idx = dav1d_msac_decode_symbol_adapt4(&ts->msac, + ts->cdf.m.txtp_intra2[t_dim->min][y_mode_nofilt], 4); + *txtp = dav1d_tx_types_per_set[idx + 0]; + } else { + idx = dav1d_msac_decode_symbol_adapt8(&ts->msac, + ts->cdf.m.txtp_intra1[t_dim->min][y_mode_nofilt], 6); + *txtp = dav1d_tx_types_per_set[idx + 5]; + } + if (dbg) + printf("Post-txtp-intra[%d->%d][%d][%d->%d]: r=%d\n", + tx, t_dim->min, y_mode_nofilt, idx, *txtp, ts->msac.rng); + } else { + if (f->frame_hdr->reduced_txtp_set || t_dim->max == TX_32X32) { + idx = dav1d_msac_decode_bool_adapt(&ts->msac, + ts->cdf.m.txtp_inter3[t_dim->min]); + *txtp = (idx - 1) & IDTX; /* idx ? DCT_DCT : IDTX */ + } else if (t_dim->min == TX_16X16) { + idx = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.txtp_inter2, 11); + *txtp = dav1d_tx_types_per_set[idx + 12]; + } else { + idx = dav1d_msac_decode_symbol_adapt16(&ts->msac, + ts->cdf.m.txtp_inter1[t_dim->min], 15); + *txtp = dav1d_tx_types_per_set[idx + 24]; + } + if (dbg) + printf("Post-txtp-inter[%d->%d][%d->%d]: r=%d\n", + tx, t_dim->min, idx, *txtp, ts->msac.rng); + } + } + + // find end-of-block (eob) + int eob_bin; + const int tx2dszctx = imin(t_dim->lw, TX_32X32) + imin(t_dim->lh, TX_32X32); + const enum TxClass tx_class = dav1d_tx_type_class[*txtp]; + const int is_1d = tx_class != TX_CLASS_2D; + switch (tx2dszctx) { +#define case_sz(sz, bin, ns, is_1d) \ + case sz: { \ + uint16_t *const eob_bin_cdf = ts->cdf.coef.eob_bin_##bin[chroma]is_1d; \ + eob_bin = dav1d_msac_decode_symbol_adapt##ns(&ts->msac, eob_bin_cdf, 4 + sz); \ + break; \ + } + case_sz(0, 16, 4, [is_1d]); + case_sz(1, 32, 8, [is_1d]); + case_sz(2, 64, 8, [is_1d]); + case_sz(3, 128, 8, [is_1d]); + case_sz(4, 256, 16, [is_1d]); + case_sz(5, 512, 16, ); + case_sz(6, 1024, 16, ); +#undef case_sz + } + if (dbg) + printf("Post-eob_bin_%d[%d][%d][%d]: r=%d\n", + 16 << tx2dszctx, chroma, is_1d, eob_bin, ts->msac.rng); + int eob; + if (eob_bin > 1) { + uint16_t *const eob_hi_bit_cdf = + ts->cdf.coef.eob_hi_bit[t_dim->ctx][chroma][eob_bin]; + const int eob_hi_bit = dav1d_msac_decode_bool_adapt(&ts->msac, eob_hi_bit_cdf); + if (dbg) + printf("Post-eob_hi_bit[%d][%d][%d][%d]: r=%d\n", + t_dim->ctx, chroma, eob_bin, eob_hi_bit, ts->msac.rng); + eob = ((eob_hi_bit | 2) << (eob_bin - 2)) | + dav1d_msac_decode_bools(&ts->msac, eob_bin - 2); + if (dbg) + printf("Post-eob[%d]: r=%d\n", eob, ts->msac.rng); + } else { + eob = eob_bin; + } + assert(eob >= 0); + + // base tokens + uint16_t (*const eob_cdf)[4] = ts->cdf.coef.eob_base_tok[t_dim->ctx][chroma]; + uint16_t (*const hi_cdf)[4] = ts->cdf.coef.br_tok[imin(t_dim->ctx, 3)][chroma]; + unsigned rc, dc_tok; + + if (eob) { + uint16_t (*const lo_cdf)[4] = ts->cdf.coef.base_tok[t_dim->ctx][chroma]; + uint8_t *const levels = t->scratch.levels; // bits 0-5: tok, 6-7: lo_tok + const int sw = imin(t_dim->w, 8), sh = imin(t_dim->h, 8); + + /* eob */ + unsigned ctx = 1 + (eob > sw * sh * 2) + (eob > sw * sh * 4); + int eob_tok = dav1d_msac_decode_symbol_adapt4(&ts->msac, eob_cdf[ctx], 2); + int tok = eob_tok + 1; + int level_tok = tok * 0x41; + unsigned mag; + +#define DECODE_COEFS_CLASS(tx_class) \ + unsigned x, y; \ + if (tx_class == TX_CLASS_2D) \ + rc = scan[eob], x = rc >> shift, y = rc & mask; \ + else if (tx_class == TX_CLASS_H) \ + /* Transposing reduces the stride and padding requirements */ \ + x = eob & mask, y = eob >> shift, rc = eob; \ + else /* tx_class == TX_CLASS_V */ \ + x = eob & mask, y = eob >> shift, rc = (x << shift2) | y; \ + if (dbg) \ + printf("Post-lo_tok[%d][%d][%d][%d=%d=%d]: r=%d\n", \ + t_dim->ctx, chroma, ctx, eob, rc, tok, ts->msac.rng); \ + if (eob_tok == 2) { \ + ctx = (tx_class == TX_CLASS_2D ? (x | y) > 1 : y != 0) ? 14 : 7; \ + tok = dav1d_msac_decode_hi_tok(&ts->msac, hi_cdf[ctx]); \ + level_tok = tok + (3 << 6); \ + if (dbg) \ + printf("Post-hi_tok[%d][%d][%d][%d=%d=%d]: r=%d\n", \ + imin(t_dim->ctx, 3), chroma, ctx, eob, rc, tok, \ + ts->msac.rng); \ + } \ + cf[rc] = tok << 11; \ + levels[x * stride + y] = (uint8_t) level_tok; \ + for (int i = eob - 1; i > 0; i--) { /* ac */ \ + unsigned rc_i; \ + if (tx_class == TX_CLASS_2D) \ + rc_i = scan[i], x = rc_i >> shift, y = rc_i & mask; \ + else if (tx_class == TX_CLASS_H) \ + x = i & mask, y = i >> shift, rc_i = i; \ + else /* tx_class == TX_CLASS_V */ \ + x = i & mask, y = i >> shift, rc_i = (x << shift2) | y; \ + assert(x < 32 && y < 32); \ + uint8_t *const level = levels + x * stride + y; \ + ctx = get_lo_ctx(level, tx_class, &mag, lo_ctx_offsets, x, y, stride); \ + if (tx_class == TX_CLASS_2D) \ + y |= x; \ + tok = dav1d_msac_decode_symbol_adapt4(&ts->msac, lo_cdf[ctx], 3); \ + if (dbg) \ + printf("Post-lo_tok[%d][%d][%d][%d=%d=%d]: r=%d\n", \ + t_dim->ctx, chroma, ctx, i, rc_i, tok, ts->msac.rng); \ + if (tok == 3) { \ + mag &= 63; \ + ctx = (y > (tx_class == TX_CLASS_2D) ? 14 : 7) + \ + (mag > 12 ? 6 : (mag + 1) >> 1); \ + tok = dav1d_msac_decode_hi_tok(&ts->msac, hi_cdf[ctx]); \ + if (dbg) \ + printf("Post-hi_tok[%d][%d][%d][%d=%d=%d]: r=%d\n", \ + imin(t_dim->ctx, 3), chroma, ctx, i, rc_i, tok, \ + ts->msac.rng); \ + *level = (uint8_t) (tok + (3 << 6)); \ + cf[rc_i] = (tok << 11) | rc; \ + rc = rc_i; \ + } else { \ + /* 0x1 for tok, 0x7ff as bitmask for rc, 0x41 for level_tok */ \ + tok *= 0x17ff41; \ + *level = (uint8_t) tok; \ + /* tok ? (tok << 11) | rc : 0 */ \ + tok = (tok >> 9) & (rc + ~0x7ffu); \ + if (tok) rc = rc_i; \ + cf[rc_i] = tok; \ + } \ + } \ + /* dc */ \ + ctx = (tx_class == TX_CLASS_2D) ? 0 : \ + get_lo_ctx(levels, tx_class, &mag, lo_ctx_offsets, 0, 0, stride); \ + dc_tok = dav1d_msac_decode_symbol_adapt4(&ts->msac, lo_cdf[ctx], 3); \ + if (dbg) \ + printf("Post-dc_lo_tok[%d][%d][%d][%d]: r=%d\n", \ + t_dim->ctx, chroma, ctx, dc_tok, ts->msac.rng); \ + if (dc_tok == 3) { \ + if (tx_class == TX_CLASS_2D) \ + mag = levels[0 * stride + 1] + levels[1 * stride + 0] + \ + levels[1 * stride + 1]; \ + mag &= 63; \ + ctx = mag > 12 ? 6 : (mag + 1) >> 1; \ + dc_tok = dav1d_msac_decode_hi_tok(&ts->msac, hi_cdf[ctx]); \ + if (dbg) \ + printf("Post-dc_hi_tok[%d][%d][0][%d]: r=%d\n", \ + imin(t_dim->ctx, 3), chroma, dc_tok, ts->msac.rng); \ + } \ + break + + const uint16_t *scan; + switch (tx_class) { + case TX_CLASS_2D: { + const unsigned nonsquare_tx = tx >= RTX_4X8; + const uint8_t (*const lo_ctx_offsets)[5] = + dav1d_lo_ctx_offsets[nonsquare_tx + (tx & nonsquare_tx)]; + scan = dav1d_scans[tx]; + const ptrdiff_t stride = 4 * sh; + const unsigned shift = t_dim->lh < 4 ? t_dim->lh + 2 : 5, shift2 = 0; + const unsigned mask = 4 * sh - 1; + memset(levels, 0, stride * (4 * sw + 2)); + DECODE_COEFS_CLASS(TX_CLASS_2D); + } + case TX_CLASS_H: { + const uint8_t (*const lo_ctx_offsets)[5] = NULL; + const ptrdiff_t stride = 16; + const unsigned shift = t_dim->lh + 2, shift2 = 0; + const unsigned mask = 4 * sh - 1; + memset(levels, 0, stride * (4 * sh + 2)); + DECODE_COEFS_CLASS(TX_CLASS_H); + } + case TX_CLASS_V: { + const uint8_t (*const lo_ctx_offsets)[5] = NULL; + const ptrdiff_t stride = 16; + const unsigned shift = t_dim->lw + 2, shift2 = t_dim->lh + 2; + const unsigned mask = 4 * sw - 1; + memset(levels, 0, stride * (4 * sw + 2)); + DECODE_COEFS_CLASS(TX_CLASS_V); + } +#undef DECODE_COEFS_CLASS + default: assert(0); + } + } else { // dc-only + int tok_br = dav1d_msac_decode_symbol_adapt4(&ts->msac, eob_cdf[0], 2); + dc_tok = 1 + tok_br; + if (dbg) + printf("Post-dc_lo_tok[%d][%d][%d][%d]: r=%d\n", + t_dim->ctx, chroma, 0, dc_tok, ts->msac.rng); + if (tok_br == 2) { + dc_tok = dav1d_msac_decode_hi_tok(&ts->msac, hi_cdf[0]); + if (dbg) + printf("Post-dc_hi_tok[%d][%d][0][%d]: r=%d\n", + imin(t_dim->ctx, 3), chroma, dc_tok, ts->msac.rng); + } + rc = 0; + } + + // residual and sign + const uint16_t *const dq_tbl = ts->dq[b->seg_id][plane]; + const uint8_t *const qm_tbl = *txtp < IDTX ? f->qm[tx][plane] : NULL; + const int dq_shift = imax(0, t_dim->ctx - 2); + const int cf_max = ~(~127U << (BITDEPTH == 8 ? 8 : f->cur.p.bpc)); + unsigned cul_level, dc_sign_level; + + if (!dc_tok) { + cul_level = 0; + dc_sign_level = 1 << 6; + if (qm_tbl) goto ac_qm; + goto ac_noqm; + } + + const int dc_sign_ctx = get_dc_sign_ctx(tx, a, l); + uint16_t *const dc_sign_cdf = ts->cdf.coef.dc_sign[chroma][dc_sign_ctx]; + const int dc_sign = dav1d_msac_decode_bool_adapt(&ts->msac, dc_sign_cdf); + if (dbg) + printf("Post-dc_sign[%d][%d][%d]: r=%d\n", + chroma, dc_sign_ctx, dc_sign, ts->msac.rng); + + int dc_dq = dq_tbl[0]; + dc_sign_level = (dc_sign - 1) & (2 << 6); + + if (qm_tbl) { + dc_dq = (dc_dq * qm_tbl[0] + 16) >> 5; + + if (dc_tok == 15) { + dc_tok = read_golomb(&ts->msac) + 15; + if (dbg) + printf("Post-dc_residual[%d->%d]: r=%d\n", + dc_tok - 15, dc_tok, ts->msac.rng); + + dc_tok &= 0xfffff; + dc_dq = (dc_dq * dc_tok) & 0xffffff; + } else { + dc_dq *= dc_tok; + assert(dc_dq <= 0xffffff); + } + cul_level = dc_tok; + dc_dq >>= dq_shift; + dc_dq = umin(dc_dq, cf_max + dc_sign); + cf[0] = (coef) (dc_sign ? -dc_dq : dc_dq); + + if (rc) ac_qm: { + const unsigned ac_dq = dq_tbl[1]; + do { + const int sign = dav1d_msac_decode_bool_equi(&ts->msac); + if (dbg) + printf("Post-sign[%d=%d]: r=%d\n", rc, sign, ts->msac.rng); + const unsigned rc_tok = cf[rc]; + unsigned tok, dq = (ac_dq * qm_tbl[rc] + 16) >> 5; + int dq_sat; + + if (rc_tok >= (15 << 11)) { + tok = read_golomb(&ts->msac) + 15; + if (dbg) + printf("Post-residual[%d=%d->%d]: r=%d\n", + rc, tok - 15, tok, ts->msac.rng); + + tok &= 0xfffff; + dq = (dq * tok) & 0xffffff; + } else { + tok = rc_tok >> 11; + dq *= tok; + assert(dq <= 0xffffff); + } + cul_level += tok; + dq >>= dq_shift; + dq_sat = umin(dq, cf_max + sign); + cf[rc] = (coef) (sign ? -dq_sat : dq_sat); + + rc = rc_tok & 0x3ff; + } while (rc); + } + } else { + // non-qmatrix is the common case and allows for additional optimizations + if (dc_tok == 15) { + dc_tok = read_golomb(&ts->msac) + 15; + if (dbg) + printf("Post-dc_residual[%d->%d]: r=%d\n", + dc_tok - 15, dc_tok, ts->msac.rng); + + dc_tok &= 0xfffff; + dc_dq = ((dc_dq * dc_tok) & 0xffffff) >> dq_shift; + dc_dq = umin(dc_dq, cf_max + dc_sign); + } else { + dc_dq = ((dc_dq * dc_tok) >> dq_shift); + assert(dc_dq <= cf_max); + } + cul_level = dc_tok; + cf[0] = (coef) (dc_sign ? -dc_dq : dc_dq); + + if (rc) ac_noqm: { + const unsigned ac_dq = dq_tbl[1]; + do { + const int sign = dav1d_msac_decode_bool_equi(&ts->msac); + if (dbg) + printf("Post-sign[%d=%d]: r=%d\n", rc, sign, ts->msac.rng); + const unsigned rc_tok = cf[rc]; + unsigned tok; + int dq; + + // residual + if (rc_tok >= (15 << 11)) { + tok = read_golomb(&ts->msac) + 15; + if (dbg) + printf("Post-residual[%d=%d->%d]: r=%d\n", + rc, tok - 15, tok, ts->msac.rng); + + // coefficient parsing, see 5.11.39 + tok &= 0xfffff; + + // dequant, see 7.12.3 + dq = ((ac_dq * tok) & 0xffffff) >> dq_shift; + dq = umin(dq, cf_max + sign); + } else { + // cannot exceed cf_max, so we can avoid the clipping + tok = rc_tok >> 11; + dq = ((ac_dq * tok) >> dq_shift); + assert(dq <= cf_max); + } + cul_level += tok; + cf[rc] = (coef) (sign ? -dq : dq); + + rc = rc_tok & 0x3ff; // next non-zero rc, zero if eob + } while (rc); + } + } + + // context + *res_ctx = umin(cul_level, 63) | dc_sign_level; + + return eob; +} + +static void read_coef_tree(Dav1dTaskContext *const t, + const enum BlockSize bs, const Av1Block *const b, + const enum RectTxfmSize ytx, const int depth, + const uint16_t *const tx_split, + const int x_off, const int y_off, pixel *dst) +{ + const Dav1dFrameContext *const f = t->f; + Dav1dTileState *const ts = t->ts; + const Dav1dDSPContext *const dsp = f->dsp; + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[ytx]; + const int txw = t_dim->w, txh = t_dim->h; + + /* y_off can be larger than 3 since lossless blocks use TX_4X4 but can't + * be splitted. Aviods an undefined left shift. */ + if (depth < 2 && tx_split[depth] && + tx_split[depth] & (1 << (y_off * 4 + x_off))) + { + const enum RectTxfmSize sub = t_dim->sub; + const TxfmInfo *const sub_t_dim = &dav1d_txfm_dimensions[sub]; + const int txsw = sub_t_dim->w, txsh = sub_t_dim->h; + + read_coef_tree(t, bs, b, sub, depth + 1, tx_split, + x_off * 2 + 0, y_off * 2 + 0, dst); + t->bx += txsw; + if (txw >= txh && t->bx < f->bw) + read_coef_tree(t, bs, b, sub, depth + 1, tx_split, x_off * 2 + 1, + y_off * 2 + 0, dst ? &dst[4 * txsw] : NULL); + t->bx -= txsw; + t->by += txsh; + if (txh >= txw && t->by < f->bh) { + if (dst) + dst += 4 * txsh * PXSTRIDE(f->cur.stride[0]); + read_coef_tree(t, bs, b, sub, depth + 1, tx_split, + x_off * 2 + 0, y_off * 2 + 1, dst); + t->bx += txsw; + if (txw >= txh && t->bx < f->bw) + read_coef_tree(t, bs, b, sub, depth + 1, tx_split, x_off * 2 + 1, + y_off * 2 + 1, dst ? &dst[4 * txsw] : NULL); + t->bx -= txsw; + } + t->by -= txsh; + } else { + const int bx4 = t->bx & 31, by4 = t->by & 31; + enum TxfmType txtp; + uint8_t cf_ctx; + int eob; + coef *cf; + struct CodedBlockInfo *cbi; + + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].cf); + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; + cbi = &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; + } else { + cf = bitfn(t->cf); + } + if (t->frame_thread.pass != 2) { + eob = decode_coefs(t, &t->a->lcoef[bx4], &t->l.lcoef[by4], + ytx, bs, b, 0, 0, cf, &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", + ytx, txtp, eob, ts->msac.rng); +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir lcoef[off], cf_ctx, sz) + case_set_upto16_with_default(imin(txh, f->bh - t->by), l., 1, by4); + case_set_upto16_with_default(imin(txw, f->bw - t->bx), a->, 0, bx4); +#undef default_memset +#undef set_ctx +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + for (int y = 0; y < txh; y++) { \ + rep_macro(type, txtp_map, 0, mul * txtp); \ + txtp_map += 32; \ + } + uint8_t *txtp_map = &t->txtp_map[by4 * 32 + bx4]; + case_set_upto16(txw,,,); +#undef set_ctx + if (t->frame_thread.pass == 1) { + cbi->eob[0] = eob; + cbi->txtp[0] = txtp; + } + } else { + eob = cbi->eob[0]; + txtp = cbi->txtp[0]; + } + if (!(t->frame_thread.pass & 1)) { + assert(dst); + if (eob >= 0) { + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + coef_dump(cf, imin(t_dim->h, 8) * 4, imin(t_dim->w, 8) * 4, 3, "dq"); + dsp->itx.itxfm_add[ytx][txtp](dst, f->cur.stride[0], cf, eob + HIGHBD_CALL_SUFFIX); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + hex_dump(dst, f->cur.stride[0], t_dim->w * 4, t_dim->h * 4, "recon"); + } + } + } +} + +void bytefn(dav1d_read_coef_blocks)(Dav1dTaskContext *const t, + const enum BlockSize bs, const Av1Block *const b) +{ + const Dav1dFrameContext *const f = t->f; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int bx4 = t->bx & 31, by4 = t->by & 31; + const int cbx4 = bx4 >> ss_hor, cby4 = by4 >> ss_ver; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + const int cbw4 = (bw4 + ss_hor) >> ss_hor, cbh4 = (bh4 + ss_ver) >> ss_ver; + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 && + (bw4 > ss_hor || t->bx & 1) && + (bh4 > ss_ver || t->by & 1); + + if (b->skip) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * 0x40) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[0], off, mul * 0x40); \ + rep_macro(type, t->dir ccoef[1], off, mul * 0x40) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + return; + } + + Dav1dTileState *const ts = t->ts; + const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); + const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; + assert(t->frame_thread.pass == 1); + assert(!b->skip); + const TxfmInfo *const uv_t_dim = &dav1d_txfm_dimensions[b->uvtx]; + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[b->intra ? b->tx : b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; + + for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); + for (int init_x = 0; init_x < w4; init_x += 16) { + const int sub_w4 = imin(w4, init_x + 16); + int y_off = !!init_y, y, x; + for (y = init_y, t->by += init_y; y < sub_h4; + y += t_dim->h, t->by += t_dim->h, y_off++) + { + struct CodedBlockInfo *const cbi = + &f->frame_thread.cbi[t->by * f->b4_stride]; + int x_off = !!init_x; + for (x = init_x, t->bx += init_x; x < sub_w4; + x += t_dim->w, t->bx += t_dim->w, x_off++) + { + if (!b->intra) { + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, + x_off, y_off, NULL); + } else { + uint8_t cf_ctx = 0x40; + enum TxfmType txtp; + const int eob = cbi[t->bx].eob[0] = + decode_coefs(t, &t->a->lcoef[bx4 + x], + &t->l.lcoef[by4 + y], b->tx, bs, b, 1, + 0, ts->frame_thread[1].cf, &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", + b->tx, txtp, eob, ts->msac.rng); + cbi[t->bx].txtp[0] = txtp; + ts->frame_thread[1].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir lcoef[off], cf_ctx, sz) + case_set_upto16_with_default(imin(t_dim->h, f->bh - t->by), + l., 1, by4 + y); + case_set_upto16_with_default(imin(t_dim->w, f->bw - t->bx), + a->, 0, bx4 + x); +#undef default_memset +#undef set_ctx + } + } + t->bx -= x; + } + t->by -= y; + + if (!has_chroma) continue; + + const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); + const int sub_cw4 = imin(cw4, (init_x + 16) >> ss_hor); + for (int pl = 0; pl < 2; pl++) { + for (y = init_y >> ss_ver, t->by += init_y; y < sub_ch4; + y += uv_t_dim->h, t->by += uv_t_dim->h << ss_ver) + { + struct CodedBlockInfo *const cbi = + &f->frame_thread.cbi[t->by * f->b4_stride]; + for (x = init_x >> ss_hor, t->bx += init_x; x < sub_cw4; + x += uv_t_dim->w, t->bx += uv_t_dim->w << ss_hor) + { + uint8_t cf_ctx = 0x40; + enum TxfmType txtp; + if (!b->intra) + txtp = t->txtp_map[(by4 + (y << ss_ver)) * 32 + + bx4 + (x << ss_hor)]; + const int eob = cbi[t->bx].eob[1 + pl] = + decode_coefs(t, &t->a->ccoef[pl][cbx4 + x], + &t->l.ccoef[pl][cby4 + y], b->uvtx, bs, + b, b->intra, 1 + pl, ts->frame_thread[1].cf, + &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-uv-cf-blk[pl=%d,tx=%d," + "txtp=%d,eob=%d]: r=%d\n", + pl, b->uvtx, txtp, eob, ts->msac.rng); + cbi[t->bx].txtp[1 + pl] = txtp; + ts->frame_thread[1].cf += uv_t_dim->w * uv_t_dim->h * 16; +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir ccoef[pl][off], cf_ctx, sz) + case_set_upto16_with_default( \ + imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver), + l., 1, cby4 + y); + case_set_upto16_with_default( \ + imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor), + a->, 0, cbx4 + x); +#undef default_memset +#undef set_ctx + } + t->bx -= x << ss_hor; + } + t->by -= y << ss_ver; + } + } + } +} + +static int mc(Dav1dTaskContext *const t, + pixel *const dst8, int16_t *const dst16, const ptrdiff_t dst_stride, + const int bw4, const int bh4, + const int bx, const int by, const int pl, + const mv mv, const Dav1dThreadPicture *const refp, const int refidx, + const enum Filter2d filter_2d) +{ + assert((dst8 != NULL) ^ (dst16 != NULL)); + const Dav1dFrameContext *const f = t->f; + const int ss_ver = !!pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = !!pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + const int mvx = mv.x, mvy = mv.y; + const int mx = mvx & (15 >> !ss_hor), my = mvy & (15 >> !ss_ver); + ptrdiff_t ref_stride = refp->p.stride[!!pl]; + const pixel *ref; + + if (refp->p.p.w == f->cur.p.w && refp->p.p.h == f->cur.p.h) { + const int dx = bx * h_mul + (mvx >> (3 + ss_hor)); + const int dy = by * v_mul + (mvy >> (3 + ss_ver)); + int w, h; + + if (refp->p.data[0] != f->cur.data[0]) { // i.e. not for intrabc + w = (f->cur.p.w + ss_hor) >> ss_hor; + h = (f->cur.p.h + ss_ver) >> ss_ver; + } else { + w = f->bw * 4 >> ss_hor; + h = f->bh * 4 >> ss_ver; + } + if (dx < !!mx * 3 || dy < !!my * 3 || + dx + bw4 * h_mul + !!mx * 4 > w || + dy + bh4 * v_mul + !!my * 4 > h) + { + pixel *const emu_edge_buf = bitfn(t->scratch.emu_edge); + f->dsp->mc.emu_edge(bw4 * h_mul + !!mx * 7, bh4 * v_mul + !!my * 7, + w, h, dx - !!mx * 3, dy - !!my * 3, + emu_edge_buf, 192 * sizeof(pixel), + refp->p.data[pl], ref_stride); + ref = &emu_edge_buf[192 * !!my * 3 + !!mx * 3]; + ref_stride = 192 * sizeof(pixel); + } else { + ref = ((pixel *) refp->p.data[pl]) + PXSTRIDE(ref_stride) * dy + dx; + } + + if (dst8 != NULL) { + f->dsp->mc.mc[filter_2d](dst8, dst_stride, ref, ref_stride, bw4 * h_mul, + bh4 * v_mul, mx << !ss_hor, my << !ss_ver + HIGHBD_CALL_SUFFIX); + } else { + f->dsp->mc.mct[filter_2d](dst16, ref, ref_stride, bw4 * h_mul, + bh4 * v_mul, mx << !ss_hor, my << !ss_ver + HIGHBD_CALL_SUFFIX); + } + } else { + assert(refp != &f->sr_cur); + + const int orig_pos_y = (by * v_mul << 4) + mvy * (1 << !ss_ver); + const int orig_pos_x = (bx * h_mul << 4) + mvx * (1 << !ss_hor); +#define scale_mv(res, val, scale) do { \ + const int64_t tmp = (int64_t)(val) * scale + (scale - 0x4000) * 8; \ + res = apply_sign64((int) ((llabs(tmp) + 128) >> 8), tmp) + 32; \ + } while (0) + int pos_y, pos_x; + scale_mv(pos_x, orig_pos_x, f->svc[refidx][0].scale); + scale_mv(pos_y, orig_pos_y, f->svc[refidx][1].scale); +#undef scale_mv + const int left = pos_x >> 10; + const int top = pos_y >> 10; + const int right = + ((pos_x + (bw4 * h_mul - 1) * f->svc[refidx][0].step) >> 10) + 1; + const int bottom = + ((pos_y + (bh4 * v_mul - 1) * f->svc[refidx][1].step) >> 10) + 1; + + if (DEBUG_BLOCK_INFO) + printf("Off %dx%d [%d,%d,%d], size %dx%d [%d,%d]\n", + left, top, orig_pos_x, f->svc[refidx][0].scale, refidx, + right-left, bottom-top, + f->svc[refidx][0].step, f->svc[refidx][1].step); + + const int w = (refp->p.p.w + ss_hor) >> ss_hor; + const int h = (refp->p.p.h + ss_ver) >> ss_ver; + if (left < 3 || top < 3 || right + 4 > w || bottom + 4 > h) { + pixel *const emu_edge_buf = bitfn(t->scratch.emu_edge); + f->dsp->mc.emu_edge(right - left + 7, bottom - top + 7, + w, h, left - 3, top - 3, + emu_edge_buf, 320 * sizeof(pixel), + refp->p.data[pl], ref_stride); + ref = &emu_edge_buf[320 * 3 + 3]; + ref_stride = 320 * sizeof(pixel); + if (DEBUG_BLOCK_INFO) printf("Emu\n"); + } else { + ref = ((pixel *) refp->p.data[pl]) + PXSTRIDE(ref_stride) * top + left; + } + + if (dst8 != NULL) { + f->dsp->mc.mc_scaled[filter_2d](dst8, dst_stride, ref, ref_stride, + bw4 * h_mul, bh4 * v_mul, + pos_x & 0x3ff, pos_y & 0x3ff, + f->svc[refidx][0].step, + f->svc[refidx][1].step + HIGHBD_CALL_SUFFIX); + } else { + f->dsp->mc.mct_scaled[filter_2d](dst16, ref, ref_stride, + bw4 * h_mul, bh4 * v_mul, + pos_x & 0x3ff, pos_y & 0x3ff, + f->svc[refidx][0].step, + f->svc[refidx][1].step + HIGHBD_CALL_SUFFIX); + } + } + + return 0; +} + +static int obmc(Dav1dTaskContext *const t, + pixel *const dst, const ptrdiff_t dst_stride, + const uint8_t *const b_dim, const int pl, + const int bx4, const int by4, const int w4, const int h4) +{ + assert(!(t->bx & 1) && !(t->by & 1)); + const Dav1dFrameContext *const f = t->f; + /*const*/ refmvs_block **r = &t->rt.r[(t->by & 31) + 5]; + pixel *const lap = bitfn(t->scratch.lap); + const int ss_ver = !!pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = !!pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + int res; + + if (t->by > t->ts->tiling.row_start && + (!pl || b_dim[0] * h_mul + b_dim[1] * v_mul >= 16)) + { + for (int i = 0, x = 0; x < w4 && i < imin(b_dim[2], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const a_r = &r[-1][t->bx + x + 1]; + const uint8_t *const a_b_dim = dav1d_block_dimensions[a_r->bs]; + const int step4 = iclip(a_b_dim[0], 2, 16); + + if (a_r->ref.ref[0] > 0) { + const int ow4 = imin(step4, b_dim[0]); + const int oh4 = imin(b_dim[1], 16) >> 1; + res = mc(t, lap, NULL, ow4 * h_mul * sizeof(pixel), ow4, (oh4 * 3 + 3) >> 2, + t->bx + x, t->by, pl, a_r->mv.mv[0], + &f->refp[a_r->ref.ref[0] - 1], a_r->ref.ref[0] - 1, + dav1d_filter_2d[t->a->filter[1][bx4 + x + 1]][t->a->filter[0][bx4 + x + 1]]); + if (res) return res; + f->dsp->mc.blend_h(&dst[x * h_mul], dst_stride, lap, + h_mul * ow4, v_mul * oh4); + i++; + } + x += step4; + } + } + + if (t->bx > t->ts->tiling.col_start) + for (int i = 0, y = 0; y < h4 && i < imin(b_dim[3], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const l_r = &r[y + 1][t->bx - 1]; + const uint8_t *const l_b_dim = dav1d_block_dimensions[l_r->bs]; + const int step4 = iclip(l_b_dim[1], 2, 16); + + if (l_r->ref.ref[0] > 0) { + const int ow4 = imin(b_dim[0], 16) >> 1; + const int oh4 = imin(step4, b_dim[1]); + res = mc(t, lap, NULL, h_mul * ow4 * sizeof(pixel), ow4, oh4, + t->bx, t->by + y, pl, l_r->mv.mv[0], + &f->refp[l_r->ref.ref[0] - 1], l_r->ref.ref[0] - 1, + dav1d_filter_2d[t->l.filter[1][by4 + y + 1]][t->l.filter[0][by4 + y + 1]]); + if (res) return res; + f->dsp->mc.blend_v(&dst[y * v_mul * PXSTRIDE(dst_stride)], + dst_stride, lap, h_mul * ow4, v_mul * oh4); + i++; + } + y += step4; + } + return 0; +} + +static int warp_affine(Dav1dTaskContext *const t, + pixel *dst8, int16_t *dst16, const ptrdiff_t dstride, + const uint8_t *const b_dim, const int pl, + const Dav1dThreadPicture *const refp, + const Dav1dWarpedMotionParams *const wmp) +{ + assert((dst8 != NULL) ^ (dst16 != NULL)); + const Dav1dFrameContext *const f = t->f; + const Dav1dDSPContext *const dsp = f->dsp; + const int ss_ver = !!pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = !!pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + assert(!((b_dim[0] * h_mul) & 7) && !((b_dim[1] * v_mul) & 7)); + const int32_t *const mat = wmp->matrix; + const int width = (refp->p.p.w + ss_hor) >> ss_hor; + const int height = (refp->p.p.h + ss_ver) >> ss_ver; + + for (int y = 0; y < b_dim[1] * v_mul; y += 8) { + const int src_y = t->by * 4 + ((y + 4) << ss_ver); + const int64_t mat3_y = (int64_t) mat[3] * src_y + mat[0]; + const int64_t mat5_y = (int64_t) mat[5] * src_y + mat[1]; + for (int x = 0; x < b_dim[0] * h_mul; x += 8) { + // calculate transformation relative to center of 8x8 block in + // luma pixel units + const int src_x = t->bx * 4 + ((x + 4) << ss_hor); + const int64_t mvx = ((int64_t) mat[2] * src_x + mat3_y) >> ss_hor; + const int64_t mvy = ((int64_t) mat[4] * src_x + mat5_y) >> ss_ver; + + const int dx = (int) (mvx >> 16) - 4; + const int mx = (((int) mvx & 0xffff) - wmp->u.p.alpha * 4 - + wmp->u.p.beta * 7) & ~0x3f; + const int dy = (int) (mvy >> 16) - 4; + const int my = (((int) mvy & 0xffff) - wmp->u.p.gamma * 4 - + wmp->u.p.delta * 4) & ~0x3f; + + const pixel *ref_ptr; + ptrdiff_t ref_stride = refp->p.stride[!!pl]; + + if (dx < 3 || dx + 8 + 4 > width || dy < 3 || dy + 8 + 4 > height) { + pixel *const emu_edge_buf = bitfn(t->scratch.emu_edge); + f->dsp->mc.emu_edge(15, 15, width, height, dx - 3, dy - 3, + emu_edge_buf, 32 * sizeof(pixel), + refp->p.data[pl], ref_stride); + ref_ptr = &emu_edge_buf[32 * 3 + 3]; + ref_stride = 32 * sizeof(pixel); + } else { + ref_ptr = ((pixel *) refp->p.data[pl]) + PXSTRIDE(ref_stride) * dy + dx; + } + if (dst16 != NULL) + dsp->mc.warp8x8t(&dst16[x], dstride, ref_ptr, ref_stride, + wmp->u.abcd, mx, my HIGHBD_CALL_SUFFIX); + else + dsp->mc.warp8x8(&dst8[x], dstride, ref_ptr, ref_stride, + wmp->u.abcd, mx, my HIGHBD_CALL_SUFFIX); + } + if (dst8) dst8 += 8 * PXSTRIDE(dstride); + else dst16 += 8 * dstride; + } + return 0; +} + +void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize bs, + const enum EdgeFlags intra_edge_flags, + const Av1Block *const b) +{ + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + const Dav1dDSPContext *const dsp = f->dsp; + const int bx4 = t->bx & 31, by4 = t->by & 31; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int cbx4 = bx4 >> ss_hor, cby4 = by4 >> ss_ver; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); + const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 && + (bw4 > ss_hor || t->bx & 1) && + (bh4 > ss_ver || t->by & 1); + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[b->tx]; + const TxfmInfo *const uv_t_dim = &dav1d_txfm_dimensions[b->uvtx]; + + // coefficient coding + pixel *const edge = bitfn(t->scratch.edge) + 128; + const int cbw4 = (bw4 + ss_hor) >> ss_hor, cbh4 = (bh4 + ss_ver) >> ss_ver; + + const int intra_edge_filter_flag = f->seq_hdr->intra_edge_filter << 10; + + for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); + const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); + for (int init_x = 0; init_x < w4; init_x += 16) { + if (b->pal_sz[0]) { + pixel *dst = ((pixel *) f->cur.data[0]) + + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + t->bx); + const uint8_t *pal_idx; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += bw4 * bh4 * 16; + } else { + pal_idx = t->scratch.pal_idx; + } + const uint16_t *const pal = t->frame_thread.pass ? + f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + + ((t->bx >> 1) + (t->by & 1))][0] : t->scratch.pal[0]; + f->dsp->ipred.pal_pred(dst, f->cur.stride[0], pal, + pal_idx, bw4 * 4, bh4 * 4); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + hex_dump(dst, PXSTRIDE(f->cur.stride[0]), + bw4 * 4, bh4 * 4, "y-pal-pred"); + } + + const int intra_flags = (sm_flag(t->a, bx4) | + sm_flag(&t->l, by4) | + intra_edge_filter_flag); + const int sb_has_tr = init_x + 16 < w4 ? 1 : init_y ? 0 : + intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT; + const int sb_has_bl = init_x ? 0 : init_y + 16 < h4 ? 1 : + intra_edge_flags & EDGE_I444_LEFT_HAS_BOTTOM; + int y, x; + const int sub_w4 = imin(w4, init_x + 16); + for (y = init_y, t->by += init_y; y < sub_h4; + y += t_dim->h, t->by += t_dim->h) + { + pixel *dst = ((pixel *) f->cur.data[0]) + + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + + t->bx + init_x); + for (x = init_x, t->bx += init_x; x < sub_w4; + x += t_dim->w, t->bx += t_dim->w) + { + if (b->pal_sz[0]) goto skip_y_pred; + + int angle = b->y_angle; + const enum EdgeFlags edge_flags = + (((y > init_y || !sb_has_tr) && (x + t_dim->w >= sub_w4)) ? + 0 : EDGE_I444_TOP_HAS_RIGHT) | + ((x > init_x || (!sb_has_bl && y + t_dim->h >= sub_h4)) ? + 0 : EDGE_I444_LEFT_HAS_BOTTOM); + const pixel *top_sb_edge = NULL; + if (!(t->by & (f->sb_step - 1))) { + top_sb_edge = f->ipred_edge[0]; + const int sby = t->by >> f->sb_shift; + top_sb_edge += f->sb128w * 128 * (sby - 1); + } + const enum IntraPredMode m = + bytefn(dav1d_prepare_intra_edges)(t->bx, + t->bx > ts->tiling.col_start, + t->by, + t->by > ts->tiling.row_start, + ts->tiling.col_end, + ts->tiling.row_end, + edge_flags, dst, + f->cur.stride[0], top_sb_edge, + b->y_mode, &angle, + t_dim->w, t_dim->h, + f->seq_hdr->intra_edge_filter, + edge HIGHBD_CALL_SUFFIX); + dsp->ipred.intra_pred[m](dst, f->cur.stride[0], edge, + t_dim->w * 4, t_dim->h * 4, + angle | intra_flags, + 4 * f->bw - 4 * t->bx, + 4 * f->bh - 4 * t->by + HIGHBD_CALL_SUFFIX); + + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) { + hex_dump(edge - t_dim->h * 4, t_dim->h * 4, + t_dim->h * 4, 2, "l"); + hex_dump(edge, 0, 1, 1, "tl"); + hex_dump(edge + 1, t_dim->w * 4, + t_dim->w * 4, 2, "t"); + hex_dump(dst, f->cur.stride[0], + t_dim->w * 4, t_dim->h * 4, "y-intra-pred"); + } + + skip_y_pred: {} + if (!b->skip) { + coef *cf; + int eob; + enum TxfmType txtp; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; + const struct CodedBlockInfo *const cbi = + &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; + eob = cbi->eob[0]; + txtp = cbi->txtp[0]; + } else { + uint8_t cf_ctx; + cf = bitfn(t->cf); + eob = decode_coefs(t, &t->a->lcoef[bx4 + x], + &t->l.lcoef[by4 + y], b->tx, bs, + b, 1, 0, cf, &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", + b->tx, txtp, eob, ts->msac.rng); +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir lcoef[off], cf_ctx, sz) + case_set_upto16_with_default(imin(t_dim->h, f->bh - t->by), \ + l., 1, by4 + y); + case_set_upto16_with_default(imin(t_dim->w, f->bw - t->bx), \ + a->, 0, bx4 + x); +#undef default_memset +#undef set_ctx + } + if (eob >= 0) { + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + coef_dump(cf, imin(t_dim->h, 8) * 4, + imin(t_dim->w, 8) * 4, 3, "dq"); + dsp->itx.itxfm_add[b->tx] + [txtp](dst, + f->cur.stride[0], + cf, eob HIGHBD_CALL_SUFFIX); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + hex_dump(dst, f->cur.stride[0], + t_dim->w * 4, t_dim->h * 4, "recon"); + } + } else if (!t->frame_thread.pass) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * 0x40) + case_set_upto16(t_dim->h, l., 1, by4 + y); + case_set_upto16(t_dim->w, a->, 0, bx4 + x); +#undef set_ctx + } + dst += 4 * t_dim->w; + } + t->bx -= x; + } + t->by -= y; + + if (!has_chroma) continue; + + const ptrdiff_t stride = f->cur.stride[1]; + + if (b->uv_mode == CFL_PRED) { + assert(!init_x && !init_y); + + int16_t *const ac = t->scratch.ac; + pixel *y_src = ((pixel *) f->cur.data[0]) + 4 * (t->bx & ~ss_hor) + + 4 * (t->by & ~ss_ver) * PXSTRIDE(f->cur.stride[0]); + const ptrdiff_t uv_off = 4 * ((t->bx >> ss_hor) + + (t->by >> ss_ver) * PXSTRIDE(stride)); + pixel *const uv_dst[2] = { ((pixel *) f->cur.data[1]) + uv_off, + ((pixel *) f->cur.data[2]) + uv_off }; + + const int furthest_r = + ((cw4 << ss_hor) + t_dim->w - 1) & ~(t_dim->w - 1); + const int furthest_b = + ((ch4 << ss_ver) + t_dim->h - 1) & ~(t_dim->h - 1); + dsp->ipred.cfl_ac[f->cur.p.layout - 1](ac, y_src, f->cur.stride[0], + cbw4 - (furthest_r >> ss_hor), + cbh4 - (furthest_b >> ss_ver), + cbw4 * 4, cbh4 * 4); + for (int pl = 0; pl < 2; pl++) { + if (!b->cfl_alpha[pl]) continue; + int angle = 0; + const pixel *top_sb_edge = NULL; + if (!((t->by & ~ss_ver) & (f->sb_step - 1))) { + top_sb_edge = f->ipred_edge[pl + 1]; + const int sby = t->by >> f->sb_shift; + top_sb_edge += f->sb128w * 128 * (sby - 1); + } + const int xpos = t->bx >> ss_hor, ypos = t->by >> ss_ver; + const int xstart = ts->tiling.col_start >> ss_hor; + const int ystart = ts->tiling.row_start >> ss_ver; + const enum IntraPredMode m = + bytefn(dav1d_prepare_intra_edges)(xpos, xpos > xstart, + ypos, ypos > ystart, + ts->tiling.col_end >> ss_hor, + ts->tiling.row_end >> ss_ver, + 0, uv_dst[pl], stride, + top_sb_edge, DC_PRED, &angle, + uv_t_dim->w, uv_t_dim->h, 0, + edge HIGHBD_CALL_SUFFIX); + dsp->ipred.cfl_pred[m](uv_dst[pl], stride, edge, + uv_t_dim->w * 4, + uv_t_dim->h * 4, + ac, b->cfl_alpha[pl] + HIGHBD_CALL_SUFFIX); + } + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) { + ac_dump(ac, 4*cbw4, 4*cbh4, "ac"); + hex_dump(uv_dst[0], stride, cbw4 * 4, cbh4 * 4, "u-cfl-pred"); + hex_dump(uv_dst[1], stride, cbw4 * 4, cbh4 * 4, "v-cfl-pred"); + } + } else if (b->pal_sz[1]) { + const ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + + (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); + const uint16_t (*pal)[8]; + const uint8_t *pal_idx; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal = f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + + ((t->bx >> 1) + (t->by & 1))]; + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += cbw4 * cbh4 * 16; + } else { + pal = t->scratch.pal; + pal_idx = &t->scratch.pal_idx[bw4 * bh4 * 16]; + } + + f->dsp->ipred.pal_pred(((pixel *) f->cur.data[1]) + uv_dstoff, + f->cur.stride[1], pal[1], + pal_idx, cbw4 * 4, cbh4 * 4); + f->dsp->ipred.pal_pred(((pixel *) f->cur.data[2]) + uv_dstoff, + f->cur.stride[1], pal[2], + pal_idx, cbw4 * 4, cbh4 * 4); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) { + hex_dump(((pixel *) f->cur.data[1]) + uv_dstoff, + PXSTRIDE(f->cur.stride[1]), + cbw4 * 4, cbh4 * 4, "u-pal-pred"); + hex_dump(((pixel *) f->cur.data[2]) + uv_dstoff, + PXSTRIDE(f->cur.stride[1]), + cbw4 * 4, cbh4 * 4, "v-pal-pred"); + } + } + + const int sm_uv_fl = sm_uv_flag(t->a, cbx4) | + sm_uv_flag(&t->l, cby4); + const int uv_sb_has_tr = + ((init_x + 16) >> ss_hor) < cw4 ? 1 : init_y ? 0 : + intra_edge_flags & (EDGE_I420_TOP_HAS_RIGHT >> (f->cur.p.layout - 1)); + const int uv_sb_has_bl = + init_x ? 0 : ((init_y + 16) >> ss_ver) < ch4 ? 1 : + intra_edge_flags & (EDGE_I420_LEFT_HAS_BOTTOM >> (f->cur.p.layout - 1)); + const int sub_cw4 = imin(cw4, (init_x + 16) >> ss_hor); + for (int pl = 0; pl < 2; pl++) { + for (y = init_y >> ss_ver, t->by += init_y; y < sub_ch4; + y += uv_t_dim->h, t->by += uv_t_dim->h << ss_ver) + { + pixel *dst = ((pixel *) f->cur.data[1 + pl]) + + 4 * ((t->by >> ss_ver) * PXSTRIDE(stride) + + ((t->bx + init_x) >> ss_hor)); + for (x = init_x >> ss_hor, t->bx += init_x; x < sub_cw4; + x += uv_t_dim->w, t->bx += uv_t_dim->w << ss_hor) + { + if ((b->uv_mode == CFL_PRED && b->cfl_alpha[pl]) || + b->pal_sz[1]) + { + goto skip_uv_pred; + } + + int angle = b->uv_angle; + // this probably looks weird because we're using + // luma flags in a chroma loop, but that's because + // prepare_intra_edges() expects luma flags as input + const enum EdgeFlags edge_flags = + (((y > (init_y >> ss_ver) || !uv_sb_has_tr) && + (x + uv_t_dim->w >= sub_cw4)) ? + 0 : EDGE_I444_TOP_HAS_RIGHT) | + ((x > (init_x >> ss_hor) || + (!uv_sb_has_bl && y + uv_t_dim->h >= sub_ch4)) ? + 0 : EDGE_I444_LEFT_HAS_BOTTOM); + const pixel *top_sb_edge = NULL; + if (!((t->by & ~ss_ver) & (f->sb_step - 1))) { + top_sb_edge = f->ipred_edge[1 + pl]; + const int sby = t->by >> f->sb_shift; + top_sb_edge += f->sb128w * 128 * (sby - 1); + } + const enum IntraPredMode uv_mode = + b->uv_mode == CFL_PRED ? DC_PRED : b->uv_mode; + const int xpos = t->bx >> ss_hor, ypos = t->by >> ss_ver; + const int xstart = ts->tiling.col_start >> ss_hor; + const int ystart = ts->tiling.row_start >> ss_ver; + const enum IntraPredMode m = + bytefn(dav1d_prepare_intra_edges)(xpos, xpos > xstart, + ypos, ypos > ystart, + ts->tiling.col_end >> ss_hor, + ts->tiling.row_end >> ss_ver, + edge_flags, dst, stride, + top_sb_edge, uv_mode, + &angle, uv_t_dim->w, + uv_t_dim->h, + f->seq_hdr->intra_edge_filter, + edge HIGHBD_CALL_SUFFIX); + angle |= intra_edge_filter_flag; + dsp->ipred.intra_pred[m](dst, stride, edge, + uv_t_dim->w * 4, + uv_t_dim->h * 4, + angle | sm_uv_fl, + (4 * f->bw + ss_hor - + 4 * (t->bx & ~ss_hor)) >> ss_hor, + (4 * f->bh + ss_ver - + 4 * (t->by & ~ss_ver)) >> ss_ver + HIGHBD_CALL_SUFFIX); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) { + hex_dump(edge - uv_t_dim->h * 4, uv_t_dim->h * 4, + uv_t_dim->h * 4, 2, "l"); + hex_dump(edge, 0, 1, 1, "tl"); + hex_dump(edge + 1, uv_t_dim->w * 4, + uv_t_dim->w * 4, 2, "t"); + hex_dump(dst, stride, uv_t_dim->w * 4, + uv_t_dim->h * 4, pl ? "v-intra-pred" : "u-intra-pred"); + } + + skip_uv_pred: {} + if (!b->skip) { + enum TxfmType txtp; + int eob; + coef *cf; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += uv_t_dim->w * uv_t_dim->h * 16; + const struct CodedBlockInfo *const cbi = + &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; + eob = cbi->eob[pl + 1]; + txtp = cbi->txtp[pl + 1]; + } else { + uint8_t cf_ctx; + cf = bitfn(t->cf); + eob = decode_coefs(t, &t->a->ccoef[pl][cbx4 + x], + &t->l.ccoef[pl][cby4 + y], + b->uvtx, bs, b, 1, 1 + pl, cf, + &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-uv-cf-blk[pl=%d,tx=%d," + "txtp=%d,eob=%d]: r=%d [x=%d,cbx4=%d]\n", + pl, b->uvtx, txtp, eob, ts->msac.rng, x, cbx4); +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir ccoef[pl][off], cf_ctx, sz) + case_set_upto16_with_default( \ + imin(uv_t_dim->h, (f->bh - t->by + ss_ver) >> ss_ver), + l., 1, cby4 + y); + case_set_upto16_with_default( \ + imin(uv_t_dim->w, (f->bw - t->bx + ss_hor) >> ss_hor), + a->, 0, cbx4 + x); +#undef default_memset +#undef set_ctx + } + if (eob >= 0) { + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + coef_dump(cf, uv_t_dim->h * 4, + uv_t_dim->w * 4, 3, "dq"); + dsp->itx.itxfm_add[b->uvtx] + [txtp](dst, stride, + cf, eob HIGHBD_CALL_SUFFIX); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + hex_dump(dst, stride, uv_t_dim->w * 4, + uv_t_dim->h * 4, "recon"); + } + } else if (!t->frame_thread.pass) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[pl], off, mul * 0x40) + case_set_upto16(uv_t_dim->h, l., 1, cby4 + y); + case_set_upto16(uv_t_dim->w, a->, 0, cbx4 + x); +#undef set_ctx + } + dst += uv_t_dim->w * 4; + } + t->bx -= x << ss_hor; + } + t->by -= y << ss_ver; + } + } + } +} + +int bytefn(dav1d_recon_b_inter)(Dav1dTaskContext *const t, const enum BlockSize bs, + const Av1Block *const b) +{ + Dav1dTileState *const ts = t->ts; + const Dav1dFrameContext *const f = t->f; + const Dav1dDSPContext *const dsp = f->dsp; + const int bx4 = t->bx & 31, by4 = t->by & 31; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int cbx4 = bx4 >> ss_hor, cby4 = by4 >> ss_ver; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], bh4 = b_dim[1]; + const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 && + (bw4 > ss_hor || t->bx & 1) && + (bh4 > ss_ver || t->by & 1); + const int chr_layout_idx = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I400 ? 0 : + DAV1D_PIXEL_LAYOUT_I444 - f->cur.p.layout; + int res; + + // prediction + const int cbh4 = (bh4 + ss_ver) >> ss_ver, cbw4 = (bw4 + ss_hor) >> ss_hor; + pixel *dst = ((pixel *) f->cur.data[0]) + + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + t->bx); + const ptrdiff_t uvdstoff = + 4 * ((t->bx >> ss_hor) + (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); + if (IS_KEY_OR_INTRA(f->frame_hdr)) { + // intrabc + assert(!f->frame_hdr->super_res.enabled); + res = mc(t, dst, NULL, f->cur.stride[0], bw4, bh4, t->bx, t->by, 0, + b->mv[0], &f->sr_cur, 0 /* unused */, FILTER_2D_BILINEAR); + if (res) return res; + if (has_chroma) for (int pl = 1; pl < 3; pl++) { + res = mc(t, ((pixel *)f->cur.data[pl]) + uvdstoff, NULL, f->cur.stride[1], + bw4 << (bw4 == ss_hor), bh4 << (bh4 == ss_ver), + t->bx & ~ss_hor, t->by & ~ss_ver, pl, b->mv[0], + &f->sr_cur, 0 /* unused */, FILTER_2D_BILINEAR); + if (res) return res; + } + } else if (b->comp_type == COMP_INTER_NONE) { + const Dav1dThreadPicture *const refp = &f->refp[b->ref[0]]; + const enum Filter2d filter_2d = b->filter2d; + + if (imin(bw4, bh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + res = warp_affine(t, dst, NULL, f->cur.stride[0], b_dim, 0, refp, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + if (res) return res; + } else { + res = mc(t, dst, NULL, f->cur.stride[0], + bw4, bh4, t->bx, t->by, 0, b->mv[0], refp, b->ref[0], filter_2d); + if (res) return res; + if (b->motion_mode == MM_OBMC) { + res = obmc(t, dst, f->cur.stride[0], b_dim, 0, bx4, by4, w4, h4); + if (res) return res; + } + } + if (b->interintra_type) { + pixel *const tl_edge = bitfn(t->scratch.edge) + 32; + enum IntraPredMode m = b->interintra_mode == II_SMOOTH_PRED ? + SMOOTH_PRED : b->interintra_mode; + pixel *const tmp = bitfn(t->scratch.interintra); + int angle = 0; + const pixel *top_sb_edge = NULL; + if (!(t->by & (f->sb_step - 1))) { + top_sb_edge = f->ipred_edge[0]; + const int sby = t->by >> f->sb_shift; + top_sb_edge += f->sb128w * 128 * (sby - 1); + } + m = bytefn(dav1d_prepare_intra_edges)(t->bx, t->bx > ts->tiling.col_start, + t->by, t->by > ts->tiling.row_start, + ts->tiling.col_end, ts->tiling.row_end, + 0, dst, f->cur.stride[0], top_sb_edge, + m, &angle, bw4, bh4, 0, tl_edge + HIGHBD_CALL_SUFFIX); + dsp->ipred.intra_pred[m](tmp, 4 * bw4 * sizeof(pixel), + tl_edge, bw4 * 4, bh4 * 4, 0, 0, 0 + HIGHBD_CALL_SUFFIX); + const uint8_t *const ii_mask = + b->interintra_type == INTER_INTRA_BLEND ? + dav1d_ii_masks[bs][0][b->interintra_mode] : + dav1d_wedge_masks[bs][0][0][b->wedge_idx]; + dsp->mc.blend(dst, f->cur.stride[0], tmp, + bw4 * 4, bh4 * 4, ii_mask); + } + + if (!has_chroma) goto skip_inter_chroma_pred; + + // sub8x8 derivation + int is_sub8x8 = bw4 == ss_hor || bh4 == ss_ver; + refmvs_block *const *r; + if (is_sub8x8) { + assert(ss_hor == 1); + r = &t->rt.r[(t->by & 31) + 5]; + if (bw4 == 1) is_sub8x8 &= r[0][t->bx - 1].ref.ref[0] > 0; + if (bh4 == ss_ver) is_sub8x8 &= r[-1][t->bx].ref.ref[0] > 0; + if (bw4 == 1 && bh4 == ss_ver) + is_sub8x8 &= r[-1][t->bx - 1].ref.ref[0] > 0; + } + + // chroma prediction + if (is_sub8x8) { + assert(ss_hor == 1); + ptrdiff_t h_off = 0, v_off = 0; + if (bw4 == 1 && bh4 == ss_ver) { + for (int pl = 0; pl < 2; pl++) { + res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, + NULL, f->cur.stride[1], + bw4, bh4, t->bx - 1, t->by - 1, 1 + pl, + r[-1][t->bx - 1].mv.mv[0], + &f->refp[r[-1][t->bx - 1].ref.ref[0] - 1], + r[-1][t->bx - 1].ref.ref[0] - 1, + t->frame_thread.pass != 2 ? t->tl_4x4_filter : + f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx - 1].filter2d); + if (res) return res; + } + v_off = 2 * PXSTRIDE(f->cur.stride[1]); + h_off = 2; + } + if (bw4 == 1) { + const enum Filter2d left_filter_2d = + dav1d_filter_2d[t->l.filter[1][by4]][t->l.filter[0][by4]]; + for (int pl = 0; pl < 2; pl++) { + res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + v_off, NULL, + f->cur.stride[1], bw4, bh4, t->bx - 1, + t->by, 1 + pl, r[0][t->bx - 1].mv.mv[0], + &f->refp[r[0][t->bx - 1].ref.ref[0] - 1], + r[0][t->bx - 1].ref.ref[0] - 1, + t->frame_thread.pass != 2 ? left_filter_2d : + f->frame_thread.b[(t->by * f->b4_stride) + t->bx - 1].filter2d); + if (res) return res; + } + h_off = 2; + } + if (bh4 == ss_ver) { + const enum Filter2d top_filter_2d = + dav1d_filter_2d[t->a->filter[1][bx4]][t->a->filter[0][bx4]]; + for (int pl = 0; pl < 2; pl++) { + res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + h_off, NULL, + f->cur.stride[1], bw4, bh4, t->bx, t->by - 1, + 1 + pl, r[-1][t->bx].mv.mv[0], + &f->refp[r[-1][t->bx].ref.ref[0] - 1], + r[-1][t->bx].ref.ref[0] - 1, + t->frame_thread.pass != 2 ? top_filter_2d : + f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx].filter2d); + if (res) return res; + } + v_off = 2 * PXSTRIDE(f->cur.stride[1]); + } + for (int pl = 0; pl < 2; pl++) { + res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + h_off + v_off, NULL, f->cur.stride[1], + bw4, bh4, t->bx, t->by, 1 + pl, b->mv[0], + refp, b->ref[0], filter_2d); + if (res) return res; + } + } else { + if (imin(cbw4, cbh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + for (int pl = 0; pl < 2; pl++) { + res = warp_affine(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, NULL, + f->cur.stride[1], b_dim, 1 + pl, refp, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + if (res) return res; + } + } else { + for (int pl = 0; pl < 2; pl++) { + res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, + NULL, f->cur.stride[1], + bw4 << (bw4 == ss_hor), bh4 << (bh4 == ss_ver), + t->bx & ~ss_hor, t->by & ~ss_ver, + 1 + pl, b->mv[0], refp, b->ref[0], filter_2d); + if (res) return res; + if (b->motion_mode == MM_OBMC) { + res = obmc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, + f->cur.stride[1], b_dim, 1 + pl, bx4, by4, w4, h4); + if (res) return res; + } + } + } + if (b->interintra_type) { + // FIXME for 8x32 with 4:2:2 subsampling, this probably does + // the wrong thing since it will select 4x16, not 4x32, as a + // transform size... + const uint8_t *const ii_mask = + b->interintra_type == INTER_INTRA_BLEND ? + dav1d_ii_masks[bs][chr_layout_idx][b->interintra_mode] : + dav1d_wedge_masks[bs][chr_layout_idx][0][b->wedge_idx]; + + for (int pl = 0; pl < 2; pl++) { + pixel *const tmp = bitfn(t->scratch.interintra); + pixel *const tl_edge = bitfn(t->scratch.edge) + 32; + enum IntraPredMode m = + b->interintra_mode == II_SMOOTH_PRED ? + SMOOTH_PRED : b->interintra_mode; + int angle = 0; + pixel *const uvdst = ((pixel *) f->cur.data[1 + pl]) + uvdstoff; + const pixel *top_sb_edge = NULL; + if (!(t->by & (f->sb_step - 1))) { + top_sb_edge = f->ipred_edge[pl + 1]; + const int sby = t->by >> f->sb_shift; + top_sb_edge += f->sb128w * 128 * (sby - 1); + } + m = bytefn(dav1d_prepare_intra_edges)(t->bx >> ss_hor, + (t->bx >> ss_hor) > + (ts->tiling.col_start >> ss_hor), + t->by >> ss_ver, + (t->by >> ss_ver) > + (ts->tiling.row_start >> ss_ver), + ts->tiling.col_end >> ss_hor, + ts->tiling.row_end >> ss_ver, + 0, uvdst, f->cur.stride[1], + top_sb_edge, m, + &angle, cbw4, cbh4, 0, tl_edge + HIGHBD_CALL_SUFFIX); + dsp->ipred.intra_pred[m](tmp, cbw4 * 4 * sizeof(pixel), + tl_edge, cbw4 * 4, cbh4 * 4, 0, 0, 0 + HIGHBD_CALL_SUFFIX); + dsp->mc.blend(uvdst, f->cur.stride[1], tmp, + cbw4 * 4, cbh4 * 4, ii_mask); + } + } + } + + skip_inter_chroma_pred: {} + t->tl_4x4_filter = filter_2d; + } else { + const enum Filter2d filter_2d = b->filter2d; + // Maximum super block size is 128x128 + int16_t (*tmp)[128 * 128] = t->scratch.compinter; + int jnt_weight; + uint8_t *const seg_mask = t->scratch.seg_mask; + const uint8_t *mask; + + for (int i = 0; i < 2; i++) { + const Dav1dThreadPicture *const refp = &f->refp[b->ref[i]]; + + if (b->inter_mode == GLOBALMV_GLOBALMV && f->gmv_warp_allowed[b->ref[i]]) { + res = warp_affine(t, NULL, tmp[i], bw4 * 4, b_dim, 0, refp, + &f->frame_hdr->gmv[b->ref[i]]); + if (res) return res; + } else { + res = mc(t, NULL, tmp[i], 0, bw4, bh4, t->bx, t->by, 0, + b->mv[i], refp, b->ref[i], filter_2d); + if (res) return res; + } + } + switch (b->comp_type) { + case COMP_INTER_AVG: + dsp->mc.avg(dst, f->cur.stride[0], tmp[0], tmp[1], + bw4 * 4, bh4 * 4 HIGHBD_CALL_SUFFIX); + break; + case COMP_INTER_WEIGHTED_AVG: + jnt_weight = f->jnt_weights[b->ref[0]][b->ref[1]]; + dsp->mc.w_avg(dst, f->cur.stride[0], tmp[0], tmp[1], + bw4 * 4, bh4 * 4, jnt_weight HIGHBD_CALL_SUFFIX); + break; + case COMP_INTER_SEG: + dsp->mc.w_mask[chr_layout_idx](dst, f->cur.stride[0], + tmp[b->mask_sign], tmp[!b->mask_sign], + bw4 * 4, bh4 * 4, seg_mask, + b->mask_sign HIGHBD_CALL_SUFFIX); + mask = seg_mask; + break; + case COMP_INTER_WEDGE: + mask = dav1d_wedge_masks[bs][0][0][b->wedge_idx]; + dsp->mc.mask(dst, f->cur.stride[0], + tmp[b->mask_sign], tmp[!b->mask_sign], + bw4 * 4, bh4 * 4, mask HIGHBD_CALL_SUFFIX); + if (has_chroma) + mask = dav1d_wedge_masks[bs][chr_layout_idx][b->mask_sign][b->wedge_idx]; + break; + } + + // chroma + if (has_chroma) for (int pl = 0; pl < 2; pl++) { + for (int i = 0; i < 2; i++) { + const Dav1dThreadPicture *const refp = &f->refp[b->ref[i]]; + if (b->inter_mode == GLOBALMV_GLOBALMV && + imin(cbw4, cbh4) > 1 && f->gmv_warp_allowed[b->ref[i]]) + { + res = warp_affine(t, NULL, tmp[i], bw4 * 4 >> ss_hor, + b_dim, 1 + pl, + refp, &f->frame_hdr->gmv[b->ref[i]]); + if (res) return res; + } else { + res = mc(t, NULL, tmp[i], 0, bw4, bh4, t->bx, t->by, + 1 + pl, b->mv[i], refp, b->ref[i], filter_2d); + if (res) return res; + } + } + pixel *const uvdst = ((pixel *) f->cur.data[1 + pl]) + uvdstoff; + switch (b->comp_type) { + case COMP_INTER_AVG: + dsp->mc.avg(uvdst, f->cur.stride[1], tmp[0], tmp[1], + bw4 * 4 >> ss_hor, bh4 * 4 >> ss_ver + HIGHBD_CALL_SUFFIX); + break; + case COMP_INTER_WEIGHTED_AVG: + dsp->mc.w_avg(uvdst, f->cur.stride[1], tmp[0], tmp[1], + bw4 * 4 >> ss_hor, bh4 * 4 >> ss_ver, jnt_weight + HIGHBD_CALL_SUFFIX); + break; + case COMP_INTER_WEDGE: + case COMP_INTER_SEG: + dsp->mc.mask(uvdst, f->cur.stride[1], + tmp[b->mask_sign], tmp[!b->mask_sign], + bw4 * 4 >> ss_hor, bh4 * 4 >> ss_ver, mask + HIGHBD_CALL_SUFFIX); + break; + } + } + } + + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) { + hex_dump(dst, f->cur.stride[0], b_dim[0] * 4, b_dim[1] * 4, "y-pred"); + if (has_chroma) { + hex_dump(&((pixel *) f->cur.data[1])[uvdstoff], f->cur.stride[1], + cbw4 * 4, cbh4 * 4, "u-pred"); + hex_dump(&((pixel *) f->cur.data[2])[uvdstoff], f->cur.stride[1], + cbw4 * 4, cbh4 * 4, "v-pred"); + } + } + + const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; + + if (b->skip) { + // reset coef contexts +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir lcoef, off, mul * 0x40) + case_set(bh4, l., 1, by4); + case_set(bw4, a->, 0, bx4); +#undef set_ctx + if (has_chroma) { +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[0], off, mul * 0x40); \ + rep_macro(type, t->dir ccoef[1], off, mul * 0x40) + case_set(cbh4, l., 1, cby4); + case_set(cbw4, a->, 0, cbx4); +#undef set_ctx + } + return 0; + } + + const TxfmInfo *const uvtx = &dav1d_txfm_dimensions[b->uvtx]; + const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; + + for (int init_y = 0; init_y < bh4; init_y += 16) { + for (int init_x = 0; init_x < bw4; init_x += 16) { + // coefficient coding & inverse transforms + int y_off = !!init_y, y; + dst += PXSTRIDE(f->cur.stride[0]) * 4 * init_y; + for (y = init_y, t->by += init_y; y < imin(h4, init_y + 16); + y += ytx->h, y_off++) + { + int x, x_off = !!init_x; + for (x = init_x, t->bx += init_x; x < imin(w4, init_x + 16); + x += ytx->w, x_off++) + { + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, + x_off, y_off, &dst[x * 4]); + t->bx += ytx->w; + } + dst += PXSTRIDE(f->cur.stride[0]) * 4 * ytx->h; + t->bx -= x; + t->by += ytx->h; + } + dst -= PXSTRIDE(f->cur.stride[0]) * 4 * y; + t->by -= y; + + // chroma coefs and inverse transform + if (has_chroma) for (int pl = 0; pl < 2; pl++) { + pixel *uvdst = ((pixel *) f->cur.data[1 + pl]) + uvdstoff + + (PXSTRIDE(f->cur.stride[1]) * init_y * 4 >> ss_ver); + for (y = init_y >> ss_ver, t->by += init_y; + y < imin(ch4, (init_y + 16) >> ss_ver); y += uvtx->h) + { + int x; + for (x = init_x >> ss_hor, t->bx += init_x; + x < imin(cw4, (init_x + 16) >> ss_hor); x += uvtx->w) + { + coef *cf; + int eob; + enum TxfmType txtp; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += uvtx->w * uvtx->h * 16; + const struct CodedBlockInfo *const cbi = + &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; + eob = cbi->eob[1 + pl]; + txtp = cbi->txtp[1 + pl]; + } else { + uint8_t cf_ctx; + cf = bitfn(t->cf); + txtp = t->txtp_map[(by4 + (y << ss_ver)) * 32 + + bx4 + (x << ss_hor)]; + eob = decode_coefs(t, &t->a->ccoef[pl][cbx4 + x], + &t->l.ccoef[pl][cby4 + y], + b->uvtx, bs, b, 0, 1 + pl, + cf, &txtp, &cf_ctx); + if (DEBUG_BLOCK_INFO) + printf("Post-uv-cf-blk[pl=%d,tx=%d," + "txtp=%d,eob=%d]: r=%d\n", + pl, b->uvtx, txtp, eob, ts->msac.rng); +#define set_ctx(type, dir, diridx, off, mul, rep_macro) \ + rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) +#define default_memset(dir, diridx, off, sz) \ + memset(&t->dir ccoef[pl][off], cf_ctx, sz) + case_set_upto16_with_default( \ + imin(uvtx->h, (f->bh - t->by + ss_ver) >> ss_ver), + l., 1, cby4 + y); + case_set_upto16_with_default( \ + imin(uvtx->w, (f->bw - t->bx + ss_hor) >> ss_hor), + a->, 0, cbx4 + x); +#undef default_memset +#undef set_ctx + } + if (eob >= 0) { + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + coef_dump(cf, uvtx->h * 4, uvtx->w * 4, 3, "dq"); + dsp->itx.itxfm_add[b->uvtx] + [txtp](&uvdst[4 * x], + f->cur.stride[1], + cf, eob HIGHBD_CALL_SUFFIX); + if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) + hex_dump(&uvdst[4 * x], f->cur.stride[1], + uvtx->w * 4, uvtx->h * 4, "recon"); + } + t->bx += uvtx->w << ss_hor; + } + uvdst += PXSTRIDE(f->cur.stride[1]) * 4 * uvtx->h; + t->bx -= x << ss_hor; + t->by += uvtx->h << ss_ver; + } + t->by -= y << ss_ver; + } + } + } + return 0; +} + +void bytefn(dav1d_filter_sbrow_deblock_cols)(Dav1dFrameContext *const f, const int sby) { + if (!(f->c->inloop_filters & DAV1D_INLOOPFILTER_DEBLOCK) || + (!f->frame_hdr->loopfilter.level_y[0] && !f->frame_hdr->loopfilter.level_y[1])) + { + return; + } + const int y = sby * f->sb_step * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *const p[3] = { + f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), + f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) + }; + Av1Filter *mask = f->lf.mask + (sby >> !f->seq_hdr->sb128) * f->sb128w; + bytefn(dav1d_loopfilter_sbrow_cols)(f, p, mask, sby, + f->lf.start_of_tile_row[sby]); +} + +void bytefn(dav1d_filter_sbrow_deblock_rows)(Dav1dFrameContext *const f, const int sby) { + const int y = sby * f->sb_step * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *const p[3] = { + f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), + f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) + }; + Av1Filter *mask = f->lf.mask + (sby >> !f->seq_hdr->sb128) * f->sb128w; + if (f->c->inloop_filters & DAV1D_INLOOPFILTER_DEBLOCK && + (f->frame_hdr->loopfilter.level_y[0] || f->frame_hdr->loopfilter.level_y[1])) + { + bytefn(dav1d_loopfilter_sbrow_rows)(f, p, mask, sby); + } + if (f->seq_hdr->cdef || f->lf.restore_planes) { + // Store loop filtered pixels required by CDEF / LR + bytefn(dav1d_copy_lpf)(f, p, sby); + } +} + +void bytefn(dav1d_filter_sbrow_cdef)(Dav1dTaskContext *const tc, const int sby) { + const Dav1dFrameContext *const f = tc->f; + if (!(f->c->inloop_filters & DAV1D_INLOOPFILTER_CDEF)) return; + const int sbsz = f->sb_step; + const int y = sby * sbsz * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *const p[3] = { + f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), + f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) + }; + Av1Filter *prev_mask = f->lf.mask + ((sby - 1) >> !f->seq_hdr->sb128) * f->sb128w; + Av1Filter *mask = f->lf.mask + (sby >> !f->seq_hdr->sb128) * f->sb128w; + const int start = sby * sbsz; + if (sby) { + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *p_up[3] = { + p[0] - 8 * PXSTRIDE(f->cur.stride[0]), + p[1] - (8 * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + p[2] - (8 * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + }; + bytefn(dav1d_cdef_brow)(tc, p_up, prev_mask, start - 2, start, 1, sby); + } + const int n_blks = sbsz - 2 * (sby + 1 < f->sbh); + const int end = imin(start + n_blks, f->bh); + bytefn(dav1d_cdef_brow)(tc, p, mask, start, end, 0, sby); +} + +void bytefn(dav1d_filter_sbrow_resize)(Dav1dFrameContext *const f, const int sby) { + const int sbsz = f->sb_step; + const int y = sby * sbsz * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const pixel *const p[3] = { + f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), + f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) + }; + pixel *const sr_p[3] = { + f->lf.sr_p[0] + y * PXSTRIDE(f->sr_cur.p.stride[0]), + f->lf.sr_p[1] + (y * PXSTRIDE(f->sr_cur.p.stride[1]) >> ss_ver), + f->lf.sr_p[2] + (y * PXSTRIDE(f->sr_cur.p.stride[1]) >> ss_ver) + }; + const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400; + for (int pl = 0; pl < 1 + 2 * has_chroma; pl++) { + const int ss_ver = pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int h_start = 8 * !!sby >> ss_ver; + const ptrdiff_t dst_stride = f->sr_cur.p.stride[!!pl]; + pixel *dst = sr_p[pl] - h_start * PXSTRIDE(dst_stride); + const ptrdiff_t src_stride = f->cur.stride[!!pl]; + const pixel *src = p[pl] - h_start * PXSTRIDE(src_stride); + const int h_end = 4 * (sbsz - 2 * (sby + 1 < f->sbh)) >> ss_ver; + const int ss_hor = pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int dst_w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; + const int src_w = (4 * f->bw + ss_hor) >> ss_hor; + const int img_h = (f->cur.p.h - sbsz * 4 * sby + ss_ver) >> ss_ver; + + f->dsp->mc.resize(dst, dst_stride, src, src_stride, dst_w, + imin(img_h, h_end) + h_start, src_w, + f->resize_step[!!pl], f->resize_start[!!pl] + HIGHBD_CALL_SUFFIX); + } +} + +void bytefn(dav1d_filter_sbrow_lr)(Dav1dFrameContext *const f, const int sby) { + if (!(f->c->inloop_filters & DAV1D_INLOOPFILTER_RESTORATION)) return; + const int y = sby * f->sb_step * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *const sr_p[3] = { + f->lf.sr_p[0] + y * PXSTRIDE(f->sr_cur.p.stride[0]), + f->lf.sr_p[1] + (y * PXSTRIDE(f->sr_cur.p.stride[1]) >> ss_ver), + f->lf.sr_p[2] + (y * PXSTRIDE(f->sr_cur.p.stride[1]) >> ss_ver) + }; + bytefn(dav1d_lr_sbrow)(f, sr_p, sby); +} + +void bytefn(dav1d_filter_sbrow)(Dav1dFrameContext *const f, const int sby) { + bytefn(dav1d_filter_sbrow_deblock_cols)(f, sby); + bytefn(dav1d_filter_sbrow_deblock_rows)(f, sby); + if (f->seq_hdr->cdef) + bytefn(dav1d_filter_sbrow_cdef)(f->c->tc, sby); + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) + bytefn(dav1d_filter_sbrow_resize)(f, sby); + if (f->lf.restore_planes) + bytefn(dav1d_filter_sbrow_lr)(f, sby); +} + +void bytefn(dav1d_backup_ipred_edge)(Dav1dTaskContext *const t) { + const Dav1dFrameContext *const f = t->f; + Dav1dTileState *const ts = t->ts; + const int sby = t->by >> f->sb_shift; + const int sby_off = f->sb128w * 128 * sby; + const int x_off = ts->tiling.col_start; + + const pixel *const y = + ((const pixel *) f->cur.data[0]) + x_off * 4 + + ((t->by + f->sb_step) * 4 - 1) * PXSTRIDE(f->cur.stride[0]); + pixel_copy(&f->ipred_edge[0][sby_off + x_off * 4], y, + 4 * (ts->tiling.col_end - x_off)); + + if (f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400) { + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + + const ptrdiff_t uv_off = (x_off * 4 >> ss_hor) + + (((t->by + f->sb_step) * 4 >> ss_ver) - 1) * PXSTRIDE(f->cur.stride[1]); + for (int pl = 1; pl <= 2; pl++) + pixel_copy(&f->ipred_edge[pl][sby_off + (x_off * 4 >> ss_hor)], + &((const pixel *) f->cur.data[pl])[uv_off], + 4 * (ts->tiling.col_end - x_off) >> ss_hor); + } +} diff --git a/third_party/dav1d/src/ref.c b/third_party/dav1d/src/ref.c new file mode 100644 index 0000000000..46462b4c80 --- /dev/null +++ b/third_party/dav1d/src/ref.c @@ -0,0 +1,107 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "src/ref.h" + +static void default_free_callback(const uint8_t *const data, void *const user_data) { + assert(data == user_data); + dav1d_free_aligned(user_data); +} + +Dav1dRef *dav1d_ref_create(size_t size) { + size = (size + sizeof(void*) - 1) & ~(sizeof(void*) - 1); + + uint8_t *const data = dav1d_alloc_aligned(size + sizeof(Dav1dRef), 64); + if (!data) return NULL; + + Dav1dRef *const res = (Dav1dRef*)(data + size); + res->const_data = res->user_data = res->data = data; + atomic_init(&res->ref_cnt, 1); + res->free_ref = 0; + res->free_callback = default_free_callback; + + return res; +} + +static void pool_free_callback(const uint8_t *const data, void *const user_data) { + dav1d_mem_pool_push((Dav1dMemPool*)data, user_data); +} + +Dav1dRef *dav1d_ref_create_using_pool(Dav1dMemPool *const pool, size_t size) { + size = (size + sizeof(void*) - 1) & ~(sizeof(void*) - 1); + + Dav1dMemPoolBuffer *const buf = + dav1d_mem_pool_pop(pool, size + sizeof(Dav1dRef)); + if (!buf) return NULL; + + Dav1dRef *const res = &((Dav1dRef*)buf)[-1]; + res->data = buf->data; + res->const_data = pool; + atomic_init(&res->ref_cnt, 1); + res->free_ref = 0; + res->free_callback = pool_free_callback; + res->user_data = buf; + + return res; +} + +Dav1dRef *dav1d_ref_wrap(const uint8_t *const ptr, + void (*free_callback)(const uint8_t *data, void *user_data), + void *const user_data) +{ + Dav1dRef *res = malloc(sizeof(Dav1dRef)); + if (!res) return NULL; + + res->data = NULL; + res->const_data = ptr; + atomic_init(&res->ref_cnt, 1); + res->free_ref = 1; + res->free_callback = free_callback; + res->user_data = user_data; + + return res; +} + +void dav1d_ref_dec(Dav1dRef **const pref) { + assert(pref != NULL); + + Dav1dRef *const ref = *pref; + if (!ref) return; + + *pref = NULL; + if (atomic_fetch_sub(&ref->ref_cnt, 1) == 1) { + const int free_ref = ref->free_ref; + ref->free_callback(ref->const_data, ref->user_data); + if (free_ref) free(ref); + } +} + +int dav1d_ref_is_writable(Dav1dRef *const ref) { + return atomic_load(&ref->ref_cnt) == 1 && ref->data; +} diff --git a/third_party/dav1d/src/ref.h b/third_party/dav1d/src/ref.h new file mode 100644 index 0000000000..ec070a0a9a --- /dev/null +++ b/third_party/dav1d/src/ref.h @@ -0,0 +1,60 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_REF_H +#define DAV1D_SRC_REF_H + +#include "dav1d/dav1d.h" + +#include "src/mem.h" +#include "src/thread.h" + +#include +#include + +struct Dav1dRef { + void *data; + const void *const_data; + atomic_int ref_cnt; + int free_ref; + void (*free_callback)(const uint8_t *data, void *user_data); + void *user_data; +}; + +Dav1dRef *dav1d_ref_create(size_t size); +Dav1dRef *dav1d_ref_create_using_pool(Dav1dMemPool *pool, size_t size); +Dav1dRef *dav1d_ref_wrap(const uint8_t *ptr, + void (*free_callback)(const uint8_t *data, void *user_data), + void *user_data); +void dav1d_ref_dec(Dav1dRef **ref); +int dav1d_ref_is_writable(Dav1dRef *ref); + +static inline void dav1d_ref_inc(Dav1dRef *const ref) { + atomic_fetch_add_explicit(&ref->ref_cnt, 1, memory_order_relaxed); +} + +#endif /* DAV1D_SRC_REF_H */ diff --git a/third_party/dav1d/src/refmvs.c b/third_party/dav1d/src/refmvs.c new file mode 100644 index 0000000000..5398d396d1 --- /dev/null +++ b/third_party/dav1d/src/refmvs.c @@ -0,0 +1,940 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "dav1d/common.h" + +#include "common/intops.h" + +#include "src/env.h" +#include "src/mem.h" +#include "src/refmvs.h" + +static void add_spatial_candidate(refmvs_candidate *const mvstack, int *const cnt, + const int weight, const refmvs_block *const b, + const union refmvs_refpair ref, const mv gmv[2], + int *const have_newmv_match, + int *const have_refmv_match) +{ + if (b->mv.mv[0].n == INVALID_MV) return; // intra block, no intrabc + + if (ref.ref[1] == -1) { + for (int n = 0; n < 2; n++) { + if (b->ref.ref[n] == ref.ref[0]) { + const mv cand_mv = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? + gmv[0] : b->mv.mv[n]; + + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + + const int last = *cnt; + for (int m = 0; m < last; m++) + if (mvstack[m].mv.mv[0].n == cand_mv.n) { + mvstack[m].weight += weight; + return; + } + + if (last < 8) { + mvstack[last].mv.mv[0] = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + return; + } + } + } else if (b->ref.pair == ref.pair) { + const refmvs_mvpair cand_mv = { .mv = { + [0] = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? gmv[0] : b->mv.mv[0], + [1] = ((b->mf & 1) && gmv[1].n != INVALID_MV) ? gmv[1] : b->mv.mv[1], + }}; + + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + + const int last = *cnt; + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == cand_mv.n) { + mvstack[n].weight += weight; + return; + } + + if (last < 8) { + mvstack[last].mv = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + } +} + +static int scan_row(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + const refmvs_block *b, const int bw4, const int w4, + const int max_rows, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = b; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bw4 = first_cand_b_dim[0]; + int len = imax(step, imin(bw4, cand_bw4)); + + if (bw4 <= cand_bw4) { + // FIXME weight can be higher for odd blocks (bx4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for row_offset=-3/-5 + // FIXME why can this not be cand_bw4? + const int weight = bw4 == 1 ? 2 : + imax(2, imin(2 * max_rows, first_cand_b_dim[1])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int x = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + x += len; + if (x >= w4) return 1; + cand_b = &b[x]; + cand_bw4 = dav1d_block_dimensions[cand_b->bs][0]; + assert(cand_bw4 < bw4); + len = imax(step, cand_bw4); + } +} + +static int scan_col(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + /*const*/ refmvs_block *const *b, const int bh4, const int h4, + const int bx4, const int max_cols, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = &b[0][bx4]; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bh4 = first_cand_b_dim[1]; + int len = imax(step, imin(bh4, cand_bh4)); + + if (bh4 <= cand_bh4) { + // FIXME weight can be higher for odd blocks (by4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for col_offset=-3/-5 + // FIXME why can this not be cand_bh4? + const int weight = bh4 == 1 ? 2 : + imax(2, imin(2 * max_cols, first_cand_b_dim[0])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int y = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + y += len; + if (y >= h4) return 1; + cand_b = &b[y][bx4]; + cand_bh4 = dav1d_block_dimensions[cand_b->bs][1]; + assert(cand_bh4 < bh4); + len = imax(step, cand_bh4); + } +} + +static inline union mv mv_projection(const union mv mv, const int num, const int den) { + static const uint16_t div_mult[32] = { + 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, + 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, + 1024, 963, 910, 862, 819, 780, 744, 712, + 682, 655, 630, 606, 585, 564, 546, 528 + }; + assert(den > 0 && den < 32); + assert(num > -32 && num < 32); + const int frac = num * div_mult[den]; + const int y = mv.y * frac, x = mv.x * frac; + // Round and clip according to AV1 spec section 7.9.3 + return (union mv) { // 0x3fff == (1 << 14) - 1 + .y = iclip((y + 8192 + (y >> 31)) >> 14, -0x3fff, 0x3fff), + .x = iclip((x + 8192 + (x >> 31)) >> 14, -0x3fff, 0x3fff) + }; +} + +static void add_temporal_candidate(const refmvs_frame *const rf, + refmvs_candidate *const mvstack, int *const cnt, + const refmvs_temporal_block *const rb, + const union refmvs_refpair ref, int *const globalmv_ctx, + const union mv gmv[]) +{ + if (rb->mv.n == INVALID_MV) return; + + union mv mv = mv_projection(rb->mv, rf->pocdiff[ref.ref[0] - 1], rb->ref); + fix_mv_precision(rf->frm_hdr, &mv); + + const int last = *cnt; + if (ref.ref[1] == -1) { + if (globalmv_ctx) + *globalmv_ctx = (abs(mv.x - gmv[0].x) | abs(mv.y - gmv[0].y)) >= 16; + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.mv[0].n == mv.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv.mv[0] = mv; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } else { + refmvs_mvpair mvp = { .mv = { + [0] = mv, + [1] = mv_projection(rb->mv, rf->pocdiff[ref.ref[1] - 1], rb->ref), + }}; + fix_mv_precision(rf->frm_hdr, &mvp.mv[1]); + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == mvp.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv = mvp; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } +} + +static void add_compound_extended_candidate(refmvs_candidate *const same, + int *const same_count, + const refmvs_block *const cand_b, + const int sign0, const int sign1, + const union refmvs_refpair ref, + const uint8_t *const sign_bias) +{ + refmvs_candidate *const diff = &same[2]; + int *const diff_count = &same_count[2]; + + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + + mv cand_mv = cand_b->mv.mv[n]; + if (cand_ref == ref.ref[0]) { + if (same_count[0] < 2) + same[same_count[0]++].mv.mv[0] = cand_mv; + if (diff_count[1] < 2) { + if (sign1 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[1]++].mv.mv[1] = cand_mv; + } + } else if (cand_ref == ref.ref[1]) { + if (same_count[1] < 2) + same[same_count[1]++].mv.mv[1] = cand_mv; + if (diff_count[0] < 2) { + if (sign0 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[0]++].mv.mv[0] = cand_mv; + } + } else { + mv i_cand_mv = (union mv) { + .x = -cand_mv.x, + .y = -cand_mv.y + }; + + if (diff_count[0] < 2) { + diff[diff_count[0]++].mv.mv[0] = + sign0 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + + if (diff_count[1] < 2) { + diff[diff_count[1]++].mv.mv[1] = + sign1 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + } + } +} + +static void add_single_extended_candidate(refmvs_candidate mvstack[8], int *const cnt, + const refmvs_block *const cand_b, + const int sign, const uint8_t *const sign_bias) +{ + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + // we need to continue even if cand_ref == ref.ref[0], since + // the candidate could have been added as a globalmv variant, + // which changes the value + // FIXME if scan_{row,col}() returned a mask for the nearest + // edge, we could skip the appropriate ones here + + mv cand_mv = cand_b->mv.mv[n]; + if (sign ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + + int m; + const int last = *cnt; + for (m = 0; m < last; m++) + if (cand_mv.n == mvstack[m].mv.mv[0].n) + break; + if (m == last) { + mvstack[m].mv.mv[0] = cand_mv; + mvstack[m].weight = 2; // "minimal" + *cnt = last + 1; + } + } +} + +/* + * refmvs_frame allocates memory for one sbrow (32 blocks high, whole frame + * wide) of 4x4-resolution refmvs_block entries for spatial MV referencing. + * mvrefs_tile[] keeps a list of 35 (32 + 3 above) pointers into this memory, + * and each sbrow, the bottom entries (y=27/29/31) are exchanged with the top + * (-5/-3/-1) pointers by calling dav1d_refmvs_tile_sbrow_init() at the start + * of each tile/sbrow. + * + * For temporal MV referencing, we call dav1d_refmvs_save_tmvs() at the end of + * each tile/sbrow (when tile column threading is enabled), or at the start of + * each interleaved sbrow (i.e. once for all tile columns together, when tile + * column threading is disabled). This will copy the 4x4-resolution spatial MVs + * into 8x8-resolution refmvs_temporal_block structures. Then, for subsequent + * frames, at the start of each tile/sbrow (when tile column threading is + * enabled) or at the start of each interleaved sbrow (when tile column + * threading is disabled), we call load_tmvs(), which will project the MVs to + * their respective position in the current frame. + */ + +void dav1d_refmvs_find(const refmvs_tile *const rt, + refmvs_candidate mvstack[8], int *const cnt, + int *const ctx, + const union refmvs_refpair ref, const enum BlockSize bs, + const enum EdgeFlags edge_flags, + const int by4, const int bx4) +{ + const refmvs_frame *const rf = rt->rf; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], w4 = imin(imin(bw4, 16), rt->tile_col.end - bx4); + const int bh4 = b_dim[1], h4 = imin(imin(bh4, 16), rt->tile_row.end - by4); + mv gmv[2], tgmv[2]; + + *cnt = 0; + assert(ref.ref[0] >= 0 && ref.ref[0] <= 8 && + ref.ref[1] >= -1 && ref.ref[1] <= 8); + if (ref.ref[0] > 0) { + tgmv[0] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[0] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[0] = rf->frm_hdr->gmv[ref.ref[0] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[0] : (mv) { .n = INVALID_MV }; + } else { + tgmv[0] = (mv) { .n = 0 }; + gmv[0] = (mv) { .n = INVALID_MV }; + } + if (ref.ref[1] > 0) { + tgmv[1] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[1] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[1] = rf->frm_hdr->gmv[ref.ref[1] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[1] : (mv) { .n = INVALID_MV }; + } + + // top + int have_newmv = 0, have_col_mvs = 0, have_row_mvs = 0; + unsigned max_rows = 0, n_rows = ~0; + const refmvs_block *b_top; + if (by4 > rt->tile_row.start) { + max_rows = imin((by4 - rt->tile_row.start + 1) >> 1, 2 + (bh4 > 1)); + b_top = &rt->r[(by4 & 31) - 1 + 5][bx4]; + n_rows = scan_row(mvstack, cnt, ref, gmv, b_top, + bw4, w4, max_rows, bw4 >= 16 ? 4 : 1, + &have_newmv, &have_row_mvs); + } + + // left + unsigned max_cols = 0, n_cols = ~0U; + refmvs_block *const *b_left; + if (bx4 > rt->tile_col.start) { + max_cols = imin((bx4 - rt->tile_col.start + 1) >> 1, 2 + (bw4 > 1)); + b_left = &rt->r[(by4 & 31) + 5]; + n_cols = scan_col(mvstack, cnt, ref, gmv, b_left, + bh4, h4, bx4 - 1, max_cols, bh4 >= 16 ? 4 : 1, + &have_newmv, &have_col_mvs); + } + + // top/right + if (n_rows != ~0U && edge_flags & EDGE_I444_TOP_HAS_RIGHT && + imax(bw4, bh4) <= 16 && bw4 + bx4 < rt->tile_col.end) + { + add_spatial_candidate(mvstack, cnt, 4, &b_top[bw4], ref, gmv, + &have_newmv, &have_row_mvs); + } + + const int nearest_match = have_col_mvs + have_row_mvs; + const int nearest_cnt = *cnt; + for (int n = 0; n < nearest_cnt; n++) + mvstack[n].weight += 640; + + // temporal + int globalmv_ctx = rf->frm_hdr->use_ref_frame_mvs; + if (rf->use_ref_frame_mvs) { + const ptrdiff_t stride = rf->rp_stride; + const int by8 = by4 >> 1, bx8 = bx4 >> 1; + const refmvs_temporal_block *const rbi = &rt->rp_proj[(by8 & 15) * stride + bx8]; + const refmvs_temporal_block *rb = rbi; + const int step_h = bw4 >= 16 ? 2 : 1, step_v = bh4 >= 16 ? 2 : 1; + const int w8 = imin((w4 + 1) >> 1, 8), h8 = imin((h4 + 1) >> 1, 8); + for (int y = 0; y < h8; y += step_v) { + for (int x = 0; x < w8; x+= step_h) { + add_temporal_candidate(rf, mvstack, cnt, &rb[x], ref, + !(x | y) ? &globalmv_ctx : NULL, tgmv); + } + rb += stride * step_v; + } + if (imin(bw4, bh4) >= 2 && imax(bw4, bh4) < 16) { + const int bh8 = bh4 >> 1, bw8 = bw4 >> 1; + rb = &rbi[bh8 * stride]; + const int has_bottom = by8 + bh8 < imin(rt->tile_row.end >> 1, + (by8 & ~7) + 8); + if (has_bottom && bx8 - 1 >= imax(rt->tile_col.start >> 1, bx8 & ~7)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[-1], ref, + NULL, NULL); + } + if (bx8 + bw8 < imin(rt->tile_col.end >> 1, (bx8 & ~7) + 8)) { + if (has_bottom) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8], ref, + NULL, NULL); + } + if (by8 + bh8 - 1 < imin(rt->tile_row.end >> 1, (by8 & ~7) + 8)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8 - stride], + ref, NULL, NULL); + } + } + } + } + assert(*cnt <= 8); + + // top/left (which, confusingly, is part of "secondary" references) + int have_dummy_newmv_match; + if ((n_rows | n_cols) != ~0U) { + add_spatial_candidate(mvstack, cnt, 4, &b_top[-1], ref, gmv, + &have_dummy_newmv_match, &have_row_mvs); + } + + // "secondary" (non-direct neighbour) top & left edges + // what is different about secondary is that everything is now in 8x8 resolution + for (int n = 2; n <= 3; n++) { + if ((unsigned) n > n_rows && (unsigned) n <= max_rows) { + n_rows += scan_row(mvstack, cnt, ref, gmv, + &rt->r[(((by4 & 31) - 2 * n + 1) | 1) + 5][bx4 | 1], + bw4, w4, 1 + max_rows - n, bw4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_row_mvs); + } + + if ((unsigned) n > n_cols && (unsigned) n <= max_cols) { + n_cols += scan_col(mvstack, cnt, ref, gmv, &rt->r[((by4 & 31) | 1) + 5], + bh4, h4, (bx4 - n * 2 + 1) | 1, + 1 + max_cols - n, bh4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_col_mvs); + } + } + assert(*cnt <= 8); + + const int ref_match_count = have_col_mvs + have_row_mvs; + + // context build-up + int refmv_ctx, newmv_ctx; + switch (nearest_match) { + case 0: + refmv_ctx = imin(2, ref_match_count); + newmv_ctx = ref_match_count > 0; + break; + case 1: + refmv_ctx = imin(ref_match_count * 3, 4); + newmv_ctx = 3 - have_newmv; + break; + case 2: + refmv_ctx = 5; + newmv_ctx = 5 - have_newmv; + break; + } + + // sorting (nearest, then "secondary") + int len = nearest_cnt; + while (len) { + int last = 0; + for (int n = 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { +#define EXCHANGE(a, b) do { refmvs_candidate tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(mvstack[n - 1], mvstack[n]); + last = n; + } + } + len = last; + } + len = *cnt; + while (len > nearest_cnt) { + int last = nearest_cnt; + for (int n = nearest_cnt + 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { + EXCHANGE(mvstack[n - 1], mvstack[n]); +#undef EXCHANGE + last = n; + } + } + len = last; + } + + if (ref.ref[1] > 0) { + if (*cnt < 2) { + const int sign0 = rf->sign_bias[ref.ref[0] - 1]; + const int sign1 = rf->sign_bias[ref.ref[1] - 1]; + const int sz4 = imin(w4, h4); + refmvs_candidate *const same = &mvstack[*cnt]; + int same_count[4] = { 0 }; + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4;) { + const refmvs_block *const cand_b = &b_top[x]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + + refmvs_candidate *const diff = &same[2]; + const int *const diff_count = &same_count[2]; + + // merge together + for (int n = 0; n < 2; n++) { + int m = same_count[n]; + + if (m >= 2) continue; + + const int l = diff_count[n]; + if (l) { + same[m].mv.mv[n] = diff[0].mv.mv[n]; + if (++m == 2) continue; + if (l == 2) { + same[1].mv.mv[n] = diff[1].mv.mv[n]; + continue; + } + } + do { + same[m].mv.mv[n] = tgmv[n]; + } while (++m < 2); + } + + // if the first extended was the same as the non-extended one, + // then replace it with the second extended one + int n = *cnt; + if (n == 1 && mvstack[0].mv.n == same[0].mv.n) + mvstack[1].mv = mvstack[2].mv; + do { + mvstack[n].weight = 2; + } while (++n < 2); + *cnt = 2; + } + + // clamping + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + const int n_refmvs = *cnt; + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + mvstack[n].mv.mv[1].x = iclip(mvstack[n].mv.mv[1].x, left, right); + mvstack[n].mv.mv[1].y = iclip(mvstack[n].mv.mv[1].y, top, bottom); + } while (++n < n_refmvs); + + switch (refmv_ctx >> 1) { + case 0: + *ctx = imin(newmv_ctx, 1); + break; + case 1: + *ctx = 1 + imin(newmv_ctx, 3); + break; + case 2: + *ctx = iclip(3 + newmv_ctx, 4, 7); + break; + } + + return; + } else if (*cnt < 2 && ref.ref[0] > 0) { + const int sign = rf->sign_bias[ref.ref[0] - 1]; + const int sz4 = imin(w4, h4); + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_top[x]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + } + assert(*cnt <= 8); + + // clamping + int n_refmvs = *cnt; + if (n_refmvs) { + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + } while (++n < n_refmvs); + } + + for (int n = *cnt; n < 2; n++) + mvstack[n].mv.mv[0] = tgmv[0]; + + *ctx = (refmv_ctx << 4) | (globalmv_ctx << 3) | newmv_ctx; +} + +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *const rt, const refmvs_frame *const rf, + const int tile_col_start4, const int tile_col_end4, + const int tile_row_start4, const int tile_row_end4, + const int sby, int tile_row_idx, const int pass) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + rt->rp_proj = &rf->rp_proj[16 * rf->rp_stride * tile_row_idx]; + const int uses_2pass = rf->n_tile_threads > 1 && rf->n_frame_threads > 1; + const ptrdiff_t pass_off = (uses_2pass && pass == 2) ? + 35 * rf->r_stride * rf->n_tile_rows : 0; + refmvs_block *r = &rf->r[35 * rf->r_stride * tile_row_idx + pass_off]; + const int sbsz = rf->sbsz; + const int off = (sbsz * sby) & 16; + for (int i = 0; i < sbsz; i++, r += rf->r_stride) + rt->r[off + 5 + i] = r; + rt->r[off + 0] = r; + r += rf->r_stride; + rt->r[off + 1] = NULL; + rt->r[off + 2] = r; + r += rf->r_stride; + rt->r[off + 3] = NULL; + rt->r[off + 4] = r; + if (sby & 1) { +#define EXCHANGE(a, b) do { void *const tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(rt->r[off + 0], rt->r[off + sbsz + 0]); + EXCHANGE(rt->r[off + 2], rt->r[off + sbsz + 2]); + EXCHANGE(rt->r[off + 4], rt->r[off + sbsz + 4]); +#undef EXCHANGE + } + + rt->rf = rf; + rt->tile_row.start = tile_row_start4; + rt->tile_row.end = imin(tile_row_end4, rf->ih4); + rt->tile_col.start = tile_col_start4; + rt->tile_col.end = imin(tile_col_end4, rf->iw4); +} + +static void load_tmvs_c(const refmvs_frame *const rf, int tile_row_idx, + const int col_start8, const int col_end8, + const int row_start8, int row_end8) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + const int col_start8i = imax(col_start8 - 8, 0); + const int col_end8i = imin(col_end8 + 8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + refmvs_temporal_block *rp_proj = + &rf->rp_proj[16 * stride * tile_row_idx + (row_start8 & 15) * stride]; + for (int y = row_start8; y < row_end8; y++) { + for (int x = col_start8; x < col_end8; x++) + rp_proj[x].mv.n = INVALID_MV; + rp_proj += stride; + } + + rp_proj = &rf->rp_proj[16 * stride * tile_row_idx]; + for (int n = 0; n < rf->n_mfmvs; n++) { + const int ref2cur = rf->mfmv_ref2cur[n]; + if (ref2cur == INT_MIN) continue; + + const int ref = rf->mfmv_ref[n]; + const int ref_sign = ref - 4; + const refmvs_temporal_block *r = &rf->rp_ref[ref][row_start8 * stride]; + for (int y = row_start8; y < row_end8; y++) { + const int y_sb_align = y & ~7; + const int y_proj_start = imax(y_sb_align, row_start8); + const int y_proj_end = imin(y_sb_align + 8, row_end8); + for (int x = col_start8i; x < col_end8i; x++) { + const refmvs_temporal_block *rb = &r[x]; + const int b_ref = rb->ref; + if (!b_ref) continue; + const int ref2ref = rf->mfmv_ref2ref[n][b_ref - 1]; + if (!ref2ref) continue; + const mv b_mv = rb->mv; + const mv offset = mv_projection(b_mv, ref2cur, ref2ref); + int pos_x = x + apply_sign(abs(offset.x) >> 6, + offset.x ^ ref_sign); + const int pos_y = y + apply_sign(abs(offset.y) >> 6, + offset.y ^ ref_sign); + if (pos_y >= y_proj_start && pos_y < y_proj_end) { + const ptrdiff_t pos = (pos_y & 15) * stride; + for (;;) { + const int x_sb_align = x & ~7; + if (pos_x >= imax(x_sb_align - 8, col_start8) && + pos_x < imin(x_sb_align + 16, col_end8)) + { + rp_proj[pos + pos_x].mv = rb->mv; + rp_proj[pos + pos_x].ref = ref2ref; + } + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + pos_x++; + } + } else { + for (;;) { + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + } + } + x--; + } + r += stride; + } + } +} + +static void save_tmvs_c(refmvs_temporal_block *rp, const ptrdiff_t stride, + refmvs_block *const *const rr, + const uint8_t *const ref_sign, + const int col_end8, const int row_end8, + const int col_start8, const int row_start8) +{ + for (int y = row_start8; y < row_end8; y++) { + const refmvs_block *const b = rr[(y & 15) * 2]; + + for (int x = col_start8; x < col_end8;) { + const refmvs_block *const cand_b = &b[x * 2 + 1]; + const int bw8 = (dav1d_block_dimensions[cand_b->bs][0] + 1) >> 1; + + if (cand_b->ref.ref[1] > 0 && ref_sign[cand_b->ref.ref[1] - 1] && + (abs(cand_b->mv.mv[1].y) | abs(cand_b->mv.mv[1].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[1], + .ref = cand_b->ref.ref[1] }; + } else if (cand_b->ref.ref[0] > 0 && ref_sign[cand_b->ref.ref[0] - 1] && + (abs(cand_b->mv.mv[0].y) | abs(cand_b->mv.mv[0].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[0], + .ref = cand_b->ref.ref[0] }; + } else { + for (int n = 0; n < bw8; n++, x++) { + rp[x].mv.n = 0; + rp[x].ref = 0; // "invalid" + } + } + } + rp += stride; + } +} + +int dav1d_refmvs_init_frame(refmvs_frame *const rf, + const Dav1dSequenceHeader *const seq_hdr, + const Dav1dFrameHeader *const frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *const rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + const int n_tile_threads, const int n_frame_threads) +{ + rf->sbsz = 16 << seq_hdr->sb128; + rf->frm_hdr = frm_hdr; + rf->iw8 = (frm_hdr->width[0] + 7) >> 3; + rf->ih8 = (frm_hdr->height + 7) >> 3; + rf->iw4 = rf->iw8 << 1; + rf->ih4 = rf->ih8 << 1; + + const ptrdiff_t r_stride = ((frm_hdr->width[0] + 127) & ~127) >> 2; + const int n_tile_rows = n_tile_threads > 1 ? frm_hdr->tiling.rows : 1; + if (r_stride != rf->r_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->r) dav1d_freep_aligned(&rf->r); + const int uses_2pass = n_tile_threads > 1 && n_frame_threads > 1; + rf->r = dav1d_alloc_aligned(sizeof(*rf->r) * 35 * r_stride * n_tile_rows * (1 + uses_2pass), 64); + if (!rf->r) return DAV1D_ERR(ENOMEM); + rf->r_stride = r_stride; + } + + const ptrdiff_t rp_stride = r_stride >> 1; + if (rp_stride != rf->rp_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->rp_proj) dav1d_freep_aligned(&rf->rp_proj); + rf->rp_proj = dav1d_alloc_aligned(sizeof(*rf->rp_proj) * 16 * rp_stride * n_tile_rows, 64); + if (!rf->rp_proj) return DAV1D_ERR(ENOMEM); + rf->rp_stride = rp_stride; + } + rf->n_tile_rows = n_tile_rows; + rf->n_tile_threads = n_tile_threads; + rf->n_frame_threads = n_frame_threads; + rf->rp = rp; + rf->rp_ref = rp_ref; + const unsigned poc = frm_hdr->frame_offset; + for (int i = 0; i < 7; i++) { + const int poc_diff = get_poc_diff(seq_hdr->order_hint_n_bits, + ref_poc[i], poc); + rf->sign_bias[i] = poc_diff > 0; + rf->mfmv_sign[i] = poc_diff < 0; + rf->pocdiff[i] = iclip(get_poc_diff(seq_hdr->order_hint_n_bits, + poc, ref_poc[i]), -31, 31); + } + + // temporal MV setup + rf->n_mfmvs = 0; + if (frm_hdr->use_ref_frame_mvs && seq_hdr->order_hint_n_bits) { + int total = 2; + if (rp_ref[0] && ref_ref_poc[0][6] != ref_poc[3] /* alt-of-last != gold */) { + rf->mfmv_ref[rf->n_mfmvs++] = 0; // last + total = 3; + } + if (rp_ref[4] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[4], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 4; // bwd + } + if (rp_ref[5] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[5], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 5; // altref2 + } + if (rf->n_mfmvs < total && rp_ref[6] && + get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[6], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 6; // altref + } + if (rf->n_mfmvs < total && rp_ref[1]) + rf->mfmv_ref[rf->n_mfmvs++] = 1; // last2 + + for (int n = 0; n < rf->n_mfmvs; n++) { + const unsigned rpoc = ref_poc[rf->mfmv_ref[n]]; + const int diff1 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, frm_hdr->frame_offset); + if (abs(diff1) > 31) { + rf->mfmv_ref2cur[n] = INT_MIN; + } else { + rf->mfmv_ref2cur[n] = rf->mfmv_ref[n] < 4 ? -diff1 : diff1; + for (int m = 0; m < 7; m++) { + const unsigned rrpoc = ref_ref_poc[rf->mfmv_ref[n]][m]; + const int diff2 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, rrpoc); + // unsigned comparison also catches the < 0 case + rf->mfmv_ref2ref[n][m] = (unsigned) diff2 > 31U ? 0 : diff2; + } + } + } + } + rf->use_ref_frame_mvs = rf->n_mfmvs > 0; + + return 0; +} + +void dav1d_refmvs_init(refmvs_frame *const rf) { + rf->r = NULL; + rf->r_stride = 0; + rf->rp_proj = NULL; + rf->rp_stride = 0; +} + +void dav1d_refmvs_clear(refmvs_frame *const rf) { + if (rf->r) dav1d_freep_aligned(&rf->r); + if (rf->rp_proj) dav1d_freep_aligned(&rf->rp_proj); +} + +static void splat_mv_c(refmvs_block **rr, const refmvs_block *const rmv, + const int bx4, const int bw4, int bh4) +{ + do { + refmvs_block *const r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = *rmv; + } while (--bh4); +} + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM +#include "src/arm/refmvs.h" +#elif ARCH_X86 +#include "src/x86/refmvs.h" +#endif +#endif + +COLD void dav1d_refmvs_dsp_init(Dav1dRefmvsDSPContext *const c) +{ + c->load_tmvs = load_tmvs_c; + c->save_tmvs = save_tmvs_c; + c->splat_mv = splat_mv_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + refmvs_dsp_init_arm(c); +#elif ARCH_X86 + refmvs_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/refmvs.h b/third_party/dav1d/src/refmvs.h new file mode 100644 index 0000000000..70dc9678dd --- /dev/null +++ b/third_party/dav1d/src/refmvs.h @@ -0,0 +1,176 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_REF_MVS_H +#define DAV1D_SRC_REF_MVS_H + +#include + +#include "dav1d/headers.h" + +#include "common/intops.h" + +#include "src/intra_edge.h" +#include "src/tables.h" + +#define INVALID_MV 0x80008000 + +PACKED(typedef struct refmvs_temporal_block { + mv mv; + int8_t ref; +}) refmvs_temporal_block; + +typedef union refmvs_refpair { + int8_t ref[2]; // [0] = 0: intra=1, [1] = -1: comp=0 + uint16_t pair; +} refmvs_refpair; + +typedef union refmvs_mvpair { + mv mv[2]; + uint64_t n; +} refmvs_mvpair; + +PACKED(typedef struct refmvs_block { + refmvs_mvpair mv; + refmvs_refpair ref; + uint8_t bs, mf; // 1 = globalmv+affine, 2 = newmv +}) ALIGN(refmvs_block, 4); + +typedef struct refmvs_frame { + const Dav1dFrameHeader *frm_hdr; + int iw4, ih4, iw8, ih8; + int sbsz; + int use_ref_frame_mvs; + uint8_t sign_bias[7], mfmv_sign[7]; + int8_t pocdiff[7]; + uint8_t mfmv_ref[3]; + int mfmv_ref2cur[3]; + int mfmv_ref2ref[3][7]; + int n_mfmvs; + + refmvs_temporal_block *rp; + /*const*/ refmvs_temporal_block *const *rp_ref; + refmvs_temporal_block *rp_proj; + ptrdiff_t rp_stride; + + refmvs_block *r; // 35 x r_stride memory + ptrdiff_t r_stride; + int n_tile_rows, n_tile_threads, n_frame_threads; +} refmvs_frame; + +typedef struct refmvs_tile { + const refmvs_frame *rf; + refmvs_block *r[32 + 5]; + refmvs_temporal_block *rp_proj; + struct { + int start, end; + } tile_col, tile_row; +} refmvs_tile; + +typedef struct refmvs_candidate { + refmvs_mvpair mv; + int weight; +} refmvs_candidate; + +// initialize temporal MVs; this can be done in any configuration, e.g. one +// tile/sbrow at a time, where col_{start,end}8 are the tile boundaries; or +// it can just be for the whole frame's sbrow, where col_{start,end}8 are the +// frame boundaries. row_{start,end}8 are the superblock row boundaries. +#define decl_load_tmvs_fn(name) \ +void (name)(const refmvs_frame *rf, int tile_row_idx, \ + int col_start8, int col_end8, int row_start8, int row_end8) +typedef decl_load_tmvs_fn(*load_tmvs_fn); + +#define decl_save_tmvs_fn(name) \ +void (name)(refmvs_temporal_block *rp, const ptrdiff_t stride, \ + refmvs_block *const *const rr, const uint8_t *const ref_sign, \ + int col_end8, int row_end8, int col_start8, int row_start8) +typedef decl_save_tmvs_fn(*save_tmvs_fn); + +#define decl_splat_mv_fn(name) \ +void (name)(refmvs_block **rr, const refmvs_block *rmv, int bx4, int bw4, int bh4) +typedef decl_splat_mv_fn(*splat_mv_fn); + +typedef struct Dav1dRefmvsDSPContext { + load_tmvs_fn load_tmvs; + save_tmvs_fn save_tmvs; + splat_mv_fn splat_mv; +} Dav1dRefmvsDSPContext; + +// call once per frame thread +void dav1d_refmvs_init(refmvs_frame *rf); +void dav1d_refmvs_clear(refmvs_frame *rf); + +// call once per frame +int dav1d_refmvs_init_frame(refmvs_frame *rf, + const Dav1dSequenceHeader *seq_hdr, + const Dav1dFrameHeader *frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + int n_tile_threads, int n_frame_threads); + +// cache the current tile/sbrow (or frame/sbrow)'s projectable motion vectors +// into buffers for use in future frame's temporal MV prediction +static inline void dav1d_refmvs_save_tmvs(const Dav1dRefmvsDSPContext *const dsp, + const refmvs_tile *const rt, + const int col_start8, int col_end8, + const int row_start8, int row_end8) +{ + const refmvs_frame *const rf = rt->rf; + + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + col_end8 = imin(col_end8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + const uint8_t *const ref_sign = rf->mfmv_sign; + refmvs_temporal_block *rp = &rf->rp[row_start8 * stride]; + + dsp->save_tmvs(rp, stride, rt->r + 6, ref_sign, + col_end8, row_end8, col_start8, row_start8); +} + +// initialize tile boundaries and refmvs_block pointers for one tile/sbrow +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *rt, const refmvs_frame *rf, + int tile_col_start4, int tile_col_end4, + int tile_row_start4, int tile_row_end4, + int sby, int tile_row_idx, int pass); + +// call for each block +void dav1d_refmvs_find(const refmvs_tile *rt, + refmvs_candidate mvstack[8], int *cnt, + int *ctx, const refmvs_refpair ref, enum BlockSize bs, + enum EdgeFlags edge_flags, int by4, int bx4); + +void dav1d_refmvs_dsp_init(Dav1dRefmvsDSPContext *dsp); +void dav1d_refmvs_dsp_init_arm(Dav1dRefmvsDSPContext *dsp); +void dav1d_refmvs_dsp_init_x86(Dav1dRefmvsDSPContext *dsp); + +#endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/scan.c b/third_party/dav1d/src/scan.c new file mode 100644 index 0000000000..5261ccd3d1 --- /dev/null +++ b/third_party/dav1d/src/scan.c @@ -0,0 +1,299 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "common/attributes.h" +#include "src/scan.h" + +static const uint16_t ALIGN(scan_4x4[], 32) = { + 0, 4, 1, 2, + 5, 8, 12, 9, + 6, 3, 7, 10, + 13, 14, 11, 15, +}; + +static const uint16_t ALIGN(scan_4x8[], 32) = { + 0, 8, 1, 16, + 9, 2, 24, 17, + 10, 3, 25, 18, + 11, 4, 26, 19, + 12, 5, 27, 20, + 13, 6, 28, 21, + 14, 7, 29, 22, + 15, 30, 23, 31, +}; + +static const uint16_t ALIGN(scan_4x16[], 32) = { + 0, 16, 1, 32, + 17, 2, 48, 33, + 18, 3, 49, 34, + 19, 4, 50, 35, + 20, 5, 51, 36, + 21, 6, 52, 37, + 22, 7, 53, 38, + 23, 8, 54, 39, + 24, 9, 55, 40, + 25, 10, 56, 41, + 26, 11, 57, 42, + 27, 12, 58, 43, + 28, 13, 59, 44, + 29, 14, 60, 45, + 30, 15, 61, 46, + 31, 62, 47, 63, +}; + +static const uint16_t ALIGN(scan_8x4[], 32) = { + 0, 1, 4, 2, 5, 8, 3, 6, + 9, 12, 7, 10, 13, 16, 11, 14, + 17, 20, 15, 18, 21, 24, 19, 22, + 25, 28, 23, 26, 29, 27, 30, 31, +}; + +static const uint16_t ALIGN(scan_8x8[], 32) = { + 0, 8, 1, 2, 9, 16, 24, 17, + 10, 3, 4, 11, 18, 25, 32, 40, + 33, 26, 19, 12, 5, 6, 13, 20, + 27, 34, 41, 48, 56, 49, 42, 35, + 28, 21, 14, 7, 15, 22, 29, 36, + 43, 50, 57, 58, 51, 44, 37, 30, + 23, 31, 38, 45, 52, 59, 60, 53, + 46, 39, 47, 54, 61, 62, 55, 63, +}; + +static const uint16_t ALIGN(scan_8x16[], 32) = { + 0, 16, 1, 32, 17, 2, 48, 33, + 18, 3, 64, 49, 34, 19, 4, 80, + 65, 50, 35, 20, 5, 96, 81, 66, + 51, 36, 21, 6, 112, 97, 82, 67, + 52, 37, 22, 7, 113, 98, 83, 68, + 53, 38, 23, 8, 114, 99, 84, 69, + 54, 39, 24, 9, 115, 100, 85, 70, + 55, 40, 25, 10, 116, 101, 86, 71, + 56, 41, 26, 11, 117, 102, 87, 72, + 57, 42, 27, 12, 118, 103, 88, 73, + 58, 43, 28, 13, 119, 104, 89, 74, + 59, 44, 29, 14, 120, 105, 90, 75, + 60, 45, 30, 15, 121, 106, 91, 76, + 61, 46, 31, 122, 107, 92, 77, 62, + 47, 123, 108, 93, 78, 63, 124, 109, + 94, 79, 125, 110, 95, 126, 111, 127, +}; + +static const uint16_t ALIGN(scan_8x32[], 32) = { + 0, 32, 1, 64, 33, 2, 96, 65, + 34, 3, 128, 97, 66, 35, 4, 160, + 129, 98, 67, 36, 5, 192, 161, 130, + 99, 68, 37, 6, 224, 193, 162, 131, + 100, 69, 38, 7, 225, 194, 163, 132, + 101, 70, 39, 8, 226, 195, 164, 133, + 102, 71, 40, 9, 227, 196, 165, 134, + 103, 72, 41, 10, 228, 197, 166, 135, + 104, 73, 42, 11, 229, 198, 167, 136, + 105, 74, 43, 12, 230, 199, 168, 137, + 106, 75, 44, 13, 231, 200, 169, 138, + 107, 76, 45, 14, 232, 201, 170, 139, + 108, 77, 46, 15, 233, 202, 171, 140, + 109, 78, 47, 16, 234, 203, 172, 141, + 110, 79, 48, 17, 235, 204, 173, 142, + 111, 80, 49, 18, 236, 205, 174, 143, + 112, 81, 50, 19, 237, 206, 175, 144, + 113, 82, 51, 20, 238, 207, 176, 145, + 114, 83, 52, 21, 239, 208, 177, 146, + 115, 84, 53, 22, 240, 209, 178, 147, + 116, 85, 54, 23, 241, 210, 179, 148, + 117, 86, 55, 24, 242, 211, 180, 149, + 118, 87, 56, 25, 243, 212, 181, 150, + 119, 88, 57, 26, 244, 213, 182, 151, + 120, 89, 58, 27, 245, 214, 183, 152, + 121, 90, 59, 28, 246, 215, 184, 153, + 122, 91, 60, 29, 247, 216, 185, 154, + 123, 92, 61, 30, 248, 217, 186, 155, + 124, 93, 62, 31, 249, 218, 187, 156, + 125, 94, 63, 250, 219, 188, 157, 126, + 95, 251, 220, 189, 158, 127, 252, 221, + 190, 159, 253, 222, 191, 254, 223, 255, +}; + +static const uint16_t ALIGN(scan_16x4[], 32) = { + 0, 1, 4, 2, 5, 8, 3, 6, 9, 12, 7, 10, 13, 16, 11, 14, + 17, 20, 15, 18, 21, 24, 19, 22, 25, 28, 23, 26, 29, 32, 27, 30, + 33, 36, 31, 34, 37, 40, 35, 38, 41, 44, 39, 42, 45, 48, 43, 46, + 49, 52, 47, 50, 53, 56, 51, 54, 57, 60, 55, 58, 61, 59, 62, 63, +}; + +static const uint16_t ALIGN(scan_16x8[], 32) = { + 0, 1, 8, 2, 9, 16, 3, 10, 17, 24, 4, 11, 18, 25, 32, 5, + 12, 19, 26, 33, 40, 6, 13, 20, 27, 34, 41, 48, 7, 14, 21, 28, + 35, 42, 49, 56, 15, 22, 29, 36, 43, 50, 57, 64, 23, 30, 37, 44, + 51, 58, 65, 72, 31, 38, 45, 52, 59, 66, 73, 80, 39, 46, 53, 60, + 67, 74, 81, 88, 47, 54, 61, 68, 75, 82, 89, 96, 55, 62, 69, 76, + 83, 90, 97, 104, 63, 70, 77, 84, 91, 98, 105, 112, 71, 78, 85, 92, + 99, 106, 113, 120, 79, 86, 93, 100, 107, 114, 121, 87, 94, 101, 108, 115, + 122, 95, 102, 109, 116, 123, 103, 110, 117, 124, 111, 118, 125, 119, 126, 127, +}; + +static const uint16_t ALIGN(scan_16x16[], 32) = { + 0, 16, 1, 2, 17, 32, 48, 33, 18, 3, 4, 19, 34, 49, 64, 80, + 65, 50, 35, 20, 5, 6, 21, 36, 51, 66, 81, 96, 112, 97, 82, 67, + 52, 37, 22, 7, 8, 23, 38, 53, 68, 83, 98, 113, 128, 144, 129, 114, + 99, 84, 69, 54, 39, 24, 9, 10, 25, 40, 55, 70, 85, 100, 115, 130, + 145, 160, 176, 161, 146, 131, 116, 101, 86, 71, 56, 41, 26, 11, 12, 27, + 42, 57, 72, 87, 102, 117, 132, 147, 162, 177, 192, 208, 193, 178, 163, 148, + 133, 118, 103, 88, 73, 58, 43, 28, 13, 14, 29, 44, 59, 74, 89, 104, + 119, 134, 149, 164, 179, 194, 209, 224, 240, 225, 210, 195, 180, 165, 150, 135, + 120, 105, 90, 75, 60, 45, 30, 15, 31, 46, 61, 76, 91, 106, 121, 136, + 151, 166, 181, 196, 211, 226, 241, 242, 227, 212, 197, 182, 167, 152, 137, 122, + 107, 92, 77, 62, 47, 63, 78, 93, 108, 123, 138, 153, 168, 183, 198, 213, + 228, 243, 244, 229, 214, 199, 184, 169, 154, 139, 124, 109, 94, 79, 95, 110, + 125, 140, 155, 170, 185, 200, 215, 230, 245, 246, 231, 216, 201, 186, 171, 156, + 141, 126, 111, 127, 142, 157, 172, 187, 202, 217, 232, 247, 248, 233, 218, 203, + 188, 173, 158, 143, 159, 174, 189, 204, 219, 234, 249, 250, 235, 220, 205, 190, + 175, 191, 206, 221, 236, 251, 252, 237, 222, 207, 223, 238, 253, 254, 239, 255, +}; + +static const uint16_t ALIGN(scan_16x32[], 32) = { + 0, 32, 1, 64, 33, 2, 96, 65, 34, 3, 128, 97, 66, 35, 4, 160, + 129, 98, 67, 36, 5, 192, 161, 130, 99, 68, 37, 6, 224, 193, 162, 131, + 100, 69, 38, 7, 256, 225, 194, 163, 132, 101, 70, 39, 8, 288, 257, 226, + 195, 164, 133, 102, 71, 40, 9, 320, 289, 258, 227, 196, 165, 134, 103, 72, + 41, 10, 352, 321, 290, 259, 228, 197, 166, 135, 104, 73, 42, 11, 384, 353, + 322, 291, 260, 229, 198, 167, 136, 105, 74, 43, 12, 416, 385, 354, 323, 292, + 261, 230, 199, 168, 137, 106, 75, 44, 13, 448, 417, 386, 355, 324, 293, 262, + 231, 200, 169, 138, 107, 76, 45, 14, 480, 449, 418, 387, 356, 325, 294, 263, + 232, 201, 170, 139, 108, 77, 46, 15, 481, 450, 419, 388, 357, 326, 295, 264, + 233, 202, 171, 140, 109, 78, 47, 16, 482, 451, 420, 389, 358, 327, 296, 265, + 234, 203, 172, 141, 110, 79, 48, 17, 483, 452, 421, 390, 359, 328, 297, 266, + 235, 204, 173, 142, 111, 80, 49, 18, 484, 453, 422, 391, 360, 329, 298, 267, + 236, 205, 174, 143, 112, 81, 50, 19, 485, 454, 423, 392, 361, 330, 299, 268, + 237, 206, 175, 144, 113, 82, 51, 20, 486, 455, 424, 393, 362, 331, 300, 269, + 238, 207, 176, 145, 114, 83, 52, 21, 487, 456, 425, 394, 363, 332, 301, 270, + 239, 208, 177, 146, 115, 84, 53, 22, 488, 457, 426, 395, 364, 333, 302, 271, + 240, 209, 178, 147, 116, 85, 54, 23, 489, 458, 427, 396, 365, 334, 303, 272, + 241, 210, 179, 148, 117, 86, 55, 24, 490, 459, 428, 397, 366, 335, 304, 273, + 242, 211, 180, 149, 118, 87, 56, 25, 491, 460, 429, 398, 367, 336, 305, 274, + 243, 212, 181, 150, 119, 88, 57, 26, 492, 461, 430, 399, 368, 337, 306, 275, + 244, 213, 182, 151, 120, 89, 58, 27, 493, 462, 431, 400, 369, 338, 307, 276, + 245, 214, 183, 152, 121, 90, 59, 28, 494, 463, 432, 401, 370, 339, 308, 277, + 246, 215, 184, 153, 122, 91, 60, 29, 495, 464, 433, 402, 371, 340, 309, 278, + 247, 216, 185, 154, 123, 92, 61, 30, 496, 465, 434, 403, 372, 341, 310, 279, + 248, 217, 186, 155, 124, 93, 62, 31, 497, 466, 435, 404, 373, 342, 311, 280, + 249, 218, 187, 156, 125, 94, 63, 498, 467, 436, 405, 374, 343, 312, 281, 250, + 219, 188, 157, 126, 95, 499, 468, 437, 406, 375, 344, 313, 282, 251, 220, 189, + 158, 127, 500, 469, 438, 407, 376, 345, 314, 283, 252, 221, 190, 159, 501, 470, + 439, 408, 377, 346, 315, 284, 253, 222, 191, 502, 471, 440, 409, 378, 347, 316, + 285, 254, 223, 503, 472, 441, 410, 379, 348, 317, 286, 255, 504, 473, 442, 411, + 380, 349, 318, 287, 505, 474, 443, 412, 381, 350, 319, 506, 475, 444, 413, 382, + 351, 507, 476, 445, 414, 383, 508, 477, 446, 415, 509, 478, 447, 510, 479, 511, +}; + +static const uint16_t ALIGN(scan_32x8[], 32) = { + 0, 1, 8, 2, 9, 16, 3, 10, 17, 24, 4, 11, 18, 25, 32, 5, 12, 19, 26, 33, 40, 6, 13, 20, 27, 34, 41, 48, 7, 14, 21, 28, + 35, 42, 49, 56, 15, 22, 29, 36, 43, 50, 57, 64, 23, 30, 37, 44, 51, 58, 65, 72, 31, 38, 45, 52, 59, 66, 73, 80, 39, 46, 53, 60, + 67, 74, 81, 88, 47, 54, 61, 68, 75, 82, 89, 96, 55, 62, 69, 76, 83, 90, 97, 104, 63, 70, 77, 84, 91, 98, 105, 112, 71, 78, 85, 92, + 99, 106, 113, 120, 79, 86, 93, 100, 107, 114, 121, 128, 87, 94, 101, 108, 115, 122, 129, 136, 95, 102, 109, 116, 123, 130, 137, 144, 103, 110, 117, 124, + 131, 138, 145, 152, 111, 118, 125, 132, 139, 146, 153, 160, 119, 126, 133, 140, 147, 154, 161, 168, 127, 134, 141, 148, 155, 162, 169, 176, 135, 142, 149, 156, + 163, 170, 177, 184, 143, 150, 157, 164, 171, 178, 185, 192, 151, 158, 165, 172, 179, 186, 193, 200, 159, 166, 173, 180, 187, 194, 201, 208, 167, 174, 181, 188, + 195, 202, 209, 216, 175, 182, 189, 196, 203, 210, 217, 224, 183, 190, 197, 204, 211, 218, 225, 232, 191, 198, 205, 212, 219, 226, 233, 240, 199, 206, 213, 220, + 227, 234, 241, 248, 207, 214, 221, 228, 235, 242, 249, 215, 222, 229, 236, 243, 250, 223, 230, 237, 244, 251, 231, 238, 245, 252, 239, 246, 253, 247, 254, 255, +}; + +static const uint16_t ALIGN(scan_32x16[], 32) = { + 0, 1, 16, 2, 17, 32, 3, 18, 33, 48, 4, 19, 34, 49, 64, 5, 20, 35, 50, 65, 80, 6, 21, 36, 51, 66, 81, 96, 7, 22, 37, 52, + 67, 82, 97, 112, 8, 23, 38, 53, 68, 83, 98, 113, 128, 9, 24, 39, 54, 69, 84, 99, 114, 129, 144, 10, 25, 40, 55, 70, 85, 100, 115, 130, + 145, 160, 11, 26, 41, 56, 71, 86, 101, 116, 131, 146, 161, 176, 12, 27, 42, 57, 72, 87, 102, 117, 132, 147, 162, 177, 192, 13, 28, 43, 58, 73, + 88, 103, 118, 133, 148, 163, 178, 193, 208, 14, 29, 44, 59, 74, 89, 104, 119, 134, 149, 164, 179, 194, 209, 224, 15, 30, 45, 60, 75, 90, 105, 120, + 135, 150, 165, 180, 195, 210, 225, 240, 31, 46, 61, 76, 91, 106, 121, 136, 151, 166, 181, 196, 211, 226, 241, 256, 47, 62, 77, 92, 107, 122, 137, 152, + 167, 182, 197, 212, 227, 242, 257, 272, 63, 78, 93, 108, 123, 138, 153, 168, 183, 198, 213, 228, 243, 258, 273, 288, 79, 94, 109, 124, 139, 154, 169, 184, + 199, 214, 229, 244, 259, 274, 289, 304, 95, 110, 125, 140, 155, 170, 185, 200, 215, 230, 245, 260, 275, 290, 305, 320, 111, 126, 141, 156, 171, 186, 201, 216, + 231, 246, 261, 276, 291, 306, 321, 336, 127, 142, 157, 172, 187, 202, 217, 232, 247, 262, 277, 292, 307, 322, 337, 352, 143, 158, 173, 188, 203, 218, 233, 248, + 263, 278, 293, 308, 323, 338, 353, 368, 159, 174, 189, 204, 219, 234, 249, 264, 279, 294, 309, 324, 339, 354, 369, 384, 175, 190, 205, 220, 235, 250, 265, 280, + 295, 310, 325, 340, 355, 370, 385, 400, 191, 206, 221, 236, 251, 266, 281, 296, 311, 326, 341, 356, 371, 386, 401, 416, 207, 222, 237, 252, 267, 282, 297, 312, + 327, 342, 357, 372, 387, 402, 417, 432, 223, 238, 253, 268, 283, 298, 313, 328, 343, 358, 373, 388, 403, 418, 433, 448, 239, 254, 269, 284, 299, 314, 329, 344, + 359, 374, 389, 404, 419, 434, 449, 464, 255, 270, 285, 300, 315, 330, 345, 360, 375, 390, 405, 420, 435, 450, 465, 480, 271, 286, 301, 316, 331, 346, 361, 376, + 391, 406, 421, 436, 451, 466, 481, 496, 287, 302, 317, 332, 347, 362, 377, 392, 407, 422, 437, 452, 467, 482, 497, 303, 318, 333, 348, 363, 378, 393, 408, 423, + 438, 453, 468, 483, 498, 319, 334, 349, 364, 379, 394, 409, 424, 439, 454, 469, 484, 499, 335, 350, 365, 380, 395, 410, 425, 440, 455, 470, 485, 500, 351, 366, + 381, 396, 411, 426, 441, 456, 471, 486, 501, 367, 382, 397, 412, 427, 442, 457, 472, 487, 502, 383, 398, 413, 428, 443, 458, 473, 488, 503, 399, 414, 429, 444, + 459, 474, 489, 504, 415, 430, 445, 460, 475, 490, 505, 431, 446, 461, 476, 491, 506, 447, 462, 477, 492, 507, 463, 478, 493, 508, 479, 494, 509, 495, 510, 511, +}; + +static const uint16_t ALIGN(scan_32x32[], 32) = { + 0, 32, 1, 2, 33, 64, 96, 65, 34, 3, 4, 35, 66, 97, 128, 160, 129, 98, 67, 36, 5, 6, 37, 68, 99, 130, 161, 192, 224, 193, 162, 131, + 100, 69, 38, 7, 8, 39, 70, 101, 132, 163, 194, 225, 256, 288, 257, 226, 195, 164, 133, 102, 71, 40, 9, 10, 41, 72, 103, 134, 165, 196, 227, 258, + 289, 320, 352, 321, 290, 259, 228, 197, 166, 135, 104, 73, 42, 11, 12, 43, 74, 105, 136, 167, 198, 229, 260, 291, 322, 353, 384, 416, 385, 354, 323, 292, + 261, 230, 199, 168, 137, 106, 75, 44, 13, 14, 45, 76, 107, 138, 169, 200, 231, 262, 293, 324, 355, 386, 417, 448, 480, 449, 418, 387, 356, 325, 294, 263, + 232, 201, 170, 139, 108, 77, 46, 15, 16, 47, 78, 109, 140, 171, 202, 233, 264, 295, 326, 357, 388, 419, 450, 481, 512, 544, 513, 482, 451, 420, 389, 358, + 327, 296, 265, 234, 203, 172, 141, 110, 79, 48, 17, 18, 49, 80, 111, 142, 173, 204, 235, 266, 297, 328, 359, 390, 421, 452, 483, 514, 545, 576, 608, 577, + 546, 515, 484, 453, 422, 391, 360, 329, 298, 267, 236, 205, 174, 143, 112, 81, 50, 19, 20, 51, 82, 113, 144, 175, 206, 237, 268, 299, 330, 361, 392, 423, + 454, 485, 516, 547, 578, 609, 640, 672, 641, 610, 579, 548, 517, 486, 455, 424, 393, 362, 331, 300, 269, 238, 207, 176, 145, 114, 83, 52, 21, 22, 53, 84, + 115, 146, 177, 208, 239, 270, 301, 332, 363, 394, 425, 456, 487, 518, 549, 580, 611, 642, 673, 704, 736, 705, 674, 643, 612, 581, 550, 519, 488, 457, 426, 395, + 364, 333, 302, 271, 240, 209, 178, 147, 116, 85, 54, 23, 24, 55, 86, 117, 148, 179, 210, 241, 272, 303, 334, 365, 396, 427, 458, 489, 520, 551, 582, 613, + 644, 675, 706, 737, 768, 800, 769, 738, 707, 676, 645, 614, 583, 552, 521, 490, 459, 428, 397, 366, 335, 304, 273, 242, 211, 180, 149, 118, 87, 56, 25, 26, + 57, 88, 119, 150, 181, 212, 243, 274, 305, 336, 367, 398, 429, 460, 491, 522, 553, 584, 615, 646, 677, 708, 739, 770, 801, 832, 864, 833, 802, 771, 740, 709, + 678, 647, 616, 585, 554, 523, 492, 461, 430, 399, 368, 337, 306, 275, 244, 213, 182, 151, 120, 89, 58, 27, 28, 59, 90, 121, 152, 183, 214, 245, 276, 307, + 338, 369, 400, 431, 462, 493, 524, 555, 586, 617, 648, 679, 710, 741, 772, 803, 834, 865, 896, 928, 897, 866, 835, 804, 773, 742, 711, 680, 649, 618, 587, 556, + 525, 494, 463, 432, 401, 370, 339, 308, 277, 246, 215, 184, 153, 122, 91, 60, 29, 30, 61, 92, 123, 154, 185, 216, 247, 278, 309, 340, 371, 402, 433, 464, + 495, 526, 557, 588, 619, 650, 681, 712, 743, 774, 805, 836, 867, 898, 929, 960, 992, 961, 930, 899, 868, 837, 806, 775, 744, 713, 682, 651, 620, 589, 558, 527, + 496, 465, 434, 403, 372, 341, 310, 279, 248, 217, 186, 155, 124, 93, 62, 31, 63, 94, 125, 156, 187, 218, 249, 280, 311, 342, 373, 404, 435, 466, 497, 528, + 559, 590, 621, 652, 683, 714, 745, 776, 807, 838, 869, 900, 931, 962, 993, 994, 963, 932, 901, 870, 839, 808, 777, 746, 715, 684, 653, 622, 591, 560, 529, 498, + 467, 436, 405, 374, 343, 312, 281, 250, 219, 188, 157, 126, 95, 127, 158, 189, 220, 251, 282, 313, 344, 375, 406, 437, 468, 499, 530, 561, 592, 623, 654, 685, + 716, 747, 778, 809, 840, 871, 902, 933, 964, 995, 996, 965, 934, 903, 872, 841, 810, 779, 748, 717, 686, 655, 624, 593, 562, 531, 500, 469, 438, 407, 376, 345, + 314, 283, 252, 221, 190, 159, 191, 222, 253, 284, 315, 346, 377, 408, 439, 470, 501, 532, 563, 594, 625, 656, 687, 718, 749, 780, 811, 842, 873, 904, 935, 966, + 997, 998, 967, 936, 905, 874, 843, 812, 781, 750, 719, 688, 657, 626, 595, 564, 533, 502, 471, 440, 409, 378, 347, 316, 285, 254, 223, 255, 286, 317, 348, 379, + 410, 441, 472, 503, 534, 565, 596, 627, 658, 689, 720, 751, 782, 813, 844, 875, 906, 937, 968, 999, 1000, 969, 938, 907, 876, 845, 814, 783, 752, 721, 690, 659, + 628, 597, 566, 535, 504, 473, 442, 411, 380, 349, 318, 287, 319, 350, 381, 412, 443, 474, 505, 536, 567, 598, 629, 660, 691, 722, 753, 784, 815, 846, 877, 908, + 939, 970, 1001, 1002, 971, 940, 909, 878, 847, 816, 785, 754, 723, 692, 661, 630, 599, 568, 537, 506, 475, 444, 413, 382, 351, 383, 414, 445, 476, 507, 538, 569, + 600, 631, 662, 693, 724, 755, 786, 817, 848, 879, 910, 941, 972, 1003, 1004, 973, 942, 911, 880, 849, 818, 787, 756, 725, 694, 663, 632, 601, 570, 539, 508, 477, + 446, 415, 447, 478, 509, 540, 571, 602, 633, 664, 695, 726, 757, 788, 819, 850, 881, 912, 943, 974, 1005, 1006, 975, 944, 913, 882, 851, 820, 789, 758, 727, 696, + 665, 634, 603, 572, 541, 510, 479, 511, 542, 573, 604, 635, 666, 697, 728, 759, 790, 821, 852, 883, 914, 945, 976, 1007, 1008, 977, 946, 915, 884, 853, 822, 791, + 760, 729, 698, 667, 636, 605, 574, 543, 575, 606, 637, 668, 699, 730, 761, 792, 823, 854, 885, 916, 947, 978, 1009, 1010, 979, 948, 917, 886, 855, 824, 793, 762, + 731, 700, 669, 638, 607, 639, 670, 701, 732, 763, 794, 825, 856, 887, 918, 949, 980, 1011, 1012, 981, 950, 919, 888, 857, 826, 795, 764, 733, 702, 671, 703, 734, + 765, 796, 827, 858, 889, 920, 951, 982, 1013, 1014, 983, 952, 921, 890, 859, 828, 797, 766, 735, 767, 798, 829, 860, 891, 922, 953, 984, 1015, 1016, 985, 954, 923, + 892, 861, 830, 799, 831, 862, 893, 924, 955, 986, 1017, 1018, 987, 956, 925, 894, 863, 895, 926, 957, 988, 1019, 1020, 989, 958, 927, 959, 990, 1021, 1022, 991, 1023, +}; + +const uint16_t *const dav1d_scans[N_RECT_TX_SIZES] = { + [ TX_4X4 ] = scan_4x4, + [ TX_8X8 ] = scan_8x8, + [ TX_16X16] = scan_16x16, + [ TX_32X32] = scan_32x32, + [ TX_64X64] = scan_32x32, + [RTX_4X8 ] = scan_4x8, + [RTX_8X4 ] = scan_8x4, + [RTX_8X16 ] = scan_8x16, + [RTX_16X8 ] = scan_16x8, + [RTX_16X32] = scan_16x32, + [RTX_32X16] = scan_32x16, + [RTX_32X64] = scan_32x32, + [RTX_64X32] = scan_32x32, + [RTX_4X16 ] = scan_4x16, + [RTX_16X4 ] = scan_16x4, + [RTX_8X32 ] = scan_8x32, + [RTX_32X8 ] = scan_32x8, + [RTX_16X64] = scan_16x32, + [RTX_64X16] = scan_32x16, +}; diff --git a/third_party/dav1d/src/scan.h b/third_party/dav1d/src/scan.h new file mode 100644 index 0000000000..09df988779 --- /dev/null +++ b/third_party/dav1d/src/scan.h @@ -0,0 +1,37 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_SCAN_H +#define DAV1D_SRC_SCAN_H + +#include + +#include "src/levels.h" + +EXTERN const uint16_t *const dav1d_scans[N_RECT_TX_SIZES]; + +#endif /* DAV1D_SRC_SCAN_H */ diff --git a/third_party/dav1d/src/tables.c b/third_party/dav1d/src/tables.c new file mode 100644 index 0000000000..9752f15c40 --- /dev/null +++ b/third_party/dav1d/src/tables.c @@ -0,0 +1,1013 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" + +#include "src/levels.h" +#include "src/tables.h" + +const uint8_t dav1d_al_part_ctx[2][N_BL_LEVELS][N_PARTITIONS] = { + { + // partitions: + // none, h, v, split, tts, tbs, tls, trs, h4, v4 + { 0x00, 0x00, 0x10, -1, 0x00, 0x10, 0x10, 0x10, -1, -1 }, // bl128 + { 0x10, 0x10, 0x18, -1, 0x10, 0x18, 0x18, 0x18, 0x10, 0x1c }, // bl64 + { 0x18, 0x18, 0x1c, -1, 0x18, 0x1c, 0x1c, 0x1c, 0x18, 0x1e }, // bl32 + { 0x1c, 0x1c, 0x1e, -1, 0x1c, 0x1e, 0x1e, 0x1e, 0x1c, 0x1f }, // bl16 + { 0x1e, 0x1e, 0x1f, 0x1f, -1, -1, -1, -1, -1, -1 }, // bl8 + }, { + { 0x00, 0x10, 0x00, -1, 0x10, 0x10, 0x00, 0x10, -1, -1 }, // bl128 + { 0x10, 0x18, 0x10, -1, 0x18, 0x18, 0x10, 0x18, 0x1c, 0x10 }, // bl64 + { 0x18, 0x1c, 0x18, -1, 0x1c, 0x1c, 0x18, 0x1c, 0x1e, 0x18 }, // bl32 + { 0x1c, 0x1e, 0x1c, -1, 0x1e, 0x1e, 0x1c, 0x1e, 0x1f, 0x1c }, // bl16 + { 0x1e, 0x1f, 0x1e, 0x1f, -1, -1, -1, -1, -1, -1 }, // bl8 + } +}; + +const uint8_t /* enum BlockSize */ + dav1d_block_sizes[N_BL_LEVELS][N_PARTITIONS][2] = +{ + [BL_128X128] = { + [PARTITION_NONE] = { BS_128x128 }, + [PARTITION_H] = { BS_128x64 }, + [PARTITION_V] = { BS_64x128 }, + [PARTITION_T_TOP_SPLIT] = { BS_64x64, BS_128x64 }, + [PARTITION_T_BOTTOM_SPLIT] = { BS_128x64, BS_64x64 }, + [PARTITION_T_LEFT_SPLIT] = { BS_64x64, BS_64x128 }, + [PARTITION_T_RIGHT_SPLIT] = { BS_64x128, BS_64x64 }, + }, [BL_64X64] = { + [PARTITION_NONE] = { BS_64x64 }, + [PARTITION_H] = { BS_64x32 }, + [PARTITION_V] = { BS_32x64 }, + [PARTITION_T_TOP_SPLIT] = { BS_32x32, BS_64x32 }, + [PARTITION_T_BOTTOM_SPLIT] = { BS_64x32, BS_32x32 }, + [PARTITION_T_LEFT_SPLIT] = { BS_32x32, BS_32x64 }, + [PARTITION_T_RIGHT_SPLIT] = { BS_32x64, BS_32x32 }, + [PARTITION_H4] = { BS_64x16 }, + [PARTITION_V4] = { BS_16x64 }, + }, [BL_32X32] = { + [PARTITION_NONE] = { BS_32x32 }, + [PARTITION_H] = { BS_32x16 }, + [PARTITION_V] = { BS_16x32 }, + [PARTITION_T_TOP_SPLIT] = { BS_16x16, BS_32x16 }, + [PARTITION_T_BOTTOM_SPLIT] = { BS_32x16, BS_16x16 }, + [PARTITION_T_LEFT_SPLIT] = { BS_16x16, BS_16x32 }, + [PARTITION_T_RIGHT_SPLIT] = { BS_16x32, BS_16x16 }, + [PARTITION_H4] = { BS_32x8 }, + [PARTITION_V4] = { BS_8x32 }, + }, [BL_16X16] = { + [PARTITION_NONE] = { BS_16x16 }, + [PARTITION_H] = { BS_16x8 }, + [PARTITION_V] = { BS_8x16 }, + [PARTITION_T_TOP_SPLIT] = { BS_8x8, BS_16x8 }, + [PARTITION_T_BOTTOM_SPLIT] = { BS_16x8, BS_8x8 }, + [PARTITION_T_LEFT_SPLIT] = { BS_8x8, BS_8x16 }, + [PARTITION_T_RIGHT_SPLIT] = { BS_8x16, BS_8x8 }, + [PARTITION_H4] = { BS_16x4 }, + [PARTITION_V4] = { BS_4x16 }, + }, [BL_8X8] = { + [PARTITION_NONE] = { BS_8x8 }, + [PARTITION_H] = { BS_8x4 }, + [PARTITION_V] = { BS_4x8 }, + [PARTITION_SPLIT] = { BS_4x4 }, + } +}; + +const uint8_t dav1d_block_dimensions[N_BS_SIZES][4] = { + [BS_128x128] = { 32, 32, 5, 5 }, + [BS_128x64] = { 32, 16, 5, 4 }, + [BS_64x128] = { 16, 32, 4, 5 }, + [BS_64x64] = { 16, 16, 4, 4 }, + [BS_64x32] = { 16, 8, 4, 3 }, + [BS_64x16] = { 16, 4, 4, 2 }, + [BS_32x64] = { 8, 16, 3, 4 }, + [BS_32x32] = { 8, 8, 3, 3 }, + [BS_32x16] = { 8, 4, 3, 2 }, + [BS_32x8] = { 8, 2, 3, 1 }, + [BS_16x64] = { 4, 16, 2, 4 }, + [BS_16x32] = { 4, 8, 2, 3 }, + [BS_16x16] = { 4, 4, 2, 2 }, + [BS_16x8] = { 4, 2, 2, 1 }, + [BS_16x4] = { 4, 1, 2, 0 }, + [BS_8x32] = { 2, 8, 1, 3 }, + [BS_8x16] = { 2, 4, 1, 2 }, + [BS_8x8] = { 2, 2, 1, 1 }, + [BS_8x4] = { 2, 1, 1, 0 }, + [BS_4x16] = { 1, 4, 0, 2 }, + [BS_4x8] = { 1, 2, 0, 1 }, + [BS_4x4] = { 1, 1, 0, 0 }, +}; + +const TxfmInfo dav1d_txfm_dimensions[N_RECT_TX_SIZES] = { + [ TX_4X4] = { .w = 1, .h = 1, .lw = 0, .lh = 0, + .min = 0, .max = 0, .ctx = 0 }, + [ TX_8X8] = { .w = 2, .h = 2, .lw = 1, .lh = 1, + .min = 1, .max = 1, .sub = TX_4X4, .ctx = 1 }, + [ TX_16X16] = { .w = 4, .h = 4, .lw = 2, .lh = 2, + .min = 2, .max = 2, .sub = TX_8X8, .ctx = 2 }, + [ TX_32X32] = { .w = 8, .h = 8, .lw = 3, .lh = 3, + .min = 3, .max = 3, .sub = TX_16X16, .ctx = 3 }, + [ TX_64X64] = { .w = 16, .h = 16, .lw = 4, .lh = 4, + .min = 4, .max = 4, .sub = TX_32X32, .ctx = 4 }, + [RTX_4X8] = { .w = 1, .h = 2, .lw = 0, .lh = 1, + .min = 0, .max = 1, .sub = TX_4X4, .ctx = 1 }, + [RTX_8X4] = { .w = 2, .h = 1, .lw = 1, .lh = 0, + .min = 0, .max = 1, .sub = TX_4X4, .ctx = 1 }, + [RTX_8X16] = { .w = 2, .h = 4, .lw = 1, .lh = 2, + .min = 1, .max = 2, .sub = TX_8X8, .ctx = 2 }, + [RTX_16X8] = { .w = 4, .h = 2, .lw = 2, .lh = 1, + .min = 1, .max = 2, .sub = TX_8X8, .ctx = 2 }, + [RTX_16X32] = { .w = 4, .h = 8, .lw = 2, .lh = 3, + .min = 2, .max = 3, .sub = TX_16X16, .ctx = 3 }, + [RTX_32X16] = { .w = 8, .h = 4, .lw = 3, .lh = 2, + .min = 2, .max = 3, .sub = TX_16X16, .ctx = 3 }, + [RTX_32X64] = { .w = 8, .h = 16, .lw = 3, .lh = 4, + .min = 3, .max = 4, .sub = TX_32X32, .ctx = 4 }, + [RTX_64X32] = { .w = 16, .h = 8, .lw = 4, .lh = 3, + .min = 3, .max = 4, .sub = TX_32X32, .ctx = 4 }, + [RTX_4X16] = { .w = 1, .h = 4, .lw = 0, .lh = 2, + .min = 0, .max = 2, .sub = RTX_4X8, .ctx = 1 }, + [RTX_16X4] = { .w = 4, .h = 1, .lw = 2, .lh = 0, + .min = 0, .max = 2, .sub = RTX_8X4, .ctx = 1 }, + [RTX_8X32] = { .w = 2, .h = 8, .lw = 1, .lh = 3, + .min = 1, .max = 3, .sub = RTX_8X16, .ctx = 2 }, + [RTX_32X8] = { .w = 8, .h = 2, .lw = 3, .lh = 1, + .min = 1, .max = 3, .sub = RTX_16X8, .ctx = 2 }, + [RTX_16X64] = { .w = 4, .h = 16, .lw = 2, .lh = 4, + .min = 2, .max = 4, .sub = RTX_16X32, .ctx = 3 }, + [RTX_64X16] = { .w = 16, .h = 4, .lw = 4, .lh = 2, + .min = 2, .max = 4, .sub = RTX_32X16, .ctx = 3 }, +}; + +const uint8_t /* enum (Rect)TxfmSize */ + dav1d_max_txfm_size_for_bs[N_BS_SIZES][4 /* y, 420, 422, 444 */] = +{ + [BS_128x128] = { TX_64X64, TX_32X32, TX_32X32, TX_32X32 }, + [BS_128x64] = { TX_64X64, TX_32X32, TX_32X32, TX_32X32 }, + [BS_64x128] = { TX_64X64, TX_32X32, 0, TX_32X32 }, + [BS_64x64] = { TX_64X64, TX_32X32, TX_32X32, TX_32X32 }, + [BS_64x32] = { RTX_64X32, RTX_32X16, TX_32X32, TX_32X32 }, + [BS_64x16] = { RTX_64X16, RTX_32X8, RTX_32X16, RTX_32X16 }, + [BS_32x64] = { RTX_32X64, RTX_16X32, 0, TX_32X32 }, + [BS_32x32] = { TX_32X32, TX_16X16, RTX_16X32, TX_32X32 }, + [BS_32x16] = { RTX_32X16, RTX_16X8, TX_16X16, RTX_32X16 }, + [BS_32x8] = { RTX_32X8, RTX_16X4, RTX_16X8, RTX_32X8 }, + [BS_16x64] = { RTX_16X64, RTX_8X32, 0, RTX_16X32 }, + [BS_16x32] = { RTX_16X32, RTX_8X16, 0, RTX_16X32 }, + [BS_16x16] = { TX_16X16, TX_8X8, RTX_8X16, TX_16X16 }, + [BS_16x8] = { RTX_16X8, RTX_8X4, TX_8X8, RTX_16X8 }, + [BS_16x4] = { RTX_16X4, RTX_8X4, RTX_8X4, RTX_16X4 }, + [BS_8x32] = { RTX_8X32, RTX_4X16, 0, RTX_8X32 }, + [BS_8x16] = { RTX_8X16, RTX_4X8, 0, RTX_8X16 }, + [BS_8x8] = { TX_8X8, TX_4X4, RTX_4X8, TX_8X8 }, + [BS_8x4] = { RTX_8X4, TX_4X4, TX_4X4, RTX_8X4 }, + [BS_4x16] = { RTX_4X16, RTX_4X8, 0, RTX_4X16 }, + [BS_4x8] = { RTX_4X8, TX_4X4, 0, RTX_4X8 }, + [BS_4x4] = { TX_4X4, TX_4X4, TX_4X4, TX_4X4 }, +}; + +const uint8_t /* enum TxfmType */ + dav1d_txtp_from_uvmode[N_UV_INTRA_PRED_MODES] = +{ + [DC_PRED] = DCT_DCT, + [VERT_PRED] = ADST_DCT, + [HOR_PRED] = DCT_ADST, + [DIAG_DOWN_LEFT_PRED] = DCT_DCT, + [DIAG_DOWN_RIGHT_PRED] = ADST_ADST, + [VERT_RIGHT_PRED] = ADST_DCT, + [HOR_DOWN_PRED] = DCT_ADST, + [HOR_UP_PRED] = DCT_ADST, + [VERT_LEFT_PRED] = ADST_DCT, + [SMOOTH_PRED] = ADST_ADST, + [SMOOTH_V_PRED] = ADST_DCT, + [SMOOTH_H_PRED] = DCT_ADST, + [PAETH_PRED] = ADST_ADST, +}; + +const uint8_t /* enum InterPredMode */ + dav1d_comp_inter_pred_modes[N_COMP_INTER_PRED_MODES][2] = +{ + [NEARESTMV_NEARESTMV] = { NEARESTMV, NEARESTMV }, + [NEARMV_NEARMV] = { NEARMV, NEARMV }, + [NEWMV_NEWMV] = { NEWMV, NEWMV }, + [GLOBALMV_GLOBALMV] = { GLOBALMV, GLOBALMV }, + [NEWMV_NEARESTMV] = { NEWMV, NEARESTMV }, + [NEWMV_NEARMV] = { NEWMV, NEARMV }, + [NEARESTMV_NEWMV] = { NEARESTMV, NEWMV }, + [NEARMV_NEWMV] = { NEARMV, NEWMV }, +}; + +const uint8_t dav1d_partition_type_count[N_BL_LEVELS] = { + [BL_128X128] = N_PARTITIONS - 3, + [BL_64X64] = N_PARTITIONS - 1, + [BL_32X32] = N_PARTITIONS - 1, + [BL_16X16] = N_PARTITIONS - 1, + [BL_8X8] = N_SUB8X8_PARTITIONS - 1, +}; + +const uint8_t /* enum TxfmType */ dav1d_tx_types_per_set[40] = { + /* Intra2 */ + IDTX, DCT_DCT, ADST_ADST, ADST_DCT, DCT_ADST, + /* Intra1 */ + IDTX, DCT_DCT, V_DCT, H_DCT, ADST_ADST, ADST_DCT, DCT_ADST, + /* Inter2 */ + IDTX, V_DCT, H_DCT, DCT_DCT, ADST_DCT, DCT_ADST, FLIPADST_DCT, + DCT_FLIPADST, ADST_ADST, FLIPADST_FLIPADST, ADST_FLIPADST, FLIPADST_ADST, + /* Inter1 */ + IDTX, V_DCT, H_DCT, V_ADST, H_ADST, V_FLIPADST, H_FLIPADST, + DCT_DCT, ADST_DCT, DCT_ADST, FLIPADST_DCT, DCT_FLIPADST, + ADST_ADST, FLIPADST_FLIPADST, ADST_FLIPADST, FLIPADST_ADST, +}; + +const uint8_t dav1d_ymode_size_context[N_BS_SIZES] = { + [BS_128x128] = 3, + [BS_128x64] = 3, + [BS_64x128] = 3, + [BS_64x64] = 3, + [BS_64x32] = 3, + [BS_64x16] = 2, + [BS_32x64] = 3, + [BS_32x32] = 3, + [BS_32x16] = 2, + [BS_32x8 ] = 1, + [BS_16x64] = 2, + [BS_16x32] = 2, + [BS_16x16] = 2, + [BS_16x8 ] = 1, + [BS_16x4 ] = 0, + [BS_8x32 ] = 1, + [BS_8x16 ] = 1, + [BS_8x8 ] = 1, + [BS_8x4 ] = 0, + [BS_4x16 ] = 0, + [BS_4x8 ] = 0, + [BS_4x4 ] = 0, +}; + +const uint8_t dav1d_lo_ctx_offsets[3][5][5] = { + { /* w == h */ + { 0, 1, 6, 6, 21 }, + { 1, 6, 6, 21, 21 }, + { 6, 6, 21, 21, 21 }, + { 6, 21, 21, 21, 21 }, + { 21, 21, 21, 21, 21 }, + }, { /* w > h */ + { 0, 16, 6, 6, 21 }, + { 16, 16, 6, 21, 21 }, + { 16, 16, 21, 21, 21 }, + { 16, 16, 21, 21, 21 }, + { 16, 16, 21, 21, 21 }, + }, { /* w < h */ + { 0, 11, 11, 11, 11 }, + { 11, 11, 11, 11, 11 }, + { 6, 6, 21, 21, 21 }, + { 6, 21, 21, 21, 21 }, + { 21, 21, 21, 21, 21 }, + }, +}; + +const uint8_t dav1d_skip_ctx[5][5] = { + { 1, 2, 2, 2, 3 }, + { 2, 4, 4, 4, 5 }, + { 2, 4, 4, 4, 5 }, + { 2, 4, 4, 4, 5 }, + { 3, 5, 5, 5, 6 }, +}; + +const uint8_t /* enum TxClass */ dav1d_tx_type_class[N_TX_TYPES_PLUS_LL] = { + [DCT_DCT] = TX_CLASS_2D, + [ADST_DCT] = TX_CLASS_2D, + [DCT_ADST] = TX_CLASS_2D, + [ADST_ADST] = TX_CLASS_2D, + [FLIPADST_DCT] = TX_CLASS_2D, + [DCT_FLIPADST] = TX_CLASS_2D, + [FLIPADST_FLIPADST] = TX_CLASS_2D, + [ADST_FLIPADST] = TX_CLASS_2D, + [FLIPADST_ADST] = TX_CLASS_2D, + [IDTX] = TX_CLASS_2D, + [V_DCT] = TX_CLASS_V, + [H_DCT] = TX_CLASS_H, + [V_ADST] = TX_CLASS_V, + [H_ADST] = TX_CLASS_H, + [V_FLIPADST] = TX_CLASS_V, + [H_FLIPADST] = TX_CLASS_H, + [WHT_WHT] = TX_CLASS_2D, +}; + +const uint8_t /* enum Filter2d */ dav1d_filter_2d[DAV1D_N_FILTERS][DAV1D_N_FILTERS] = { + [DAV1D_FILTER_8TAP_REGULAR] = { + [DAV1D_FILTER_8TAP_REGULAR] = FILTER_2D_8TAP_REGULAR, + [DAV1D_FILTER_8TAP_SHARP] = FILTER_2D_8TAP_REGULAR_SHARP, + [DAV1D_FILTER_8TAP_SMOOTH] = FILTER_2D_8TAP_REGULAR_SMOOTH, + }, [DAV1D_FILTER_8TAP_SHARP] = { + [DAV1D_FILTER_8TAP_REGULAR] = FILTER_2D_8TAP_SHARP_REGULAR, + [DAV1D_FILTER_8TAP_SHARP] = FILTER_2D_8TAP_SHARP, + [DAV1D_FILTER_8TAP_SMOOTH] = FILTER_2D_8TAP_SHARP_SMOOTH, + }, [DAV1D_FILTER_8TAP_SMOOTH] = { + [DAV1D_FILTER_8TAP_REGULAR] = FILTER_2D_8TAP_SMOOTH_REGULAR, + [DAV1D_FILTER_8TAP_SHARP] = FILTER_2D_8TAP_SMOOTH_SHARP, + [DAV1D_FILTER_8TAP_SMOOTH] = FILTER_2D_8TAP_SMOOTH, + }, [DAV1D_FILTER_BILINEAR] = { + [DAV1D_FILTER_BILINEAR] = FILTER_2D_BILINEAR, + } +}; + +const uint8_t /* enum Dav1dFilterMode */ dav1d_filter_dir[N_2D_FILTERS][2] = { + [FILTER_2D_8TAP_REGULAR] = { DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_REGULAR }, + [FILTER_2D_8TAP_REGULAR_SMOOTH] = { DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_REGULAR }, + [FILTER_2D_8TAP_REGULAR_SHARP] = { DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_REGULAR }, + [FILTER_2D_8TAP_SHARP_REGULAR] = { DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SHARP }, + [FILTER_2D_8TAP_SHARP_SMOOTH] = { DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SHARP }, + [FILTER_2D_8TAP_SHARP] = { DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SHARP }, + [FILTER_2D_8TAP_SMOOTH_REGULAR] = { DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SMOOTH }, + [FILTER_2D_8TAP_SMOOTH] = { DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SMOOTH }, + [FILTER_2D_8TAP_SMOOTH_SHARP] = { DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SMOOTH }, + [FILTER_2D_BILINEAR] = { DAV1D_FILTER_BILINEAR, DAV1D_FILTER_BILINEAR }, +}; + +const uint8_t dav1d_filter_mode_to_y_mode[5] = { + DC_PRED, VERT_PRED, HOR_PRED, HOR_DOWN_PRED, DC_PRED +}; + +const uint8_t dav1d_intra_mode_context[N_INTRA_PRED_MODES] = { + [DC_PRED] = 0, + [VERT_PRED] = 1, + [HOR_PRED] = 2, + [DIAG_DOWN_LEFT_PRED] = 3, + [DIAG_DOWN_RIGHT_PRED] = 4, + [VERT_RIGHT_PRED] = 4, + [HOR_DOWN_PRED] = 4, + [HOR_UP_PRED] = 4, + [VERT_LEFT_PRED] = 3, + [SMOOTH_PRED] = 0, + [SMOOTH_V_PRED] = 1, + [SMOOTH_H_PRED] = 2, + [PAETH_PRED] = 0, +}; + +const uint8_t dav1d_wedge_ctx_lut[N_BS_SIZES] = { + [BS_32x32] = 6, + [BS_32x16] = 5, + [BS_32x8] = 8, + [BS_16x32] = 4, + [BS_16x16] = 3, + [BS_16x8] = 2, + [BS_8x32] = 7, + [BS_8x16] = 1, + [BS_8x8] = 0, +}; + +const Dav1dWarpedMotionParams dav1d_default_wm_params = { + .type = DAV1D_WM_TYPE_IDENTITY, + .matrix = { + 0, 0, 1 << 16, + 0, 0, 1 << 16, + }, + .u.p.alpha = 0, + .u.p.beta = 0, + .u.p.gamma = 0, + .u.p.delta = 0, +}; + +const int8_t dav1d_cdef_directions[2 + 8 + 2 /* dir */][2 /* pass */] = { + { 1 * 12 + 0, 2 * 12 + 0 }, // 6 + { 1 * 12 + 0, 2 * 12 - 1 }, // 7 + { -1 * 12 + 1, -2 * 12 + 2 }, // 0 + { 0 * 12 + 1, -1 * 12 + 2 }, // 1 + { 0 * 12 + 1, 0 * 12 + 2 }, // 2 + { 0 * 12 + 1, 1 * 12 + 2 }, // 3 + { 1 * 12 + 1, 2 * 12 + 2 }, // 4 + { 1 * 12 + 0, 2 * 12 + 1 }, // 5 + { 1 * 12 + 0, 2 * 12 + 0 }, // 6 + { 1 * 12 + 0, 2 * 12 - 1 }, // 7 + { -1 * 12 + 1, -2 * 12 + 2 }, // 0 + { 0 * 12 + 1, -1 * 12 + 2 }, // 1 +}; + +const uint16_t ALIGN(dav1d_sgr_params[16][2], 4) = { + { 140, 3236 }, { 112, 2158 }, { 93, 1618 }, { 80, 1438 }, + { 70, 1295 }, { 58, 1177 }, { 47, 1079 }, { 37, 996 }, + { 30, 925 }, { 25, 863 }, { 0, 2589 }, { 0, 1618 }, + { 0, 1177 }, { 0, 925 }, { 56, 0 }, { 22, 0 }, +}; + +const uint8_t ALIGN(dav1d_sgr_x_by_x[256], 64) = { + 255, 128, 85, 64, 51, 43, 37, 32, 28, 26, 23, 21, 20, 18, 17, + 16, 15, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 9, 9, + 8, 8, 8, 8, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, + 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0 +}; + +const int8_t ALIGN(dav1d_mc_subpel_filters[6][15][8], 8) = { + [DAV1D_FILTER_8TAP_REGULAR] = { + { 0, 1, -3, 63, 4, -1, 0, 0 }, + { 0, 1, -5, 61, 9, -2, 0, 0 }, + { 0, 1, -6, 58, 14, -4, 1, 0 }, + { 0, 1, -7, 55, 19, -5, 1, 0 }, + { 0, 1, -7, 51, 24, -6, 1, 0 }, + { 0, 1, -8, 47, 29, -6, 1, 0 }, + { 0, 1, -7, 42, 33, -6, 1, 0 }, + { 0, 1, -7, 38, 38, -7, 1, 0 }, + { 0, 1, -6, 33, 42, -7, 1, 0 }, + { 0, 1, -6, 29, 47, -8, 1, 0 }, + { 0, 1, -6, 24, 51, -7, 1, 0 }, + { 0, 1, -5, 19, 55, -7, 1, 0 }, + { 0, 1, -4, 14, 58, -6, 1, 0 }, + { 0, 0, -2, 9, 61, -5, 1, 0 }, + { 0, 0, -1, 4, 63, -3, 1, 0 } + }, [DAV1D_FILTER_8TAP_SMOOTH] = { + { 0, 1, 14, 31, 17, 1, 0, 0 }, + { 0, 0, 13, 31, 18, 2, 0, 0 }, + { 0, 0, 11, 31, 20, 2, 0, 0 }, + { 0, 0, 10, 30, 21, 3, 0, 0 }, + { 0, 0, 9, 29, 22, 4, 0, 0 }, + { 0, 0, 8, 28, 23, 5, 0, 0 }, + { 0, -1, 8, 27, 24, 6, 0, 0 }, + { 0, -1, 7, 26, 26, 7, -1, 0 }, + { 0, 0, 6, 24, 27, 8, -1, 0 }, + { 0, 0, 5, 23, 28, 8, 0, 0 }, + { 0, 0, 4, 22, 29, 9, 0, 0 }, + { 0, 0, 3, 21, 30, 10, 0, 0 }, + { 0, 0, 2, 20, 31, 11, 0, 0 }, + { 0, 0, 2, 18, 31, 13, 0, 0 }, + { 0, 0, 1, 17, 31, 14, 1, 0 } + }, [DAV1D_FILTER_8TAP_SHARP] = { + { -1, 1, -3, 63, 4, -1, 1, 0 }, + { -1, 3, -6, 62, 8, -3, 2, -1 }, + { -1, 4, -9, 60, 13, -5, 3, -1 }, + { -2, 5, -11, 58, 19, -7, 3, -1 }, + { -2, 5, -11, 54, 24, -9, 4, -1 }, + { -2, 5, -12, 50, 30, -10, 4, -1 }, + { -2, 5, -12, 45, 35, -11, 5, -1 }, + { -2, 6, -12, 40, 40, -12, 6, -2 }, + { -1, 5, -11, 35, 45, -12, 5, -2 }, + { -1, 4, -10, 30, 50, -12, 5, -2 }, + { -1, 4, -9, 24, 54, -11, 5, -2 }, + { -1, 3, -7, 19, 58, -11, 5, -2 }, + { -1, 3, -5, 13, 60, -9, 4, -1 }, + { -1, 2, -3, 8, 62, -6, 3, -1 }, + { 0, 1, -1, 4, 63, -3, 1, -1 } + /* width <= 4 */ + }, [3 + DAV1D_FILTER_8TAP_REGULAR] = { + { 0, 0, -2, 63, 4, -1, 0, 0 }, + { 0, 0, -4, 61, 9, -2, 0, 0 }, + { 0, 0, -5, 58, 14, -3, 0, 0 }, + { 0, 0, -6, 55, 19, -4, 0, 0 }, + { 0, 0, -6, 51, 24, -5, 0, 0 }, + { 0, 0, -7, 47, 29, -5, 0, 0 }, + { 0, 0, -6, 42, 33, -5, 0, 0 }, + { 0, 0, -6, 38, 38, -6, 0, 0 }, + { 0, 0, -5, 33, 42, -6, 0, 0 }, + { 0, 0, -5, 29, 47, -7, 0, 0 }, + { 0, 0, -5, 24, 51, -6, 0, 0 }, + { 0, 0, -4, 19, 55, -6, 0, 0 }, + { 0, 0, -3, 14, 58, -5, 0, 0 }, + { 0, 0, -2, 9, 61, -4, 0, 0 }, + { 0, 0, -1, 4, 63, -2, 0, 0 } + }, [3 + DAV1D_FILTER_8TAP_SMOOTH] = { + { 0, 0, 15, 31, 17, 1, 0, 0 }, + { 0, 0, 13, 31, 18, 2, 0, 0 }, + { 0, 0, 11, 31, 20, 2, 0, 0 }, + { 0, 0, 10, 30, 21, 3, 0, 0 }, + { 0, 0, 9, 29, 22, 4, 0, 0 }, + { 0, 0, 8, 28, 23, 5, 0, 0 }, + { 0, 0, 7, 27, 24, 6, 0, 0 }, + { 0, 0, 6, 26, 26, 6, 0, 0 }, + { 0, 0, 6, 24, 27, 7, 0, 0 }, + { 0, 0, 5, 23, 28, 8, 0, 0 }, + { 0, 0, 4, 22, 29, 9, 0, 0 }, + { 0, 0, 3, 21, 30, 10, 0, 0 }, + { 0, 0, 2, 20, 31, 11, 0, 0 }, + { 0, 0, 2, 18, 31, 13, 0, 0 }, + { 0, 0, 1, 17, 31, 15, 0, 0 } + /* Bilin scaled being very rarely used, add a new table entry + * and use the put/prep_8tap_scaled code, thus acting as a + * scaled bilinear filter. */ + }, [5] = { + { 0, 0, 0, 60, 4, 0, 0, 0 }, + { 0, 0, 0, 56, 8, 0, 0, 0 }, + { 0, 0, 0, 52, 12, 0, 0, 0 }, + { 0, 0, 0, 48, 16, 0, 0, 0 }, + { 0, 0, 0, 44, 20, 0, 0, 0 }, + { 0, 0, 0, 40, 24, 0, 0, 0 }, + { 0, 0, 0, 36, 28, 0, 0, 0 }, + { 0, 0, 0, 32, 32, 0, 0, 0 }, + { 0, 0, 0, 28, 36, 0, 0, 0 }, + { 0, 0, 0, 24, 40, 0, 0, 0 }, + { 0, 0, 0, 20, 44, 0, 0, 0 }, + { 0, 0, 0, 16, 48, 0, 0, 0 }, + { 0, 0, 0, 12, 52, 0, 0, 0 }, + { 0, 0, 0, 8, 56, 0, 0, 0 }, + { 0, 0, 0, 4, 60, 0, 0, 0 } + } +}; + +const int8_t ALIGN(dav1d_mc_warp_filter[193][8], 8) = { + // [-1, 0) + { 0, 0, 127, 1, 0, 0, 0, 0 }, { 0, -1, 127, 2, 0, 0, 0, 0 }, + { 1, -3, 127, 4, - 1, 0, 0, 0 }, { 1, -4, 126, 6, -2, 1, 0, 0 }, + { 1, -5, 126, 8, - 3, 1, 0, 0 }, { 1, -6, 125, 11, -4, 1, 0, 0 }, + { 1, -7, 124, 13, - 4, 1, 0, 0 }, { 2, -8, 123, 15, -5, 1, 0, 0 }, + { 2, -9, 122, 18, - 6, 1, 0, 0 }, { 2, -10, 121, 20, -6, 1, 0, 0 }, + { 2, -11, 120, 22, - 7, 2, 0, 0 }, { 2, -12, 119, 25, -8, 2, 0, 0 }, + { 3, -13, 117, 27, - 8, 2, 0, 0 }, { 3, -13, 116, 29, -9, 2, 0, 0 }, + { 3, -14, 114, 32, -10, 3, 0, 0 }, { 3, -15, 113, 35, -10, 2, 0, 0 }, + { 3, -15, 111, 37, -11, 3, 0, 0 }, { 3, -16, 109, 40, -11, 3, 0, 0 }, + { 3, -16, 108, 42, -12, 3, 0, 0 }, { 4, -17, 106, 45, -13, 3, 0, 0 }, + { 4, -17, 104, 47, -13, 3, 0, 0 }, { 4, -17, 102, 50, -14, 3, 0, 0 }, + { 4, -17, 100, 52, -14, 3, 0, 0 }, { 4, -18, 98, 55, -15, 4, 0, 0 }, + { 4, -18, 96, 58, -15, 3, 0, 0 }, { 4, -18, 94, 60, -16, 4, 0, 0 }, + { 4, -18, 91, 63, -16, 4, 0, 0 }, { 4, -18, 89, 65, -16, 4, 0, 0 }, + { 4, -18, 87, 68, -17, 4, 0, 0 }, { 4, -18, 85, 70, -17, 4, 0, 0 }, + { 4, -18, 82, 73, -17, 4, 0, 0 }, { 4, -18, 80, 75, -17, 4, 0, 0 }, + { 4, -18, 78, 78, -18, 4, 0, 0 }, { 4, -17, 75, 80, -18, 4, 0, 0 }, + { 4, -17, 73, 82, -18, 4, 0, 0 }, { 4, -17, 70, 85, -18, 4, 0, 0 }, + { 4, -17, 68, 87, -18, 4, 0, 0 }, { 4, -16, 65, 89, -18, 4, 0, 0 }, + { 4, -16, 63, 91, -18, 4, 0, 0 }, { 4, -16, 60, 94, -18, 4, 0, 0 }, + { 3, -15, 58, 96, -18, 4, 0, 0 }, { 4, -15, 55, 98, -18, 4, 0, 0 }, + { 3, -14, 52, 100, -17, 4, 0, 0 }, { 3, -14, 50, 102, -17, 4, 0, 0 }, + { 3, -13, 47, 104, -17, 4, 0, 0 }, { 3, -13, 45, 106, -17, 4, 0, 0 }, + { 3, -12, 42, 108, -16, 3, 0, 0 }, { 3, -11, 40, 109, -16, 3, 0, 0 }, + { 3, -11, 37, 111, -15, 3, 0, 0 }, { 2, -10, 35, 113, -15, 3, 0, 0 }, + { 3, -10, 32, 114, -14, 3, 0, 0 }, { 2, - 9, 29, 116, -13, 3, 0, 0 }, + { 2, -8, 27, 117, -13, 3, 0, 0 }, { 2, - 8, 25, 119, -12, 2, 0, 0 }, + { 2, -7, 22, 120, -11, 2, 0, 0 }, { 1, - 6, 20, 121, -10, 2, 0, 0 }, + { 1, -6, 18, 122, - 9, 2, 0, 0 }, { 1, - 5, 15, 123, - 8, 2, 0, 0 }, + { 1, -4, 13, 124, - 7, 1, 0, 0 }, { 1, - 4, 11, 125, - 6, 1, 0, 0 }, + { 1, -3, 8, 126, - 5, 1, 0, 0 }, { 1, - 2, 6, 126, - 4, 1, 0, 0 }, + { 0, -1, 4, 127, - 3, 1, 0, 0 }, { 0, 0, 2, 127, - 1, 0, 0, 0 }, + // [0, 1) + { 0, 0, 0, 127, 1, 0, 0, 0 }, { 0, 0, -1, 127, 2, 0, 0, 0 }, + { 0, 1, -3, 127, 4, -2, 1, 0 }, { 0, 1, -5, 127, 6, -2, 1, 0 }, + { 0, 2, -6, 126, 8, -3, 1, 0 }, { -1, 2, -7, 126, 11, -4, 2, -1 }, + { -1, 3, -8, 125, 13, -5, 2, -1 }, { -1, 3, -10, 124, 16, -6, 3, -1 }, + { -1, 4, -11, 123, 18, -7, 3, -1 }, { -1, 4, -12, 122, 20, -7, 3, -1 }, + { -1, 4, -13, 121, 23, -8, 3, -1 }, { -2, 5, -14, 120, 25, -9, 4, -1 }, + { -1, 5, -15, 119, 27, -10, 4, -1 }, { -1, 5, -16, 118, 30, -11, 4, -1 }, + { -2, 6, -17, 116, 33, -12, 5, -1 }, { -2, 6, -17, 114, 35, -12, 5, -1 }, + { -2, 6, -18, 113, 38, -13, 5, -1 }, { -2, 7, -19, 111, 41, -14, 6, -2 }, + { -2, 7, -19, 110, 43, -15, 6, -2 }, { -2, 7, -20, 108, 46, -15, 6, -2 }, + { -2, 7, -20, 106, 49, -16, 6, -2 }, { -2, 7, -21, 104, 51, -16, 7, -2 }, + { -2, 7, -21, 102, 54, -17, 7, -2 }, { -2, 8, -21, 100, 56, -18, 7, -2 }, + { -2, 8, -22, 98, 59, -18, 7, -2 }, { -2, 8, -22, 96, 62, -19, 7, -2 }, + { -2, 8, -22, 94, 64, -19, 7, -2 }, { -2, 8, -22, 91, 67, -20, 8, -2 }, + { -2, 8, -22, 89, 69, -20, 8, -2 }, { -2, 8, -22, 87, 72, -21, 8, -2 }, + { -2, 8, -21, 84, 74, -21, 8, -2 }, { -2, 8, -22, 82, 77, -21, 8, -2 }, + { -2, 8, -21, 79, 79, -21, 8, -2 }, { -2, 8, -21, 77, 82, -22, 8, -2 }, + { -2, 8, -21, 74, 84, -21, 8, -2 }, { -2, 8, -21, 72, 87, -22, 8, -2 }, + { -2, 8, -20, 69, 89, -22, 8, -2 }, { -2, 8, -20, 67, 91, -22, 8, -2 }, + { -2, 7, -19, 64, 94, -22, 8, -2 }, { -2, 7, -19, 62, 96, -22, 8, -2 }, + { -2, 7, -18, 59, 98, -22, 8, -2 }, { -2, 7, -18, 56, 100, -21, 8, -2 }, + { -2, 7, -17, 54, 102, -21, 7, -2 }, { -2, 7, -16, 51, 104, -21, 7, -2 }, + { -2, 6, -16, 49, 106, -20, 7, -2 }, { -2, 6, -15, 46, 108, -20, 7, -2 }, + { -2, 6, -15, 43, 110, -19, 7, -2 }, { -2, 6, -14, 41, 111, -19, 7, -2 }, + { -1, 5, -13, 38, 113, -18, 6, -2 }, { -1, 5, -12, 35, 114, -17, 6, -2 }, + { -1, 5, -12, 33, 116, -17, 6, -2 }, { -1, 4, -11, 30, 118, -16, 5, -1 }, + { -1, 4, -10, 27, 119, -15, 5, -1 }, { -1, 4, -9, 25, 120, -14, 5, -2 }, + { -1, 3, -8, 23, 121, -13, 4, -1 }, { -1, 3, -7, 20, 122, -12, 4, -1 }, + { -1, 3, -7, 18, 123, -11, 4, -1 }, { -1, 3, -6, 16, 124, -10, 3, -1 }, + { -1, 2, -5, 13, 125, -8, 3, -1 }, { -1, 2, -4, 11, 126, -7, 2, -1 }, + { 0, 1, -3, 8, 126, -6, 2, 0 }, { 0, 1, -2, 6, 127, -5, 1, 0 }, + { 0, 1, -2, 4, 127, -3, 1, 0 }, { 0, 0, 0, 2, 127, -1, 0, 0 }, + // [1, 2) + { 0, 0, 0, 1, 127, 0, 0, 0 }, { 0, 0, 0, -1, 127, 2, 0, 0 }, + { 0, 0, 1, -3, 127, 4, -1, 0 }, { 0, 0, 1, -4, 126, 6, -2, 1 }, + { 0, 0, 1, -5, 126, 8, -3, 1 }, { 0, 0, 1, -6, 125, 11, -4, 1 }, + { 0, 0, 1, -7, 124, 13, -4, 1 }, { 0, 0, 2, -8, 123, 15, -5, 1 }, + { 0, 0, 2, -9, 122, 18, -6, 1 }, { 0, 0, 2, -10, 121, 20, -6, 1 }, + { 0, 0, 2, -11, 120, 22, -7, 2 }, { 0, 0, 2, -12, 119, 25, -8, 2 }, + { 0, 0, 3, -13, 117, 27, -8, 2 }, { 0, 0, 3, -13, 116, 29, -9, 2 }, + { 0, 0, 3, -14, 114, 32, -10, 3 }, { 0, 0, 3, -15, 113, 35, -10, 2 }, + { 0, 0, 3, -15, 111, 37, -11, 3 }, { 0, 0, 3, -16, 109, 40, -11, 3 }, + { 0, 0, 3, -16, 108, 42, -12, 3 }, { 0, 0, 4, -17, 106, 45, -13, 3 }, + { 0, 0, 4, -17, 104, 47, -13, 3 }, { 0, 0, 4, -17, 102, 50, -14, 3 }, + { 0, 0, 4, -17, 100, 52, -14, 3 }, { 0, 0, 4, -18, 98, 55, -15, 4 }, + { 0, 0, 4, -18, 96, 58, -15, 3 }, { 0, 0, 4, -18, 94, 60, -16, 4 }, + { 0, 0, 4, -18, 91, 63, -16, 4 }, { 0, 0, 4, -18, 89, 65, -16, 4 }, + { 0, 0, 4, -18, 87, 68, -17, 4 }, { 0, 0, 4, -18, 85, 70, -17, 4 }, + { 0, 0, 4, -18, 82, 73, -17, 4 }, { 0, 0, 4, -18, 80, 75, -17, 4 }, + { 0, 0, 4, -18, 78, 78, -18, 4 }, { 0, 0, 4, -17, 75, 80, -18, 4 }, + { 0, 0, 4, -17, 73, 82, -18, 4 }, { 0, 0, 4, -17, 70, 85, -18, 4 }, + { 0, 0, 4, -17, 68, 87, -18, 4 }, { 0, 0, 4, -16, 65, 89, -18, 4 }, + { 0, 0, 4, -16, 63, 91, -18, 4 }, { 0, 0, 4, -16, 60, 94, -18, 4 }, + { 0, 0, 3, -15, 58, 96, -18, 4 }, { 0, 0, 4, -15, 55, 98, -18, 4 }, + { 0, 0, 3, -14, 52, 100, -17, 4 }, { 0, 0, 3, -14, 50, 102, -17, 4 }, + { 0, 0, 3, -13, 47, 104, -17, 4 }, { 0, 0, 3, -13, 45, 106, -17, 4 }, + { 0, 0, 3, -12, 42, 108, -16, 3 }, { 0, 0, 3, -11, 40, 109, -16, 3 }, + { 0, 0, 3, -11, 37, 111, -15, 3 }, { 0, 0, 2, -10, 35, 113, -15, 3 }, + { 0, 0, 3, -10, 32, 114, -14, 3 }, { 0, 0, 2, -9, 29, 116, -13, 3 }, + { 0, 0, 2, -8, 27, 117, -13, 3 }, { 0, 0, 2, -8, 25, 119, -12, 2 }, + { 0, 0, 2, -7, 22, 120, -11, 2 }, { 0, 0, 1, -6, 20, 121, -10, 2 }, + { 0, 0, 1, -6, 18, 122, -9, 2 }, { 0, 0, 1, -5, 15, 123, -8, 2 }, + { 0, 0, 1, -4, 13, 124, -7, 1 }, { 0, 0, 1, -4, 11, 125, -6, 1 }, + { 0, 0, 1, -3, 8, 126, -5, 1 }, { 0, 0, 1, -2, 6, 126, -4, 1 }, + { 0, 0, 0, -1, 4, 127, -3, 1 }, { 0, 0, 0, 0, 2, 127, -1, 0 }, + // dummy (replicate row index 191) + { 0, 0, 0, 0, 2, 127, -1, 0 }, +}; + +const int8_t ALIGN(dav1d_resize_filter[64][8], 8) = { + { 0, 0, 0, -128, 0, 0, 0, 0 }, { 0, 0, 1, -128, -2, 1, 0, 0 }, + { 0, -1, 3, -127, -4, 2, -1, 0 }, { 0, -1, 4, -127, -6, 3, -1, 0 }, + { 0, -2, 6, -126, -8, 3, -1, 0 }, { 0, -2, 7, -125, -11, 4, -1, 0 }, + { 1, -2, 8, -125, -13, 5, -2, 0 }, { 1, -3, 9, -124, -15, 6, -2, 0 }, + { 1, -3, 10, -123, -18, 6, -2, 1 }, { 1, -3, 11, -122, -20, 7, -3, 1 }, + { 1, -4, 12, -121, -22, 8, -3, 1 }, { 1, -4, 13, -120, -25, 9, -3, 1 }, + { 1, -4, 14, -118, -28, 9, -3, 1 }, { 1, -4, 15, -117, -30, 10, -4, 1 }, + { 1, -5, 16, -116, -32, 11, -4, 1 }, { 1, -5, 16, -114, -35, 12, -4, 1 }, + { 1, -5, 17, -112, -38, 12, -4, 1 }, { 1, -5, 18, -111, -40, 13, -5, 1 }, + { 1, -5, 18, -109, -43, 14, -5, 1 }, { 1, -6, 19, -107, -45, 14, -5, 1 }, + { 1, -6, 19, -105, -48, 15, -5, 1 }, { 1, -6, 19, -103, -51, 16, -5, 1 }, + { 1, -6, 20, -101, -53, 16, -6, 1 }, { 1, -6, 20, -99, -56, 17, -6, 1 }, + { 1, -6, 20, -97, -58, 17, -6, 1 }, { 1, -6, 20, -95, -61, 18, -6, 1 }, + { 2, -7, 20, -93, -64, 18, -6, 2 }, { 2, -7, 20, -91, -66, 19, -6, 1 }, + { 2, -7, 20, -88, -69, 19, -6, 1 }, { 2, -7, 20, -86, -71, 19, -6, 1 }, + { 2, -7, 20, -84, -74, 20, -7, 2 }, { 2, -7, 20, -81, -76, 20, -7, 1 }, + { 2, -7, 20, -79, -79, 20, -7, 2 }, { 1, -7, 20, -76, -81, 20, -7, 2 }, + { 2, -7, 20, -74, -84, 20, -7, 2 }, { 1, -6, 19, -71, -86, 20, -7, 2 }, + { 1, -6, 19, -69, -88, 20, -7, 2 }, { 1, -6, 19, -66, -91, 20, -7, 2 }, + { 2, -6, 18, -64, -93, 20, -7, 2 }, { 1, -6, 18, -61, -95, 20, -6, 1 }, + { 1, -6, 17, -58, -97, 20, -6, 1 }, { 1, -6, 17, -56, -99, 20, -6, 1 }, + { 1, -6, 16, -53, -101, 20, -6, 1 }, { 1, -5, 16, -51, -103, 19, -6, 1 }, + { 1, -5, 15, -48, -105, 19, -6, 1 }, { 1, -5, 14, -45, -107, 19, -6, 1 }, + { 1, -5, 14, -43, -109, 18, -5, 1 }, { 1, -5, 13, -40, -111, 18, -5, 1 }, + { 1, -4, 12, -38, -112, 17, -5, 1 }, { 1, -4, 12, -35, -114, 16, -5, 1 }, + { 1, -4, 11, -32, -116, 16, -5, 1 }, { 1, -4, 10, -30, -117, 15, -4, 1 }, + { 1, -3, 9, -28, -118, 14, -4, 1 }, { 1, -3, 9, -25, -120, 13, -4, 1 }, + { 1, -3, 8, -22, -121, 12, -4, 1 }, { 1, -3, 7, -20, -122, 11, -3, 1 }, + { 1, -2, 6, -18, -123, 10, -3, 1 }, { 0, -2, 6, -15, -124, 9, -3, 1 }, + { 0, -2, 5, -13, -125, 8, -2, 1 }, { 0, -1, 4, -11, -125, 7, -2, 0 }, + { 0, -1, 3, -8, -126, 6, -2, 0 }, { 0, -1, 3, -6, -127, 4, -1, 0 }, + { 0, -1, 2, -4, -127, 3, -1, 0 }, { 0, 0, 1, -2, -128, 1, 0, 0 }, +}; + +const uint8_t ALIGN(dav1d_sm_weights[128], 16) = { + // Unused, because we always offset by bs, which is at least 2. + 0, 0, + // bs = 2 + 255, 128, + // bs = 4 + 255, 149, 85, 64, + // bs = 8 + 255, 197, 146, 105, 73, 50, 37, 32, + // bs = 16 + 255, 225, 196, 170, 145, 123, 102, 84, + 68, 54, 43, 33, 26, 20, 17, 16, + // bs = 32 + 255, 240, 225, 210, 196, 182, 169, 157, + 145, 133, 122, 111, 101, 92, 83, 74, + 66, 59, 52, 45, 39, 34, 29, 25, + 21, 17, 14, 12, 10, 9, 8, 8, + // bs = 64 + 255, 248, 240, 233, 225, 218, 210, 203, + 196, 189, 182, 176, 169, 163, 156, 150, + 144, 138, 133, 127, 121, 116, 111, 106, + 101, 96, 91, 86, 82, 77, 73, 69, + 65, 61, 57, 54, 50, 47, 44, 41, + 38, 35, 32, 29, 27, 25, 22, 20, + 18, 16, 15, 13, 12, 10, 9, 8, + 7, 6, 6, 5, 5, 4, 4, 4 +}; + +const uint16_t dav1d_dr_intra_derivative[44] = { + // Values that are 0 will never be used + 0, // Angles: + 1023, 0, // 3, 93, 183 + 547, // 6, 96, 186 + 372, 0, 0, // 9, 99, 189 + 273, // 14, 104, 194 + 215, 0, // 17, 107, 197 + 178, // 20, 110, 200 + 151, 0, // 23, 113, 203 (113 & 203 are base angles) + 132, // 26, 116, 206 + 116, 0, // 29, 119, 209 + 102, 0, // 32, 122, 212 + 90, // 36, 126, 216 + 80, 0, // 39, 129, 219 + 71, // 42, 132, 222 + 64, 0, // 45, 135, 225 (45 & 135 are base angles) + 57, // 48, 138, 228 + 51, 0, // 51, 141, 231 + 45, 0, // 54, 144, 234 + 40, // 58, 148, 238 + 35, 0, // 61, 151, 241 + 31, // 64, 154, 244 + 27, 0, // 67, 157, 247 (67 & 157 are base angles) + 23, // 70, 160, 250 + 19, 0, // 73, 163, 253 + 15, 0, // 76, 166, 256 + 11, 0, // 81, 171, 261 + 7, // 84, 174, 264 + 3 // 87, 177, 267 +}; + +#if ARCH_X86 +#define F(idx, f0, f1, f2, f3, f4, f5, f6) \ + [2*idx+0] = f0, [2*idx+1] = f1, \ + [2*idx+16] = f2, [2*idx+17] = f3, \ + [2*idx+32] = f4, [2*idx+33] = f5, \ + [2*idx+48] = f6 +#else +#define F(idx, f0, f1, f2, f3, f4, f5, f6) \ + [1*idx+0] = f0, [1*idx+8] = f1, \ + [1*idx+16] = f2, [1*idx+24] = f3, \ + [1*idx+32] = f4, [1*idx+40] = f5, \ + [1*idx+48] = f6 +#endif +const int8_t ALIGN(dav1d_filter_intra_taps[5][64], 64) = { + { + F( 0, -6, 10, 0, 0, 0, 12, 0 ), + F( 1, -5, 2, 10, 0, 0, 9, 0 ), + F( 2, -3, 1, 1, 10, 0, 7, 0 ), + F( 3, -3, 1, 1, 2, 10, 5, 0 ), + F( 4, -4, 6, 0, 0, 0, 2, 12 ), + F( 5, -3, 2, 6, 0, 0, 2, 9 ), + F( 6, -3, 2, 2, 6, 0, 2, 7 ), + F( 7, -3, 1, 2, 2, 6, 3, 5 ), + }, { + F( 0, -10, 16, 0, 0, 0, 10, 0 ), + F( 1, -6, 0, 16, 0, 0, 6, 0 ), + F( 2, -4, 0, 0, 16, 0, 4, 0 ), + F( 3, -2, 0, 0, 0, 16, 2, 0 ), + F( 4, -10, 16, 0, 0, 0, 0, 10 ), + F( 5, -6, 0, 16, 0, 0, 0, 6 ), + F( 6, -4, 0, 0, 16, 0, 0, 4 ), + F( 7, -2, 0, 0, 0, 16, 0, 2 ), + }, { + F( 0, -8, 8, 0, 0, 0, 16, 0 ), + F( 1, -8, 0, 8, 0, 0, 16, 0 ), + F( 2, -8, 0, 0, 8, 0, 16, 0 ), + F( 3, -8, 0, 0, 0, 8, 16, 0 ), + F( 4, -4, 4, 0, 0, 0, 0, 16 ), + F( 5, -4, 0, 4, 0, 0, 0, 16 ), + F( 6, -4, 0, 0, 4, 0, 0, 16 ), + F( 7, -4, 0, 0, 0, 4, 0, 16 ), + }, { + F( 0, -2, 8, 0, 0, 0, 10, 0 ), + F( 1, -1, 3, 8, 0, 0, 6, 0 ), + F( 2, -1, 2, 3, 8, 0, 4, 0 ), + F( 3, 0, 1, 2, 3, 8, 2, 0 ), + F( 4, -1, 4, 0, 0, 0, 3, 10 ), + F( 5, -1, 3, 4, 0, 0, 4, 6 ), + F( 6, -1, 2, 3, 4, 0, 4, 4 ), + F( 7, -1, 2, 2, 3, 4, 3, 3 ), + }, { + F( 0, -12, 14, 0, 0, 0, 14, 0 ), + F( 1, -10, 0, 14, 0, 0, 12, 0 ), + F( 2, -9, 0, 0, 14, 0, 11, 0 ), + F( 3, -8, 0, 0, 0, 14, 10, 0 ), + F( 4, -10, 12, 0, 0, 0, 0, 14 ), + F( 5, -9, 1, 12, 0, 0, 0, 12 ), + F( 6, -8, 0, 0, 12, 0, 1, 11 ), + F( 7, -7, 0, 0, 1, 12, 1, 9 ), + } +}; + +const uint8_t ALIGN(dav1d_obmc_masks[64], 16) = { + /* Unused */ + 0, 0, + /* 2 */ + 19, 0, + /* 4 */ + 25, 14, 5, 0, + /* 8 */ + 28, 22, 16, 11, 7, 3, 0, 0, + /* 16 */ + 30, 27, 24, 21, 18, 15, 12, 10, 8, 6, 4, 3, 0, 0, 0, 0, + /* 32 */ + 31, 29, 28, 26, 24, 23, 21, 20, 19, 17, 16, 14, 13, 12, 11, 9, + 8, 7, 6, 5, 4, 4, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// Taken from the spec. Range is [-2048, 2047], mean is 0 and stddev is 512 +const int16_t dav1d_gaussian_sequence[2048] = { + 56, 568, -180, 172, 124, -84, 172, -64, -900, 24, 820, + 224, 1248, 996, 272, -8, -916, -388, -732, -104, -188, 800, + 112, -652, -320, -376, 140, -252, 492, -168, 44, -788, 588, + -584, 500, -228, 12, 680, 272, -476, 972, -100, 652, 368, + 432, -196, -720, -192, 1000, -332, 652, -136, -552, -604, -4, + 192, -220, -136, 1000, -52, 372, -96, -624, 124, -24, 396, + 540, -12, -104, 640, 464, 244, -208, -84, 368, -528, -740, + 248, -968, -848, 608, 376, -60, -292, -40, -156, 252, -292, + 248, 224, -280, 400, -244, 244, -60, 76, -80, 212, 532, + 340, 128, -36, 824, -352, -60, -264, -96, -612, 416, -704, + 220, -204, 640, -160, 1220, -408, 900, 336, 20, -336, -96, + -792, 304, 48, -28, -1232, -1172, -448, 104, -292, -520, 244, + 60, -948, 0, -708, 268, 108, 356, -548, 488, -344, -136, + 488, -196, -224, 656, -236, -1128, 60, 4, 140, 276, -676, + -376, 168, -108, 464, 8, 564, 64, 240, 308, -300, -400, + -456, -136, 56, 120, -408, -116, 436, 504, -232, 328, 844, + -164, -84, 784, -168, 232, -224, 348, -376, 128, 568, 96, + -1244, -288, 276, 848, 832, -360, 656, 464, -384, -332, -356, + 728, -388, 160, -192, 468, 296, 224, 140, -776, -100, 280, + 4, 196, 44, -36, -648, 932, 16, 1428, 28, 528, 808, + 772, 20, 268, 88, -332, -284, 124, -384, -448, 208, -228, + -1044, -328, 660, 380, -148, -300, 588, 240, 540, 28, 136, + -88, -436, 256, 296, -1000, 1400, 0, -48, 1056, -136, 264, + -528, -1108, 632, -484, -592, -344, 796, 124, -668, -768, 388, + 1296, -232, -188, -200, -288, -4, 308, 100, -168, 256, -500, + 204, -508, 648, -136, 372, -272, -120, -1004, -552, -548, -384, + 548, -296, 428, -108, -8, -912, -324, -224, -88, -112, -220, + -100, 996, -796, 548, 360, -216, 180, 428, -200, -212, 148, + 96, 148, 284, 216, -412, -320, 120, -300, -384, -604, -572, + -332, -8, -180, -176, 696, 116, -88, 628, 76, 44, -516, + 240, -208, -40, 100, -592, 344, -308, -452, -228, 20, 916, + -1752, -136, -340, -804, 140, 40, 512, 340, 248, 184, -492, + 896, -156, 932, -628, 328, -688, -448, -616, -752, -100, 560, + -1020, 180, -800, -64, 76, 576, 1068, 396, 660, 552, -108, + -28, 320, -628, 312, -92, -92, -472, 268, 16, 560, 516, + -672, -52, 492, -100, 260, 384, 284, 292, 304, -148, 88, + -152, 1012, 1064, -228, 164, -376, -684, 592, -392, 156, 196, + -524, -64, -884, 160, -176, 636, 648, 404, -396, -436, 864, + 424, -728, 988, -604, 904, -592, 296, -224, 536, -176, -920, + 436, -48, 1176, -884, 416, -776, -824, -884, 524, -548, -564, + -68, -164, -96, 692, 364, -692, -1012, -68, 260, -480, 876, + -1116, 452, -332, -352, 892, -1088, 1220, -676, 12, -292, 244, + 496, 372, -32, 280, 200, 112, -440, -96, 24, -644, -184, + 56, -432, 224, -980, 272, -260, 144, -436, 420, 356, 364, + -528, 76, 172, -744, -368, 404, -752, -416, 684, -688, 72, + 540, 416, 92, 444, 480, -72, -1416, 164, -1172, -68, 24, + 424, 264, 1040, 128, -912, -524, -356, 64, 876, -12, 4, + -88, 532, 272, -524, 320, 276, -508, 940, 24, -400, -120, + 756, 60, 236, -412, 100, 376, -484, 400, -100, -740, -108, + -260, 328, -268, 224, -200, -416, 184, -604, -564, -20, 296, + 60, 892, -888, 60, 164, 68, -760, 216, -296, 904, -336, + -28, 404, -356, -568, -208, -1480, -512, 296, 328, -360, -164, + -1560, -776, 1156, -428, 164, -504, -112, 120, -216, -148, -264, + 308, 32, 64, -72, 72, 116, 176, -64, -272, 460, -536, + -784, -280, 348, 108, -752, -132, 524, -540, -776, 116, -296, + -1196, -288, -560, 1040, -472, 116, -848, -1116, 116, 636, 696, + 284, -176, 1016, 204, -864, -648, -248, 356, 972, -584, -204, + 264, 880, 528, -24, -184, 116, 448, -144, 828, 524, 212, + -212, 52, 12, 200, 268, -488, -404, -880, 824, -672, -40, + 908, -248, 500, 716, -576, 492, -576, 16, 720, -108, 384, + 124, 344, 280, 576, -500, 252, 104, -308, 196, -188, -8, + 1268, 296, 1032, -1196, 436, 316, 372, -432, -200, -660, 704, + -224, 596, -132, 268, 32, -452, 884, 104, -1008, 424, -1348, + -280, 4, -1168, 368, 476, 696, 300, -8, 24, 180, -592, + -196, 388, 304, 500, 724, -160, 244, -84, 272, -256, -420, + 320, 208, -144, -156, 156, 364, 452, 28, 540, 316, 220, + -644, -248, 464, 72, 360, 32, -388, 496, -680, -48, 208, + -116, -408, 60, -604, -392, 548, -840, 784, -460, 656, -544, + -388, -264, 908, -800, -628, -612, -568, 572, -220, 164, 288, + -16, -308, 308, -112, -636, -760, 280, -668, 432, 364, 240, + -196, 604, 340, 384, 196, 592, -44, -500, 432, -580, -132, + 636, -76, 392, 4, -412, 540, 508, 328, -356, -36, 16, + -220, -64, -248, -60, 24, -192, 368, 1040, 92, -24, -1044, + -32, 40, 104, 148, 192, -136, -520, 56, -816, -224, 732, + 392, 356, 212, -80, -424, -1008, -324, 588, -1496, 576, 460, + -816, -848, 56, -580, -92, -1372, -112, -496, 200, 364, 52, + -140, 48, -48, -60, 84, 72, 40, 132, -356, -268, -104, + -284, -404, 732, -520, 164, -304, -540, 120, 328, -76, -460, + 756, 388, 588, 236, -436, -72, -176, -404, -316, -148, 716, + -604, 404, -72, -88, -888, -68, 944, 88, -220, -344, 960, + 472, 460, -232, 704, 120, 832, -228, 692, -508, 132, -476, + 844, -748, -364, -44, 1116, -1104, -1056, 76, 428, 552, -692, + 60, 356, 96, -384, -188, -612, -576, 736, 508, 892, 352, + -1132, 504, -24, -352, 324, 332, -600, -312, 292, 508, -144, + -8, 484, 48, 284, -260, -240, 256, -100, -292, -204, -44, + 472, -204, 908, -188, -1000, -256, 92, 1164, -392, 564, 356, + 652, -28, -884, 256, 484, -192, 760, -176, 376, -524, -452, + -436, 860, -736, 212, 124, 504, -476, 468, 76, -472, 552, + -692, -944, -620, 740, -240, 400, 132, 20, 192, -196, 264, + -668, -1012, -60, 296, -316, -828, 76, -156, 284, -768, -448, + -832, 148, 248, 652, 616, 1236, 288, -328, -400, -124, 588, + 220, 520, -696, 1032, 768, -740, -92, -272, 296, 448, -464, + 412, -200, 392, 440, -200, 264, -152, -260, 320, 1032, 216, + 320, -8, -64, 156, -1016, 1084, 1172, 536, 484, -432, 132, + 372, -52, -256, 84, 116, -352, 48, 116, 304, -384, 412, + 924, -300, 528, 628, 180, 648, 44, -980, -220, 1320, 48, + 332, 748, 524, -268, -720, 540, -276, 564, -344, -208, -196, + 436, 896, 88, -392, 132, 80, -964, -288, 568, 56, -48, + -456, 888, 8, 552, -156, -292, 948, 288, 128, -716, -292, + 1192, -152, 876, 352, -600, -260, -812, -468, -28, -120, -32, + -44, 1284, 496, 192, 464, 312, -76, -516, -380, -456, -1012, + -48, 308, -156, 36, 492, -156, -808, 188, 1652, 68, -120, + -116, 316, 160, -140, 352, 808, -416, 592, 316, -480, 56, + 528, -204, -568, 372, -232, 752, -344, 744, -4, 324, -416, + -600, 768, 268, -248, -88, -132, -420, -432, 80, -288, 404, + -316, -1216, -588, 520, -108, 92, -320, 368, -480, -216, -92, + 1688, -300, 180, 1020, -176, 820, -68, -228, -260, 436, -904, + 20, 40, -508, 440, -736, 312, 332, 204, 760, -372, 728, + 96, -20, -632, -520, -560, 336, 1076, -64, -532, 776, 584, + 192, 396, -728, -520, 276, -188, 80, -52, -612, -252, -48, + 648, 212, -688, 228, -52, -260, 428, -412, -272, -404, 180, + 816, -796, 48, 152, 484, -88, -216, 988, 696, 188, -528, + 648, -116, -180, 316, 476, 12, -564, 96, 476, -252, -364, + -376, -392, 556, -256, -576, 260, -352, 120, -16, -136, -260, + -492, 72, 556, 660, 580, 616, 772, 436, 424, -32, -324, + -1268, 416, -324, -80, 920, 160, 228, 724, 32, -516, 64, + 384, 68, -128, 136, 240, 248, -204, -68, 252, -932, -120, + -480, -628, -84, 192, 852, -404, -288, -132, 204, 100, 168, + -68, -196, -868, 460, 1080, 380, -80, 244, 0, 484, -888, + 64, 184, 352, 600, 460, 164, 604, -196, 320, -64, 588, + -184, 228, 12, 372, 48, -848, -344, 224, 208, -200, 484, + 128, -20, 272, -468, -840, 384, 256, -720, -520, -464, -580, + 112, -120, 644, -356, -208, -608, -528, 704, 560, -424, 392, + 828, 40, 84, 200, -152, 0, -144, 584, 280, -120, 80, + -556, -972, -196, -472, 724, 80, 168, -32, 88, 160, -688, + 0, 160, 356, 372, -776, 740, -128, 676, -248, -480, 4, + -364, 96, 544, 232, -1032, 956, 236, 356, 20, -40, 300, + 24, -676, -596, 132, 1120, -104, 532, -1096, 568, 648, 444, + 508, 380, 188, -376, -604, 1488, 424, 24, 756, -220, -192, + 716, 120, 920, 688, 168, 44, -460, 568, 284, 1144, 1160, + 600, 424, 888, 656, -356, -320, 220, 316, -176, -724, -188, + -816, -628, -348, -228, -380, 1012, -452, -660, 736, 928, 404, + -696, -72, -268, -892, 128, 184, -344, -780, 360, 336, 400, + 344, 428, 548, -112, 136, -228, -216, -820, -516, 340, 92, + -136, 116, -300, 376, -244, 100, -316, -520, -284, -12, 824, + 164, -548, -180, -128, 116, -924, -828, 268, -368, -580, 620, + 192, 160, 0, -1676, 1068, 424, -56, -360, 468, -156, 720, + 288, -528, 556, -364, 548, -148, 504, 316, 152, -648, -620, + -684, -24, -376, -384, -108, -920, -1032, 768, 180, -264, -508, + -1268, -260, -60, 300, -240, 988, 724, -376, -576, -212, -736, + 556, 192, 1092, -620, -880, 376, -56, -4, -216, -32, 836, + 268, 396, 1332, 864, -600, 100, 56, -412, -92, 356, 180, + 884, -468, -436, 292, -388, -804, -704, -840, 368, -348, 140, + -724, 1536, 940, 372, 112, -372, 436, -480, 1136, 296, -32, + -228, 132, -48, -220, 868, -1016, -60, -1044, -464, 328, 916, + 244, 12, -736, -296, 360, 468, -376, -108, -92, 788, 368, + -56, 544, 400, -672, -420, 728, 16, 320, 44, -284, -380, + -796, 488, 132, 204, -596, -372, 88, -152, -908, -636, -572, + -624, -116, -692, -200, -56, 276, -88, 484, -324, 948, 864, + 1000, -456, -184, -276, 292, -296, 156, 676, 320, 160, 908, + -84, -1236, -288, -116, 260, -372, -644, 732, -756, -96, 84, + 344, -520, 348, -688, 240, -84, 216, -1044, -136, -676, -396, + -1500, 960, -40, 176, 168, 1516, 420, -504, -344, -364, -360, + 1216, -940, -380, -212, 252, -660, -708, 484, -444, -152, 928, + -120, 1112, 476, -260, 560, -148, -344, 108, -196, 228, -288, + 504, 560, -328, -88, 288, -1008, 460, -228, 468, -836, -196, + 76, 388, 232, 412, -1168, -716, -644, 756, -172, -356, -504, + 116, 432, 528, 48, 476, -168, -608, 448, 160, -532, -272, + 28, -676, -12, 828, 980, 456, 520, 104, -104, 256, -344, + -4, -28, -368, -52, -524, -572, -556, -200, 768, 1124, -208, + -512, 176, 232, 248, -148, -888, 604, -600, -304, 804, -156, + -212, 488, -192, -804, -256, 368, -360, -916, -328, 228, -240, + -448, -472, 856, -556, -364, 572, -12, -156, -368, -340, 432, + 252, -752, -152, 288, 268, -580, -848, -592, 108, -76, 244, + 312, -716, 592, -80, 436, 360, 4, -248, 160, 516, 584, + 732, 44, -468, -280, -292, -156, -588, 28, 308, 912, 24, + 124, 156, 180, -252, 944, -924, -772, -520, -428, -624, 300, + -212, -1144, 32, -724, 800, -1128, -212, -1288, -848, 180, -416, + 440, 192, -576, -792, -76, -1080, 80, -532, -352, -132, 380, + -820, 148, 1112, 128, 164, 456, 700, -924, 144, -668, -384, + 648, -832, 508, 552, -52, -100, -656, 208, -568, 748, -88, + 680, 232, 300, 192, -408, -1012, -152, -252, -268, 272, -876, + -664, -648, -332, -136, 16, 12, 1152, -28, 332, -536, 320, + -672, -460, -316, 532, -260, 228, -40, 1052, -816, 180, 88, + -496, -556, -672, -368, 428, 92, 356, 404, -408, 252, 196, + -176, -556, 792, 268, 32, 372, 40, 96, -332, 328, 120, + 372, -900, -40, 472, -264, -592, 952, 128, 656, 112, 664, + -232, 420, 4, -344, -464, 556, 244, -416, -32, 252, 0, + -412, 188, -696, 508, -476, 324, -1096, 656, -312, 560, 264, + -136, 304, 160, -64, -580, 248, 336, -720, 560, -348, -288, + -276, -196, -500, 852, -544, -236, -1128, -992, -776, 116, 56, + 52, 860, 884, 212, -12, 168, 1020, 512, -552, 924, -148, + 716, 188, 164, -340, -520, -184, 880, -152, -680, -208, -1156, + -300, -528, -472, 364, 100, -744, -1056, -32, 540, 280, 144, + -676, -32, -232, -280, -224, 96, 568, -76, 172, 148, 148, + 104, 32, -296, -32, 788, -80, 32, -16, 280, 288, 944, + 428, -484 +}; diff --git a/third_party/dav1d/src/tables.h b/third_party/dav1d/src/tables.h new file mode 100644 index 0000000000..f3c00cfb00 --- /dev/null +++ b/third_party/dav1d/src/tables.h @@ -0,0 +1,125 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_TABLES_H +#define DAV1D_SRC_TABLES_H + +#include + +#include "common/intops.h" + +#include "src/levels.h" + +EXTERN const uint8_t dav1d_al_part_ctx[2][N_BL_LEVELS][N_PARTITIONS]; +EXTERN const uint8_t /* enum BlockSize */ + dav1d_block_sizes[N_BL_LEVELS][N_PARTITIONS][2]; +// width, height (in 4px blocks), log2 versions of these two +EXTERN const uint8_t dav1d_block_dimensions[N_BS_SIZES][4]; +typedef struct TxfmInfo { + // width, height (in 4px blocks), log2 of them, min/max of log2, sub, pad + uint8_t w, h, lw, lh, min, max, sub, ctx; +} TxfmInfo; +EXTERN const TxfmInfo dav1d_txfm_dimensions[N_RECT_TX_SIZES]; +EXTERN const uint8_t /* enum (Rect)TxfmSize */ + dav1d_max_txfm_size_for_bs[N_BS_SIZES][4 /* y, 420, 422, 444 */]; +EXTERN const uint8_t /* enum TxfmType */ + dav1d_txtp_from_uvmode[N_UV_INTRA_PRED_MODES]; + +EXTERN const uint8_t /* enum InterPredMode */ + dav1d_comp_inter_pred_modes[N_COMP_INTER_PRED_MODES][2]; + +EXTERN const uint8_t dav1d_partition_type_count[N_BL_LEVELS]; +EXTERN const uint8_t /* enum TxfmType */ dav1d_tx_types_per_set[40]; + +EXTERN const uint8_t dav1d_filter_mode_to_y_mode[5]; +EXTERN const uint8_t dav1d_ymode_size_context[N_BS_SIZES]; +EXTERN const uint8_t dav1d_lo_ctx_offsets[3][5][5]; +EXTERN const uint8_t dav1d_skip_ctx[5][5]; +EXTERN const uint8_t /* enum TxClass */ + dav1d_tx_type_class[N_TX_TYPES_PLUS_LL]; +EXTERN const uint8_t /* enum Filter2d */ + dav1d_filter_2d[DAV1D_N_FILTERS /* h */][DAV1D_N_FILTERS /* v */]; +EXTERN const uint8_t /* enum Dav1dFilterMode */ dav1d_filter_dir[N_2D_FILTERS][2]; +EXTERN const uint8_t dav1d_intra_mode_context[N_INTRA_PRED_MODES]; +EXTERN const uint8_t dav1d_wedge_ctx_lut[N_BS_SIZES]; + +static const unsigned cfl_allowed_mask = + (1 << BS_32x32) | + (1 << BS_32x16) | + (1 << BS_32x8) | + (1 << BS_16x32) | + (1 << BS_16x16) | + (1 << BS_16x8) | + (1 << BS_16x4) | + (1 << BS_8x32) | + (1 << BS_8x16) | + (1 << BS_8x8) | + (1 << BS_8x4) | + (1 << BS_4x16) | + (1 << BS_4x8) | + (1 << BS_4x4); + +static const unsigned wedge_allowed_mask = + (1 << BS_32x32) | + (1 << BS_32x16) | + (1 << BS_32x8) | + (1 << BS_16x32) | + (1 << BS_16x16) | + (1 << BS_16x8) | + (1 << BS_8x32) | + (1 << BS_8x16) | + (1 << BS_8x8); + +static const unsigned interintra_allowed_mask = + (1 << BS_32x32) | + (1 << BS_32x16) | + (1 << BS_16x32) | + (1 << BS_16x16) | + (1 << BS_16x8) | + (1 << BS_8x16) | + (1 << BS_8x8); + +EXTERN const Dav1dWarpedMotionParams dav1d_default_wm_params; + +EXTERN const int8_t dav1d_cdef_directions[12][2]; + +EXTERN const uint16_t dav1d_sgr_params[16][2]; +EXTERN const uint8_t dav1d_sgr_x_by_x[256]; + +EXTERN const int8_t dav1d_mc_subpel_filters[6][15][8]; +EXTERN const int8_t dav1d_mc_warp_filter[193][8]; +EXTERN const int8_t dav1d_resize_filter[64][8]; + +EXTERN const uint8_t dav1d_sm_weights[128]; +EXTERN const uint16_t dav1d_dr_intra_derivative[44]; +EXTERN const int8_t dav1d_filter_intra_taps[5][64]; + +EXTERN const uint8_t dav1d_obmc_masks[64]; + +EXTERN const int16_t dav1d_gaussian_sequence[2048]; // for fgs + +#endif /* DAV1D_SRC_TABLES_H */ diff --git a/third_party/dav1d/src/thread.h b/third_party/dav1d/src/thread.h new file mode 100644 index 0000000000..b091e4f26d --- /dev/null +++ b/third_party/dav1d/src/thread.h @@ -0,0 +1,188 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_THREAD_H +#define DAV1D_SRC_THREAD_H + +#if defined(_WIN32) + +#include +#include + +#define PTHREAD_ONCE_INIT INIT_ONCE_STATIC_INIT + +typedef struct { + HANDLE h; + void *(*func)(void*); + void *arg; +} pthread_t; + +typedef struct { + unsigned stack_size; +} pthread_attr_t; + +typedef SRWLOCK pthread_mutex_t; +typedef CONDITION_VARIABLE pthread_cond_t; +typedef INIT_ONCE pthread_once_t; + +void dav1d_init_thread(void); +void dav1d_set_thread_name(const wchar_t *name); +#define dav1d_set_thread_name(name) dav1d_set_thread_name(L##name) + +int dav1d_pthread_create(pthread_t *thread, const pthread_attr_t *attr, + void *(*func)(void*), void *arg); +int dav1d_pthread_join(pthread_t *thread, void **res); +int dav1d_pthread_once(pthread_once_t *once_control, + void (*init_routine)(void)); + +#define pthread_create dav1d_pthread_create +#define pthread_join(thread, res) dav1d_pthread_join(&(thread), res) +#define pthread_once dav1d_pthread_once + +static inline int pthread_attr_init(pthread_attr_t *const attr) { + attr->stack_size = 0; + return 0; +} + +static inline int pthread_attr_destroy(pthread_attr_t *const attr) { + return 0; +} + +static inline int pthread_attr_setstacksize(pthread_attr_t *const attr, + const size_t stack_size) +{ + if (stack_size > UINT_MAX) return 1; + attr->stack_size = (unsigned) stack_size; + return 0; +} + +static inline int pthread_mutex_init(pthread_mutex_t *const mutex, + const void *const attr) +{ + InitializeSRWLock(mutex); + return 0; +} + +static inline int pthread_mutex_destroy(pthread_mutex_t *const mutex) { + return 0; +} + +static inline int pthread_mutex_lock(pthread_mutex_t *const mutex) { + AcquireSRWLockExclusive(mutex); + return 0; +} + +static inline int pthread_mutex_unlock(pthread_mutex_t *const mutex) { + ReleaseSRWLockExclusive(mutex); + return 0; +} + +static inline int pthread_cond_init(pthread_cond_t *const cond, + const void *const attr) +{ + InitializeConditionVariable(cond); + return 0; +} + +static inline int pthread_cond_destroy(pthread_cond_t *const cond) { + return 0; +} + +static inline int pthread_cond_wait(pthread_cond_t *const cond, + pthread_mutex_t *const mutex) +{ + return !SleepConditionVariableSRW(cond, mutex, INFINITE, 0); +} + +static inline int pthread_cond_signal(pthread_cond_t *const cond) { + WakeConditionVariable(cond); + return 0; +} + +static inline int pthread_cond_broadcast(pthread_cond_t *const cond) { + WakeAllConditionVariable(cond); + return 0; +} + +#else + +#include + +#define dav1d_init_thread() do {} while (0) + +/* Thread naming support */ + +#ifdef __linux__ + +#include + +static inline void dav1d_set_thread_name(const char *const name) { + prctl(PR_SET_NAME, name); +} + +#elif defined(__APPLE__) + +static inline void dav1d_set_thread_name(const char *const name) { + pthread_setname_np(name); +} + +#elif defined(__DragonFly__) || defined(__FreeBSD__) || defined(__OpenBSD__) + +#if defined(__FreeBSD__) + /* ALIGN from conflicts with ALIGN from "common/attributes.h" */ +#define _SYS_PARAM_H_ +#include +#endif +#include + +static inline void dav1d_set_thread_name(const char *const name) { + pthread_set_name_np(pthread_self(), name); +} + +#elif defined(__NetBSD__) + +static inline void dav1d_set_thread_name(const char *const name) { + pthread_setname_np(pthread_self(), "%s", (void*)name); +} + +#elif defined(__HAIKU__) + +#include + +static inline void dav1d_set_thread_name(const char *const name) { + rename_thread(find_thread(NULL), name); +} + +#else + +#define dav1d_set_thread_name(name) do {} while (0) + +#endif + +#endif + +#endif /* DAV1D_SRC_THREAD_H */ diff --git a/third_party/dav1d/src/thread_data.h b/third_party/dav1d/src/thread_data.h new file mode 100644 index 0000000000..62814e6348 --- /dev/null +++ b/third_party/dav1d/src/thread_data.h @@ -0,0 +1,40 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_THREAD_DATA_H +#define DAV1D_SRC_THREAD_DATA_H + +#include "src/thread.h" + +struct thread_data { + pthread_t thread; + pthread_cond_t cond; + pthread_mutex_t lock; + int inited; +}; + +#endif /* DAV1D_SRC_THREAD_DATA_H */ diff --git a/third_party/dav1d/src/thread_task.c b/third_party/dav1d/src/thread_task.c new file mode 100644 index 0000000000..bfedf6e5bb --- /dev/null +++ b/third_party/dav1d/src/thread_task.c @@ -0,0 +1,936 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "common/frame.h" + +#include "src/thread_task.h" +#include "src/fg_apply.h" + +// This function resets the cur pointer to the first frame theoretically +// executable after a task completed (ie. each time we update some progress or +// insert some tasks in the queue). +// When frame_idx is set, it can be either from a completed task, or from tasks +// inserted in the queue, in which case we have to make sure the cur pointer +// isn't past this insert. +// The special case where frame_idx is UINT_MAX is to handle the reset after +// completing a task and locklessly signaling progress. In this case we don't +// enter a critical section, which is needed for this function, so we set an +// atomic for a delayed handling, happening here. Meaning we can call this +// function without any actual update other than what's in the atomic, hence +// this special case. +static inline int reset_task_cur(const Dav1dContext *const c, + struct TaskThreadData *const ttd, + unsigned frame_idx) +{ + const unsigned first = atomic_load(&ttd->first); + unsigned reset_frame_idx = atomic_exchange(&ttd->reset_task_cur, UINT_MAX); + if (reset_frame_idx < first) { + if (frame_idx == UINT_MAX) return 0; + reset_frame_idx = UINT_MAX; + } + if (!ttd->cur && c->fc[first].task_thread.task_cur_prev == NULL) + return 0; + if (reset_frame_idx != UINT_MAX) { + if (frame_idx == UINT_MAX) { + if (reset_frame_idx > first + ttd->cur) + return 0; + ttd->cur = reset_frame_idx - first; + goto cur_found; + } + } else if (frame_idx == UINT_MAX) + return 0; + if (frame_idx < first) frame_idx += c->n_fc; + const unsigned min_frame_idx = umin(reset_frame_idx, frame_idx); + const unsigned cur_frame_idx = first + ttd->cur; + if (ttd->cur < c->n_fc && cur_frame_idx < min_frame_idx) + return 0; + for (ttd->cur = min_frame_idx - first; ttd->cur < c->n_fc; ttd->cur++) + if (c->fc[(first + ttd->cur) % c->n_fc].task_thread.task_head) + break; +cur_found: + for (unsigned i = ttd->cur; i < c->n_fc; i++) + c->fc[(first + i) % c->n_fc].task_thread.task_cur_prev = NULL; + return 1; +} + +static inline void reset_task_cur_async(struct TaskThreadData *const ttd, + unsigned frame_idx, unsigned n_frames) +{ + const unsigned first = atomic_load(&ttd->first); + if (frame_idx < first) frame_idx += n_frames; + unsigned last_idx = frame_idx; + do { + frame_idx = last_idx; + last_idx = atomic_exchange(&ttd->reset_task_cur, frame_idx); + } while (last_idx < frame_idx); + if (frame_idx == first && atomic_load(&ttd->first) != first) { + unsigned expected = frame_idx; + atomic_compare_exchange_strong(&ttd->reset_task_cur, &expected, UINT_MAX); + } +} + +static void insert_tasks_between(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + Dav1dTask *const a, Dav1dTask *const b, + const int cond_signal) +{ + struct TaskThreadData *const ttd = f->task_thread.ttd; + if (atomic_load(f->c->flush)) return; + assert(!a || a->next == b); + if (!a) f->task_thread.task_head = first; + else a->next = first; + if (!b) f->task_thread.task_tail = last; + last->next = b; + reset_task_cur(f->c, ttd, first->frame_idx); + if (cond_signal && !atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); +} + +static void insert_tasks(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + const int cond_signal) +{ + // insert task back into task queue + Dav1dTask *t_ptr, *prev_t = NULL; + for (t_ptr = f->task_thread.task_head; + t_ptr; prev_t = t_ptr, t_ptr = t_ptr->next) + { + // entropy coding precedes other steps + if (t_ptr->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + if (first->type > DAV1D_TASK_TYPE_TILE_ENTROPY) continue; + // both are entropy + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + } else { + if (first->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + if (first->type > t_ptr->type) continue; + if (first->type < t_ptr->type) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same task type + } + + // sort by tile-id + assert(first->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION || + first->type == DAV1D_TASK_TYPE_TILE_ENTROPY); + assert(first->type == t_ptr->type); + assert(t_ptr->sby == first->sby); + const int p = first->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int t_tile_idx = (int) (first - f->task_thread.tile_tasks[p]); + const int p_tile_idx = (int) (t_ptr - f->task_thread.tile_tasks[p]); + assert(t_tile_idx != p_tile_idx); + if (t_tile_idx > p_tile_idx) continue; + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // append at the end + insert_tasks_between(f, first, last, prev_t, NULL, cond_signal); +} + +static inline void insert_task(Dav1dFrameContext *const f, + Dav1dTask *const t, const int cond_signal) +{ + insert_tasks(f, t, t, cond_signal); +} + +static inline void add_pending(Dav1dFrameContext *const f, Dav1dTask *const t) { + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + t->next = NULL; + if (!f->task_thread.pending_tasks.head) + f->task_thread.pending_tasks.head = t; + else + f->task_thread.pending_tasks.tail->next = t; + f->task_thread.pending_tasks.tail = t; + atomic_store(&f->task_thread.pending_tasks.merge, 1); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); +} + +static inline int merge_pending_frame(Dav1dFrameContext *const f) { + int const merge = atomic_load(&f->task_thread.pending_tasks.merge); + if (merge) { + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + Dav1dTask *t = f->task_thread.pending_tasks.head; + f->task_thread.pending_tasks.head = NULL; + f->task_thread.pending_tasks.tail = NULL; + atomic_store(&f->task_thread.pending_tasks.merge, 0); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); + while (t) { + Dav1dTask *const tmp = t->next; + insert_task(f, t, 0); + t = tmp; + } + } + return merge; +} + +static inline int merge_pending(const Dav1dContext *const c) { + int res = 0; + for (unsigned i = 0; i < c->n_fc; i++) + res |= merge_pending_frame(&c->fc[i]); + return res; +} + +static int create_filter_sbrow(Dav1dFrameContext *const f, + const int pass, Dav1dTask **res_t) +{ + const int has_deblock = f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]; + const int has_cdef = f->seq_hdr->cdef; + const int has_resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; + const int has_lr = f->lf.restore_planes; + + Dav1dTask *tasks = f->task_thread.tasks; + const int uses_2pass = f->c->n_fc > 1; + int num_tasks = f->sbh * (1 + uses_2pass); + if (num_tasks > f->task_thread.num_tasks) { + const size_t size = sizeof(Dav1dTask) * num_tasks; + tasks = realloc(f->task_thread.tasks, size); + if (!tasks) return -1; + memset(tasks, 0, size); + f->task_thread.tasks = tasks; + f->task_thread.num_tasks = num_tasks; + } + tasks += f->sbh * (pass & 1); + + if (pass & 1) { + f->frame_thread.entropy_progress = 0; + } else { + const int prog_sz = ((f->sbh + 31) & ~31) >> 5; + if (prog_sz > f->frame_thread.prog_sz) { + atomic_uint *const prog = realloc(f->frame_thread.frame_progress, + 2 * prog_sz * sizeof(*prog)); + if (!prog) return -1; + f->frame_thread.frame_progress = prog; + f->frame_thread.copy_lpf_progress = prog + prog_sz; + } + f->frame_thread.prog_sz = prog_sz; + memset(f->frame_thread.frame_progress, 0, prog_sz * sizeof(atomic_uint)); + memset(f->frame_thread.copy_lpf_progress, 0, prog_sz * sizeof(atomic_uint)); + atomic_store(&f->frame_thread.deblock_progress, 0); + } + f->frame_thread.next_tile_row[pass & 1] = 0; + + Dav1dTask *t = &tasks[0]; + t->sby = 0; + t->recon_progress = 1; + t->deblock_progress = 0; + t->type = pass == 1 ? DAV1D_TASK_TYPE_ENTROPY_PROGRESS : + has_deblock ? DAV1D_TASK_TYPE_DEBLOCK_COLS : + has_cdef || has_lr /* i.e. LR backup */ ? DAV1D_TASK_TYPE_DEBLOCK_ROWS : + has_resize ? DAV1D_TASK_TYPE_SUPER_RESOLUTION : + DAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS; + t->frame_idx = (int)(f - f->c->fc); + + *res_t = t; + return 0; +} + +int dav1d_task_create_tile_sbrow(Dav1dFrameContext *const f, const int pass, + const int cond_signal) +{ + Dav1dTask *tasks = f->task_thread.tile_tasks[0]; + const int uses_2pass = f->c->n_fc > 1; + const int num_tasks = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; + if (pass < 2) { + int alloc_num_tasks = num_tasks * (1 + uses_2pass); + if (alloc_num_tasks > f->task_thread.num_tile_tasks) { + const size_t size = sizeof(Dav1dTask) * alloc_num_tasks; + tasks = realloc(f->task_thread.tile_tasks[0], size); + if (!tasks) return -1; + memset(tasks, 0, size); + f->task_thread.tile_tasks[0] = tasks; + f->task_thread.num_tile_tasks = alloc_num_tasks; + } + f->task_thread.tile_tasks[1] = tasks + num_tasks; + } + tasks += num_tasks * (pass & 1); + + Dav1dTask *pf_t; + if (create_filter_sbrow(f, pass, &pf_t)) + return -1; + + Dav1dTask *prev_t = NULL; + for (int tile_idx = 0; tile_idx < num_tasks; tile_idx++) { + Dav1dTileState *const ts = &f->ts[tile_idx]; + Dav1dTask *t = &tasks[tile_idx]; + t->sby = ts->tiling.row_start >> f->sb_shift; + if (pf_t && t->sby) { + prev_t->next = pf_t; + prev_t = pf_t; + pf_t = NULL; + } + t->recon_progress = 0; + t->deblock_progress = 0; + t->deps_skip = 0; + t->type = pass != 1 ? DAV1D_TASK_TYPE_TILE_RECONSTRUCTION : + DAV1D_TASK_TYPE_TILE_ENTROPY; + t->frame_idx = (int)(f - f->c->fc); + if (prev_t) prev_t->next = t; + prev_t = t; + } + if (pf_t) { + prev_t->next = pf_t; + prev_t = pf_t; + } + prev_t->next = NULL; + + atomic_store(&f->task_thread.done[pass & 1], 0); + + // XXX in theory this could be done locklessly, at this point they are no + // tasks in the frameQ, so no other runner should be using this lock, but + // we must add both passes at once + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + assert(f->task_thread.pending_tasks.head == NULL || pass == 2); + if (!f->task_thread.pending_tasks.head) + f->task_thread.pending_tasks.head = &tasks[0]; + else + f->task_thread.pending_tasks.tail->next = &tasks[0]; + f->task_thread.pending_tasks.tail = prev_t; + atomic_store(&f->task_thread.pending_tasks.merge, 1); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); + + return 0; +} + +void dav1d_task_frame_init(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + + atomic_store(&f->task_thread.init_done, 0); + // schedule init task, which will schedule the remaining tasks + Dav1dTask *const t = &f->task_thread.init_task; + t->type = DAV1D_TASK_TYPE_INIT; + t->frame_idx = (int)(f - c->fc); + t->sby = 0; + t->recon_progress = t->deblock_progress = 0; + insert_task(f, t, 1); +} + +void dav1d_task_delayed_fg(Dav1dContext *const c, Dav1dPicture *const out, + const Dav1dPicture *const in) +{ + struct TaskThreadData *const ttd = &c->task_thread; + ttd->delayed_fg.in = in; + ttd->delayed_fg.out = out; + ttd->delayed_fg.type = DAV1D_TASK_TYPE_FG_PREP; + atomic_init(&ttd->delayed_fg.progress[0], 0); + atomic_init(&ttd->delayed_fg.progress[1], 0); + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 1; + pthread_cond_signal(&ttd->cond); + pthread_cond_wait(&ttd->delayed_fg.cond, &ttd->lock); + pthread_mutex_unlock(&ttd->lock); +} + +static inline int ensure_progress(struct TaskThreadData *const ttd, + Dav1dFrameContext *const f, + Dav1dTask *const t, const enum TaskType type, + atomic_int *const state, int *const target) +{ + // deblock_rows (non-LR portion) depends on deblock of previous sbrow, + // so ensure that completed. if not, re-add to task-queue; else, fall-through + int p1 = atomic_load(state); + if (p1 < t->sby) { + t->type = type; + t->recon_progress = t->deblock_progress = 0; + *target = t->sby; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + return 1; + } + return 0; +} + +static inline int check_tile(Dav1dTask *const t, Dav1dFrameContext *const f, + const int frame_mt) +{ + const int tp = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[tp]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + const int p1 = atomic_load(&ts->progress[tp]); + if (p1 < t->sby) return 1; + int error = p1 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + if (!error && frame_mt && !tp) { + const int p2 = atomic_load(&ts->progress[1]); + if (p2 <= t->sby) return 1; + error = p2 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + } + if (!error && frame_mt && !IS_KEY_OR_INTRA(f->frame_hdr)) { + // check reference state + const Dav1dThreadPicture *p = &f->sr_cur; + const int ss_ver = p->p.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const unsigned p_b = (t->sby + 1) << (f->sb_shift + 2); + const int tile_sby = t->sby - (ts->tiling.row_start >> f->sb_shift); + const int (*const lowest_px)[2] = ts->lowest_pixel[tile_sby]; + for (int n = t->deps_skip; n < 7; n++, t->deps_skip++) { + unsigned lowest; + if (tp) { + // if temporal mv refs are disabled, we only need this + // for the primary ref; if segmentation is disabled, we + // don't even need that + lowest = p_b; + } else { + // +8 is postfilter-induced delay + const int y = lowest_px[n][0] == INT_MIN ? INT_MIN : + lowest_px[n][0] + 8; + const int uv = lowest_px[n][1] == INT_MIN ? INT_MIN : + lowest_px[n][1] * (1 << ss_ver) + 8; + const int max = imax(y, uv); + if (max == INT_MIN) continue; + lowest = iclip(max, 1, f->refp[n].p.p.h); + } + const unsigned p3 = atomic_load(&f->refp[n].progress[!tp]); + if (p3 < lowest) return 1; + atomic_fetch_or(&f->task_thread.error, p3 == FRAME_ERROR); + } + } + return 0; +} + +static inline int get_frame_progress(const Dav1dContext *const c, + const Dav1dFrameContext *const f) +{ + unsigned frame_prog = c->n_fc > 1 ? atomic_load(&f->sr_cur.progress[1]) : 0; + if (frame_prog >= FRAME_ERROR) + return f->sbh - 1; + int idx = frame_prog >> (f->sb_shift + 7); + int prog; + do { + atomic_uint *state = &f->frame_thread.frame_progress[idx]; + const unsigned val = ~atomic_load(state); + prog = val ? ctz(val) : 32; + if (prog != 32) break; + prog = 0; + } while (++idx < f->frame_thread.prog_sz); + return ((idx << 5) | prog) - 1; +} + +static inline void abort_frame(Dav1dFrameContext *const f, const int error) { + atomic_store(&f->task_thread.error, error == DAV1D_ERR(EINVAL) ? 1 : -1); + atomic_store(&f->task_thread.task_counter, 0); + atomic_store(&f->task_thread.done[0], 1); + atomic_store(&f->task_thread.done[1], 1); + atomic_store(&f->sr_cur.progress[0], FRAME_ERROR); + atomic_store(&f->sr_cur.progress[1], FRAME_ERROR); + dav1d_decode_frame_exit(f, error); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); +} + +static inline void delayed_fg_task(const Dav1dContext *const c, + struct TaskThreadData *const ttd) +{ + const Dav1dPicture *const in = ttd->delayed_fg.in; + Dav1dPicture *const out = ttd->delayed_fg.out; +#if CONFIG_16BPC + int off; + if (out->p.bpc != 8) + off = (out->p.bpc >> 1) - 4; +#endif + switch (ttd->delayed_fg.type) { + case DAV1D_TASK_TYPE_FG_PREP: + ttd->delayed_fg.exec = 0; + if (atomic_load(&ttd->cond_signaled)) + pthread_cond_signal(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + switch (out->p.bpc) { +#if CONFIG_8BPC + case 8: + dav1d_prep_grain_8bpc(&c->dsp[0].fg, out, in, + ttd->delayed_fg.scaling_8bpc, + ttd->delayed_fg.grain_lut_8bpc); + break; +#endif +#if CONFIG_16BPC + case 10: + case 12: + dav1d_prep_grain_16bpc(&c->dsp[off].fg, out, in, + ttd->delayed_fg.scaling_16bpc, + ttd->delayed_fg.grain_lut_16bpc); + break; +#endif + default: abort(); + } + ttd->delayed_fg.type = DAV1D_TASK_TYPE_FG_APPLY; + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 1; + // fall-through + case DAV1D_TASK_TYPE_FG_APPLY:; + int row = atomic_fetch_add(&ttd->delayed_fg.progress[0], 1); + pthread_mutex_unlock(&ttd->lock); + int progmax = (out->p.h + 31) >> 5; + fg_apply_loop: + if (row + 1 < progmax) + pthread_cond_signal(&ttd->cond); + else if (row + 1 >= progmax) { + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 0; + if (row >= progmax) goto end_add; + pthread_mutex_unlock(&ttd->lock); + } + switch (out->p.bpc) { +#if CONFIG_8BPC + case 8: + dav1d_apply_grain_row_8bpc(&c->dsp[0].fg, out, in, + ttd->delayed_fg.scaling_8bpc, + ttd->delayed_fg.grain_lut_8bpc, row); + break; +#endif +#if CONFIG_16BPC + case 10: + case 12: + dav1d_apply_grain_row_16bpc(&c->dsp[off].fg, out, in, + ttd->delayed_fg.scaling_16bpc, + ttd->delayed_fg.grain_lut_16bpc, row); + break; +#endif + default: abort(); + } + row = atomic_fetch_add(&ttd->delayed_fg.progress[0], 1); + int done = atomic_fetch_add(&ttd->delayed_fg.progress[1], 1) + 1; + if (row < progmax) goto fg_apply_loop; + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 0; + end_add: + done = atomic_fetch_add(&ttd->delayed_fg.progress[1], 1) + 1; + progmax = atomic_load(&ttd->delayed_fg.progress[0]); + // signal for completion only once the last runner reaches this + if (done < progmax) + break; + pthread_cond_signal(&ttd->delayed_fg.cond); + break; + default: abort(); + } +} + +void *dav1d_worker_task(void *data) { + Dav1dTaskContext *const tc = data; + const Dav1dContext *const c = tc->c; + struct TaskThreadData *const ttd = tc->task_thread.ttd; + + dav1d_set_thread_name("dav1d-worker"); + + pthread_mutex_lock(&ttd->lock); + for (;;) { + if (tc->task_thread.die) break; + if (atomic_load(c->flush)) goto park; + + merge_pending(c); + if (ttd->delayed_fg.exec) { // run delayed film grain first + delayed_fg_task(c, ttd); + continue; + } + Dav1dFrameContext *f; + Dav1dTask *t, *prev_t = NULL; + if (c->n_fc > 1) { // run init tasks second + for (unsigned i = 0; i < c->n_fc; i++) { + const unsigned first = atomic_load(&ttd->first); + f = &c->fc[(first + i) % c->n_fc]; + if (atomic_load(&f->task_thread.init_done)) continue; + t = f->task_thread.task_head; + if (!t) continue; + if (t->type == DAV1D_TASK_TYPE_INIT) goto found; + if (t->type == DAV1D_TASK_TYPE_INIT_CDF) { + // XXX This can be a simple else, if adding tasks of both + // passes at once (in dav1d_task_create_tile_sbrow). + // Adding the tasks to the pending Q can result in a + // thread merging them before setting init_done. + // We will need to set init_done before adding to the + // pending Q, so maybe return the tasks, set init_done, + // and add to pending Q only then. + const int p1 = f->in_cdf.progress ? + atomic_load(f->in_cdf.progress) : 1; + if (p1) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } + } + } + while (ttd->cur < c->n_fc) { // run decoding tasks last + const unsigned first = atomic_load(&ttd->first); + f = &c->fc[(first + ttd->cur) % c->n_fc]; + merge_pending_frame(f); + prev_t = f->task_thread.task_cur_prev; + t = prev_t ? prev_t->next : f->task_thread.task_head; + while (t) { + if (t->type == DAV1D_TASK_TYPE_INIT_CDF) goto next; + else if (t->type == DAV1D_TASK_TYPE_TILE_ENTROPY || + t->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION) + { + // if not bottom sbrow of tile, this task will be re-added + // after it's finished + if (!check_tile(t, f, c->n_fc > 1)) + goto found; + } else if (t->recon_progress) { + const int p = t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS; + int error = atomic_load(&f->task_thread.error); + assert(!atomic_load(&f->task_thread.done[p]) || error); + const int tile_row_base = f->frame_hdr->tiling.cols * + f->frame_thread.next_tile_row[p]; + if (p) { + atomic_int *const prog = &f->frame_thread.entropy_progress; + const int p1 = atomic_load(prog); + if (p1 < t->sby) goto next; + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + } + for (int tc = 0; tc < f->frame_hdr->tiling.cols; tc++) { + Dav1dTileState *const ts = &f->ts[tile_row_base + tc]; + const int p2 = atomic_load(&ts->progress[p]); + if (p2 < t->recon_progress) goto next; + atomic_fetch_or(&f->task_thread.error, p2 == TILE_ERROR); + } + if (t->sby + 1 < f->sbh) { + // add sby+1 to list to replace this one + Dav1dTask *next_t = &t[1]; + *next_t = *t; + next_t->sby++; + const int ntr = f->frame_thread.next_tile_row[p] + 1; + const int start = f->frame_hdr->tiling.row_start_sb[ntr]; + if (next_t->sby == start) + f->frame_thread.next_tile_row[p] = ntr; + next_t->recon_progress = next_t->sby + 1; + insert_task(f, next_t, 0); + } + goto found; + } else if (t->type == DAV1D_TASK_TYPE_CDEF) { + atomic_uint *prog = f->frame_thread.copy_lpf_progress; + const int p1 = atomic_load(&prog[(t->sby - 1) >> 5]); + if (p1 & (1U << ((t->sby - 1) & 31))) + goto found; + } else { + assert(t->deblock_progress); + const int p1 = atomic_load(&f->frame_thread.deblock_progress); + if (p1 >= t->deblock_progress) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } + next: + prev_t = t; + t = t->next; + f->task_thread.task_cur_prev = prev_t; + } + ttd->cur++; + } + if (reset_task_cur(c, ttd, UINT_MAX)) continue; + if (merge_pending(c)) continue; + park: + tc->task_thread.flushed = 1; + pthread_cond_signal(&tc->task_thread.td.cond); + // we want to be woken up next time progress is signaled + atomic_store(&ttd->cond_signaled, 0); + pthread_cond_wait(&ttd->cond, &ttd->lock); + tc->task_thread.flushed = 0; + reset_task_cur(c, ttd, UINT_MAX); + continue; + + found: + // remove t from list + if (prev_t) prev_t->next = t->next; + else f->task_thread.task_head = t->next; + if (!t->next) f->task_thread.task_tail = prev_t; + if (t->type > DAV1D_TASK_TYPE_INIT_CDF && !f->task_thread.task_head) + ttd->cur++; + t->next = NULL; + // we don't need to check cond_signaled here, since we found a task + // after the last signal so we want to re-signal the next waiting thread + // and again won't need to signal after that + atomic_store(&ttd->cond_signaled, 1); + pthread_cond_signal(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + found_unlocked:; + const int flush = atomic_load(c->flush); + int error = atomic_fetch_or(&f->task_thread.error, flush) | flush; + + // run it + tc->f = f; + int sby = t->sby; + switch (t->type) { + case DAV1D_TASK_TYPE_INIT: { + assert(c->n_fc > 1); + int res = dav1d_decode_frame_init(f); + int p1 = f->in_cdf.progress ? atomic_load(f->in_cdf.progress) : 1; + if (res || p1 == TILE_ERROR) { + pthread_mutex_lock(&ttd->lock); + abort_frame(f, res ? res : DAV1D_ERR(EINVAL)); + reset_task_cur(c, ttd, t->frame_idx); + } else { + t->type = DAV1D_TASK_TYPE_INIT_CDF; + if (p1) goto found_unlocked; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + } + continue; + } + case DAV1D_TASK_TYPE_INIT_CDF: { + assert(c->n_fc > 1); + int res = DAV1D_ERR(EINVAL); + if (!atomic_load(&f->task_thread.error)) + res = dav1d_decode_frame_init_cdf(f); + if (f->frame_hdr->refresh_context && !f->task_thread.update_set) { + atomic_store(f->out_cdf.progress, res < 0 ? TILE_ERROR : 1); + } + if (!res) { + assert(c->n_fc > 1); + for (int p = 1; p <= 2; p++) { + const int res = dav1d_task_create_tile_sbrow(f, p, 0); + if (res) { + pthread_mutex_lock(&ttd->lock); + // memory allocation failed + atomic_store(&f->task_thread.done[2 - p], 1); + atomic_store(&f->task_thread.error, -1); + atomic_fetch_sub(&f->task_thread.task_counter, + f->frame_hdr->tiling.cols * + f->frame_hdr->tiling.rows + f->sbh); + atomic_store(&f->sr_cur.progress[p - 1], FRAME_ERROR); + if (p == 2 && atomic_load(&f->task_thread.done[1])) { + assert(!atomic_load(&f->task_thread.task_counter)); + dav1d_decode_frame_exit(f, DAV1D_ERR(ENOMEM)); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + atomic_store(&f->task_thread.init_done, 1); + continue; + } else { + pthread_mutex_unlock(&ttd->lock); + } + } + } + atomic_store(&f->task_thread.init_done, 1); + pthread_mutex_lock(&ttd->lock); + } else { + pthread_mutex_lock(&ttd->lock); + abort_frame(f, res); + reset_task_cur(c, ttd, t->frame_idx); + atomic_store(&f->task_thread.init_done, 1); + } + continue; + } + case DAV1D_TASK_TYPE_TILE_ENTROPY: + case DAV1D_TASK_TYPE_TILE_RECONSTRUCTION: { + const int p = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[p]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + + tc->ts = ts; + tc->by = sby << f->sb_shift; + const int uses_2pass = c->n_fc > 1; + tc->frame_thread.pass = !uses_2pass ? 0 : + 1 + (t->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION); + if (!error) error = dav1d_decode_tile_sbrow(tc); + const int progress = error ? TILE_ERROR : 1 + sby; + + // signal progress + atomic_fetch_or(&f->task_thread.error, error); + if (((sby + 1) << f->sb_shift) < ts->tiling.row_end) { + t->sby++; + t->deps_skip = 0; + if (!check_tile(t, f, uses_2pass)) { + atomic_store(&ts->progress[p], progress); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + goto found_unlocked; + } + atomic_store(&ts->progress[p], progress); + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + } else { + pthread_mutex_lock(&ttd->lock); + atomic_store(&ts->progress[p], progress); + reset_task_cur(c, ttd, t->frame_idx); + error = atomic_load(&f->task_thread.error); + if (f->frame_hdr->refresh_context && + tc->frame_thread.pass <= 1 && f->task_thread.update_set && + f->frame_hdr->tiling.update == tile_idx) + { + if (!error) + dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, + &f->ts[f->frame_hdr->tiling.update].cdf); + if (c->n_fc > 1) + atomic_store(f->out_cdf.progress, error ? TILE_ERROR : 1); + } + if (atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1 == 0 && + atomic_load(&f->task_thread.done[0]) && + (!uses_2pass || atomic_load(&f->task_thread.done[1]))) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + assert(atomic_load(&f->task_thread.task_counter) >= 0); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + continue; + } + case DAV1D_TASK_TYPE_DEBLOCK_COLS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_cols(f, sby); + if (ensure_progress(ttd, f, t, DAV1D_TASK_TYPE_DEBLOCK_ROWS, + &f->frame_thread.deblock_progress, + &t->deblock_progress)) continue; + // fall-through + case DAV1D_TASK_TYPE_DEBLOCK_ROWS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_rows(f, sby); + // signal deblock progress + if (f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]) + { + error = atomic_load(&f->task_thread.error); + atomic_store(&f->frame_thread.deblock_progress, + error ? TILE_ERROR : sby + 1); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } else if (f->seq_hdr->cdef || f->lf.restore_planes) { + atomic_fetch_or(&f->frame_thread.copy_lpf_progress[sby >> 5], + 1U << (sby & 31)); + // CDEF needs the top buffer to be saved by lr_copy_lpf of the + // previous sbrow + if (sby) { + int prog = atomic_load(&f->frame_thread.copy_lpf_progress[(sby - 1) >> 5]); + if (~prog & (1U << ((sby - 1) & 31))) { + t->type = DAV1D_TASK_TYPE_CDEF; + t->recon_progress = t->deblock_progress = 0; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + continue; + } + } + } + // fall-through + case DAV1D_TASK_TYPE_CDEF: + if (f->seq_hdr->cdef) { + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_cdef(tc, sby); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + // fall-through + case DAV1D_TASK_TYPE_SUPER_RESOLUTION: + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_resize(f, sby); + // fall-through + case DAV1D_TASK_TYPE_LOOP_RESTORATION: + if (!atomic_load(&f->task_thread.error) && f->lf.restore_planes) + f->bd_fn.filter_sbrow_lr(f, sby); + // fall-through + case DAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS: + // dummy to cover for no post-filters + case DAV1D_TASK_TYPE_ENTROPY_PROGRESS: + // dummy to convert tile progress to frame + break; + default: abort(); + } + // if task completed [typically LR], signal picture progress as per below + const int uses_2pass = c->n_fc > 1; + const int sbh = f->sbh; + const int sbsz = f->sb_step * 4; + if (t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS) { + error = atomic_load(&f->task_thread.error); + const unsigned y = sby + 1 == sbh ? UINT_MAX : (unsigned)(sby + 1) * sbsz; + assert(c->n_fc > 1); + if (f->sr_cur.p.data[0] /* upon flush, this can be free'ed already */) + atomic_store(&f->sr_cur.progress[0], error ? FRAME_ERROR : y); + atomic_store(&f->frame_thread.entropy_progress, + error ? TILE_ERROR : sby + 1); + if (sby + 1 == sbh) + atomic_store(&f->task_thread.done[1], 1); + pthread_mutex_lock(&ttd->lock); + const int num_tasks = atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1; + if (sby + 1 < sbh && num_tasks) { + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + if (!num_tasks && atomic_load(&f->task_thread.done[0]) && + atomic_load(&f->task_thread.done[1])) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + // t->type != DAV1D_TASK_TYPE_ENTROPY_PROGRESS + atomic_fetch_or(&f->frame_thread.frame_progress[sby >> 5], + 1U << (sby & 31)); + pthread_mutex_lock(&f->task_thread.lock); + sby = get_frame_progress(c, f); + error = atomic_load(&f->task_thread.error); + const unsigned y = sby + 1 == sbh ? UINT_MAX : (unsigned)(sby + 1) * sbsz; + if (c->n_fc > 1 && f->sr_cur.p.data[0] /* upon flush, this can be free'ed already */) + atomic_store(&f->sr_cur.progress[1], error ? FRAME_ERROR : y); + pthread_mutex_unlock(&f->task_thread.lock); + if (sby + 1 == sbh) + atomic_store(&f->task_thread.done[0], 1); + pthread_mutex_lock(&ttd->lock); + const int num_tasks = atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1; + if (sby + 1 < sbh && num_tasks) { + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + if (!num_tasks && atomic_load(&f->task_thread.done[0]) && + (!uses_2pass || atomic_load(&f->task_thread.done[1]))) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); + } + pthread_mutex_unlock(&ttd->lock); + + return NULL; +} diff --git a/third_party/dav1d/src/thread_task.h b/third_party/dav1d/src/thread_task.h new file mode 100644 index 0000000000..257da1a470 --- /dev/null +++ b/third_party/dav1d/src/thread_task.h @@ -0,0 +1,53 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_THREAD_TASK_H +#define DAV1D_SRC_THREAD_TASK_H + +#include + +#include "src/internal.h" + +#define FRAME_ERROR (UINT_MAX - 1) +#define TILE_ERROR (INT_MAX - 1) + +// these functions assume the task scheduling lock is already taken +int dav1d_task_create_tile_sbrow(Dav1dFrameContext *f, int pass, int cond_signal); +void dav1d_task_frame_init(Dav1dFrameContext *f); + +void dav1d_task_delayed_fg(Dav1dContext *c, Dav1dPicture *out, const Dav1dPicture *in); + +void *dav1d_worker_task(void *data); + +int dav1d_decode_frame_init(Dav1dFrameContext *f); +int dav1d_decode_frame_init_cdf(Dav1dFrameContext *f); +int dav1d_decode_frame_main(Dav1dFrameContext *f); +void dav1d_decode_frame_exit(Dav1dFrameContext *f, int retval); +int dav1d_decode_frame(Dav1dFrameContext *f); +int dav1d_decode_tile_sbrow(Dav1dTaskContext *t); + +#endif /* DAV1D_SRC_THREAD_TASK_H */ diff --git a/third_party/dav1d/src/warpmv.c b/third_party/dav1d/src/warpmv.c new file mode 100644 index 0000000000..439c4304c7 --- /dev/null +++ b/third_party/dav1d/src/warpmv.c @@ -0,0 +1,209 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/intops.h" + +#include "src/warpmv.h" + +static const uint16_t div_lut[257] = { + 16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768, + 15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142, + 15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564, + 14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028, + 13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530, + 13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066, + 13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633, + 12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228, + 12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848, + 11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491, + 11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155, + 11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838, + 10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538, + 10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255, + 10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986, + 9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732, + 9709, 9687, 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489, + 9468, 9447, 9425, 9404, 9383, 9362, 9341, 9321, 9300, 9279, 9259, + 9239, 9218, 9198, 9178, 9158, 9138, 9118, 9098, 9079, 9059, 9039, + 9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, 8867, 8849, 8830, + 8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, 8630, + 8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439, + 8422, 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257, + 8240, 8224, 8208, 8192, +}; + +static inline int iclip_wmp(const int v) { + const int cv = iclip(v, INT16_MIN, INT16_MAX); + + return apply_sign((abs(cv) + 32) >> 6, cv) * (1 << 6); +} + +static inline int resolve_divisor_32(const unsigned d, int *const shift) { + *shift = ulog2(d); + const int e = d - (1 << *shift); + const int f = *shift > 8 ? (e + (1 << (*shift - 9))) >> (*shift - 8) : + e << (8 - *shift); + assert(f <= 256); + *shift += 14; + // Use f as lookup into the precomputed table of multipliers + return div_lut[f]; +} + +int dav1d_get_shear_params(Dav1dWarpedMotionParams *const wm) { + const int32_t *const mat = wm->matrix; + + if (mat[2] <= 0) return 1; + + wm->u.p.alpha = iclip_wmp(mat[2] - 0x10000); + wm->u.p.beta = iclip_wmp(mat[3]); + + int shift; + const int y = apply_sign(resolve_divisor_32(abs(mat[2]), &shift), mat[2]); + const int64_t v1 = ((int64_t) mat[4] * 0x10000) * y; + const int rnd = (1 << shift) >> 1; + wm->u.p.gamma = iclip_wmp(apply_sign64((int) ((llabs(v1) + rnd) >> shift), v1)); + const int64_t v2 = ((int64_t) mat[3] * mat[4]) * y; + wm->u.p.delta = iclip_wmp(mat[5] - + apply_sign64((int) ((llabs(v2) + rnd) >> shift), v2) - + 0x10000); + + return (4 * abs(wm->u.p.alpha) + 7 * abs(wm->u.p.beta) >= 0x10000) || + (4 * abs(wm->u.p.gamma) + 4 * abs(wm->u.p.delta) >= 0x10000); +} + +static int resolve_divisor_64(const uint64_t d, int *const shift) { + *shift = u64log2(d); + const int64_t e = d - (1LL << *shift); + const int64_t f = *shift > 8 ? (e + (1LL << (*shift - 9))) >> (*shift - 8) : + e << (8 - *shift); + assert(f <= 256); + *shift += 14; + // Use f as lookup into the precomputed table of multipliers + return div_lut[f]; +} + +static int get_mult_shift_ndiag(const int64_t px, + const int idet, const int shift) +{ + const int64_t v1 = px * idet; + const int v2 = apply_sign64((int) ((llabs(v1) + + ((1LL << shift) >> 1)) >> shift), + v1); + return iclip(v2, -0x1fff, 0x1fff); +} + +static int get_mult_shift_diag(const int64_t px, + const int idet, const int shift) +{ + const int64_t v1 = px * idet; + const int v2 = apply_sign64((int) ((llabs(v1) + + ((1LL << shift) >> 1)) >> shift), + v1); + return iclip(v2, 0xe001, 0x11fff); +} + +void dav1d_set_affine_mv2d(const int bw4, const int bh4, + const mv mv, Dav1dWarpedMotionParams *const wm, + const int bx4, const int by4) +{ + int32_t *const mat = wm->matrix; + const int rsuy = 2 * bh4 - 1; + const int rsux = 2 * bw4 - 1; + const int isuy = by4 * 4 + rsuy; + const int isux = bx4 * 4 + rsux; + + mat[0] = iclip(mv.x * 0x2000 - (isux * (mat[2] - 0x10000) + isuy * mat[3]), + -0x800000, 0x7fffff); + mat[1] = iclip(mv.y * 0x2000 - (isux * mat[4] + isuy * (mat[5] - 0x10000)), + -0x800000, 0x7fffff); +} + +int dav1d_find_affine_int(const int (*pts)[2][2], const int np, + const int bw4, const int bh4, + const mv mv, Dav1dWarpedMotionParams *const wm, + const int bx4, const int by4) +{ + int32_t *const mat = wm->matrix; + int a[2][2] = { { 0, 0 }, { 0, 0 } }; + int bx[2] = { 0, 0 }; + int by[2] = { 0, 0 }; + const int rsuy = 2 * bh4 - 1; + const int rsux = 2 * bw4 - 1; + const int suy = rsuy * 8; + const int sux = rsux * 8; + const int duy = suy + mv.y; + const int dux = sux + mv.x; + const int isuy = by4 * 4 + rsuy; + const int isux = bx4 * 4 + rsux; + + for (int i = 0; i < np; i++) { + const int dx = pts[i][1][0] - dux; + const int dy = pts[i][1][1] - duy; + const int sx = pts[i][0][0] - sux; + const int sy = pts[i][0][1] - suy; + if (abs(sx - dx) < 256 && abs(sy - dy) < 256) { + a[0][0] += ((sx * sx) >> 2) + sx * 2 + 8; + a[0][1] += ((sx * sy) >> 2) + sx + sy + 4; + a[1][1] += ((sy * sy) >> 2) + sy * 2 + 8; + bx[0] += ((sx * dx) >> 2) + sx + dx + 8; + bx[1] += ((sy * dx) >> 2) + sy + dx + 4; + by[0] += ((sx * dy) >> 2) + sx + dy + 4; + by[1] += ((sy * dy) >> 2) + sy + dy + 8; + } + } + + // compute determinant of a + const int64_t det = (int64_t) a[0][0] * a[1][1] - (int64_t) a[0][1] * a[0][1]; + if (det == 0) return 1; + int shift, idet = apply_sign64(resolve_divisor_64(llabs(det), &shift), det); + shift -= 16; + if (shift < 0) { + idet <<= -shift; + shift = 0; + } + + // solve the least-squares + mat[2] = get_mult_shift_diag((int64_t) a[1][1] * bx[0] - + (int64_t) a[0][1] * bx[1], idet, shift); + mat[3] = get_mult_shift_ndiag((int64_t) a[0][0] * bx[1] - + (int64_t) a[0][1] * bx[0], idet, shift); + mat[4] = get_mult_shift_ndiag((int64_t) a[1][1] * by[0] - + (int64_t) a[0][1] * by[1], idet, shift); + mat[5] = get_mult_shift_diag((int64_t) a[0][0] * by[1] - + (int64_t) a[0][1] * by[0], idet, shift); + + mat[0] = iclip(mv.x * 0x2000 - (isux * (mat[2] - 0x10000) + isuy * mat[3]), + -0x800000, 0x7fffff); + mat[1] = iclip(mv.y * 0x2000 - (isux * mat[4] + isuy * (mat[5] - 0x10000)), + -0x800000, 0x7fffff); + + return 0; +} diff --git a/third_party/dav1d/src/warpmv.h b/third_party/dav1d/src/warpmv.h new file mode 100644 index 0000000000..08e841d1ca --- /dev/null +++ b/third_party/dav1d/src/warpmv.h @@ -0,0 +1,39 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_WARPMV_H +#define DAV1D_SRC_WARPMV_H + +#include "src/levels.h" + +int dav1d_get_shear_params(Dav1dWarpedMotionParams *wm); +int dav1d_find_affine_int(const int (*pts)[2][2], int np, int bw4, int bh4, + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); +void dav1d_set_affine_mv2d(int bw4, int bh4, + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); + +#endif /* DAV1D_SRC_WARPMV_H */ diff --git a/third_party/dav1d/src/wedge.c b/third_party/dav1d/src/wedge.c new file mode 100644 index 0000000000..6466068f32 --- /dev/null +++ b/third_party/dav1d/src/wedge.c @@ -0,0 +1,342 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/intops.h" + +#include "src/wedge.h" + +enum WedgeDirectionType { + WEDGE_HORIZONTAL = 0, + WEDGE_VERTICAL = 1, + WEDGE_OBLIQUE27 = 2, + WEDGE_OBLIQUE63 = 3, + WEDGE_OBLIQUE117 = 4, + WEDGE_OBLIQUE153 = 5, + N_WEDGE_DIRECTIONS +}; + +typedef struct { + uint8_t /* enum WedgeDirectionType */ direction; + uint8_t x_offset; + uint8_t y_offset; +} wedge_code_type; + +static const wedge_code_type wedge_codebook_16_hgtw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 6 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static const wedge_code_type wedge_codebook_16_hltw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 4, 4 }, + { WEDGE_VERTICAL, 6, 4 }, { WEDGE_HORIZONTAL, 4, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static const wedge_code_type wedge_codebook_16_heqw[16] = { + { WEDGE_OBLIQUE27, 4, 4 }, { WEDGE_OBLIQUE63, 4, 4 }, + { WEDGE_OBLIQUE117, 4, 4 }, { WEDGE_OBLIQUE153, 4, 4 }, + { WEDGE_HORIZONTAL, 4, 2 }, { WEDGE_HORIZONTAL, 4, 6 }, + { WEDGE_VERTICAL, 2, 4 }, { WEDGE_VERTICAL, 6, 4 }, + { WEDGE_OBLIQUE27, 4, 2 }, { WEDGE_OBLIQUE27, 4, 6 }, + { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, + { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, + { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, +}; + +static uint8_t ALIGN(wedge_masks_444_32x32[2 * 16 * 32 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_32x16[2 * 16 * 32 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_32x8 [2 * 16 * 32 * 8], 64); +static uint8_t ALIGN(wedge_masks_444_16x32[2 * 16 * 16 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_444_8x32 [2 * 16 * 8 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_8x8 [2 * 16 * 8 * 8], 64); + +static uint8_t ALIGN(wedge_masks_422_16x32[2 * 16 * 16 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_422_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_422_8x32 [2 * 16 * 8 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_422_8x8 [2 * 16 * 8 * 8], 64); +static uint8_t ALIGN(wedge_masks_422_4x32 [2 * 16 * 4 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_4x16 [2 * 16 * 4 * 16], 64); +static uint8_t ALIGN(wedge_masks_422_4x8 [2 * 16 * 4 * 8], 32); + +static uint8_t ALIGN(wedge_masks_420_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_420_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_420_16x4 [2 * 16 * 16 * 4], 64); +static uint8_t ALIGN(wedge_masks_420_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_420_8x8 [2 * 16 * 8 * 8], 64); +static uint8_t ALIGN(wedge_masks_420_8x4 [2 * 16 * 8 * 4], 64); +static uint8_t ALIGN(wedge_masks_420_4x16 [2 * 16 * 4 * 16], 64); +static uint8_t ALIGN(wedge_masks_420_4x8 [2 * 16 * 4 * 8], 32); +static uint8_t ALIGN(wedge_masks_420_4x4 [2 * 16 * 4 * 4], 16); + +const uint8_t *dav1d_wedge_masks[N_BS_SIZES][3][2][16]; + +static void insert_border(uint8_t *const dst, const uint8_t *const src, + const int ctr) +{ + if (ctr > 4) memset(dst, 0, ctr - 4); + memcpy(dst + imax(ctr, 4) - 4, src + imax(4 - ctr, 0), imin(64 - ctr, 8)); + if (ctr < 64 - 4) + memset(dst + ctr + 4, 64, 64 - 4 - ctr); +} + +static void transpose(uint8_t *const dst, const uint8_t *const src) { + for (int y = 0, y_off = 0; y < 64; y++, y_off += 64) + for (int x = 0, x_off = 0; x < 64; x++, x_off += 64) + dst[x_off + y] = src[y_off + x]; +} + +static void hflip(uint8_t *const dst, const uint8_t *const src) { + for (int y = 0, y_off = 0; y < 64; y++, y_off += 64) + for (int x = 0; x < 64; x++) + dst[y_off + 64 - 1 - x] = src[y_off + x]; +} + +static void invert(uint8_t *const dst, const uint8_t *const src, + const int w, const int h) +{ + for (int y = 0, y_off = 0; y < h; y++, y_off += w) + for (int x = 0; x < w; x++) + dst[y_off + x] = 64 - src[y_off + x]; +} + +static void copy2d(uint8_t *dst, const uint8_t *src, + const int w, const int h, const int x_off, const int y_off) +{ + src += y_off * 64 + x_off; + for (int y = 0; y < h; y++) { + memcpy(dst, src, w); + src += 64; + dst += w; + } +} + +static COLD void init_chroma(uint8_t *chroma, const uint8_t *luma, + const int sign, const int w, const int h, + const int ss_ver) +{ + for (int y = 0; y < h; y += 1 + ss_ver) { + for (int x = 0; x < w; x += 2) { + int sum = luma[x] + luma[x + 1] + 1; + if (ss_ver) sum += luma[w + x] + luma[w + x + 1] + 1; + chroma[x >> 1] = (sum - sign) >> (1 + ss_ver); + } + luma += w << ss_ver; + chroma += w >> 1; + } +} + +static COLD void fill2d_16x2(uint8_t *dst, const int w, const int h, + const enum BlockSize bs, + const uint8_t (*const master)[64 * 64], + const wedge_code_type *const cb, + uint8_t *masks_444, uint8_t *masks_422, + uint8_t *masks_420, const unsigned signs) +{ + uint8_t *ptr = dst; + for (int n = 0; n < 16; n++) { + copy2d(ptr, master[cb[n].direction], w, h, + 32 - (w * cb[n].x_offset >> 3), 32 - (h * cb[n].y_offset >> 3)); + ptr += w * h; + } + for (int n = 0, off = 0; n < 16; n++, off += w * h) + invert(ptr + off, dst + off, w, h); + + const int n_stride_444 = (w * h); + const int n_stride_422 = n_stride_444 >> 1; + const int n_stride_420 = n_stride_444 >> 2; + const int sign_stride_444 = 16 * n_stride_444; + const int sign_stride_422 = 16 * n_stride_422; + const int sign_stride_420 = 16 * n_stride_420; + // assign pointers in externally visible array + for (int n = 0; n < 16; n++) { + const int sign = (signs >> n) & 1; + dav1d_wedge_masks[bs][0][0][n] = &masks_444[ sign * sign_stride_444]; + // not using !sign is intentional here, since 444 does not require + // any rounding since no chroma subsampling is applied. + dav1d_wedge_masks[bs][0][1][n] = &masks_444[ sign * sign_stride_444]; + dav1d_wedge_masks[bs][1][0][n] = &masks_422[ sign * sign_stride_422]; + dav1d_wedge_masks[bs][1][1][n] = &masks_422[!sign * sign_stride_422]; + dav1d_wedge_masks[bs][2][0][n] = &masks_420[ sign * sign_stride_420]; + dav1d_wedge_masks[bs][2][1][n] = &masks_420[!sign * sign_stride_420]; + masks_444 += n_stride_444; + masks_422 += n_stride_422; + masks_420 += n_stride_420; + + // since the pointers come from inside, we know that + // violation of the const is OK here. Any other approach + // means we would have to duplicate the sign correction + // logic in two places, which isn't very nice, or mark + // the table faced externally as non-const, which also sucks + init_chroma((uint8_t *)dav1d_wedge_masks[bs][1][0][n], + dav1d_wedge_masks[bs][0][0][n], 0, w, h, 0); + init_chroma((uint8_t *)dav1d_wedge_masks[bs][1][1][n], + dav1d_wedge_masks[bs][0][0][n], 1, w, h, 0); + init_chroma((uint8_t *)dav1d_wedge_masks[bs][2][0][n], + dav1d_wedge_masks[bs][0][0][n], 0, w, h, 1); + init_chroma((uint8_t *)dav1d_wedge_masks[bs][2][1][n], + dav1d_wedge_masks[bs][0][0][n], 1, w, h, 1); + } +} + +COLD void dav1d_init_wedge_masks(void) { + // This function is guaranteed to be called only once + + enum WedgeMasterLineType { + WEDGE_MASTER_LINE_ODD, + WEDGE_MASTER_LINE_EVEN, + WEDGE_MASTER_LINE_VERT, + N_WEDGE_MASTER_LINES, + }; + static const uint8_t wedge_master_border[N_WEDGE_MASTER_LINES][8] = { + [WEDGE_MASTER_LINE_ODD] = { 1, 2, 6, 18, 37, 53, 60, 63 }, + [WEDGE_MASTER_LINE_EVEN] = { 1, 4, 11, 27, 46, 58, 62, 63 }, + [WEDGE_MASTER_LINE_VERT] = { 0, 2, 7, 21, 43, 57, 62, 64 }, + }; + uint8_t master[6][64 * 64]; + + // create master templates + for (int y = 0, off = 0; y < 64; y++, off += 64) + insert_border(&master[WEDGE_VERTICAL][off], + wedge_master_border[WEDGE_MASTER_LINE_VERT], 32); + for (int y = 0, off = 0, ctr = 48; y < 64; y += 2, off += 128, ctr--) + { + insert_border(&master[WEDGE_OBLIQUE63][off], + wedge_master_border[WEDGE_MASTER_LINE_EVEN], ctr); + insert_border(&master[WEDGE_OBLIQUE63][off + 64], + wedge_master_border[WEDGE_MASTER_LINE_ODD], ctr - 1); + } + + transpose(master[WEDGE_OBLIQUE27], master[WEDGE_OBLIQUE63]); + transpose(master[WEDGE_HORIZONTAL], master[WEDGE_VERTICAL]); + hflip(master[WEDGE_OBLIQUE117], master[WEDGE_OBLIQUE63]); + hflip(master[WEDGE_OBLIQUE153], master[WEDGE_OBLIQUE27]); + +#define fill(w, h, sz_422, sz_420, hvsw, signs) \ + fill2d_16x2((uint8_t *) wedge_masks_444_##w##x##h, w, h, BS_##w##x##h, \ + master, wedge_codebook_16_##hvsw, wedge_masks_444_##w##x##h, \ + wedge_masks_422_##sz_422, wedge_masks_420_##sz_420, signs) + + fill(32, 32, 16x32, 16x16, heqw, 0x7bfb); + fill(32, 16, 16x16, 16x8, hltw, 0x7beb); + fill(32, 8, 16x8, 16x4, hltw, 0x6beb); + fill(16, 32, 8x32, 8x16, hgtw, 0x7beb); + fill(16, 16, 8x16, 8x8, heqw, 0x7bfb); + fill(16, 8, 8x8, 8x4, hltw, 0x7beb); + fill( 8, 32, 4x32, 4x16, hgtw, 0x7aeb); + fill( 8, 16, 4x16, 4x8, hgtw, 0x7beb); + fill( 8, 8, 4x8, 4x4, heqw, 0x7bfb); +#undef fill +} + +#define N_II_PRED_MODES (N_INTER_INTRA_PRED_MODES - 1) +static uint8_t ALIGN(ii_dc_mask[32 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_32x32[N_II_PRED_MODES][32 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_16x32[N_II_PRED_MODES][16 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_16x16[N_II_PRED_MODES][16 * 16], 64); +static uint8_t ALIGN(ii_nondc_mask_8x32 [N_II_PRED_MODES][ 8 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_8x16 [N_II_PRED_MODES][ 8 * 16], 64); +static uint8_t ALIGN(ii_nondc_mask_8x8 [N_II_PRED_MODES][ 8 * 8], 64); +static uint8_t ALIGN(ii_nondc_mask_4x16 [N_II_PRED_MODES][ 4 * 16], 64); +static uint8_t ALIGN(ii_nondc_mask_4x8 [N_II_PRED_MODES][ 4 * 8], 32); +static uint8_t ALIGN(ii_nondc_mask_4x4 [N_II_PRED_MODES][ 4 * 4], 16); +#undef N_II_PRED_MODES + +#define set1(sz) \ + [II_DC_PRED] = ii_dc_mask, \ + [II_VERT_PRED] = ii_nondc_mask_##sz[II_VERT_PRED - 1], \ + [II_HOR_PRED] = ii_nondc_mask_##sz[II_HOR_PRED - 1], \ + [II_SMOOTH_PRED] = ii_nondc_mask_##sz[II_SMOOTH_PRED - 1] +#define set(sz_444, sz_422, sz_420) \ + { { set1(sz_444) }, { set1(sz_422) }, { set1(sz_420) } } +const uint8_t *dav1d_ii_masks[N_BS_SIZES][3][N_INTER_INTRA_PRED_MODES] = { + [BS_8x8] = set( 8x8, 4x8, 4x4), + [BS_8x16] = set( 8x16, 4x16, 4x8), + [BS_16x8] = set(16x16, 8x8, 8x8), + [BS_16x16] = set(16x16, 8x16, 8x8), + [BS_16x32] = set(16x32, 8x32, 8x16), + [BS_32x16] = set(32x32, 16x16, 16x16), + [BS_32x32] = set(32x32, 16x32, 16x16), +}; +#undef set +#undef set1 + +static COLD void build_nondc_ii_masks(uint8_t *const mask_v, + uint8_t *const mask_h, + uint8_t *const mask_sm, + const int w, const int h, const int step) +{ + static const uint8_t ii_weights_1d[] = { + 60, 52, 45, 39, 34, 30, 26, 22, 19, 17, 15, 13, 11, 10, 8, 7, + 6, 6, 5, 4, 4, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1, + }; + + for (int y = 0, off = 0; y < h; y++, off += w) { + memset(&mask_v[off], ii_weights_1d[y * step], w); + for (int x = 0; x < w; x++) { + mask_sm[off + x] = ii_weights_1d[imin(x, y) * step]; + mask_h[off + x] = ii_weights_1d[x * step]; + } + } +} + +COLD void dav1d_init_interintra_masks(void) { + // This function is guaranteed to be called only once + + memset(ii_dc_mask, 32, 32 * 32); +#define set(a) a[II_VERT_PRED - 1], a[II_HOR_PRED - 1], a[II_SMOOTH_PRED - 1] + build_nondc_ii_masks(set(ii_nondc_mask_32x32), 32, 32, 1); + build_nondc_ii_masks(set(ii_nondc_mask_16x32), 16, 32, 1); + build_nondc_ii_masks(set(ii_nondc_mask_16x16), 16, 16, 2); + build_nondc_ii_masks(set(ii_nondc_mask_8x32), 8, 32, 1); + build_nondc_ii_masks(set(ii_nondc_mask_8x16), 8, 16, 2); + build_nondc_ii_masks(set(ii_nondc_mask_8x8), 8, 8, 4); + build_nondc_ii_masks(set(ii_nondc_mask_4x16), 4, 16, 2); + build_nondc_ii_masks(set(ii_nondc_mask_4x8), 4, 8, 4); + build_nondc_ii_masks(set(ii_nondc_mask_4x4), 4, 4, 8); +#undef set +} diff --git a/third_party/dav1d/src/wedge.h b/third_party/dav1d/src/wedge.h new file mode 100644 index 0000000000..586be98c42 --- /dev/null +++ b/third_party/dav1d/src/wedge.h @@ -0,0 +1,41 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_WEDGE_H +#define DAV1D_SRC_WEDGE_H + +#include "src/levels.h" + +void dav1d_init_wedge_masks(void); +EXTERN const uint8_t *dav1d_wedge_masks[N_BS_SIZES][3 /* 444/luma, 422, 420 */] + [2 /* sign */][16 /* wedge_idx */]; + +void dav1d_init_interintra_masks(void); +EXTERN const uint8_t *dav1d_ii_masks[N_BS_SIZES][3 /* 444/luma, 422, 420 */] + [N_INTER_INTRA_PRED_MODES]; + +#endif /* DAV1D_SRC_WEDGE_H */ diff --git a/third_party/dav1d/src/win32/thread.c b/third_party/dav1d/src/win32/thread.c new file mode 100644 index 0000000000..b89bd6b165 --- /dev/null +++ b/third_party/dav1d/src/win32/thread.c @@ -0,0 +1,99 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if defined(_WIN32) + +#include +#include +#include + +#include "common/attributes.h" + +#include "src/thread.h" + +static HRESULT (WINAPI *set_thread_description)(HANDLE, PCWSTR); + +COLD void dav1d_init_thread(void) { +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + HANDLE kernel32 = GetModuleHandleW(L"kernel32.dll"); + if (kernel32) + set_thread_description = + (void*)GetProcAddress(kernel32, "SetThreadDescription"); +#endif +} + +#undef dav1d_set_thread_name +COLD void dav1d_set_thread_name(const wchar_t *const name) { + if (set_thread_description) /* Only available since Windows 10 1607 */ + set_thread_description(GetCurrentThread(), name); +} + +static COLD unsigned __stdcall thread_entrypoint(void *const data) { + pthread_t *const t = data; + t->arg = t->func(t->arg); + return 0; +} + +COLD int dav1d_pthread_create(pthread_t *const thread, + const pthread_attr_t *const attr, + void *(*const func)(void*), void *const arg) +{ + const unsigned stack_size = attr ? attr->stack_size : 0; + thread->func = func; + thread->arg = arg; + thread->h = (HANDLE)_beginthreadex(NULL, stack_size, thread_entrypoint, thread, + STACK_SIZE_PARAM_IS_A_RESERVATION, NULL); + return !thread->h; +} + +COLD int dav1d_pthread_join(pthread_t *const thread, void **const res) { + if (WaitForSingleObject(thread->h, INFINITE)) + return 1; + + if (res) + *res = thread->arg; + + return !CloseHandle(thread->h); +} + +COLD int dav1d_pthread_once(pthread_once_t *const once_control, + void (*const init_routine)(void)) +{ + BOOL pending = FALSE; + + if (InitOnceBeginInitialize(once_control, 0, &pending, NULL) != TRUE) + return 1; + + if (pending == TRUE) + init_routine(); + + return !InitOnceComplete(once_control, 0, NULL); +} + +#endif diff --git a/third_party/dav1d/src/x86/cdef.h b/third_party/dav1d/src/x86/cdef.h new file mode 100644 index 0000000000..553d650741 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef.h @@ -0,0 +1,87 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/cdef.h" + +#define decl_cdef_fns(ext) \ + decl_cdef_fn(BF(dav1d_cdef_filter_4x4, ext)); \ + decl_cdef_fn(BF(dav1d_cdef_filter_4x8, ext)); \ + decl_cdef_fn(BF(dav1d_cdef_filter_8x8, ext)) + +decl_cdef_fns(avx512icl); +decl_cdef_fns(avx2); +decl_cdef_fns(sse4); +decl_cdef_fns(ssse3); +decl_cdef_fns(sse2); + +decl_cdef_dir_fn(BF(dav1d_cdef_dir, avx2)); +decl_cdef_dir_fn(BF(dav1d_cdef_dir, sse4)); +decl_cdef_dir_fn(BF(dav1d_cdef_dir, ssse3)); + +static ALWAYS_INLINE void cdef_dsp_init_x86(Dav1dCdefDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + +#if BITDEPTH == 8 + if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; + + c->fb[0] = BF(dav1d_cdef_filter_8x8, sse2); + c->fb[1] = BF(dav1d_cdef_filter_4x8, sse2); + c->fb[2] = BF(dav1d_cdef_filter_4x4, sse2); +#endif + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + + c->dir = BF(dav1d_cdef_dir, ssse3); + c->fb[0] = BF(dav1d_cdef_filter_8x8, ssse3); + c->fb[1] = BF(dav1d_cdef_filter_4x8, ssse3); + c->fb[2] = BF(dav1d_cdef_filter_4x4, ssse3); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; + + c->dir = BF(dav1d_cdef_dir, sse4); +#if BITDEPTH == 8 + c->fb[0] = BF(dav1d_cdef_filter_8x8, sse4); + c->fb[1] = BF(dav1d_cdef_filter_4x8, sse4); + c->fb[2] = BF(dav1d_cdef_filter_4x4, sse4); +#endif + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->dir = BF(dav1d_cdef_dir, avx2); + c->fb[0] = BF(dav1d_cdef_filter_8x8, avx2); + c->fb[1] = BF(dav1d_cdef_filter_4x8, avx2); + c->fb[2] = BF(dav1d_cdef_filter_4x4, avx2); + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + + c->fb[0] = BF(dav1d_cdef_filter_8x8, avx512icl); + c->fb[1] = BF(dav1d_cdef_filter_4x8, avx512icl); + c->fb[2] = BF(dav1d_cdef_filter_4x4, avx512icl); +#endif +} diff --git a/third_party/dav1d/src/x86/cdef16_avx2.asm b/third_party/dav1d/src/x86/cdef16_avx2.asm new file mode 100644 index 0000000000..4c8d3bca43 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef16_avx2.asm @@ -0,0 +1,877 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA + +%macro DIR_TABLE 1 ; stride + db 1 * %1 + 0, 2 * %1 + 0 + db 1 * %1 + 0, 2 * %1 - 2 + db -1 * %1 + 2, -2 * %1 + 4 + db 0 * %1 + 2, -1 * %1 + 4 + db 0 * %1 + 2, 0 * %1 + 4 + db 0 * %1 + 2, 1 * %1 + 4 + db 1 * %1 + 2, 2 * %1 + 4 + db 1 * %1 + 0, 2 * %1 + 2 + db 1 * %1 + 0, 2 * %1 + 0 + db 1 * %1 + 0, 2 * %1 - 2 + db -1 * %1 + 2, -2 * %1 + 4 + db 0 * %1 + 2, -1 * %1 + 4 +%endmacro + +dir_table4: DIR_TABLE 16 +dir_table8: DIR_TABLE 32 +pri_taps: dw 4, 4, 3, 3, 2, 2, 3, 3 + +dir_shift: times 2 dw 0x4000 + times 2 dw 0x1000 + +pw_2048: times 2 dw 2048 +pw_m16384: times 2 dw -16384 + +cextern cdef_dir_8bpc_avx2.main + +SECTION .text + +%macro CDEF_FILTER 2 ; w, h + DEFINE_ARGS dst, stride, _, dir, pridmp, pri, sec, tmp + movifnidn prid, r5m + movifnidn secd, r6m + mov dird, r7m + vpbroadcastd m8, [base+pw_2048] + lea dirq, [base+dir_table%1+dirq*2] + test prid, prid + jz .sec_only +%if WIN64 + vpbroadcastw m6, prim + movaps [rsp+16*0], xmm9 + movaps [rsp+16*1], xmm10 +%else + movd xm6, prid + vpbroadcastw m6, xm6 +%endif + lzcnt pridmpd, prid + rorx tmpd, prid, 2 + cmp dword r10m, 0xfff ; if (bpc == 12) + cmove prid, tmpd ; pri >>= 2 + mov tmpd, r8m ; damping + and prid, 4 + sub tmpd, 31 + vpbroadcastd m9, [base+pri_taps+priq+8*0] + vpbroadcastd m10, [base+pri_taps+priq+8*1] + test secd, secd + jz .pri_only +%if WIN64 + movaps r8m, xmm13 + vpbroadcastw m13, secm + movaps r4m, xmm11 + movaps r6m, xmm12 +%else + movd xm0, secd + vpbroadcastw m13, xm0 +%endif + lzcnt secd, secd + xor prid, prid + add pridmpd, tmpd + cmovs pridmpd, prid + add secd, tmpd + lea tmpq, [px] + mov [pri_shift], pridmpq + mov [sec_shift], secq +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri_sec +%endrep +%if WIN64 + movaps xmm11, r4m + movaps xmm12, r6m + movaps xmm13, r8m +%endif + jmp .pri_end +.pri_only: + add pridmpd, tmpd + cmovs pridmpd, secd + lea tmpq, [px] + mov [pri_shift], pridmpq +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri +%endrep +.pri_end: +%if WIN64 + movaps xmm9, [rsp+16*0] + movaps xmm10, [rsp+16*1] +%endif +.end: + RET +.sec_only: + mov tmpd, r8m ; damping +%if WIN64 + vpbroadcastw m6, secm +%else + movd xm6, secd + vpbroadcastw m6, xm6 +%endif + tzcnt secd, secd + sub tmpd, secd + mov [sec_shift], tmpq + lea tmpq, [px] +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).sec +%endrep + jmp .end +%if %1 == %2 +ALIGN function_align +.pri: + movsx offq, byte [dirq+4] ; off_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] ; 0 2 1 3 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0p0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0p1 +%else + mova xm1, [tmpq+32*0] + vinserti128 m1, [tmpq+32*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+5] ; off_k1 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m0, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1p0 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1p1 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 ; adiff_k1p0 + paddw m0, m2 ; constrain(diff_k0) + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m0, m9 ; pri_tap_k0 + pmullw m7, m10 ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + paddw m0, m1 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r9 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] +%endif + ret +ALIGN function_align +.sec: + movsx offq, byte [dirq+8] ; off1_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0s1 +%else + mova xm1, [tmpq+32*0] + vinserti128 m1, [tmpq+32*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+0] ; off2_k0 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k0s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k0s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+9] ; off1_k1 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + psignw m2, m3 ; constrain(diff_k0s1) + pabsw m3, m4 ; adiff_k0s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k1s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k1s1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+1] ; off2_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + paddw m0, m7 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k1s3) + paddw m0, m4 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + paddw m0, m1 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r9 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] +%endif + ret +ALIGN function_align +.pri_sec: + movsx offq, byte [dirq+8] ; off2_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0s1 +%else + mova xm1, [dstq+strideq*0] + vinserti128 m1, [dstq+strideq*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+0] ; off3_k0 + pmaxsw m11, m2, m3 + pminuw m12, m2, m3 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m13, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m13, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k0s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k0s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+9] ; off2_k1 + psignw m2, m3 ; constrain(diff_k0s1) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + paddw m0, m2 + pabsw m3, m4 ; adiff_k0s2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m13, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m13, m3 + pminsw m4, m2 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k1s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k1s1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+1] ; off3_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + pmaxsw m11, m2 + pminuw m12, m2 + pmaxsw m11, m3 + pminuw m12, m3 + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m13, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m13, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+4] ; off1_k0 + paddw m0, m7 + psignw m2, m3 ; constrain(diff_k1s1) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m13, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m13, m3 + pminsw m4, m2 + paddw m0, m7 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0p0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0p1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+5] ; off1_k1 + psignw m4, m5 ; constrain(diff_k1s3) + pmaxsw m11, m2 + pminuw m12, m2 + pmaxsw m11, m3 + pminuw m12, m3 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + paddw m0, m4 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m7, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m7, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1p0 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1p1 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + psignw m2, m3 ; constrain(diff_k0p1) + paddw m7, m2 ; constrain(diff_k0) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + pabsw m3, m4 ; adiff_k1p0 + pmullw m7, m9 ; pri_tap_k0 + paddw m0, m7 + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m7, m10 ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + pmaxsw m11, m1 + pminuw m12, m1 + paddw m0, m1 + pminsw m0, m11 + pmaxsw m0, m12 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r9 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] +%endif + ret +%endif +%endmacro + +INIT_YMM avx2 +cglobal cdef_filter_4x4_16bpc, 5, 10, 9, 16*10, dst, stride, left, top, bot, \ + pri, sec, edge +%if WIN64 + %define px rsp+16*6 + %define offq r8 + %define pri_shift rsp+16*2 + %define sec_shift rsp+16*3 +%else + %define px rsp+16*4 + %define offq r4 + %define pri_shift rsp+16*0 + %define sec_shift rsp+16*1 +%endif + %define base r8-dir_table4 + mov edged, r9m + lea r8, [dir_table4] + movu xm0, [dstq+strideq*0] + movu xm1, [dstq+strideq*1] + lea r9, [strideq*3] + movu xm2, [dstq+strideq*2] + movu xm3, [dstq+r9 ] + vpbroadcastd m7, [base+pw_m16384] + mova [px+16*0+0], xm0 + mova [px+16*1+0], xm1 + mova [px+16*2+0], xm2 + mova [px+16*3+0], xm3 + test edgeb, 4 ; HAVE_TOP + jz .no_top + movu xm0, [topq+strideq*0] + movu xm1, [topq+strideq*1] + mova [px-16*2+0], xm0 + mova [px-16*1+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-16*2-4], xm0 + movd [px-16*1-4], xm1 + jmp .top_done +.no_top: + mova [px-16*2+0], m7 +.top_no_left: + movd [px-16*2-4], xm7 + movd [px-16*1-4], xm7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movu xm0, [botq+strideq*0] + movu xm1, [botq+strideq*1] + mova [px+16*4+0], xm0 + mova [px+16*5+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd xm0, [botq+strideq*0-4] + movd xm1, [botq+strideq*1-4] + movd [px+16*4-4], xm0 + movd [px+16*5-4], xm1 + jmp .bottom_done +.no_bottom: + mova [px+16*4+0], m7 +.bottom_no_left: + movd [px+16*4-4], xm7 + movd [px+16*5-4], xm7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+16*0-4], xm0 + movd [px+16*1-4], xm1 + movd [px+16*2-4], xm2 + movd [px+16*3-4], xm3 + jmp .left_done +.no_left: + REPX {movd [px+16*x-4], xm7}, 0, 1, 2, 3 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+16*x+8], xm7}, -2, -1, 0, 1, 2, 3, 4, 5 +.padding_done: + CDEF_FILTER 4, 4 + +cglobal cdef_filter_4x8_16bpc, 5, 10, 9, 16*14, dst, stride, left, top, bot, \ + pri, sec, edge + mov edged, r9m + movu xm0, [dstq+strideq*0] + movu xm1, [dstq+strideq*1] + lea r9, [strideq*3] + movu xm2, [dstq+strideq*2] + movu xm3, [dstq+r9 ] + lea r6, [dstq+strideq*4] + movu xm4, [r6 +strideq*0] + movu xm5, [r6 +strideq*1] + movu xm6, [r6 +strideq*2] + movu xm7, [r6 +r9 ] + lea r8, [dir_table4] + mova [px+16*0+0], xm0 + mova [px+16*1+0], xm1 + mova [px+16*2+0], xm2 + mova [px+16*3+0], xm3 + mova [px+16*4+0], xm4 + mova [px+16*5+0], xm5 + mova [px+16*6+0], xm6 + mova [px+16*7+0], xm7 + vpbroadcastd m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movu xm0, [topq+strideq*0] + movu xm1, [topq+strideq*1] + mova [px-16*2+0], xm0 + mova [px-16*1+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-16*2-4], xm0 + movd [px-16*1-4], xm1 + jmp .top_done +.no_top: + mova [px-16*2+0], m7 +.top_no_left: + movd [px-16*2-4], xm7 + movd [px-16*1-4], xm7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movu xm0, [botq+strideq*0] + movu xm1, [botq+strideq*1] + mova [px+16*8+0], xm0 + mova [px+16*9+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd xm0, [botq+strideq*0-4] + movd xm1, [botq+strideq*1-4] + movd [px+16*8-4], xm0 + movd [px+16*9-4], xm1 + jmp .bottom_done +.no_bottom: + mova [px+16*8+0], m7 +.bottom_no_left: + movd [px+16*8-4], xm7 + movd [px+16*9-4], xm7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+16*0-4], xm0 + movd [px+16*1-4], xm1 + movd [px+16*2-4], xm2 + movd [px+16*3-4], xm3 + movd xm0, [leftq+4*4] + movd xm1, [leftq+4*5] + movd xm2, [leftq+4*6] + movd xm3, [leftq+4*7] + movd [px+16*4-4], xm0 + movd [px+16*5-4], xm1 + movd [px+16*6-4], xm2 + movd [px+16*7-4], xm3 + jmp .left_done +.no_left: + REPX {movd [px+16*x-4], xm7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+16*x+8], xm7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 4, 8 + +cglobal cdef_filter_8x8_16bpc, 5, 9, 9, 32*13, dst, stride, left, top, bot, \ + pri, sec, edge +%if WIN64 + %define px rsp+32*4 +%else + %define px rsp+32*3 +%endif + %define base r8-dir_table8 + mov edged, r9m + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea r6, [dstq+strideq*2] + movu m2, [r6 +strideq*0] + movu m3, [r6 +strideq*1] + lea r6, [r6 +strideq*2] + movu m4, [r6 +strideq*0] + movu m5, [r6 +strideq*1] + lea r6, [r6 +strideq*2] + movu m6, [r6 +strideq*0] + movu m7, [r6 +strideq*1] + lea r8, [dir_table8] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + mova [px+32*4+0], m4 + mova [px+32*5+0], m5 + mova [px+32*6+0], m6 + mova [px+32*7+0], m7 + vpbroadcastd m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-32*2-4], xm0 + movd [px-32*1-4], xm1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], xm7 + movd [px-32*1-4], xm7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movu m0, [botq+strideq*0] + movu m1, [botq+strideq*1] + mova [px+32*8+0], m0 + mova [px+32*9+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd xm0, [botq+strideq*0-4] + movd xm1, [botq+strideq*1-4] + movd [px+32*8-4], xm0 + movd [px+32*9-4], xm1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+0], m7 + mova [px+32*9+0], m7 +.bottom_no_left: + movd [px+32*8-4], xm7 + movd [px+32*9-4], xm7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+32*0-4], xm0 + movd [px+32*1-4], xm1 + movd [px+32*2-4], xm2 + movd [px+32*3-4], xm3 + movd xm0, [leftq+4*4] + movd xm1, [leftq+4*5] + movd xm2, [leftq+4*6] + movd xm3, [leftq+4*7] + movd [px+32*4-4], xm0 + movd [px+32*5-4], xm1 + movd [px+32*6-4], xm2 + movd [px+32*7-4], xm3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], xm7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+16], xm7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 8, 8 + +cglobal cdef_dir_16bpc, 4, 7, 6, src, stride, var, bdmax + lea r6, [dir_shift] + shr bdmaxd, 11 ; 0 for 10bpc, 1 for 12bpc + vpbroadcastd m4, [r6+bdmaxq*4] + lea r6, [strideq*3] + mova xm0, [srcq+strideq*0] + mova xm1, [srcq+strideq*1] + mova xm2, [srcq+strideq*2] + mova xm3, [srcq+r6 ] + lea srcq, [srcq+strideq*4] + vinserti128 m0, [srcq+r6 ], 1 + vinserti128 m1, [srcq+strideq*2], 1 + vinserti128 m2, [srcq+strideq*1], 1 + vinserti128 m3, [srcq+strideq*0], 1 + REPX {pmulhuw x, m4}, m0, m1, m2, m3 + jmp mangle(private_prefix %+ _cdef_dir_8bpc %+ SUFFIX).main + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef16_avx512.asm b/third_party/dav1d/src/x86/cdef16_avx512.asm new file mode 100644 index 0000000000..6d625a02a0 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef16_avx512.asm @@ -0,0 +1,622 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +cdef_perm: db 2, 18, 16, 18, 24, 19, 0, 19, 25, 20, 1, 20, 26, 21, 2, 21 + db 3, 26, 3, 26, 28, 27, 4, 27, 29, 28, -1, 28, 30, 29, -1, 29 + db 0, 34, 17, 34, 16, 35, 8, 35, 17, 36, 9, 36, 18, 37, 10, 37 + db 1, 42, 11, 42, 20, 43, 12, 43, 21, 44, -1, 44, 22, 45, -1, 45 +end_perm4: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30 + db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62 +edge_mask4: dw 0xff99, 0xff88, 0xff11, 0xff00 ; 0100, 0101, 0110, 0111 + dw 0x99ff, 0x88ff, 0x11ff, 0x00ff ; 1000, 1001, 1010, 1011 + dw 0x9999, 0x8888, 0x1111, 0x0000 ; 1100, 1101, 1110, 1111 +pri_taps4: dw 64, 32, 48, 48 ; left-shifted by 4 +cdef_dirs4: dw 8, 16, 8, 15, -7,-14, 1, -6 + dw 1, 2, 1, 10, 9, 18, 8, 17 + dw 8, 16, 8, 15, -7,-14, 1, -6 +deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +cdef_dirs8: db 32, 64, 32, 62,-30,-60, 2,-28 + db 2, 4, 2, 36, 34, 68, 32, 66 + db 32, 64, 32, 62,-30,-60, 2,-28 +pri_taps8: dw 4, 4, 2, 2, 3, 3, 3, 3 +sec_taps4: dw 32, 16 +pw_m16384: times 2 dw -16384 +pw_2048: times 2 dw 2048 +pd_268435568: dd 268435568 ; (1 << 28) + (7 << 4) +edge_mask8: dw 0x2121, 0x2020, 0x0101 + +SECTION .text + +%macro CONSTRAIN 7 ; dst, p, px, zero, tresh, shift, tmp + psubw %1, %2, %3 + pabsw %1, %1 + vpcmpgtw k1, %3, %2 + vpsrlvw %7, %1, %6 + psubusw %7, %5, %7 + pminsw %1, %7 + vpsubw %1{k1}, %4, %1 +%endmacro + +; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 +; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 +; L0 L1 00 01 02 03 04 05 b0 b1 b2 b3 b4 b5 b6 b7 +; L2 L3 10 11 12 13 14 15 B0 B1 B2 B3 B4 B5 B6 B7 + +INIT_ZMM avx512icl +cglobal cdef_filter_4x4_16bpc, 5, 7, 16, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%define base r6-cdef_dirs4 + lea r6, [cdef_dirs4] + movu xm3, [dstq+strideq*0] + vinserti32x4 ym3, [dstq+strideq*1], 1 + mova xm2, [leftq] + lea r2, [dstq+strideq*2] + vinserti32x4 m3, [r2+strideq*0], 2 + mova m5, [base+cdef_perm] + vinserti32x4 m3, [r2+strideq*1], 3 + vpermt2d m2, m5, m3 + vinserti32x4 m1, m2, [topq+strideq*0-4], 0 + vinserti32x4 m1, [topq+strideq*1-4], 1 + mov r3d, edgem + movifnidn prid, prim + punpcklwd m3, m3 ; px + psrlw m5, 8 + vpbroadcastd m0, [base+pd_268435568] + pxor m12, m12 + cmp r3d, 0x0f + jne .mask_edges + vinserti32x4 m2, [botq+strideq*0-4], 2 + vinserti32x4 m2, [botq+strideq*1-4], 3 +.main: + test prid, prid + jz .sec_only + lzcnt r4d, prid + rorx r3d, prid, 2 + vpbroadcastw m13, prim + cmp dword r10m, 0xfff ; if (bpc == 12) + cmove prid, r3d ; pri >>= 2 + mov r3d, dampingm + and prid, 4 + sub r3d, 31 + vpbroadcastd m15, [base+pri_taps4+priq] + xor prid, prid + add r4d, r3d + cmovns prid, r4d ; pri_shift + mov r4d, dirm + vpbroadcastw m14, prid + mov r5d, secm + vpbroadcastd m9, [base+cdef_dirs4+(r4+2)*4] + call .constrain + test r5d, r5d + jz .end_no_clip + lzcnt r5d, r5d + vpbroadcastw m13, secm + add r3d, r5d + pminuw m6, m3, m8 + pmaxsw m7, m3, m8 + pminuw m6, m9 + pmaxsw m7, m9 + call .constrain_sec + pminuw m6, m8 + pmaxsw m7, m8 + pminuw m6, m9 + pmaxsw m7, m9 + vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4] + call .constrain + pminuw m6, m8 + pmaxsw m7, m8 + pminuw m6, m9 + pmaxsw m7, m9 + psrldq m8, m6, 2 + vpshldd m3, m0, 8 + psrldq m9, m7, 2 + paddd m0, m3 + pminuw m6, m8 + psrldq m0, 1 + pmaxsw m7, m9 + pmaxsw m0, m6 + pminsw m0, m7 + vpmovdw ym0, m0 + jmp .end +.sec_only: + tzcnt r5d, secm + mov r3d, dampingm + vpbroadcastw m13, secm + mov r4d, dirm + sub r3d, r5d ; sec_shift + call .constrain_sec + vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4] + call .constrain +.end_no_clip: + mova ym1, [base+end_perm4] + vpshldd m3, m0, 8 ; (px << 8) + ((sum > -8) << 4) + paddd m0, m3 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + vpermb m0, m1, m0 +.end: + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm0, ym0, 1 + movq [r2+strideq*0], xm0 + movhps [r2+strideq*1], xm0 + RET +.mask_edges: + vpbroadcastd m6, [base+pw_m16384] + test r3b, 0x08 + jz .mask_edges_no_bottom ; avoid buffer overread + vinserti32x4 m2, [botq+strideq*0-4], 2 + vinserti32x4 m2, [botq+strideq*1-4], 3 + kmovw k1, [base+edge_mask4-8+r3*2] + jmp .mask_edges_main +.mask_edges_no_bottom: + kmovw k1, [base+edge_mask4+8+r3*2] +.mask_edges_main: + or r3d, 0x04 + vmovdqa32 m1{k1}, m6 ; edge pixels = -16384 + kmovw k1, [base+edge_mask4-8+r3*2] + vmovdqa32 m2{k1}, m6 + jmp .main +.constrain_sec: + vpbroadcastd m9, [base+cdef_dirs4+(r4+4)*4] + vpbroadcastw m14, r3d + vpbroadcastd m15, [base+sec_taps4] +.constrain: + paddw m8, m5, m9 + vpermi2w m8, m1, m2 ; k0p0 k1p0 + psubw m9, m5, m9 + vpermi2w m9, m1, m2 ; k0p1 k1p1 + CONSTRAIN m10, m8, m3, m12, m13, m14, m11 + vpdpwssd m0, m10, m15 + CONSTRAIN m10, m9, m3, m12, m13, m14, m11 + vpdpwssd m0, m10, m15 + ret + +; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65 +; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75 +; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 b0 b1 b2 b3 b4 b5 b6 b7 +; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 B0 B1 B2 B3 B4 B5 B6 B7 + +cglobal cdef_filter_4x8_16bpc, 5, 7, 22, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge + lea r6, [cdef_dirs4] + movu xm18, [dstq+strideq*0] + vinserti128 ym18, [dstq+strideq*1], 1 + mova xm1, [leftq+16*0] + mova xm2, [leftq+16*1] + lea r2, [strideq*3] + vinserti32x4 m18, [dstq+strideq*2], 2 + mova m5, [base+cdef_perm] + vinserti32x4 m18, [dstq+r2 ], 3 + vpermt2d m1, m5, m18 + vinserti32x4 m0, m1, [topq+strideq*0-4], 0 + vinserti32x4 m0, [topq+strideq*1-4], 1 + lea r3, [dstq+strideq*4] + movu xm19, [r3+strideq*0] + vinserti128 ym19, [r3+strideq*1], 1 + vinserti32x4 m19, [r3+strideq*2], 2 + vinserti32x4 m19, [r3+r2 ], 3 + mov r3d, edgem + movifnidn prid, prim + vpermt2d m2, m5, m19 + vpbroadcastd m16, [base+pd_268435568] + pxor m12, m12 + punpcklwd m18, m18 ; px (top) + psrlw m5, 8 + punpcklwd m19, m19 ; px (bottom) + mova m17, m16 + vshufi32x4 m1, m2, q3210 + cmp r3d, 0x0f + jne .mask_edges + vinserti32x4 m2, [botq+strideq*0-4], 2 + vinserti32x4 m2, [botq+strideq*1-4], 3 +.main: + test prid, prid + jz .sec_only + lzcnt r4d, prid + rorx r3d, prid, 2 + vpbroadcastw m13, prim + cmp dword r10m, 0xfff ; if (bpc == 12) + cmove prid, r3d ; pri >>= 2 + mov r3d, dampingm + and prid, 4 + sub r3d, 31 + vpbroadcastd m15, [base+pri_taps4+priq] + xor prid, prid + add r4d, r3d + cmovns prid, r4d ; pri_shift + mov r4d, dirm + vpbroadcastw m14, prid + mov r5d, secm + vpbroadcastd m9, [base+cdef_dirs4+(r4+2)*4] + call .constrain + test r5d, r5d + jz .end_no_clip + lzcnt r5d, r5d + vpbroadcastw m13, secm + add r3d, r5d + pminuw m3, m18, m6 + pmaxsw m4, m18, m6 + pminuw m20, m19, m7 + pmaxsw m21, m19, m7 + pminuw m3, m8 + pmaxsw m4, m8 + pminuw m20, m9 + pmaxsw m21, m9 + call .constrain_sec + pminuw m3, m6 + pmaxsw m4, m6 + pminuw m20, m7 + pmaxsw m21, m7 + pminuw m3, m8 + pmaxsw m4, m8 + pminuw m20, m9 + pmaxsw m21, m9 + vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4] + call .constrain + pminuw m3, m6 + pmaxsw m4, m6 + mov r3, 0xcccccccccccccccc + pminuw m20, m7 + pmaxsw m21, m7 + kmovq k1, r3 + pminuw m3, m8 + pmaxsw m4, m8 + pminuw m20, m9 + pmaxsw m21, m9 + vbroadcasti32x4 m0, [base+deint_shuf] + vpshldd m6, m20, m3, 16 + vmovdqu8 m3{k1}, m20 + vpshldd m18, m16, 8 + vpshldd m7, m21, m4, 16 + vmovdqu8 m4{k1}, m21 + vpshldd m19, m17, 8 + pminuw m3, m6 + paddd m16, m18 + pmaxsw m4, m7 + paddd m17, m19 + psrldq m16, 1 + palignr m16{k1}, m17, m17, 15 + lea r6, [dstq+strideq*4] + pmaxsw m16, m3 + pminsw m16, m4 + pshufb m16, m0 + movq [dstq+strideq*0], xm16 + movhps [r6 +strideq*0], xm16 + vextracti128 xm17, ym16, 1 + movq [dstq+strideq*1], xm17 + movhps [r6 +strideq*1], xm17 + vextracti32x4 xm17, m16, 2 + movq [dstq+strideq*2], xm17 + movhps [r6 +strideq*2], xm17 + vextracti32x4 xm16, m16, 3 + movq [dstq+r2 ], xm16 + movhps [r6 +r2 ], xm16 + RET +.sec_only: + mov r4d, dirm + tzcnt r5d, secm + mov r3d, dampingm + vpbroadcastw m13, secm + sub r3d, r5d ; sec_shift + call .constrain_sec + vpbroadcastd m9, [base+cdef_dirs4+(r4+0)*4] + call .constrain +.end_no_clip: + mova ym20, [base+end_perm4] + vpshldd m18, m16, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m19, m17, 8 + paddd m16, m18 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddd m17, m19 + vpermb m16, m20, m16 + vpermb m17, m20, m17 + movq [dstq+strideq*0], xm16 + movhps [dstq+strideq*1], xm16 + vextracti128 xm16, ym16, 1 + movq [dstq+strideq*2], xm16 + movhps [dstq+r2 ], xm16 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm17 + movhps [dstq+strideq*1], xm17 + vextracti128 xm17, ym17, 1 + movq [dstq+strideq*2], xm17 + movhps [dstq+r2 ], xm17 + RET +.mask_edges: + vpbroadcastd m6, [base+pw_m16384] + test r3b, 0x08 + jz .mask_edges_no_bottom ; avoid buffer overread + vinserti32x4 m2, [botq+strideq*0-4], 2 + vinserti32x4 m2, [botq+strideq*1-4], 3 + kmovw k1, [base+edge_mask4-8+r3*2] + jmp .mask_edges_main +.mask_edges_no_bottom: + kmovw k1, [base+edge_mask4+8+r3*2] +.mask_edges_main: + mov r4d, r3d + or r3d, 0x0c + vmovdqa32 m0{k1}, m6 ; edge pixels = -16384 + kmovw k1, [base+edge_mask4-8+r3*2] + or r4d, 0x04 + vmovdqa32 m1{k1}, m6 + kmovw k1, [base+edge_mask4-8+r4*2] + vmovdqa32 m2{k1}, m6 + jmp .main +.constrain_sec: + vpbroadcastd m9, [base+cdef_dirs4+(r4+4)*4] + vpbroadcastw m14, r3d + vpbroadcastd m15, [base+sec_taps4] +.constrain: + paddw m7, m5, m9 + mova m6, m0 + vpermt2w m6, m7, m1 ; k0p0 k1p0 (top) + psubw m9, m5, m9 + mova m8, m0 + vpermi2w m7, m1, m2 ; k0p0 k1p0 (bottom) + CONSTRAIN m10, m6, m18, m12, m13, m14, m11 + vpermt2w m8, m9, m1 ; k0p1 k1p1 (top) + vpdpwssd m16, m10, m15 + CONSTRAIN m10, m7, m19, m12, m13, m14, m11 + vpermi2w m9, m1, m2 ; k0p1 k1p1 (bottom) + vpdpwssd m17, m10, m15 + CONSTRAIN m10, m8, m18, m12, m13, m14, m11 + vpdpwssd m16, m10, m15 + CONSTRAIN m10, m9, m19, m12, m13, m14, m11 + vpdpwssd m17, m10, m15 + ret + +cglobal cdef_filter_8x8_16bpc, 5, 7, 22, 64*6, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%define base r6-cdef_dirs8 + lea r6, [cdef_dirs8] + movu ym17, [dstq+strideq*0] + vinserti32x8 m17, [dstq+strideq*1], 1 + movq xm4, [leftq+8*0] + movq xm5, [leftq+8*1] + psrld m2, [base+cdef_perm], 16 + movq xm6, [leftq+8*2] + movq xm7, [leftq+8*3] + lea r2, [strideq*3] + movu ym16, [topq+strideq*0-4] + vinserti32x8 m16, [topq+strideq*1-4], 1 + lea r3, [dstq+strideq*4] + movu ym18, [dstq+strideq*2] + vinserti32x8 m18, [dstq+r2 ], 1 + movu ym19, [r3+strideq*0] + vinserti32x8 m19, [r3+strideq*1], 1 + movu ym20, [r3+strideq*2] + vinserti32x8 m20, [r3+r2 ], 1 + vshufi32x4 m0, m17, m18, q2020 ; px (top) + mov r3d, edgem + vshufi32x4 m1, m19, m20, q2020 ; px (bottom) + movifnidn prid, prim + vpermt2d m17, m2, m4 + vpermt2d m18, m2, m5 + pxor m12, m12 + vpermt2d m19, m2, m6 + vpermt2d m20, m2, m7 + cmp r3d, 0x0f + jne .mask_edges + movu ym21, [botq+strideq*0-4] + vinserti32x8 m21, [botq+strideq*1-4], 1 +.main: + mova [rsp+64*0], m16 ; top + mova [rsp+64*1], m17 ; 0 1 + mova [rsp+64*2], m18 ; 2 3 + mova [rsp+64*3], m19 ; 4 5 + mova [rsp+64*4], m20 ; 6 7 + mova [rsp+64*5], m21 ; bottom + test prid, prid + jz .sec_only + lzcnt r4d, prid + rorx r3d, prid, 2 + vpbroadcastw m13, prim + cmp dword r10m, 0xfff ; if (bpc == 12) + cmove prid, r3d ; pri >>= 2 + mov r3d, dampingm + and prid, 4 + sub r3d, 31 + add r4d, r3d ; pri_shift + vpbroadcastw m14, r4d + mov r4d, dirm + vpbroadcastd m2, [base+pri_taps8+priq*2+0] + vpbroadcastd m3, [base+pri_taps8+priq*2+4] + movsx r5, byte [base+cdef_dirs8+(r4+2)*2+0] ; k0off1 + pmaxsw m14, m12 + call .constrain + mov r5d, secm + pmullw m16, m8, m2 + pmullw m17, m9, m2 + test r5d, r5d + jnz .pri_sec + movsx r5, byte [base+cdef_dirs8+(r4+2)*2+1] ; k1off1 + call .constrain + pmullw m8, m3 + pmullw m9, m3 + jmp .end_no_clip +.pri_sec: + lzcnt r5d, r5d + add r3d, r5d ; sec_shift + movsx r5, byte [base+cdef_dirs8+(r4+2)*2+1] ; k1off1 + pminuw m18, m0, m4 + pmaxsw m19, m0, m4 + pminuw m20, m1, m5 + pmaxsw m21, m1, m5 + call .min_max_constrain2 + movsx r5, byte [base+cdef_dirs8+(r4+0)*2+0] ; k0off2 + pmullw m8, m3 + pmullw m9, m3 + vpbroadcastw m13, secm + vpbroadcastw m14, r3d + paddw m16, m8 + paddw m17, m9 + call .min_max_constrain + movsx r5, byte [base+cdef_dirs8+(r4+4)*2+0] ; k0off3 + mova m2, m8 + mova m3, m9 + call .min_max_constrain + movsx r5, byte [base+cdef_dirs8+(r4+0)*2+1] ; k1off2 + paddw m2, m8 + paddw m3, m9 + call .min_max_constrain + movsx r5, byte [base+cdef_dirs8+(r4+4)*2+1] ; k1off3 + paddw m2, m2 + paddw m3, m3 + paddw m16, m8 + paddw m17, m9 + call .min_max_constrain + vpbroadcastd m10, [base+pw_2048] + paddw m16, m2 + paddw m17, m3 + paddw m16, m8 + paddw m17, m9 + psraw m8, m16, 15 + psraw m9, m17, 15 + paddw m16, m8 + paddw m17, m9 + pmulhrsw m16, m10 + pmulhrsw m17, m10 + pminuw m18, m4 + pmaxsw m19, m4 + pminuw m20, m5 + pmaxsw m21, m5 + pminuw m18, m6 + pmaxsw m19, m6 + pminuw m20, m7 + pmaxsw m21, m7 + paddw m16, m0 + paddw m17, m1 + pmaxsw m16, m18 + pmaxsw m17, m20 + pminsw m16, m19 + pminsw m17, m21 + jmp .end +.sec_only: + tzcnt r5d, secm + mov r4d, dirm + mov r3d, dampingm + vpbroadcastw m13, secm + sub r3d, r5d + movsx r5, byte [base+cdef_dirs8+(r4+0)*2+0] + vpbroadcastw m14, r3d + call .constrain + movsx r5, byte [base+cdef_dirs8+(r4+4)*2+0] + mova m16, m8 + mova m17, m9 + call .constrain + movsx r5, byte [base+cdef_dirs8+(r4+0)*2+1] + paddw m16, m8 + paddw m17, m9 + call .constrain + movsx r5, byte [base+cdef_dirs8+(r4+4)*2+1] + paddw m16, m16 + paddw m17, m17 + paddw m16, m8 + paddw m17, m9 + call .constrain +.end_no_clip: + vpbroadcastd m10, [base+pw_2048] + paddw m16, m8 + paddw m17, m9 + psraw m8, m16, 15 + psraw m9, m17, 15 + paddw m16, m8 + paddw m17, m9 + pmulhrsw m16, m10 + pmulhrsw m17, m10 + paddw m16, m0 + paddw m17, m1 +.end: + mova [dstq+strideq*0], xm16 + vextracti128 [dstq+strideq*1], ym16, 1 + vextracti32x4 [dstq+strideq*2], m16, 2 + vextracti32x4 [dstq+r2 ], m16, 3 + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm17 + vextracti128 [dstq+strideq*1], ym17, 1 + vextracti32x4 [dstq+strideq*2], m17, 2 + vextracti32x4 [dstq+r2 ], m17, 3 + RET +.mask_edges: + vpbroadcastd m2, [base+pw_m16384] + test r3b, 0x08 + jz .mask_edges_no_bottom ; avoid buffer overread + movu ym21, [botq+strideq*0-4] + vinserti32x8 m21, [botq+strideq*1-4], 1 + jmp .mask_edges_top +.mask_edges_no_bottom: + mova m21, m2 +.mask_edges_top: + test r3b, 0x04 + jnz .mask_edges_main + mova m16, m2 +.mask_edges_main: + and r3d, 0x03 + cmp r3d, 0x03 + je .main + kmovw k1, [base+edge_mask8+r3*2] + vmovdqa32 m16{k1}, m2 ; edge pixels = -16384 + vmovdqa32 m17{k1}, m2 + vmovdqa32 m18{k1}, m2 + vmovdqa32 m19{k1}, m2 + vmovdqa32 m20{k1}, m2 + vmovdqa32 m21{k1}, m2 + jmp .main +ALIGN function_align +.min_max_constrain: + pminuw m18, m4 + pmaxsw m19, m4 + pminuw m20, m5 + pmaxsw m21, m5 +.min_max_constrain2: + pminuw m18, m6 + pmaxsw m19, m6 + pminuw m20, m7 + pmaxsw m21, m7 +.constrain: + %define tmp rsp+gprsize+68 + movu m4, [tmp+r5+64*0] + vshufi32x4 m4, [tmp+r5+64*1], q2020 ; k0p0 (top) + movu m5, [tmp+r5+64*2] + vshufi32x4 m5, [tmp+r5+64*3], q2020 ; k0p0 (bottom) + neg r5 + movu m6, [tmp+r5+64*0] + vshufi32x4 m6, [tmp+r5+64*1], q2020 ; k0p1 (top) + movu m7, [tmp+r5+64*2] + vshufi32x4 m7, [tmp+r5+64*3], q2020 ; k0p1 (bottom) + CONSTRAIN m8, m4, m0, m12, m13, m14, m15 + CONSTRAIN m9, m5, m1, m12, m13, m14, m15 + CONSTRAIN m10, m6, m0, m12, m13, m14, m15 + CONSTRAIN m11, m7, m1, m12, m13, m14, m15 + paddw m8, m10 + paddw m9, m11 + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef16_sse.asm b/third_party/dav1d/src/x86/cdef16_sse.asm new file mode 100644 index 0000000000..1bd67ace64 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef16_sse.asm @@ -0,0 +1,1033 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; Copyright (c) 2017-2021, The rav1e contributors +; Copyright (c) 2021, Nathan Egge +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +%macro DUP8 1-* + %rep %0 + times 8 dw %1 + %rotate 1 + %endrep +%endmacro + +pri_taps: DUP8 4, 2, 3, 3 +dir_table: db 1 * 32 + 0, 2 * 32 + 0 + db 1 * 32 + 0, 2 * 32 - 2 + db -1 * 32 + 2, -2 * 32 + 4 + db 0 * 32 + 2, -1 * 32 + 4 + db 0 * 32 + 2, 0 * 32 + 4 + db 0 * 32 + 2, 1 * 32 + 4 + db 1 * 32 + 2, 2 * 32 + 4 + db 1 * 32 + 0, 2 * 32 + 2 + db 1 * 32 + 0, 2 * 32 + 0 + db 1 * 32 + 0, 2 * 32 - 2 + db -1 * 32 + 2, -2 * 32 + 4 + db 0 * 32 + 2, -1 * 32 + 4 + +dir_shift: times 4 dw 0x4000 + times 4 dw 0x1000 + +pw_128: times 4 dw 128 +pw_2048: times 8 dw 2048 +pw_m16384: times 8 dw -16384 + +cextern cdef_dir_8bpc_ssse3.main +cextern cdef_dir_8bpc_sse4.main +cextern shufw_6543210x + +SECTION .text + +%if ARCH_X86_32 +DECLARE_REG_TMP 5, 3 +%elif WIN64 +DECLARE_REG_TMP 8, 4 +%else +DECLARE_REG_TMP 8, 6 +%endif + +%macro CDEF_FILTER 2 ; w, h +%if ARCH_X86_64 + DEFINE_ARGS dst, stride, _, tmp, pridmp, pri, sec, dir + mova m8, [base+pw_2048] +%else + DEFINE_ARGS dst, pridmp, tmp, sec, pri, _, dir + %define m8 [base+pw_2048] + %define m9 [rsp+16*1+gprsize] + %define m10 [rsp+16*2+gprsize] +%endif + movifnidn prid, r5m + movifnidn secd, r6m + test prid, prid + jz .sec_only + movd m6, r5m +%if ARCH_X86_32 + mov [rsp+24], pridmpd +%endif + bsr pridmpd, prid + lea tmpd, [priq*4] + cmp dword r10m, 0x3ff ; if (bpc == 10) + cmove prid, tmpd ; pri <<= 2 + mov tmpd, r8m ; damping + mov dird, r7m + and prid, 16 + pshufb m6, m7 ; splat + lea dirq, [base+dir_table+dirq*2] + lea priq, [base+pri_taps+priq*2] + test secd, secd + jz .pri_only + mova [rsp], m6 + movd m6, secd + tzcnt secd, secd + sub pridmpd, tmpd + sub tmpd, secd + pshufb m6, m7 + xor secd, secd + neg pridmpd + cmovs pridmpd, secd +%if ARCH_X86_32 + mov [pri_shift+4], secd + mov [sec_shift+4], secd +%endif + mov [pri_shift+0], pridmpq + mov [sec_shift+0], tmpq + lea tmpq, [px] +%if WIN64 + movaps r4m, m9 + movaps r6m, m10 +%elif ARCH_X86_32 + mov pridmpd, [rsp+24] +%endif +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri_sec +%endrep +%if WIN64 + movaps m9, r4m + movaps m10, r6m +%endif + jmp .end +.pri_only: + sub tmpd, pridmpd + cmovs tmpd, secd +%if ARCH_X86_32 + mov pridmpd, [rsp+24] + mov [pri_shift+4], secd +%endif + mov [pri_shift+0], tmpq + lea tmpq, [px] +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri +%endrep +.end: + RET +.sec_only: + mov tmpd, r8m ; damping + movd m6, r6m + tzcnt secd, secd + mov dird, r7m + pshufb m6, m7 + sub tmpd, secd + lea dirq, [base+dir_table+dirq*2] +%if ARCH_X86_32 + mov [sec_shift+4], prid +%endif + mov [sec_shift+0], tmpq + lea tmpq, [px] +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).sec +%endrep + jmp .end +%if %1 == %2 + %if ARCH_X86_64 + DEFINE_ARGS dst, stride, _, tmp, off, pri, _, dir + %else + DEFINE_ARGS dst, stride, tmp, off, pri, _, dir + %endif +ALIGN function_align +.pri: + movsx offq, byte [dirq+4] ; off_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0p0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0p1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+5] ; off_k1 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m0, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1p0 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1p1 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 ; adiff_k1p0 + paddw m0, m2 ; constrain(diff_k0) + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m0, [priq+16*0] ; pri_tap_k0 + pmullw m7, [priq+16*1] ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +ALIGN function_align +.sec: + movsx offq, byte [dirq+8] ; off1_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0s1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+0] ; off2_k0 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k0s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k0s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+9] ; off1_k1 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + psignw m2, m3 ; constrain(diff_k0s1) + pabsw m3, m4 ; adiff_k0s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k1s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k1s1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+1] ; off2_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + paddw m0, m7 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k1s3) + paddw m0, m4 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +ALIGN function_align +.pri_sec: + movsx offq, byte [dirq+8] ; off2_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0s1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+0] ; off3_k0 + pabsw m4, m2 +%if ARCH_X86_64 + pabsw m10, m3 + pmaxsw m9, m2, m3 + pminsw m10, m4 +%else + pabsw m7, m3 + pmaxsw m5, m2, m3 + pminsw m4, m7 + mova m9, m5 + mova m10, m4 +%endif + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k0s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k0s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+9] ; off2_k1 + pabsw m7, m4 + psignw m2, m3 + pabsw m3, m5 ; constrain(diff_k0s1) +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m7 + pmaxsw m9, m5 + pminsw m10, m3 +%else + pminsw m7, m10 + pminsw m7, m3 + pmaxsw m3, m9, m4 + pmaxsw m3, m5 + mova m10, m7 + mova m9, m3 +%endif + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + paddw m0, m2 + pabsw m3, m4 ; adiff_k0s2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k1s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k1s1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+1] ; off3_k1 + paddw m0, m7 + pabsw m7, m2 + psignw m4, m5 ; constrain(diff_k0s3) + pabsw m5, m3 +%if ARCH_X86_64 + pmaxsw m9, m2 + pminsw m10, m7 + pmaxsw m9, m3 + pminsw m10, m5 +%else + pminsw m7, m10 + pminsw m7, m5 + pmaxsw m5, m9, m2 + pmaxsw m5, m3 + mova m10, m7 + mova m9, m5 +%endif + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+4] ; off1_k0 + paddw m0, m7 + pabsw m7, m4 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m5 +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m7 + pmaxsw m9, m5 + pminsw m10, m3 +%else + pminsw m7, m10 + pminsw m7, m3 + pmaxsw m3, m9, m4 + pmaxsw m3, m5 + mova m10, m7 + mova m9, m3 +%endif + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k0p0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0p1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+5] ; off1_k1 + pabsw m7, m2 + psignw m4, m5 ; constrain(diff_k1s3) + pabsw m5, m3 +%if ARCH_X86_64 + pmaxsw m9, m2 + pminsw m10, m7 + pmaxsw m9, m3 + pminsw m10, m5 +%else + pminsw m7, m10 + pminsw m7, m5 + pmaxsw m5, m9, m2 + pmaxsw m5, m3 + mova m10, m7 + mova m9, m5 +%endif + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + paddw m0, m4 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m7, [rsp+gprsize], m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m7, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m7, m2 ; constrain(diff_k0p0) + psubusw m2, [rsp+gprsize], m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1p0 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1p1 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 + paddw m7, m2 ; constrain(diff_k0) + pabsw m2, m5 +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m3 + pmaxsw m9, m5 + pminsw m10, m2 +%else + pminsw m3, m10 + pminsw m3, m2 + pmaxsw m2, m9, m4 + pmaxsw m2, m5 + mova m10, m3 + mova m9, m2 +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + pabsw m3, m4 ; adiff_k1p0 + pmullw m7, [priq+16*0] ; pri_tap_k0 + paddw m0, m7 + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, [rsp+16*0+gprsize], m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, [rsp+16*0+gprsize], m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m7, [priq+16*1] ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if ARCH_X86_64 + pmaxsw m9, m1 + pminsw m0, m9 +%else + pmaxsw m2, m9, m1 + pminsw m0, m2 +%endif + pminsw m1, m10 + pmaxsw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +%endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal cdef_filter_4x4_16bpc, 5, 9, 9, 32*10, dst, stride, left, top, bot, \ + pri, sec, edge + %define px rsp+32*4 +%else +cglobal cdef_filter_4x4_16bpc, 2, 7, 8, -32*11, dst, stride, edge, top, left + %define botq topq + %define px rsp+32*5 +%endif + %define base t0-dir_table + %define pri_shift px-16*6 + %define sec_shift px-16*5 + mov edged, r9m + LEA t0, dir_table + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea t1, [dstq+strideq*2] + movu m2, [t1 +strideq*0] + movu m3, [t1 +strideq*1] + movddup m7, [base+pw_m16384] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], m7 + movd [px-32*1-4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movifnidn botq, r4mp + movu m0, [botq+strideq*0] + movu m1, [botq+strideq*1] + mova [px+32*4+0], m0 + mova [px+32*5+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [botq+strideq*0-4] + movd m1, [botq+strideq*1-4] + movd [px+32*4-4], m0 + movd [px+32*5-4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*4+0], m7 + mova [px+32*5+0], m7 +.bottom_no_left: + movd [px+32*4-4], m7 + movd [px+32*5-4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0-4], m0 + movd [px+32*1-4], m1 + movd [px+32*2-4], m2 + movd [px+32*3-4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], m7}, 0, 1, 2, 3 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+8], m7}, -2, -1, 0, 1, 2, 3, 4, 5 +.padding_done: + CDEF_FILTER 4, 4 + +%if ARCH_X86_64 +cglobal cdef_filter_4x8_16bpc, 5, 9, 9, 32*14, dst, stride, left, top, bot, \ + pri, sec, edge +%else +cglobal cdef_filter_4x8_16bpc, 2, 7, 8, -32*15, dst, stride, edge, top, left +%endif + mov edged, r9m + LEA t0, dir_table + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea t1, [dstq+strideq*2] + movu m2, [t1 +strideq*0] + movu m3, [t1 +strideq*1] + lea t1, [t1 +strideq*2] + movu m4, [t1 +strideq*0] + movu m5, [t1 +strideq*1] + lea t1, [t1 +strideq*2] + movu m6, [t1 +strideq*0] + movu m7, [t1 +strideq*1] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + mova [px+32*4+0], m4 + mova [px+32*5+0], m5 + mova [px+32*6+0], m6 + mova [px+32*7+0], m7 + movddup m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], m7 + movd [px-32*1-4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movifnidn botq, r4mp + movu m0, [botq+strideq*0] + movu m1, [botq+strideq*1] + mova [px+32*8+0], m0 + mova [px+32*9+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [botq+strideq*0-4] + movd m1, [botq+strideq*1-4] + movd [px+32*8-4], m0 + movd [px+32*9-4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+0], m7 + mova [px+32*9+0], m7 +.bottom_no_left: + movd [px+32*8-4], m7 + movd [px+32*9-4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0-4], m0 + movd [px+32*1-4], m1 + movd [px+32*2-4], m2 + movd [px+32*3-4], m3 + movd m0, [leftq+4*4] + movd m1, [leftq+4*5] + movd m2, [leftq+4*6] + movd m3, [leftq+4*7] + movd [px+32*4-4], m0 + movd [px+32*5-4], m1 + movd [px+32*6-4], m2 + movd [px+32*7-4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], m7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+8], m7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 4, 8 + +%if ARCH_X86_64 +cglobal cdef_filter_8x8_16bpc, 5, 9, 9, 32*14, dst, stride, left, top, bot, \ + pri, sec, edge +%else +cglobal cdef_filter_8x8_16bpc, 2, 7, 8, -32*15, dst, stride, edge, top, left +%endif + mov edged, r9m + LEA t0, dir_table + mova m0, [dstq+strideq*0+ 0] + movd m1, [dstq+strideq*0+16] + mova m2, [dstq+strideq*1+ 0] + movd m3, [dstq+strideq*1+16] + lea t1, [dstq+strideq*2] + mova m4, [t1 +strideq*0+ 0] + movd m5, [t1 +strideq*0+16] + mova m6, [t1 +strideq*1+ 0] + movd m7, [t1 +strideq*1+16] + lea t1, [t1 +strideq*2] + mova [px+32*0+ 0], m0 + movd [px+32*0+16], m1 + mova [px+32*1+ 0], m2 + movd [px+32*1+16], m3 + mova [px+32*2+ 0], m4 + movd [px+32*2+16], m5 + mova [px+32*3+ 0], m6 + movd [px+32*3+16], m7 + mova m0, [t1 +strideq*0+ 0] + movd m1, [t1 +strideq*0+16] + mova m2, [t1 +strideq*1+ 0] + movd m3, [t1 +strideq*1+16] + lea t1, [t1 +strideq*2] + mova m4, [t1 +strideq*0+ 0] + movd m5, [t1 +strideq*0+16] + mova m6, [t1 +strideq*1+ 0] + movd m7, [t1 +strideq*1+16] + mova [px+32*4+ 0], m0 + movd [px+32*4+16], m1 + mova [px+32*5+ 0], m2 + movd [px+32*5+16], m3 + mova [px+32*6+ 0], m4 + movd [px+32*6+16], m5 + mova [px+32*7+ 0], m6 + movd [px+32*7+16], m7 + movddup m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + mova m0, [topq+strideq*0+ 0] + mova m1, [topq+strideq*0+16] + mova m2, [topq+strideq*1+ 0] + mova m3, [topq+strideq*1+16] + mova [px-32*2+ 0], m0 + movd [px-32*2+16], m1 + mova [px-32*1+ 0], m2 + movd [px-32*1+16], m3 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+ 0], m7 + movd [px-32*2+16], m7 + mova [px-32*1+ 0], m7 + movd [px-32*1+16], m7 +.top_no_left: + movd [px-32*2- 4], m7 + movd [px-32*1- 4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + movifnidn botq, r4mp + mova m0, [botq+strideq*0+ 0] + movd m1, [botq+strideq*0+16] + mova m2, [botq+strideq*1+ 0] + movd m3, [botq+strideq*1+16] + mova [px+32*8+ 0], m0 + movd [px+32*8+16], m1 + mova [px+32*9+ 0], m2 + movd [px+32*9+16], m3 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [botq+strideq*0-4] + movd m1, [botq+strideq*1-4] + movd [px+32*8- 4], m0 + movd [px+32*9- 4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+ 0], m7 + movd [px+32*8+16], m7 + mova [px+32*9+ 0], m7 + movd [px+32*9+16], m7 +.bottom_no_left: + movd [px+32*8- 4], m7 + movd [px+32*9- 4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0- 4], m0 + movd [px+32*1- 4], m1 + movd [px+32*2- 4], m2 + movd [px+32*3- 4], m3 + movd m0, [leftq+4*4] + movd m1, [leftq+4*5] + movd m2, [leftq+4*6] + movd m3, [leftq+4*7] + movd [px+32*4- 4], m0 + movd [px+32*5- 4], m1 + movd [px+32*6- 4], m2 + movd [px+32*7- 4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x- 4], m7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+16], m7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 8, 8 + +%macro CDEF_DIR 0 +%if ARCH_X86_64 +cglobal cdef_dir_16bpc, 4, 7, 16, src, stride, var, bdmax + lea r6, [dir_shift] + shr bdmaxd, 11 ; 0 for 10bpc, 1 for 12bpc + movddup m7, [r6+bdmaxq*8] + lea r6, [strideq*3] + mova m0, [srcq+strideq*0] + mova m1, [srcq+strideq*1] + mova m2, [srcq+strideq*2] + mova m3, [srcq+r6 ] + lea srcq, [srcq+strideq*4] + mova m4, [srcq+strideq*0] + mova m5, [srcq+strideq*1] + mova m6, [srcq+strideq*2] + REPX {pmulhuw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhuw m7, [srcq+r6 ] + pxor m8, m8 + packuswb m9, m0, m1 + packuswb m10, m2, m3 + packuswb m11, m4, m5 + packuswb m12, m6, m7 + REPX {psadbw x, m8}, m9, m10, m11, m12 + packssdw m9, m10 + packssdw m11, m12 + packssdw m9, m11 + jmp mangle(private_prefix %+ _cdef_dir_8bpc %+ SUFFIX).main +%else +cglobal cdef_dir_16bpc, 2, 4, 8, 96, src, stride, var, bdmax + mov bdmaxd, bdmaxm + LEA r2, dir_shift + shr bdmaxd, 11 + movddup m7, [r2+bdmaxq*8] + lea r3, [strideq*3] + pmulhuw m3, m7, [srcq+strideq*0] + pmulhuw m4, m7, [srcq+strideq*1] + pmulhuw m5, m7, [srcq+strideq*2] + pmulhuw m6, m7, [srcq+r3 ] + movddup m1, [r2-dir_shift+pw_128] + lea srcq, [srcq+strideq*4] + pxor m0, m0 + packuswb m2, m3, m4 + psubw m3, m1 + psubw m4, m1 + mova [esp+0x00], m3 + mova [esp+0x10], m4 + packuswb m3, m5, m6 + psadbw m2, m0 + psadbw m3, m0 + psubw m5, m1 + psubw m6, m1 + packssdw m2, m3 + mova [esp+0x20], m5 + mova [esp+0x50], m6 + pmulhuw m4, m7, [srcq+strideq*0] + pmulhuw m5, m7, [srcq+strideq*1] + pmulhuw m6, m7, [srcq+strideq*2] + pmulhuw m7, [srcq+r3 ] + packuswb m3, m4, m5 + packuswb m1, m6, m7 + psadbw m3, m0 + psadbw m1, m0 + packssdw m3, m1 + movddup m1, [r2-dir_shift+pw_128] + LEA r2, shufw_6543210x + jmp mangle(private_prefix %+ _cdef_dir_8bpc %+ SUFFIX).main +%endif +%endmacro + +INIT_XMM ssse3 +CDEF_DIR + +INIT_XMM sse4 +CDEF_DIR diff --git a/third_party/dav1d/src/x86/cdef_avx2.asm b/third_party/dav1d/src/x86/cdef_avx2.asm new file mode 100644 index 0000000000..1f30f8a3b7 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx2.asm @@ -0,0 +1,1772 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +%macro JMP_TABLE 2-* + %xdefine %1_jmptable %%table + %xdefine %%base mangle(private_prefix %+ _%1_avx2) + %%table: + %rep %0 - 1 + dd %%base %+ .%2 - %%table + %rotate 1 + %endrep +%endmacro + +%macro CDEF_FILTER_JMP_TABLE 1 +JMP_TABLE cdef_filter_%1_8bpc, \ + d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \ + d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1 +%endmacro + +SECTION_RODATA 32 + +pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 +blend_4x4: dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00 + dd 0x80, 0x00, 0x00 +blend_4x8_0: dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +blend_4x8_1: dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + dd 0x00, 0x00 +blend_4x8_2: dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000 +blend_4x8_3: dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000, 0x0000 +blend_8x8_0: dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80 +blend_8x8_1: dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000 +div_table: dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105 +shufw_6543210x:db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 +shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 +pw_128: times 2 dw 128 +pw_2048: times 2 dw 2048 +tap_table: ; masks for 8 bit shifts + db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 + ; weights + db 4, 2, 3, 3, 2, 1 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + db 1 * 16 + 0, 2 * 16 + 0 + db 1 * 16 + 0, 2 * 16 - 1 + ; the last 6 are repeats of the first 6 so we don't need to & 7 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + +CDEF_FILTER_JMP_TABLE 4x4 +CDEF_FILTER_JMP_TABLE 4x8 +CDEF_FILTER_JMP_TABLE 8x8 + +SECTION .text + +%macro PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r7m + lea tableq, [cdef_filter_%1x%2_8bpc_jmptable] + lea dirq, [tableq+dirq*2*4] +%if %1 == 4 + %if %2 == 4 + DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \ + table, dir, dirjmp, stride3, k + %else + DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \ + table, dir, dirjmp, dst4, stride3, k + lea dst4q, [dstq+strideq*4] + %endif +%else + DEFINE_ARGS dst, stride, h, top1, bot, pri, sec, \ + table, dir, dirjmp, top2, stride3, k + mov hq, -8 + lea top1q, [top1q+strideq*0] + lea top2q, [top1q+strideq*1] +%endif +%if %1 == 4 + lea stride3q, [strideq*3] +%endif +%endmacro + +%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 + pxor m15, m15 ; sum +%if %2 == 8 + pxor m12, m12 + %if %1 == 4 + movd xm4, [dstq +strideq*0] + movd xm6, [dstq +strideq*1] + movd xm5, [dstq +strideq*2] + movd xm7, [dstq +stride3q ] + vinserti128 m4, [dst4q+strideq*0], 1 + vinserti128 m6, [dst4q+strideq*1], 1 + vinserti128 m5, [dst4q+strideq*2], 1 + vinserti128 m7, [dst4q+stride3q ], 1 + punpckldq m4, m6 + punpckldq m5, m7 + %else + movq xm4, [dstq+strideq*0] + movq xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + %endif + punpcklqdq m4, m5 +%else + movd xm4, [dstq+strideq*0] + movd xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + punpckldq m4, m5 +%endif +%if %3 == 1 + mova m7, m4 ; min + mova m8, m4 ; max +%endif +%endmacro + +%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, h, clip + ; load p0/p1 + movsxd dirjmpq, [dirq+kq*4+%1*2*4] + add dirjmpq, tableq + call dirjmpq + +%if %8 == 1 + pmaxub m7, m5 + pminub m8, m5 + pmaxub m7, m6 + pminub m8, m6 +%endif + + ; accumulate sum[m15] over p0/p1 +%if %7 == 4 + punpcklbw m5, m6 + punpcklbw m6, m4, m4 + psubusb m9, m5, m6 + psubusb m5, m6, m5 + por m9, m5 ; abs_diff_p01(p01 - px) + pcmpeqb m5, m9 + por m5, %5 + psignb m6, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m6 + paddw m15, m5 +%else + psubusb m9, m5, m4 + psubusb m5, m4, m5 + psubusb m11, m6, m4 + psubusb m6, m4, m6 + por m9, m5 ; abs_diff_p0(p0 - px) + por m11, m6 ; abs_diff_p1(p1 - px) + pcmpeqb m5, m9 + pcmpeqb m6, m11 + punpckhbw m10, m9, m11 + punpcklbw m9, m11 + por m5, %5 + por m11, m6, %5 + punpckhbw m6, m5, m11 + punpcklbw m5, m11 + psignb m11, %5, m6 + psrlw m6, m10, %2 ; emulate 8-bit shift + pand m6, %3 + psubusb m6, %4, m6 + pminub m6, m10 + pmaddubsw m6, m11 + paddw m12, m6 + psignb m11, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m11 + paddw m15, m5 +%endif +%endmacro + +%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip +%if %2 == 4 + %if %5 == 1 + punpcklbw m4, %3 + %endif + pcmpgtw %3, m15 + paddw m15, %3 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m15 + paddb m4, m15 + %else + paddw m4, m15 + packuswb m4, m4 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + movd [dstq+strideq*0], xm4 + movd [dstq+strideq*2], xm5 + pextrd [dstq+strideq*1], xm4, 1 + pextrd [dstq+stride3q ], xm5, 1 +%else + pcmpgtw m6, %3, m12 + pcmpgtw m5, %3, m15 + paddw m12, m6 + paddw m15, m5 + %if %5 == 1 + punpckhbw m5, m4, %3 + punpcklbw m4, %3 + %endif + pmulhrsw m12, %4 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m12 + paddb m4, m15 + %else + paddw m5, m12 + paddw m4, m15 + packuswb m4, m5 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + %if %1 == 4 + movd [dstq +strideq*0], xm4 + movd [dst4q+strideq*0], xm5 + pextrd [dstq +strideq*1], xm4, 1 + pextrd [dst4q+strideq*1], xm5, 1 + pextrd [dstq +strideq*2], xm4, 2 + pextrd [dst4q+strideq*2], xm5, 2 + pextrd [dstq +stride3q ], xm4, 3 + pextrd [dst4q+stride3q ], xm5, 3 + %else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm5 + movhps [dstq+strideq*1], xm4 + movhps [dstq+stride3q ], xm5 + %endif +%endif +%endmacro + +%macro BORDER_PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r7m + lea dirq, [tableq+dirq*2+14] +%if %1*%2*2/mmsize > 1 + %if %1 == 4 + DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, h, off + %else + DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, h, off + %endif + mov hd, %1*%2*2/mmsize +%else + DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, off +%endif + lea stkq, [px] + pxor m11, m11 +%endmacro + +%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 +%if %1 == 4 + movq xm4, [stkq+32*0] + movhps xm4, [stkq+32*1] + movq xm5, [stkq+32*2] + movhps xm5, [stkq+32*3] + vinserti128 m4, xm5, 1 +%else + mova xm4, [stkq+32*0] ; px + vinserti128 m4, [stkq+32*1], 1 +%endif + pxor m15, m15 ; sum +%if %3 == 1 + mova m7, m4 ; max + mova m8, m4 ; min +%endif +%endmacro + +%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, clip + ; load p0/p1 + movsx offq, byte [dirq+kq+%1] ; off1 +%if %6 == 4 + movq xm5, [stkq+offq*2+32*0] ; p0 + movq xm6, [stkq+offq*2+32*2] + movhps xm5, [stkq+offq*2+32*1] + movhps xm6, [stkq+offq*2+32*3] + vinserti128 m5, xm6, 1 +%else + movu xm5, [stkq+offq*2+32*0] ; p0 + vinserti128 m5, [stkq+offq*2+32*1], 1 +%endif + neg offq ; -off1 +%if %6 == 4 + movq xm6, [stkq+offq*2+32*0] ; p1 + movq xm9, [stkq+offq*2+32*2] + movhps xm6, [stkq+offq*2+32*1] + movhps xm9, [stkq+offq*2+32*3] + vinserti128 m6, xm9, 1 +%else + movu xm6, [stkq+offq*2+32*0] ; p1 + vinserti128 m6, [stkq+offq*2+32*1], 1 +%endif +%if %7 == 1 + ; out of bounds values are set to a value that is a both a large unsigned + ; value and a negative signed value. + ; use signed max and unsigned min to remove them + pmaxsw m7, m5 ; max after p0 + pminuw m8, m5 ; min after p0 + pmaxsw m7, m6 ; max after p1 + pminuw m8, m6 ; min after p1 +%endif + + ; accumulate sum[m15] over p0/p1 + ; calculate difference before converting + psubw m5, m4 ; diff_p0(p0 - px) + psubw m6, m4 ; diff_p1(p1 - px) + + ; convert to 8-bits with signed saturation + ; saturating to large diffs has no impact on the results + packsswb m5, m6 + + ; group into pairs so we can accumulate using maddubsw + pshufb m5, m12 + pabsb m9, m5 + psignb m10, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + + ; use unsigned min since abs diff can equal 0x80 + pminub m5, m9 + pmaddubsw m5, m10 + paddw m15, m5 +%endmacro + +%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip + pcmpgtw m9, m11, m15 + paddw m15, m9 + pmulhrsw m15, %2 + paddw m4, m15 +%if %3 == 1 + pminsw m4, m7 + pmaxsw m4, m8 +%endif + packuswb m4, m4 + vextracti128 xm5, m4, 1 +%if %1 == 4 + movd [dstq+strideq*0], xm4 + pextrd [dstq+strideq*1], xm4, 1 + movd [dstq+strideq*2], xm5 + pextrd [dstq+stride3q ], xm5, 1 +%else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*1], xm5 +%endif +%endmacro + +%macro CDEF_FILTER 2 ; w, h +INIT_YMM avx2 +cglobal cdef_filter_%1x%2_8bpc, 5, 11, 0, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%assign stack_offset_entry stack_offset + mov edged, edgem + cmp edged, 0xf + jne .border_block + + PUSH r11 + PUSH r12 +%if %2 == 4 +%assign regs_used 13 + ALLOC_STACK 0x60, 16 + pmovzxbw xm0, [leftq+1] + vpermq m0, m0, q0110 + psrldq m1, m0, 4 + vpalignr m2, m0, m0, 12 + movu [rsp+0x10], m0 + movu [rsp+0x28], m1 + movu [rsp+0x40], m2 +%elif %1 == 4 +%assign regs_used 14 + PUSH r13 + ALLOC_STACK 8*2+%1*%2*1, 16 + pmovzxwd m0, [leftq] + mova [rsp+0x10], m0 +%else +%assign regs_used 15 + PUSH r13 + PUSH r14 + ALLOC_STACK 8*4+%1*%2*2+32, 16 + lea r11, [strideq*3] + movu xm4, [dstq+strideq*2] + pmovzxwq m0, [leftq+0] + pmovzxwq m1, [leftq+8] + vinserti128 m4, [dstq+r11], 1 + pmovzxbd m2, [leftq+1] + pmovzxbd m3, [leftq+9] + mov [rsp+16], botq + mova [rsp+0x20], m0 + mova [rsp+0x40], m1 + mova [rsp+0x60], m2 + mova [rsp+0x80], m3 + mova [rsp+0xa0], m4 + lea botq, [dstq+strideq*4] +%endif + + DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, zero, pridmp, damping + mov dampingd, r8m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + test prid, prid + jz .sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + test secdmpd, secdmpd + jz .pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, bot, pri, sec, table, dir + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + PREP_REGS %1, %2 +%if %1*%2 > mmsize +.v_loop: +%endif + LOAD_BLOCK %1, %2, 1 +.k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0 + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2 + dec kq + jge .k_loop + + vpbroadcastd m10, [pw_2048] + pxor m9, m9 + ADJUST_PIXEL %1, %2, m9, m10, 1 +%if %1*%2 > mmsize + lea dstq, [dstq+strideq*4] + lea top1q, [rsp+0xa0] + lea top2q, [rsp+0xb0] + mov botq, [rsp+16] + add hq, 4 + jl .v_loop +%endif + RET + +.pri_only: + DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, dir + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + PREP_REGS %1, %2 + vpbroadcastd m3, [pw_2048] + pxor m1, m1 +%if %1*%2 > mmsize +.pri_v_loop: +%endif + LOAD_BLOCK %1, %2 +.pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0 + dec kq + jge .pri_k_loop + ADJUST_PIXEL %1, %2, m1, m3 +%if %1*%2 > mmsize + lea dstq, [dstq+strideq*4] + lea top1q, [rsp+0xa0] + lea top2q, [rsp+0xb0] + mov botq, [rsp+16] + add hq, 4 + jl .pri_v_loop +%endif + RET + +.sec_only: + DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, zero, _, damping + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, table + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, bot, _, sec, table, dir + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + PREP_REGS %1, %2 + vpbroadcastd m2, [pw_2048] + pxor m0, m0 +%if %1*%2 > mmsize +.sec_v_loop: +%endif + LOAD_BLOCK %1, %2 +.sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2 + dec kq + jge .sec_k_loop + ADJUST_PIXEL %1, %2, m0, m2 +%if %1*%2 > mmsize + lea dstq, [dstq+strideq*4] + lea top1q, [rsp+0xa0] + lea top2q, [rsp+0xb0] + mov botq, [rsp+16] + add hq, 4 + jl .sec_v_loop +%endif + RET + +.d0k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-1] + vpbroadcastq m10, [dstq+strideq*2-1] + movd xm5, [topq+strideq*1+1] + movd xm9, [dstq+strideq*0+1] + psrldq m11, m6, 2 + psrldq m12, m10, 2 + vinserti128 m6, [dstq+stride3q -1], 1 + vinserti128 m10, [botq -1], 1 + vpblendd m5, m11, 0x10 + vpblendd m9, m12, 0x10 + movu m11, [blend_4x4+16] + punpckldq m6, m10 + punpckldq m5, m9 + vpblendvb m6, [rsp+gprsize+0x28], m11 + %else + movd xm5, [topq +strideq*1+1] + movq xm6, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm5, [dstq +strideq*0+1], 1 + movhps xm6, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm9, xm6, 2 + shufps xm5, xm9, q2010 ; -1 +0 +1 +2 + shufps xm6, xm10, q2020 ; +1 +2 +3 +4 + psrldq xm9, xm11, 2 + psrldq xm10, 2 + shufps xm10, xm9, q2020 ; +3 +4 +5 +6 + movd xm9, [dst4q+stride3q -1] + pinsrd xm9, [botq -1], 1 + shufps xm11, xm9, q1020 ; +5 +6 +7 +8 + pmovzxbw m9, [leftq+3] + vinserti128 m6, xm11, 1 + movu m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [top2q +1] + vbroadcasti128 m10, [dstq+strideq*1-1] + vbroadcasti128 m11, [dstq+strideq*2-1] + movhps xm5, [dstq+strideq*0+1] + vinserti128 m6, m10, [dstq+stride3q-1], 1 + vinserti128 m9, m11, [botq -1], 1 + psrldq m10, 2 + psrldq m11, 2 + punpcklqdq m6, m9 + movu m9, [r13+hq*2*1+16*1] + punpcklqdq m10, m11 + vpblendd m5, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+0x60+hq*8+64+8*1], m9 +%endif + ret +.d1k0: +.d2k0: +.d3k0: +%if %1 == 4 + %if %2 == 4 + movq xm6, [dstq+strideq*0-1] + movq xm9, [dstq+strideq*1-1] + vinserti128 m6, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m11, [rsp+gprsize+0x10] + pcmpeqd m12, m12 + psrldq m5, m6, 2 + psrldq m10, m9, 2 + psrld m12, 24 + punpckldq m6, m9 + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movq xm6, [dstq +strideq*0-1] + movq xm9, [dstq +strideq*2-1] + movhps xm6, [dstq +strideq*1-1] + movhps xm9, [dstq +stride3q -1] + movq xm10, [dst4q+strideq*0-1] + movhps xm10, [dst4q+strideq*1-1] + psrldq xm5, xm6, 2 + psrldq xm11, xm9, 2 + shufps xm5, xm11, q2020 + movq xm11, [dst4q+strideq*2-1] + movhps xm11, [dst4q+stride3q -1] + shufps xm6, xm9, q2020 + shufps xm9, xm10, xm11, q2020 + vinserti128 m6, xm9, 1 + pmovzxbw m9, [leftq+1] + psrldq xm10, 2 + psrldq xm11, 2 + shufps xm10, xm11, q2020 + vpbroadcastd m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + movu xm5, [dstq+strideq*0-1] + movu xm9, [dstq+strideq*1-1] + vinserti128 m5, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m10, [blend_8x8_0+16] + punpcklqdq m6, m5, m9 + vpblendvb m6, [rsp+gprsize+0x60+hq*8+64], m10 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d4k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m10, [dstq+strideq*1-1] + vpbroadcastq m11, [dstq+strideq*2-1] + movd xm6, [topq+strideq*1-1] + movd xm9, [dstq+strideq*0-1] + psrldq m5, m10, 2 + psrldq m12, m11, 2 + vpblendd m6, m10, 0x10 + vpblendd m9, m11, 0x10 + movu m10, [blend_4x4] + vinserti128 m5, [dstq+stride3q +1], 1 + vinserti128 m12, [botq +1], 1 + punpckldq m6, m9 + punpckldq m5, m12 + vpblendvb m6, [rsp+gprsize+0x40], m10 + %else + movd xm6, [topq +strideq*1-1] + movq xm9, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm6, [dstq +strideq*0-1], 1 + movhps xm9, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm5, xm9, 2 + shufps xm6, xm9, q2010 + psrldq xm9, xm10, 2 + shufps xm5, xm9, q2020 + shufps xm10, xm11, q2020 + movd xm9, [dst4q+stride3q +1] + vinserti128 m6, xm10, 1 + pinsrd xm9, [botq +1], 1 + psrldq xm11, 2 + pmovzxbw m10, [leftq-1] + shufps xm11, xm9, q1020 + movu m9, [blend_4x8_0] + vinserti128 m5, xm11, 1 + vpblendvb m6, m10, m9 + %endif +%else + lea r13, [blend_8x8_0+8] + movq xm6, [top2q -1] + vbroadcasti128 m5, [dstq+strideq*1-1] + vbroadcasti128 m9, [dstq+strideq*2-1] + movhps xm6, [dstq+strideq*0-1] + movu m11, [r13+hq*2*1+16*1] + punpcklqdq m10, m5, m9 + vinserti128 m5, [dstq+stride3q -1], 1 + vinserti128 m9, [botq -1], 1 + vpblendd m6, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+0x60+hq*8+64-8*1], m11 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d5k0: +.d6k0: +.d7k0: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq+strideq*1 ] + vpbroadcastd m5, [dstq+strideq*1 ] + vpbroadcastd m9, [dstq+strideq*2 ] + vpblendd xm6, [dstq+strideq*0-4], 0x2 + vpblendd m5, m9, 0x22 + vpblendd m6, m5, 0x30 + vinserti128 m5, [dstq+stride3q ], 1 + vpblendd m5, [botq -20], 0x20 + %else + movd xm6, [topq +strideq*1] + movd xm5, [dstq +strideq*1] + movd xm9, [dstq +stride3q ] + movd xm10, [dst4q+strideq*1] + movd xm11, [dst4q+stride3q ] + pinsrd xm6, [dstq +strideq*0], 1 + pinsrd xm5, [dstq +strideq*2], 1 + pinsrd xm9, [dst4q+strideq*0], 1 + pinsrd xm10, [dst4q+strideq*2], 1 + pinsrd xm11, [botq ], 1 + punpcklqdq xm6, xm5 + punpcklqdq xm5, xm9 + punpcklqdq xm9, xm10 + punpcklqdq xm10, xm11 + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + %endif +%else + movq xm6, [top2q ] + movq xm5, [dstq+strideq*1] + movq xm9, [dstq+stride3q ] + movhps xm6, [dstq+strideq*0] + movhps xm5, [dstq+strideq*2] + movhps xm9, [botq ] + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 +%endif + ret +.d0k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [dstq+strideq*2-2] + movd xm9, [dstq+stride3q -2] + movd xm5, [topq+strideq*0+2] + movd xm10, [topq+strideq*1+2] + pinsrw xm6, [leftq+4], 0 + pinsrw xm9, [leftq+6], 0 + vinserti128 m5, [dstq+strideq*0+2], 1 + vinserti128 m10, [dstq+strideq*1+2], 1 + vinserti128 m6, [botq+strideq*0-2], 1 + vinserti128 m9, [botq+strideq*1-2], 1 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movq xm6, [dstq +strideq*2-2] + movd xm10, [dst4q+strideq*2-2] + movd xm5, [topq +strideq*0+2] + movq xm9, [dst4q+strideq*0-2] + movhps xm6, [dstq +stride3q -2] + pinsrw xm10, [dst4q+stride3q ], 3 + pinsrd xm5, [topq +strideq*1+2], 1 + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [botq +strideq*0-2], 2 + pinsrd xm5, [dstq +strideq*0+2], 2 + pinsrd xm10, [botq +strideq*1-2], 3 + pinsrd xm5, [dstq +strideq*1+2], 3 + shufps xm11, xm6, xm9, q3131 + shufps xm6, xm9, q2020 + movu m9, [blend_4x8_3+8] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm11, 1 + vpblendvb m6, [rsp+gprsize+0x10+8], m9 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm6, [dstq+strideq*2-2] + movq xm9, [dstq+stride3q -2] + movq xm5, [top1q +2] + movq xm10, [top2q +2] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m6, [botq+strideq*0-2], 1 + vinserti128 m9, [botq+strideq*1-2], 1 + vinserti128 m5, [dstq+strideq*0+2], 1 + vinserti128 m10, [dstq+strideq*1+2], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+0x20+hq*8+64+8*2], m11 +%endif + ret +.d1k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-2] + vpbroadcastq m9, [dstq+strideq*2-2] + movd xm5, [topq+strideq*1+2] + movd xm10, [dstq+strideq*0+2] + psrldq m11, m6, 4 + psrldq m12, m9, 4 + vpblendd m5, m11, 0x10 + movq xm11, [leftq+2] + vinserti128 m6, [dstq+stride3q-2], 1 + punpckldq xm11, xm11 + vpblendd m10, m12, 0x10 + pcmpeqd m12, m12 + pmovzxwd m11, xm11 + psrld m12, 16 + punpckldq m6, m9 + vpbroadcastd m9, [botq-2] + vpblendvb m6, m11, m12 + punpckldq m5, m10 + vpblendd m6, m9, 0x20 + %else + movd xm5, [topq +strideq*1+2] + movq xm6, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q -2] + pinsrd xm5, [dstq +strideq*0+2], 1 + movhps xm6, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [botq -2], 1 + shufps xm5, xm6, q3110 + shufps xm6, xm9, q2020 + shufps xm9, xm10, q3131 + shufps xm10, xm11, q1020 + movu m11, [blend_4x8_2+4] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm9, 1 + vpblendvb m6, [rsp+gprsize+0x10+4], m11 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm5, [top2q +2] + vbroadcasti128 m6, [dstq+strideq*1-2] + vbroadcasti128 m9, [dstq+strideq*2-2] + movhps xm5, [dstq+strideq*0+2] + shufps m10, m6, m9, q2121 + vinserti128 m6, [dstq+stride3q -2], 1 + vinserti128 m9, [botq -2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m5, m10, 0xF0 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+0x20+hq*8+64+8*1], m11 +%endif + ret +.d2k1: +%if %1 == 4 + %if %2 == 4 + movq xm11, [leftq] + movq xm6, [dstq+strideq*0-2] + movq xm9, [dstq+strideq*1-2] + vinserti128 m6, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + punpckldq xm11, xm11 + psrldq m5, m6, 4 + psrldq m10, m9, 4 + pmovzxwd m11, xm11 + punpckldq m6, m9 + punpckldq m5, m10 + pblendw m6, m11, 0x05 + %else + movq xm5, [dstq +strideq*0-2] + movq xm9, [dstq +strideq*2-2] + movq xm10, [dst4q+strideq*0-2] + movq xm11, [dst4q+strideq*2-2] + movhps xm5, [dstq +strideq*1-2] + movhps xm9, [dstq +stride3q -2] + movhps xm10, [dst4q+strideq*1-2] + movhps xm11, [dst4q+stride3q -2] + shufps xm6, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + shufps xm9, xm10, xm11, q2020 + shufps xm10, xm11, q3131 + pmovzxwd m11, [leftq] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + pblendw m6, m11, 0x55 + %endif +%else + mova m11, [rsp+gprsize+0x20+hq*8+64] + movu xm5, [dstq+strideq*0-2] + movu xm9, [dstq+strideq*1-2] + vinserti128 m5, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + shufps m6, m5, m9, q1010 + shufps m5, m9, q2121 + pblendw m6, m11, 0x11 +%endif + ret +.d3k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m11, [dstq+strideq*1-2] + vpbroadcastq m12, [dstq+strideq*2-2] + movd xm6, [topq+strideq*1-2] + movd xm9, [dstq+strideq*0-2] + pblendw m11, [leftq-16+2], 0x01 + pblendw m12, [leftq-16+4], 0x01 + pinsrw xm9, [leftq- 0+0], 0 + psrldq m5, m11, 4 + psrldq m10, m12, 4 + vinserti128 m5, [dstq+stride3q +2], 1 + vinserti128 m10, [botq +2], 1 + vpblendd m6, m11, 0x10 + vpblendd m9, m12, 0x10 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm6, [topq +strideq*1-2] + movq xm5, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q +2] + pinsrw xm6, [dstq +strideq*0 ], 3 + movhps xm5, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [botq +2], 1 + shufps xm6, xm5, q2010 + shufps xm5, xm9, q3131 + shufps xm9, xm10, q2020 + shufps xm10, xm11, q1031 + movu m11, [blend_4x8_2] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+0x10-4], m11 + %endif +%else + lea r13, [blend_8x8_1+8] + movq xm6, [top2q -2] + vbroadcasti128 m5, [dstq+strideq*1-2] + vbroadcasti128 m10, [dstq+strideq*2-2] + movhps xm6, [dstq+strideq*0-2] + punpcklqdq m9, m5, m10 + vinserti128 m5, [dstq+stride3q -2], 1 + vinserti128 m10, [botq -2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m6, m9, 0xF0 + shufps m5, m10, q2121 + vpblendvb m6, [rsp+gprsize+0x20+hq*8+64-8*1], m11 +%endif + ret +.d4k1: +%if %1 == 4 + %if %2 == 4 + vinserti128 m6, [dstq+strideq*0-2], 1 + vinserti128 m9, [dstq+strideq*1-2], 1 + movd xm5, [dstq+strideq*2+2] + movd xm10, [dstq+stride3q +2] + pblendw m6, [leftq-16+0], 0x01 + pblendw m9, [leftq-16+2], 0x01 + vinserti128 m5, [botq+strideq*0+2], 1 + vinserti128 m10, [botq+strideq*1+2], 1 + vpblendd m6, [topq+strideq*0-2], 0x01 + vpblendd m9, [topq+strideq*1-2], 0x01 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movd xm6, [topq +strideq*0-2] + movq xm5, [dstq +strideq*2-2] + movq xm9, [dst4q+strideq*0-2] + movd xm10, [dst4q+strideq*2+2] + pinsrd xm6, [topq +strideq*1-2], 1 + movhps xm5, [dstq +stride3q -2] + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [dst4q+stride3q +2], 1 + pinsrd xm6, [dstq +strideq*0-2], 2 + pinsrd xm10, [botq +strideq*0+2], 2 + pinsrd xm6, [dstq +strideq*1-2], 3 + pinsrd xm10, [botq +strideq*1+2], 3 + shufps xm11, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + movu m9, [blend_4x8_3] + vinserti128 m6, xm11, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+0x10-8], m9 + %endif +%else + lea r13, [blend_8x8_1] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -2] + movq xm9, [top2q -2] + movq xm5, [dstq+strideq*2+2] + movq xm10, [dstq+stride3q +2] + vinserti128 m6, [dstq+strideq*0-2], 1 + vinserti128 m9, [dstq+strideq*1-2], 1 + vinserti128 m5, [botq+strideq*0+2], 1 + vinserti128 m10, [botq+strideq*1+2], 1 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+0x20+hq*8+64-8*2], m11 + punpcklqdq m5, m10 +%endif + ret +.d5k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq+strideq*0-1] + movd xm9, [topq+strideq*1-1] + movd xm5, [dstq+strideq*2+1] + movd xm10, [dstq+stride3q +1] + pcmpeqd m12, m12 + pmovzxbw m11, [leftq-8+1] + psrld m12, 24 + vinserti128 m6, [dstq+strideq*0-1], 1 + vinserti128 m9, [dstq+strideq*1-1], 1 + vinserti128 m5, [botq+strideq*0+1], 1 + vinserti128 m10, [botq+strideq*1+1], 1 + punpckldq m6, m9 + pxor m9, m9 + vpblendd m12, m9, 0x0F + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movd xm6, [topq +strideq*0-1] + movq xm5, [dstq +strideq*2-1] + movq xm9, [dst4q+strideq*0-1] + movd xm10, [dst4q+strideq*2+1] + pinsrd xm6, [topq +strideq*1-1], 1 + movhps xm5, [dstq +stride3q -1] + movhps xm9, [dst4q+strideq*1-1] + pinsrd xm10, [dst4q+stride3q +1], 1 + pinsrd xm6, [dstq +strideq*0-1], 2 + pinsrd xm10, [botq +strideq*0+1], 2 + pinsrd xm6, [dstq +strideq*1-1], 3 + pinsrd xm10, [botq +strideq*1+1], 3 + shufps xm11, xm5, xm9, q2020 + vinserti128 m6, xm11, 1 + pmovzxbw m11, [leftq-3] + psrldq xm5, 2 + psrldq xm9, 2 + shufps xm5, xm9, q2020 + movu m9, [blend_4x8_1] + vinserti128 m5, xm10, 1 + vpblendvb m6, m11, m9 + %endif +%else + lea r13, [blend_8x8_0] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -1] + movq xm9, [top2q -1] + movq xm5, [dstq+strideq*2+1] + movq xm10, [dstq+stride3q +1] + vinserti128 m6, [dstq+strideq*0-1], 1 + vinserti128 m9, [dstq+strideq*1-1], 1 + vinserti128 m5, [botq+strideq*0+1], 1 + vinserti128 m10, [botq+strideq*1+1], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+0x60+hq*8+64-8*2], m11 +%endif + ret +.d6k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq+strideq*0] + movd xm9, [topq+strideq*1] + movd xm5, [dstq+strideq*2] + movd xm10, [dstq+stride3q ] + vinserti128 m6, [dstq+strideq*0], 1 + vinserti128 m9, [dstq+strideq*1], 1 + vinserti128 m5, [botq+strideq*0], 1 + vinserti128 m10, [botq+strideq*1], 1 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm5, [dstq +strideq*2] + movd xm6, [topq +strideq*0] + movd xm9, [dst4q+strideq*2] + pinsrd xm5, [dstq +stride3q ], 1 + pinsrd xm6, [topq +strideq*1], 1 + pinsrd xm9, [dst4q+stride3q ], 1 + pinsrd xm5, [dst4q+strideq*0], 2 + pinsrd xm6, [dstq +strideq*0], 2 + pinsrd xm9, [botq +strideq*0], 2 + pinsrd xm5, [dst4q+strideq*1], 3 + pinsrd xm6, [dstq +strideq*1], 3 + pinsrd xm9, [botq +strideq*1], 3 + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 + %endif +%else + movq xm5, [dstq+strideq*2] + movq xm9, [botq+strideq*0] + movq xm6, [top1q ] + movq xm10, [dstq+strideq*0] + movhps xm5, [dstq+stride3q ] + movhps xm9, [botq+strideq*1] + movhps xm6, [top2q ] + movhps xm10, [dstq+strideq*1] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 +%endif + ret +.d7k1: +%if %1 == 4 + %if %2 == 4 + movd xm5, [dstq+strideq*2-1] + movd xm9, [dstq+stride3q -1] + movd xm6, [topq+strideq*0+1] + movd xm10, [topq+strideq*1+1] + pinsrb xm5, [leftq+ 5], 0 + pinsrb xm9, [leftq+ 7], 0 + vinserti128 m6, [dstq+strideq*0+1], 1 + vinserti128 m10, [dstq+strideq*1+1], 1 + vinserti128 m5, [botq+strideq*0-1], 1 + vinserti128 m9, [botq+strideq*1-1], 1 + punpckldq m6, m10 + punpckldq m5, m9 + %else + movd xm6, [topq +strideq*0+1] + movq xm9, [dstq +strideq*2-1] + movq xm10, [dst4q+strideq*0-1] + movd xm11, [dst4q+strideq*2-1] + pinsrd xm6, [topq +strideq*1+1], 1 + movhps xm9, [dstq +stride3q -1] + movhps xm10, [dst4q+strideq*1-1] + pinsrd xm11, [dst4q+stride3q -1], 1 + pinsrd xm6, [dstq +strideq*0+1], 2 + pinsrd xm11, [botq +strideq*0-1], 2 + pinsrd xm6, [dstq +strideq*1+1], 3 + pinsrd xm11, [botq +strideq*1-1], 3 + shufps xm5, xm9, xm10, q2020 + vinserti128 m5, xm11, 1 + pmovzxbw m11, [leftq+5] + psrldq xm9, 2 + psrldq xm10, 2 + shufps xm9, xm10, q2020 + movu m10, [blend_4x8_1+8] + vinserti128 m6, xm9, 1 + vpblendvb m5, m11, m10 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [dstq+strideq*2-1] + movq xm9, [botq+strideq*0-1] + movq xm6, [top1q +1] + movq xm10, [dstq+strideq*0+1] + movhps xm5, [dstq+stride3q -1] + movhps xm9, [botq+strideq*1-1] + movhps xm6, [top2q +1] + movhps xm10, [dstq+strideq*1+1] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 + vpblendvb m5, [rsp+gprsize+0x60+hq*8+64+8*2], m11 +%endif + ret + +.border_block: + DEFINE_ARGS dst, stride, left, top, bot, pri, sec, stride3, dst4, edge +%define rstk rsp +%assign stack_offset stack_offset_entry +%assign regs_used 11 + ALLOC_STACK 2*16+(%2+4)*32, 16 +%define px rsp+2*16+2*32 + + pcmpeqw m14, m14 + psllw m14, 15 ; 0x8000 + + ; prepare pixel buffers - body/right +%if %1 == 4 + INIT_XMM avx2 +%endif +%if %2 == 8 + lea dst4q, [dstq+strideq*4] +%endif + lea stride3q, [strideq*3] + test edgeb, 2 ; have_right + jz .no_right + pmovzxbw m1, [dstq+strideq*0] + pmovzxbw m2, [dstq+strideq*1] + pmovzxbw m3, [dstq+strideq*2] + pmovzxbw m4, [dstq+stride3q] + mova [px+0*32], m1 + mova [px+1*32], m2 + mova [px+2*32], m3 + mova [px+3*32], m4 +%if %2 == 8 + pmovzxbw m1, [dst4q+strideq*0] + pmovzxbw m2, [dst4q+strideq*1] + pmovzxbw m3, [dst4q+strideq*2] + pmovzxbw m4, [dst4q+stride3q] + mova [px+4*32], m1 + mova [px+5*32], m2 + mova [px+6*32], m3 + mova [px+7*32], m4 +%endif + jmp .body_done +.no_right: +%if %1 == 4 + movd xm1, [dstq+strideq*0] + movd xm2, [dstq+strideq*1] + movd xm3, [dstq+strideq*2] + movd xm4, [dstq+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+0*32], xm1 + movq [px+1*32], xm2 + movq [px+2*32], xm3 + movq [px+3*32], xm4 +%else + pmovzxbw xm1, [dstq+strideq*0] + pmovzxbw xm2, [dstq+strideq*1] + pmovzxbw xm3, [dstq+strideq*2] + pmovzxbw xm4, [dstq+stride3q] + mova [px+0*32], xm1 + mova [px+1*32], xm2 + mova [px+2*32], xm3 + mova [px+3*32], xm4 +%endif + movd [px+0*32+%1*2], xm14 + movd [px+1*32+%1*2], xm14 + movd [px+2*32+%1*2], xm14 + movd [px+3*32+%1*2], xm14 +%if %2 == 8 + %if %1 == 4 + movd xm1, [dst4q+strideq*0] + movd xm2, [dst4q+strideq*1] + movd xm3, [dst4q+strideq*2] + movd xm4, [dst4q+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+4*32], xm1 + movq [px+5*32], xm2 + movq [px+6*32], xm3 + movq [px+7*32], xm4 + %else + pmovzxbw xm1, [dst4q+strideq*0] + pmovzxbw xm2, [dst4q+strideq*1] + pmovzxbw xm3, [dst4q+strideq*2] + pmovzxbw xm4, [dst4q+stride3q] + mova [px+4*32], xm1 + mova [px+5*32], xm2 + mova [px+6*32], xm3 + mova [px+7*32], xm4 + %endif + movd [px+4*32+%1*2], xm14 + movd [px+5*32+%1*2], xm14 + movd [px+6*32+%1*2], xm14 + movd [px+7*32+%1*2], xm14 +%endif +.body_done: + + ; top + test edgeb, 4 ; have_top + jz .no_top + test edgeb, 1 ; have_left + jz .top_no_left + test edgeb, 2 ; have_right + jz .top_no_right + pmovzxbw m1, [topq+strideq*0-(%1/2)] + pmovzxbw m2, [topq+strideq*1-(%1/2)] + movu [px-2*32-%1], m1 + movu [px-1*32-%1], m2 + jmp .top_done +.top_no_right: + pmovzxbw m1, [topq+strideq*0-%1] + pmovzxbw m2, [topq+strideq*1-%1] + movu [px-2*32-%1*2], m1 + movu [px-1*32-%1*2], m2 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.top_no_left: + test edgeb, 2 ; have_right + jz .top_no_left_right + pmovzxbw m1, [topq+strideq*0] + pmovzxbw m2, [topq+strideq*1] + mova [px-2*32+0], m1 + mova [px-1*32+0], m2 + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + jmp .top_done +.top_no_left_right: +%if %1 == 4 + movd xm1, [topq+strideq*0] + pinsrd xm1, [topq+strideq*1], 1 + pmovzxbw xm1, xm1 + movq [px-2*32+0], xm1 + movhps [px-1*32+0], xm1 +%else + pmovzxbw xm1, [topq+strideq*0] + pmovzxbw xm2, [topq+strideq*1] + mova [px-2*32+0], xm1 + mova [px-1*32+0], xm2 +%endif + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.no_top: + movu [px-2*32-%1], m14 + movu [px-1*32-%1], m14 +.top_done: + + ; left + test edgeb, 1 ; have_left + jz .no_left + pmovzxbw xm1, [leftq+ 0] +%if %2 == 8 + pmovzxbw xm2, [leftq+ 8] +%endif + movd [px+0*32-4], xm1 + pextrd [px+1*32-4], xm1, 1 + pextrd [px+2*32-4], xm1, 2 + pextrd [px+3*32-4], xm1, 3 +%if %2 == 8 + movd [px+4*32-4], xm2 + pextrd [px+5*32-4], xm2, 1 + pextrd [px+6*32-4], xm2, 2 + pextrd [px+7*32-4], xm2, 3 +%endif + jmp .left_done +.no_left: + movd [px+0*32-4], xm14 + movd [px+1*32-4], xm14 + movd [px+2*32-4], xm14 + movd [px+3*32-4], xm14 +%if %2 == 8 + movd [px+4*32-4], xm14 + movd [px+5*32-4], xm14 + movd [px+6*32-4], xm14 + movd [px+7*32-4], xm14 +%endif +.left_done: + + ; bottom + DEFINE_ARGS dst, stride, _, _, bot, pri, sec, stride3, _, edge + test edgeb, 8 ; have_bottom + jz .no_bottom + test edgeb, 1 ; have_left + jz .bottom_no_left + test edgeb, 2 ; have_right + jz .bottom_no_right + pmovzxbw m1, [botq+strideq*0-(%1/2)] + pmovzxbw m2, [botq+strideq*1-(%1/2)] + movu [px+(%2+0)*32-%1], m1 + movu [px+(%2+1)*32-%1], m2 + jmp .bottom_done +.bottom_no_right: + pmovzxbw m1, [botq+strideq*0-%1] + pmovzxbw m2, [botq+strideq*1-%1] + movu [px+(%2+0)*32-%1*2], m1 + movu [px+(%2+1)*32-%1*2], m2 +%if %1 == 8 + movd [px+(%2-1)*32+%1*2], xm14 ; overwritten by previous movu +%endif + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.bottom_no_left: + test edgeb, 2 ; have_right + jz .bottom_no_left_right + pmovzxbw m1, [botq+strideq*0] + pmovzxbw m2, [botq+strideq*1] + mova [px+(%2+0)*32+0], m1 + mova [px+(%2+1)*32+0], m2 + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + jmp .bottom_done +.bottom_no_left_right: +%if %1 == 4 + movd xm1, [botq+strideq*0] + pinsrd xm1, [botq+strideq*1], 1 + pmovzxbw xm1, xm1 + movq [px+(%2+0)*32+0], xm1 + movhps [px+(%2+1)*32+0], xm1 +%else + pmovzxbw xm1, [botq+strideq*0] + pmovzxbw xm2, [botq+strideq*1] + mova [px+(%2+0)*32+0], xm1 + mova [px+(%2+1)*32+0], xm2 +%endif + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.no_bottom: + movu [px+(%2+0)*32-%1], m14 + movu [px+(%2+1)*32-%1], m14 +.bottom_done: + + ; actual filter + INIT_YMM avx2 + DEFINE_ARGS dst, stride, _, pridmp, damping, pri, secdmp, stride3, zero +%undef edged + ; register to shuffle values into after packing + vbroadcasti128 m12, [shufb_lohi] + + mov dampingd, r8m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + test prid, prid + jz .border_sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + test secdmpd, secdmpd + jz .border_pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, _, pridmp, table, pri, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, _, dir, table, pri, sec, stride3 + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + BORDER_PREP_REGS %1, %2 +%if %1*%2*2/mmsize > 1 +.border_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2, 1 +.border_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1 + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1 + dec kq + jge .border_k_loop + + vpbroadcastd m10, [pw_2048] + BORDER_ADJUST_PIXEL %1, m10, 1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_v_loop +%endif + RET + +.border_pri_only: + DEFINE_ARGS dst, stride, _, pridmp, table, pri, _, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + DEFINE_ARGS dst, stride, _, dir, table, pri, _, stride3 + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m1, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_pri_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1 + dec kq + jge .border_pri_k_loop + BORDER_ADJUST_PIXEL %1, m1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_pri_v_loop +%endif + RET + +.border_sec_only: + DEFINE_ARGS dst, stride, _, _, damping, _, secdmp, stride3 + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, _, _, table, _, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + DEFINE_ARGS dst, stride, _, dir, table, _, sec, stride3 + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m0, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_sec_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1 + dec kq + jge .border_sec_k_loop + BORDER_ADJUST_PIXEL %1, m0 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_sec_v_loop +%endif + RET +%endmacro + +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 + +INIT_YMM avx2 +cglobal cdef_dir_8bpc, 3, 4, 6, src, stride, var, stride3 + lea stride3q, [strideq*3] + movq xm0, [srcq+strideq*0] + movq xm1, [srcq+strideq*1] + movq xm2, [srcq+strideq*2] + movq xm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+stride3q ] + vpbroadcastq m5, [srcq+strideq*2] + vpblendd m0, m4, 0xf0 + vpblendd m1, m5, 0xf0 + vpbroadcastq m4, [srcq+strideq*1] + vpbroadcastq m5, [srcq+strideq*0] + vpblendd m2, m4, 0xf0 + vpblendd m3, m5, 0xf0 + pxor m4, m4 + punpcklbw m0, m4 + punpcklbw m1, m4 + punpcklbw m2, m4 + punpcklbw m3, m4 +cglobal_label .main + vpbroadcastd m4, [pw_128] + PROLOGUE 3, 4, 15 + psubw m0, m4 + psubw m1, m4 + psubw m2, m4 + psubw m3, m4 + + ; shuffle registers to generate partial_sum_diag[0-1] together + vperm2i128 m7, m0, m0, 0x01 + vperm2i128 m6, m1, m1, 0x01 + vperm2i128 m5, m2, m2, 0x01 + vperm2i128 m4, m3, m3, 0x01 + + ; start with partial_sum_hv[0-1] + paddw m8, m0, m1 + paddw m9, m2, m3 + phaddw m10, m0, m1 + phaddw m11, m2, m3 + paddw m8, m9 + phaddw m10, m11 + vextracti128 xm9, m8, 1 + vextracti128 xm11, m10, 1 + paddw xm8, xm9 ; partial_sum_hv[1] + phaddw xm10, xm11 ; partial_sum_hv[0] + vinserti128 m8, xm10, 1 + vpbroadcastd m9, [div_table+44] + pmaddwd m8, m8 + pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] + + ; create aggregates [lower half]: + ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ + ; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 + ; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ + ; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x + ; and [upper half]: + ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ + ; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 + ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ + ; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx + ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd + + pslldq m9, m1, 2 + psrldq m10, m1, 14 + pslldq m11, m2, 4 + psrldq m12, m2, 12 + pslldq m13, m3, 6 + psrldq m14, m3, 10 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m4, 8 + psrldq m12, m4, 8 + pslldq m13, m5, 10 + psrldq m14, m5, 6 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m6, 12 + psrldq m12, m6, 4 + pslldq m13, m7, 14 + psrldq m14, m7, 2 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] + vbroadcasti128 m14, [shufw_6543210x] + vbroadcasti128 m13, [div_table+16] + vbroadcasti128 m12, [div_table+0] + paddw m9, m0 ; partial_sum_diag[0/1][0-7] + pshufb m10, m14 + punpckhwd m11, m9, m10 + punpcklwd m9, m10 + pmaddwd m11, m11 + pmaddwd m9, m9 + pmulld m11, m13 + pmulld m9, m12 + paddd m9, m11 ; cost0[a-d] | cost4[a-d] + + ; merge horizontally and vertically for partial_sum_alt[0-3] + paddw m10, m0, m1 + paddw m11, m2, m3 + paddw m12, m4, m5 + paddw m13, m6, m7 + phaddw m0, m4 + phaddw m1, m5 + phaddw m2, m6 + phaddw m3, m7 + + ; create aggregates [lower half]: + ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 + ; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx + ; and [upper half]: + ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 + ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx + ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m4, m11, 2 + psrldq m11, 14 + pslldq m5, m12, 4 + psrldq m12, 12 + pslldq m6, m13, 6 + psrldq m13, 10 + paddw m4, m10 + paddw m11, m12 + vpbroadcastd m12, [div_table+44] + paddw m5, m6 + paddw m11, m13 ; partial_sum_alt[3/2] right + vbroadcasti128 m13, [div_table+32] + paddw m4, m5 ; partial_sum_alt[3/2] left + pshuflw m5, m11, q3012 + punpckhwd m6, m11, m4 + punpcklwd m4, m5 + pmaddwd m6, m6 + pmaddwd m4, m4 + pmulld m6, m12 + pmulld m4, m13 + paddd m4, m6 ; cost7[a-d] | cost5[a-d] + + ; create aggregates [lower half]: + ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 + ; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx + ; and [upper half]: + ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 + ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx + ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m5, m1, 2 + psrldq m1, 14 + pslldq m6, m2, 4 + psrldq m2, 12 + pslldq m7, m3, 6 + psrldq m3, 10 + paddw m5, m0 + paddw m1, m2 + paddw m6, m7 + paddw m1, m3 ; partial_sum_alt[0/1] right + paddw m5, m6 ; partial_sum_alt[0/1] left + pshuflw m0, m1, q3012 + punpckhwd m1, m5 + punpcklwd m5, m0 + pmaddwd m1, m1 + pmaddwd m5, m5 + pmulld m1, m12 + pmulld m5, m13 + paddd m5, m1 ; cost1[a-d] | cost3[a-d] + + mova xm0, [pd_47130256+ 16] + mova m1, [pd_47130256] + phaddd m9, m8 + phaddd m5, m4 + phaddd m9, m5 + vpermd m0, m9 ; cost[0-3] + vpermd m1, m9 ; cost[4-7] | cost[0-3] + + ; now find the best cost + pmaxsd xm2, xm0, xm1 + pshufd xm3, xm2, q1032 + pmaxsd xm2, xm3 + pshufd xm3, xm2, q2301 + pmaxsd xm2, xm3 ; best cost + + ; find the idx using minpos + ; make everything other than the best cost negative via subtraction + ; find the min of unsigned 16-bit ints to sort out the negative values + psubd xm4, xm1, xm2 + psubd xm3, xm0, xm2 + packssdw xm3, xm4 + phminposuw xm3, xm3 + + ; convert idx to 32-bits + psrld xm3, 16 + movd eax, xm3 + + ; get idx^4 complement + vpermd m3, m1 + psubd xm2, xm3 + psrld xm2, 10 + movd [varq], xm2 + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_avx512.asm b/third_party/dav1d/src/x86/cdef_avx512.asm new file mode 100644 index 0000000000..b4f9c008ca --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx512.asm @@ -0,0 +1,860 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +%macro DUP4 1-* + %rep %0 + times 4 db %1 + %rotate 1 + %endrep +%endmacro + +%macro DIRS 16 ; cdef_directions[] + %rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1 + ; masking away unused bits allows us to use a single vpaddd {1to16} + ; instruction instead of having to do vpbroadcastd + paddb + db %13 & 0x3f, -%13 & 0x3f + %rotate 1 + %endrep +%endmacro + +SECTION_RODATA 64 + +lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13 + db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37 + db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57 +lut_perm_4x8a: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 96, 97, 0, 1, 2, 3, 4, 5, 98, 99, 8, 9, 10, 11, 12, 13 +lut_perm_4x8b:db 100,101, 16, 17, 18, 19, 20, 21,102,103, 24, 25, 26, 27, 28, 29 + db 104,105, 32, 33, 34, 35, 36, 37,106,107, 40, 41, 42, 43, 44, 45 + db 108,109, 48, 49, 50, 51, 52, 53,110,111, 56, 57, 58, 59, 60, 61 + db 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95 +pd_01234567: dd 0, 1, 2, 3, 4, 5, 6, 7 +lut_perm_8x8a: db 32, 33, 34, 35, 36, 37, 38, 39, 48, 49, 50, 51, 52, 53, 54, 55 + db 36, 37, 38, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56, 57, 58, 59 +lut_perm_8x8b: db 12, 13, 0, 1, 2, 3, 4, 5, 14, 15, 16, 17, 18, 19, 20, 21 + db 2, 3, 4, 5, 6, 7, 8, 9, 18, 19, 20, 21, 22, 23, 24, 25 + db 28, 29, 32, 33, 34, 35, 36, 37, 30, 31, 48, 49, 50, 51, 52, 53 + db 34, 35, 36, 37, 38, 39, 40, 41, 50, 51, 52, 53, 54, 55, 56, 57 +end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 +end_perm_clip: db 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30 + db 32, 36, 40, 44, 34, 38, 42, 46, 48, 52, 56, 60, 50, 54, 58, 62 + db 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31 + db 33, 37, 41, 45, 35, 39, 43, 47, 49, 53, 57, 61, 51, 55, 59, 63 +edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001 + dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011 + dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101 + dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111 + dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001 + dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011 + dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101 + dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111 +px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45 +cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15 +gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0 + dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2 + dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4 + dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6 +pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4 +sec_tap: db 32, 32, 16, 16 +pd_268435568: dd 268435568 + +SECTION .text + +%if WIN64 +DECLARE_REG_TMP 4 +%else +DECLARE_REG_TMP 8 +%endif + +; lut: +; t0 t1 t2 t3 t4 t5 t6 t7 +; T0 T1 T2 T3 T4 T5 T6 T7 +; L0 L1 00 01 02 03 04 05 +; L2 L3 10 11 12 13 14 15 +; L4 L5 20 21 22 23 24 25 +; L6 L7 30 31 32 33 34 35 +; b0 b1 b2 b3 b4 b5 b6 b7 +; B0 B1 B2 B3 B4 B5 B6 B7 + +INIT_ZMM avx512icl +cglobal cdef_filter_4x4_8bpc, 5, 8, 13, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%define base r7-edge_mask + movq xmm0, [dstq+strideq*0] + movhps xmm0, [dstq+strideq*1] + lea r7, [edge_mask] + movq xmm1, [topq+strideq*0-2] + movhps xmm1, [topq+strideq*1-2] + mov r6d, edgem + vinserti32x4 ym0, ymm0, [leftq], 1 + lea r2, [strideq*3] + vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1 + mova m5, [base+lut_perm_4x4] + vinserti32x4 m0, [dstq+r2], 2 + test r6b, 0x08 ; avoid buffer overread + jz .main + vinserti32x4 m1, [botq+strideq*0-4], 2 + vinserti32x4 m0, [botq+strideq*1-4], 3 +.main: + movifnidn prid, prim + mov t0d, dirm + mova m3, [base+px_idx] + mov r3d, dampingm + vpermi2b m5, m0, m1 ; lut + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m7, m7 + lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m6, m3, m5 ; px + cmp r6d, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1 +%macro CDEF_FILTER_4x4_PRI 0 + vpcmpub k1, m6, m1, 6 ; px > pN + psubb m2, m1, m6 + lzcnt r6d, prid + vpsubb m2{k1}, m6, m1 ; abs(diff) + vpbroadcastb m4, prid + and prid, 1 + vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift + movifnidn secd, secm + vpbroadcastd m10, [base+pri_tap+priq*4] + vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap) + psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift))) + pminub m2, m4 + vpdpbusd m0, m2, m10 ; sum +%endmacro + CDEF_FILTER_4x4_PRI + test secd, secd + jz .end_no_clip + call .sec +.end_clip: + pminub m4, m6, m1 + pmaxub m1, m6 + pminub m5, m2, m3 + pmaxub m2, m3 + pminub m4, m5 + pmaxub m2, m1 + psrldq m1, m4, 2 + psrldq m3, m2, 2 + pminub m1, m4 + vpcmpw k1, m0, m7, 1 + vpshldd m6, m0, 8 + pmaxub m2, m3 + pslldq m3, m1, 1 + psubw m7, m0 + paddusw m0, m6 ; clip >0xff + vpsubusw m0{k1}, m6, m7 ; clip <0x00 + pslldq m4, m2, 1 + pminub m1, m3 + pmaxub m2, m4 + pmaxub m0, m1 + pminub m0, m2 + jmp .end +.sec_only: + movifnidn secd, secm + call .sec +.end_no_clip: + vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4) + paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) +.end: + mova xm1, [base+end_perm] + vpermb m0, m1, m0 ; output in bits 8-15 of each dword + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + RET +.mask_edges_sec_only: + movifnidn secd, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + vpbroadcastq m8, [base+edge_mask+r6*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m8, m2 ; index in-range + mova m1, m6 + vpermb m1{k1}, m2, m5 + CDEF_FILTER_4x4_PRI + test secd, secd + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m8, m4 + mova m2, m6 + vpermb m2{k1}, m4, m5 + vpshufbitqmb k1, m8, m9 + mova m3, m6 + vpermb m3{k1}, m9, m5 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1 + vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3 +.sec_main: + vpbroadcastd m8, [base+sec_tap] + vpcmpub k1, m6, m2, 6 + psubb m4, m2, m6 + vpbroadcastb m12, secd + lzcnt secd, secd + vpsubb m4{k1}, m6, m2 + vpcmpub k2, m6, m3, 6 + vpbroadcastq m11, [r3+secq*8] + gf2p8affineqb m10, m4, m11, 0 + psubb m5, m3, m6 + mova m9, m8 + vpsubb m8{k1}, m7, m8 + psubusb m10, m12, m10 + vpsubb m5{k2}, m6, m3 + pminub m4, m10 + vpdpbusd m0, m4, m8 + gf2p8affineqb m11, m5, m11, 0 + vpsubb m9{k2}, m7, m9 + psubusb m12, m11 + pminub m5, m12 + vpdpbusd m0, m5, m9 + ret + +DECLARE_REG_TMP 2, 7 + +; lut top lut bottom +; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 +; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 +; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 +; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 +; L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65 +; L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75 +; L8 L9 40 41 42 43 44 45 b0 b1 b2 b3 b4 b5 b6 b7 +; La Lb 50 51 52 53 54 55 B0 B1 B2 B3 B4 B5 B6 B7 + +cglobal cdef_filter_4x8_8bpc, 5, 9, 22, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + vpbroadcastd ym21, strided + mov r6d, edgem + lea r8, [edge_mask] + movq xm1, [topq+strideq*0-2] + pmulld ym21, [base+pd_01234567] + kxnorb k1, k1, k1 + movq xm2, [topq+strideq*1-2] + vpgatherdq m0{k1}, [dstq+ym21] ; +0+1 +2+3 +4+5 +6+7 + mova m14, [base+lut_perm_4x8a] + movu m15, [base+lut_perm_4x8b] + test r6b, 0x08 ; avoid buffer overread + jz .main + vinserti32x4 ym1, [botq+strideq*0-2], 1 + vinserti32x4 ym2, [botq+strideq*1-2], 1 +.main: + punpcklqdq ym1, ym2 + vinserti32x4 m1, [leftq], 2 ; -2-1 +8+9 left ____ + movifnidn prid, prim + mov t0d, dirm + mova m16, [base+px_idx] + mov r3d, dampingm + vpermi2b m14, m0, m1 ; lut top + vpermi2b m15, m0, m1 ; lut bottom + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m20, m20 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m2, m16, m14 ; pxt + vpermb m3, m16, m15 ; pxb + mova m1, m0 + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m4, m6, m14 ; pNt k0p0 k0p1 k1p0 k1p1 + vpermb m5, m6, m15 ; pNb +%macro CDEF_FILTER_4x8_PRI 0 + vpcmpub k1, m2, m4, 6 ; pxt > pNt + vpcmpub k2, m3, m5, 6 ; pxb > pNb + psubb m6, m4, m2 + psubb m7, m5, m3 + lzcnt r6d, prid + vpsubb m6{k1}, m2, m4 ; abs(diff_top) + vpsubb m7{k2}, m3, m5 ; abs(diff_bottom) + vpbroadcastb m13, prid + vpbroadcastq m9, [r3+r6*8] + and prid, 1 + vpbroadcastd m11, [base+pri_tap+priq*4] + vgf2p8affineqb m8, m6, m9, 0 ; abs(dt) >> shift + vgf2p8affineqb m9, m7, m9, 0 ; abs(db) >> shift + mova m10, m11 + movifnidn t1d, secm + vpsubb m10{k1}, m20, m11 ; apply_sign(pri_tap_top) + vpsubb m11{k2}, m20, m11 ; apply_sign(pri_tap_bottom) + psubusb m12, m13, m8 ; imax(0, pri_strength - (abs(dt) >> shift))) + psubusb m13, m13, m9 ; imax(0, pri_strength - (abs(db) >> shift))) + pminub m6, m12 + pminub m7, m13 + vpdpbusd m0, m6, m10 ; sum top + vpdpbusd m1, m7, m11 ; sum bottom +%endmacro + CDEF_FILTER_4x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m10, m4, m2 + pminub m12, m6, m8 + pminub m11, m5, m3 + pminub m13, m7, m9 + pmaxub m4, m2 + pmaxub m6, m8 + pmaxub m5, m3 + pmaxub m7, m9 + pminub m10, m12 + pminub m11, m13 + pmaxub m4, m6 + pmaxub m5, m7 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + kxnorb k2, k2, k2 ; hw lw + vpshrdd m12, m0, m1, 16 ; m1lw m0hw + vpshrdd m6, m10, m11, 16 ; m11lw m10hw + vpshrdd m8, m4, m5, 16 ; m5lw m4hw + vpblendmw m7{k1}, m10, m11 ; m11hw m10lw + vpblendmw m9{k1}, m4, m5 ; m5hw m4lw + vpblendmw m4{k1}, m0, m12 ; m1lw m0lw + vpblendmw m5{k1}, m12, m1 ; m1hw m0hw + vpshrdd m2, m3, 16 + pminub m6, m7 + pmaxub m8, m9 + mova ym14, [base+end_perm] + vpcmpw k1, m4, m20, 1 + vpshldw m2, m5, 8 + pslldq m7, m6, 1 + pslldq m9, m8, 1 + psubw m5, m20, m4 + paddusw m0, m4, m2 ; clip >0xff + pminub m6, m7 + pmaxub m8, m9 + psubusw m0{k1}, m2, m5 ; clip <0x00 + pmaxub m0, m6 + pminub m0, m8 + vpermb m0, m14, m0 + vpscatterdd [dstq+ym21]{k2}, ym0 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova ym4, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m2, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m3, m1, 8 + paddw m0, m2 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m3 + pslld m0, 16 + vpshrdd m0, m1, 16 + vpermb m0, m4, m0 ; output in bits 8-15 of each word + vpscatterdd [dstq+ym21]{k1}, ym0 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t1d, r6d + or r6d, 8 ; top 4x4 has bottom + or t1d, 4 ; bottom 4x4 has top + vpbroadcastq m17, [base+edge_mask+r6*8] + vpbroadcastq m18, [base+edge_mask+t1*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m17, m6 ; index in-range + vpshufbitqmb k2, m18, m6 + mova m4, m2 + mova m5, m3 + vpermb m4{k1}, m6, m14 + vpermb m5{k2}, m6, m15 + CDEF_FILTER_4x8_PRI + test t1d, t1d + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m10, m16, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m11, m16, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m17, m10 + vpshufbitqmb k2, m18, m10 + vpshufbitqmb k3, m17, m11 + vpshufbitqmb k4, m18, m11 + mova m6, m2 + mova m7, m3 + mova m8, m2 + mova m9, m3 + vpermb m6{k1}, m10, m14 + vpermb m7{k2}, m10, m15 + vpermb m8{k3}, m11, m14 + vpermb m9{k4}, m11, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m8, m16, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m9, m16, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m6, m8, m14 ; pNt k0s0 k0s1 k1s0 k1s1 + vpermb m7, m8, m15 ; pNb + vpermb m8, m9, m14 ; pNt k0s2 k0s3 k1s2 k1s3 + vpermb m9, m9, m15 ; pNb +.sec_main: + vpbroadcastb m18, t1d + lzcnt t1d, t1d + vpcmpub k1, m2, m6, 6 + vpcmpub k2, m3, m7, 6 + vpcmpub k3, m2, m8, 6 + vpcmpub k4, m3, m9, 6 + vpbroadcastq m17, [r3+t1*8] + psubb m10, m6, m2 + psubb m11, m7, m3 + psubb m12, m8, m2 + psubb m13, m9, m3 + vpsubb m10{k1}, m2, m6 ; abs(dt0) + vpsubb m11{k2}, m3, m7 ; abs(db0) + vpsubb m12{k3}, m2, m8 ; abs(dt1) + vpsubb m13{k4}, m3, m9 ; abs(db1) + vpbroadcastd m19, [base+sec_tap] + gf2p8affineqb m14, m10, m17, 0 ; abs(dt0) >> shift + gf2p8affineqb m15, m11, m17, 0 ; abs(db0) >> shift + gf2p8affineqb m16, m12, m17, 0 ; abs(dt1) >> shift + gf2p8affineqb m17, m13, m17, 0 ; abs(db1) >> shift + psubusb m14, m18, m14 ; imax(0, sec_strength - (abs(dt0) >> shift))) + psubusb m15, m18, m15 ; imax(0, sec_strength - (abs(db0) >> shift))) + psubusb m16, m18, m16 ; imax(0, sec_strength - (abs(dt1) >> shift))) + psubusb m17, m18, m17 ; imax(0, sec_strength - (abs(db1) >> shift))) + pminub m10, m14 + pminub m11, m15 + pminub m12, m16 + pminub m13, m17 + mova m14, m19 + mova m15, m19 + mova m16, m19 + vpsubb m14{k1}, m20, m19 ; apply_sign(sec_tap_top_0) + vpsubb m15{k2}, m20, m19 ; apply_sign(sec_tap_bottom_0) + vpsubb m16{k3}, m20, m19 ; apply_sign(sec_tap_top_1) + vpsubb m19{k4}, m20, m19 ; apply_sign(sec_tap_bottom_1) + vpdpbusd m0, m10, m14 + vpdpbusd m1, m11, m15 + vpdpbusd m0, m12, m16 + vpdpbusd m1, m13, m19 + ret + +; lut tl lut tr +; t0 t1 t2 t3 t4 t5 t6 t7 t4 t5 t6 t7 t8 t9 ta tb +; T0 T1 T2 T3 T4 T5 T6 T7 T4 T5 T6 T7 T8 T9 Ta Tb +; L0 L1 00 01 02 03 04 05 02 03 04 05 06 07 08 09 +; L2 L3 10 11 12 13 14 15 12 13 14 15 16 17 18 19 +; L4 L5 20 21 22 23 24 25 22 23 24 25 26 27 28 29 +; L6 L7 30 31 32 33 34 35 32 33 34 35 36 37 38 39 +; L8 L9 40 41 42 43 44 45 42 43 44 45 46 47 48 49 +; La Lb 50 51 52 53 54 55 52 53 54 55 56 57 58 59 +; lut bl lut br +; L4 L5 20 21 22 23 24 25 22 23 24 25 26 27 28 29 +; L6 L7 30 31 32 33 34 35 32 33 34 35 36 37 38 39 +; L8 L9 40 41 42 43 44 45 42 43 44 45 46 47 48 49 +; La Lb 50 51 52 53 54 55 52 53 54 55 56 57 58 59 +; Lc Ld 60 61 62 63 64 65 62 63 64 65 66 67 68 69 +; Le Lf 70 71 72 73 74 75 72 73 74 75 76 77 78 79 +; b0 b1 b2 b3 b4 b5 b6 b7 b4 b5 b6 b7 b8 b9 ba bb +; B0 B1 B2 B3 B4 B5 B6 B7 B4 B5 B6 B7 B8 B9 Ba Bb + +cglobal cdef_filter_8x8_8bpc, 5, 11, 32, 4*64, dst, stride, left, top, bot, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + movu xm16, [dstq+strideq*0] + pinsrd xm16, [leftq+4*0], 3 + mov r6d, edgem + vinserti128 ym16, [dstq+strideq*1], 1 + lea r10, [dstq+strideq*4] + movu xm17, [dstq+strideq*2] + vinserti32x4 m16, [topq+strideq*0-2], 2 + lea r9, [strideq*3] + pinsrd xm17, [leftq+4*1], 3 + vinserti32x4 m16, [topq+strideq*1-2], 3 ; 0 1 t T + lea r8, [edge_mask] + vinserti128 ym17, [dstq+r9 ], 1 + vpbroadcastd ym18, [leftq+4*2] + vpblendd ym17, ym18, 0x80 + movu xm18, [r10 +strideq*2] + vinserti32x4 m17, [r10 +strideq*0], 2 + pinsrd xm18, [leftq+4*3], 3 + vinserti32x4 m17, [r10 +strideq*1], 3 ; 2 3 4 5 + vinserti128 ym18, [r10 +r9 ], 1 + test r6b, 0x08 ; avoid buffer overread + jz .main + vinserti32x4 m18, [botq+strideq*0-2], 2 + vinserti32x4 m18, [botq+strideq*1-2], 3 ; 6 7 b B +.main: + mova m0, [base+lut_perm_8x8a] + movu m1, [base+lut_perm_8x8b] + mova m30, [base+px_idx] + vpermb m16, m0, m16 + movifnidn prid, prim + vpermb m17, m1, m17 + mov t0d, dirm + vpermb m18, m0, m18 + mov r3d, dampingm + vshufi32x4 m12, m16, m17, q2020 ; lut tl + vshufi32x4 m13, m16, m17, q3131 ; lut tr + vshufi32x4 m14, m17, m18, q0220 ; lut bl + vshufi32x4 m15, m17, m18, q1331 ; lut br + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m31, m31 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m4, m30, m12 ; pxtl + mova m1, m0 + vpermb m5, m30, m13 ; pxtr + mova m2, m0 + vpermb m6, m30, m14 ; pxbl + mova m3, m0 + vpermb m7, m30, m15 ; pxbr + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m11, m30, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m8, m11, m12 ; pNtl k0p0 k0p1 k1p0 k1p1 + vpermb m9, m11, m13 ; pNtr + vpermb m10, m11, m14 ; pNbl + vpermb m11, m11, m15 ; pNbr +%macro CDEF_FILTER_8x8_PRI 0 + vpcmpub k1, m4, m8, 6 ; pxtl > pNtl + vpcmpub k2, m5, m9, 6 ; pxtr > pNtr + vpcmpub k3, m6, m10, 6 ; pxbl > pNbl + vpcmpub k4, m7, m11, 6 ; pxbr > pNbr + psubb m16, m8, m4 + psubb m17, m9, m5 + psubb m18, m10, m6 + psubb m19, m11, m7 + lzcnt r6d, prid + vpsubb m16{k1}, m4, m8 ; abs(diff_tl) + vpsubb m17{k2}, m5, m9 ; abs(diff_tr) + vpsubb m18{k3}, m6, m10 ; abs(diff_bl) + vpsubb m19{k4}, m7, m11 ; abs(diff_br) + vpbroadcastq m28, [r3+r6*8] + vpbroadcastb m29, prid + and prid, 1 + vpbroadcastd m27, [base+pri_tap+priq*4] + vgf2p8affineqb m20, m16, m28, 0 ; abs(dtl) >> shift + vgf2p8affineqb m21, m17, m28, 0 ; abs(dtr) >> shift + vgf2p8affineqb m22, m18, m28, 0 ; abs(dbl) >> shift + vgf2p8affineqb m23, m19, m28, 0 ; abs(dbl) >> shift + mova m24, m27 + mova m25, m27 + mova m26, m27 + movifnidn t1d, secm + vpsubb m24{k1}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m25{k2}, m31, m27 ; apply_sign(pri_tap_tr) + vpsubb m26{k3}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m27{k4}, m31, m27 ; apply_sign(pri_tap_tr) + psubusb m20, m29, m20 ; imax(0, pri_strength - (abs(dtl) >> shift))) + psubusb m21, m29, m21 ; imax(0, pri_strength - (abs(dtr) >> shift))) + psubusb m22, m29, m22 ; imax(0, pri_strength - (abs(dbl) >> shift))) + psubusb m23, m29, m23 ; imax(0, pri_strength - (abs(dbr) >> shift))) + pminub m16, m20 + pminub m17, m21 + pminub m18, m22 + pminub m19, m23 + vpdpbusd m0, m16, m24 ; sum tl + vpdpbusd m1, m17, m25 ; sum tr + vpdpbusd m2, m18, m26 ; sum bl + vpdpbusd m3, m19, m27 ; sum br +%endmacro + CDEF_FILTER_8x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m20, m8, m4 + pminub m24, m12, m16 + pminub m21, m9, m5 + pminub m25, m13, m17 + pminub m22, m10, m6 + pminub m26, m14, m18 + pminub m23, m11, m7 + pminub m27, m15, m19 + pmaxub m8, m4 + pmaxub m12, m16 + pmaxub m9, m5 + pmaxub m13, m17 + pmaxub m10, m6 + pmaxub m14, m18 + pmaxub m11, m7 + pmaxub m15, m19 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + pmaxub m8, m12 + pmaxub m9, m13 + pmaxub m10, m14 + pmaxub m11, m15 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + vpshrdd m24, m0, m1, 16 + vpshrdd m25, m2, m3, 16 + vpshrdd m12, m20, m21, 16 + vpshrdd m14, m22, m23, 16 + vpshrdd m16, m8, m9, 16 + vpshrdd m18, m10, m11, 16 + vpblendmw m13{k1}, m20, m21 + vpblendmw m15{k1}, m22, m23 + vpblendmw m17{k1}, m8, m9 + vpblendmw m19{k1}, m10, m11 + vpblendmw m20{k1}, m0, m24 + vpblendmw m21{k1}, m24, m1 + vpblendmw m22{k1}, m2, m25 + vpblendmw m23{k1}, m25, m3 + vpshrdd m4, m5, 16 + vpshrdd m6, m7, 16 + pminub m12, m13 + pminub m14, m15 + pmaxub m16, m17 + pmaxub m18, m19 + mova m8, [base+end_perm_clip] + vpcmpw k2, m20, m31, 1 + vpcmpw k3, m22, m31, 1 + vpshldw m4, m21, 8 + vpshldw m6, m23, 8 + kunpckdq k1, k1, k1 + kxnorb k4, k4, k4 + vpshrdw m11, m12, m14, 8 + vpshrdw m15, m16, m18, 8 + vpblendmb m13{k1}, m12, m14 + vpblendmb m17{k1}, m16, m18 + psubw m21, m31, m20 + psubw m23, m31, m22 + paddusw m0, m20, m4 ; clip >0xff + paddusw m1, m22, m6 + pminub m11, m13 + pmaxub m15, m17 + psubusw m0{k2}, m4, m21 ; clip <0x00 + psubusw m1{k3}, m6, m23 + psrlw m0, 8 + vmovdqu8 m0{k1}, m1 + pmaxub m0, m11 + pminub m0, m15 + vpermb m0, m8, m0 + vextracti32x4 xm1, m0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*2], xm1 + movq [r10 +strideq*0], xm2 + movq [r10 +strideq*2], xm3 + movhps [dstq+strideq*1], xm0 + movhps [dstq+r9 ], xm1 + movhps [r10 +strideq*1], xm2 + movhps [r10 +r9 ], xm3 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova xm8, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m4, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m5, m1, 8 + vpshldd m6, m2, 8 + vpshldd m7, m3, 8 + paddw m0, m4 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + vpermb m0, m8, m0 + vpermb m1, m8, m1 + vpermb m2, m8, m2 + vpermb m3, m8, m3 + punpckldq m4, m0, m1 + punpckhdq m0, m1 + punpckldq m5, m2, m3 + punpckhdq m2, m3 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm0 + movq [r10 +strideq*0], xm5 + movq [r10 +strideq*2], xm2 + movhps [dstq+strideq*1], xm4 + movhps [dstq+r9 ], xm0 + movhps [r10 +strideq*1], xm5 + movhps [r10 +r9 ], xm2 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t0d, r6d + mov t1d, r6d + or t0d, 0xA ; top-left 4x4 has bottom and right + or t1d, 0x9 ; top-right 4x4 has bottom and left + vpbroadcastq m26, [base+edge_mask+t0*8] + vpbroadcastq m27, [base+edge_mask+t1*8] + mov t1d, r6d + or r6d, 0x6 ; bottom-left 4x4 has top and right + or t1d, 0x5 ; bottom-right 4x4 has top and left + vpbroadcastq m28, [base+edge_mask+r6*8] + vpbroadcastq m29, [base+edge_mask+t1*8] + mov t0d, dirm + test prid, prid + jz .mask_edges_sec_only + vpaddd m20, m30, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m26, m20 ; index in-range + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m8, m4 + mova m9, m5 + mova m10, m6 + mova m11, m7 + vpermb m8{k1}, m20, m12 + vpermb m9{k2}, m20, m13 + vpermb m10{k3}, m20, m14 + vpermb m11{k4}, m20, m15 + mova [rsp+0x00], m26 + mova [rsp+0x40], m27 + mova [rsp+0x80], m28 + mova [rsp+0xC0], m29 + CDEF_FILTER_8x8_PRI + test t1d, t1d + jz .end_no_clip + mova m26, [rsp+0x00] + mova m27, [rsp+0x40] + mova m28, [rsp+0x80] + mova m29, [rsp+0xC0] + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m26, m20 + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m16, m4 + mova m17, m5 + mova m18, m6 + mova m19, m7 + vpermb m16{k1}, m20, m12 + vpermb m17{k2}, m20, m13 + vpermb m18{k3}, m20, m14 + vpermb m19{k4}, m20, m15 + vpshufbitqmb k1, m26, m21 + vpshufbitqmb k2, m27, m21 + vpshufbitqmb k3, m28, m21 + vpshufbitqmb k4, m29, m21 + vpermb m12, m21, m12 + vpermb m13, m21, m13 + vpermb m14, m21, m14 + vpermb m15, m21, m15 + vpblendmb m12{k1}, m4, m12 + vpblendmb m13{k2}, m5, m13 + vpblendmb m14{k3}, m6, m14 + vpblendmb m15{k4}, m7, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m16, m20, m12 ; pNtl k0s0 k0s1 k1s0 k1s1 + vpermb m17, m20, m13 ; pNtr + vpermb m18, m20, m14 ; pNbl + vpermb m19, m20, m15 ; pNbr + vpermb m12, m21, m12 ; pNtl k0s2 k0s3 k1s2 k1s3 + vpermb m13, m21, m13 ; pNtr + vpermb m14, m21, m14 ; pNbl + vpermb m15, m21, m15 ; pNbr +.sec_main: +%macro CDEF_FILTER_8x8_SEC 4-5 0 ; load constants + vpcmpub k1, m4, %1, 6 + vpcmpub k2, m5, %2, 6 + vpcmpub k3, m6, %3, 6 + vpcmpub k4, m7, %4, 6 + psubb m20, %1, m4 + psubb m21, %2, m5 + psubb m22, %3, m6 + psubb m23, %4, m7 +%if %5 + vpbroadcastb m28, t1d + lzcnt t1d, t1d + vpbroadcastq m29, [r3+t1*8] +%endif + vpsubb m20{k1}, m4, %1 + vpsubb m21{k2}, m5, %2 + vpsubb m22{k3}, m6, %3 + vpsubb m23{k4}, m7, %4 + gf2p8affineqb m24, m20, m29, 0 + gf2p8affineqb m25, m21, m29, 0 + gf2p8affineqb m26, m22, m29, 0 + gf2p8affineqb m27, m23, m29, 0 +%if %5 + vpbroadcastd m30, [base+sec_tap] +%endif + psubusb m24, m28, m24 + psubusb m25, m28, m25 + psubusb m26, m28, m26 + psubusb m27, m28, m27 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + mova m24, m30 + mova m25, m30 + mova m26, m30 + mova m27, m30 + vpsubb m24{k1}, m31, m30 + vpsubb m25{k2}, m31, m30 + vpsubb m26{k3}, m31, m30 + vpsubb m27{k4}, m31, m30 + vpdpbusd m0, m20, m24 + vpdpbusd m1, m21, m25 + vpdpbusd m2, m22, m26 + vpdpbusd m3, m23, m27 +%endmacro + CDEF_FILTER_8x8_SEC m16, m17, m18, m19, 1 + CDEF_FILTER_8x8_SEC m12, m13, m14, m15 + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_sse.asm b/third_party/dav1d/src/x86/cdef_sse.asm new file mode 100644 index 0000000000..1b353121f4 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_sse.asm @@ -0,0 +1,1357 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; Copyright © 2019, VideoLabs +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +%macro DUP8 1-* + %rep %0 + times 8 db %1 + %rotate 1 + %endrep +%endmacro + +div_table_sse4: dd 840, 420, 280, 210, 168, 140, 120, 105 + dd 420, 210, 140, 105, 105, 105, 105, 105 +div_table_ssse3: dw 840, 840, 420, 420, 280, 280, 210, 210 + dw 168, 168, 140, 140, 120, 120, 105, 105 + dw 420, 420, 210, 210, 140, 140, 105, 105 + dw 105, 105, 105, 105, 105, 105, 105, 105 +const shufw_6543210x, \ + db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 +shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 +pw_8: times 8 dw 8 +pw_128: times 8 dw 128 +pw_256: times 8 dw 256 +pw_2048: times 8 dw 2048 +pw_0x7FFF: times 8 dw 0x7FFF +pw_0x8000: times 8 dw 0x8000 +tap_table: ; masks for 8-bit shift emulation + DUP8 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80 + ; weights + DUP8 4, 2, 3, 3, 2, 1 + ; taps indices + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + db 1 * 16 + 0, 2 * 16 + 0 + db 1 * 16 + 0, 2 * 16 - 1 + ; the last 6 are repeats of the first 6 so we don't need to & 7 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + +SECTION .text + +%macro movif32 2 + %if ARCH_X86_32 + mov %1, %2 + %endif +%endmacro + +%macro PMOVZXBW 2-3 0 ; %3 = half + %if cpuflag(sse4) && %3 == 0 + pmovzxbw %1, %2 + %else + %if %3 == 1 + movd %1, %2 + %else + movq %1, %2 + %endif + punpcklbw %1, m7 + %endif +%endmacro + +%macro PSHUFB_0 2 + %if cpuflag(ssse3) + pshufb %1, %2 + %else + punpcklbw %1, %1 + pshuflw %1, %1, q0000 + punpcklqdq %1, %1 + %endif +%endmacro + +%macro MOVDDUP 2 +%if cpuflag(ssse3) + movddup %1, %2 +%else + movq %1, %2 + punpcklqdq %1, %1 +%endif +%endmacro + +%macro ACCUMULATE_TAP 7 ; tap_offset, shift, shift_mask, strength, mul_tap, w, minmax + ; load p0/p1 + movsx offq, byte [dirq+kq+%1+14*8] ; off1 + %if %6 == 4 + movq m5, [stkq+offq*2+32*0] ; p0 + movhps m5, [stkq+offq*2+32*1] + %else + movu m5, [stkq+offq*2+32*0] ; p0 + %endif + neg offq ; -off1 + %if %6 == 4 + movq m6, [stkq+offq*2+32*0] ; p1 + movhps m6, [stkq+offq*2+32*1] + %else + movu m6, [stkq+offq*2+32*0] ; p1 + %endif + %if %7 + %if cpuflag(sse4) + ; out of bounds values are set to a value that is a both a large unsigned + ; value and a negative signed value. + ; use signed max and unsigned min to remove them + pmaxsw m7, m5 + pminuw m8, m5 + pmaxsw m7, m6 + pminuw m8, m6 + %else + pcmpeqw m3, m14, m5 + pminsw m8, m5 ; min after p0 + pandn m3, m5 + pmaxsw m7, m3 ; max after p0 + pcmpeqw m3, m14, m6 + pminsw m8, m6 ; min after p1 + pandn m3, m6 + pmaxsw m7, m3 ; max after p1 + %endif + %endif + + ; accumulate sum[m13] over p0/p1 + psubw m5, m4 ; diff_p0(p0 - px) + psubw m6, m4 ; diff_p1(p1 - px) + packsswb m5, m6 ; convert pixel diff to 8-bit + %if cpuflag(ssse3) + pshufb m5, m13 ; group diffs p0 and p1 into pairs + pabsb m6, m5 + psignb m3, %5, m5 + %else + movlhps m6, m5 + punpckhbw m6, m5 + pxor m5, m5 + pcmpgtb m5, m6 + paddb m6, m5 + pxor m6, m5 + paddb m3, %5, m5 + pxor m3, m5 + %endif + pand m9, %3, m6 ; emulate 8-bit shift + psrlw m9, %2 + psubusb m5, %4, m9 + pminub m5, m6 ; constrain(diff_p) + %if cpuflag(ssse3) + pmaddubsw m5, m3 ; constrain(diff_p) * taps + %else + psrlw m9, m5, 8 + psraw m6, m3, 8 + psllw m5, 8 + psllw m3, 8 + pmullw m9, m6 + pmulhw m5, m3 + paddw m5, m9 + %endif + paddw m0, m5 +%endmacro + +%macro LOAD_BODY 3 ; dst, src, block_width + %if %3 == 4 + PMOVZXBW m0, [%2+strideq*0] + PMOVZXBW m1, [%2+strideq*1] + PMOVZXBW m2, [%2+strideq*2] + PMOVZXBW m3, [%2+stride3q] + mova [%1+32*0], m0 + mova [%1+32*1], m1 + mova [%1+32*2], m2 + mova [%1+32*3], m3 + %else + movu m0, [%2+strideq*0] + movu m1, [%2+strideq*1] + movu m2, [%2+strideq*2] + movu m3, [%2+stride3q] + punpcklbw m4, m0, m7 + punpckhbw m0, m7 + mova [%1+32*0+ 0], m4 + mova [%1+32*0+16], m0 + punpcklbw m4, m1, m7 + punpckhbw m1, m7 + mova [%1+32*1+ 0], m4 + mova [%1+32*1+16], m1 + punpcklbw m4, m2, m7 + punpckhbw m2, m7 + mova [%1+32*2+ 0], m4 + mova [%1+32*2+16], m2 + punpcklbw m4, m3, m7 + punpckhbw m3, m7 + mova [%1+32*3+ 0], m4 + mova [%1+32*3+16], m3 + %endif +%endmacro + +%macro CDEF_FILTER_END 2 ; w, minmax + pxor m6, m6 + pcmpgtw m6, m0 + paddw m0, m6 + %if cpuflag(ssse3) + pmulhrsw m0, m15 + %else + paddw m0, m15 + psraw m0, 4 + %endif + paddw m4, m0 + %if %2 + pminsw m4, m7 + pmaxsw m4, m8 + %endif + packuswb m4, m4 + %if %1 == 4 + movd [dstq+strideq*0], m4 + psrlq m4, 32 + movd [dstq+strideq*1], m4 + add stkq, 32*2 + lea dstq, [dstq+strideq*2] + %else + movq [dstq], m4 + add stkq, 32 + add dstq, strideq + %endif +%endmacro + +%macro CDEF_FILTER 2 ; w, h + %if ARCH_X86_64 +cglobal cdef_filter_%1x%2_8bpc, 5, 9, 16, 3 * 16 + (%2+4)*32, \ + dst, stride, left, top, bot, pri, dst4, edge, \ + stride3 + %define px rsp+3*16+2*32 + %define base 0 + %else +cglobal cdef_filter_%1x%2_8bpc, 2, 7, 8, - 7 * 16 - (%2+4)*32, \ + dst, stride, left, edge, stride3 + %define topq r2 + %define botq r2 + %define dst4q r2 + LEA r5, tap_table + %define px esp+7*16+2*32 + %define base r5-tap_table + %endif + mov edged, r9m + %if cpuflag(sse4) + %define OUT_OF_BOUNDS_MEM [base+pw_0x8000] + %else + %define OUT_OF_BOUNDS_MEM [base+pw_0x7FFF] + %endif + mova m6, OUT_OF_BOUNDS_MEM + pxor m7, m7 + + ; prepare pixel buffers - body/right + %if %2 == 8 + lea dst4q, [dstq+strideq*4] + %endif + lea stride3q, [strideq*3] + test edgeb, 2 ; have_right + jz .no_right + LOAD_BODY px, dstq, %1 + %if %2 == 8 + LOAD_BODY px+4*32, dst4q, %1 + %endif + jmp .body_done +.no_right: + PMOVZXBW m0, [dstq+strideq*0], %1 == 4 + PMOVZXBW m1, [dstq+strideq*1], %1 == 4 + PMOVZXBW m2, [dstq+strideq*2], %1 == 4 + PMOVZXBW m3, [dstq+stride3q ], %1 == 4 + mova [px+32*0], m0 + mova [px+32*1], m1 + mova [px+32*2], m2 + mova [px+32*3], m3 + movd [px+32*0+%1*2], m6 + movd [px+32*1+%1*2], m6 + movd [px+32*2+%1*2], m6 + movd [px+32*3+%1*2], m6 + %if %2 == 8 + PMOVZXBW m0, [dst4q+strideq*0], %1 == 4 + PMOVZXBW m1, [dst4q+strideq*1], %1 == 4 + PMOVZXBW m2, [dst4q+strideq*2], %1 == 4 + PMOVZXBW m3, [dst4q+stride3q ], %1 == 4 + mova [px+32*4], m0 + mova [px+32*5], m1 + mova [px+32*6], m2 + mova [px+32*7], m3 + movd [px+32*4+%1*2], m6 + movd [px+32*5+%1*2], m6 + movd [px+32*6+%1*2], m6 + movd [px+32*7+%1*2], m6 + %endif +.body_done: + + ; top + movifnidn topq, r3mp + test edgeb, 4 ; have_top + jz .no_top + test edgeb, 1 ; have_left + jz .top_no_left + test edgeb, 2 ; have_right + jz .top_no_right + %if %1 == 4 + PMOVZXBW m0, [topq+strideq*0-2] + PMOVZXBW m1, [topq+strideq*1-2] + %else + movu m0, [topq+strideq*0-4] + movu m1, [topq+strideq*1-4] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + movu [px-32*2+8], m2 + movu [px-32*1+8], m3 + %endif + movu [px-32*2-%1], m0 + movu [px-32*1-%1], m1 + jmp .top_done +.top_no_right: + %if %1 == 4 + PMOVZXBW m0, [topq+strideq*0-%1] + PMOVZXBW m1, [topq+strideq*1-%1] + movu [px-32*2-8], m0 + movu [px-32*1-8], m1 + %else + movu m0, [topq+strideq*0-%1] + movu m1, [topq+strideq*1-%2] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + mova [px-32*2-16], m0 + mova [px-32*2+ 0], m2 + mova [px-32*1-16], m1 + mova [px-32*1+ 0], m3 + %endif + movd [px-32*2+%1*2], m6 + movd [px-32*1+%1*2], m6 + jmp .top_done +.top_no_left: + test edgeb, 2 ; have_right + jz .top_no_left_right + %if %1 == 4 + PMOVZXBW m0, [topq+strideq*0] + PMOVZXBW m1, [topq+strideq*1] + %else + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + movd [px-32*2+16], m2 + movd [px-32*1+16], m3 + %endif + movd [px-32*2- 4], m6 + movd [px-32*1- 4], m6 + mova [px-32*2+ 0], m0 + mova [px-32*1+ 0], m1 + jmp .top_done +.top_no_left_right: + PMOVZXBW m0, [topq+strideq*0], %1 == 4 + PMOVZXBW m1, [topq+strideq*1], %1 == 4 + movd [px-32*2-4], m6 + movd [px-32*1-4], m6 + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + movd [px-32*2+%1*2], m6 + movd [px-32*1+%1*2], m6 + jmp .top_done +.no_top: + movu [px-32*2- 4], m6 + movu [px-32*1- 4], m6 + %if %1 == 8 + movq [px-32*2+12], m6 + movq [px-32*1+12], m6 + %endif +.top_done: + + ; left + test edgeb, 1 ; have_left + jz .no_left + movifnidn leftq, leftmp + %if %2 == 4 + movq m0, [leftq] + %else + movu m0, [leftq] + %endif + %if %2 == 4 + punpcklbw m0, m7 + %else + punpckhbw m1, m0, m7 + punpcklbw m0, m7 + movhlps m3, m1 + movd [px+32*4-4], m1 + movd [px+32*6-4], m3 + psrlq m1, 32 + psrlq m3, 32 + movd [px+32*5-4], m1 + movd [px+32*7-4], m3 + %endif + movhlps m2, m0 + movd [px+32*0-4], m0 + movd [px+32*2-4], m2 + psrlq m0, 32 + psrlq m2, 32 + movd [px+32*1-4], m0 + movd [px+32*3-4], m2 + jmp .left_done +.no_left: + movd [px+32*0-4], m6 + movd [px+32*1-4], m6 + movd [px+32*2-4], m6 + movd [px+32*3-4], m6 + %if %2 == 8 + movd [px+32*4-4], m6 + movd [px+32*5-4], m6 + movd [px+32*6-4], m6 + movd [px+32*7-4], m6 + %endif +.left_done: + + ; bottom + movifnidn botq, r4mp + test edgeb, 8 ; have_bottom + jz .no_bottom + test edgeb, 1 ; have_left + jz .bottom_no_left + test edgeb, 2 ; have_right + jz .bottom_no_right + %if %1 == 4 + PMOVZXBW m0, [botq+strideq*0-(%1/2)] + PMOVZXBW m1, [botq+strideq*1-(%1/2)] + %else + movu m0, [botq+strideq*0-4] + movu m1, [botq+strideq*1-4] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + movu [px+32*(%2+0)+8], m2 + movu [px+32*(%2+1)+8], m3 + %endif + movu [px+32*(%2+0)-%1], m0 + movu [px+32*(%2+1)-%1], m1 + jmp .bottom_done +.bottom_no_right: + %if %1 == 4 + PMOVZXBW m0, [botq+strideq*0-4] + PMOVZXBW m1, [botq+strideq*1-4] + movu [px+32*(%2+0)-8], m0 + movu [px+32*(%2+1)-8], m1 + %else + movu m0, [botq+strideq*0-8] + movu m1, [botq+strideq*1-8] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + mova [px+32*(%2+0)-16], m0 + mova [px+32*(%2+0)+ 0], m2 + mova [px+32*(%2+1)-16], m1 + mova [px+32*(%2+1)+ 0], m3 + movd [px+32*(%2-1)+16], m6 ; overwritten by first mova + %endif + movd [px+32*(%2+0)+%1*2], m6 + movd [px+32*(%2+1)+%1*2], m6 + jmp .bottom_done +.bottom_no_left: + test edgeb, 2 ; have_right + jz .bottom_no_left_right + %if %1 == 4 + PMOVZXBW m0, [botq+strideq*0] + PMOVZXBW m1, [botq+strideq*1] + %else + movu m0, [botq+strideq*0] + movu m1, [botq+strideq*1] + punpckhbw m2, m0, m7 + punpcklbw m0, m7 + punpckhbw m3, m1, m7 + punpcklbw m1, m7 + mova [px+32*(%2+0)+16], m2 + mova [px+32*(%2+1)+16], m3 + %endif + mova [px+32*(%2+0)+ 0], m0 + mova [px+32*(%2+1)+ 0], m1 + movd [px+32*(%2+0)- 4], m6 + movd [px+32*(%2+1)- 4], m6 + jmp .bottom_done +.bottom_no_left_right: + PMOVZXBW m0, [botq+strideq*0], %1 == 4 + PMOVZXBW m1, [botq+strideq*1], %1 == 4 + mova [px+32*(%2+0)+ 0], m0 + mova [px+32*(%2+1)+ 0], m1 + movd [px+32*(%2+0)+%1*2], m6 + movd [px+32*(%2+1)+%1*2], m6 + movd [px+32*(%2+0)- 4], m6 + movd [px+32*(%2+1)- 4], m6 + jmp .bottom_done +.no_bottom: + movu [px+32*(%2+0)- 4], m6 + movu [px+32*(%2+1)- 4], m6 + %if %1 == 8 + movq [px+32*(%2+0)+12], m6 + movq [px+32*(%2+1)+12], m6 + %endif +.bottom_done: + + ; actual filter + %if ARCH_X86_64 + DEFINE_ARGS dst, stride, _, pridmp, damping, pri, sec + mova m13, [shufb_lohi] + %if cpuflag(ssse3) + mova m15, [pw_2048] + %else + mova m15, [pw_8] + %endif + mova m14, m6 + %else + DEFINE_ARGS dst, pridmp, sec, damping, pri, tap + %xdefine m8 m1 + %xdefine m9 m2 + %xdefine m10 m0 + %xdefine m13 [base+shufb_lohi] + %xdefine m14 OUT_OF_BOUNDS_MEM + %if cpuflag(ssse3) + %xdefine m15 [base+pw_2048] + %else + %xdefine m15 [base+pw_8] + %endif + %endif + movifnidn prid, r5m + movifnidn secd, r6m + mov dampingd, r8m + movif32 [esp+0x3C], r1d + test prid, prid + jz .sec_only + movd m1, r5m + bsr pridmpd, prid + test secd, secd + jz .pri_only + movd m10, r6m + tzcnt secd, secd + and prid, 1 + sub pridmpd, dampingd + sub secd, dampingd + xor dampingd, dampingd + add prid, prid + neg pridmpd + cmovs pridmpd, dampingd + neg secd + PSHUFB_0 m1, m7 + PSHUFB_0 m10, m7 + %if ARCH_X86_64 + DEFINE_ARGS dst, stride, _, pridmp, tap, pri, sec + lea tapq, [tap_table] + MOVDDUP m11, [tapq+pridmpq*8] ; pri_shift_mask + MOVDDUP m12, [tapq+secq*8] ; sec_shift_mask + mov [rsp+0x00], pridmpq ; pri_shift + mov [rsp+0x10], secq ; sec_shift + DEFINE_ARGS dst, stride, h, dir, tap, pri, stk, k, off + %else + MOVDDUP m2, [tapq+pridmpq*8] + MOVDDUP m3, [tapq+secq*8] + mov [esp+0x04], dampingd ; zero upper 32 bits of psrlw + mov [esp+0x34], dampingd ; source operand in ACCUMULATE_TAP + mov [esp+0x00], pridmpd + mov [esp+0x30], secd + DEFINE_ARGS dst, stride, dir, stk, pri, tap, h + %define offq dstq + %define kd strided + %define kq strideq + mova [esp+0x10], m2 + mova [esp+0x40], m3 + mova [esp+0x20], m1 + mova [esp+0x50], m10 + %endif + mov dird, r7m + lea stkq, [px] + lea priq, [tapq+8*8+priq*8] ; pri_taps + mov hd, %1*%2/8 + lea dirq, [tapq+dirq*2] +.v_loop: + movif32 [esp+0x38], dstd + mov kd, 1 + %if %1 == 4 + movq m4, [stkq+32*0] + movhps m4, [stkq+32*1] + %else + mova m4, [stkq+32*0] ; px + %endif + pxor m0, m0 ; sum + mova m7, m4 ; max + mova m8, m4 ; min +.k_loop: + MOVDDUP m2, [priq+kq*8] + %if ARCH_X86_64 + ACCUMULATE_TAP 0*2, [rsp+0x00], m11, m1, m2, %1, 1 + MOVDDUP m2, [tapq+12*8+kq*8] + ACCUMULATE_TAP 2*2, [rsp+0x10], m12, m10, m2, %1, 1 + ACCUMULATE_TAP 6*2, [rsp+0x10], m12, m10, m2, %1, 1 + %else + ACCUMULATE_TAP 0*2, [esp+0x00], [esp+0x10], [esp+0x20], m2, %1, 1 + MOVDDUP m2, [tapq+12*8+kq*8] + ACCUMULATE_TAP 2*2, [esp+0x30], [esp+0x40], [esp+0x50], m2, %1, 1 + MOVDDUP m2, [tapq+12*8+kq*8] + ACCUMULATE_TAP 6*2, [esp+0x30], [esp+0x40], [esp+0x50], m2, %1, 1 + %endif + dec kd + jge .k_loop + movif32 dstq, [esp+0x38] + movif32 strideq, [esp+0x3C] + CDEF_FILTER_END %1, 1 + dec hd + jg .v_loop + RET + +.pri_only: +%if ARCH_X86_64 + DEFINE_ARGS dst, stride, zero, pridmp, damping, pri, tap + lea tapq, [tap_table] + %else + DEFINE_ARGS dst, pridmp, zero, damping, pri, tap + %endif + and prid, 1 + xor zerod, zerod + sub dampingd, pridmpd + cmovs dampingd, zerod + add prid, prid + PSHUFB_0 m1, m7 + MOVDDUP m7, [tapq+dampingq*8] + mov [rsp+0x00], dampingq + %if ARCH_X86_64 + DEFINE_ARGS dst, stride, h, dir, stk, pri, tap, k, off + %else + mov [rsp+0x04], zerod + DEFINE_ARGS dst, stride, dir, stk, pri, tap, h + %endif + mov dird, r7m + lea stkq, [px] + lea priq, [tapq+8*8+priq*8] + mov hd, %1*%2/8 + lea dirq, [tapq+dirq*2] +.pri_v_loop: + movif32 [esp+0x38], dstd + mov kd, 1 + %if %1 == 4 + movq m4, [stkq+32*0] + movhps m4, [stkq+32*1] + %else + mova m4, [stkq+32*0] + %endif + pxor m0, m0 +.pri_k_loop: + MOVDDUP m2, [priq+kq*8] + ACCUMULATE_TAP 0*2, [rsp], m7, m1, m2, %1, 0 + dec kd + jge .pri_k_loop + movif32 dstq, [esp+0x38] + movif32 strideq, [esp+0x3C] + CDEF_FILTER_END %1, 0 + dec hd + jg .pri_v_loop + RET + +.sec_only: +%if ARCH_X86_64 + DEFINE_ARGS dst, stride, zero, dir, damping, tap, sec +%else + DEFINE_ARGS dst, stride, sec, damping, dir, tap, zero +%endif + movd m1, r6m + tzcnt secd, secd + mov dird, r7m + xor zerod, zerod + sub dampingd, secd + cmovs dampingd, zerod + PSHUFB_0 m1, m7 + %if ARCH_X86_64 + lea tapq, [tap_table] + %else + mov [rsp+0x04], zerod + %endif + mov [rsp+0x00], dampingq + MOVDDUP m7, [tapq+dampingq*8] + lea dirq, [tapq+dirq*2] + %if ARCH_X86_64 + DEFINE_ARGS dst, stride, h, dir, stk, tap, off, k + %else + DEFINE_ARGS dst, stride, off, stk, dir, tap, h + %endif + lea stkq, [px] + mov hd, %1*%2/8 +.sec_v_loop: + mov kd, 1 + %if %1 == 4 + movq m4, [stkq+32*0] + movhps m4, [stkq+32*1] + %else + mova m4, [stkq+32*0] + %endif + pxor m0, m0 +.sec_k_loop: + MOVDDUP m2, [tapq+12*8+kq*8] + ACCUMULATE_TAP 2*2, [rsp], m7, m1, m2, %1, 0 + %if ARCH_X86_32 + MOVDDUP m2, [tapq+12*8+kq*8] + %endif + ACCUMULATE_TAP 6*2, [rsp], m7, m1, m2, %1, 0 + dec kd + jge .sec_k_loop + movif32 strideq, [esp+0x3C] + CDEF_FILTER_END %1, 0 + dec hd + jg .sec_v_loop + RET +%endmacro + +%macro MULLD 2 + %if cpuflag(sse4) + pmulld %1, %2 + %else + %if ARCH_X86_32 + %define m15 m1 + %endif + pmulhuw m15, %1, %2 + pmullw %1, %2 + pslld m15, 16 + paddd %1, m15 + %endif +%endmacro + +%macro CDEF_DIR 0 + %if ARCH_X86_64 +cglobal cdef_dir_8bpc, 3, 7, 16, src, stride, var + lea r6, [strideq*3] + movq m1, [srcq+strideq*0] + movhps m1, [srcq+strideq*1] + movq m3, [srcq+strideq*2] + movhps m3, [srcq+r6 ] + lea srcq, [srcq+strideq*4] + movq m5, [srcq+strideq*0] + movhps m5, [srcq+strideq*1] + movq m7, [srcq+strideq*2] + movhps m7, [srcq+r6 ] + + pxor m8, m8 + psadbw m9, m1, m8 + psadbw m2, m3, m8 + psadbw m4, m5, m8 + psadbw m6, m7, m8 + packssdw m9, m2 + packssdw m4, m6 + packssdw m9, m4 + + punpcklbw m0, m1, m8 + punpckhbw m1, m8 + punpcklbw m2, m3, m8 + punpckhbw m3, m8 + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + punpcklbw m6, m7, m8 + punpckhbw m7, m8 +cglobal_label .main + mova m8, [pw_128] + psubw m0, m8 + psubw m1, m8 + psubw m2, m8 + psubw m3, m8 + psubw m4, m8 + psubw m5, m8 + psubw m6, m8 + psubw m7, m8 + psllw m8, 3 + psubw m9, m8 ; partial_sum_hv[0] + + paddw m8, m0, m1 + paddw m10, m2, m3 + paddw m8, m4 + paddw m10, m5 + paddw m8, m6 + paddw m10, m7 + paddw m8, m10 ; partial_sum_hv[1] + + pmaddwd m8, m8 + pmaddwd m9, m9 + phaddd m9, m8 + SWAP m8, m9 + MULLD m8, [div_table%+SUFFIX+48] + + pslldq m9, m1, 2 + psrldq m10, m1, 14 + pslldq m11, m2, 4 + psrldq m12, m2, 12 + pslldq m13, m3, 6 + psrldq m14, m3, 10 + paddw m9, m0 + paddw m10, m12 + paddw m11, m13 + paddw m10, m14 ; partial_sum_diag[0] top/right half + paddw m9, m11 ; partial_sum_diag[0] top/left half + pslldq m11, m4, 8 + psrldq m12, m4, 8 + pslldq m13, m5, 10 + psrldq m14, m5, 6 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m6, 12 + psrldq m12, m6, 4 + pslldq m13, m7, 14 + psrldq m14, m7, 2 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 ; partial_sum_diag[0][0-7] + paddw m10, m14 ; partial_sum_diag[0][8-14,zero] + pshufb m10, [shufw_6543210x] + punpckhwd m11, m9, m10 + punpcklwd m9, m10 + pmaddwd m11, m11 + pmaddwd m9, m9 + MULLD m11, [div_table%+SUFFIX+16] + MULLD m9, [div_table%+SUFFIX+0] + paddd m9, m11 ; cost[0a-d] + + pslldq m10, m0, 14 + psrldq m11, m0, 2 + pslldq m12, m1, 12 + psrldq m13, m1, 4 + pslldq m14, m2, 10 + psrldq m15, m2, 6 + paddw m10, m12 + paddw m11, m13 + paddw m10, m14 + paddw m11, m15 + pslldq m12, m3, 8 + psrldq m13, m3, 8 + pslldq m14, m4, 6 + psrldq m15, m4, 10 + paddw m10, m12 + paddw m11, m13 + paddw m10, m14 + paddw m11, m15 + pslldq m12, m5, 4 + psrldq m13, m5, 12 + pslldq m14, m6, 2 + psrldq m15, m6, 14 + paddw m10, m12 + paddw m11, m13 + paddw m10, m14 + paddw m11, m15 ; partial_sum_diag[1][8-14,zero] + paddw m10, m7 ; partial_sum_diag[1][0-7] + pshufb m11, [shufw_6543210x] + punpckhwd m12, m10, m11 + punpcklwd m10, m11 + pmaddwd m12, m12 + pmaddwd m10, m10 + MULLD m12, [div_table%+SUFFIX+16] + MULLD m10, [div_table%+SUFFIX+0] + paddd m10, m12 ; cost[4a-d] + phaddd m9, m10 ; cost[0a/b,4a/b] + + paddw m10, m0, m1 + paddw m11, m2, m3 + paddw m12, m4, m5 + paddw m13, m6, m7 + phaddw m0, m4 + phaddw m1, m5 + phaddw m2, m6 + phaddw m3, m7 + + ; m0-3 are horizontal sums (x >> 1), m10-13 are vertical sums (y >> 1) + pslldq m4, m11, 2 + psrldq m5, m11, 14 + pslldq m6, m12, 4 + psrldq m7, m12, 12 + pslldq m14, m13, 6 + psrldq m15, m13, 10 + paddw m4, m10 + paddw m5, m7 + paddw m4, m6 + paddw m5, m15 ; partial_sum_alt[3] right + paddw m4, m14 ; partial_sum_alt[3] left + pshuflw m6, m5, q3012 + punpckhwd m5, m4 + punpcklwd m4, m6 + pmaddwd m5, m5 + pmaddwd m4, m4 + MULLD m5, [div_table%+SUFFIX+48] + MULLD m4, [div_table%+SUFFIX+32] + paddd m4, m5 ; cost[7a-d] + + pslldq m5, m10, 6 + psrldq m6, m10, 10 + pslldq m7, m11, 4 + psrldq m10, m11, 12 + pslldq m11, m12, 2 + psrldq m12, 14 + paddw m5, m7 + paddw m6, m10 + paddw m5, m11 + paddw m6, m12 + paddw m5, m13 + pshuflw m7, m6, q3012 + punpckhwd m6, m5 + punpcklwd m5, m7 + pmaddwd m6, m6 + pmaddwd m5, m5 + MULLD m6, [div_table%+SUFFIX+48] + MULLD m5, [div_table%+SUFFIX+32] + paddd m5, m6 ; cost[5a-d] + + pslldq m6, m1, 2 + psrldq m7, m1, 14 + pslldq m10, m2, 4 + psrldq m11, m2, 12 + pslldq m12, m3, 6 + psrldq m13, m3, 10 + paddw m6, m0 + paddw m7, m11 + paddw m6, m10 + paddw m7, m13 ; partial_sum_alt[3] right + paddw m6, m12 ; partial_sum_alt[3] left + pshuflw m10, m7, q3012 + punpckhwd m7, m6 + punpcklwd m6, m10 + pmaddwd m7, m7 + pmaddwd m6, m6 + MULLD m7, [div_table%+SUFFIX+48] + MULLD m6, [div_table%+SUFFIX+32] + paddd m6, m7 ; cost[1a-d] + + pshufd m0, m0, q1032 + pshufd m1, m1, q1032 + pshufd m2, m2, q1032 + pshufd m3, m3, q1032 + + pslldq m10, m0, 6 + psrldq m11, m0, 10 + pslldq m12, m1, 4 + psrldq m13, m1, 12 + pslldq m14, m2, 2 + psrldq m2, 14 + paddw m10, m12 + paddw m11, m13 + paddw m10, m14 + paddw m11, m2 + paddw m10, m3 + pshuflw m12, m11, q3012 + punpckhwd m11, m10 + punpcklwd m10, m12 + pmaddwd m11, m11 + pmaddwd m10, m10 + MULLD m11, [div_table%+SUFFIX+48] + MULLD m10, [div_table%+SUFFIX+32] + paddd m10, m11 ; cost[3a-d] + + phaddd m9, m8 ; cost[0,4,2,6] + phaddd m6, m10 + phaddd m5, m4 + phaddd m6, m5 ; cost[1,3,5,7] + pshufd m4, m9, q3120 + + ; now find the best cost + %if cpuflag(sse4) + pmaxsd m9, m6 + pshufd m0, m9, q1032 + pmaxsd m0, m9 + pshufd m1, m0, q2301 + pmaxsd m0, m1 ; best cost + %else + pcmpgtd m0, m9, m6 + pand m9, m0 + pandn m0, m6 + por m9, m0 + pshufd m1, m9, q1032 + pcmpgtd m0, m9, m1 + pand m9, m0 + pandn m0, m1 + por m9, m0 + pshufd m1, m9, q2301 + pcmpgtd m0, m9, m1 + pand m9, m0 + pandn m0, m1 + por m0, m9 + %endif + + ; get direction and variance + punpckhdq m1, m4, m6 + punpckldq m4, m6 + psubd m2, m0, m1 + psubd m3, m0, m4 +%if WIN64 + WIN64_RESTORE_XMM + %define tmp rsp+stack_offset+8 +%else + %define tmp rsp-40 +%endif + mova [tmp+0x00], m2 ; emulate ymm in stack + mova [tmp+0x10], m3 + pcmpeqd m1, m0 ; compute best cost mask + pcmpeqd m4, m0 + packssdw m4, m1 + pmovmskb eax, m4 ; get byte-idx from mask + tzcnt eax, eax + mov r1d, [tmp+rax*2] ; get idx^4 complement from emulated ymm + shr eax, 1 ; get direction by converting byte-idx to word-idx + shr r1d, 10 + mov [varq], r1d + %else +cglobal cdef_dir_8bpc, 2, 4, 8, 96, src, stride, var, stride3 +%define base r2-shufw_6543210x + LEA r2, shufw_6543210x + pxor m0, m0 + lea stride3q, [strideq*3] + movq m5, [srcq+strideq*0] + movhps m5, [srcq+strideq*1] + movq m7, [srcq+strideq*2] + movhps m7, [srcq+stride3q] + mova m1, [base+pw_128] + psadbw m2, m5, m0 + psadbw m3, m7, m0 + packssdw m2, m3 + punpcklbw m4, m5, m0 + punpckhbw m5, m0 + punpcklbw m6, m7, m0 + punpckhbw m7, m0 + psubw m4, m1 + psubw m5, m1 + psubw m6, m1 + psubw m7, m1 + + mova [esp+0x00], m4 + mova [esp+0x10], m5 + mova [esp+0x20], m6 + mova [esp+0x50], m7 + + lea srcq, [srcq+strideq*4] + movq m5, [srcq+strideq*0] + movhps m5, [srcq+strideq*1] + movq m7, [srcq+strideq*2] + movhps m7, [srcq+stride3q] + psadbw m3, m5, m0 + psadbw m0, m7 + packssdw m3, m0 + pxor m0, m0 + punpcklbw m4, m5, m0 + punpckhbw m5, m0 + punpcklbw m6, m7, m0 + punpckhbw m7, m0 +cglobal_label .main + psubw m4, m1 + psubw m5, m1 + psubw m6, m1 + psubw m7, m1 + packssdw m2, m3 + psllw m1, 3 + psubw m2, m1 ; partial_sum_hv[0] + pmaddwd m2, m2 + + mova m3, [esp+0x50] + mova m0, [esp+0x00] + paddw m0, [esp+0x10] + paddw m1, m3, [esp+0x20] + paddw m0, m4 + paddw m1, m5 + paddw m0, m6 + paddw m1, m7 + paddw m0, m1 ; partial_sum_hv[1] + pmaddwd m0, m0 + + phaddd m2, m0 + MULLD m2, [base+div_table%+SUFFIX+48] + mova [esp+0x30], m2 + + mova m1, [esp+0x10] + pslldq m0, m1, 2 + psrldq m1, 14 + paddw m0, [esp+0x00] + pslldq m2, m3, 6 + psrldq m3, 10 + paddw m0, m2 + paddw m1, m3 + mova m3, [esp+0x20] + pslldq m2, m3, 4 + psrldq m3, 12 + paddw m0, m2 ; partial_sum_diag[0] top/left half + paddw m1, m3 ; partial_sum_diag[0] top/right half + pslldq m2, m4, 8 + psrldq m3, m4, 8 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m5, 10 + psrldq m3, m5, 6 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m6, 12 + psrldq m3, m6, 4 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m7, 14 + psrldq m3, m7, 2 + paddw m0, m2 ; partial_sum_diag[0][0-7] + paddw m1, m3 ; partial_sum_diag[0][8-14,zero] + mova m3, [esp+0x50] + pshufb m1, [base+shufw_6543210x] + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmaddwd m2, m2 + pmaddwd m0, m0 + MULLD m2, [base+div_table%+SUFFIX+16] + MULLD m0, [base+div_table%+SUFFIX+ 0] + paddd m0, m2 ; cost[0a-d] + mova [esp+0x40], m0 + + mova m1, [esp+0x00] + pslldq m0, m1, 14 + psrldq m1, 2 + paddw m0, m7 + pslldq m2, m3, 8 + psrldq m3, 8 + paddw m0, m2 + paddw m1, m3 + mova m3, [esp+0x20] + pslldq m2, m3, 10 + psrldq m3, 6 + paddw m0, m2 + paddw m1, m3 + mova m3, [esp+0x10] + pslldq m2, m3, 12 + psrldq m3, 4 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m4, 6 + psrldq m3, m4, 10 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m5, 4 + psrldq m3, m5, 12 + paddw m0, m2 + paddw m1, m3 + pslldq m2, m6, 2 + psrldq m3, m6, 14 + paddw m0, m2 ; partial_sum_diag[1][0-7] + paddw m1, m3 ; partial_sum_diag[1][8-14,zero] + mova m3, [esp+0x50] + pshufb m1, [base+shufw_6543210x] + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmaddwd m2, m2 + pmaddwd m0, m0 + MULLD m2, [base+div_table%+SUFFIX+16] + MULLD m0, [base+div_table%+SUFFIX+ 0] + paddd m0, m2 ; cost[4a-d] + phaddd m1, [esp+0x40], m0 ; cost[0a/b,4a/b] + phaddd m1, [esp+0x30] ; cost[0,4,2,6] + mova [esp+0x30], m1 + + phaddw m0, [esp+0x00], m4 + phaddw m1, [esp+0x10], m5 + paddw m4, m5 + mova m2, [esp+0x20] + paddw m5, m2, m3 + phaddw m2, m6 + paddw m6, m7 + phaddw m3, m7 + mova m7, [esp+0x00] + paddw m7, [esp+0x10] + mova [esp+0x00], m0 + mova [esp+0x10], m1 + mova [esp+0x20], m2 + + pslldq m1, m4, 4 + pslldq m2, m6, 6 + pslldq m0, m5, 2 + paddw m1, m2 + paddw m0, m7 + psrldq m2, m5, 14 + paddw m0, m1 ; partial_sum_alt[3] left + psrldq m1, m4, 12 + paddw m1, m2 + psrldq m2, m6, 10 + paddw m1, m2 ; partial_sum_alt[3] right + pshuflw m1, m1, q3012 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmaddwd m2, m2 + pmaddwd m0, m0 + MULLD m2, [base+div_table%+SUFFIX+48] + MULLD m0, [base+div_table%+SUFFIX+32] + paddd m0, m2 ; cost[7a-d] + mova [esp+0x40], m0 + + pslldq m0, m7, 6 + psrldq m7, 10 + pslldq m1, m5, 4 + psrldq m5, 12 + pslldq m2, m4, 2 + psrldq m4, 14 + paddw m0, m6 + paddw m7, m5 + paddw m0, m1 + paddw m7, m4 + paddw m0, m2 + pshuflw m2, m7, q3012 + punpckhwd m7, m0 + punpcklwd m0, m2 + pmaddwd m7, m7 + pmaddwd m0, m0 + MULLD m7, [base+div_table%+SUFFIX+48] + MULLD m0, [base+div_table%+SUFFIX+32] + paddd m0, m7 ; cost[5a-d] + mova [esp+0x50], m0 + + mova m7, [esp+0x10] + mova m2, [esp+0x20] + pslldq m0, m7, 2 + psrldq m7, 14 + pslldq m4, m2, 4 + psrldq m2, 12 + pslldq m5, m3, 6 + psrldq m6, m3, 10 + paddw m0, [esp+0x00] + paddw m7, m2 + paddw m4, m5 + paddw m7, m6 ; partial_sum_alt[3] right + paddw m0, m4 ; partial_sum_alt[3] left + pshuflw m2, m7, q3012 + punpckhwd m7, m0 + punpcklwd m0, m2 + pmaddwd m7, m7 + pmaddwd m0, m0 + MULLD m7, [base+div_table%+SUFFIX+48] + MULLD m0, [base+div_table%+SUFFIX+32] + paddd m0, m7 ; cost[1a-d] + SWAP m0, m4 + + pshufd m0, [esp+0x00], q1032 + pshufd m1, [esp+0x10], q1032 + pshufd m2, [esp+0x20], q1032 + pshufd m3, m3, q1032 + mova [esp+0x00], m4 + + pslldq m4, m0, 6 + psrldq m0, 10 + pslldq m5, m1, 4 + psrldq m1, 12 + pslldq m6, m2, 2 + psrldq m2, 14 + paddw m4, m3 + paddw m0, m1 + paddw m5, m6 + paddw m0, m2 + paddw m4, m5 + pshuflw m2, m0, q3012 + punpckhwd m0, m4 + punpcklwd m4, m2 + pmaddwd m0, m0 + pmaddwd m4, m4 + MULLD m0, [base+div_table%+SUFFIX+48] + MULLD m4, [base+div_table%+SUFFIX+32] + paddd m4, m0 ; cost[3a-d] + + mova m1, [esp+0x00] + mova m2, [esp+0x50] + mova m0, [esp+0x30] ; cost[0,4,2,6] + phaddd m1, m4 + phaddd m2, [esp+0x40] ; cost[1,3,5,7] + phaddd m1, m2 + pshufd m2, m0, q3120 + + ; now find the best cost + %if cpuflag(sse4) + pmaxsd m0, m1 + pshufd m3, m0, q1032 + pmaxsd m3, m0 + pshufd m0, m3, q2301 + pmaxsd m0, m3 + %else + pcmpgtd m3, m0, m1 + pand m0, m3 + pandn m3, m1 + por m0, m3 + pshufd m4, m0, q1032 + pcmpgtd m3, m0, m4 + pand m0, m3 + pandn m3, m4 + por m0, m3 + pshufd m4, m0, q2301 + pcmpgtd m3, m0, m4 + pand m0, m3 + pandn m3, m4 + por m0, m3 + %endif + + ; get direction and variance + mov vard, varm + punpckhdq m3, m2, m1 + punpckldq m2, m1 + psubd m1, m0, m3 + psubd m4, m0, m2 + mova [esp+0x00], m1 ; emulate ymm in stack + mova [esp+0x10], m4 + pcmpeqd m3, m0 ; compute best cost mask + pcmpeqd m2, m0 + packssdw m2, m3 + pmovmskb eax, m2 ; get byte-idx from mask + tzcnt eax, eax + mov r1d, [esp+eax*2] ; get idx^4 complement from emulated ymm + shr eax, 1 ; get direction by converting byte-idx to word-idx + shr r1d, 10 + mov [vard], r1d + %endif + + RET +%endmacro + +INIT_XMM sse4 +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 +CDEF_DIR + +INIT_XMM ssse3 +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 +CDEF_DIR + +INIT_XMM sse2 +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 diff --git a/third_party/dav1d/src/x86/cpu.c b/third_party/dav1d/src/x86/cpu.c new file mode 100644 index 0000000000..764d8be8ef --- /dev/null +++ b/third_party/dav1d/src/x86/cpu.c @@ -0,0 +1,100 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "common/attributes.h" + +#include "src/x86/cpu.h" + +typedef struct { + uint32_t eax, ebx, edx, ecx; +} CpuidRegisters; + +void dav1d_cpu_cpuid(CpuidRegisters *regs, unsigned leaf, unsigned subleaf); +uint64_t dav1d_cpu_xgetbv(unsigned xcr); + +#define X(reg, mask) (((reg) & (mask)) == (mask)) + +COLD unsigned dav1d_get_cpu_flags_x86(void) { + union { + CpuidRegisters r; + struct { + uint32_t max_leaf; + char vendor[12]; + }; + } cpu; + dav1d_cpu_cpuid(&cpu.r, 0, 0); + unsigned flags = 0; + + if (cpu.max_leaf >= 1) { + CpuidRegisters r; + dav1d_cpu_cpuid(&r, 1, 0); + const unsigned model = ((r.eax >> 4) & 0x0f) + ((r.eax >> 12) & 0xf0); + const unsigned family = ((r.eax >> 8) & 0x0f) + ((r.eax >> 20) & 0xff); + + if (X(r.edx, 0x06008000)) /* CMOV/SSE/SSE2 */ { + flags |= DAV1D_X86_CPU_FLAG_SSE2; + if (X(r.ecx, 0x00000201)) /* SSE3/SSSE3 */ { + flags |= DAV1D_X86_CPU_FLAG_SSSE3; + if (X(r.ecx, 0x00080000)) /* SSE4.1 */ + flags |= DAV1D_X86_CPU_FLAG_SSE41; + } + } +#if ARCH_X86_64 + /* We only support >128-bit SIMD on x86-64. */ + if (X(r.ecx, 0x18000000)) /* OSXSAVE/AVX */ { + const uint64_t xcr0 = dav1d_cpu_xgetbv(0); + if (X(xcr0, 0x00000006)) /* XMM/YMM */ { + if (cpu.max_leaf >= 7) { + dav1d_cpu_cpuid(&r, 7, 0); + if (X(r.ebx, 0x00000128)) /* BMI1/BMI2/AVX2 */ { + flags |= DAV1D_X86_CPU_FLAG_AVX2; + if (X(xcr0, 0x000000e0)) /* ZMM/OPMASK */ { + if (X(r.ebx, 0xd0230000) && X(r.ecx, 0x00005f42)) + flags |= DAV1D_X86_CPU_FLAG_AVX512ICL; + } + } + } + } + } +#endif + if (!memcmp(cpu.vendor, "AuthenticAMD", sizeof(cpu.vendor))) { + if ((flags & DAV1D_X86_CPU_FLAG_AVX2) && (family < 0x19 || + (family == 0x19 && (model < 0x10 || (model >= 0x20 && model < 0x60))))) + { + /* Excavator, Zen, Zen+, Zen 2, Zen 3, Zen 3+ */ + flags |= DAV1D_X86_CPU_FLAG_SLOW_GATHER; + } + } + } + + return flags; +} diff --git a/third_party/dav1d/src/x86/cpu.h b/third_party/dav1d/src/x86/cpu.h new file mode 100644 index 0000000000..8529c77c9b --- /dev/null +++ b/third_party/dav1d/src/x86/cpu.h @@ -0,0 +1,44 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_X86_CPU_H +#define DAV1D_SRC_X86_CPU_H + +enum CpuFlags { + DAV1D_X86_CPU_FLAG_SSE2 = 1 << 0, + DAV1D_X86_CPU_FLAG_SSSE3 = 1 << 1, + DAV1D_X86_CPU_FLAG_SSE41 = 1 << 2, + DAV1D_X86_CPU_FLAG_AVX2 = 1 << 3, + DAV1D_X86_CPU_FLAG_AVX512ICL = 1 << 4, /* F/CD/BW/DQ/VL/VNNI/IFMA/VBMI/VBMI2/ + * VPOPCNTDQ/BITALG/GFNI/VAES/VPCLMULQDQ */ + DAV1D_X86_CPU_FLAG_SLOW_GATHER = 1 << 5, /* Flag CPUs where gather instructions are slow enough + * to cause performance regressions. */ +}; + +unsigned dav1d_get_cpu_flags_x86(void); + +#endif /* DAV1D_SRC_X86_CPU_H */ diff --git a/third_party/dav1d/src/x86/cpuid.asm b/third_party/dav1d/src/x86/cpuid.asm new file mode 100644 index 0000000000..e1d9228660 --- /dev/null +++ b/third_party/dav1d/src/x86/cpuid.asm @@ -0,0 +1,55 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION .text + +cglobal cpu_cpuid, 0, 5, 0, regs, leaf, subleaf + mov r4, regsmp + mov eax, leafm + mov ecx, subleafm +%if ARCH_X86_64 + mov r5, rbx +%endif + cpuid + mov [r4+4*0], eax + mov [r4+4*1], ebx + mov [r4+4*2], edx + mov [r4+4*3], ecx +%if ARCH_X86_64 + mov rbx, r5 +%endif + RET + +cglobal cpu_xgetbv, 0, 0, 0, xcr + movifnidn ecx, xcrm + xgetbv +%if ARCH_X86_64 + shl rdx, 32 + or rax, rdx +%endif + RET diff --git a/third_party/dav1d/src/x86/filmgrain.h b/third_party/dav1d/src/x86/filmgrain.h new file mode 100644 index 0000000000..eeaa328d1e --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain.h @@ -0,0 +1,81 @@ +/* + * Copyright © 2018-2022, VideoLAN and dav1d authors + * Copyright © 2018-2022, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/filmgrain.h" + +#define decl_fg_fns(ext) \ +decl_generate_grain_y_fn(BF(dav1d_generate_grain_y, ext)); \ +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_420, ext)); \ +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_422, ext)); \ +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_444, ext)); \ +decl_fgy_32x32xn_fn(BF(dav1d_fgy_32x32xn, ext)); \ +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i420, ext)); \ +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i422, ext)); \ +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i444, ext)) + +decl_fg_fns(ssse3); +decl_fg_fns(avx2); +decl_fg_fns(avx512icl); + +static ALWAYS_INLINE void film_grain_dsp_init_x86(Dav1dFilmGrainDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + + c->generate_grain_y = BF(dav1d_generate_grain_y, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, ssse3); + c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, ssse3); + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->generate_grain_y = BF(dav1d_generate_grain_y, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, avx2); + + if (!(flags & DAV1D_X86_CPU_FLAG_SLOW_GATHER)) { + c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx2); + } + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + + c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, avx512icl); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx512icl); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx512icl); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx512icl); +#endif +} diff --git a/third_party/dav1d/src/x86/filmgrain16_avx2.asm b/third_party/dav1d/src/x86/filmgrain16_avx2.asm new file mode 100644 index 0000000000..a1d4c41f27 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain16_avx2.asm @@ -0,0 +1,2248 @@ +; Copyright © 2021-2022, VideoLAN and dav1d authors +; Copyright © 2021-2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 16 +pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 +gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 +gen_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 +next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +pw_27_17_17_27: dw 27, 17, 17, 27 +pw_23_22: dw 23, 22, 0, 32 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +gen_ar0_shift: times 4 db 128 + times 4 db 64 + times 4 db 32 + times 4 db 16 +pd_16: dd 16 +pd_m65536: dd -65536 +pb_1: times 4 db 1 +grain_max: times 2 dw 511 + times 2 dw 2047 +grain_min: times 2 dw -512 + times 2 dw -2048 +fg_max: times 2 dw 1023 + times 2 dw 4095 + times 2 dw 960 + times 2 dw 3840 + times 2 dw 940 + times 2 dw 3760 +fg_min: times 2 dw 0 + times 2 dw 64 + times 2 dw 256 +uv_offset_mul: dd 256 + dd 1024 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16, 8 +round_vals: dw 32, 64, 128, 256, 512, 1024 +pb_8_9_0_1: db 8, 9, 0, 1 + +%macro JMP_TABLE 1-* + %xdefine %1_table %%table + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .ar%2 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y_16bpc_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420_16bpc_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_16bpc_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_16bpc_avx2, 0, 1, 2, 3 + +SECTION .text + +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +INIT_YMM avx2 +cglobal generate_grain_y_16bpc, 3, 9, 14, buf, fg_data, bdmax +%define base r4-generate_grain_y_16bpc_avx2_table + lea r4, [generate_grain_y_16bpc_avx2_table] + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r6d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + mov r3, -73*82*2 + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + lea r7d, [bdmaxq+1] + movq xm4, [base+mul_bits] + shr r7d, 11 ; 0 for 10bpc, 2 for 12bpc + movq xm5, [base+hmul_bits] + sub r6, r7 + mova xm6, [base+pb_mask] + sub bufq, r3 + vpbroadcastw xm7, [base+round+r6*2-2] + lea r6, [gaussian_sequence] + movsxd r5, [r4+r5*4] +.loop: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm3, xm2 ; 4 next output seeds + pshuflw xm0, xm3, q3333 + psrlw xm3, 5 + pand xm2, xm0, xm1 + movq r7, xm3 + psrlw xm3, xm2, 10 + por xm2, xm3 + pmullw xm2, xm4 + pmulhuw xm0, xm5 + movzx r8d, r7w + pshufb xm3, xm6, xm2 + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm0, xm2 + movd xm2, [r6+r8*2] + rorx r8, r7, 32 + por xm3, xm0 + shr r7d, 16 + pinsrw xm2, [r6+r7*2], 1 + pshuflw xm0, xm3, q3333 + movzx r7d, r8w + psrlw xm3, 5 + pinsrw xm2, [r6+r7*2], 2 + shr r8d, 16 + movq r7, xm3 + pinsrw xm2, [r6+r8*2], 3 + movzx r8d, r7w + pinsrw xm2, [r6+r8*2], 4 + rorx r8, r7, 32 + shr r7d, 16 + pinsrw xm2, [r6+r7*2], 5 + movzx r7d, r8w + pinsrw xm2, [r6+r7*2], 6 + shr r8d, 16 + pinsrw xm2, [r6+r8*2], 7 + paddw xm2, xm2 ; otherwise bpc=12 w/ grain_scale_shift=0 + pmulhrsw xm2, xm7 ; shifts by 0, which pmulhrsw does not support + mova [bufq+r3], xm2 + add r3, 8*2 + jl .loop + + ; auto-regression code + add r5, r4 + jmp r5 + +.ar1: + DEFINE_ARGS buf, fg_data, max, shift, val3, min, cf3, x, val0 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd xm4, [fg_dataq+FGData.ar_coeffs_y] + DEFINE_ARGS buf, h, max, shift, val3, min, cf3, x, val0 + pinsrb xm4, [base+pb_1], 3 + pmovsxbw xm4, xm4 + pshufd xm5, xm4, q1111 + pshufd xm4, xm4, q0000 + vpbroadcastw xm3, [base+round_vals+shiftq*2-12] ; rnd + sub bufq, 2*(82*73-(82*3+79)) + mov hd, 70 + sar maxd, 1 + mov mind, maxd + xor mind, -1 +.y_loop_ar1: + mov xq, -76 + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu xm0, [bufq+xq*2-82*2-2] ; top/left + psrldq xm2, xm0, 2 ; top + psrldq xm1, xm0, 4 ; top/right + punpcklwd xm0, xm2 + punpcklwd xm1, xm3 + pmaddwd xm0, xm4 + pmaddwd xm1, xm5 + paddd xm0, xm1 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + sarx val3d, val3d, shiftd + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xb, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 +.x_loop_ar1_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar1 +.ar0: + RET + +.ar2: + DEFINE_ARGS buf, fg_data, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movq xm0, [fg_dataq+FGData.ar_coeffs_y+5] ; cf5-11 + vinserti128 m0, [fg_dataq+FGData.ar_coeffs_y+0], 1 ; cf0-4 + vpbroadcastw xm10, [base+round_vals-12+shiftq*2] + pxor m1, m1 + punpcklwd xm10, xm1 + pcmpgtb m1, m0 + punpcklbw m0, m1 ; cf5-11,0-4 + vpermq m1, m0, q3333 ; cf4 + vbroadcasti128 m11, [base+gen_shufA] + pshufd m6, m0, q0000 ; cf[5,6], cf[0-1] + vbroadcasti128 m12, [base+gen_shufB] + pshufd m7, m0, q1111 ; cf[7,8], cf[2-3] + punpckhwd xm1, xm0 + pshufhw xm9, xm0, q2121 + pshufd xm8, xm1, q0000 ; cf[4,9] + sar bdmaxd, 1 + punpckhqdq xm9, xm9 ; cf[10,11] + movd xm4, bdmaxd ; max_grain + pcmpeqd xm5, xm5 + sub bufq, 2*(82*73-(82*3+79)) + pxor xm5, xm4 ; min_grain + DEFINE_ARGS buf, fg_data, h, x + mov hd, 70 +.y_loop_ar2: + mov xq, -76 +.x_loop_ar2: + vbroadcasti128 m2, [bufq+xq*2-82*4-4] ; y=-2,x=[-2,+5] + vinserti128 m1, m2, [bufq+xq*2-82*2-4], 0 ; y=-1,x=[-2,+5] + pshufb m0, m1, m11 ; y=-1/-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + pmaddwd m0, m6 + punpckhwd xm2, xm1 ; y=-2/-1 interleaved, x=[+2,+5] + pshufb m1, m12 ; y=-1/-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + pmaddwd m1, m7 + pmaddwd xm2, xm8 + paddd m0, m1 + vextracti128 xm1, m0, 1 + paddd xm0, xm10 + paddd xm2, xm0 + movu xm0, [bufq+xq*2-4] ; y=0,x=[-2,+5] + paddd xm2, xm1 + pmovsxwd xm1, [bufq+xq*2] ; in dwords, y=0,x=[0,3] +.x_loop_ar2_inner: + pmaddwd xm3, xm9, xm0 + psrldq xm0, 2 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + ; skip packssdw because we only care about one value + paddd xm3, xm1 + pminsd xm3, xm4 + psrldq xm1, 4 + pmaxsd xm3, xm5 + pextrw [bufq+xq*2], xm3, 0 + punpcklwd xm3, xm3 + pblendw xm0, xm3, 0010b + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 +.x_loop_ar2_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar2 + RET + +.ar3: + DEFINE_ARGS buf, fg_data, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + sar bdmaxd, 1 + movq xm7, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-6 + movd xm0, [fg_dataq+FGData.ar_coeffs_y+14] ; cf14-16 + pinsrb xm7, [fg_dataq+FGData.ar_coeffs_y+13], 7 ; cf0-6,13 + pinsrb xm0, [base+pb_1], 3 ; cf14-16,pb_1 + movd xm1, [fg_dataq+FGData.ar_coeffs_y+21] ; cf21-23 + vinserti128 m7, [fg_dataq+FGData.ar_coeffs_y+ 7], 1 ; cf7-13 + vinserti128 m0, [fg_dataq+FGData.ar_coeffs_y+17], 1 ; cf17-20 + vpbroadcastw xm11, [base+round_vals+shiftq*2-12] + movd xm12, bdmaxd ; max_grain + punpcklbw m7, m7 ; sign-extension + punpcklbw m0, m0 ; sign-extension + punpcklbw xm1, xm1 + REPX {psraw x, 8}, m7, m0, xm1 + pshufd m4, m7, q0000 ; cf[0,1] | cf[7,8] + pshufd m5, m7, q1111 ; cf[2,3] | cf[9,10] + pshufd m6, m7, q2222 ; cf[4,5] | cf[11,12] + pshufd xm7, xm7, q3333 ; cf[6,13] + pshufd m8, m0, q0000 ; cf[14,15] | cf[17,18] + pshufd m9, m0, q1111 ; cf[16],pw_1 | cf[19,20] + paddw xm0, xm11, xm11 + pcmpeqd xm13, xm13 + pblendw xm10, xm1, xm0, 00001000b + pxor xm13, xm12 ; min_grain + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 2*(82*73-(82*3+79)) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 +.x_loop_ar3: + movu xm0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + vinserti128 m0, [bufq+xq*2-82*4-6+ 0], 1 ; y=-3/-2,x=[-3,+4] + movq xm1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+8] + vinserti128 m1, [bufq+xq*2-82*4-6+16], 1 ; y=-3/-2,x=[+5,+12] + palignr m3, m1, m0, 2 ; y=-3/-2,x=[-2,+5] + palignr m1, m0, 12 ; y=-3/-2,x=[+3,+6] + punpckhwd m2, m0, m3 ; y=-3/-2,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m0, m3 ; y=-3/-2,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m3, m0, m2, q1032 ; y=-3/-2,x=[-1/+0,+0/+1,+1/+2,+2/+3] + pmaddwd m0, m4 + pmaddwd m2, m6 + pmaddwd m3, m5 + paddd m0, m2 + movu xm2, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + vinserti128 m2, [bufq+xq*2-82*2-6+ 6], 1 ; y=-1,x=[+1,+8] + paddd m0, m3 + psrldq m3, m2, 2 + punpcklwd m3, m2, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + pmaddwd m3, m8 ; x=[+0/+1,+1/+2,+2/+3,+3/+4] + paddd m0, m3 + psrldq m3, m2, 4 + psrldq m2, 6 + vpblendd m2, m11, 0x0f ; rounding constant + punpcklwd m3, m2 ; y=-1,x=[-1/rnd,+0/rnd,+1/rnd,+2/rnd] + pmaddwd m3, m9 ; x=[+2/+3,+3/+4,+4/+5,+5,+6] + vextracti128 xm2, m1, 1 + punpcklwd xm1, xm2 + pmaddwd xm1, xm7 ; y=-3/-2 interleaved,x=[+3,+4,+5,+6] + paddd m0, m3 + vextracti128 xm2, m0, 1 + paddd xm0, xm1 + movu xm1, [bufq+xq*2-6] ; y=0,x=[-3,+4] + paddd xm0, xm2 +.x_loop_ar3_inner: + pmaddwd xm2, xm1, xm10 + pshuflw xm3, xm2, q1032 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + ; skip packssdw because we only care about one value + pminsd xm2, xm12 + pmaxsd xm2, xm13 + pextrw [bufq+xq*2], xm2, 0 + pslldq xm2, 4 + psrldq xm1, 2 + pblendw xm1, xm2, 0100b + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar3 + RET + +%macro GEN_GRAIN_UV_FN 3 ; ss_name, ss_x, ss_y +INIT_XMM avx2 +cglobal generate_grain_uv_%1_16bpc, 4, 11, 8, buf, bufy, fg_data, uv, bdmax +%define base r8-generate_grain_uv_%1_16bpc_avx2_table + lea r8, [generate_grain_uv_%1_16bpc_avx2_table] + movifnidn bdmaxd, bdmaxm + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r5d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + lea r6d, [bdmaxq+1] + movq xm4, [base+mul_bits] + shr r6d, 11 ; 0 for 10bpc, 2 for 12bpc + movq xm5, [base+hmul_bits] + sub r5, r6 + mova xm6, [base+pb_mask] + vpbroadcastd xm2, [base+pw_seed_xor+uvq*4] + vpbroadcastw xm7, [base+round+r5*2-2] + pxor xm0, xm2 + lea r6, [gaussian_sequence] +%if %2 + mov r7d, 73-35*%3 + add bufq, 44*2 +.loop_y: + mov r5, -44*2 +%else + mov r5, -82*73*2 + sub bufq, r5 +%endif +.loop_x: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm2, xm3 ; 4 next output seeds + pshuflw xm0, xm2, q3333 + psrlw xm2, 5 + movq r10, xm2 + movzx r9d, r10w + movd xm2, [r6+r9*2] + rorx r9, r10, 32 + shr r10d, 16 + pinsrw xm2, [r6+r10*2], 1 + movzx r10d, r9w + pinsrw xm2, [r6+r10*2], 2 + shr r9d, 16 + pinsrw xm2, [r6+r9*2], 3 + paddw xm2, xm2 ; otherwise bpc=12 w/ grain_scale_shift=0 + pmulhrsw xm2, xm7 ; shifts by 0, which pmulhrsw does not support + movq [bufq+r5], xm2 + add r5, 8 + jl .loop_x +%if %2 + add bufq, 82*2 + dec r7d + jg .loop_y +%endif + + ; auto-regression code + movsxd r6, [fg_dataq+FGData.ar_coeff_lag] + movsxd r6, [r8+r6*4] + add r6, r8 + jmp r6 + +INIT_YMM avx2 +.ar0: + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + vpbroadcastb m0, [fg_dataq+FGData.ar_coeffs_uv+uvq] + sar bdmaxd, 1 + vpbroadcastd m4, [base+gen_ar0_shift-24+shiftq*4] + movd xm6, bdmaxd + pcmpeqw m7, m7 + pmaddubsw m4, m0 ; ar_coeff << (14 - shift) + vpbroadcastw m6, xm6 ; max_gain + pxor m7, m6 ; min_grain + DEFINE_ARGS buf, bufy, h, x +%if %2 + vpbroadcastw m5, [base+hmul_bits+2+%3*2] + sub bufq, 2*(82*(73-35*%3)+82-(82*3+41)) +%else + sub bufq, 2*(82*70-3) +%endif + add bufyq, 2*(3+82*3) + mov hd, 70-35*%3 +.y_loop_ar0: +%if %2 + ; first 32 pixels + movu xm0, [bufyq+16*0] + vinserti128 m0, [bufyq+16*2], 1 + movu xm1, [bufyq+16*1] + vinserti128 m1, [bufyq+16*3], 1 +%if %3 + movu xm2, [bufyq+82*2+16*0] + vinserti128 m2, [bufyq+82*2+16*2], 1 + movu xm3, [bufyq+82*2+16*1] + vinserti128 m3, [bufyq+82*2+16*3], 1 + paddw m0, m2 + paddw m1, m3 +%endif + phaddw m0, m1 + movu xm1, [bufyq+16*4] + vinserti128 m1, [bufyq+16*6], 1 + movu xm2, [bufyq+16*5] + vinserti128 m2, [bufyq+16*7], 1 +%if %3 + movu xm3, [bufyq+82*2+16*4] + vinserti128 m3, [bufyq+82*2+16*6], 1 + paddw m1, m3 + movu xm3, [bufyq+82*2+16*5] + vinserti128 m3, [bufyq+82*2+16*7], 1 + paddw m2, m3 +%endif + phaddw m1, m2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 +%else + xor xd, xd +.x_loop_ar0: + movu m0, [bufyq+xq*2] + movu m1, [bufyq+xq*2+32] +%endif + paddw m0, m0 + paddw m1, m1 + pmulhrsw m0, m4 + pmulhrsw m1, m4 +%if %2 + paddw m0, [bufq+ 0] + paddw m1, [bufq+32] +%else + paddw m0, [bufq+xq*2+ 0] + paddw m1, [bufq+xq*2+32] +%endif + pminsw m0, m6 + pminsw m1, m6 + pmaxsw m0, m7 + pmaxsw m1, m7 +%if %2 + movu [bufq+ 0], m0 + movu [bufq+32], m1 + + ; last 6 pixels + movu xm0, [bufyq+32*4] + movu xm1, [bufyq+32*4+16] +%if %3 + paddw xm0, [bufyq+32*4+82*2] + paddw xm1, [bufyq+32*4+82*2+16] +%endif + phaddw xm0, xm1 + movu xm1, [bufq+32*2] + pmulhrsw xm0, xm5 + paddw xm0, xm0 + pmulhrsw xm0, xm4 + paddw xm0, xm1 + pminsw xm0, xm6 + pmaxsw xm0, xm7 + vpblendd xm0, xm1, 0x08 + movu [bufq+32*2], xm0 +%else + movu [bufq+xq*2+ 0], m0 + movu [bufq+xq*2+32], m1 + add xd, 32 + cmp xd, 64 + jl .x_loop_ar0 + + ; last 12 pixels + movu m0, [bufyq+64*2] + movu m1, [bufq+64*2] + paddw m0, m0 + pmulhrsw m0, m4 + paddw m0, m1 + pminsw m0, m6 + pmaxsw m0, m7 + vpblendd m0, m1, 0xc0 + movu [bufq+64*2], m0 +%endif + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar0 + RET + +INIT_XMM avx2 +.ar1: + DEFINE_ARGS buf, bufy, fg_data, uv, max, cf3, min, val3, x, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] + pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 + DEFINE_ARGS buf, bufy, h, val0, max, cf3, min, val3, x, shift + pmovsxbw xm4, xm4 + pshufd xm5, xm4, q1111 + pshufd xm4, xm4, q0000 + pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd + vpbroadcastw xm6, [base+hmul_bits+2+%3*2] + vpbroadcastd xm3, xm3 +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 + sar maxd, 1 + mov mind, maxd + xor mind, -1 +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu xm0, [bufq+xq*2-82*2-2] ; top/left +%if %2 + movu xm2, [bufyq+xq*4] +%else + movq xm2, [bufyq+xq*2] +%endif +%if %2 +%if %3 + phaddw xm2, [bufyq+xq*4+82*2] + punpckhqdq xm1, xm2, xm2 + paddw xm2, xm1 +%else + phaddw xm2, xm2 +%endif + pmulhrsw xm2, xm6 +%endif + psrldq xm1, xm0, 4 ; top/right + punpcklwd xm1, xm2 + psrldq xm2, xm0, 2 ; top + punpcklwd xm0, xm2 + pmaddwd xm1, xm5 + pmaddwd xm0, xm4 + paddd xm1, xm3 + paddd xm0, xm1 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + sarx val3d, val3d, shiftd + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xb, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 +.x_loop_ar1_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar1 + RET + +INIT_YMM avx2 +.ar2: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign xmm_regs_used 13 + %2 + %assign stack_size_padded 136 + SUB rsp, stack_size_padded + movaps [rsp+16*2], xmm8 + movaps [rsp+16*3], xmm9 + movaps [rsp+16*4], xmm10 + movaps [rsp+16*5], xmm11 + movaps [rsp+16*6], xmm12 +%if %2 + movaps [rsp+16*7], xmm13 +%endif +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + vbroadcasti128 m10, [base+gen_shufA] + sar bdmaxd, 1 + vbroadcasti128 m11, [base+gen_shufB] + movd xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 5] + pinsrb xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+12], 4 + pinsrb xm7, [base+pb_1], 5 + pinsrw xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+10], 3 + movhps xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] + pinsrb xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 9], 13 + pmovsxbw m7, xm7 + movd xm8, bdmaxd ; max_grain + pshufd m4, m7, q0000 + vpbroadcastw xm12, [base+round_vals-12+shiftq*2] + pshufd m5, m7, q1111 + pcmpeqd xm9, xm9 + pshufd m6, m7, q2222 + pxor xm9, xm8 ; min_grain + pshufd xm7, xm7, q3333 + DEFINE_ARGS buf, bufy, fg_data, h, x +%if %2 + vpbroadcastw xm13, [base+hmul_bits+2+%3*2] + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) +.x_loop_ar2: + vbroadcasti128 m3, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] + vinserti128 m2, m3, [bufq+xq*2-82*4-4], 1 ; y=-2,x=[-2,+5] + pshufb m0, m2, m10 ; y=-1/-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + pmaddwd m0, m4 + pshufb m1, m2, m11 ; y=-1/-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + pmaddwd m1, m5 + punpckhwd m2, m3 ; y=-2/-1 interleaved, x=[+2,+5] +%if %2 + movu xm3, [bufyq+xq*4] +%if %3 + paddw xm3, [bufyq+xq*4+82*2] +%endif + phaddw xm3, xm3 + pmulhrsw xm3, xm13 +%else + movq xm3, [bufyq+xq*2] +%endif + punpcklwd xm3, xm12 ; luma, round interleaved + vpblendd m2, m3, 0x0f + pmaddwd m2, m6 + paddd m1, m0 + movu xm0, [bufq+xq*2-4] ; y=0,x=[-2,+5] + paddd m2, m1 + vextracti128 xm1, m2, 1 + paddd xm2, xm1 + pshufd xm1, xm0, q3321 + pmovsxwd xm1, xm1 ; y=0,x=[0,3] in dword +.x_loop_ar2_inner: + pmaddwd xm3, xm7, xm0 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + ; we do not need to packssdw since we only care about one value + paddd xm3, xm1 + psrldq xm1, 4 + pminsd xm3, xm8 + pmaxsd xm3, xm9 + pextrw [bufq+xq*2], xm3, 0 + psrldq xm0, 2 + pslldq xm3, 2 + pblendw xm0, xm3, 00000010b + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 +.x_loop_ar2_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar2 + RET + +.ar3: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign stack_offset 32 + %assign xmm_regs_used 14 + %2 + %assign stack_size_padded 152 + SUB rsp, stack_size_padded + movaps [rsp+16*2], xmm8 + movaps [rsp+16*3], xmm9 + movaps [rsp+16*4], xmm10 + movaps [rsp+16*5], xmm11 + movaps [rsp+16*6], xmm12 + movaps [rsp+16*7], xmm13 +%if %2 + movaps [rsp+16*8], xmm14 +%endif +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + vpbroadcastw xm11, [base+round_vals-12+shiftq*2] + sar bdmaxd, 1 + movq xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] + pinsrb xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+24], 7 ; luma + movhps xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 7] + pmovsxbw m7, xm7 +%if %2 + vpbroadcastw xm14, [base+hmul_bits+2+%3*2] +%endif + pshufd m4, m7, q0000 + pshufd m5, m7, q1111 + pshufd m6, m7, q2222 + pshufd m7, m7, q3333 + movd xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+14] + pinsrb xm0, [base+pb_1], 3 + pinsrd xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+21], 1 + pinsrd xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+17], 2 + pmovsxbw m0, xm0 + movd xm12, bdmaxd ; max_grain + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pcmpeqd xm13, xm13 + punpckhqdq xm10, xm0, xm0 + pxor xm13, xm12 ; min_grain + pinsrw xm10, [base+round_vals-10+shiftq*2], 3 + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) +.x_loop_ar3: + movu xm2, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + vinserti128 m2, [bufq+xq*2-82*4-6+ 0], 1 ; y=-3/-2,x=[-3,+4] + movq xm1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+8] + vinserti128 m1, [bufq+xq*2-82*4-6+16], 1 ; y=-3/-2,x=[+5,+12] + palignr m3, m1, m2, 2 ; y=-3/-2,x=[-2,+5] + palignr m1, m2, 12 ; y=-3/-2,x=[+3,+6] + punpcklwd m0, m2, m3 ; y=-3/-2,x=[-3/-2,-2/-1,-1/+0,+0/+1] + punpckhwd m2, m3 ; y=-3/-2,x=[+1/+2,+2/+3,+3/+4,+4/+5] + shufps m3, m0, m2, q1032 ; y=-3/-2,x=[-1/+0,+0/+1,+1/+2,+2/+3] + pmaddwd m0, m4 + pmaddwd m2, m6 + pmaddwd m3, m5 + paddd m0, m2 + paddd m0, m3 + movu xm2, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + vinserti128 m2, [bufq+xq*2-82*2-6+ 6], 1 ; y=-1,x=[+1,+8] +%if %2 + movu xm3, [bufyq+xq*4] +%if %3 + paddw xm3, [bufyq+xq*4+82*2] +%endif + phaddw xm3, xm3 + pmulhrsw xm3, xm14 +%else + movq xm3, [bufyq+xq*2] +%endif + punpcklwd m1, m3 + pmaddwd m1, m7 + paddd m0, m1 + psrldq m1, m2, 4 + psrldq m3, m2, 6 + vpblendd m3, m11, 0x0f ; rounding constant + punpcklwd m1, m3 ; y=-1,x=[-1/rnd,+0/rnd,+1/rnd,+2/rnd] + pmaddwd m1, m9 ; x=[+2/+3,+3/+4,+4/+5,+5,+6] + psrldq m3, m2, 2 + punpcklwd m2, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + pmaddwd m2, m8 ; x=[+0/+1,+1/+2,+2/+3,+3/+4] + paddd m0, m1 + movu xm1, [bufq+xq*2-6] ; y=0,x=[-3,+4] + paddd m0, m2 + vextracti128 xm2, m0, 1 + paddd xm0, xm2 +.x_loop_ar3_inner: + pmaddwd xm2, xm1, xm10 + pshuflw xm3, xm2, q1032 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + psrldq xm1, 2 + ; no need to packssdw since we only care about one value + pminsd xm2, xm12 + pmaxsd xm2, xm13 + pextrw [bufq+xq*2], xm2, 0 + pslldq xm2, 4 + pblendw xm1, xm2, 00000100b + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar3 + RET +%endmacro + +cglobal fgy_32x32xn_16bpc, 6, 14, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, unused, sby, see +%define base r11-grain_min + lea r11, [grain_min] + mov r6d, r9m ; bdmax + mov r9d, [fg_dataq+FGData.clip_to_restricted_range] + mov r7d, [fg_dataq+FGData.scaling_shift] + mov sbyd, sbym + vpbroadcastd m8, r9m + shr r6d, 11 ; is_12bpc + vpbroadcastd m9, [base+grain_min+r6*4] + shlx r10d, r9d, r6d + vpbroadcastd m10, [base+grain_max+r6*4] + lea r9d, [r6+r9*4] + vpbroadcastw m11, [base+mul_bits+r7*2-12] + vpbroadcastd m12, [base+fg_min+r10*4] + vpbroadcastd m13, [base+fg_max+r9*4] + test sbyd, sbyd + setnz r7b + vpbroadcastd m14, [base+pd_16] + test r7b, [fg_dataq+FGData.overlap_flag] + jnz .vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak + + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y: + ; scaling[src] + mova m0, [srcq+ 0] + mova m1, [srcq+32] + pand m4, m8, m0 + psrld m3, m0, 16 + mova m6, m9 + vpgatherdd m2, [scalingq+m4-0], m9 + pand m3, m8 + mova m9, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pand m5, m8, m1 + mova m6, m9 + vpgatherdd m3, [scalingq+m5-0], m9 + pblendw m4, m2, 0x55 + psrld m2, m1, 16 + mova m9, m6 + pand m2, m8 + vpgatherdd m5, [scalingq+m2-2], m6 + pblendw m5, m3, 0x55 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m4, [grain_lutq+offxyq*2] + pmulhrsw m5, [grain_lutq+offxyq*2+32] + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+32], m1 + + add srcq, strideq + add grain_lutq, 82*2 + dec hd + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je .loop_x + movq xm7, [pw_27_17_17_27] + cmp dword r8m, 0 ; sby + jne .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy + + lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + ; scaling[src] + mova m0, [srcq+ 0] + mova m1, [srcq+32] + pand m4, m8, m0 + psrld m3, m0, 16 + mova m6, m9 + vpgatherdd m2, [scalingq+m4-0], m9 + pand m3, m8 + mova m9, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pand m5, m8, m1 + mova m6, m9 + vpgatherdd m3, [scalingq+m5-0], m9 + pblendw m4, m2, 0x55 + psrld m2, m1, 16 + mova m9, m6 + pand m2, m8 + vpgatherdd m5, [scalingq+m2-2], m6 + pblendw m5, m3, 0x55 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq*2] + movd xm6, [grain_lutq+left_offxyq*2] + punpcklwd xm6, xm3 + pmaddwd xm6, xm7 + paddd xm6, xm14 + psrad xm6, 5 + packssdw xm6, xm6 + pmaxsw xm6, xm9 + pminsw xm6, xm10 + vpblendd m3, m6, 0x01 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m4, m3 + pmulhrsw m5, [grain_lutq+offxyq*2+32] + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+32], m1 + + add srcq, strideq + add grain_lutq, 82*2 + dec hd + jg .loop_y_h_overlap + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + cmp dword r8m, 0 ; sby + jne .loop_x_hv_overlap + jmp .loop_x_h_overlap + +.vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ + sby, see, src_bak + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + +.loop_x_v_overlap: + vpbroadcastd m15, [pw_27_17_17_27] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, unused, top_offxy + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, unused, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_v_overlap: + ; scaling[src] + mova m0, [srcq+ 0] + mova m1, [srcq+32] + pand m4, m8, m0 + psrld m3, m0, 16 + mova m6, m9 + vpgatherdd m2, [scalingq+m4-0], m9 + pand m3, m8 + mova m9, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pand m5, m8, m1 + mova m6, m9 + vpgatherdd m3, [scalingq+m5-0], m9 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m9, m6 + pand m4, m8 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq*2] + movu m5, [grain_lutq+top_offxyq*2] + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + pmaddwd m4, m15 + pmaddwd m5, m15 + movu m7, [grain_lutq+offxyq*2+32] + movu m6, [grain_lutq+top_offxyq*2+32] + paddd m4, m14 + paddd m5, m14 + psrad m4, 5 + psrad m5, 5 + packssdw m4, m5 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 + pmaddwd m5, m15 + pmaddwd m6, m15 + paddd m5, m14 + paddd m6, m14 + psrad m5, 5 + psrad m6, 5 + packssdw m5, m6 + pmaxsw m4, m9 + pmaxsw m5, m9 + pminsw m4, m10 + pminsw m5, m10 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m11 + pmaddubsw m3, m11 + paddw m2, m2 + paddw m3, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+32], m1 + + add srcq, strideq + add grain_lutq, 82*2 + dec hb + jz .end_y_v_overlap + vpbroadcastd m15, [pw_27_17_17_27+4] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_v_overlap + jmp .loop_y +.end_y_v_overlap: + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap + +.loop_x_hv_overlap: + vpbroadcastd m15, [pw_27_17_17_27] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+32] + lea left_offxyd, [offyq+32] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_hv_overlap: + ; scaling[src] + mova m0, [srcq+ 0] + mova m1, [srcq+32] + pand m4, m8, m0 + psrld m3, m0, 16 + mova m6, m9 + vpgatherdd m2, [scalingq+m4-0], m9 + pand m3, m8 + mova m9, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pand m5, m8, m1 + mova m6, m9 + vpgatherdd m3, [scalingq+m5-0], m9 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m9, m6 + pand m4, m8 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m7, [grain_lutq+offxyq*2] + movd xm6, [grain_lutq+left_offxyq*2] + movu m5, [grain_lutq+top_offxyq*2] + movd xm4, [grain_lutq+topleft_offxyq*2] + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklwd xm6, xm7 + punpcklwd xm4, xm5 + punpcklqdq xm6, xm4 + movddup xm4, [pw_27_17_17_27] + pmaddwd xm6, xm4 + paddd xm6, xm14 + psrad xm6, 5 + packssdw xm6, xm6 + pmaxsw xm6, xm9 + pminsw xm6, xm10 + pshuflw xm4, xm6, q1032 + vpblendd m6, m7, 0xfe + vpblendd m4, m5, 0xfe + ; followed by v interpolation (top | cur -> cur) + punpckhwd m5, m7 + pmaddwd m5, m15 + punpcklwd m4, m6 + pmaddwd m4, m15 + movu m7, [grain_lutq+offxyq*2+32] + movu m6, [grain_lutq+top_offxyq*2+32] + paddd m5, m14 + paddd m4, m14 + psrad m5, 5 + psrad m4, 5 + packssdw m4, m5 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 + pmaddwd m5, m15 + pmaddwd m6, m15 + paddd m5, m14 + paddd m6, m14 + psrad m5, 5 + psrad m6, 5 + packssdw m5, m6 + pmaxsw m4, m9 + pmaxsw m5, m9 + pminsw m4, m10 + pminsw m5, m10 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m11 + pmaddubsw m3, m11 + paddw m2, m2 + paddw m3, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+32], m1 + + add srcq, strideq + add grain_lutq, 82*2 + dec hb + jz .end_y_hv_overlap + vpbroadcastd m15, [pw_27_17_17_27+4] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_hv_overlap + movq xm7, [pw_27_17_17_27] + jmp .loop_y_h_overlap +.end_y_hv_overlap: + add wq, 32 + lea srcq, [src_bakq+wq*2] + jl .loop_x_hv_overlap +.end: + RET + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r12-grain_min + lea r12, [grain_min] + mov r9d, r13m ; bdmax + mov r7d, [fg_dataq+FGData.scaling_shift] + mov r11d, is_idm + mov sbyd, sbym + vpbroadcastw m11, [base+mul_bits+r7*2-12] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + shr r9d, 11 ; is_12bpc + vpbroadcastd m8, [base+grain_min+r9*4] + shlx r10d, r6d, r9d + vpbroadcastd m9, [base+grain_max+r9*4] + vpbroadcastw m10, r13m + shlx r6d, r6d, r11d + vpbroadcastd m12, [base+fg_min+r10*4] + lea r6d, [r9+r6*2] + vpbroadcastd m13, [base+fg_max+r6*4] + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused, sby, see, overlap + +%if %1 + mov r6d, r11m + vpbroadcastd m0, [base+pb_8_9_0_1] + vpbroadcastd m1, [base+uv_offset_mul+r9*4] + vbroadcasti128 m14, [fg_dataq+FGData.uv_mult+r6*4] + vpbroadcastd m15, [fg_dataq+FGData.uv_offset+r6*4] + pshufb m14, m0 ; { uv_luma_mult, uv_mult } + pmaddwd m15, m1 +%else +%if %2 + vpbroadcastq m15, [base+pw_23_22] +%else + vpbroadcastq m15, [base+pw_27_17_17_27] +%endif + vpbroadcastd m14, [base+pd_16] +%endif + test r7b, [fg_dataq+FGData.overlap_flag] + jnz %%vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused2, unused3, see, unused4, unused5, unused6, luma, lstride + + mov lumaq, r9mp + mov lstrideq, r10mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r9mp, r10 + mov r11mp, r11 + mov r12mp, r12 + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, unused2, unused3, luma, lstride + + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, unused2, unused3, luma, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: + ; luma_src +%if %2 + mova xm2, [lumaq+lstrideq*0+ 0] + vinserti128 m2, [lumaq+lstrideq*0+32], 1 + mova xm4, [lumaq+lstrideq*0+16] + vinserti128 m4, [lumaq+lstrideq*0+48], 1 + mova xm3, [lumaq+lstrideq*(1<<%3)+ 0] + vinserti128 m3, [lumaq+lstrideq*(1<<%3)+32], 1 + mova xm5, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m5, [lumaq+lstrideq*(1<<%3)+48], 1 + phaddw m2, m4 + phaddw m3, m5 + pxor m4, m4 + pavgw m2, m4 + pavgw m3, m4 +%elif %1 + mova m2, [lumaq+ 0] + mova m3, [lumaq+32] +%endif +%if %1 + mova m0, [srcq] +%if %2 + mova m1, [srcq+strideq] +%else + mova m1, [srcq+32] +%endif + punpckhwd m4, m2, m0 + punpcklwd m2, m0 + punpckhwd m5, m3, m1 + punpcklwd m3, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m4, m2, m5, m3 + REPX {paddd x, m15}, m4, m2, m5, m3 + REPX {psrad x, 6 }, m4, m2, m5, m3 + packusdw m2, m4 + packusdw m3, m5 + pminuw m2, m10 + pminuw m3, m10 ; clip_pixel() +%elif %2 + pand m2, m10 + pand m3, m10 +%else + pand m2, m10, [lumaq+ 0] + pand m3, m10, [lumaq+32] +%endif + + ; scaling[luma_src] + vpbroadcastd m7, [pd_m65536] + pandn m4, m7, m2 + mova m6, m7 + vpgatherdd m5, [scalingq+m4-0], m7 + psrld m2, 16 + mova m7, m6 + vpgatherdd m4, [scalingq+m2-2], m6 + pblendw m4, m5, 0x55 + pandn m5, m7, m3 + mova m6, m7 + vpgatherdd m2, [scalingq+m5-0], m7 + psrld m3, 16 + vpgatherdd m5, [scalingq+m3-2], m6 + pblendw m5, m2, 0x55 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m4, [grain_lutq+offxyq*2] +%if %2 + pmulhrsw m5, [grain_lutq+offxyq*2+82*2] +%else + pmulhrsw m5, [grain_lutq+offxyq*2+32] +%endif + + ; dst = clip_pixel(src, noise) +%if %1 + paddw m0, m4 + paddw m1, m5 +%else + paddw m0, m4, [srcq] +%if %2 + paddw m1, m5, [srcq+strideq] +%else + paddw m1, m5, [srcq+32] +%endif +%endif + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq], m0 +%if %2 + mova [dstq+strideq], m1 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(2<<%2) +%if %2 + sub hb, 2 +%else + dec hb +%endif + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je %%loop_x + cmp dword r8m, 0 ; sby + jne %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, luma, lstride + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, luma, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: + ; luma_src +%if %2 + mova xm2, [lumaq+lstrideq*0+ 0] + vinserti128 m2, [lumaq+lstrideq*0+32], 1 + mova xm4, [lumaq+lstrideq*0+16] + vinserti128 m4, [lumaq+lstrideq*0+48], 1 + mova xm3, [lumaq+lstrideq*(1<<%3)+ 0] + vinserti128 m3, [lumaq+lstrideq*(1<<%3)+32], 1 + mova xm5, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m5, [lumaq+lstrideq*(1<<%3)+48], 1 + phaddw m2, m4 + phaddw m3, m5 + pxor m4, m4 + pavgw m2, m4 + pavgw m3, m4 +%elif %1 + mova m2, [lumaq] + mova m3, [lumaq+32] +%endif +%if %1 + mova m0, [srcq] +%if %2 + mova m1, [srcq+strideq] +%else + mova m1, [srcq+32] +%endif + punpckhwd m4, m2, m0 + punpcklwd m2, m0 + punpckhwd m5, m3, m1 + punpcklwd m3, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m4, m2, m5, m3 + REPX {paddd x, m15}, m4, m2, m5, m3 + REPX {psrad x, 6 }, m4, m2, m5, m3 + packusdw m2, m4 + packusdw m3, m5 + pminuw m2, m10 ; clip_pixel() + pminuw m3, m10 +%elif %2 + pand m2, m10 + pand m3, m10 +%else + pand m2, m10, [lumaq+ 0] + pand m3, m10, [lumaq+32] +%endif + + ; scaling[luma_src] + vpbroadcastd m7, [pd_m65536] + pandn m4, m7, m2 + mova m6, m7 + vpgatherdd m5, [scalingq+m4-0], m7 + psrld m2, 16 + mova m7, m6 + vpgatherdd m4, [scalingq+m2-2], m6 + pblendw m4, m5, 0x55 + pandn m5, m7, m3 + mova m6, m7 + vpgatherdd m2, [scalingq+m5-0], m7 + psrld m3, 16 + vpgatherdd m5, [scalingq+m3-2], m6 + pblendw m5, m2, 0x55 + + ; grain = grain_lut[offy+y][offx+x] + movu m2, [grain_lutq+offxyq*2] +%if %2 + movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif + movd xm6, [grain_lutq+left_offxyq*2] +%if %2 + pinsrw xm6, [grain_lutq+left_offxyq*2+82*2], 2 ; {left0, left1} + punpckldq xm7, xm2, xm3 ; {cur0, cur1} + punpcklwd xm6, xm7 ; {left0, cur0, left1, cur1} +%else + punpcklwd xm6, xm2 +%endif +%if %1 +%if %2 + vpbroadcastq xm7, [pw_23_22] +%else + movq xm7, [pw_27_17_17_27] +%endif + pmaddwd xm6, xm7 + vpbroadcastd xm7, [pd_16] + paddd xm6, xm7 +%else + pmaddwd xm6, xm15 + paddd xm6, xm14 +%endif + psrad xm6, 5 + packssdw xm6, xm6 + pmaxsw xm6, xm8 + pminsw xm6, xm9 + vpblendd m2, m6, 0x01 +%if %2 + pshuflw xm6, xm6, q1032 + vpblendd m3, m6, 0x01 +%endif + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + + ; dst = clip_pixel(src, noise) +%if %1 + paddw m0, m2 + paddw m1, m3 +%else + paddw m0, m2, [srcq] +%if %2 + paddw m1, m3, [srcq+strideq] +%else + paddw m1, m3, [srcq+32] +%endif +%endif + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq], m0 +%if %2 + mova [dstq+strideq], m1 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + add srcq, strideq + add dstq, strideq + add lumaq, r10mp +%endif + add grain_lutq, 82*(2<<%2) +%if %2 + sub hb, 2 +%else + dec hb +%endif + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp dword r8m, 0 ; sby + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap + +%%vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ + sby, see, unused1, unused2, unused3, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, top_offxy, unused2, luma, lstride + + mov lumaq, r9mp + mov lstrideq, r10mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r9mp, r10 + mov r11mp, r11 + mov r12mp, r12 + neg wq + +%%loop_x_v_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, top_offxy, unused2, luma, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%if %2 == 0 + lea r10, [pw_27_17_17_27] +%endif +%%loop_y_v_overlap: + ; luma_src +%if %2 + mova xm2, [lumaq+lstrideq*0+ 0] + vinserti128 m2, [lumaq+lstrideq*0+32], 1 + mova xm4, [lumaq+lstrideq*0+16] + vinserti128 m4, [lumaq+lstrideq*0+48], 1 + mova xm3, [lumaq+lstrideq*(1<<%3)+ 0] + vinserti128 m3, [lumaq+lstrideq*(1<<%3)+32], 1 + mova xm5, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m5, [lumaq+lstrideq*(1<<%3)+48], 1 + phaddw m2, m4 + phaddw m3, m5 + pxor m4, m4 + pavgw m2, m4 + pavgw m3, m4 +%elif %1 + mova m2, [lumaq] + mova m3, [lumaq+32] +%endif +%if %1 + mova m0, [srcq] +%if %2 + mova m1, [srcq+strideq] +%else + mova m1, [srcq+32] +%endif + punpckhwd m4, m2, m0 + punpcklwd m2, m0 + punpckhwd m5, m3, m1 + punpcklwd m3, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m4, m2, m5, m3 + REPX {paddd x, m15}, m4, m2, m5, m3 + REPX {psrad x, 6 }, m4, m2, m5, m3 + packusdw m2, m4 + packusdw m3, m5 + pminuw m2, m10 ; clip_pixel() + pminuw m3, m10 +%elif %2 + pand m2, m10 + pand m3, m10 +%else + pand m2, m10, [lumaq+ 0] + pand m3, m10, [lumaq+32] +%endif + + ; scaling[luma_src] + vpbroadcastd m7, [pd_m65536] + pandn m4, m7, m2 + mova m6, m7 + vpgatherdd m5, [scalingq+m4-0], m7 + psrld m2, 16 + mova m7, m6 + vpgatherdd m4, [scalingq+m2-2], m6 + pblendw m4, m5, 0x55 + pandn m5, m7, m3 + mova m6, m7 + vpgatherdd m2, [scalingq+m5-0], m7 + psrld m3, 16 + vpgatherdd m5, [scalingq+m3-2], m6 + pblendw m5, m2, 0x55 + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq*2] + movu m3, [grain_lutq+top_offxyq*2] + punpcklwd m2, m3, m6 + punpckhwd m3, m6 ; { top, cur } +%if %3 + vpbroadcastd m0, [pw_23_22] +%elif %2 + vpbroadcastd m0, [pw_27_17_17_27] +%else + vpbroadcastd m0, [r10] +%endif + REPX {pmaddwd x, m0}, m2, m3 +%if %1 + vpbroadcastd m1, [pd_16] + REPX {paddd x, m1}, m2, m3 +%else + REPX {paddd x, m14}, m2, m3 +%endif + REPX {psrad x, 5}, m2, m3 + packssdw m2, m3 +%if %2 + movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif +%if %3 + pmaxsw m2, m8 + pminsw m2, m9 +%else +%if %2 + movu m7, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m6, m3, m7 ; { cur, top } + punpcklwd m3, m7 +%else + movu m7, [grain_lutq+top_offxyq*2+32] + punpckhwd m6, m7, m3 + punpcklwd m3, m7, m3 ; { top, cur } +%endif + pmaddwd m6, m0 + pmaddwd m3, m0 +%if %1 + paddd m6, m1 + paddd m3, m1 +%else + paddd m6, m14 + paddd m3, m14 +%endif + psrad m6, 5 + psrad m3, 5 + packssdw m3, m6 + pmaxsw m2, m8 + pmaxsw m3, m8 + pminsw m2, m9 + pminsw m3, m9 +%endif + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m2, [srcq] +%if %2 + paddw m1, m3, [srcq+strideq] +%else + paddw m1, m3, [srcq+32] +%endif + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq], m0 +%if %2 + mova [dstq+strideq], m1 + sub hb, 2 +%else + mova [dstq+32], m1 + dec hb +%endif + jle %%end_y_v_overlap +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(2<<%2) +%if %2 + jmp %%loop_y +%else + add hd, 0x80000000 + jc %%loop_y + add r10, 4 + jmp %%loop_y_v_overlap +%endif +%%end_y_v_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +%%loop_x_hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride + +%if %2 == 0 + lea r14, [pw_27_17_17_27] +%endif + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%%loop_y_hv_overlap: + ; luma_src +%if %2 + mova xm2, [lumaq+lstrideq*0+ 0] + vinserti128 m2, [lumaq+lstrideq*0+32], 1 + mova xm4, [lumaq+lstrideq*0+16] + vinserti128 m4, [lumaq+lstrideq*0+48], 1 + mova xm3, [lumaq+lstrideq*(1<<%3)+ 0] + vinserti128 m3, [lumaq+lstrideq*(1<<%3)+32], 1 + mova xm5, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m5, [lumaq+lstrideq*(1<<%3)+48], 1 + phaddw m2, m4 + phaddw m3, m5 + pxor m4, m4 + pavgw m2, m4 + pavgw m3, m4 +%elif %1 + mova m2, [lumaq] + mova m3, [lumaq+32] +%endif +%if %1 + mova m0, [srcq] +%if %2 + mova m1, [srcq+strideq] +%else + mova m1, [srcq+32] +%endif + punpckhwd m4, m2, m0 + punpcklwd m2, m0 + punpckhwd m5, m3, m1 + punpcklwd m3, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m4, m2, m5, m3 + REPX {paddd x, m15}, m4, m2, m5, m3 + REPX {psrad x, 6 }, m4, m2, m5, m3 + packusdw m2, m4 + packusdw m3, m5 + pminuw m2, m10 ; clip_pixel() + pminuw m3, m10 +%elif %2 + pand m2, m10 + pand m3, m10 +%else + pand m2, m10, [lumaq+ 0] + pand m3, m10, [lumaq+32] +%endif + + ; scaling[luma_src] + vpbroadcastd m7, [pd_m65536] + pandn m4, m7, m2 + mova m6, m7 + vpgatherdd m5, [scalingq+m4-0], m7 + psrld m2, 16 + mova m7, m6 + vpgatherdd m4, [scalingq+m2-2], m6 + pblendw m4, m5, 0x55 + pandn m5, m7, m3 + mova m6, m7 + vpgatherdd m2, [scalingq+m5-0], m7 + psrld m3, 16 + vpgatherdd m5, [scalingq+m3-2], m6 + pblendw m5, m2, 0x55 + + ; grain = grain_lut[offy+y][offx+x] + movu m0, [grain_lutq+offxyq*2] + movd xm2, [grain_lutq+left_offxyq*2] + movu m6, [grain_lutq+top_offxyq*2] +%if %2 + pinsrw xm2, [grain_lutq+left_offxyq*2+82*2], 2 + movu m3, [grain_lutq+offxyq*2+82*2] + punpckldq xm1, xm0, xm3 ; { cur0, cur1 } +%if %3 + vinserti128 m2, [grain_lutq+topleft_offxyq*2], 1 ; { left0, left1, top/left } + vinserti128 m1, [grain_lutq+top_offxyq*2], 1 ; { cur0, cur1, top0 } +%else + vinserti128 m2, [grain_lutq+topleft_offxyq*2+82*2], 1 + vpbroadcastd m7, [grain_lutq+topleft_offxyq*2] + vpblendd m2, m7, 0x20 + movd xm7, [grain_lutq+top_offxyq*2+82*2] + punpckldq xm7, xm6 + vinserti128 m1, xm7, 1 + movu m7, [grain_lutq+top_offxyq*2+82*2] +%endif + punpcklwd m2, m1 ; { cur, left } +%if %1 + vpbroadcastq m1, [pw_23_22] + pmaddwd m2, m1 + vpbroadcastd m1, [pd_16] + paddd m2, m1 + psrad m2, 5 + packssdw m2, m2 + vpermq m2, m2, q3120 +%else + pmaddwd m2, m15 + paddd m2, m14 + psrad m2, 5 + vextracti128 xm1, m2, 1 + packssdw xm2, xm1 +%endif +%else + pinsrd xm2, [grain_lutq+topleft_offxyq*2], 1 + movu m3, [grain_lutq+offxyq*2+32] + movu m7, [grain_lutq+top_offxyq*2+32] + punpckldq xm1, xm0, xm6 + punpcklwd xm2, xm1 ; { cur, left } +%if %1 + movddup xm1, [pw_27_17_17_27] + pmaddwd xm2, xm1 + vpbroadcastd m1, [pd_16] + paddd xm2, xm1 +%else + pmaddwd xm2, xm15 + paddd xm2, xm14 +%endif + psrad xm2, 5 + packssdw xm2, xm2 +%endif + pmaxsw xm2, xm8 + pminsw xm2, xm9 + vpblendd m0, m2, 0x01 +%if %2 + pshufd xm2, xm2, q0321 + vpblendd m3, m2, 0x01 +%if %3 == 0 + pshufd xm2, xm2, q0321 + vpblendd m7, m2, 0x01 +%endif +%endif + pshuflw xm2, xm2, q1032 + vpblendd m2, m6, 0xfe + punpckhwd m6, m0 ; { top, cur } + punpcklwd m2, m0 +%if %3 + vpbroadcastd m0, [pw_23_22] +%elif %2 + vpbroadcastd m0, [pw_27_17_17_27] +%else + vpbroadcastd m0, [r14] +%endif + pmaddwd m6, m0 + pmaddwd m2, m0 +%if %1 + paddd m6, m1 + paddd m2, m1 +%else + paddd m6, m14 + paddd m2, m14 +%endif + psrad m6, 5 + psrad m2, 5 + packssdw m2, m6 + +%if %3 + pmaxsw m2, m8 + pminsw m2, m9 +%else +%if %2 + punpckhwd m6, m3, m7 + punpcklwd m3, m7 ; { cur, top } +%else + punpckhwd m6, m7, m3 + punpcklwd m3, m7, m3 ; { top, cur } +%endif + REPX {pmaddwd x, m0}, m6, m3 +%if %1 + REPX {paddd x, m1}, m6, m3 +%else + REPX {paddd x, m14}, m6, m3 +%endif + REPX {psrad x, 5}, m6, m3 + packssdw m3, m6 + pmaxsw m2, m8 + pmaxsw m3, m8 + pminsw m2, m9 + pminsw m3, m9 +%endif + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m4, m11 + pmaddubsw m5, m11 + paddw m4, m4 + paddw m5, m5 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m2, [srcq] +%if %2 + paddw m1, m3, [srcq+strideq] +%else + paddw m1, m3, [srcq+32] +%endif + pmaxsw m0, m12 + pmaxsw m1, m12 + pminsw m0, m13 + pminsw m1, m13 + mova [dstq], m0 +%if %2 + mova [dstq+strideq], m1 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + add srcq, strideq + add dstq, strideq + add lumaq, r10mp +%endif + add grain_lutq, 82*(2<<%2) +%if %2 + sub hb, 2 + jg %%loop_y_h_overlap +%else + dec hb + jle %%end_y_hv_overlap + add hd, 0x80000000 + jc %%loop_y_h_overlap + add r14, 4 + jmp %%loop_y_hv_overlap +%endif +%%end_y_hv_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + jmp %%loop_x_hv_overlap +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +GEN_GRAIN_UV_FN 420, 1, 1 +FGUV_FN 420, 1, 1 +GEN_GRAIN_UV_FN 422, 1, 0 +FGUV_FN 422, 1, 0 +GEN_GRAIN_UV_FN 444, 0, 0 +FGUV_FN 444, 0, 0 + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/filmgrain16_avx512.asm b/third_party/dav1d/src/x86/filmgrain16_avx512.asm new file mode 100644 index 0000000000..00dd6af599 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain16_avx512.asm @@ -0,0 +1,932 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 +pb_0to63: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + db 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 + db 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 +scale_mask: db -1, -1, 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1 +scale_shift: dw 7, 7, 6, 6, 5, 5, 4, 4 +pw_27_17_17_27: dw 108, 68, 68, 108, 27, 17, 17, 27 +pw_23_22: dw 92, 88, 0, 128, 23, 22, 0, 32 +fg_min: times 2 dw 0 + times 2 dw 64 + times 2 dw 256 +fg_max: times 2 dw 1023 + times 2 dw 4095 + times 2 dw 960 + times 2 dw 3840 + times 2 dw 940 + times 2 dw 3760 +scale_rnd: dd 64 + dd 16 +uv_offset_mul: dd 256 + dd 1024 +pb_8_9_0_1: db 8, 9, 0, 1 + +SECTION .text + +INIT_ZMM avx512icl +cglobal fgy_32x32xn_16bpc, 6, 15, 21, dst, src, stride, fg_data, w, scaling, \ + grain_lut, offx, sby, see, offy, src_bak +%define base r11-fg_min + lea r11, [fg_min] + mov r6d, r9m ; bdmax + mov r9d, [fg_dataq+FGData.clip_to_restricted_range] + mov r7d, [fg_dataq+FGData.scaling_shift] + mov sbyd, sbym + vpbroadcastd m6, r9m + shr r6d, 11 ; is_12bpc + vbroadcasti32x4 m7, [base+scale_mask] + shlx r10d, r9d, r6d + vpbroadcastd m10, [base+scale_shift+r7*4-32] + lea r9d, [r6+r9*4] + vpbroadcastd m8, [base+fg_min+r10*4] + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m9, [base+fg_max+r9*4] + mov r12, 0xeeeeeeeeeeeeeeee + vpbroadcastd m19, [base+scale_rnd+r6*4] + kshiftrb k2, k1, 4 ; 0xf + vpbroadcastq xm20, [base+pw_27_17_17_27+r6*8] + kmovq k3, r12 + vpbroadcastd m11, [base+scale_shift+r6*8+4] + test sbyd, sbyd + setnz r7b + vpbroadcastd m12, [base+pw_27_17_17_27+r6*8+0] + vpbroadcastd m13, [base+pw_27_17_17_27+r6*8+4] + test r7b, [fg_dataq+FGData.overlap_flag] + jnz .v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + call .add_noise + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je .loop_x + test sbyd, sbyd + jnz .hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy + + lea left_offxyd, [offyq+73] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+left_offxyq*2+82*1], 1 + punpckldq xm16, xm4, xm5 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + psrad xm16, 1 + packssdw xm16, xm16 + vpsravw xm16, xm11 + vmovdqu8 m4{k2}, m16 + vpalignr m5{k2}, m16, m16, 4 + call .add_noise + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + test sbyd, sbyd + jnz .hv_overlap + jmp .loop_x_h_overlap + +.v_overlap: + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, _, top_offxy + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m16, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movu m17, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m4, m0, m16 + punpcklwd m0, m16 + punpckhwd m5, m1, m17 + punpcklwd m1, m17 + call .add_noise_v + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to .v_overlap, and instead always fall-through to .hv_overlap +.hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+73] + lea left_offxyd, [offyq+73] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m5, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+topleft_offxyq*2-82*1], 1 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + movd xm18, [grain_lutq+left_offxyq*2+82*1] + pinsrd xm18, [grain_lutq+topleft_offxyq*2+82*1], 1 + punpckldq xm16, xm5, xm0 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + punpckldq xm17, xm2, xm1 + punpcklwd xm18, xm17 + mova xm17, xm19 + vpdpwssd xm17, xm20, xm18 + punpckhwd m4, m0, m5 + punpcklwd m0, m5 + punpckhwd m5, m1, m2 + punpcklwd m1, m2 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm11 + vpshuflw m0{k2}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m1{k2}, m16, q1302 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq*2] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: + mova m2, m19 + vpdpwssd m2, m12, m4 + mova m3, m19 + vpdpwssd m3, m13, m5 + mova m4, m19 + vpdpwssd m4, m12, m0 + mova m5, m19 + vpdpwssd m5, m13, m1 + REPX {psrad x, 1}, m2, m3, m4, m5 + packssdw m4, m2 + packssdw m5, m3 + vpsravw m4, m11 + vpsravw m5, m11 +.add_noise: + mova m0, [srcq+strideq*0] + mova m1, [srcq+strideq*1] + kmovw k4, k1 + pand m16, m6, m0 + psrld m3, m0, 16 + vpgatherdd m2{k4}, [scalingq+m16] + vpcmpud k4, m3, m6, 2 ; px <= bdmax + vpgatherdd m16{k4}, [scalingq+m3] + kmovw k4, k1 + pand m17, m6, m1 + vpgatherdd m3{k4}, [scalingq+m17] + vpshufb m2{k3}, m16, m7 + psrld m16, m1, 16 + vpcmpud k4, m16, m6, 2 + vpgatherdd m17{k4}, [scalingq+m16] + vpshufb m3{k3}, m17, m7 + vpsllvw m2, m10 + vpsllvw m3, m10 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + add grain_lutq, 82*4 + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m8 + pmaxsw m1, m8 + pminsw m0, m9 + pminsw m1, m9 + mova [dstq+srcq], m0 + add srcq, strideq + mova [dstq+srcq], m1 + add srcq, strideq + ret + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 22, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r12-fg_min + lea r12, [fg_min] + mov r9d, r13m ; bdmax + mov r7d, [fg_dataq+FGData.scaling_shift] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + mov r11d, is_idm + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m5, r13m + mov r13, 0xeeeeeeeeeeeeeeee + vbroadcasti32x4 m6, [base+scale_mask] + shr r9d, 11 ; is_12bpc + vpbroadcastd m7, [base+scale_shift+r7*4-32] + shlx r10d, r6d, r9d + mov sbyd, sbym + shlx r6d, r6d, r11d + vpbroadcastd m8, [base+fg_min+r10*4] + lea r6d, [r9+r6*2] + vpbroadcastd m9, [base+fg_max+r6*4] + kmovq k2, r13 + vpbroadcastd m20, [base+scale_rnd+r9*4] + packssdw m4, m5, m5 + vpbroadcastd m21, [base+scale_shift+r9*8+4] +%if %2 + mova m12, [base+pb_0to63] ; pw_even + mov r13d, 0x0101 + vpbroadcastq m10, [base+pw_23_22+r9*8] + kmovw k3, r13d +%if %3 + pshufd m11, m10, q0000 +%else + vpbroadcastd ym16, [base+pw_27_17_17_27+r9*8+0] + vpbroadcastd m11, [base+pw_27_17_17_27+r9*8+4] + vmovdqu16 m11{k1}, m16 +%endif + psrlw m13, m12, 8 ; pw_odd +%else + vpbroadcastq m10, [base+pw_27_17_17_27+r9*8] + kshiftrb k3, k1, 7 ; 0x01 + kshiftrb k4, k1, 4 ; 0x0f + pshufd m11, m10, q0000 +%endif + mov lstrideq, r10mp + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + +%if %1 + mov r6d, r11m + vpbroadcastd m0, [base+uv_offset_mul+r9*4] + vpbroadcastd m1, [base+pb_8_9_0_1] + vpbroadcastd m14, [fg_dataq+FGData.uv_offset+r6*4] + vbroadcasti32x4 m15, [fg_dataq+FGData.uv_mult+r6*4] + pmaddwd m14, m0 + pshufb m15, m1 ; { uv_luma_mult, uv_mult } +%endif + test r7b, [fg_dataq+FGData.overlap_flag] + jnz %%v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] +%endif + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je %%loop_x + cmp dword r8m, 0 ; sby + jne %%hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + mova m17, m20 + vpdpwssd m17, m16, m10 + psrad m17, 1 + packssdw m17, m17 + vpsravw m17, m21 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm16, [grain_lutq+left_offxyq*2+82*2], 1 + punpckldq xm17, xm18, xm19 + punpcklwd xm16, xm17 + mova xm17, xm20 + vpdpwssd xm17, xm16, xm10 + psrad xm17, 1 + packssdw xm17, xm17 + vpsravw xm17, xm21 +%endif + vmovdqa32 m18{k3}, m17 + vpshufd m19{k3}, m17, q0321 + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp dword r8m, 0 ; sby + jne %%hv_overlap + jmp %%loop_x_h_overlap + +%%v_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, _, top_offxy + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + +%if %3 + movu ym16, [grain_lutq+offxyq*2+82*0] + movu ym1, [grain_lutq+top_offxyq*2+82*0] + vbroadcasti32x8 m18, [grain_lutq+offxyq*2+82*2] + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd ym17, ym1, ym16 + punpckhwd ym1, ym16 +%elif %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym17, [grain_lutq+top_offxyq*2+82*0] + vinserti32x8 m17, [grain_lutq+top_offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd m16, m17, m18 + punpckhwd m17, m18 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movu m2, [grain_lutq+offxyq*2+82*2] + movu m16, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m16 + punpcklwd m2, m16 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to %%v_overlap, and instead always fall-through to %%hv_overlap +%%hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + movu ym1, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+topleft_offxyq*2+82*0] + mova m0, m20 + vpdpwssd m0, m16, m10 +%if %3 + punpcklwd xm17, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + psrad xm16, 1 +%else + vinserti32x8 m1, [grain_lutq+top_offxyq*2+82*2], 1 + vinserti32x4 m17, [grain_lutq+topleft_offxyq*2+82*2], 2 + punpcklwd m17, m1 + mova m16, m20 + vpdpwssd m16, m17, m10 + psrad m16, 1 +%endif + psrad m0, 1 + packssdw m0, m16 + vpsravw m0, m21 + vmovdqa32 m18{k3}, m0 + vpshufd m19{k3}, m0, q0321 +%if %3 + vpunpckhdq ym1{k3}, ym0, ym0 + punpcklwd ym17, ym1, ym18 + punpckhwd ym1, ym18 +%else + vpunpckhdq m1{k3}, m0, m0 + punpcklwd m16, m1, m18 + punpckhwd m17, m1, m18 +%endif +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm17, [grain_lutq+topleft_offxyq*2+82*0], 1 + punpckldq xm16, xm18, xm19 + punpcklwd xm17, xm16 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m0, [grain_lutq+top_offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*2] + pinsrd xm16, [grain_lutq+topleft_offxyq*2+82*2], 1 + punpckldq xm1, xm2, xm0 + punpcklwd xm1, xm16, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + mova xm17, xm20 + vpdpwssd xm17, xm1, xm10 + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m0 + punpcklwd m2, m0 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm21 + vpshuflw m19{k4}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m2{k4}, m16, q3120 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + jmp %%hv_overlap + +ALIGN function_align +%%add_noise_v: +%if %3 + mova ym16, ym20 + vpdpwssd ym16, ym17, ym11 + mova ym17, ym20 + vpdpwssd ym17, ym1, ym11 + psrad ym16, 1 + psrad ym17, 1 + packssdw ym16, ym17 + vpsravw m18{k1}, m16, m21 +%elif %2 + mova m18, m20 + vpdpwssd m18, m16, m11 + mova m16, m20 + vpdpwssd m16, m17, m11 + psrad m18, 1 + psrad m16, 1 + packssdw m18, m16 + vpsravw m18, m21 +%else + mova m16, m20 + vpdpwssd m16, m1, m11 + mova m17, m20 + vpdpwssd m17, m18, m11 + mova m18, m20 + vpdpwssd m18, m19, m11 + mova m19, m20 + vpdpwssd m19, m2, m11 + REPX {psrad x, 1}, m16, m17, m18, m19 + packssdw m18, m16 + packssdw m19, m17 + vpsravw m18, m21 + vpsravw m19, m21 +%endif +%%add_noise: +%if %2 + mova m2, [lumaq+lstrideq*(0<<%3)] + mova m0, [lumaq+lstrideq*(1<<%3)] + lea lumaq, [lumaq+lstrideq*(2<<%3)] + mova m3, [lumaq+lstrideq*(0<<%3)] + mova m1, [lumaq+lstrideq*(1<<%3)] + mova m16, m12 + vpermi2w m16, m2, m0 + vpermt2w m2, m13, m0 + mova m17, m12 + vpermi2w m17, m3, m1 + vpermt2w m3, m13, m1 + pavgw m2, m16 + pavgw m3, m17 +%elif %1 + mova m2, [lumaq+lstrideq*0] + mova m3, [lumaq+lstrideq*1] +%endif +%if %2 + mova ym16, [srcq+strideq*0] + vinserti32x8 m16, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] +%else + mova m16, [srcq+strideq*0] +%endif +%if %1 + punpckhwd m17, m2, m16 + mova m0, m14 + vpdpwssd m0, m17, m15 + punpcklwd m17, m2, m16 + mova m2, m14 + vpdpwssd m2, m17, m15 +%endif +%if %2 + mova ym17, [srcq+strideq*0] + vinserti32x8 m17, [srcq+strideq*1], 1 +%else + mova m17, [srcq+strideq*1] +%endif +%if %1 + psrad m0, 6 + psrad m2, 6 + packusdw m2, m0 + punpckhwd m0, m3, m17 + mova m1, m14 + vpdpwssd m1, m15, m0 + punpcklwd m0, m3, m17 + mova m3, m14 + vpdpwssd m3, m15, m0 + psrad m1, 6 + psrad m3, 6 + packusdw m3, m1 + pminuw m2, m4 + pminuw m3, m4 + +.add_noise_main: + ; scaling[luma_src] + kmovw k5, k1 + pand m1, m5, m2 + vpgatherdd m0{k5}, [scalingq+m1] + kmovw k5, k1 + psrld m2, 16 + vpgatherdd m1{k5}, [scalingq+m2] + vpshufb m0{k2}, m1, m6 + kmovw k5, k1 + psrld m1, m3, 16 + vpgatherdd m2{k5}, [scalingq+m1] + kmovw k5, k1 + pand m3, m5 + vpgatherdd m1{k5}, [scalingq+m3] + vpshufb m1{k2}, m2, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + vpsllvw m0, m7 + vpsllvw m1, m7 + pmulhrsw m18, m0 + pmulhrsw m19, m1 + add grain_lutq, 82*(4<<%2) + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*2] + paddw m16, m18 + paddw m17, m19 + pmaxsw m16, m8 + pmaxsw m17, m8 + pminsw m16, m9 + pminsw m17, m9 +%if %2 + mova [dstq+strideq*0], ym16 + vextracti32x8 [dstq+strideq*1], m16, 1 + lea dstq, [dstq+strideq*2] + mova [dstq+strideq*0], ym17 + vextracti32x8 [dstq+strideq*1], m17, 1 +%else + mova [dstq+strideq*0], m16 + mova [dstq+strideq*1], m17 +%endif + lea dstq, [dstq+strideq*2] + ret +%else +%if %2 + pand m2, m4 + pand m3, m4 +%else + pand m2, m4, [lumaq+lstrideq*0] + pand m3, m4, [lumaq+lstrideq*1] +%endif + jmp .add_noise_main +%endif +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 + +%endif diff --git a/third_party/dav1d/src/x86/filmgrain16_sse.asm b/third_party/dav1d/src/x86/filmgrain16_sse.asm new file mode 100644 index 0000000000..6b0daaac0b --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain16_sse.asm @@ -0,0 +1,3421 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +SECTION_RODATA 16 +pd_16: times 4 dd 16 +pw_1: times 8 dw 1 +pw_16384: times 8 dw 16384 +pw_8192: times 8 dw 8192 +pw_23_22: dw 23, 22 + times 3 dw 0, 32 +pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 +pw_27_17_17_27: dw 27, 17, 17, 27 + times 2 dw 0, 32 +rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +pb_1: times 4 db 1 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512, 1024 +max: dw 256*4-1, 240*4, 235*4, 256*16-1, 240*16, 235*16 +min: dw 0, 16*4, 16*16 +; these two should be next to each other +pw_4: times 2 dw 4 +pw_16: times 2 dw 16 + +%macro JMP_TABLE 1-* + %xdefine %1_table %%table + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .ar%2 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_16bpc_ssse3, 0, 1, 2, 3 + +SECTION .text + +%if ARCH_X86_32 +%undef base +%define PIC_ptr(a) base+a +%else +%define PIC_ptr(a) a +%endif + +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +%macro vpgatherdw 5-8 8, 1 ; dst, src, base, tmp_gpr[x2], cnt, stride, tmp_xmm_reg +%assign %%idx 0 +%define %%tmp %2 +%if %0 == 8 +%define %%tmp %8 +%endif +%rep (%6/2) +%if %%idx == 0 + movd %5 %+ d, %2 + pshuflw %%tmp, %2, q3232 +%else + movd %5 %+ d, %%tmp +%if %6 == 8 +%if %%idx == 2 + punpckhqdq %%tmp, %%tmp +%elif %%idx == 4 + psrlq %%tmp, 32 +%endif +%endif +%endif + movzx %4 %+ d, %5 %+ w + shr %5 %+ d, 16 + +%if %%idx == 0 + movd %1, [%3+%4*%7] +%else + pinsrw %1, [%3+%4*%7], %%idx + 0 +%endif + pinsrw %1, [%3+%5*%7], %%idx + 1 +%assign %%idx %%idx+2 +%endrep +%endmacro + +%macro SPLATD 2 ; dst, src +%ifnidn %1, %2 + movd %1, %2 +%endif + pshufd %1, %1, q0000 +%endmacro + +%macro SPLATW 2 ; dst, src +%ifnidn %1, %2 + movd %1, %2 +%endif + pshuflw %1, %1, q0000 + punpcklqdq %1, %1 +%endmacro + + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal generate_grain_y_16bpc, 3, 8, 16, buf, fg_data, bdmax + lea r4, [pb_mask] +%define base r4-pb_mask +%else +cglobal generate_grain_y_16bpc, 3, 6, 8, buf, fg_data, bdmax + LEA r4, $$ +%define base r4-$$ +%endif + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r3d, [fg_dataq+FGData.grain_scale_shift] + lea r5d, [bdmaxq+1] + shr r5d, 11 ; 0 for 10bpc, 2 for 12bpc + sub r3, r5 + SPLATW m6, [base+round+r3*2-2] + mova m5, [base+pb_mask] + SPLATW m0, [fg_dataq+FGData.seed] + mov r3, -73*82*2 + sub bufq, r3 +%if ARCH_X86_64 + lea r6, [gaussian_sequence] +%endif +.loop: + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m2, m3 + psllq m3, m2, 15 + por m2, m3 ; aggregate each bit into next seed's high bit + pmulhuw m3, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + vpgatherdw m3, m2, r6, r5, r7, 4, 2 +%else + vpgatherdw m3, m2, base+gaussian_sequence, r5, r2, 4, 2 +%endif + paddw m3, m3 ; otherwise bpc=12 w/ grain_scale_shift=0 + ; shifts by 0, which pmulhrsw does not support + pmulhrsw m3, m6 + movq [bufq+r3], m3 + add r3, 4*2 + jl .loop + + ; auto-regression code + movsxd r3, [fg_dataq+FGData.ar_coeff_lag] + movsxd r3, [base+generate_grain_y_16bpc_ssse3_table+r3*4] + lea r3, [r3+base+generate_grain_y_16bpc_ssse3_table] + jmp r3 + +.ar1: +%if WIN64 + DEFINE_ARGS shift, fg_data, max, buf, val3, min, cf3, x, val0 + lea bufq, [r0-2*(82*73-(82*3+79))] + PUSH r8 +%else +%if ARCH_X86_64 + DEFINE_ARGS buf, fg_data, max, shift, val3, min, cf3, x, val0 +%else ; x86-32 + DEFINE_ARGS buf, fg_data, min, val3, x, cf3, val0 + PUSH r6 +%define shiftd r1d +%endif + sub bufq, 2*(82*73-(82*3+79)) +%endif + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd m4, [fg_dataq+FGData.ar_coeffs_y] + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] +%if WIN64 + DEFINE_ARGS shift, h, max, buf, val3, min, cf3, x, val0 +%elif ARCH_X86_64 + DEFINE_ARGS buf, h, max, shift, val3, min, cf3, x, val0 +%else ; x86-32 +%undef shiftd + DEFINE_ARGS buf, shift, min, val3, x, cf3, val0 +%define hd dword r0m +%define maxd dword minm +%endif +%if cpuflag(sse4) + pmovsxbw m4, m4 +%else + pxor m3, m3 + pcmpgtb m3, m4 + punpcklbw m4, m3 +%endif + pinsrw m4, [base+pw_1], 3 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + SPLATW m3, [base+round_vals+shiftq*2-12] ; rnd + mov hd, 70 + sar maxd, 1 + mov mind, maxd + xor mind, -1 +.y_loop_ar1: + mov xq, -76 + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu m0, [bufq+xq*2-82*2-2] ; top/left + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 + punpcklwd m1, m3 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar1 +%if WIN64 + POP r8 +%elif ARCH_X86_32 + POP r6 +%undef maxd +%undef hd +%endif +.ar0: + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -16*8 +%endif + DEFINE_ARGS buf, fg_data, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m0, [base+round_vals-12+shiftq*2] + pshuflw m0, m0, q0000 + movu m6, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-11 + pxor m2, m2 + punpcklwd m0, m2 + pcmpgtb m2, m6 + punpckhbw m3, m6, m2 + punpcklbw m6, m2 + pshufd m2, m6, q3333 + pshufd m1, m6, q2222 + pshufd m7, m6, q1111 + pshufd m6, m6, q0000 + pshufd m4, m3, q1111 + pshufd m3, m3, q0000 +%if ARCH_X86_64 + SWAP 0, 12 + SWAP 1, 8 + SWAP 2, 9 + SWAP 3, 10 + SWAP 4, 11 +%else +%define m12 [rsp+0*16] +%define m8 [rsp+1*16] +%define m9 [rsp+2*16] +%define m10 [rsp+3*16] +%define m11 [rsp+4*16] + mova m12, m0 + mova m8, m1 + mova m9, m2 + mova m10, m3 + mova m11, m4 + mov bdmaxd, bdmaxm +%endif + sar bdmaxd, 1 + SPLATW m0, bdmaxd ; max_grain + pcmpeqw m1, m1 +%if !cpuflag(sse4) + pcmpeqw m2, m2 + psrldq m2, 14 + pslldq m2, 2 + pxor m2, m1 +%endif + pxor m1, m0 ; min_grain +%if ARCH_X86_64 + SWAP 0, 13 + SWAP 1, 14 + SWAP 2, 15 +%else +%define m13 [rsp+5*16] +%define m14 [rsp+6*16] + mova m13, m0 + mova m14, m1 +%if !cpuflag(sse4) +%define m15 [rsp+7*16] + mova m15, m2 +%endif +%endif + sub bufq, 2*(82*73-(82*3+79)) + DEFINE_ARGS buf, fg_data, h, x + mov hd, 70 +.y_loop_ar2: + mov xq, -76 + +.x_loop_ar2: + movu m0, [bufq+xq*2-82*4-4] ; y=-2,x=[-2,+5] + movu m1, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] + psrldq m2, m0, 2 + psrldq m3, m0, 4 + psrldq m4, m0, 6 + psrldq m5, m0, 8 + punpcklwd m0, m2 + punpcklwd m3, m4 + punpcklwd m5, m1 + psrldq m2, m1, 2 + psrldq m4, m1, 4 + punpcklwd m2, m4 + psrldq m4, m1, 6 + psrldq m1, 8 + punpcklwd m4, m1 + pmaddwd m0, m6 + pmaddwd m3, m7 + pmaddwd m5, m8 + pmaddwd m2, m9 + pmaddwd m4, m10 + paddd m0, m3 + paddd m5, m2 + paddd m0, m4 + paddd m0, m5 ; accumulated top 2 rows + paddd m0, m12 + + movu m1, [bufq+xq*2-4] ; y=0,x=[-2,+5] + pshufd m4, m1, q3321 + pxor m2, m2 + pcmpgtw m2, m4 + punpcklwd m4, m2 ; in dwords, y=0,x=[0,3] +.x_loop_ar2_inner: + pmaddwd m2, m1, m11 + paddd m2, m0 + psrldq m0, 4 ; shift top to next pixel + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + paddd m2, m4 + packssdw m2, m2 + pminsw m2, m13 + pmaxsw m2, m14 + psrldq m4, 4 + pslldq m2, 2 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000010b +%else + pand m1, m15 + pandn m3, m15, m2 + por m1, m3 +%endif + ; overwrite previous pixel, this should be ok + movd [bufq+xq*2-2], m1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar2 +%if ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +.ar3: + DEFINE_ARGS buf, fg_data, bdmax, shift +%if WIN64 + mov r6, rsp + and rsp, ~15 + sub rsp, 64 + %define tmp rsp +%elif ARCH_X86_64 + %define tmp rsp+stack_offset-72 +%else +%assign stack_offset stack_offset_old + ALLOC_STACK -16*12 + %define tmp rsp + mov bdmaxd, bdmaxm +%endif + sar bdmaxd, 1 + SPLATW m7, bdmaxd ; max_grain + pcmpeqw m6, m6 +%if !cpuflag(sse4) + pcmpeqw m4, m4 + psrldq m4, 14 + pslldq m4, 4 + pxor m4, m6 +%endif + pxor m6, m7 ; min_grain + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + +%if ARCH_X86_64 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m14 [rsp+10*16] +%define m15 [esp+11*16] + mova m14, m6 + mova m15, m7 +%endif + + ; build cf0-1 until 18-19 in m5-12 and r0/1 + pxor m1, m1 + movu m0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + +%if cpuflag(sse4) + pshufd m4, m2, q3333 +%else + pshufd m5, m2, q3333 + mova [tmp+48], m5 +%endif + pshufd m3, m2, q2222 + pshufd m1, m2, q0000 + pshufd m2, m2, q1111 + pshufd m7, m0, q2222 + pshufd m6, m0, q1111 + pshufd m5, m0, q0000 + pshufd m0, m0, q3333 + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 + SWAP 4, 12 +%else +%define m8 [rsp+4*16] +%define m9 [esp+5*16] +%define m10 [rsp+6*16] +%define m11 [esp+7*16] +%define m12 [rsp+8*16] + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 + mova m12, m4 +%endif + + ; build cf20,round in r2 + ; build cf21-23,round*2 in m13 + pxor m1, m1 + movq m0, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pcmpgtb m1, m0 + punpcklbw m0, m1 + pshufd m1, m0, q0000 + pshufd m2, m0, q1111 + mova [tmp+ 0], m1 + mova [tmp+16], m2 + psrldq m3, m0, 10 + pinsrw m3, [base+round_vals+shiftq*2-10], 3 + +%if ARCH_X86_64 + SWAP 3, 13 +%else +%define m13 [esp+9*16] + mova m13, m3 +%endif + + pinsrw m0, [base+round_vals+shiftq*2-12], 5 + pshufd m3, m0, q2222 + mova [tmp+32], m3 + + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 2*(82*73-(82*3+79)) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 + +.x_loop_ar3: + movu m0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + movd m1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+6] + palignr m2, m1, m0, 2 ; y=-3,x=[-2,+5] + palignr m1, m1, m0, 12 ; y=-3,x=[+3,+6] + punpckhwd m3, m0, m2 ; y=-3,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m0, m2 ; y=-3,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m2, m0, m3, q1032 ; y=-3,x=[-1/+0,+0/+1,+1/+2,+2/+3] + + pmaddwd m0, m5 + pmaddwd m2, m6 + pmaddwd m3, m7 + paddd m0, m2 + paddd m0, m3 + ; m0 = top line first 6 multiplied by cf, m1 = top line last entry + + movu m2, [bufq+xq*2-82*4-6+ 0] ; y=-2,x=[-3,+4] + movd m3, [bufq+xq*2-82*4-6+16] ; y=-2,x=[+5,+6] + punpcklwd m1, m2 ; y=-3/-2,x=[+3/-3,+4/-2,+5/-1,+6/+0] + palignr m4, m3, m2, 2 ; y=-3,x=[-2,+5] + palignr m3, m3, m2, 4 ; y=-3,x=[-1,+6] + punpckhwd m2, m4, m3 ; y=-2,x=[+2/+3,+3/+4,+4/+5,+5/+6] + punpcklwd m4, m3 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + shufps m3, m4, m2, q1032 ; y=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + + pmaddwd m1, m8 + pmaddwd m4, m9 + pmaddwd m3, m10 + pmaddwd m2, m11 + paddd m1, m4 + paddd m3, m2 + paddd m0, m1 + paddd m0, m3 + ; m0 = top 2 lines multiplied by cf + + movu m1, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + movd m2, [bufq+xq*2-82*2-6+16] ; y=-1,x=[+5,+6] + palignr m3, m2, m1, 2 ; y=-1,x=[-2,+5] + palignr m2, m2, m1, 12 ; y=-1,x=[+3,+6] + punpckhwd m4, m1, m3 ; y=-1,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m1, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m3, m1, m4, q1032 ; y=-1,x=[-1/+0,+0/+1,+1/+2,+2/+3] + punpcklwd m2, [base+pw_1] + +%if cpuflag(sse4) + pmaddwd m1, m12 +%else + pmaddwd m1, [tmp+48] +%endif + pmaddwd m3, [tmp+ 0] + pmaddwd m4, [tmp+16] + pmaddwd m2, [tmp+32] + paddd m1, m3 + paddd m4, m2 + paddd m0, m1 + paddd m0, m4 + ; m0 = top 3 lines multiplied by cf plus rounding for downshift + + movu m1, [bufq+xq*2-6] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pmaddwd m2, m1, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + packssdw m2, m2 + pminsw m2, m15 + pmaxsw m2, m14 + pslldq m2, 4 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000100b +%else + pand m1, m12 + pandn m3, m12, m2 + por m1, m3 +%endif + ; overwrite a couple of pixels, should be ok + movq [bufq+xq*2-4], m1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar3 +%if WIN64 + mov rsp, r6 +%elif ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal generate_grain_uv_%1_16bpc, 4, 11, 16, buf, bufy, fg_data, uv, bdmax, x, gaussian_reg, h, pic_reg +%define base r8-pb_mask + lea r8, [pb_mask] + movifnidn bdmaxd, bdmaxm + lea r6d, [bdmaxq+1] +%else +cglobal generate_grain_uv_%1_16bpc, 1, 7, 8, buf, x, pic_reg, fg_data, h +%define base r2-$$ + LEA r2, $$ + mov fg_dataq, r2m + mov r6d, r4m + inc r6d +%endif + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r5d, [fg_dataq+FGData.grain_scale_shift] + shr r6d, 11 ; 0 for 10bpc, 2 for 12bpc + sub r5, r6 + SPLATW m6, [base+round+r5*2-2] + mova m5, [base+pb_mask] + SPLATW m0, [fg_dataq+FGData.seed] +%if ARCH_X86_64 + SPLATW m2, [base+pw_seed_xor+uvq*4] +%else + mov r5d, r3m + SPLATW m2, [base+pw_seed_xor+r5*4] +%endif + pxor m0, m2 +%if ARCH_X86_64 + lea r6, [gaussian_sequence] +%endif +%if %2 + mov hd, 73-35*%3 + add bufq, 44*2 +.loop_y: + mov xq, -44 +%else + mov xq, -82*73 + add bufq, 82*73*2 +%endif +.loop_x: + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m2, m3 + psllq m3, m2, 15 + por m2, m3 ; aggregate each bit into next seed's high bit + pmulhuw m3, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + vpgatherdw m3, m2, r6, r9, r10, 4, 2 +%else + vpgatherdw m3, m2, base+gaussian_sequence, r5, r6, 4, 2 +%endif + paddw m3, m3 ; otherwise bpc=12 w/ grain_scale_shift=0 + ; shifts by 0, which pmulhrsw does not support + pmulhrsw m3, m6 + movq [bufq+xq*2], m3 + add xq, 4 + jl .loop_x +%if %2 + add bufq, 82*2 + dec hd + jg .loop_y +%endif + + ; auto-regression code + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movsxd r5, [base+generate_grain_uv_%1_16bpc_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_16bpc_ssse3_table] + jmp r5 + +.ar0: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift +%assign stack_offset_old stack_offset + ALLOC_STACK -16*2 + mov bufyq, r1m + mov uvd, r3m +%endif + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m4, [fg_dataq+FGData.ar_coeffs_uv+uvq] + SPLATW m3, [base+hmul_bits+shiftq*2-10] +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m1, bdmaxd ; max_gain +%else + SPLATW m1, r4m + psraw m1, 1 +%endif + pcmpeqw m7, m7 + pxor m7, m1 ; min_grain +%if ARCH_X86_64 + SWAP 1, 14 + DEFINE_ARGS buf, bufy, h, x +%else +%define m14 [rsp+0*16] + mova m14, m1 + DEFINE_ARGS buf, bufy, pic_reg, h, x +%endif + pxor m5, m5 + pcmpgtb m5, m4 + punpcklbw m4, m5 +%if %2 + SPLATW m6, [base+hmul_bits+2+%3*2] +%endif + SPLATW m4, m4 + pxor m5, m5 +%if %2 +%if !cpuflag(sse4) + pcmpeqw m2, m2 + pslldq m2, 12 +%if ARCH_X86_64 + SWAP 2, 12 +%else +%define m12 [rsp+1*16] + mova m12, m2 +%endif +%endif +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+82-(82*3+41)) +%else + sub bufq, 2*(82*70-3) +%endif + add bufyq, 2*(3+82*3) + mov hd, 70-35*%3 +.y_loop_ar0: + ; first 32 pixels + xor xd, xd +.x_loop_ar0: + movu m0, [bufyq+xq*(2<<%2)] +%if %2 +%if %3 + movu m2, [bufyq+xq*4+82*2] + paddw m0, m2 +%endif + movu m1, [bufyq+xq*4 +16] +%if %3 + movu m2, [bufyq+xq*4+82*2+16] + paddw m1, m2 +%endif + phaddw m0, m1 + pmulhrsw m0, m6 +%endif + punpckhwd m1, m0, m5 + punpcklwd m0, m5 + REPX {pmaddwd x, m4}, m0, m1 + REPX {psrad x, 5}, m0, m1 + packssdw m0, m1 + pmulhrsw m0, m3 + movu m1, [bufq+xq*2] + paddw m0, m1 + pminsw m0, m14 + pmaxsw m0, m7 + cmp xd, 72-40*%2 + je .end + movu [bufq+xq*2], m0 + add xd, 8 + jmp .x_loop_ar0 + + ; last 6/4 pixels +.end: +%if %2 +%if cpuflag(sse4) + pblendw m0, m1, 11000000b +%else + pand m1, m12 + pandn m2, m12, m0 + por m0, m1, m2 +%endif + movu [bufq+xq*2], m0 +%else + movq [bufq+xq*2], m0 +%endif + + add bufq, 82*2 + add bufyq, 82*(2<<%3) + dec hd + jg .y_loop_ar0 +%if ARCH_X86_32 +%undef m12 +%undef m14 +%endif + RET + +.ar1: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, max, cf3, min, val3, x +%else +%assign stack_offset stack_offset_old +%xdefine rstk rsp +%assign stack_size_padded 0 + DEFINE_ARGS buf, shift, pic_reg, fg_data, uv, bufy, cf3 + mov bufyq, r1m + mov uvd, r3m +%endif + imul uvd, 28 + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movq m4, [fg_dataq+FGData.ar_coeffs_uv+uvq] +%if WIN64 + DEFINE_ARGS shift, bufy, h, buf, max, cf3, min, val3, x, val0 +%if %2 + lea bufq, [r0-2*(82*(73-35*%3)+44-(82*3+41))] +%else + lea bufq, [r0-2*(82*69+3)] +%endif +%else +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, h, shift, max, cf3, min, val3, x, val0 +%else + DEFINE_ARGS buf, shift, pic_reg, fg_data, val0, bufy, cf3 +%define hd dword r1m +%define mind dword r3m +%define maxd dword r4m +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif +%endif +%if ARCH_X86_64 + mov shiftd, [r2+FGData.ar_coeff_shift] +%else + mov shiftd, [r3+FGData.ar_coeff_shift] +%endif + pxor m5, m5 + pcmpgtb m5, m4 + punpcklbw m4, m5 ; cf0-4 in words + pshuflw m4, m4, q2100 + psrldq m4, 2 ; cf0-3,4 in words + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + movd m3, [base+round_vals+shiftq*2-12] ; rnd + pxor m6, m6 + punpcklwd m3, m6 +%if %2 + SPLATW m6, [base+hmul_bits+2+%3*2] +%endif + SPLATD m3, m3 + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 + sar maxd, 1 +%if ARCH_X86_64 + mov mind, maxd + xor mind, -1 +%else + DEFINE_ARGS buf, shift, val3, x, val0, bufy, cf3 + mov r2, maxd + xor r2, -1 + mov mind, r2 +%endif +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu m0, [bufq+xq*2-82*2-2] ; top/left +%if %2 + movu m7, [bufyq+xq*4] +%if %3 + movu m1, [bufyq+xq*4+82*2] + phaddw m7, m1 +%else + phaddw m7, m7 +%endif +%else + movq m7, [bufyq+xq*2] +%endif + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 +%if %2 +%if %3 + pshufd m2, m7, q3232 + paddw m7, m2 +%endif + pmulhrsw m7, m6 +%endif + punpcklwd m1, m7 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 + paddd m0, m3 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar1 +%if ARCH_X86_32 +%undef maxd +%undef mind +%undef hd +%endif + RET + +.ar2: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift + ALLOC_STACK -16*8 + mov bufyq, r1m + mov uvd, r3m +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m5, bdmaxd ; max_grain +%else + SPLATW m5, r4m + psraw m5, 1 +%endif + pcmpeqw m6, m6 +%if !cpuflag(sse4) + pcmpeqw m7, m7 + psrldq m7, 14 + pslldq m7, 2 + pxor m7, m6 +%endif + pxor m6, m5 ; min_grain +%if %2 && cpuflag(sse4) + SPLATW m7, [base+hmul_bits+2+%3*2] +%endif + +%if ARCH_X86_64 + SWAP 5, 13 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m13 [rsp+5*16] +%define m14 [rsp+6*16] +%define m15 [rsp+7*16] + mova m13, m5 + mova m14, m6 + mova m15, m7 +%endif + + ; coef values + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pinsrw m2, [base+round_vals-12+shiftq*2], 5 + + pshufd m6, m0, q0000 + pshufd m7, m0, q1111 + pshufd m1, m0, q3333 + pshufd m0, m0, q2222 + pshufd m3, m2, q1111 + pshufd m4, m2, q2222 + pshufd m2, m2, q0000 + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 + SWAP 4, 12 +%else +%define m8 [rsp+0*16] +%define m9 [rsp+1*16] +%define m10 [rsp+2*16] +%define m11 [rsp+3*16] +%define m12 [rsp+4*16] + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 + mova m12, m4 +%endif + +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, h, x +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, h, x +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + movu m0, [bufq+xq*2-82*4-4] ; y=-2,x=[-2,+5] + movu m5, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] + psrldq m4, m0, 2 ; y=-2,x=[-1,+5] + psrldq m1, m0, 4 ; y=-2,x=[-0,+5] + psrldq m3, m0, 6 ; y=-2,x=[+1,+5] + psrldq m2, m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m0, m4 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + punpcklwd m1, m3 ; y=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + punpcklwd m2, m5 ; y=-2/-1,x=[+2/-2,+3/-1,+4/+0,+5/+1] + pmaddwd m0, m6 + pmaddwd m1, m7 + pmaddwd m2, m8 + paddd m0, m1 + paddd m0, m2 + psrldq m3, m5, 2 ; y=-1,x=[-1,+5] + psrldq m1, m5, 4 ; y=-1,x=[-0,+5] + psrldq m4, m5, 6 ; y=-1,x=[+1,+5] + psrldq m2, m5, 8 ; y=-1,x=[+2,+5] + punpcklwd m3, m1 + punpcklwd m4, m2 + pmaddwd m3, m9 + pmaddwd m4, m10 + paddd m3, m4 + paddd m0, m3 + + ; luma component & rounding +%if %2 + movu m1, [bufyq+xq*4] +%if %3 + movu m2, [bufyq+xq*4+82*2] + phaddw m1, m2 + pshufd m2, m1, q3232 + paddw m1, m2 +%else + phaddw m1, m1 +%endif +%if cpuflag(sse4) + pmulhrsw m1, m15 +%elif %3 + pmulhrsw m1, [base+pw_8192] +%else + pmulhrsw m1, [base+pw_16384] +%endif +%else + movq m1, [bufyq+xq*2] +%endif + punpcklwd m1, [base+pw_1] + pmaddwd m1, m12 + paddd m0, m1 + + movu m1, [bufq+xq*2-4] ; y=0,x=[-2,+5] + pshufd m2, m1, q3321 + pxor m3, m3 + pcmpgtw m3, m2 + punpcklwd m2, m3 ; y=0,x=[0,3] in dword +.x_loop_ar2_inner: + pmaddwd m3, m1, m11 + paddd m3, m0 + psrldq m0, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + ; we do not need to packssdw since we only care about one value + paddd m3, m2 + packssdw m3, m3 + pminsw m3, m13 + pmaxsw m3, m14 + psrldq m1, 2 + pslldq m3, 2 + psrldq m2, 4 +%if cpuflag(sse4) + pblendw m1, m3, 00000010b +%else + pand m1, m15 + pandn m4, m15, m3 + por m1, m4 +%endif + ; overwrite previous pixel, should be ok + movd [bufq+xq*2-2], m1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar2 +%if ARCH_X86_32 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +.ar3: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%if WIN64 + mov r6, rsp + and rsp, ~15 + sub rsp, 96 + %define tmp rsp +%else + %define tmp rsp+stack_offset-120 +%endif +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift +%assign stack_offset stack_offset_old + ALLOC_STACK -16*14 + mov bufyq, r1m + mov uvd, r3m + %define tmp rsp +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + SPLATW m4, [base+round_vals-12+shiftq*2] + pxor m5, m5 + pcmpgtw m5, m4 + punpcklwd m4, m5 +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m6, bdmaxd ; max_grain +%else + SPLATW m6, r4m + psraw m6, 1 +%endif + pcmpeqw m7, m7 +%if !cpuflag(sse4) + pcmpeqw m3, m3 + psrldq m3, 14 + pslldq m3, 4 + pxor m3, m7 +%endif + pxor m7, m6 ; min_grain +%if %2 && cpuflag(sse4) + SPLATW m3, [base+hmul_bits+2+%3*2] +%endif + +%if ARCH_X86_64 + SWAP 3, 11 + SWAP 4, 12 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m11 [rsp+ 9*16] +%define m12 [rsp+10*16] +%define m14 [rsp+12*16] +%define m15 [rsp+13*16] + mova m11, m3 + mova m12, m4 + mova m14, m6 + mova m15, m7 +%endif + + ; cf from y=-3,x=-3 until y=-3,x=-2 + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pshufd m1, m0, q0000 + pshufd m3, m0, q1111 + pshufd m4, m0, q2222 + pshufd m0, m0, q3333 + pshufd m5, m2, q0000 + pshufd m6, m2, q1111 + mova [tmp+16*0], m1 + mova [tmp+16*1], m3 + mova [tmp+16*2], m4 + mova [tmp+16*3], m0 + mova [tmp+16*4], m5 + mova [tmp+16*5], m6 + pshufd m6, m2, q2222 + pshufd m7, m2, q3333 + + ; cf from y=-1,x=-1 to y=0,x=-1 + luma component + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 ; luma + punpcklbw m0, m1 + pshufd m3, m0, q3232 + psrldq m5, m0, 10 + ; y=0,x=[-3 to -1] + "1.0" for current pixel + pinsrw m5, [base+round_vals-10+shiftq*2], 3 + ; y=-1,x=[-1 to +2] + pshufd m1, m0, q0000 + pshufd m0, m0, q1111 + ; y=-1,x=+3 + luma + punpcklwd m3, m2 + pshufd m3, m3, q0000 + +%if ARCH_X86_64 + SWAP 1, 8 + SWAP 0, 9 + SWAP 3, 10 + SWAP 5, 13 + DEFINE_ARGS buf, bufy, fg_data, h, x +%else +%define m8 [rsp+ 6*16] +%define m9 [rsp+ 7*16] +%define m10 [rsp+ 8*16] +%define m13 [rsp+11*16] + mova m8, m1 + mova m9, m0 + mova m10, m3 + mova m13, m5 + DEFINE_ARGS buf, bufy, pic_reg, fg_data, h, x +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) + +.x_loop_ar3: + ; first line + movu m0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + movd m1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+6] + palignr m2, m1, m0, 2 ; y=-3,x=[-2,+5] + palignr m1, m1, m0, 12 ; y=-3,x=[+3,+6] + punpckhwd m3, m0, m2 ; y=-3,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m0, m2 ; y=-3,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m2, m0, m3, q1032 ; y=-3,x=[-1/+0,+0/+1,+1/+2,+2/+3] + + pmaddwd m0, [tmp+0*16] + pmaddwd m2, [tmp+1*16] + pmaddwd m3, [tmp+2*16] + paddd m0, m2 + paddd m0, m3 ; first 6 x of top y + + ; second line [m0/1 are busy] + movu m2, [bufq+xq*2-82*4-6+ 0] ; y=-2,x=[-3,+4] + movd m3, [bufq+xq*2-82*4-6+16] ; y=-2,x=[+5,+6] + punpcklwd m1, m2 ; y=-3/-2,x=[+3/-3,+4/-2,+5/-1,+6/+0] + palignr m4, m3, m2, 2 ; y=-2,x=[-2,+5] + palignr m3, m3, m2, 4 ; y=-2,x=[-2,+5] + punpckhwd m5, m4, m3 ; y=-2,x=[+2/+3,+3/+4,+4/+5,+5/+6] + punpcklwd m4, m3 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + shufps m3, m4, m5, q1032 ; t=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + pmaddwd m1, [tmp+3*16] + pmaddwd m4, [tmp+4*16] + pmaddwd m3, [tmp+5*16] + pmaddwd m5, m6 + paddd m1, m4 + paddd m3, m5 + paddd m0, m1 + paddd m0, m3 ; top 2 lines + + ; third line [m0 is busy] & luma + round + movu m1, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + movd m2, [bufq+xq*2-82*2-6+16] ; y=-1,x=[+5,+6] +%if %2 + movu m5, [bufyq+xq*4] +%if %3 + movu m4, [bufyq+xq*4+82*2] + phaddw m5, m4 +%else + phaddw m5, m5 +%endif +%else + movq m5, [bufyq+xq*2] +%endif + palignr m3, m2, m1, 2 ; y=-1,x=[-2,+5] + palignr m2, m2, m1, 12 ; y=-1,x=[+3,+6] +%if %3 + pshufd m4, m5, q3232 + paddw m5, m4 +%endif +%if %2 +%if cpuflag(sse4) + pmulhrsw m5, m11 +%elif %3 + pmulhrsw m5, [base+pw_8192] +%else + pmulhrsw m5, [base+pw_16384] +%endif +%endif + punpckhwd m4, m1, m3 ; y=-1,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m1, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m3, m1, m4, q1032 ; y=-1,x=[-1/+0,+0/+1,+1/+2,+2/+3] + punpcklwd m2, m5 + pmaddwd m1, m7 + pmaddwd m3, m8 + pmaddwd m4, m9 + pmaddwd m2, m10 + paddd m1, m3 + paddd m4, m2 + paddd m0, m12 ; += round + paddd m1, m4 + paddd m0, m1 + + movu m1, [bufq+xq*2-6] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pmaddwd m2, m1, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + packssdw m2, m2 + pminsw m2, m14 + pmaxsw m2, m15 + pslldq m2, 4 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000100b +%else + pand m1, m11 + pandn m3, m11, m2 + por m1, m3 +%endif + ; overwrite previous pixels, should be ok + movq [bufq+xq*2-4], m1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar3 +%if WIN64 + mov rsp, r6 +%elif ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fgy_32x32xn_16bpc, 0, 7, 8, 0-(8 * mmsize + 12 * gprsize), \ + dst, src, scaling, unused1, fg_data, picptr, unused2 + ; copy stack arguments to new position post-alignment, so that we + ; don't have to keep the old stack location in a separate register + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov r5, r8m + +%define r0m [rsp+8*mmsize+ 3*gprsize] +%define r2m [rsp+8*mmsize+ 5*gprsize] +%define r4m [rsp+8*mmsize+ 7*gprsize] +%define r6m [rsp+8*mmsize+ 9*gprsize] +%define r7m [rsp+8*mmsize+10*gprsize] +%define r8m [rsp+8*mmsize+11*gprsize] + + mov r0m, r0 + mov r2m, r1 + mov r4m, r2 + mov r6m, r3 + mov r7m, r4 + mov r8m, r5 +%else +cglobal fgy_32x32xn_16bpc, 0, 7, 8, 8 * mmsize + 4 * gprsize, \ + dst, src, scaling, unused1, fg_data, picptr, unused2 +%endif + mov srcq, srcm + mov scalingq, r5m + mov fg_dataq, r3m +%if STACK_ALIGNMENT < mmsize + mov r6, r9m + +%define r9m [rsp+8*mmsize+ 4*gprsize] +%define r3m [rsp+8*mmsize+ 6*gprsize] +%define r5m [rsp+8*mmsize+ 8*gprsize] + + mov r9m, r6 +%endif + LEA r5, $$ +%define base r5-$$ + mov r5m, picptrq +%else +cglobal fgy_32x32xn_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut + lea r8, [pb_mask] +%define base r8-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + SPLATW m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] +%if ARCH_X86_32 + DECLARE_REG_TMP 0, 3 +%else + DECLARE_REG_TMP 9, 10 +%endif + mov t0d, r9m ; bdmax + sar t0d, 11 ; is_12bpc + inc t0d + mov t1d, r6d + imul t1d, t0d + dec t0d + SPLATW m5, [base+min+t1*2] + lea t0d, [t0d*3] + lea t0d, [r6d*2+t0d] + SPLATW m4, [base+max+t0*2] + SPLATW m2, r9m + + pcmpeqw m1, m1 + psraw m7, m2, 1 ; max_grain + pxor m1, m7 ; min_grain + SPLATD m6, [base+pd_16] + + SCRATCH 1, 9, 0 + SCRATCH 2, 10, 1 + SCRATCH 3, 11, 2 + SCRATCH 4, 12, 3 + SCRATCH 5, 13, 4 + SCRATCH 6, 14, 5 + SCRATCH 7, 15, 6 + + mova m6, [base+pw_27_17_17_27] ; for horizontal filter + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, unused2 + DECLARE_REG_TMP 0 +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ + sby, see + DECLARE_REG_TMP 7 +%endif + + mov sbyd, r8m + movzx t0d, byte [fg_dataq+FGData.overlap_flag] + test t0d, t0d + jz .no_vertical_overlap + test sbyd, sbyd + jnz .vertical_overlap +.no_vertical_overlap: + mov dword r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, src_bak +%endif + + lea src_bakq, [srcq+wq*2] + mov r9mp, src_bakq + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r4m, wq +%endif + +.loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak +%endif + +.loop_x_odd: + movzx hd, word r7m + mov grain_lutq, grain_lutmp +.loop_y: + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m2, m0, scalingq-1, r0, r5, 8, 1, m4 + vpgatherdw m3, m1, scalingq-1, r0, r5, 8, 1, m4 +%else + vpgatherdw m2, m0, scalingq-1, r11, r13, 8, 1, m4 + vpgatherdw m3, m1, scalingq-1, r11, r13, 8, 1, m4 +%endif + REPX {psrlw x, 8}, m2, m3 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2] + movu m5, [grain_lutq+offxyq*2+16] + + ; noise = round2(scaling[src] * grain, scaling_shift) + REPX {pmullw x, m11}, m2, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp ; src += stride + add grain_lutq, 82*2 + dec hd + jg .loop_y + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + btc dword r8m, 2 + jc .next_blk + add offxyd, 16 + test dword r8m, 2 + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r12d, 16 ; top_offxy += 16 +%endif + jmp .loop_x_odd_v_overlap + +.next_blk: + test dword r8m, 1 + jz .loop_x + + ; r8m = sbym + test dword r8m, 2 + jnz .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: +%if ARCH_X86_32 + add offxyd, 16 + mov [rsp+8*mmsize+0*gprsize], offxyd + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + mov seed, r3m +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy +%endif + + mov hd, dword r7m + mov grain_lutq, grain_lutmp +.loop_y_h_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] + movd m4, [grain_lutq+r5*2] +%else + movd m4, [grain_lutq+left_offxyq*2] +%endif + punpcklwd m4, m5 + pmaddwd m4, m6 + paddd m4, m14 + psrad m4, 5 + packssdw m4, m4 + pminsw m4, m15 + pmaxsw m4, m9 + shufps m4, m5, q3210 + + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m2, m0, scalingq-1, r0, r5, 8, 1, m5 + vpgatherdw m3, m1, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m2, m0, scalingq-1, r13, r14, 8, 1, m5 + vpgatherdw m3, m1, scalingq-1, r13, r14, 8, 1, m5 +%endif + REPX {psrlw x, 8}, m2, m3 + + ; noise = round2(scaling[src] * grain, scaling_shift) + movu m5, [grain_lutq+offxyq*2+16] + REPX {pmullw x, m11}, m2, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hd + jg .loop_y_h_overlap + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + or dword r8m, 4 + add offxyd, 16 + + ; r8m = sbym + test dword r8m, 2 + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r12d, 16 ; top_offxy += 16 +%endif + jmp .loop_x_odd_v_overlap + +.end: + RET + +.vertical_overlap: + or t0d, 2 + mov r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, unused +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ + sby, see +%endif + + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS dst, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul t0d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add t0d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and t0d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, t0d +%if ARCH_X86_32 + xor sbyd, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, src_bak +%endif + + lea src_bakq, [srcq+wq*2] + mov r9mp, src_bakq + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r4m, wq +%endif + +.loop_x_v_overlap: +%if ARCH_X86_32 + mov r5, r5m + SPLATD m7, [base+pw_27_17_17_27] + mov seed, r3m +%else + SPLATD m7, [pw_27_17_17_27] +%endif + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, unused, top_offxy + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, unused, top_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +.loop_x_odd_v_overlap: +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)] + mov hd, dword r7m + mov grain_lutq, grain_lutmp +.loop_y_v_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+1*gprsize] + movu m2, [grain_lutq+r5*2] +%else + movu m2, [grain_lutq+top_offxyq*2] +%endif + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + REPX {pmaddwd x, m7}, m4, m2 + REPX {paddd x, m14}, m4, m2 + REPX {psrad x, 5}, m4, m2 + packssdw m2, m4 + pminsw m2, m15 + pmaxsw m2, m9 + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m3, [grain_lutq+r5*2+16] +%else + movu m3, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m5, m3, m4 + punpcklwd m3, m4 + REPX {pmaddwd x, m7}, m5, m3 + REPX {paddd x, m14}, m5, m3 + REPX {psrad x, 5}, m5, m3 + packssdw m3, m5 + pminsw m3, m15 + pmaxsw m3, m9 + + ; src + pand m0, m10, [srcq+ 0] ; m0-1: src as word + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] + ; noise = round2(scaling[src] * grain, scaling_shift) +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m4, m0, scalingq-1, r11, r13, 8, 1, m5 +%endif + psrlw m4, 8 + pmullw m4, m11 + pmulhrsw m4, m2 +%if ARCH_X86_32 + vpgatherdw m5, m1, scalingq-1, r0, r5, 8, 1, m2 +%else + vpgatherdw m5, m1, scalingq-1, r11, r13, 8, 1, m2 +%endif + psrlw m5, 8 + pmullw m5, m11 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hw + jz .end_y_v_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)+4] + xor hd, 0x10000 + test hd, 0x10000 + jnz .loop_y_v_overlap + jmp .loop_y + +.end_y_v_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + btc dword r8m, 2 + jc .next_blk_v +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + jmp .loop_x_odd_v_overlap + +.next_blk_v: + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap + +.loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r0, [rsp+8*mmsize+1*gprsize] + add r3, 16 + add r0, 16 + mov [rsp+8*mmsize+0*gprsize], r3 ; left_offxy + mov [rsp+8*mmsize+2*gprsize], r0 ; topleft_offxy + + mov seed, r3m + xor r0, r0 +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, w, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy, top_offxy, topleft_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)] + + movzx hd, word r7m + mov grain_lutq, grain_lutmp +.loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m2, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r0, [rsp+8*mmsize+1*gprsize] ; top_offxy + mov r5, [rsp+8*mmsize+0*gprsize] ; left_offxy + movu m4, [grain_lutq+r0*2] + movd m5, [grain_lutq+r5*2] + mov r5, [rsp+8*mmsize+2*gprsize] ; topleft_offxy + movd m3, [grain_lutq+r5*2] +%else + movu m4, [grain_lutq+top_offxyq*2] + movd m5, [grain_lutq+left_offxyq*2] + movd m3, [grain_lutq+topleft_offxyq*2] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklwd m5, m2 + punpcklwd m3, m4 + REPX {pmaddwd x, m6}, m5, m3 + REPX {paddd x, m14}, m5, m3 + REPX {psrad x, 5}, m5, m3 + packssdw m5, m3 + pminsw m5, m15 + pmaxsw m5, m9 + shufps m3, m5, m2, q3210 + shufps m5, m4, q3232 + ; followed by v interpolation (top | cur -> cur) + movu m0, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m1, [grain_lutq+r0*2+16] +%else + movu m1, [grain_lutq+top_offxyq*2+16] +%endif + punpcklwd m2, m5, m3 + punpckhwd m5, m3 + punpcklwd m3, m1, m0 + punpckhwd m1, m0 + REPX {pmaddwd x, m7}, m2, m5, m3, m1 + REPX {paddd x, m14}, m2, m5, m3, m1 + REPX {psrad x, 5}, m2, m5, m3, m1 + packssdw m2, m5 + packssdw m3, m1 + REPX {pminsw x, m15}, m2, m3 + REPX {pmaxsw x, m9}, m2, m3 + + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] + ; noise = round2(scaling[src] * grain, scaling_shift) +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m4, m0, scalingq-1, r14, r10, 8, 1, m5 +%endif + psrlw m4, 8 + pmullw m4, m11 + pmulhrsw m2, m4 +%if ARCH_X86_32 + vpgatherdw m5, m1, scalingq-1, r0, r5, 8, 1, m4 +%else + vpgatherdw m5, m1, scalingq-1, r14, r10, 8, 1, m4 +%endif + psrlw m5, 8 + pmullw m5, m11 + pmulhrsw m3, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hw + jz .end_y_hv_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)+4] + xor hd, 0x10000 + test hd, 0x10000 + jnz .loop_y_hv_overlap + jmp .loop_y_h_overlap + +.end_y_hv_overlap: + or dword r8m, 4 +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov r5, r5m + add offxyd, 16 + add dword [rsp+8*mmsize+1*gprsize], 16 ; top_offxy += 16 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + add offxyd, 16 + add top_offxyd, 16 + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + jmp .loop_x_odd_v_overlap + +.end_hv: + RET +%if ARCH_X86_32 + DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +%endif + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +INIT_XMM ssse3 +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fguv_32x32xn_i%1_16bpc, 0, 7, 8, 0-(8 * mmsize + 16 * gprsize), \ + tmp, src, scaling, h, fg_data, picptr, unused + mov r0, r0m + mov r1, r1m + mov r2, r2m + mov r4, r3m + mov r3, r4m + mov r5, r5m +%define r0m [rsp+8*mmsize+ 3*gprsize] +%define r1m [rsp+8*mmsize+ 4*gprsize] +%define r2m [rsp+8*mmsize+ 5*gprsize] +%define r3m [rsp+8*mmsize+ 6*gprsize] +%define r4m [rsp+8*mmsize+ 7*gprsize] +%define r5m [rsp+8*mmsize+ 8*gprsize] + mov r0m, r0 + mov r2m, r2 + mov r4m, r3 + mov r5m, r5 + + mov r0, r6m + mov r2, r7m + mov r3, r8m + mov r5, r9m +%define r6m [rsp+8*mmsize+ 9*gprsize] +%define r7m [rsp+8*mmsize+10*gprsize] +%define r8m [rsp+8*mmsize+11*gprsize] +%define r9m [rsp+8*mmsize+12*gprsize] + mov r6m, r0 + mov r7m, r2 + mov r8m, r3 + mov r9m, r5 + + mov r2, r10m + mov r3, r11m + mov r5, r12m + mov r0, r13m +%define r10m [rsp+8*mmsize+13*gprsize] +%define r11m [rsp+8*mmsize+14*gprsize] +%define r12m [rsp+8*mmsize+15*gprsize] + mov r10m, r2 + mov r11m, r3 + mov r12m, r5 + + SPLATW m2, r13m +%else +cglobal fguv_32x32xn_i%1_16bpc, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ + tmp, src, scaling, h, fg_data, picptr, unused + mov srcq, srcm + mov fg_dataq, r3m +%endif + LEA r5, $$ +%define base r5-$$ + + DECLARE_REG_TMP 0, 2, 3 +%else +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r8-pb_mask + lea r8, [pb_mask] + + DECLARE_REG_TMP 9, 10, 11 +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + SPLATW m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] +%if STACK_ALIGNMENT >= mmsize + mov t0d, r13m ; bdmax +%endif + sar t0d, 11 ; is_12bpc + inc t0d + mov t1d, r6d + imul t1d, t0d + dec t0d + SPLATW m5, [base+min+t1*2] + lea t1d, [t0d*3] + mov t2d, r12m + inc t2d + imul r6d, t2d + add t1d, r6d + SPLATW m4, [base+max+t1*2] +%if STACK_ALIGNMENT >= mmsize + SPLATW m2, r13m +%endif + + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + +%define mzero m7 + +%if %3 + SPLATD m2, [base+pw_23_22] +%endif + +%if ARCH_X86_32 + mov scalingq, r5m + mov r5m, r5 +%else + mov r13mp, strideq +%endif + + pcmpeqw m0, m0 + psraw m1, m10, 1 + pxor m0, m1 + + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_h, ss_v +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap + + DECLARE_REG_TMP 0 +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap + + DECLARE_REG_TMP 9 +%endif + +%if %1 + mov r6d, r11m + SPLATW m0, [fg_dataq+FGData.uv_mult+r6*4] + SPLATW m1, [fg_dataq+FGData.uv_luma_mult+r6*4] + punpcklwd m6, m1, m0 + SPLATW m5, [fg_dataq+FGData.uv_offset+r6*4] + SPLATD m7, [base+pw_4+t0*4] + pmullw m5, m7 +%else + SPLATD m6, [base+pd_16] +%if %2 + mova m5, [base+pw_23_22] +%else + mova m5, [base+pw_27_17_17_27] +%endif +%endif + + SCRATCH 6, 14, 6 + SCRATCH 5, 15, 7 + +%if ARCH_X86_32 + DECLARE_REG_TMP 0 +%else + DECLARE_REG_TMP 7 +%endif + + mov sbyd, r8m + mov t0d, [fg_dataq+FGData.overlap_flag] + test t0d, t0d + jz %%no_vertical_overlap + test sbyd, sbyd + jnz %%vertical_overlap + +%%no_vertical_overlap: + mov r8m, t0d +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, luma + + mov dstq, r0mp + mov lumaq, r9mp + mov wq, r4m + lea r3, [srcq+wq*2] + mov r1mp, r3 + lea r3, [dstq+wq*2] + mov r11mp, r3 + lea r3, [lumaq+wq*(2<<%2)] + mov r12mp, r3 +%if %3 + shl r10mp, 1 +%endif +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused2, unused3, see, unused4, unused5, unused6, luma, lstride + + mov lstrideq, r10mp +%if %3 + add lstrideq, lstrideq +%endif + mov lumaq, r9mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r10mp, r10 + mov r11mp, r11 + mov r12mp, r12 +%endif + neg wq +%if ARCH_X86_32 + mov r4mp, wq +%endif + +%%loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, unused2, unused3, luma, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, unused2, unused3, luma, lstride +%endif + +%if %2 == 0 +%%loop_x_odd: +%endif + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y: + ; src + mova m0, [srcq] + mova m1, [srcq+16] ; m0-1: src as word + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9m +%endif + mova m4, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m4, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9m, lumaq +%endif +%if %2 + pavgw m4, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m3, m4, m0 + punpcklwd m4, m0 + punpckhwd m5, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m3, m4, m5, m6 + REPX {psrad x, 6}, m3, m4, m5, m6 + packssdw m4, m3 + packssdw m6, m5 + REPX {paddw x, m15}, m4, m6 + REPX {pmaxsw x, mzero}, m4, m6 + REPX {pminsw x, m10}, m4, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m4, m6 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m3, m4, scalingq-1, r0, r5, 8, 1 + vpgatherdw m5, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m3, m4, scalingq-1, r10, r12, 8, 1 + vpgatherdw m5, m6, scalingq-1, r10, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m3, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2] + movu m6, [grain_lutq+offxyq*2+16] + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m3, m5 + pmulhrsw m4, m3 + pmulhrsw m6, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hd + jg %%loop_y + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, picptr, luma + + mov wq, r4mp +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0m, dstq + mov r9m, lumaq + mov r4m, wq +%endif +%if %2 == 0 + btc dword r8m, 2 + jc %%next_blk + add offxyd, 16 + test dword r8m, 2 + jz %%loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + jmp %%loop_x_odd_v_overlap +%%next_blk: +%endif + test dword r8m, 1 + je %%loop_x + + ; r8m = sbym + test dword r8m, 2 + jnz %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: +%if ARCH_X86_32 + add offxyd, 16 + mov [rsp+8*mmsize+0*gprsize], offxyd + + DEFINE_ARGS dst, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, luma, lstride + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, luma, lstride +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_h_overlap: + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + mov lumaq, r9m +%endif + mova m4, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m4, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9m, lumaq +%endif +%if %2 + pavgw m4, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m3, m4, m0 + punpcklwd m4, m0 + punpckhwd m5, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m3, m4, m5, m6 + REPX {psrad x, 6}, m3, m4, m5, m6 + packssdw m4, m3 + packssdw m6, m5 + REPX {paddw x, m15}, m4, m6 + REPX {pmaxsw x, mzero}, m4, m6 + REPX {pminsw x, m10}, m4, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m4, m6 +%endif + + ; grain = grain_lut[offy+y][offx+x] + movu m7, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] + movd m5, [grain_lutq+r5*2] +%else + movd m5, [grain_lutq+left_offxyq*2+ 0] +%endif + punpcklwd m5, m7 ; {left0, cur0} +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %2 + pmaddwd m5, [PIC_ptr(pw_23_22)] +%else + pmaddwd m5, [PIC_ptr(pw_27_17_17_27)] +%endif + paddd m5, [PIC_ptr(pd_16)] +%else + pmaddwd m5, m15 + paddd m5, m14 +%endif + psrad m5, 5 + packssdw m5, m5 + pmaxsw m5, m8 + pminsw m5, m9 + shufps m5, m7, q3210 + movu m3, [grain_lutq+offxyq*2+16] + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5, 8, 1 + vpgatherdw m4, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m7, m4, scalingq-1, r2, r12, 8, 1 + vpgatherdw m4, m6, scalingq-1, r2, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m7, m4 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m4 + pmulhrsw m5, m7 + pmulhrsw m3, m4 + + ; dst = clip_pixel(src, noise) + paddw m0, m5 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hd + jg %%loop_y_h_overlap + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + mov wq, r4mp +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0mp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif + +%if %2 + ; r8m = sbym + test dword r8m, 2 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap +%else + or dword r8m, 4 + add offxyd, 16 + + ; r8m = sbym + test dword r8m, 2 + jz %%loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxy += 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%end: + RET + +%%vertical_overlap: + or t0d, 2 + mov r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ + sby, see, unused1, unused2, unused3, lstride +%endif + + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul t0d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add t0d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and t0d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, t0d +%if ARCH_X86_32 + xor sbyd, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, luma + + mov r3m, seed + mov dstq, r0mp + mov lumaq, r9mp + mov wq, r4m + lea r3, [srcq+wq*2] + mov r1mp, r3 + lea r3, [dstq+wq*2] + mov r11mp, r3 + lea r3, [lumaq+wq*(2<<%2)] + mov r12mp, r3 +%if %3 + shl r10mp, 1 +%endif +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, unused3, unused4, unused5, luma, lstride + + mov lstrideq, r10mp +%if %3 + add lstrideq, lstrideq +%endif + mov lumaq, r9mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r10mp, r10 + mov r11mp, r11 + mov r12mp, r12 +%endif + neg wq +%if ARCH_X86_32 + mov r4m, wq +%endif + +%%loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m + xor t0d, t0d +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, top_offxy, unused2, luma, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, top_offxy, unused2, luma, lstride +%endif + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if %2 == 0 +%%loop_x_odd_v_overlap: +%endif +%if %3 == 0 +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)] +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_v_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r0, [rsp+mmsize*8+gprsize*1] ; top_offxy + movu m5, [grain_lutq+r0*2] +%else + movu m5, [grain_lutq+top_offxyq*2] +%endif + punpckhwd m7, m5, m3 + punpcklwd m5, m3 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif + REPX {paddd x, [PIC_ptr(pd_16)]}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif + REPX {psrad x, 5}, m7, m5 + packssdw m3, m5, m7 + pmaxsw m3, m8 + pminsw m3, m9 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m5, [grain_lutq+r0*2+16] +%else + movu m5, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m7, m5, m4 + punpcklwd m5, m4 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif + REPX {psrad x, 5}, m7, m5 + packssdw m4, m5, m7 + pmaxsw m4, m8 + pminsw m4, m9 + + ; src + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9mp +%endif + mova m5, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m5, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif +%if %2 + pavgw m5, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m7, m5, m0 + punpcklwd m5, m0 + REPX {pmaddwd x, m14}, m7, m5 + REPX {psrad x, 6}, m7, m5 + packssdw m5, m7 + punpckhwd m7, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m7, m6 + REPX {psrad x, 6}, m7, m6 + packssdw m6, m7 + pxor mzero, mzero + REPX {paddw x, m15}, m5, m6 + REPX {pmaxsw x, mzero}, m5, m6 + REPX {pminsw x, m10}, m5, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m5, m6 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m5, scalingq-1, r0, r5, 8, 1 + vpgatherdw m5, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m7, m5, scalingq-1, r10, r12, 8, 1 + vpgatherdw m5, m6, scalingq-1, r10, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m7, m5 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m5 + pmulhrsw m3, m7 + pmulhrsw m4, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + + dec hw + jle %%end_y_v_overlap +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 +%if %3 + jmp %%loop_y +%else + btc hd, 16 + jc %%loop_y +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)+4] + jmp %%loop_y_v_overlap +%endif + +%%end_y_v_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0mp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif + +%if %2 + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +%else + btc dword r8m, 2 + jc %%loop_x_hv_overlap + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, picptr, grain_lut + + mov t0d, [rsp+mmsize*8+gprsize*1] ; top_offxy + add offxyd, 16 + add t0d, 16 + mov [rsp+mmsize*8+gprsize*0], offxyd ; left_offxyd + mov [rsp+mmsize*8+gprsize*2], t0d ; topleft_offxyd + + DEFINE_ARGS dst, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m + xor t0d, t0d +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, top_offxy +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride +%endif + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if %3 == 0 +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)] +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] ; left_offxy + mov r0, [rsp+8*mmsize+1*gprsize] ; top_offxy + movd m5, [grain_lutq+r5*2] +%else + movd m5, [grain_lutq+left_offxyq*2] +%endif + movu m7, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+2*gprsize] + movu m4, [grain_lutq+r0*2] +%if %2 + pinsrw m5, [grain_lutq+r5*2], 2 +%else + movd m3, [grain_lutq+r5*2] +%endif +%else + movu m4, [grain_lutq+top_offxyq*2] +%if %2 + pinsrw m5, [grain_lutq+topleft_offxyq*2], 2 ; { left, _, top/left } +%else + movd m3, [grain_lutq+topleft_offxyq*2] +%endif +%endif +%if %2 == 0 + punpckldq m5, m3 +%endif + punpckldq m3, m7, m4 ; { cur0/1,top0/1,cur2/3,top2/3 } + punpcklwd m5, m3 ; { left/cur0,_/cur1,topleft/top0,_/top1 } +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %2 + movddup m0, [PIC_ptr(pw_23_22)] +%else + movddup m0, [PIC_ptr(pw_27_17_17_27)] +%endif +%else + pshufd m0, m15, q1010 +%endif + pmaddwd m5, m0 +%if %1 + paddd m5, [PIC_ptr(pd_16)] +%else + paddd m5, m14 +%endif + psrad m5, 5 + packssdw m5, m5 + pmaxsw m5, m8 + pminsw m5, m9 + shufps m5, m3, q3210 ; cur0/1,top0/1,cur2/3,top2/3 + shufps m3, m5, m7, q3220 ; cur0-7 post-h_filter + shufps m5, m4, q3231 ; top0-7 post-h_filter + + punpckhwd m7, m5, m3 + punpcklwd m5, m3 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m5, m7 +%else + REPX {paddd x, m14}, m5, m7 +%endif + REPX {psrad x, 5}, m5, m7 + packssdw m3, m5, m7 + pmaxsw m3, m8 + pminsw m3, m9 + + ; right half + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m0, [grain_lutq+r0*2+16] +%else + movu m0, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m1, m0, m4 + punpcklwd m0, m4 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m1, m0 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m1, m0 +%else + REPX {paddd x, m14}, m1, m0 +%endif + REPX {psrad x, 5}, m1, m0 + packssdw m4, m0, m1 + pmaxsw m4, m8 + pminsw m4, m9 + + ; src + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9mp +%endif + mova m6, [lumaq+ 0] + mova m5, [lumaq+(16<<%2)] +%if %2 + phaddw m6, [lumaq+16] + phaddw m5, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif +%if %2 + pavgw m6, mzero + pavgw m5, mzero +%endif + +%if %1 + punpckhwd m7, m6, m0 + punpcklwd m6, m0 + REPX {pmaddwd x, m14}, m7, m6 + REPX {psrad x, 6}, m7, m6 + packssdw m6, m7 + punpckhwd m7, m5, m1 + punpcklwd m5, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m7, m5 + REPX {psrad x, 6}, m7, m5 + packssdw m5, m7 + pxor mzero, mzero + REPX {paddw x, m15}, m6, m5 + REPX {pmaxsw x, mzero}, m6, m5 + REPX {pminsw x, m10}, m6, m5 ; clip_pixel() +%else + REPX {pand x, m10}, m6, m5 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m6, scalingq-1, r0, r5, 8, 1 + vpgatherdw m6, m5, scalingq-1, r0, r5, 8, 1 +%else +%if %3 == 0 + ; register shortage :) + push r12 +%endif + vpgatherdw m7, m6, scalingq-1, r2, r12, 8, 1 + vpgatherdw m6, m5, scalingq-1, r2, r12, 8, 1 +%if %3 == 0 + pop r12 +%endif +%endif + REPX {psrlw x, 8}, m7, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m6 + pmulhrsw m3, m7 + pmulhrsw m4, m6 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hw +%if %3 + jg %%loop_y_h_overlap +%else + jle %%end_y_hv_overlap + btc hd, 16 + jc %%loop_y_h_overlap +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)+4] + jmp %%loop_y_hv_overlap +%%end_y_hv_overlap: +%endif +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov dstmp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif +%if %2 + jmp %%loop_x_hv_overlap +%else + or dword r8m, 4 + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxy += 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%end_hv: + RET +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/filmgrain_avx2.asm b/third_party/dav1d/src/x86/filmgrain_avx2.asm new file mode 100644 index 0000000000..55445cf593 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain_avx2.asm @@ -0,0 +1,2107 @@ +; Copyright © 2019-2022, VideoLAN and dav1d authors +; Copyright © 2019-2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 +pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 +gen_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 +gen_shufB: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 +gen_shufC: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 +gen_shufD: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 +; note: the order of (some of) the following constants matter +pb_27_17: times 2 db 27, 17 +byte_blend: db 0, 0, 0, -1 +pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 +pb_17_27: times 2 db 17, 27 +pb_1: times 4 db 1 +pb_23_22: db 23, 22, 0, 32, 0, 32, 0, 32 +next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +fg_min: times 4 db 0 + times 4 db 16 +fg_max: times 4 db 255 + times 4 db 240 + times 4 db 235 +pd_m65536: dd -65536 +pw_8: times 2 dw 8 +pw_1024: times 2 dw 1024 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512 +pw_1: dw 1 + +%macro JMP_TABLE 2-* + %1_8bpc_%2_table: + %xdefine %%base %1_8bpc_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + %rep %0 - 2 + dd %%prefix %+ .ar%3 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444, avx2, 0, 1, 2, 3 + +SECTION .text + +INIT_YMM avx2 +cglobal generate_grain_y_8bpc, 2, 9, 8, buf, fg_data +%define base r4-generate_grain_y_8bpc_avx2_table + lea r4, [generate_grain_y_8bpc_avx2_table] + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r6d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movq xm4, [base+mul_bits] + movq xm5, [base+hmul_bits] + mov r7, -73*82 + mova xm6, [base+pb_mask] + sub bufq, r7 + vpbroadcastw xm7, [base+round+r6*2] + lea r6, [gaussian_sequence] + movsxd r5, [r4+r5*4] +.loop: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm3, xm2 ; 4 next output seeds + pshuflw xm0, xm3, q3333 + psrlw xm3, 5 + pand xm2, xm0, xm1 + movq r2, xm3 + psrlw xm3, xm2, 10 + por xm2, xm3 + pmullw xm2, xm4 + pmulhuw xm0, xm5 + movzx r3d, r2w + pshufb xm3, xm6, xm2 + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm0, xm2 + movd xm2, [r6+r3*2] + rorx r3, r2, 32 + por xm3, xm0 + shr r2d, 16 + pinsrw xm2, [r6+r2*2], 1 + pshuflw xm0, xm3, q3333 + movzx r2d, r3w + psrlw xm3, 5 + pinsrw xm2, [r6+r2*2], 2 + shr r3d, 16 + movq r2, xm3 + pinsrw xm2, [r6+r3*2], 3 + movzx r3d, r2w + pinsrw xm2, [r6+r3*2], 4 + rorx r3, r2, 32 + shr r2d, 16 + pinsrw xm2, [r6+r2*2], 5 + movzx r2d, r3w + pinsrw xm2, [r6+r2*2], 6 + shr r3d, 16 + pinsrw xm2, [r6+r3*2], 7 + pmulhrsw xm2, xm7 + packsswb xm2, xm2 + movq [bufq+r7], xm2 + add r7, 8 + jl .loop + + ; auto-regression code + add r5, r4 + jmp r5 + +.ar1: + DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd xm5, [fg_dataq+FGData.ar_coeffs_y] + mova xm2, [base+gen_shufC] + DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 + pinsrb xm5, [base+pb_1], 3 + vpbroadcastw xm3, [base+round_vals+shiftq*2-12] ; rnd + pmovsxbw xm5, xm5 + pshufd xm4, xm5, q0000 + pshufd xm5, xm5, q1111 + sub bufq, 82*73-(82*3+79) + mov hd, 70 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -76 + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + pmovsxbw xm1, [bufq+xq-82-3] + pshufb xm0, xm1, xm2 + punpckhwd xm1, xm3 + pmaddwd xm0, xm4 + pmaddwd xm1, xm5 + paddd xm0, xm1 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + movsx val0d, byte [bufq+xq] + sarx val3d, val3d, shiftd + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xb, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 +.x_loop_ar1_end: + add bufq, 82 + dec hd + jg .y_loop_ar1 +.ar0: + RET + +.ar2: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign xmm_regs_used 16 + %assign stack_size_padded 168 + SUB rsp, stack_size_padded + movaps [rsp+16*2], xmm8 + movaps [rsp+16*3], xmm9 + movaps [rsp+16*4], xmm10 + movaps [rsp+16*5], xmm11 + movaps [rsp+16*6], xmm12 + movaps [rsp+16*7], xmm13 + movaps [rsp+16*8], xmm14 + movaps [rsp+16*9], xmm15 +%endif + DEFINE_ARGS buf, fg_data, h, x + mov r6d, [fg_dataq+FGData.ar_coeff_shift] + pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 + movd xm9, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 + vpbroadcastd xm10, [base+round_vals-14+r6*2] + movd xm11, [base+byte_blend+1] + pmovsxbw xm9, xm9 + pshufd xm4, xm7, q0000 + mova xm12, [base+gen_shufA] + pshufd xm5, xm7, q3333 + mova xm13, [base+gen_shufB] + pshufd xm6, xm7, q1111 + mova xm14, [base+gen_shufC] + pshufd xm7, xm7, q2222 + mova xm15, [base+gen_shufD] + pshufd xm8, xm9, q0000 + psrld xm10, 16 + pshufd xm9, xm9, q1111 + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar2: + mov xq, -76 +.x_loop_ar2: + pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pshufb xm2, xm0, xm12 + pmaddwd xm2, xm4 + pshufb xm3, xm1, xm13 + pmaddwd xm3, xm5 + paddd xm2, xm3 + pshufb xm3, xm0, xm14 + pmaddwd xm3, xm6 + punpckhqdq xm0, xm0 + punpcklwd xm0, xm1 + pmaddwd xm0, xm7 + pshufb xm1, xm15 + pmaddwd xm1, xm8 + paddd xm2, xm10 + paddd xm2, xm3 + paddd xm0, xm1 + paddd xm2, xm0 + movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pmovsxbw xm1, xm0 + pmaddwd xm3, xm9, xm1 + psrldq xm1, 4 ; y=0,x=0 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + paddw xm3, xm1 + packsswb xm3, xm3 + pextrb [bufq+xq], xm3, 0 + pslldq xm3, 2 + vpblendvb xm0, xm3, xm11 + psrldq xm0, 1 + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 +.x_loop_ar2_end: + add bufq, 82 + dec hd + jg .y_loop_ar2 + RET + +INIT_YMM avx2 +.ar3: +%if WIN64 + ; xmm6 and xmm7 already saved + %assign stack_offset 16 + ALLOC_STACK 16*14 + %assign stack_size stack_size - 16*4 + %assign xmm_regs_used 12 + movaps [rsp+16*12], xmm8 + movaps [rsp+16*13], xmm9 + movaps [rsp+16*14], xmm10 + movaps [rsp+16*15], xmm11 +%else + ALLOC_STACK 16*12 +%endif + mov r6d, [fg_dataq+FGData.ar_coeff_shift] + movd xm11, [base+byte_blend] + pmovsxbw m1, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pshufd m0, m1, q0000 + mova [rsp+16* 0], m0 + pshufd m0, m1, q1111 + mova [rsp+16* 2], m0 + pshufd m0, m1, q2222 + mova [rsp+16* 4], m0 + pshufd m1, m1, q3333 + mova [rsp+16* 6], m1 + pshufd xm0, xm2, q0000 + mova [rsp+16* 8], xm0 + pshufd xm0, xm2, q1111 + mova [rsp+16* 9], xm0 + psrldq xm7, xm2, 10 + mova m8, [base+gen_shufA] + pinsrw xm2, [base+pw_1], 5 + mova m9, [base+gen_shufC] + pshufd xm2, xm2, q2222 + movu m10, [base+gen_shufE] + vpbroadcastw xm6, [base+round_vals-12+r6*2] + pinsrw xm7, [base+round_vals+r6*2-10], 3 + mova [rsp+16*10], xm2 + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 +.x_loop_ar3: + movu xm5, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + vinserti128 m5, [bufq+xq-82*2-3], 1 ; y=-2,x=[-3,+12] + movu xm4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + punpcklbw m3, m5, m5 + punpckhwd m5, m4 + psraw m3, 8 + punpcklbw m5, m5 + psraw m5, 8 + punpcklbw xm4, xm4 + psraw xm4, 8 + pshufb m0, m3, m8 + pmaddwd m0, [rsp+16*0] + pshufb m1, m3, m9 + pmaddwd m1, [rsp+16*2] + shufps m2, m3, m5, q1032 + paddd m0, m1 + pshufb m1, m2, m8 + vperm2i128 m3, m4, 0x21 + pmaddwd m1, [rsp+16*4] + shufps xm2, xm3, q1021 + vpblendd m2, m3, 0xf0 + pshufb m2, m10 + paddd m0, m1 + pmaddwd m2, [rsp+16*6] + pshufb xm1, xm4, xm9 + pmaddwd xm1, [rsp+16*8] + shufps xm4, xm5, q1132 + paddd m0, m2 + pshufb xm2, xm4, xm8 + pshufd xm4, xm4, q2121 + pmaddwd xm2, [rsp+16*9] + punpcklwd xm4, xm6 + pmaddwd xm4, [rsp+16*10] + vextracti128 xm3, m0, 1 + paddd xm0, xm1 + movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] + paddd xm2, xm4 + paddd xm0, xm2 + paddd xm0, xm3 +.x_loop_ar3_inner: + pmovsxbw xm2, xm1 + pmaddwd xm2, xm7 + pshufd xm3, xm2, q1111 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + packsswb xm2, xm2 + pextrb [bufq+xq], xm2, 0 + pslldq xm2, 3 + vpblendvb xm1, xm2, xm11 + psrldq xm1, 1 + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82 + dec hd + jg .y_loop_ar3 + RET + +%macro GEN_GRAIN_UV_FN 3 ; ss_name, ss_x, ss_y +INIT_XMM avx2 +cglobal generate_grain_uv_%1_8bpc, 4, 10, 16, buf, bufy, fg_data, uv +%define base r4-generate_grain_uv_%1_8bpc_avx2_table + lea r4, [generate_grain_uv_%1_8bpc_avx2_table] + vpbroadcastw xm0, [fg_dataq+FGData.seed] + mov r6d, [fg_dataq+FGData.grain_scale_shift] + movq xm1, [base+next_upperbit_mask] + movq xm4, [base+mul_bits] + movq xm5, [base+hmul_bits] + mova xm6, [base+pb_mask] + vpbroadcastw xm7, [base+round+r6*2] + vpbroadcastd xm2, [base+pw_seed_xor+uvq*4] + pxor xm0, xm2 + lea r6, [gaussian_sequence] +%if %2 + mov r7d, 73-35*%3 + add bufq, 44 +.loop_y: + mov r5, -44 +%else + mov r5, -73*82 + sub bufq, r5 +%endif +.loop: + pand xm2, xm0, xm1 + psrlw xm3, xm2, 10 + por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw xm2, xm4 ; bits 0x0f00 are set + pmulhuw xm0, xm5 + pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds + psllq xm2, xm3, 30 + por xm2, xm3 + psllq xm3, xm2, 15 + por xm2, xm0 ; aggregate each bit into next seed's high bit + por xm2, xm3 ; 4 next output seeds + pshuflw xm0, xm2, q3333 + psrlw xm2, 5 + movq r8, xm2 + movzx r9d, r8w + movd xm2, [r6+r9*2] + rorx r9, r8, 32 + shr r8d, 16 + pinsrw xm2, [r6+r8*2], 1 + movzx r8d, r9w + pinsrw xm2, [r6+r8*2], 2 + shr r9d, 16 + pinsrw xm2, [r6+r9*2], 3 + pmulhrsw xm2, xm7 + packsswb xm2, xm2 + movd [bufq+r5], xm2 + add r5, 4 + jl .loop +%if %2 + add bufq, 82 + dec r7d + jg .loop_y +%endif + + ; auto-regression code + movsxd r6, [fg_dataq+FGData.ar_coeff_lag] + movsxd r6, [base+generate_grain_uv_%1_8bpc_avx2_table+r6*4] + add r6, r4 + jmp r6 + +INIT_YMM avx2 +.ar0: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq] + movd xm3, [base+hmul_bits+shiftq*2] + DEFINE_ARGS buf, bufy, h + pmovsxbw xm2, xm2 +%if %2 + vpbroadcastd m7, [base+pb_1] + vpbroadcastw m6, [base+hmul_bits+2+%3*2] +%endif + vpbroadcastw m2, xm2 + vpbroadcastw m3, xm3 + pxor m12, m12 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif + add bufyq, 3+82*3 + mov hd, 70-35*%3 +.y_loop_ar0: +%if %2 + ; first 32 pixels + movu xm4, [bufyq] + vinserti128 m4, [bufyq+32], 1 +%if %3 + movu xm0, [bufyq+82] + vinserti128 m0, [bufyq+82+32], 1 +%endif + movu xm5, [bufyq+16] + vinserti128 m5, [bufyq+48], 1 +%if %3 + movu xm1, [bufyq+82+16] + vinserti128 m1, [bufyq+82+48], 1 +%endif + pmaddubsw m4, m7, m4 +%if %3 + pmaddubsw m0, m7, m0 +%endif + pmaddubsw m5, m7, m5 +%if %3 + pmaddubsw m1, m7, m1 + paddw m4, m0 + paddw m5, m1 +%endif + pmulhrsw m4, m6 + pmulhrsw m5, m6 +%else + xor r3d, r3d + ; first 32x2 pixels +.x_loop_ar0: + movu m4, [bufyq+r3] + pcmpgtb m0, m12, m4 + punpckhbw m5, m4, m0 + punpcklbw m4, m0 +%endif + pmullw m4, m2 + pmullw m5, m2 + pmulhrsw m4, m3 + pmulhrsw m5, m3 +%if %2 + movu m1, [bufq] +%else + movu m1, [bufq+r3] +%endif + pcmpgtb m8, m12, m1 + punpcklbw m0, m1, m8 + punpckhbw m1, m8 + paddw m0, m4 + paddw m1, m5 + packsswb m0, m1 +%if %2 + movu [bufq], m0 +%else + movu [bufq+r3], m0 + add r3d, 32 + cmp r3d, 64 + jl .x_loop_ar0 +%endif + + ; last 6/12 pixels + movu xm4, [bufyq+32*2] +%if %2 +%if %3 + movu xm5, [bufyq+32*2+82] +%endif + pmaddubsw xm4, xm7, xm4 +%if %3 + pmaddubsw xm5, xm7, xm5 + paddw xm4, xm5 +%endif + movq xm0, [bufq+32] + pmulhrsw xm4, xm6 + pmullw xm4, xm2 + pmulhrsw xm4, xm3 + pcmpgtb xm5, xm12, xm0 + punpcklbw xm5, xm0, xm5 + paddw xm4, xm5 + packsswb xm4, xm4 + pblendw xm0, xm4, xm0, 1000b + movq [bufq+32], xm0 +%else + movu xm0, [bufq+64] + pcmpgtb xm1, xm12, xm4 + punpckhbw xm5, xm4, xm1 + punpcklbw xm4, xm1 + pmullw xm5, xm2 + pmullw xm4, xm2 + vpblendd xm1, xm3, xm12, 0x0c + pmulhrsw xm5, xm1 + pmulhrsw xm4, xm3 + pcmpgtb xm1, xm12, xm0 + punpckhbw xm8, xm0, xm1 + punpcklbw xm0, xm1 + paddw xm5, xm8 + paddw xm0, xm4 + packsswb xm0, xm5 + movu [bufq+64], xm0 +%endif + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar0 + RET + +INIT_XMM avx2 +.ar1: + DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] + pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 + DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift + pmovsxbw xm4, xm4 + pshufd xm5, xm4, q1111 + pshufd xm4, xm4, q0000 + pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 + vpbroadcastd xm7, [base+pb_1] + vpbroadcastw xm6, [base+hmul_bits+2+%3*2] +%endif + vpbroadcastd xm3, xm3 +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + pmovsxbw xm0, [bufq+xq-82-1] ; top/left +%if %2 + movq xm8, [bufyq+xq*2] +%if %3 + movq xm9, [bufyq+xq*2+82] +%endif +%endif + psrldq xm2, xm0, 2 ; top + psrldq xm1, xm0, 4 ; top/right +%if %2 + pmaddubsw xm8, xm7, xm8 +%if %3 + pmaddubsw xm9, xm7, xm9 + paddw xm8, xm9 +%endif + pmulhrsw xm8, xm6 +%else + pmovsxbw xm8, [bufyq+xq] +%endif + punpcklwd xm0, xm2 + punpcklwd xm1, xm8 + pmaddwd xm0, xm4 + pmaddwd xm1, xm5 + paddd xm0, xm1 + paddd xm0, xm3 +.x_loop_ar1_inner: + movd val0d, xm0 + psrldq xm0, 4 + imul val3d, cf3d + add val3d, val0d + sarx val3d, val3d, shiftd + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar1 + RET + +.ar2: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + vpbroadcastw xm13, [base+round_vals-12+shiftq*2] + pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 + pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 + pinsrw xm0, [base+pw_1], 5 +%if %2 + vpbroadcastw xm12, [base+hmul_bits+2+%3*2] + vpbroadcastd xm11, [base+pb_1] +%endif + DEFINE_ARGS buf, bufy, fg_data, h, unused, x + pshufd xm4, xm7, q0000 + pshufd xm5, xm7, q3333 + pshufd xm6, xm7, q1111 + pshufd xm7, xm7, q2222 + pshufd xm8, xm0, q0000 + pshufd xm9, xm0, q1111 + pshufd xm10, xm0, q2222 +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pshufb xm2, xm0, [base+gen_shufA] + pmaddwd xm2, xm4 + pshufb xm3, xm1, [base+gen_shufB] + pmaddwd xm3, xm5 + paddd xm2, xm3 + pshufb xm3, xm0, [base+gen_shufC] + pmaddwd xm3, xm6 + punpckhqdq xm0, xm0 ; y=-2,x=[+2,+5] + punpcklwd xm0, xm1 + pmaddwd xm0, xm7 + pshufb xm1, [gen_shufD] + pmaddwd xm1, xm8 + paddd xm2, xm3 + paddd xm0, xm1 + paddd xm2, xm0 + +%if %2 + movq xm0, [bufyq+xq*2] +%if %3 + movq xm3, [bufyq+xq*2+82] +%endif + pmaddubsw xm0, xm11, xm0 +%if %3 + pmaddubsw xm3, xm11, xm3 + paddw xm0, xm3 +%endif + pmulhrsw xm0, xm12 +%else + pmovsxbw xm0, [bufyq+xq] +%endif + punpcklwd xm0, xm13 + pmaddwd xm0, xm10 + paddd xm2, xm0 + + movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pmovsxbw xm0, xm0 + pmaddwd xm3, xm0, xm9 + psrldq xm0, 2 + paddd xm3, xm2 + psrldq xm2, 4 ; shift top to next pixel + psrad xm3, [fg_dataq+FGData.ar_coeff_shift] + pslldq xm3, 2 + paddw xm3, xm0 + pblendw xm0, xm3, 00000010b + packsswb xm0, xm0 + pextrb [bufq+xq], xm0, 1 + inc xq + jz .x_loop_ar2_end + test xb, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar2 + RET + +INIT_YMM avx2 +.ar3: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + pmovsxbw m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15 + pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-23 + vpbroadcastb xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq+24] ; cf24 [luma] + movd xm13, [base+round_vals-10+shiftq*2] + vpbroadcastd xm14, [base+round_vals-14+shiftq*2] + pshufd m6, m0, q0000 + pshufd m7, m0, q1111 + pshufd m8, m0, q2222 + pshufd m9, m0, q3333 + pshufd xm10, xm1, q0000 + pshufd xm11, xm1, q1111 + pshufhw xm12, xm1, q0000 + psraw xm2, 8 + palignr xm13, xm1, 10 + punpckhwd xm12, xm2 ; interleave luma cf + psrld xm14, 16 + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + vpbroadcastw xm15, [base+hmul_bits+2+%3*2] + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) +.x_loop_ar3: + vbroadcasti128 m3, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12 + palignr xm1, xm3, [bufq+xq-82*3-9], 6 ; y=-3,x=[-3,+12] + vbroadcasti128 m4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + vpblendd m3, m1, 0x0f + pxor m0, m0 + pcmpgtb m2, m0, m3 + pcmpgtb m0, m4 + punpcklbw m1, m3, m2 + punpckhbw m3, m2 + punpcklbw m2, m4, m0 + punpckhbw xm4, xm0 + pshufb m0, m1, [base+gen_shufA] + pmaddwd m0, m6 + pshufb m5, m1, [base+gen_shufC] + pmaddwd m5, m7 + shufps m1, m3, q1032 + paddd m0, m5 + pshufb m5, m1, [base+gen_shufA] + pmaddwd m5, m8 + shufps xm1, xm3, q2121 + vpblendd m1, m2, 0xf0 + pshufb m1, [base+gen_shufE] + pmaddwd m1, m9 + paddd m0, m5 + pshufb xm3, xm2, [base+gen_shufC] + paddd m0, m1 + pmaddwd xm3, xm10 + palignr xm1, xm4, xm2, 2 + punpckhwd xm1, xm2, xm1 + pmaddwd xm1, xm11 + palignr xm4, xm2, 12 + paddd xm3, xm1 +%if %2 + vpbroadcastd xm5, [base+pb_1] + movq xm1, [bufyq+xq*2] + pmaddubsw xm1, xm5, xm1 +%if %3 + movq xm2, [bufyq+xq*2+82] + pmaddubsw xm5, xm2 + paddw xm1, xm5 +%endif + pmulhrsw xm1, xm15 +%else + pmovsxbw xm1, [bufyq+xq] +%endif + punpcklwd xm4, xm1 + pmaddwd xm4, xm12 + movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] + vextracti128 xm2, m0, 1 + paddd xm0, xm14 + paddd xm3, xm4 + paddd xm0, xm3 + paddd xm0, xm2 +.x_loop_ar3_inner: + pmovsxbw xm1, xm1 + pmaddwd xm2, xm13, xm1 + pshuflw xm3, xm2, q1032 + paddd xm2, xm0 ; add top + paddd xm2, xm3 ; left+cur + psrldq xm0, 4 + psrad xm2, [fg_dataq+FGData.ar_coeff_shift] + psrldq xm1, 2 + ; don't packssdw, we only care about one value + punpckldq xm2, xm2 + pblendw xm1, xm2, 0100b + packsswb xm1, xm1 + pextrb [bufq+xq], xm1, 2 + inc xq + jz .x_loop_ar3_end + test xb, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 +.x_loop_ar3_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar3 + RET +%endmacro + +INIT_YMM avx2 +cglobal fgy_32x32xn_8bpc, 6, 13, 15, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, see, overlap +%define base r9-pd_m65536 + lea r9, [pd_m65536] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] + vpbroadcastd m8, [base+pd_m65536] + vpbroadcastw m9, [base+mul_bits+r6*2-14] + vpbroadcastd m10, [base+fg_min+r7*4] + vpbroadcastd m11, [base+fg_max+r7*8] + vpbroadcastd m12, [base+pw_1024] + movq xm13, [base+pb_27_17_17_27] + test sbyd, sbyd + setnz r7b + pxor m7, m7 + test r7b, overlapb + jnz .vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, overlap + + mov hd, hm + mov grain_lutq, grain_lutmp +.loop_y: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq] + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hd + jg .loop_y + + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + test overlapd, overlapd + jz .loop_x + + ; r8m = sbym + cmp dword r8m, 0 + jne .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy + + lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm5 + pmaddubsw xm4, xm13, xm4 + pmulhrsw xm4, xm12 + packsswb xm4, xm4 + vpblendd m4, m5, 0xfe + punpckhbw m5, m7 + punpcklbw m4, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hd + jg .loop_y_h_overlap + + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + + ; r8m = sbym + cmp dword r8m, 0 + jne .loop_x_hv_overlap + jmp .loop_x_h_overlap + +.vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused, sby, see, overlap + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq + +.loop_x_v_overlap: + vpbroadcastd m14, [pb_27_17] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_v_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] + punpcklbw m5, m4, m6 + punpckhbw m4, m6 + pmaddubsw m5, m14, m5 + pmaddubsw m4, m14, m4 + pmulhrsw m5, m12 + pmulhrsw m4, m12 + packsswb m5, m4 + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hb + jz .end_y_v_overlap + vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_v_overlap + jmp .loop_y +.end_y_v_overlap: + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +.loop_x_hv_overlap: + vpbroadcastd m14, [pb_27_17] + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+32] + lea left_offxyd, [offyq+32] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +.loop_y_hv_overlap: + ; src + mova m2, [srcq] + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + + ; scaling[src] + pandn m4, m8, m0 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, m0, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m1 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + pblendw m2, m4, 0xaa + psrld m4, m1, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m4-2], m6 + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] + movu m6, [grain_lutq+offxyq] + movd xm7, [grain_lutq+left_offxyq] + movu m4, [grain_lutq+top_offxyq] + movd xm5, [grain_lutq+topleft_offxyq] + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw xm7, xm6 + punpcklbw xm5, xm4 + pmaddubsw xm7, xm13, xm7 + pmaddubsw xm5, xm13, xm5 + pmulhrsw xm7, xm12 + pmulhrsw xm5, xm12 + packsswb xm7, xm7 + packsswb xm5, xm5 + vpblendd m7, m6, 0xfe + vpblendd m5, m4, 0xfe + ; followed by v interpolation (top | cur -> cur) + punpckhbw m4, m6 + punpcklbw m5, m7 + pmaddubsw m4, m14, m4 + pmaddubsw m5, m14, m5 + pmulhrsw m4, m12 + pmulhrsw m5, m12 + pxor m7, m7 + packsswb m5, m4 + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 + mova [dstq+srcq], m0 + + add srcq, strideq + add grain_lutq, 82 + dec hb + jz .end_y_hv_overlap + vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + add hd, 0x80000000 + jnc .loop_y_hv_overlap + jmp .loop_y_h_overlap +.end_y_hv_overlap: + add wq, 32 + lea srcq, [src_bakq+wq] + jl .loop_x_hv_overlap +.end: + RET + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, overlap, uv_pl, is_id +%define base r11-pd_m65536 + lea r11, [pd_m65536] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov r9d, is_idm + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] + vpbroadcastd m8, [base+pd_m65536] + vpbroadcastw m9, [base+mul_bits+r6*2-14] + vpbroadcastd m10, [base+fg_min+r7*4] + shlx r7d, r7d, r9d + vpbroadcastd m11, [base+fg_max+r7*4] + vpbroadcastd m12, [base+pw_1024] + pxor m7, m7 + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, sby, see, overlap, uv_pl +%if %1 + mov r6d, uv_plm + vpbroadcastd m0, [base+pw_8] + vbroadcasti128 m14, [fg_dataq+FGData.uv_mult+r6*4] + vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] + pshufb m14, m0 ; uv_luma_mult, uv_mult +%elif %2 + vpbroadcastq m15, [base+pb_23_22] +%else + vpbroadcastq xm15, [base+pb_27_17_17_27] +%endif +%if %3 + vpbroadcastw m13, [base+pb_23_22] +%elif %2 + pshufd m13, [base+pb_27_17], q0000 ; 8x27_17, 8x17_27 +%endif + test r7b, overlapb + jnz %%vertical_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused2, unused3, see, overlap, unused4, unused5, lstride + + mov lumaq, r9mp + lea r12, [srcq+wq] + lea r13, [dstq+wq] + lea r14, [lumaq+wq*(1+%2)] + mov r11mp, r12 + mov r12mp, r13 + mov lstrideq, r10mp + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, unused1, unused2, lstride + + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, unused1, unused2, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3) +0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm5, [grain_lutq+offxyq+ 0] + vinserti128 m5, [grain_lutq+offxyq+82], 1 +%else + movu m5, [grain_lutq+offxyq] +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 + jg %%loop_y + + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + test overlapd, overlapd + jz %%loop_x + + ; r8m = sbym + cmp dword r8m, 0 + jne %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm5, [grain_lutq+offxyq+ 0] + vinserti128 m5, [grain_lutq+offxyq+82], 1 + movd xm4, [grain_lutq+left_offxyq+ 0] + vinserti128 m4, [grain_lutq+left_offxyq+82], 1 + punpcklbw m4, m5 +%if %1 + vpbroadcastq m0, [pb_23_22] + pmaddubsw m4, m0, m4 +%else + pmaddubsw m4, m15, m4 +%endif + pmulhrsw m4, m12 + packsswb m4, m4 + vpblendd m4, m5, 0xee +%else + movu m5, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm5 +%if %1 + movq xm0, [pb_27_17_17_27] + pmaddubsw xm4, xm0, xm4 +%else + pmaddubsw xm4, xm15, xm4 +%endif + pmulhrsw xm4, xm12 + packsswb xm4, xm4 + vpblendd m4, m5, 0xfe +%endif + punpckhbw m5, m7 + punpcklbw m4, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(1+%2) + sub hb, 1+%2 + jg %%loop_y_h_overlap + + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + + ; r8m = sbym + cmp dword r8m, 0 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap + +%%vertical_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ + sby, see, overlap, unused1, unused2, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused1, unused2, see, overlap, unused3, unused4, lstride + + mov lumaq, r9mp + lea r12, [srcq+wq] + lea r13, [dstq+wq] + lea r14, [lumaq+wq*(1+%2)] + mov r11mp, r12 + mov r12mp, r13 + mov lstrideq, r10mp + neg wq + +%%loop_x_v_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, top_offxy, unused, lstride + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy, unused, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%if %2 == 0 + vpbroadcastd m13, [pb_27_17] +%endif +%%loop_y_v_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %3 == 0 +%if %2 + movu xm0, [grain_lutq+offxyq] + vinserti128 m0, [grain_lutq+offxyq+82], 1 + movu xm4, [grain_lutq+top_offxyq] + vinserti128 m4, [grain_lutq+top_offxyq+82], 1 +%else + movu m0, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] +%endif + punpcklbw m5, m4, m0 + punpckhbw m4, m0 + pmaddubsw m5, m13, m5 + pmaddubsw m4, m13, m4 + pmulhrsw m5, m12 + pmulhrsw m4, m12 + packsswb m5, m4 +%else + movq xm4, [grain_lutq+offxyq] + vinserti128 m4, [grain_lutq+offxyq+8], 1 + movq xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+8], 1 + punpcklbw m5, m4 + pmaddubsw m5, m13, m5 + pmulhrsw m5, m12 + vextracti128 xm4, m5, 1 + packsswb xm5, xm4 + ; only interpolate first line, insert second line unmodified + vinserti128 m5, [grain_lutq+offxyq+82], 1 +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma_source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + + sub hb, 1+%2 + jle %%end_y_v_overlap +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 +%if %2 == 0 + vpbroadcastd m13, [pb_17_27] + add hd, 0x80000000 + jnc %%loop_y_v_overlap +%endif + jmp %%loop_y + +%%end_y_v_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap + +%%loop_x_hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride + + lea topleft_offxyd, [top_offxyq+(32>>%2)] + lea left_offxyd, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 +%if %2 == 0 + vpbroadcastd m13, [pb_27_17] +%endif +%%loop_y_hv_overlap: + ; src +%if %2 + mova xm3, [lumaq+lstrideq*0+ 0] + vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 + vpbroadcastd m2, [pb_1] + mova xm0, [lumaq+lstrideq*0+16] + vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 + mova xm1, [srcq] + vinserti128 m1, [srcq+strideq], 1 + pmaddubsw m3, m2 + pmaddubsw m0, m2 + pavgw m3, m7 + pavgw m0, m7 +%else + mova m2, [lumaq] + mova m1, [srcq] +%endif +%if %1 +%if %2 + packuswb m2, m3, m0 ; luma +%endif + punpckhbw m3, m2, m1 + punpcklbw m2, m1 ; { luma, chroma } + pmaddubsw m3, m14 + pmaddubsw m2, m14 + psraw m3, 6 + psraw m2, 6 + paddw m3, m15 + paddw m2, m15 + packuswb m2, m3 ; pack+unpack = clip +%endif +%if %1 || %2 == 0 + punpcklbw m3, m2, m7 + punpckhbw m0, m2, m7 +%endif + + ; scaling[luma_src] + pandn m4, m8, m3 + mova m6, m8 + vpgatherdd m2, [scalingq+m4-0], m8 + psrld m3, 16 + mova m8, m6 + vpgatherdd m4, [scalingq+m3-2], m6 + pandn m5, m8, m0 + mova m6, m8 + vpgatherdd m3, [scalingq+m5-0], m8 + psrld m0, 16 + mova m8, m6 + vpgatherdd m5, [scalingq+m0-2], m6 + pblendw m2, m4, 0xaa + pblendw m3, m5, 0xaa + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movu xm4, [grain_lutq+offxyq] + vinserti128 m4, [grain_lutq+offxyq+82], 1 + movd xm0, [grain_lutq+left_offxyq] + vinserti128 m0, [grain_lutq+left_offxyq+82], 1 + movd xm6, [grain_lutq+topleft_offxyq] +%if %3 + movq xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+8], 1 +%else + vinserti128 m6, [grain_lutq+topleft_offxyq+82], 1 + movu xm5, [grain_lutq+top_offxyq] + vinserti128 m5, [grain_lutq+top_offxyq+82], 1 +%endif + + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m0, m4 +%if %3 + punpcklbw xm6, xm5 +%else + punpcklbw m6, m5 +%endif + punpcklqdq m0, m6 +%if %1 + vpbroadcastq m6, [pb_23_22] + pmaddubsw m0, m6, m0 +%else + pmaddubsw m0, m15, m0 +%endif + pmulhrsw m0, m12 + packsswb m0, m0 + vpblendd m4, m0, 0x11 +%if %3 + pshuflw xm0, xm0, q1032 + vpblendd m5, m0, 0x01 +%else + pshuflw m0, m0, q1032 + vpblendd m5, m0, 0x11 +%endif +%else + movu m4, [grain_lutq+offxyq] + movd xm0, [grain_lutq+left_offxyq] + movu m5, [grain_lutq+top_offxyq] + movd xm6, [grain_lutq+topleft_offxyq] + punpcklbw xm0, xm4 + punpcklbw xm6, xm5 + punpcklqdq xm0, xm6 +%if %1 + vpbroadcastq xm6, [pb_27_17_17_27] + pmaddubsw xm0, xm6, xm0 +%else + pmaddubsw xm0, xm15, xm0 +%endif + pmulhrsw xm0, xm12 + packsswb xm0, xm0 + vpblendd m4, m0, 0x01 + pshuflw xm0, xm0, q1032 + vpblendd m5, m0, 0x01 +%endif + + ; followed by v interpolation (top | cur -> cur) +%if %3 + vpermq m0, m4, q3120 + punpcklbw m5, m0 + pmaddubsw m5, m13, m5 + pmulhrsw m5, m12 + vextracti128 xm0, m5, 1 + packsswb xm5, xm0 + vpblendd m5, m4, 0xf0 +%else + punpckhbw m0, m5, m4 + punpcklbw m5, m4 + pmaddubsw m4, m13, m0 + pmaddubsw m5, m13, m5 + pmulhrsw m4, m12 + pmulhrsw m5, m12 + packsswb m5, m4 +%endif + punpcklbw m4, m5, m7 + punpckhbw m5, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmaddubsw m2, m4 + pmaddubsw m3, m5 + pmulhrsw m2, m9 + pmulhrsw m3, m9 + + ; unpack chroma source + punpcklbw m0, m1, m7 + punpckhbw m1, m7 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + pmaxub m0, m10 + pminub m0, m11 +%if %2 + mova [dstq], xm0 + vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif + +%if %2 + lea srcq, [srcq+strideq*2] + lea dstq, [dstq+strideq*2] + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 +%if %2 + jg %%loop_y_h_overlap +%else + je %%end_y_hv_overlap + vpbroadcastd m13, [pb_17_27] + add hd, 0x80000000 + jnc %%loop_y_hv_overlap + jmp %%loop_y_h_overlap +%endif + +%%end_y_hv_overlap: + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r14+wq*(1+%2)] + add srcq, wq + add dstq, wq + jmp %%loop_x_hv_overlap +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +GEN_GRAIN_UV_FN 420, 1, 1 +FGUV_FN 420, 1, 1 +GEN_GRAIN_UV_FN 422, 1, 0 +FGUV_FN 422, 1, 0 +GEN_GRAIN_UV_FN 444, 0, 0 +FGUV_FN 444, 0, 0 + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/filmgrain_avx512.asm b/third_party/dav1d/src/x86/filmgrain_avx512.asm new file mode 100644 index 0000000000..317ec118b3 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain_avx512.asm @@ -0,0 +1,813 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +pb_even: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 + db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94 + db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126 +pb_odd: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 + db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95 + db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127 +interleave_hl: db 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7 +pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 +pb_23_22_0_32: db 23, 22, 0, 32, 0, 32, 0, 32 +pb_27_17: times 2 db 27, 17 +pb_23_22: times 2 db 23, 22 +pw_8: times 2 dw 8 +pw_1024: times 2 dw 1024 +pb_17_27: times 2 db 17, 27 +fg_max: times 4 db 255 + times 4 db 240 + times 4 db 235 +fg_min: times 4 db 0 + times 4 db 16 +noise_rnd: times 2 dw 128 + times 2 dw 64 + times 2 dw 32 + times 2 dw 16 + +SECTION .text + +INIT_ZMM avx512icl +cglobal fgy_32x32xn_8bpc, 6, 13, 22, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, see, overlap +%define base r11-fg_min + lea r11, [fg_min] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] + mov r12, 0x0000000f0000000f ; h_overlap mask + mova m0, [scalingq+64*0] + mova m1, [scalingq+64*1] + mova m2, [scalingq+64*2] + mova m3, [scalingq+64*3] + kmovq k1, r12 + vbroadcasti32x4 m4, [base+interleave_hl] + vpbroadcastd ym16, [base+pb_27_17] + vpbroadcastd m12, [base+pb_17_27] + vpbroadcastd m6, [base+noise_rnd+r6*4-32] + test sbyd, sbyd + setnz r6b + vpbroadcastd m7, [base+fg_min+r7*4] + vpbroadcastd m8, [base+fg_max+r7*8] + pxor m5, m5 + vpbroadcastd m9, [base+pw_1024] + vpbroadcastq m10, [base+pb_27_17_17_27] + vmovdqa64 m12{k1}, m16 + test r6b, overlapb + jnz .v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ + h, sby, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq +.loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offxd, [offyq+offxq*2+829] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ + h, sby, see, overlap + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y: + movu ym21, [grain_lutq+offxyq-82] + vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1 + call .add_noise + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + test overlapd, overlapd + jz .loop_x + test sbyd, sbyd + jnz .hv_overlap + +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ + h, sby, see, left_offxy + + rorx offyd, seed, 8 + mov left_offxyd, offxd ; previous column's offy*stride + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offxd, [offyq+offxq*2+829] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ + h, sby, see, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + movu ym20, [grain_lutq+offxyq-82] + vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1 + movd xm19, [grain_lutq+left_offxyq-50] + vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2 + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 + pmulhrsw m19, m9 + punpckhbw m21, m20, m5 + packsswb m20{k1}, m19, m19 + punpcklbw m20, m5, m20 + call .add_noise_h + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + test sbyd, sbyd + jnz .hv_overlap + jmp .loop_x_h_overlap + +.v_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, offy, offx, \ + h, sby, see, overlap + + movzx r6d, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, r6d, 173 * 0x00010001 + imul r6d, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add r6d, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and r6d, 0xff00ff00 + xor seed, r7d + xor seed, r6d ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ + h, sby, see, overlap + + lea src_bakq, [srcq+wq] + neg wq + sub dstq, srcq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offxd, [offyq+offxq*2+0x10001*829+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ + h, sby, see, overlap, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + movu ym19, [grain_lutq+offxyq-82] + vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 + movu ym21, [grain_lutq+top_offxyq-82] + vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 + punpckhbw m20, m21, m19 + punpcklbw m21, m19 + call .add_noise_v + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to .v_overlap, and instead always fall-through to h+v overlap +.hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ + h, sby, see, left_offxy, top_offxy, topleft_offxy + + mov topleft_offxyd, top_offxyd + rorx offyd, seed, 8 + mov left_offxyd, offxd + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offxd, [offyq+offxq*2+0x10001*829+32*82] + + DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ + h, sby, see, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + movu ym19, [grain_lutq+offxyq-82] + vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 + movd xm16, [grain_lutq+left_offxyq-50] + vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2 + movu ym21, [grain_lutq+top_offxyq-82] + vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 + movd xm17, [grain_lutq+topleft_offxyq-50] + vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2 + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m16, m19 + pmaddubsw m16, m10, m16 + punpcklbw m17, m21 + pmaddubsw m17, m10, m17 + punpckhbw m20, m21, m19 + pmulhrsw m16, m9 + pmulhrsw m17, m9 + packsswb m19{k1}, m16, m16 + packsswb m21{k1}, m17, m17 + ; followed by v interpolation (top | cur -> cur) + punpcklbw m21, m19 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: + pmaddubsw m20, m12, m20 + pmaddubsw m21, m12, m21 + pmulhrsw m20, m9 + pmulhrsw m21, m9 + packsswb m21, m20 +.add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +.add_noise_h: + mova ym18, [srcq+strideq*0] + vinserti32x8 m18, [srcq+strideq*1], 1 + mova m19, m0 + punpcklbw m16, m18, m5 + vpermt2b m19, m18, m1 ; scaling[ 0..127] + vpmovb2m k2, m18 + punpckhbw m17, m18, m5 + vpermi2b m18, m2, m3 ; scaling[128..255] + vmovdqu8 m19{k2}, m18 ; scaling[src] + pshufb m19, m4 + pmaddubsw m18, m19, m20 + pmaddubsw m19, m21 + add grain_lutq, 82*2 + pmulhrsw m18, m6 ; noise + pmulhrsw m19, m6 + paddw m16, m18 + paddw m17, m19 + packuswb m16, m17 + pmaxub m16, m7 + pminub m16, m8 + mova [dstq+srcq], ym16 + add srcq, strideq + vextracti32x8 [dstq+srcq], m16, 1 + add srcq, strideq + ret + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \ + scaling, grain_lut, h, sby, luma, \ + overlap, uv_pl, is_id, _, stride3 + lea r11, [fg_min] + mov r6d, [fg_dataq+FGData.scaling_shift] + mov r7d, [fg_dataq+FGData.clip_to_restricted_range] + mov r9d, is_idm + mov sbyd, sbym + mov overlapd, [fg_dataq+FGData.overlap_flag] +%if %2 + mov r12, 0x000f000f000f000f ; h_overlap mask + vpbroadcastq m10, [base+pb_23_22_0_32] + lea stride3q, [strideq*3] +%else + mov r12, 0x0000000f0000000f + vpbroadcastq m10, [base+pb_27_17_17_27] +%endif + mova m0, [scalingq+64*0] + mova m1, [scalingq+64*1] + mova m2, [scalingq+64*2] + mova m3, [scalingq+64*3] + kmovq k1, r12 + vbroadcasti32x4 m4, [base+interleave_hl] + vpbroadcastd m6, [base+noise_rnd+r6*4-32] + vpbroadcastd m7, [base+fg_min+r7*4] + shlx r7d, r7d, r9d + vpbroadcastd m8, [base+fg_max+r7*4] + test sbyd, sbyd + setnz r7b + vpbroadcastd m9, [base+pw_1024] + mova m11, [base+pb_even] + mova m12, [base+pb_odd] + pxor m5, m5 + mov r5, r10mp ; lstride + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ + h, sby, see, overlap, uv_pl, _, _, stride3 +%if %1 + mov r6d, uv_plm + vpbroadcastd m16, [base+pw_8] + vbroadcasti32x4 m14, [fg_dataq+FGData.uv_mult+r6*4] + vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] + pshufb m14, m16 ; uv_luma_mult, uv_mult +%endif + test r7b, overlapb + jnz %%v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + offx, offy, see, overlap, _, _, _, stride3 + + mov lumaq, r9mp + lea r11, [srcq+wq] + lea r12, [dstq+wq] + lea r13, [lumaq+wq*(1+%2)] + mov r11mp, r11 + mov r12mp, r12 + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + h, offxy, see, overlap, _, _, _, stride3 + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: +%if %2 + movu xm21, [grain_lutq+offxyq+82*0] + vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 + vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 + vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 +%else + movu ym21, [grain_lutq+offxyq+82*0] + vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 +%endif + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + test overlapd, overlapd + jz %%loop_x + cmp dword r8m, 0 ; sby + jne %%hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + offx, offy, see, left_offxy, _, _, _, stride3 + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + h, offxy, see, left_offxy, _, _, _, stride3 + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: +%if %2 + movu xm20, [grain_lutq+offxyq +82*0] + movd xm19, [grain_lutq+left_offxyq+82*0] + vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1 + vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1 + vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2 + vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3 +%else + movu ym20, [grain_lutq+offxyq + 0] + movd xm19, [grain_lutq+left_offxyq+ 0] + vinserti32x8 m20, [grain_lutq+offxyq +82], 1 + vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2 +%endif + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 + punpckhbw m21, m20, m5 + pmulhrsw m19, m9 + vpacksswb m20{k1}, m19, m19 + punpcklbw m20, m5, m20 + call %%add_noise_h + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + cmp dword r8m, 0 ; sby + jne %%hv_overlap + jmp %%loop_x_h_overlap + +%%v_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ + _, sby, see, overlap, _, _, _, stride3 + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + +%if %3 + vpbroadcastd m13, [base+pb_23_22] + kxnorw k3, k3, k3 ; v_overlap mask +%elif %2 + vbroadcasti32x8 m13, [base+pb_27_17] + kxnord k3, k3, k3 + pshufd m13, m13, q0000 ; 8x27_17, 8x17_27 +%else + vpbroadcastd ym16, [base+pb_27_17] + vpbroadcastd m13, [base+pb_17_27] + vmovdqa64 m13{k1}, m16 +%endif + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + offx, offy, see, overlap, top_offxy, _, _, stride3 + + mov lumaq, r9mp + lea r11, [srcq+wq] + lea r12, [dstq+wq] + lea r13, [lumaq+wq*(1<<%2)] + mov r11mp, r11 + mov r12mp, r12 + neg wq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0x000f000f + and offxd, 0x000f000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + h, offxy, see, overlap, top_offxy, _, _, stride3 + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + +%if %3 + movu xm18, [grain_lutq+offxyq+82*0] + movu xm20, [grain_lutq+top_offxyq+82*0] + ; only interpolate first line, insert remaining line unmodified + vbroadcasti128 ym21, [grain_lutq+offxyq+82*1] + vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 + vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 + punpcklbw xm19, xm20, xm18 + punpckhbw xm20, xm18 +%elif %2 + movu xm18, [grain_lutq+offxyq+82*0] + vinserti128 ym18, [grain_lutq+offxyq+82*1], 1 + movu xm20, [grain_lutq+top_offxyq+82*0] + vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1 + vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2] + vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 + punpcklbw ym19, ym20, ym18 + punpckhbw ym20, ym18 +%else + movu ym21, [grain_lutq+offxyq+82*0] + vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 + movu ym20, [grain_lutq+top_offxyq+82*0] + vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + +%%hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 + + lea topleft_offxyd, [top_offxyq+(32>>%2)] + lea left_offxyd, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0x000f000f + and offxd, 0x000f000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + +%if %2 + movu xm21, [grain_lutq+offxyq+82*0] + movd xm16, [grain_lutq+left_offxyq+82*0] + vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 + vinserti128 ym16, [grain_lutq+left_offxyq+82*1], 1 + vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 + vinserti32x4 m16, [grain_lutq+left_offxyq+82*2], 2 + vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 + vinserti32x4 m16, [grain_lutq+left_offxyq+82*3], 3 + movd xm18, [grain_lutq+topleft_offxyq+82*0] + movu xm20, [grain_lutq+top_offxyq] + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m16, m21 +%if %3 + punpcklbw xm18, xm20 +%else + vinserti128 ym18, [grain_lutq+topleft_offxyq+82*1], 1 + vinserti128 ym20, [grain_lutq+top_offxyq+82*1], 1 + punpcklbw ym18, ym20 +%endif + punpcklqdq m16, m18 + pmaddubsw m16, m10, m16 + pmulhrsw m16, m9 + packsswb m16, m16 + vmovdqu8 m21{k1}, m16 +%if %3 + vpalignr xm20{k1}, xm16, xm16, 4 + punpcklbw xm19, xm20, xm21 + punpckhbw xm20, xm21 +%else + vpalignr ym20{k1}, ym16, ym16, 4 + punpcklbw ym19, ym20, ym21 + punpckhbw ym20, ym21 +%endif +%else + movu ym21, [grain_lutq+offxyq+82*0] + vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 + movd xm16, [grain_lutq+left_offxyq+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2 + movu ym20, [grain_lutq+top_offxyq+82*0] + vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 + movd xm18, [grain_lutq+topleft_offxyq+82*0] + vinserti32x4 m18, [grain_lutq+topleft_offxyq+82*1], 2 + punpcklbw m16, m21 + punpcklbw m18, m20 + punpcklqdq m16, m18 + pmaddubsw m16, m10, m16 + pmulhrsw m16, m9 + packsswb m16, m16 + vpalignr m20{k1}, m16, m16, 4 + vmovdqu8 m21{k1}, m16 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + jmp %%hv_overlap +ALIGN function_align +%%add_noise_v: +%if %3 + pmaddubsw xm19, xm13, xm19 + pmaddubsw xm20, xm13, xm20 + pmulhrsw xm19, xm9 + pmulhrsw xm20, xm9 + vpacksswb m21{k3}, m19, m20 +%elif %2 + pmaddubsw ym19, ym13, ym19 + pmaddubsw ym20, ym13, ym20 + pmulhrsw ym19, ym9 + pmulhrsw ym20, ym9 + vpacksswb m21{k3}, m19, m20 +%else + punpcklbw m19, m20, m21 + punpckhbw m20, m21 + pmaddubsw m19, m13, m19 + pmaddubsw m20, m13, m20 + pmulhrsw m19, m9 + pmulhrsw m20, m9 + packsswb m21, m19, m20 +%endif +%%add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +%%add_noise_h: + mova ym18, [lumaq+lstrideq*(0<<%3)] + vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 +%if %2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] + mova ym16, [lumaq+lstrideq*(0<<%3)] + vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1 + mova xm17, [srcq+strideq*0] + mova m19, m11 + vpermi2b m19, m18, m16 + vinserti128 ym17, [srcq+strideq*1], 1 + vpermt2b m18, m12, m16 + vinserti32x4 m17, [srcq+strideq*2], 2 + pavgb m18, m19 + vinserti32x4 m17, [srcq+stride3q ], 3 +%else + mova ym17, [srcq+strideq*0] + vinserti32x8 m17, [srcq+strideq*1], 1 +%endif +%if %1 + punpckhbw m19, m18, m17 + punpcklbw m18, m17 ; { luma, chroma } + pmaddubsw m19, m14 + pmaddubsw m18, m14 + psraw m19, 6 + psraw m18, 6 + paddw m19, m15 + paddw m18, m15 + packuswb m18, m19 +.add_noise_main: + mova m19, m0 + vpermt2b m19, m18, m1 ; scaling[ 0..127] + vpmovb2m k2, m18 + vpermi2b m18, m2, m3 ; scaling[128..255] + vmovdqu8 m19{k2}, m18 ; scaling[src] + pshufb m19, m4 + pmaddubsw m18, m19, m20 + pmaddubsw m19, m21 + add grain_lutq, 82*2<<%2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*(2<<%2)] + pmulhrsw m18, m6 ; noise + pmulhrsw m19, m6 + punpcklbw m16, m17, m5 ; chroma + punpckhbw m17, m5 + paddw m16, m18 + paddw m17, m19 + packuswb m16, m17 + pmaxub m16, m7 + pminub m16, m8 +%if %2 + mova [dstq+strideq*0], xm16 + vextracti128 [dstq+strideq*1], ym16, 1 + vextracti32x4 [dstq+strideq*2], m16, 2 + vextracti32x4 [dstq+stride3q ], m16, 3 +%else + mova [dstq+strideq*0], ym16 + vextracti32x8 [dstq+strideq*1], m16, 1 +%endif + lea dstq, [dstq+strideq*(2<<%2)] + ret +%else + jmp .add_noise_main +%endif +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/filmgrain_common.asm b/third_party/dav1d/src/x86/filmgrain_common.asm new file mode 100644 index 0000000000..74f7044e66 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain_common.asm @@ -0,0 +1,46 @@ +; Copyright © 2019-2022, VideoLAN and dav1d authors +; Copyright © 2019-2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +struc FGData + .seed: resd 1 + .num_y_points: resd 1 + .y_points: resb 14 * 2 + .chroma_scaling_from_luma: resd 1 + .num_uv_points: resd 2 + .uv_points: resb 2 * 10 * 2 + .scaling_shift: resd 1 + .ar_coeff_lag: resd 1 + .ar_coeffs_y: resb 24 + .ar_coeffs_uv: resb 2 * 28 ; includes padding + .ar_coeff_shift: resq 1 + .grain_scale_shift: resd 1 + .uv_mult: resd 2 + .uv_luma_mult: resd 2 + .uv_offset: resd 2 + .overlap_flag: resd 1 + .clip_to_restricted_range: resd 1 +endstruc + +cextern gaussian_sequence diff --git a/third_party/dav1d/src/x86/filmgrain_sse.asm b/third_party/dav1d/src/x86/filmgrain_sse.asm new file mode 100644 index 0000000000..0172f98760 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain_sse.asm @@ -0,0 +1,3233 @@ +; Copyright © 2019-2021, VideoLAN and dav1d authors +; Copyright © 2019, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +SECTION_RODATA + +pw_1024: times 8 dw 1024 +pb_27_17_17_27: db 27, 17, 17, 27 + times 6 db 0, 32 +pb_23_22_h: db 23, 22 + times 7 db 0, 32 +pb_27_17: times 8 db 27, 17 +pb_17_27: times 8 db 17, 27 +pb_23_22: times 8 db 23, 22 +pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 +rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +pb_1: times 4 db 1 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512 +max: dw 255, 240, 235 +min: dw 0, 16 +pw_1: dw 1 + +%macro JMP_TABLE 2-* + %xdefine %1_8bpc_%2_table %%table + %xdefine %%base %1_8bpc_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .ar%3 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444, ssse3, 0, 1, 2, 3 + +SECTION .text + +%if ARCH_X86_32 +%define PIC_ptr(a) base+a +%else +%define PIC_ptr(a) a +%endif + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +cglobal generate_grain_y_8bpc, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data + LEA r4, $$ +%define base r4-$$ + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r2d, [fg_dataq+FGData.grain_scale_shift] + movd m2, [base+round+r2*2] + movd m0, [fg_dataq+FGData.seed] + mova m5, [base+pb_mask] + pshuflw m2, m2, q0000 + pshuflw m0, m0, q0000 + mov r2, -73*82 + sub bufq, r2 + lea r3, [base+gaussian_sequence] +.loop: + pand m6, m0, m1 + psrlw m3, m6, 10 + por m6, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m6, m4 ; bits 0x0f00 are set + pshufb m3, m5, m6 ; set 15th bit for next 4 seeds + psllq m6, m3, 30 + por m3, m6 + psllq m6, m3, 15 + por m3, m6 ; aggregate each bit into next seed's high bit + pmulhuw m6, m0, m7 + por m3, m6 ; 4 next output seeds + pshuflw m0, m3, q3333 + psrlw m3, 5 +%if ARCH_X86_64 + movq r6, m3 + mov r8, r6 + movzx r5d, r6w + shr r6d, 16 + shr r8, 32 + movzx r7, r8w + shr r8, 16 + + movd m6, [r3+r5*2] + pinsrw m6, [r3+r6*2], 1 + pinsrw m6, [r3+r7*2], 2 + pinsrw m6, [r3+r8*2], 3 +%else + movd r6, m3 + pshuflw m3, m3, q3232 + movzx r5, r6w + shr r6, 16 + + movd m6, [r3+r5*2] + pinsrw m6, [r3+r6*2], 1 + + movd r6, m3 + movzx r5, r6w + shr r6, 16 + + pinsrw m6, [r3+r5*2], 2 + pinsrw m6, [r3+r6*2], 3 +%endif + pmulhrsw m6, m2 + packsswb m6, m6 + movd [bufq+r2], m6 + add r2, 4 + jl .loop + + ; auto-regression code + movsxd r2, [fg_dataq+FGData.ar_coeff_lag] + movsxd r2, [base+generate_grain_y_8bpc_ssse3_table+r2*4] + lea r2, [r2+base+generate_grain_y_8bpc_ssse3_table] + jmp r2 + +.ar1: +%if ARCH_X86_32 + DEFINE_ARGS buf, fg_data, cf3, unused, val3, min, max +%elif WIN64 + DEFINE_ARGS shift, fg_data, cf3, buf, val3, min, max, x, val0 + mov bufq, r0 +%else + DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 +%endif + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd m4, [fg_dataq+FGData.ar_coeffs_y] + mov ecx, [fg_dataq+FGData.ar_coeff_shift] +%if ARCH_X86_32 + mov r1m, cf3d + DEFINE_ARGS buf, shift, val3, min, max, x, val0 +%define hd r0mp +%define cf3d r1mp +%elif WIN64 + DEFINE_ARGS shift, h, cf3, buf, val3, min, max, x, val0 +%else + DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 +%endif + pxor m6, m6 + pcmpgtb m7, m6, m4 + punpcklbw m4, m7 + pinsrw m4, [base+pw_1], 3 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + movd m3, [base+round_vals+shiftq*2-12] ; rnd + pshuflw m3, m3, q0000 + sub bufq, 82*73-(82*3+79) + mov hd, 70 + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -76 + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: + movq m0, [bufq+xq-82-1] ; top/left + pcmpgtb m7, m6, m0 + punpcklbw m0, m7 + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 + punpcklwd m1, m3 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 + dec hd + jg .y_loop_ar1 +.ar0: + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -16*8 +%endif + DEFINE_ARGS buf, fg_data, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m6, [base+round_vals-12+shiftq*2] + movd m7, [base+byte_blend+1] + SCRATCH 7, 15, 7 + movq m0, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 + movd m1, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 + pxor m7, m7 + pshuflw m6, m6, q0000 + punpcklwd m6, m7 + pcmpgtb m4, m7, m0 + pcmpgtb m5, m7, m1 + punpcklbw m0, m4 + punpcklbw m1, m5 + DEFINE_ARGS buf, fg_data, h, x + pshufd m4, m1, q0000 + pshufd m5, m1, q1111 + pshufd m3, m0, q3333 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + SCRATCH 6, 14, 6 + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar2: + mov xq, -76 + +.x_loop_ar2: + movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pcmpgtb m2, m7, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + psrldq m5, m0, 2 ; y=-2,x=[-1,+5] + psrldq m3, m1, 2 ; y=-1,x=[-1,+5] + psrldq m4, m1, 4 ; y=-1,x=[+0,+5] + punpcklwd m2, m0, m5 + punpcklwd m3, m4 + pmaddwd m2, m8 + pmaddwd m3, m11 + paddd m2, m3 + + psrldq m4, m0, 4 ; y=-2,x=[+0,+5] + psrldq m5, m0, 6 ; y=-2,x=[+1,+5] + psrldq m6, m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m4, m5 + punpcklwd m6, m1 + psrldq m5, m1, 6 ; y=-1,x=[+1,+5] + psrldq m1, m1, 8 ; y=-1,x=[+2,+5] + punpcklwd m5, m1 + pmaddwd m4, m9 + pmaddwd m6, m10 + pmaddwd m5, m12 + paddd m4, m6 + paddd m2, m5 + paddd m2, m4 + paddd m2, m14 + + movq m0, [bufq+xq-2] ; y=0,x=[-2,+5] +.x_loop_ar2_inner: + pcmpgtb m4, m7, m0 + punpcklbw m1, m0, m4 + pmaddwd m3, m1, m13 + paddd m3, m2 + psrldq m1, 4 ; y=0,x=0 + psrldq m2, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + paddw m3, m1 + packsswb m3, m3 + pslldq m3, 2 + pand m3, m15 + pandn m1, m15, m0 + por m0, m1, m3 + psrldq m0, 1 + ; overwrite 2 pixels, but that's ok + movd [bufq+xq-1], m0 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + dec hd + jg .y_loop_ar2 + RET + +.ar3: + DEFINE_ARGS buf, fg_data, shift +%if ARCH_X86_32 +%assign stack_offset stack_offset_old + ALLOC_STACK -16*14 +%elif WIN64 + SUB rsp, 16*6 +%assign stack_size_padded (stack_size_padded+16*6) +%assign stack_size (stack_size+16*6) +%else + ALLOC_STACK -16*6 +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m6, [base+round_vals-12+shiftq*2] + movd m7, [base+byte_blend] + movu m0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + movq m2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pxor m3, m3 + pcmpgtb m4, m3, m0 + pcmpgtb m3, m2 + pshuflw m6, m6, q0000 + SCRATCH 6, 14, 12 + SCRATCH 7, 15, 13 + punpckhbw m1, m0, m4 + punpcklbw m0, m4 + punpcklbw m2, m3 + pshufd m3, m0, q1111 + pshufd m4, m0, q2222 + pshufd m5, m0, q3333 + pshufd m0, m0, q0000 + mova [rsp+ 0*16], m0 + mova [rsp+ 1*16], m3 + mova [rsp+ 2*16], m4 + mova [rsp+ 3*16], m5 + pshufd m6, m1, q1111 + pshufd m7, m1, q2222 + pshufd m5, m1, q3333 + pshufd m1, m1, q0000 + pshufd m3, m2, q1111 + psrldq m0, m2, 10 + pinsrw m2, [base+pw_1], 5 + pshufd m4, m2, q2222 + pshufd m2, m2, q0000 + pinsrw m0, [base+round_vals+shiftq*2-10], 3 + mova [rsp+ 4*16], m1 + mova [rsp+ 5*16], m6 + SCRATCH 7, 8, 6 + SCRATCH 5, 9, 7 + SCRATCH 2, 10, 8 + SCRATCH 3, 11, 9 + SCRATCH 4, 12, 10 + SCRATCH 0, 13, 11 + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 82*73-(82*3+79) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 + +.x_loop_ar3: + movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + pxor m3, m3 + pcmpgtb m3, m0 + punpckhbw m2, m0, m3 + punpcklbw m0, m3 + + psrldq m5, m0, 2 + psrldq m6, m0, 4 + psrldq m7, m0, 6 + punpcklwd m4, m0, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 0*16] + pmaddwd m6, [rsp+ 1*16] + paddd m4, m6 + + movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] + pxor m5, m5 + pcmpgtb m5, m1 + punpckhbw m3, m1, m5 + punpcklbw m1, m5 + palignr m6, m2, m0, 10 + palignr m7, m2, m0, 12 + psrldq m0, 8 + punpcklwd m0, m6 + punpcklwd m7, m1 + pmaddwd m0, [rsp+ 2*16] + pmaddwd m7, [rsp+ 3*16] + paddd m0, m7 + paddd m0, m4 + + psrldq m4, m1, 2 + psrldq m5, m1, 4 + psrldq m6, m1, 6 + psrldq m7, m1, 8 + punpcklwd m4, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 4*16] + pmaddwd m6, [rsp+ 5*16] + paddd m4, m6 + paddd m0, m4 + + movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + pxor m7, m7 + pcmpgtb m7, m2 + punpckhbw m5, m2, m7 + punpcklbw m2, m7 + palignr m7, m3, m1, 10 + palignr m3, m1, 12 + psrldq m1, m2, 2 + punpcklwd m7, m3 + punpcklwd m3, m2, m1 + pmaddwd m7, m8 + pmaddwd m3, m9 + paddd m7, m3 + paddd m0, m7 + + psrldq m6, m2, 4 + psrldq m1, m2, 6 + psrldq m3, m2, 8 + palignr m4, m5, m2, 10 + palignr m5, m5, m2, 12 + + punpcklwd m6, m1 + punpcklwd m3, m4 + punpcklwd m5, m14 + pmaddwd m6, m10 + pmaddwd m3, m11 + pmaddwd m5, m12 + paddd m0, m6 + paddd m3, m5 + paddd m0, m3 + + movq m1, [bufq+xq-3] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pxor m5, m5 + pcmpgtb m5, m1 + punpcklbw m2, m1, m5 + pmaddwd m2, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw since we only care about one value + packsswb m2, m2 + pslldq m2, 3 + pand m2, m15 + pandn m3, m15, m1 + por m1, m2, m3 + movd [bufq+xq-3], m1 + psrldq m1, 1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82 + dec hd + jg .y_loop_ar3 + RET + +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y +INIT_XMM ssse3 +cglobal generate_grain_uv_%1_8bpc, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv + movifnidn r2, r2mp + movifnidn r3, r3mp + LEA r4, $$ +%define base r4-$$ + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r5d, [fg_dataq+FGData.grain_scale_shift] + movd m6, [base+round+r5*2] + mova m5, [base+pb_mask] + movd m0, [fg_dataq+FGData.seed] + movd m2, [base+pw_seed_xor+uvq*4] + pxor m0, m2 + pshuflw m6, m6, q0000 + pshuflw m0, m0, q0000 + lea r6, [base+gaussian_sequence] +%if %2 +%if ARCH_X86_64 + mov r7d, 73-35*%3 +%else + mov r3mp, 73-35*%3 +%endif + add bufq, 44 +.loop_y: + mov r5, -44 +.loop_x: +%else + mov r5, -82*73 + sub bufq, r5 +.loop: +%endif + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m3, m2 + psllq m2, m3, 15 + por m3, m2 ; aggregate each bit into next seed's high bit + pmulhuw m2, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + movd r9d, m2 + pshuflw m2, m2, q3232 + movzx r8, r9w + shr r9, 16 + + movd m3, [r6+r8*2] + pinsrw m3, [r6+r9*2], 1 + + movd r9d, m2 + movzx r8, r9w + shr r9, 16 + + pinsrw m3, [r6+r8*2], 2 + pinsrw m3, [r6+r9*2], 3 +%else + movd r2, m2 + pshuflw m2, m2, q3232 + movzx r1, r2w + shr r2, 16 + + movd m3, [r6+r1*2] + pinsrw m3, [r6+r2*2], 1 + + movd r2, m2 + movzx r1, r2w + shr r2, 16 + + pinsrw m3, [r6+r1*2], 2 + pinsrw m3, [r6+r2*2], 3 +%endif + pmulhrsw m3, m6 + packsswb m3, m3 + movd [bufq+r5], m3 + add r5, 4 +%if %2 + jl .loop_x + add bufq, 82 +%if ARCH_X86_64 + dec r7d +%else + dec r3mp +%endif + jg .loop_y +%else + jl .loop +%endif + +%if ARCH_X86_32 + mov r2, r2mp +%endif + + ; auto-regression code + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movsxd r5, [base+generate_grain_uv_%1_8bpc_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_8bpc_ssse3_table] + jmp r5 + +.ar0: + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -2*16 +%endif + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m5, [fg_dataq+FGData.ar_coeffs_uv+uvq] + movd m4, [base+hmul_bits+shiftq*2] + DEFINE_ARGS buf, bufy, h, x + pxor m0, m0 + pcmpgtb m0, m5 + punpcklbw m5, m0 + movd m7, [base+pb_1] +%if %2 + movd m6, [base+hmul_bits+2+%3*2] +%endif + pshuflw m5, m5, q0000 + pshuflw m4, m4, q0000 + pshufd m7, m7, q0000 +%if %2 + pshuflw m6, m6, q0000 +%endif + punpcklqdq m5, m5 + punpcklqdq m4, m4 +%if %2 + punpcklqdq m6, m6 +%endif + pcmpeqw m1, m1 + pslldq m1, 12>>%2 + SCRATCH 1, 8, 0 + SCRATCH 4, 9, 1 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif + add bufyq, 3+82*3 + mov hd, 70-35*%3 +.y_loop_ar0: + xor xd, xd +.x_loop_ar0: + ; first 32 pixels +%if %2 + movu m1, [bufyq+xq*2] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif + movu m3, [bufyq+xq*2+16] +%if %3 + movu m4, [bufyq+xq*2+82+16] +%endif + pmaddubsw m0, m7, m1 +%if %3 + pmaddubsw m1, m7, m2 +%endif + pmaddubsw m2, m7, m3 +%if %3 + pmaddubsw m3, m7, m4 + paddw m0, m1 + paddw m2, m3 +%endif + pmulhrsw m0, m6 + pmulhrsw m2, m6 +%else + movu m0, [bufyq+xq] + pxor m6, m6 + pcmpgtb m6, m0 + punpckhbw m2, m0, m6 + punpcklbw m0, m6 +%endif + pmullw m0, m5 + pmullw m2, m5 + pmulhrsw m0, m9 + pmulhrsw m2, m9 + movu m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpckhbw m3, m1, m4 +%if %2 + punpcklbw m1, m4 + paddw m2, m3 + paddw m0, m1 +%else + punpcklbw m6, m1, m4 + paddw m2, m3 + paddw m0, m6 +%endif + packsswb m0, m2 +%if %2 + movu [bufq+xq], m0 + add xd, 16 + cmp xd, 32 + jl .x_loop_ar0 + + ; last 6/12 pixels + movu m1, [bufyq+xq*(1+%2)] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif + pmaddubsw m0, m7, m1 +%if %3 + pmaddubsw m1, m7, m2 + paddw m0, m1 +%endif + pmulhrsw m0, m6 + pmullw m0, m5 + pmulhrsw m0, m9 + movq m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpcklbw m2, m1, m4 + paddw m0, m2 + packsswb m0, m0 + pandn m2, m8, m0 + pand m1, m8 + por m2, m1 + movq [bufq+xq], m2 +%else + add xd, 16 + cmp xd, 80 + je .y_loop_final_ar0 + movu [bufq+xq-16], m0 + jmp .x_loop_ar0 +.y_loop_final_ar0: + pandn m2, m8, m0 + pand m1, m8 + por m2, m1 + movu [bufq+xq-16], m2 +%endif + + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar0 + RET + +.ar1: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x + imul uvd, 28 + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movd m4, [fg_dataq+FGData.ar_coeffs_uv+uvq-1] + pinsrw m4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 2 +%if ARCH_X86_32 + mov r3mp, cf3d + DEFINE_ARGS buf, shift, fg_data, val3, min, max, x +%elif WIN64 + DEFINE_ARGS shift, bufy, fg_data, buf, val3, cf3, min, max, x + mov bufq, r0 +%else + DEFINE_ARGS buf, bufy, fg_data, shift, val3, cf3, min, max, x +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 + movd m7, [base+pb_1] + movd m6, [base+hmul_bits+2+%3*2] +%endif + psrldq m4, 1 +%if ARCH_X86_32 + DEFINE_ARGS buf, shift, val0, val3, min, max, x +%elif WIN64 + DEFINE_ARGS shift, bufy, h, buf, val3, cf3, min, max, x, val0 +%else + DEFINE_ARGS buf, bufy, h, shift, val3, cf3, min, max, x, val0 +%endif + pxor m5, m5 + punpcklwd m3, m5 +%if %2 + punpcklwd m6, m6 +%endif + pcmpgtb m5, m4 + punpcklbw m4, m5 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + pshufd m3, m3, q0000 +%if %2 + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif +%if ARCH_X86_32 + add r1mp, 79+82*3 + mov r0mp, 70-35*%3 +%else + add bufyq, 79+82*3 + mov hd, 70-35*%3 +%endif + mov mind, -128 + mov maxd, 127 +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, byte [bufq+xq-1] +.x_loop_ar1: +%if %2 +%if ARCH_X86_32 + mov r2, r1mp + movq m0, [r2+xq*2] +%if %3 + movq m1, [r2+xq*2+82] +%endif +%else + movq m0, [bufyq+xq*2] +%if %3 + movq m1, [bufyq+xq*2+82] +%endif +%endif + pmaddubsw m2, m7, m0 +%if %3 + pmaddubsw m0, m7, m1 + paddw m2, m0 +%endif + pmulhrsw m2, m6 +%else +%if ARCH_X86_32 + mov r2, r1mp + movd m2, [r2+xq] +%else + movd m2, [bufyq+xq] +%endif + pxor m0, m0 + pcmpgtb m0, m2 + punpcklbw m2, m0 +%endif + + movq m0, [bufq+xq-82-1] ; top/left + pxor m1, m1 + pcmpgtb m1, m0 + punpcklbw m0, m1 + psrldq m1, m0, 4 ; top/right + punpcklwd m1, m2 + psrldq m2, m0, 2 ; top + punpcklwd m0, m2 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 + paddd m0, m3 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 +%if ARCH_X86_32 + imul val3d, r3mp +%else + imul val3d, cf3d +%endif + add val3d, val0d + sar val3d, shiftb + movsx val0d, byte [bufq+xq] + add val3d, val0d + cmp val3d, maxd + cmovns val3d, maxd + cmp val3d, mind + cmovs val3d, mind + mov byte [bufq+xq], val3b + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82 +%if ARCH_X86_32 + add r1mp, 82<<%3 + dec r0mp +%else + add bufyq, 82<<%3 + dec hd +%endif + jg .y_loop_ar1 + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp + ALLOC_STACK -8*16 +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + movd m7, [base+round_vals-12+shiftq*2] + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-12 + pxor m2, m2 + pcmpgtb m2, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + pinsrw m1, [base+pw_1], 5 + punpcklwd m7, m7 + pshufd m7, m7, q0000 + DEFINE_ARGS buf, bufy, fg_data, h, unused, x + pshufd m4, m1, q0000 + pshufd m5, m1, q1111 + pshufd m6, m1, q2222 + pshufd m3, m0, q3333 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + SCRATCH 6, 14, 6 + SCRATCH 7, 15, 7 +%if %2 + movd m7, [base+hmul_bits+2+%3*2] + movd m6, [base+pb_1] + punpcklwd m7, m7 + pshufd m6, m6, q0000 + pshufd m7, m7, q0000 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + pxor m2, m2 + movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] + movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] + pcmpgtb m2, m0 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 + psrldq m5, m0, 2 ; y=-2,x=[-1,+5] + psrldq m3, m1, 2 ; y=-1,x=[-1,+5] + psrldq m4, m1, 4 ; y=-1,x=[+0,+5] + punpcklwd m2, m0, m5 + punpcklwd m3, m4 + pmaddwd m2, m8 + pmaddwd m3, m11 + paddd m2, m3 + + psrldq m4, m0, 4 ; y=-2,x=[+0,+5] + psrldq m5, m0, 6 ; y=-2,x=[+1,+5] + psrldq m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m4, m5 + punpcklwd m0, m1 + psrldq m3, m1, 6 ; y=-1,x=[+1,+5] + psrldq m1, m1, 8 ; y=-1,x=[+2,+5] + punpcklwd m3, m1 + pmaddwd m4, m9 + pmaddwd m0, m10 + pmaddwd m3, m12 + paddd m4, m0 + paddd m2, m3 + paddd m2, m4 + +%if %2 + movq m1, [bufyq+xq*2] +%if %3 + movq m3, [bufyq+xq*2+82] +%endif + pmaddubsw m0, m6, m1 +%if %3 + pmaddubsw m1, m6, m3 + paddw m0, m1 +%endif + pmulhrsw m0, m7 +%else + movd m0, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m0 + punpcklbw m0, m1 +%endif + punpcklwd m0, m15 + pmaddwd m0, m14 + paddd m2, m0 + + movq m0, [bufq+xq-2] ; y=0,x=[-2,+5] + pxor m4, m4 + movd m5, [base+byte_blend+1] + punpcklbw m5, m5 +.x_loop_ar2_inner: + pcmpgtb m1, m4, m0 + punpcklbw m0, m1 + pmaddwd m3, m0, m13 + paddd m3, m2 + psrldq m2, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + pslldq m3, 4 + pand m3, m5 + paddw m0, m3 + packsswb m0, m0 + movd [bufq+xq-2], m0 + psrldq m0, 1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar2 + RET + +.ar3: +%if ARCH_X86_32 +%assign stack_offset stack_offset_old +%assign stack_size_padded 0 +%xdefine rstk rsp +%endif + DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift + movifnidn bufyq, bufymp +%if ARCH_X86_32 + ALLOC_STACK -15*16 +%else + SUB rsp, 16*7 +%assign stack_size_padded (stack_size_padded+16*7) +%assign stack_size (stack_size+16*7) +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15 + pxor m3, m3 + pcmpgtb m3, m0 + punpckhbw m1, m0, m3 + punpcklbw m0, m3 + pshufd m2, m0, q1111 + pshufd m3, m0, q2222 + pshufd m4, m0, q3333 + pshufd m0, m0, q0000 + pshufd m5, m1, q1111 + pshufd m6, m1, q2222 + pshufd m7, m1, q3333 + pshufd m1, m1, q0000 + mova [rsp+ 0*16], m0 + mova [rsp+ 1*16], m2 + mova [rsp+ 2*16], m3 + mova [rsp+ 3*16], m4 + mova [rsp+ 4*16], m1 + mova [rsp+ 5*16], m5 + mova [rsp+ 6*16], m6 + SCRATCH 7, 8, 7 + + movu m2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-24 [24=luma] + pxor m4, m4 + pcmpgtb m4, m2 + punpckhbw m5, m2, m4 + punpcklbw m2, m4 + pshufd m4, m2, q3232 + punpcklwd m3, m4, m5 + pshuflw m5, m4, q3321 + pshufd m4, m3, q0000 + pshufd m3, m2, q1111 + pshufd m2, m2, q0000 + pinsrw m5, [base+round_vals+shiftq*2-10], 3 + SCRATCH 2, 9, 8 + SCRATCH 3, 10, 9 + SCRATCH 4, 11, 10 + SCRATCH 5, 12, 11 + + movd m2, [base+round_vals-12+shiftq*2] +%if %2 + movd m1, [base+pb_1] + movd m3, [base+hmul_bits+2+%3*2] +%endif + pxor m0, m0 + punpcklwd m2, m0 +%if %2 + punpcklwd m3, m3 +%endif + pshufd m2, m2, q0000 +%if %2 + pshufd m1, m1, q0000 + pshufd m3, m3, q0000 + SCRATCH 1, 13, 12 +%endif + SCRATCH 2, 14, 13 +%if %2 + SCRATCH 3, 15, 14 +%endif + + DEFINE_ARGS buf, bufy, fg_data, h, unused, x +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif + add bufyq, 79+82*3 + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) + +.x_loop_ar3: + movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] + pxor m4, m4 + pcmpgtb m4, m0 + punpckhbw m3, m0, m4 + punpcklbw m0, m4 + + psrldq m5, m0, 2 + psrldq m6, m0, 4 + psrldq m7, m0, 6 + punpcklwd m4, m0, m5 + punpcklwd m6, m7 + pmaddwd m4, [rsp+ 0*16] + pmaddwd m6, [rsp+ 1*16] + paddd m4, m6 + + palignr m2, m3, m0, 10 + palignr m3, m0, 12 + psrldq m0, 8 + + movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] + pxor m6, m6 + pcmpgtb m6, m1 + punpckhbw m5, m1, m6 + punpcklbw m1, m6 + + punpcklwd m0, m2 + punpcklwd m3, m1 + pmaddwd m0, [rsp+ 2*16] + pmaddwd m3, [rsp+ 3*16] + paddd m0, m3 + paddd m0, m4 + + movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] + pxor m7, m7 + pcmpgtb m7, m2 + punpckhbw m6, m2, m7 + punpcklbw m2, m7 + + palignr m3, m5, m1, 10 + palignr m5, m1, 12 + psrldq m4, m2, 2 + + punpcklwd m3, m5 + punpcklwd m5, m2, m4 + pmaddwd m3, [rsp+ 6*16] + pmaddwd m5, m8 + paddd m3, m5 + paddd m0, m3 + + psrldq m3, m1, 2 + psrldq m4, m1, 4 + psrldq m5, m1, 6 + psrldq m1, 8 + + punpcklwd m3, m4 + punpcklwd m5, m1 + pmaddwd m3, [rsp+ 4*16] + pmaddwd m5, [rsp+ 5*16] + paddd m3, m5 + paddd m0, m3 + +%if %2 + movq m1, [bufyq+xq*2] +%if %3 + movq m3, [bufyq+xq*2+82] +%endif + pmaddubsw m7, m13, m1 +%if %3 + pmaddubsw m5, m13, m3 + paddw m7, m5 +%endif + pmulhrsw m7, m15 +%else + movd m7, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m7 + punpcklbw m7, m1 +%endif + + psrldq m1, m2, 4 + psrldq m3, m2, 6 + palignr m4, m6, m2, 10 + palignr m6, m2, 12 + psrldq m2, 8 + + punpcklwd m1, m3 + punpcklwd m2, m4 + punpcklwd m6, m7 + pmaddwd m1, m9 + pmaddwd m2, m10 + pmaddwd m6, m11 + paddd m1, m2 + paddd m0, m6 + paddd m0, m1 + paddd m0, m14 + + movq m1, [bufq+xq-3] ; y=0,x=[-3,+4] + pxor m4, m4 + movd m5, [base+byte_blend] +.x_loop_ar3_inner: + pcmpgtb m2, m4, m1 + punpcklbw m3, m1, m2 + pmaddwd m2, m3, m12 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + ; don't packssdw, we only care about one value + packsswb m2, m2 + pandn m3, m5, m1 + pslld m2, 24 + pand m2, m5 + por m1, m2, m3 + movd [bufq+xq-3], m1 + psrldq m1, 1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82 + add bufyq, 82<<%3 + dec hd + jg .y_loop_ar3 + RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 + +%macro vpgatherdw 5-6 ; dst, src, base, tmp_gpr[x2], tmp_xmm_reg +%assign %%idx 0 +%define %%tmp %2 +%if %0 == 6 +%define %%tmp %6 +%endif +%rep 4 +%if %%idx == 0 + movd %5 %+ d, %2 + pshuflw %%tmp, %2, q3232 +%else + movd %5 %+ d, %%tmp +%if %%idx == 2 + punpckhqdq %%tmp, %%tmp +%elif %%idx == 4 + psrlq %%tmp, 32 +%endif +%endif + movzx %4 %+ d, %5 %+ w + shr %5 %+ d, 16 + +%if %%idx == 0 + movd %1, [%3+%4] +%else + pinsrw %1, [%3+%4], %%idx + 0 +%endif + pinsrw %1, [%3+%5], %%idx + 1 +%assign %%idx %%idx+2 +%endrep +%endmacro + +INIT_XMM ssse3 +; fgy_32x32xn(dst, src, stride, fg_data, w, scaling, grain_lut, h, sby) +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fgy_32x32xn_8bpc, 0, 7, 16, 0 - (5 * mmsize + 16 * gprsize), \ + dst, src, scaling, unused1, fg_data, picptr, unused2 + ; copy stack arguments to new position post-alignment, so that we + ; don't have to keep the old stack location in a separate register + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov r5, r8m + + mov [rsp+5*mmsize+ 4*gprsize], r0 + mov [rsp+5*mmsize+ 6*gprsize], r1 + mov [rsp+5*mmsize+ 8*gprsize], r2 + mov [rsp+5*mmsize+10*gprsize], r3 + mov [rsp+5*mmsize+11*gprsize], r4 + mov [rsp+5*mmsize+12*gprsize], r5 +%else +cglobal fgy_32x32xn_8bpc, 0, 7, 16, 5 * mmsize + 4 * gprsize, \ + dst, src, scaling, unused1, fg_data, picptr, unused2 +%endif + mov srcq, srcm + mov fg_dataq, r3m + mov scalingq, r5m +%if STACK_ALIGNMENT < mmsize +%define r0m [rsp+5*mmsize+ 4*gprsize] +%define r1m [rsp+5*mmsize+ 5*gprsize] +%define r2m [rsp+5*mmsize+ 6*gprsize] +%define r3m [rsp+5*mmsize+ 7*gprsize] +%define r4m [rsp+5*mmsize+ 8*gprsize] +%define r5m [rsp+5*mmsize+ 9*gprsize] +%define r6m [rsp+5*mmsize+10*gprsize] +%define r7m [rsp+5*mmsize+11*gprsize] +%define r8m [rsp+5*mmsize+12*gprsize] +%endif + LEA r5, pb_mask +%define base r5-pb_mask + mov r5m, picptrq +%else +cglobal fgy_32x32xn_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut + lea r7, [pb_mask] +%define base r7-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + movd m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + movd m4, [base+max+r6*4] + movd m5, [base+min+r6*2] + punpcklwd m3, m3 + punpcklwd m4, m4 + punpcklwd m5, m5 + pshufd m3, m3, q0000 + pshufd m4, m4, q0000 + pshufd m5, m5, q0000 + SCRATCH 3, 11, 0 + SCRATCH 4, 12, 1 + SCRATCH 5, 13, 2 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap +%endif + + mov sbyd, r8m + mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 + test overlapd, overlapd + jz .no_vertical_overlap + mova m6, [base+pw_1024] + mova m7, [base+pb_27_17_17_27] + SCRATCH 6, 14, 3 + SCRATCH 7, 15, 4 + test sbyd, sbyd + jnz .vertical_overlap + ; fall-through + +.no_vertical_overlap: + mov r8m, overlapd +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + unused1, unused2, see, unused3 +%endif + + lea src_bakq, [srcq+wq] + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r4m, wq + DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3 +%endif + +.loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, unused + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr, + ; r6m=grain_lut, r7m=h, r8m=overlap_v|h + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, unused +%endif + +.loop_x_odd: + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 +%endif + REPX {psrlw x, 8}, m4, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] + pcmpgtb m7, m2, m3 + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m4 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + add srcq, r2mp + add grain_lutq, 82 + dec hd + jg .loop_y + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r1mp + add srcq, r4mp +%else + lea srcq, [src_bakq+wq] +%endif + btc dword r8m, 2 + jc .next_blk + + add offxyd, 16 + test dword r8m, 2 ; r8m & 2 = have_top_overlap + jz .loop_x_odd + +%if ARCH_X86_32 + add dword [rsp+5*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxyd +%endif + jnz .loop_x_odd_v_overlap + +.next_blk: + test dword r8m, 1 + jz .loop_x + + test dword r8m, 2 + jnz .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: +%if ARCH_X86_32 + ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr, + ; r6m=grain_lut, r7m=h, r8m=overlap_v|h + DEFINE_ARGS dst, src, scaling, offxy, unused1, unused2, unused3 + + add offxyd, 16 ; left_offxyd + mov [rsp+5*mmsize+0*gprsize], offxyd + + DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3 + + mov seed, r3m +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_h_overlap: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 +%endif + REPX {psrlw x, 8}, m4, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+0*gprsize] + movd m7, [grain_lutq+r5] +%else + movd m7, [grain_lutq+left_offxyq] +%endif + punpcklbw m7, m3 + pmaddubsw m6, m15, m7 + pmulhrsw m6, m14 + packsswb m6, m6 + shufps m6, m3, q3210 + pcmpgtb m2, m6 + punpcklbw m7, m6, m2 + punpckhbw m6, m2 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m7, m4 + pmullw m6, m5 + pmulhrsw m7, m11 + pmulhrsw m6, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m7 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + add srcq, r2mp + add grain_lutq, 82 + dec hd + jg .loop_y_h_overlap + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r1m + add srcq, r4m +%else + lea srcq, [src_bakq+wq] +%endif + xor dword r8m, 4 + add offxyd, 16 + + ; since this half-block had left-overlap, the next does not + test dword r8m, 2 ; have_top_overlap + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+5*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxyd +%endif + jmp .loop_x_odd_v_overlap + +.end: + RET + +.vertical_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap +%endif + + or overlapd, 2 ; top_overlap: overlap & 2 + mov r8m, overlapd + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul tmpd, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add tmpd, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and tmpd, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, tmpd +%if ARCH_X86_32 + xor sbyd, seed ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + tmp, unused2, see, unused3 +%endif + + lea src_bakq, [srcq+wq] + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r4m, wq + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%endif + +.loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m +%endif + ; we assume from the block above that bits 8-15 of tmpd are zero'ed, + ; because of the 'and tmpd, 0x00ff00ff' above + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, unused, top_offxy + + mov offyd, seed + mov offxd, seed +%endif + + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, unused, top_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+5*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +.loop_x_odd_v_overlap: +%if ARCH_X86_32 + mov r5, r5m + lea r5, [base+pb_27_17] + mov [rsp+5*mmsize+12], r5 +%else + mova m8, [pb_27_17] +%endif + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_v_overlap: + ; src + mova m0, [srcq] + pxor m2, m2 + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 +%else + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 +%endif + REPX {psrlw x, 8}, m4, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+1*gprsize] + movu m7, [grain_lutq+r5] +%else + movu m7, [grain_lutq+top_offxyq] +%endif + punpckhbw m6, m7, m3 + punpcklbw m7, m3 +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+12] + pmaddubsw m3, [r5], m6 + pmaddubsw m6, [r5], m7 +%else + pmaddubsw m3, m8, m6 + pmaddubsw m6, m8, m7 +%endif + pmulhrsw m3, m14 + pmulhrsw m6, m14 + packsswb m6, m3 + pcmpgtb m7, m2, m6 + punpcklbw m2, m6, m7 + punpckhbw m6, m7 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m4 + pmullw m6, m5 + pmulhrsw m2, m11 + pmulhrsw m6, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add dword [rsp+5*mmsize+12], mmsize +%else + mova m8, [pb_17_27] +%endif + add srcq, r2mp + add grain_lutq, 82 + dec hw + jz .end_y_v_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + btc hd, 16 + jnc .loop_y_v_overlap + jmp .loop_y + +.end_y_v_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r1mp + add srcq, r4mp +%else + lea srcq, [src_bakq+wq] +%endif + btc dword r8m, 2 + jc .loop_x_hv_overlap + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+5*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + jmp .loop_x_odd_v_overlap + +.loop_x_hv_overlap: +%if ARCH_X86_32 + mov r5, r5m + lea r5, [base+pb_27_17] + mov [rsp+5*mmsize+12], r5 + + DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, src_bak + + mov r5, [rsp+5*mmsize+1*gprsize] + mov r4, offxyd + add r5, 16 + add r4, 16 + mov [rsp+5*mmsize+2*gprsize], r5 ; topleft_offxy + mov [rsp+5*mmsize+0*gprsize], r4 ; left_offxy + + DEFINE_ARGS tmp, src, scaling, see, w, picptr, src_bak + + xor tmpd, tmpd + mov seed, r3m +%else + mova m8, [pb_27_17] + + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + tmp, unused2, see, unused3 + + ; we assume from the block above that bits 8-15 of tmpd are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut + + movzx r5, offxyw ; top_offxy + mov [rsp+5*mmsize+1*gprsize], r5 +%else + DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy + + movzx top_offxyd, offxyw +%endif + shr offxyd, 16 + + mov hd, r7m + mov grain_lutq, grain_lutmp +.loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+1*gprsize] ; top_offxy + mov r0, [rsp+5*mmsize+0*gprsize] ; left_offxy + movu m6, [grain_lutq+r5] + mov r5, [rsp+5*mmsize+2*gprsize] ; topleft_offxy + movd m4, [grain_lutq+r0] + movd m7, [grain_lutq+r5] +%else + movu m6, [grain_lutq+top_offxyq] + movd m4, [grain_lutq+left_offxyq] + movd m7, [grain_lutq+topleft_offxyq] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m4, m3 + punpcklbw m7, m6 + pmaddubsw m2, m15, m4 + pmaddubsw m4, m15, m7 + pmulhrsw m2, m14 + pmulhrsw m4, m14 + packsswb m2, m2 + packsswb m4, m4 + shufps m2, m3, q3210 + shufps m4, m6, q3210 + ; followed by v interpolation (top | cur -> cur) + punpcklbw m3, m4, m2 + punpckhbw m4, m2 +%if ARCH_X86_32 + mov r5, [rsp+5*mmsize+12] + pmaddubsw m7, [r5], m4 + pmaddubsw m4, [r5], m3 +%else + pmaddubsw m7, m8, m4 + pmaddubsw m4, m8, m3 +%endif + pmulhrsw m7, m14 + pmulhrsw m4, m14 + packsswb m4, m7 + pxor m2, m2 + pcmpgtb m7, m2, m4 + punpcklbw m3, m4, m7 + punpckhbw m4, m7 + + ; src + mova m0, [srcq] + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m5, m0, scalingq-1, r0, r5, m7 + vpgatherdw m6, m1, scalingq-1, r0, r5, m7 +%else + vpgatherdw m5, m0, scalingq-1, r13, r14, m7 + vpgatherdw m6, m1, scalingq-1, r13, r14, m7 +%endif + REPX {psrlw x, 8}, m5, m6 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m3, m5 + pmullw m4, m6 + pmulhrsw m3, m11 + pmulhrsw m4, m11 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add dword [rsp+5*mmsize+12], mmsize +%else + mova m8, [pb_17_27] +%endif + add srcq, r2mp + add grain_lutq, 82 + dec hw + jz .end_y_hv_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines + btc hd, 16 + jnc .loop_y_hv_overlap + jmp .loop_y_h_overlap + +.end_y_hv_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r1m + add srcq, r4m +%else + lea srcq, [src_bakq+wq] +%endif + xor dword r8m, 4 + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+5*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + jmp .loop_x_odd_v_overlap + +.end_hv: + RET + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +INIT_XMM ssse3 +%if ARCH_X86_32 +; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h, +; sby, luma, lstride, uv_pl, is_id) +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8 +cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 0 - (7 * mmsize + (13 + 3) * gprsize), \ + tmp, src, scaling, h, fg_data, picptr, unused + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov [rsp+7*mmsize+3*gprsize], r0 + mov [rsp+7*mmsize+5*gprsize], r1 + mov [rsp+7*mmsize+7*gprsize], r2 + mov [rsp+7*mmsize+9*gprsize], r3 + mov [rsp+7*mmsize+10*gprsize], r4 + + mov r0, r8m + mov r1, r9m + mov r2, r10m + mov r4, r11m + mov r3, r12m + mov [rsp+7*mmsize+11*gprsize], r0 + mov [rsp+7*mmsize+12*gprsize], r1 + mov [rsp+7*mmsize+13*gprsize], r2 + mov [rsp+7*mmsize+14*gprsize], r4 +%else +cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 7 * mmsize + (4) * gprsize, \ + tmp, src, scaling, h, fg_data, picptr, unused +%endif + mov srcq, srcm + mov fg_dataq, r3m + mov scalingq, r5m +%if STACK_ALIGNMENT < mmsize +%define r0m [rsp+7*mmsize+ 3*gprsize] +%define r1m [rsp+7*mmsize+ 4*gprsize] +%define r2m [rsp+7*mmsize+ 5*gprsize] +%define r3m [rsp+7*mmsize+ 6*gprsize] +%define r4m [rsp+7*mmsize+ 7*gprsize] +%define r5m [rsp+7*mmsize+ 8*gprsize] +%define r6m [rsp+7*mmsize+ 9*gprsize] +%define r7m [rsp+7*mmsize+10*gprsize] +%define r8m [rsp+7*mmsize+11*gprsize] +%define r9m [rsp+7*mmsize+12*gprsize] +%define r10m [rsp+7*mmsize+13*gprsize] +%define r11m [rsp+7*mmsize+14*gprsize] +%define r12m [rsp+7*mmsize+15*gprsize] +%endif + LEA r5, pb_mask +%define base r5-pb_mask + mov r5m, r5 +%else +cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, tmp, sby, luma, lstride, uv_pl, is_id + lea r8, [pb_mask] +%define base r8-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + movd m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + lea tmpd, [r6d*2] +%if ARCH_X86_32 && STACK_ALIGNMENT < mmsize + test r3, r3 +%else + cmp dword r12m, 0 ; is_idm +%endif + movd m5, [base+min+r6*2] + cmovne r6d, tmpd + movd m4, [base+max+r6*2] + punpcklwd m3, m3 + punpcklwd m5, m5 + punpcklwd m4, m4 + pshufd m3, m3, q0000 + pshufd m5, m5, q0000 + pshufd m4, m4, q0000 + SCRATCH 3, 11, 0 + SCRATCH 4, 12, 1 + SCRATCH 5, 13, 2 + + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap +%endif + +%if %1 + mov r6d, dword r11m + movd m0, [fg_dataq+FGData.uv_mult+r6*4] + movd m1, [fg_dataq+FGData.uv_luma_mult+r6*4] + punpcklbw m6, m1, m0 + movd m7, [fg_dataq+FGData.uv_offset+r6*4] + punpcklwd m6, m6 + punpcklwd m7, m7 + pshufd m6, m6, q0000 + pshufd m7, m7, q0000 + SCRATCH 6, 14, 3 + SCRATCH 7, 15, 4 +%endif + + mov sbyd, r8m + mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 + test overlapd, overlapd + jz %%no_vertical_overlap +%if ARCH_X86_32 +%if %2 + mova m1, [base+pb_23_22_h] +%else + mova m1, [base+pb_27_17_17_27] +%endif + mova m0, [base+pw_1024] +%else +%if %2 + mova m1, [pb_23_22_h] +%else + mova m1, [pb_27_17_17_27] +%endif + mova m0, [pw_1024] +%endif + SCRATCH 0, 8, 5 + SCRATCH 1, 9, 6 + test sbyd, sbyd + jnz %%vertical_overlap + ; fall-through + +%%no_vertical_overlap: + mov r8m, overlapd +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak +%define luma_bakq lumaq + + mov wq, r4m +%if %3 + shl r10mp, 1 +%endif +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + unused2, unused3, see, overlap, unused4, src_bak, lstride, luma_bak + + mov lstrideq, r10mp +%endif + + mov lumaq, r9mp + lea src_bakq, [srcq+wq] + lea luma_bakq, [lumaq+wq*(1+%2)] + neg wq + sub r0mp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r11m, luma_bakq + mov r4m, wq + + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%else + mov r11mp, src_bakq + mov r12mp, strideq +%endif + +%%loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, unused1, unused2, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, unused1, unused2, lstride, luma_bak +%endif + +%%loop_x_odd: + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y: + ; src +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 +%else + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 +%endif + REPX {psrlw x, 8}, m7, m5 + + ; unpack chroma_source + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq+ 0] + pcmpgtb m6, m2, m3 + punpcklbw m2, m3, m6 + punpckhbw m3, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; we already incremented lumaq above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 + dec hw + jg %%loop_y + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 == 0 + ; adjust top_offxy +%if ARCH_X86_32 + add dword [rsp+7*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jc %%loop_x_even + test dword r8m, 2 + jz %%loop_x_odd + jmp %%loop_x_odd_v_overlap +%%loop_x_even: +%endif + test dword r8m, 1 + jz %%loop_x + + ; r8m = sbym + test dword r8m, 2 + jne %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: +%if ARCH_X86_32 +%if %2 + lea r6, [offxyd+16] + mov [rsp+7*mmsize+0*gprsize], r6 +%else + mov [rsp+7*mmsize+0*gprsize], offxyd +%endif + + DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + +%if %2 + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx +%else + mov left_offxyd, offyd +%endif +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS luma, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, lstride, luma_bak +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_h_overlap: + ; src +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 +%else + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 +%endif + REPX {psrlw x, 8}, m7, m5 + + ; unpack chroma_source + punpckhbw m1, m0, m2 + punpcklbw m0, m2 ; m0-1: src as word + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq+ 0] +%if ARCH_X86_32 + mov r0, [rsp+7*mmsize+0*gprsize] + movd m2, [grain_lutq+r0+ 0] +%else + movd m2, [grain_lutq+left_offxyq+ 0] +%endif + punpcklbw m2, m4 + pmaddubsw m3, m9, m2 + pmulhrsw m3, m8 + packsswb m3, m3 + shufps m3, m4, q3210 + pxor m4, m4 + pcmpgtb m4, m3 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + packuswb m0, m1 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has already been incremented above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 + dec hw + jg %%loop_y_h_overlap + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 == 0 + xor dword r8m, 4 + ; adjust top_offxyd +%if ARCH_X86_32 + add dword [rsp+7*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 +%endif + + ; r8m = sbym + test dword r8m, 2 +%if %2 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap +%else + jne %%loop_x_odd_v_overlap + jmp %%loop_x_odd +%endif + +%%end: + RET + +%%vertical_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap +%endif + + or overlapd, 2 ; top_overlap: overlap & 2 + mov r8m, overlapd + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul tmpd, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add tmpd, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and tmpd, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, tmpd +%if ARCH_X86_32 + xor sbyd, seed ; (cur_seed << 16) | top_seed + + DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%if %3 + shl r10mp, 1 +%endif +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + tmp, unused2, see, overlap, unused3, src_bak, lstride, luma_bak + + mov lstrideq, r10mp +%endif + + mov lumaq, r9mp + lea src_bakq, [srcq+wq] + lea luma_bakq, [lumaq+wq*(1+%2)] + neg wq + sub r0mp, srcq +%if ARCH_X86_32 + mov r1m, src_bakq + mov r11m, luma_bakq + mov r4m, wq + + DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 +%else + mov r11mp, src_bakq + mov r12mp, strideq +%endif + +%%loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m + xor tmpd, tmpd +%endif + ; we assume from the block above that bits 8-15 of tmpd are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, overlap, top_offxy, unused, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS tmp, src, scaling, offxy, h, picptr, top_offxy +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, overlap, top_offxy, unused, lstride, luma_bak +%endif + + movzx top_offxyd, offxyw + shr offxyd, 16 +%if ARCH_X86_32 + mov [rsp+7*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut +%endif + +%%loop_x_odd_v_overlap: + mov hd, r7m + mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %3 + mova m1, [PIC_ptr(pb_23_22)] +%else + mova m1, [PIC_ptr(pb_27_17)] +%endif +%%loop_y_v_overlap: +%if ARCH_X86_32 + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 +%else + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 +%endif + REPX {psrlw x, 8}, m7, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq] +%if ARCH_X86_32 + mov r0, [rsp+7*mmsize+1*gprsize] + movu m4, [grain_lutq+r0] +%else + movu m4, [grain_lutq+top_offxyq] +%endif + punpckhbw m6, m4, m3 + punpcklbw m4, m3 + pmaddubsw m2, m1, m6 + pmaddubsw m3, m1, m4 + pmulhrsw m2, m8 + pmulhrsw m3, m8 + packsswb m3, m2 + pxor m6, m6 + pcmpgtb m6, m3 + punpcklbw m2, m3, m6 + punpckhbw m3, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m3, m5 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + + ; unpack chroma_source + pxor m4, m4 + punpckhbw m6, m0, m4 + punpcklbw m0, m4 ; m0-1: src as word + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m6, m3 + pmaxsw m0, m13 + pmaxsw m6, m13 + pminsw m0, m12 + pminsw m6, m12 + packuswb m0, m6 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + + dec hw + je %%end_y_v_overlap +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has already been incremented above +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif +%endif + add grain_lutq, 82 +%if %3 == 0 + btc hd, 16 +%if ARCH_X86_32 + mov r5, r5m +%endif + mova m1, [PIC_ptr(pb_17_27)] + jnc %%loop_y_v_overlap +%endif + jmp %%loop_y + +%%end_y_v_overlap: +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif + +%if %2 + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +%else +%if ARCH_X86_32 + add dword [rsp+7*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jnc %%loop_x_odd_v_overlap +%endif + +%%loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused + + mov r6, [rsp+7*mmsize+1*gprsize] +%if %2 + lea r0, [r3d+16] + add r6, 16 + mov [rsp+7*mmsize+0*gprsize], r0 ; left_offxy +%else + mov [rsp+7*mmsize+0*gprsize], r3 ; left_offxy +%endif + mov [rsp+7*mmsize+2*gprsize], r6 ; topleft_offxy + + DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused + + mov seed, r3m + xor tmpd, tmpd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + tmp, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride + +%if %2 + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offxyq+16] +%else + mov topleft_offxyq, top_offxyq + mov left_offxyq, offxyq +%endif + + ; we assume from the block above that bits 8-15 of tmpd are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp tmpb ; parity of top_seed + shr seed, 16 + shl tmpd, 16 + test seeb, seeh + setp tmpb ; parity of cur_seed + or r6d, 0x00010001 + xor tmpd, r6d + mov seed, tmpd + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS tmp, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride, luma_bak +%endif + + movzx top_offxyd, offxyw + shr offxyd, 16 +%if ARCH_X86_32 + mov [rsp+7*mmsize+1*gprsize], top_offxyd +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %3 + mova m3, [PIC_ptr(pb_23_22)] +%else + mova m3, [PIC_ptr(pb_27_17)] +%endif +%%loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] +%if ARCH_X86_32 + mov r0, [rsp+7*mmsize+2*gprsize] ; topleft_offxy + mov r5, [rsp+7*mmsize+1*gprsize] ; top_offxy + movd m1, [grain_lutq+r0] + mov r0, [rsp+7*mmsize+0*gprsize] ; left_offxy +%else + movd m1, [grain_lutq+topleft_offxyq] +%endif + movu m2, [grain_lutq+offxyq] +%if ARCH_X86_32 + movu m6, [grain_lutq+r5] + movd m4, [grain_lutq+r0] +%else + movu m6, [grain_lutq+top_offxyq] + movd m4, [grain_lutq+left_offxyq] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m1, m6 + punpcklbw m4, m2 + pmaddubsw m0, m9, m1 + pmaddubsw m1, m9, m4 + REPX {pmulhrsw x, m8}, m0, m1 + packsswb m0, m1 + shufps m4, m0, m2, q3232 + shufps m0, m6, q3210 + ; followed by v interpolation (top | cur -> cur) + punpcklbw m2, m0, m4 + punpckhbw m0, m4 + pmaddubsw m4, m3, m0 + pmaddubsw m1, m3, m2 + pmulhrsw m4, m8 + pmulhrsw m1, m8 + packsswb m1, m4 + + ; src +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov lumaq, r9mp +%endif +%if %2 + mova m4, [lumaq+ 0] + mova m6, [lumaq+16] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq + mov r5, r5m + movd m7, [base+pb_1] +%else + movd m7, [pb_1] +%endif + pshufd m7, m7, q0000 + pxor m2, m2 + pmaddubsw m4, m7 + pmaddubsw m6, m7 + pavgw m4, m2 + pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif + +%if %1 +%if %2 + packuswb m4, m6 ; luma +%endif + punpckhbw m6, m4, m0 + punpcklbw m4, m0 ; { luma, chroma } + pmaddubsw m6, m14 + pmaddubsw m4, m14 + psraw m6, 6 + psraw m4, 6 + paddw m6, m15 + paddw m4, m15 + packuswb m4, m6 ; pack+unpack = clip + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 +%endif + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 +%else +%if %3 + vpgatherdw m7, m4, scalingq-1, r2, r12 + vpgatherdw m5, m6, scalingq-1, r2, r12 +%else + vpgatherdw m7, m4, scalingq-1, r2, r13 + vpgatherdw m5, m6, scalingq-1, r2, r13 +%endif +%endif + REPX {psrlw x, 8}, m7, m5 + + ; unpack grain + pxor m4, m4 + pcmpgtb m4, m1 + punpcklbw m2, m1, m4 + punpckhbw m1, m4 + + ; noise = round2(scaling[src] * grain, scaling_shift) + pmullw m2, m7 + pmullw m1, m5 + pmulhrsw m2, m11 + pmulhrsw m1, m11 + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + + ; unpack chroma source + pxor m4, m4 + punpckhbw m5, m0, m4 + punpcklbw m0, m4 ; m0-1: src as word + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m5, m1 + pmaxsw m0, m13 + pmaxsw m5, m13 + pminsw m0, m12 + pminsw m5, m12 + packuswb m0, m5 + movifnidn dstq, dstmp + mova [dstq+srcq], m0 + +%if ARCH_X86_32 + add srcq, r2mp + ; lumaq has been adjusted above already +%else + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*(1+%2)] +%else + add lumaq, r10mp +%endif +%endif + add grain_lutq, 82 + dec hw +%if %3 + jg %%loop_y_h_overlap +%else + jle %%end_y_hv_overlap +%if ARCH_X86_32 + mov r5, r5m +%endif + mova m3, [PIC_ptr(pb_17_27)] + btc hd, 16 + jnc %%loop_y_hv_overlap +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif + jmp %%loop_y_h_overlap +%%end_y_hv_overlap: +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif +%endif + +%if ARCH_X86_32 + DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp + mov lumaq, r11mp +%else + mov srcq, r11mp +%endif + lea lumaq, [luma_bakq+wq*(1+%2)] + add srcq, wq +%if ARCH_X86_32 + mov r4m, wq + mov r9m, lumaq +%endif +%if %2 + jmp %%loop_x_hv_overlap +%else +%if ARCH_X86_32 + add dword [rsp+7*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + xor dword r8m, 4 + jmp %%loop_x_odd_v_overlap +%endif + +%%end_hv: + RET +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro + +FGUV_FN 420, 1, 1 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 422, 1, 0 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/ipred.h b/third_party/dav1d/src/x86/ipred.h new file mode 100644 index 0000000000..415a4d8d62 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred.h @@ -0,0 +1,151 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/ipred.h" + +#define decl_fn(type, name) \ + decl_##type##_fn(BF(dav1d_##name, ssse3)); \ + decl_##type##_fn(BF(dav1d_##name, avx2)); \ + decl_##type##_fn(BF(dav1d_##name, avx512icl)) +#define init_fn(type0, type1, name, suffix) \ + c->type0[type1] = BF(dav1d_##name, suffix) + +#define init_angular_ipred_fn(type, name, suffix) \ + init_fn(intra_pred, type, name, suffix) +#define init_cfl_pred_fn(type, name, suffix) \ + init_fn(cfl_pred, type, name, suffix) +#define init_cfl_ac_fn(type, name, suffix) \ + init_fn(cfl_ac, type, name, suffix) + +decl_fn(angular_ipred, ipred_dc); +decl_fn(angular_ipred, ipred_dc_128); +decl_fn(angular_ipred, ipred_dc_top); +decl_fn(angular_ipred, ipred_dc_left); +decl_fn(angular_ipred, ipred_h); +decl_fn(angular_ipred, ipred_v); +decl_fn(angular_ipred, ipred_paeth); +decl_fn(angular_ipred, ipred_smooth); +decl_fn(angular_ipred, ipred_smooth_h); +decl_fn(angular_ipred, ipred_smooth_v); +decl_fn(angular_ipred, ipred_z1); +decl_fn(angular_ipred, ipred_z2); +decl_fn(angular_ipred, ipred_z3); +decl_fn(angular_ipred, ipred_filter); + +decl_fn(cfl_pred, ipred_cfl); +decl_fn(cfl_pred, ipred_cfl_128); +decl_fn(cfl_pred, ipred_cfl_top); +decl_fn(cfl_pred, ipred_cfl_left); + +decl_fn(cfl_ac, ipred_cfl_ac_420); +decl_fn(cfl_ac, ipred_cfl_ac_422); +decl_fn(cfl_ac, ipred_cfl_ac_444); + +decl_fn(pal_pred, pal_pred); + +static ALWAYS_INLINE void intra_pred_dsp_init_x86(Dav1dIntraPredDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + + init_angular_ipred_fn(DC_PRED, ipred_dc, ssse3); + init_angular_ipred_fn(DC_128_PRED, ipred_dc_128, ssse3); + init_angular_ipred_fn(TOP_DC_PRED, ipred_dc_top, ssse3); + init_angular_ipred_fn(LEFT_DC_PRED, ipred_dc_left, ssse3); + init_angular_ipred_fn(HOR_PRED, ipred_h, ssse3); + init_angular_ipred_fn(VERT_PRED, ipred_v, ssse3); + init_angular_ipred_fn(PAETH_PRED, ipred_paeth, ssse3); + init_angular_ipred_fn(SMOOTH_PRED, ipred_smooth, ssse3); + init_angular_ipred_fn(SMOOTH_H_PRED, ipred_smooth_h, ssse3); + init_angular_ipred_fn(SMOOTH_V_PRED, ipred_smooth_v, ssse3); +#if BITDEPTH == 8 + init_angular_ipred_fn(Z1_PRED, ipred_z1, ssse3); + init_angular_ipred_fn(Z2_PRED, ipred_z2, ssse3); + init_angular_ipred_fn(Z3_PRED, ipred_z3, ssse3); +#endif + init_angular_ipred_fn(FILTER_PRED, ipred_filter, ssse3); + + init_cfl_pred_fn(DC_PRED, ipred_cfl, ssse3); + init_cfl_pred_fn(DC_128_PRED, ipred_cfl_128, ssse3); + init_cfl_pred_fn(TOP_DC_PRED, ipred_cfl_top, ssse3); + init_cfl_pred_fn(LEFT_DC_PRED, ipred_cfl_left, ssse3); + + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I420 - 1, ipred_cfl_ac_420, ssse3); + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I422 - 1, ipred_cfl_ac_422, ssse3); + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I444 - 1, ipred_cfl_ac_444, ssse3); + + c->pal_pred = BF(dav1d_pal_pred, ssse3); + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + init_angular_ipred_fn(DC_PRED, ipred_dc, avx2); + init_angular_ipred_fn(DC_128_PRED, ipred_dc_128, avx2); + init_angular_ipred_fn(TOP_DC_PRED, ipred_dc_top, avx2); + init_angular_ipred_fn(LEFT_DC_PRED, ipred_dc_left, avx2); + init_angular_ipred_fn(HOR_PRED, ipred_h, avx2); + init_angular_ipred_fn(VERT_PRED, ipred_v, avx2); + init_angular_ipred_fn(PAETH_PRED, ipred_paeth, avx2); + init_angular_ipred_fn(SMOOTH_PRED, ipred_smooth, avx2); + init_angular_ipred_fn(SMOOTH_H_PRED, ipred_smooth_h, avx2); + init_angular_ipred_fn(SMOOTH_V_PRED, ipred_smooth_v, avx2); + init_angular_ipred_fn(Z1_PRED, ipred_z1, avx2); + init_angular_ipred_fn(Z2_PRED, ipred_z2, avx2); + init_angular_ipred_fn(Z3_PRED, ipred_z3, avx2); + init_angular_ipred_fn(FILTER_PRED, ipred_filter, avx2); + + init_cfl_pred_fn(DC_PRED, ipred_cfl, avx2); + init_cfl_pred_fn(DC_128_PRED, ipred_cfl_128, avx2); + init_cfl_pred_fn(TOP_DC_PRED, ipred_cfl_top, avx2); + init_cfl_pred_fn(LEFT_DC_PRED, ipred_cfl_left, avx2); + + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I420 - 1, ipred_cfl_ac_420, avx2); + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I422 - 1, ipred_cfl_ac_422, avx2); + init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I444 - 1, ipred_cfl_ac_444, avx2); + + c->pal_pred = BF(dav1d_pal_pred, avx2); + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + +#if BITDEPTH == 8 + init_angular_ipred_fn(DC_PRED, ipred_dc, avx512icl); + init_angular_ipred_fn(DC_128_PRED, ipred_dc_128, avx512icl); + init_angular_ipred_fn(TOP_DC_PRED, ipred_dc_top, avx512icl); + init_angular_ipred_fn(LEFT_DC_PRED, ipred_dc_left, avx512icl); + init_angular_ipred_fn(HOR_PRED, ipred_h, avx512icl); + init_angular_ipred_fn(VERT_PRED, ipred_v, avx512icl); +#endif + init_angular_ipred_fn(PAETH_PRED, ipred_paeth, avx512icl); + init_angular_ipred_fn(SMOOTH_PRED, ipred_smooth, avx512icl); + init_angular_ipred_fn(SMOOTH_H_PRED, ipred_smooth_h, avx512icl); + init_angular_ipred_fn(SMOOTH_V_PRED, ipred_smooth_v, avx512icl); + init_angular_ipred_fn(FILTER_PRED, ipred_filter, avx512icl); + + c->pal_pred = BF(dav1d_pal_pred, avx512icl); +#endif +} diff --git a/third_party/dav1d/src/x86/ipred16_avx2.asm b/third_party/dav1d/src/x86/ipred16_avx2.asm new file mode 100644 index 0000000000..7ddb189916 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred16_avx2.asm @@ -0,0 +1,4992 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 64 + +%macro SMOOTH_WEIGHTS 1-* +const smooth_weights_1d_16bpc ; sm_weights[] << 7 + %rep %0 + dw %1*128 + %rotate 1 + %endrep +const smooth_weights_2d_16bpc ; sm_weights[], 256 - sm_weights[] + %rep %0 + dw %1, 256-%1 + %rotate 1 + %endrep +%endmacro + +SMOOTH_WEIGHTS 0, 0, 255, 128, 255, 149, 85, 64, \ + 255, 197, 146, 105, 73, 50, 37, 32, \ + 255, 225, 196, 170, 145, 123, 102, 84, \ + 68, 54, 43, 33, 26, 20, 17, 16, \ + 255, 240, 225, 210, 196, 182, 169, 157, \ + 145, 133, 122, 111, 101, 92, 83, 74, \ + 66, 59, 52, 45, 39, 34, 29, 25, \ + 21, 17, 14, 12, 10, 9, 8, 8, \ + 255, 248, 240, 233, 225, 218, 210, 203, \ + 196, 189, 182, 176, 169, 163, 156, 150, \ + 144, 138, 133, 127, 121, 116, 111, 106, \ + 101, 96, 91, 86, 82, 77, 73, 69, \ + 65, 61, 57, 54, 50, 47, 44, 41, \ + 38, 35, 32, 29, 27, 25, 22, 20, \ + 18, 16, 15, 13, 12, 10, 9, 8, \ + 7, 6, 6, 5, 5, 4, 4, 4 + +%if ARCH_X86_64 + +ipred_hv_shuf: db 6, 7, 6, 7, 0, 1, 2, 3, 2, 3, 2, 3, 8, 9, 10, 11 + db 4, 5, 4, 5, 4, 5, 6, 7, 0, 1, 0, 1, 12, 13, 14, 15 +filter_shuf1: db 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 14, 15, 12, 13, -1, -1 +filter_shuf2: db 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 2, 3, -1, -1 +filter_shuf3: db 12, 13, 0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 8, 9, -1, -1 +pal_pred_shuf: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 +z_base_inc: dw 0*64, 1*64, 2*64, 3*64, 4*64, 5*64, 6*64, 7*64 + dw 8*64, 9*64, 10*64, 11*64, 12*64, 13*64, 14*64, 15*64 +z_filter_t0: db 55,127, 39,127, 39,127, 7, 15, 31, 7, 15, 31, 0, 3, 31, 0 +z_filter_t1: db 39, 63, 19, 47, 19, 47, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0 +z_filter_wh: db 7, 7, 11, 11, 15, 15, 19, 19, 19, 23, 23, 23, 31, 31, 31, 39 + db 39, 39, 47, 47, 47, 63, 63, 63, 79, 79, 79, -1 +pw_m1024: times 2 dw -1024 +pw_1to16: dw 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +pw_16to1: dw 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +z2_ymul: dw 1, 2, 1, 2, 1, 2, 1, 2, 3, 4, 3, 4, 3, 4, 3, 4 +z2_ymul8: dw 1, 2, 5, 6, 3, 4, 7, 8, 5, 6, 16, 16, 7, 8 +pb_90: times 4 db 90 +z2_y_shuf_h4: dd 3, 7, 2, 6, 1, 5, 0, 4 +z_upsample: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +z2_x_shuf: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 +z2_y_shuf: db 6, 7, 14, 15, 4, 5, 12, 13, 4, 5, 12, 13, 2, 3, 10, 11 +z2_y_shuf_us: db 6, 7, 14, 15, 2, 3, 10, 11, 4, 5, 12, 13, 0, 1, 8, 9 +z_filter_k: dw 4, 4, 5, 5, 4, 4 + dw 8, 8, 6, 6, 4, 4 + dw 0, 0, 0, 0, 2, 2 + +%define pw_2 (z_filter_k+32) +%define pw_4 (z_filter_k+ 0) +%define pw_16 (z2_ymul8 +20) + +pw_1: times 2 dw 1 +pw_3: times 2 dw 3 +pw_62: times 2 dw 62 +pw_512: times 2 dw 512 +pw_2048: times 2 dw 2048 +pd_8: dd 8 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_16bpc_avx2_table (ipred_dc_16bpc_avx2_table + 10*4) +%define ipred_cfl_splat_16bpc_avx2_table (ipred_cfl_16bpc_avx2_table + 8*4) + +JMP_TABLE ipred_dc_16bpc, avx2, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4 +JMP_TABLE ipred_dc_left_16bpc, avx2, h4, h8, h16, h32, h64 +JMP_TABLE ipred_h_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_paeth_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_h_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_v_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z1_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z2_16bpc, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z3_16bpc, avx2, h4, h8, h16, h32, h64 +JMP_TABLE ipred_filter_16bpc, avx2, w4, w8, w16, w32 +JMP_TABLE ipred_cfl_16bpc, avx2, h4, h8, h16, h32, w4, w8, w16, w32, \ + s4-8*4, s8-8*4, s16-8*4, s32-8*4 +JMP_TABLE ipred_cfl_left_16bpc, avx2, h4, h8, h16, h32 +JMP_TABLE ipred_cfl_ac_444_16bpc, avx2, w4, w8, w16, w32 +JMP_TABLE pal_pred_16bpc, avx2, w4, w8, w16, w32, w64 + +cextern dr_intra_derivative +cextern filter_intra_taps + +SECTION .text + +INIT_YMM avx2 +cglobal ipred_dc_top_16bpc, 3, 7, 6, dst, stride, tl, w, h + movifnidn hd, hm + add tlq, 2 + movd xm4, wd + pxor xm3, xm3 + pavgw xm4, xm3 + tzcnt wd, wd + movd xm5, wd + movu m0, [tlq] + lea r5, [ipred_dc_left_16bpc_avx2_table] + movsxd r6, [r5+wq*4] + add r6, r5 + add r5, ipred_dc_splat_16bpc_avx2_table-ipred_dc_left_16bpc_avx2_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +cglobal ipred_dc_left_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + mov hd, hm + sub tlq, hq + movd xm4, hd + sub tlq, hq + pxor xm3, xm3 + pavgw xm4, xm3 + tzcnt r6d, hd + movd xm5, r6d + movu m0, [tlq] + lea r5, [ipred_dc_left_16bpc_avx2_table] + movsxd r6, [r5+r6*4] + add r6, r5 + add r5, ipred_dc_splat_16bpc_avx2_table-ipred_dc_left_16bpc_avx2_table + tzcnt wd, wd + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + paddw m0, [tlq+96] + paddw m0, [tlq+64] +.h32: + paddw m0, [tlq+32] +.h16: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 +.h8: + psrldq xm1, xm0, 8 + paddw xm0, xm1 +.h4: + punpcklwd xm0, xm3 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + paddd xm0, xm4 + psrld xm0, xm5 + lea stride3q, [strideq*3] + vpbroadcastw m0, xm0 + mova m1, m0 + mova m2, m0 + mova m3, m0 + jmp wq + +cglobal ipred_dc_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd xm4, r5d + tzcnt r5d, r5d + movd xm5, r5d + lea r5, [ipred_dc_16bpc_avx2_table] + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+5*4] + pxor m3, m3 + psrlw xm4, 1 + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movq xm0, [tlq-8] + jmp wq +.w4: + movq xm1, [tlq+2] + paddw m0, m4 + paddw m0, m1 + psrlq m1, m0, 32 + paddw m0, m1 + psrld m1, m0, 16 + paddw m0, m1 + cmp hd, 4 + jg .w4_mul + psrlw xm0, 3 + jmp .w4_end +.w4_mul: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 + lea r2d, [hq*2] + mov r6d, 0xAAAB6667 + shrx r6d, r6d, r2d + punpckhwd xm1, xm0, xm3 + punpcklwd xm0, xm3 + paddd xm0, xm1 + movd xm1, r6d + psrld xm0, 2 + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w4_end: + vpbroadcastw xm0, xm0 +.s4: + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm0 + movq [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +ALIGN function_align +.h8: + mova xm0, [tlq-16] + jmp wq +.w8: + vextracti128 xm1, m0, 1 + paddw xm0, [tlq+2] + paddw xm0, xm4 + paddw xm0, xm1 + psrld xm1, xm0, 16 + paddw xm0, xm1 + pblendw xm0, xm3, 0xAA + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 8 + je .w8_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w8_end: + vpbroadcastw xm0, xm0 +.s8: + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm0 + mova [dstq+strideq*2], xm0 + mova [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +ALIGN function_align +.h16: + mova m0, [tlq-32] + jmp wq +.w16: + paddw m0, [tlq+2] + vextracti128 xm1, m0, 1 + paddw xm0, xm4 + paddw xm0, xm1 + punpckhwd xm1, xm0, xm3 + punpcklwd xm0, xm3 + paddd xm0, xm1 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 16 + je .w16_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + test hb, 8|32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w16_end: + vpbroadcastw m0, xm0 +.s16: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +ALIGN function_align +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-32] + jmp wq +.w32: + paddw m0, [tlq+ 2] + paddw m0, [tlq+34] + vextracti128 xm1, m0, 1 + paddw xm0, xm4 + paddw xm0, xm1 + punpcklwd xm1, xm0, xm3 + punpckhwd xm0, xm3 + paddd xm0, xm1 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x6667AAAB + shrx r6d, r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w32_end: + vpbroadcastw m0, xm0 + mova m1, m0 +.s32: + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m0 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*2+32*0], m0 + mova [dstq+strideq*2+32*1], m1 + mova [dstq+stride3q +32*0], m0 + mova [dstq+stride3q +32*1], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s32 + RET +ALIGN function_align +.h64: + mova m0, [tlq-128] + mova m1, [tlq- 96] + paddw m0, [tlq- 64] + paddw m1, [tlq- 32] + paddw m0, m1 + jmp wq +.w64: + movu m1, [tlq+ 2] + paddw m0, [tlq+34] + paddw m1, [tlq+66] + paddw m0, [tlq+98] + paddw m0, m1 + vextracti128 xm1, m0, 1 + paddw xm0, xm1 + punpcklwd xm1, xm0, xm3 + punpckhwd xm0, xm3 + paddd xm1, xm4 + paddd xm0, xm1 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 64 + je .w64_end + mov r6d, 0x6667AAAB + shrx r6d, r6d, hd + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w64_end: + vpbroadcastw m0, xm0 + mova m1, m0 + mova m2, m0 + mova m3, m0 +.s64: + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*0+32*2], m2 + mova [dstq+strideq*0+32*3], m3 + mova [dstq+strideq*1+32*0], m0 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*1+32*2], m2 + mova [dstq+strideq*1+32*3], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s64 + RET + +cglobal ipred_dc_128_16bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 + mov r6d, r8m + shr r6d, 11 + lea r5, [ipred_dc_splat_16bpc_avx2_table] + tzcnt wd, wd + movifnidn hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m0, [r5-ipred_dc_splat_16bpc_avx2_table+pw_512+r6*4] + mova m1, m0 + mova m2, m0 + mova m3, m0 + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_v_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + movu m0, [tlq+ 2] + movu m1, [tlq+34] + movu m2, [tlq+66] + movu m3, [tlq+98] + lea r5, [ipred_dc_splat_16bpc_avx2_table] + tzcnt wd, wd + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +%macro IPRED_H 2 ; w, store_type + vpbroadcastw m0, [tlq-2] + vpbroadcastw m1, [tlq-4] + vpbroadcastw m2, [tlq-6] + vpbroadcastw m3, [tlq-8] + sub tlq, 8 + mov%2 [dstq+strideq*0], m0 + mov%2 [dstq+strideq*1], m1 + mov%2 [dstq+strideq*2], m2 + mov%2 [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w%1 + RET +ALIGN function_align +%endmacro + +cglobal ipred_h_16bpc, 3, 6, 4, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + lea r5, [ipred_h_16bpc_avx2_table] + tzcnt wd, wd + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +INIT_XMM avx2 +.w4: + IPRED_H 4, q +.w8: + IPRED_H 8, a +INIT_YMM avx2 +.w16: + IPRED_H 16, a +.w32: + vpbroadcastw m0, [tlq-2] + vpbroadcastw m1, [tlq-4] + vpbroadcastw m2, [tlq-6] + vpbroadcastw m3, [tlq-8] + sub tlq, 8 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m0 + mova [dstq+strideq*1+32*0], m1 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*2+32*0], m2 + mova [dstq+strideq*2+32*1], m2 + mova [dstq+stride3q +32*0], m3 + mova [dstq+stride3q +32*1], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w32 + RET +.w64: + vpbroadcastw m0, [tlq-2] + vpbroadcastw m1, [tlq-4] + sub tlq, 4 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m0 + mova [dstq+strideq*0+32*2], m0 + mova [dstq+strideq*0+32*3], m0 + mova [dstq+strideq*1+32*0], m1 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*1+32*2], m1 + mova [dstq+strideq*1+32*3], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w64 + RET + +%macro PAETH 3 ; top, signed_ldiff, ldiff + paddw m0, m%2, m1 + psubw m7, m3, m0 ; tldiff + psubw m0, m%1 ; tdiff + pabsw m7, m7 + pabsw m0, m0 + pminsw m7, m0 + pcmpeqw m0, m7 + pcmpgtw m7, m%3, m7 + vpblendvb m0, m3, m%1, m0 + vpblendvb m0, m1, m0, m7 +%endmacro + +cglobal ipred_paeth_16bpc, 3, 6, 8, dst, stride, tl, w, h +%define base r5-ipred_paeth_16bpc_avx2_table + movifnidn hd, hm + lea r5, [ipred_paeth_16bpc_avx2_table] + tzcnt wd, wd + movsxd wq, [r5+wq*4] + vpbroadcastw m3, [tlq] ; topleft + add wq, r5 + jmp wq +.w4: + vpbroadcastq m2, [tlq+2] ; top + movsldup m6, [base+ipred_hv_shuf] + lea r3, [strideq*3] + psubw m4, m2, m3 + pabsw m5, m4 +.w4_loop: + sub tlq, 8 + vpbroadcastq m1, [tlq] + pshufb m1, m6 ; left + PAETH 2, 4, 5 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_loop + RET +ALIGN function_align +.w8: + vbroadcasti128 m2, [tlq+2] + movsldup m6, [base+ipred_hv_shuf] + psubw m4, m2, m3 + pabsw m5, m4 +.w8_loop: + sub tlq, 4 + vpbroadcastd m1, [tlq] + pshufb m1, m6 + PAETH 2, 4, 5 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + movu m2, [tlq+2] + psubw m4, m2, m3 + pabsw m5, m4 +.w16_loop: + sub tlq, 2 + vpbroadcastw m1, [tlq] + PAETH 2, 4, 5 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w16_loop + RET +ALIGN function_align +.w32: + movu m2, [tlq+2] + movu m6, [tlq+34] +%if WIN64 + movaps r4m, xmm8 + movaps r6m, xmm9 +%endif + psubw m4, m2, m3 + psubw m8, m6, m3 + pabsw m5, m4 + pabsw m9, m8 +.w32_loop: + sub tlq, 2 + vpbroadcastw m1, [tlq] + PAETH 2, 4, 5 + mova [dstq+32*0], m0 + PAETH 6, 8, 9 + mova [dstq+32*1], m0 + add dstq, strideq + dec hd + jg .w32_loop +%if WIN64 + movaps xmm8, r4m + movaps xmm9, r6m +%endif + RET +ALIGN function_align +.w64: + WIN64_SPILL_XMM 16 + movu m2, [tlq+ 2] + movu m6, [tlq+34] + movu m10, [tlq+66] + movu m13, [tlq+98] + psubw m4, m2, m3 + psubw m8, m6, m3 + psubw m11, m10, m3 + psubw m14, m13, m3 + pabsw m5, m4 + pabsw m9, m8 + pabsw m12, m11 + pabsw m15, m14 +.w64_loop: + sub tlq, 2 + vpbroadcastw m1, [tlq] + PAETH 2, 4, 5 + mova [dstq+32*0], m0 + PAETH 6, 8, 9 + mova [dstq+32*1], m0 + PAETH 10, 11, 12 + mova [dstq+32*2], m0 + PAETH 13, 14, 15 + mova [dstq+32*3], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_smooth_v_16bpc, 3, 7, 6, dst, stride, tl, w, h, weights +%define base r6-ipred_smooth_v_16bpc_avx2_table + lea r6, [ipred_smooth_v_16bpc_avx2_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + lea weightsq, [base+smooth_weights_1d_16bpc+hq*4] + neg hq + vpbroadcastw m5, [tlq+hq*2] ; bottom + add wq, r6 + jmp wq +.w4: + vpbroadcastq m4, [tlq+2] ; top + movsldup m3, [base+ipred_hv_shuf] + lea r6, [strideq*3] + psubw m4, m5 ; top - bottom +.w4_loop: + vpbroadcastq m0, [weightsq+hq*2] + pshufb m0, m3 + pmulhrsw m0, m4 + paddw m0, m5 + vextracti128 xm1, m0, 1 + movhps [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movq [dstq+r6 ], xm0 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w4_loop +.ret: + RET +.w8: + vbroadcasti128 m4, [tlq+2] + movsldup m3, [base+ipred_hv_shuf] + lea r6, [strideq*3] + psubw m4, m5 +.w8_loop: + vpbroadcastd m0, [weightsq+hq*2+0] + vpbroadcastd m1, [weightsq+hq*2+4] + pshufb m0, m3 + pshufb m1, m3 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + vextracti128 [dstq+strideq*0], m0, 1 + mova [dstq+strideq*1], xm0 + vextracti128 [dstq+strideq*2], m1, 1 + mova [dstq+r6 ], xm1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w8_loop + RET +.w16: + movu m4, [tlq+2] + lea r6, [strideq*3] + psubw m4, m5 +.w16_loop: + vpbroadcastw m0, [weightsq+hq*2+0] + vpbroadcastw m1, [weightsq+hq*2+2] + vpbroadcastw m2, [weightsq+hq*2+4] + vpbroadcastw m3, [weightsq+hq*2+6] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r6 ], m3 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w16_loop + RET +.w32: + WIN64_SPILL_XMM 7 + movu m4, [tlq+ 2] + movu m6, [tlq+34] + psubw m4, m5 + psubw m6, m5 +.w32_loop: + vpbroadcastw m1, [weightsq+hq*2+0] + vpbroadcastw m3, [weightsq+hq*2+2] + pmulhrsw m0, m4, m1 + pmulhrsw m1, m6 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m6 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w32_loop + RET +.w64: + WIN64_SPILL_XMM 8 + movu m3, [tlq+ 2] + movu m4, [tlq+34] + movu m6, [tlq+66] + movu m7, [tlq+98] + REPX {psubw x, m5}, m3, m4, m6, m7 +.w64_loop: + vpbroadcastw m2, [weightsq+hq*2] + pmulhrsw m0, m3, m2 + pmulhrsw m1, m4, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*0], m0 + pmulhrsw m0, m6, m2 + mova [dstq+32*1], m1 + pmulhrsw m1, m7, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + add dstq, strideq + inc hq + jl .w64_loop + RET + +cglobal ipred_smooth_h_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 +%define base r6-ipred_smooth_h_16bpc_avx2_table + lea r6, [ipred_smooth_h_16bpc_avx2_table] + mov wd, wm + movifnidn hd, hm + vpbroadcastw m5, [tlq+wq*2] ; right + tzcnt wd, wd + add hd, hd + movsxd wq, [r6+wq*4] + sub tlq, hq + lea stride3q, [strideq*3] + add wq, r6 + jmp wq +.w4: + vpbroadcastq m4, [base+smooth_weights_1d_16bpc+4*2] + movsldup m3, [base+ipred_hv_shuf] +.w4_loop: + vpbroadcastq m0, [tlq+hq-8] ; left + pshufb m0, m3 + psubw m0, m5 ; left - right + pmulhrsw m0, m4 + paddw m0, m5 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 + jg .w4_loop + RET +.w8: + vbroadcasti128 m4, [base+smooth_weights_1d_16bpc+8*2] + movsldup m3, [base+ipred_hv_shuf] +.w8_loop: + vpbroadcastd m0, [tlq+hq-4] + vpbroadcastd m1, [tlq+hq-8] + pshufb m0, m3 + pshufb m1, m3 + psubw m0, m5 + psubw m1, m5 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 + jg .w8_loop + RET +.w16: + movu m4, [base+smooth_weights_1d_16bpc+16*2] +.w16_loop: + vpbroadcastq m3, [tlq+hq-8] + punpcklwd m3, m3 + psubw m3, m5 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 + jg .w16_loop + RET +.w32: + WIN64_SPILL_XMM 7 + movu m4, [base+smooth_weights_1d_16bpc+32*2] + movu m6, [base+smooth_weights_1d_16bpc+32*3] +.w32_loop: + vpbroadcastw m1, [tlq+hq-2] + vpbroadcastw m3, [tlq+hq-4] + psubw m1, m5 + psubw m3, m5 + pmulhrsw m0, m4, m1 + pmulhrsw m1, m6 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m6 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + lea dstq, [dstq+strideq*2] + sub hq, 2*2 + jg .w32_loop + RET +.w64: + WIN64_SPILL_XMM 8 + movu m3, [base+smooth_weights_1d_16bpc+32*4] + movu m4, [base+smooth_weights_1d_16bpc+32*5] + movu m6, [base+smooth_weights_1d_16bpc+32*6] + movu m7, [base+smooth_weights_1d_16bpc+32*7] +.w64_loop: + vpbroadcastw m2, [tlq+hq-2] + psubw m2, m5 + pmulhrsw m0, m3, m2 + pmulhrsw m1, m4, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*0], m0 + pmulhrsw m0, m6, m2 + mova [dstq+32*1], m1 + pmulhrsw m1, m7, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + add dstq, strideq + sub hq, 1*2 + jg .w64_loop + RET + +%macro SMOOTH_2D_END 6 ; src[1-2], mul[1-2], add[1-2] + pmaddwd m0, m%1, m%3 + pmaddwd m1, m%2, m%4 + paddd m0, m%5 + paddd m1, m%6 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pavgw m0, m5 +%endmacro + +cglobal ipred_smooth_16bpc, 3, 7, 6, dst, stride, tl, w, h, v_weights +%define base r6-ipred_smooth_16bpc_avx2_table + lea r6, [ipred_smooth_16bpc_avx2_table] + mov wd, wm + vpbroadcastw m4, [tlq+wq*2] ; right + tzcnt wd, wd + mov hd, hm + sub tlq, hq + sub tlq, hq + movsxd wq, [r6+wq*4] + pxor m5, m5 + add wq, r6 + lea v_weightsq, [base+smooth_weights_2d_16bpc+hq*4] + jmp wq +.w4: + WIN64_SPILL_XMM 11 + vpbroadcastw m0, [tlq] ; bottom + vpbroadcastq m6, [tlq+hq*2+2] + movsldup m7, [base+ipred_hv_shuf] + movshdup m9, [base+ipred_hv_shuf] + vbroadcasti128 m10, [base+smooth_weights_2d_16bpc+4*4] + punpcklwd m6, m0 ; top, bottom + punpcklqdq m8, m9, m9 + punpckhqdq m9, m9 + lea r3, [strideq*3] +.w4_loop: + vpbroadcastq m3, [tlq+hq*2-8] + vbroadcasti128 m1, [v_weightsq] + pshufb m3, m7 + punpcklwd m2, m3, m4 ; left, right + punpckhwd m3, m4 + pmaddwd m2, m10 + pmaddwd m3, m10 + pshufb m0, m1, m8 + pshufb m1, m9 + SMOOTH_2D_END 0, 1, 6, 6, 2, 3 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + add v_weightsq, 16 + sub hd, 4 + jg .w4_loop + RET +.w8: +%assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 12 + vpbroadcastw m0, [tlq] ; bottom + vbroadcasti128 m7, [tlq+hq*2+2] + movsldup m8, [base+ipred_hv_shuf] + movshdup m9, [base+ipred_hv_shuf] + vbroadcasti128 m10, [base+smooth_weights_2d_16bpc+8*4+16*0] + vbroadcasti128 m11, [base+smooth_weights_2d_16bpc+8*4+16*1] + punpcklwd m6, m7, m0 ; top, bottom + punpckhwd m7, m0 +.w8_loop: + vpbroadcastd m3, [tlq+hq*2-4] + vpbroadcastq m1, [v_weightsq] + pshufb m3, m8 + punpcklwd m2, m3, m4 ; left, right + punpckhwd m3, m4 + pmaddwd m2, m10 + pmaddwd m3, m11 + pshufb m1, m9 + SMOOTH_2D_END 1, 1, 6, 7, 2, 3 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + add v_weightsq, 8 + sub hd, 2 + jg .w8_loop + RET +.w16: +%assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 11 + vpbroadcastw m0, [tlq] ; bottom + movu m7, [tlq+hq*2+2] + mova xm8, [base+smooth_weights_2d_16bpc+16*4+16*0] + mova xm9, [base+smooth_weights_2d_16bpc+16*4+16*1] + vinserti128 m8, [base+smooth_weights_2d_16bpc+16*4+16*2], 1 + vinserti128 m9, [base+smooth_weights_2d_16bpc+16*4+16*3], 1 + punpcklwd m6, m7, m0 ; top, bottom + punpckhwd m7, m0 +.w16_loop: + vpbroadcastd m3, [tlq+hq*2-4] + vpbroadcastd m1, [v_weightsq+0] + punpcklwd m3, m4 ; left, right + pshufd m2, m3, q1111 + pmaddwd m10, m8, m2 + pmaddwd m2, m9 + pshufd m3, m3, q0000 + SMOOTH_2D_END 1, 1, 6, 7, 10, 2 + vpbroadcastd m1, [v_weightsq+4] + pmaddwd m2, m8, m3 + pmaddwd m3, m9 + mova [dstq+strideq*0], m0 + SMOOTH_2D_END 1, 1, 6, 7, 2, 3 + mova [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + add v_weightsq, 8 + sub hq, 2 + jg .w16_loop + RET +.w32: +%assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 15 + vpbroadcastw m0, [tlq] ; bottom + movu m7, [tlq+hq*2+ 2] + movu m9, [tlq+hq*2+34] + mova xm10, [base+smooth_weights_2d_16bpc+32*4+16*0] + mova xm11, [base+smooth_weights_2d_16bpc+32*4+16*1] + vinserti128 m10, [base+smooth_weights_2d_16bpc+32*4+16*2], 1 + vinserti128 m11, [base+smooth_weights_2d_16bpc+32*4+16*3], 1 + mova xm12, [base+smooth_weights_2d_16bpc+32*4+16*4] + mova xm13, [base+smooth_weights_2d_16bpc+32*4+16*5] + vinserti128 m12, [base+smooth_weights_2d_16bpc+32*4+16*6], 1 + vinserti128 m13, [base+smooth_weights_2d_16bpc+32*4+16*7], 1 + punpcklwd m6, m7, m0 + punpckhwd m7, m0 + punpcklwd m8, m9, m0 + punpckhwd m9, m0 +.w32_loop: + vpbroadcastw m3, [tlq+hq*2-2] + vpbroadcastd m14, [v_weightsq] + punpcklwd m3, m4 + pmaddwd m1, m10, m3 + pmaddwd m2, m11, m3 + pmaddwd m0, m6, m14 + paddd m0, m1 + pmaddwd m1, m7, m14 + paddd m1, m2 + pmaddwd m2, m12, m3 + pmaddwd m3, m13 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pavgw m0, m5 + mova [dstq+32*0], m0 + SMOOTH_2D_END 14, 14, 8, 9, 2, 3 + mova [dstq+32*1], m0 + add dstq, strideq + add v_weightsq, 4 + dec hd + jg .w32_loop + RET +.w64: +%assign stack_offset stack_offset - stack_size_padded + PROLOGUE 0, 11, 16, dst, stride, tl, tl_base, h, v_weights, dummy, v_weights_base, x, y, dst_base + mov dst_baseq, dstq + mov tl_baseq, tlq + mov v_weights_baseq, v_weightsq + xor xq, xq +.w64_loop_x: + mov yq, hq + lea tlq, [tl_baseq+hq*2] + vpbroadcastw m0, [tl_baseq] ; bottom + movu m7, [tlq+xq*2+ 2] + movu m9, [tlq+xq*2+34] + mova xm10, [base+smooth_weights_2d_16bpc+64*4+16*0] + mova xm11, [base+smooth_weights_2d_16bpc+64*4+16*1] + vinserti128 m10, [base+smooth_weights_2d_16bpc+64*4+16*2], 1 + vinserti128 m11, [base+smooth_weights_2d_16bpc+64*4+16*3], 1 + mova xm12, [base+smooth_weights_2d_16bpc+64*4+16*4] + mova xm13, [base+smooth_weights_2d_16bpc+64*4+16*5] + vinserti128 m12, [base+smooth_weights_2d_16bpc+64*4+16*6], 1 + vinserti128 m13, [base+smooth_weights_2d_16bpc+64*4+16*7], 1 + punpcklwd m6, m7, m0 + punpckhwd m7, m0 + punpcklwd m8, m9, m0 + punpckhwd m9, m0 + lea tlq, [tl_baseq-2] +.w64_loop_y: + vpbroadcastw m3, [tlq+yq*2] + vpbroadcastd m1, [v_weightsq] + punpcklwd m3, m4 + pmaddwd m14, m10, m3 + pmaddwd m15, m11, m3 + pmaddwd m2, m12, m3 + pmaddwd m3, m13 + pmaddwd m0, m6, m1 + paddd m0, m14 + pmaddwd m14, m7, m1 + paddd m14, m15 + psrld m0, 8 + psrld m14, 8 + packssdw m0, m14 + pavgw m0, m5 + mova [dstq+32*0], m0 + SMOOTH_2D_END 8, 9, 1, 1, 2, 3 + mova [dstq+32*1], m0 + add dstq, strideq + add v_weightsq, 4 + dec yq + jg .w64_loop_y + lea dstq, [dst_baseq+32*2] + add r6, 16*8 + mov v_weightsq, v_weights_baseq + add xq, 32 + test xb, 64 + jz .w64_loop_x + RET + +cglobal ipred_z1_16bpc, 3, 8, 0, dst, stride, tl, w, h, angle, dx, maxbase + %assign org_stack_offset stack_offset + lea r6, [ipred_z1_16bpc_avx2_table] + tzcnt wd, wm + movifnidn angled, anglem + movifnidn hd, hm + lea r7, [dr_intra_derivative] + movsxd wq, [r6+wq*4] + add tlq, 2 + add wq, r6 + mov dxd, angled + and dxd, 0x7e + add angled, 165 ; ~90 + movzx dxd, word [r7+dxq] + xor angled, 0x4ff ; d = 90 - angle + vpbroadcastd m5, [pw_62] + jmp wq +.w4: + ALLOC_STACK -64, 7 + cmp angleb, 40 + jae .w4_no_upsample + lea r3d, [angleq-1024] + sar r3d, 7 + add r3d, hd + jg .w4_no_upsample ; !enable_intra_edge_filter || h > 8 || (h == 8 && is_sm) + vpbroadcastw xm3, [tlq+14] + movu xm1, [tlq+ 0] ; 1 2 3 4 5 6 7 8 + palignr xm0, xm3, xm1, 4 ; 3 4 5 6 7 8 8 8 + paddw xm0, [tlq- 2] ; 0 1 2 3 4 5 6 7 + add dxd, dxd + palignr xm2, xm3, xm1, 2 ; 2 3 4 5 6 7 8 8 + paddw xm2, xm1 ; -1 * a + 9 * b + 9 * c + -1 * d + psubw xm0, xm2, xm0 ; = (b + c - a - d + (b + c) << 3 + 8) >> 4 + psraw xm0, 3 ; = ((b + c - a - d) >> 3 + b + c + 1) >> 1 + pxor xm4, xm4 + paddw xm2, xm0 + vpbroadcastw xm0, r8m ; pixel_max + mova [rsp+32], xm3 + movd xm3, dxd + pmaxsw xm2, xm4 + mov r3d, dxd + pavgw xm2, xm4 + vpbroadcastw m3, xm3 + pminsw xm2, xm0 + punpcklwd xm0, xm1, xm2 + punpckhwd xm1, xm2 + lea r5, [strideq*3] + pslldq m2, m3, 8 + mova [rsp+ 0], xm0 + mova [rsp+16], xm1 + paddw m6, m3, m3 + paddw m3, m2 + vpblendd m4, m6, 0xf0 + paddw m6, m6 + paddw m3, m4 ; xpos0 xpos1 xpos2 xpos3 + vbroadcasti128 m4, [z_upsample] +.w4_upsample_loop: + lea r2d, [r3+dxq] + shr r3d, 6 ; base0 + movu xm1, [rsp+r3*2] + lea r3d, [r2+dxq] + shr r2d, 6 ; base1 + movu xm2, [rsp+r2*2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base2 + vinserti128 m1, [rsp+r3*2], 1 ; 0 2 + lea r3d, [r2+dxq] + shr r2d, 6 ; base3 + vinserti128 m2, [rsp+r2*2], 1 ; 1 3 + pshufb m1, m4 + pshufb m2, m4 + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + pand m2, m5, m3 ; frac + psllw m2, 9 ; (a * (64 - frac) + b * frac + 32) >> 6 + psubw m1, m0 ; = a + (((b - a) * frac + 32) >> 6) + pmulhrsw m1, m2 ; = a + (((b - a) * (frac << 9) + 16384) >> 15) + paddw m3, m6 ; xpos += dx + paddw m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r5 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_upsample_loop + RET +ALIGN function_align +.filter_strength: ; w4/w8/w16 +%define base r3-z_filter_t0 + movd xm0, maxbased + lea r3, [z_filter_t0] + movd xm1, angled + shr angled, 8 ; is_sm << 1 + vpbroadcastb m0, xm0 + vpbroadcastb m1, xm1 + pcmpeqb m0, [base+z_filter_wh] + mova xm2, [r3+angleq*8] + pand m0, m1 + pcmpgtb m0, m2 + pmovmskb r5d, m0 + ret +.w4_no_upsample: + mov maxbased, 7 + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w4_main + lea maxbased, [hq+3] + call .filter_strength + mov maxbased, 7 + test r5d, r5d + jz .w4_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastw xm3, [tlq+14] + mova xm0, [tlq- 2] ; 0 1 2 3 4 5 6 7 + vpbroadcastd xm1, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd xm4, [base+z_filter_k-4+r5*4+12*0] + palignr xm2, xm3, xm0, 4 ; 2 3 4 5 6 7 8 8 + pmullw xm1, [tlq+ 0] ; 1 2 3 4 5 6 7 8 + paddw xm2, xm0 + pmullw xm2, xm4 + movd [rsp+16], xm3 + cmp r5d, 3 + jne .w4_3tap + paddw xm1, xm2 + palignr xm2, xm3, xm0, 6 ; 3 4 5 6 7 8 8 8 + pblendw xm0, [tlq-4], 0xfe ; 0 0 1 2 3 4 5 6 + movzx r3d, word [tlq+14] + movzx r2d, word [tlq+12] + inc maxbased + paddw xm2, xm0 + sub r2d, r3d + paddw xm2, xm2 + lea r2d, [r2+r3*8+4] + shr r2d, 3 ; (1 * top[6] + 7 * top[7] + 4) >> 3 + mov [rsp+16], r2w +.w4_3tap: + pxor xm0, xm0 + paddw xm1, xm2 + mov tlq, rsp + psrlw xm1, 3 + cmp hd, 8 + sbb maxbased, -1 + pavgw xm0, xm1 + mova [tlq], xm0 +.w4_main: + movd xm3, dxd + vpbroadcastq m1, [z_base_inc] + vpbroadcastw m6, [tlq+maxbaseq*2] ; top[max_base_x] + shl maxbased, 6 + vpbroadcastw m3, xm3 + movd xm0, maxbased + mov r3d, dxd ; xpos + vpbroadcastw m0, xm0 + paddw m4, m3, m3 + psubw m1, m0 ; -max_base_x + vpblendd m3, m4, 0xcc + paddw m0, m4, m3 + vpblendd m3, m0, 0xf0 ; xpos0 xpos1 xpos2 xpos3 + paddw m4, m4 + paddw m3, m1 +.w4_loop: + lea r5d, [r3+dxq] + shr r3d, 6 ; base0 + movu xm1, [tlq+r3*2] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + movu xm2, [tlq+r5*2] + lea r5d, [r3+dxq] + shr r3d, 6 ; base2 + vinserti128 m1, [tlq+r3*2], 1 ; 0 2 + lea r3d, [r5+dxq] + shr r5d, 6 ; base3 + vinserti128 m2, [tlq+r5*2], 1 ; 1 3 + punpcklqdq m0, m1, m2 + psrldq m1, 2 + pslldq m2, 6 + vpblendd m1, m2, 0xcc + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + psraw m2, m3, 15 ; xpos < max_base_x + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + sub hd, 4 + jz .w4_end + lea dstq, [dstq+strideq*2] + cmp r3d, maxbased + jb .w4_loop + lea r6, [strideq*3] +.w4_end_loop: + movq [dstq+strideq*0], xm6 + movq [dstq+strideq*1], xm6 + movq [dstq+strideq*2], xm6 + movq [dstq+r6 ], xm6 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_end_loop +.w4_end: + RET +.w8: + %assign stack_offset org_stack_offset + ALLOC_STACK -64, 7 + lea r3d, [angleq+216] + mov r3b, hb + cmp r3d, 8 + ja .w8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || h > 8 + movu m2, [tlq+2] ; 2 3 4 5 6 7 8 9 a b c d e f g _ + movu m0, [tlq+4] ; 3 4 5 6 7 8 9 a b c d e f g _ _ + movu m1, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + cmp hd, 4 + jne .w8_upsample_h8 ; awkward single-pixel edge case + vpblendd m0, m2, 0x20 ; 3 4 5 6 7 8 9 a b c c _ _ _ _ _ +.w8_upsample_h8: + paddw m2, m1 + paddw m0, [tlq-2] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + add dxd, dxd + psubw m0, m2, m0 + psraw m0, 3 + pxor m4, m4 + paddw m2, m0 + vpbroadcastw m0, r8m + movd xm3, dxd + pmaxsw m2, m4 + mov r3d, dxd + pavgw m2, m4 + vpbroadcastw m3, xm3 + pminsw m2, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + vbroadcasti128 m4, [z_upsample] + mova [rsp+ 0], xm0 + mova [rsp+16], xm1 + paddw m6, m3, m3 + vextracti128 [rsp+32], m0, 1 + vextracti128 [rsp+48], m1, 1 + vpblendd m3, m6, 0xf0 ; xpos0 xpos1 +.w8_upsample_loop: + lea r2d, [r3+dxq] + shr r3d, 6 ; base0 + movu xm1, [rsp+r3*2] + movu xm2, [rsp+r3*2+16] + lea r3d, [r2+dxq] + shr r2d, 6 ; base1 + vinserti128 m1, [rsp+r2*2], 1 + vinserti128 m2, [rsp+r2*2+16], 1 + pshufb m1, m4 + pshufb m2, m4 + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m3, m6 + paddw m0, m1 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_upsample_loop + RET +.w8_no_intra_edge_filter: + and maxbased, 7 + or maxbased, 8 ; imin(h+7, 15) + jmp .w8_main +.w8_no_upsample: + lea maxbased, [hq+7] + test angled, 0x400 + jnz .w8_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .w8_main + popcnt r5d, r5d + vpbroadcastd m1, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m4, [base+z_filter_k-4+r5*4+12*0] + mova m0, [tlq-2] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + movu m2, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pmullw m1, m2 + cmp hd, 8 + jl .w8_filter_h4 + punpckhwd m2, m2 + vpblendd m3, m2, [tlq+2], 0x7f ; 2 3 4 5 6 7 8 9 a b c d e f g g + je .w8_filter_end ; 8x4 and 8x8 are always 3-tap + movzx r3d, word [tlq+30] + mov maxbased, 16 + mov [rsp+32], r3d + cmp r5d, 3 + jne .w8_filter_end + punpcklwd xm6, xm0, xm0 + vpblendd m2, [tlq+4], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g g g + vpblendd m6, [tlq-4], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + movzx r5d, word [tlq+28] + mov [rsp+34], r3w + paddw m2, m6 + sub r5d, r3d + inc maxbased + paddw m2, m2 + lea r3d, [r5+r3*8+4] + paddw m1, m2 + shr r3d, 3 + mov [rsp+32], r3w + jmp .w8_filter_end +.w8_filter_h4: + pshuflw m3, m2, q3321 + vinserti128 m3, [tlq+2], 0 ; 2 3 4 5 6 7 8 9 a b c c _ _ _ _ +.w8_filter_end: + paddw m0, m3 + pmullw m0, m4 + mov tlq, rsp + pxor m2, m2 + paddw m0, m1 + psrlw m0, 3 + pavgw m0, m2 + mova [tlq], m0 +.w8_main: + movd xm3, dxd + vbroadcasti128 m1, [z_base_inc] + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m3, xm3 + movd xm0, maxbased + mov r3d, dxd + vpbroadcastw m0, xm0 + paddw m4, m3, m3 + psubw m1, m0 + vpblendd m3, m4, 0xf0 ; xpos0 xpos1 + paddw m3, m1 +.w8_loop: + lea r5d, [r3+dxq] + shr r3d, 6 + movu xm0, [tlq+r3*2] + movu xm1, [tlq+r3*2+2] + lea r3d, [r5+dxq] + shr r5d, 6 + vinserti128 m0, [tlq+r5*2], 1 + vinserti128 m1, [tlq+r5*2+2], 1 + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + psraw m2, m3, 15 + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + sub hd, 2 + jz .w8_end + lea dstq, [dstq+strideq*2] + cmp r3d, maxbased + jb .w8_loop +.w8_end_loop: + mova [dstq+strideq*0], xm6 + mova [dstq+strideq*1], xm6 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_end_loop +.w8_end: + RET +.w16_no_intra_edge_filter: + and maxbased, 15 + or maxbased, 16 ; imin(h+15, 31) + jmp .w16_main +.w16: + %assign stack_offset org_stack_offset + ALLOC_STACK -96, 7 + lea maxbased, [hq+15] + test angled, 0x400 + jnz .w16_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .w16_main + popcnt r5d, r5d + mova m0, [tlq-2] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m1, m0, [tlq+2] ; 2 3 4 5 6 7 8 9 a b c d e f g h + cmp r5d, 3 + jne .w16_filter_3tap + vpbroadcastd m2, [base+pw_3] + punpcklwd xm0, xm0 + vpblendd m0, [tlq-4], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + paddw m1, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + paddw m0, m2 + pavgw m0, [tlq+4] ; 3 4 5 6 7 8 9 a b c d e f g h i + paddw m0, m1 + psrlw m0, 2 + movu m3, [tlq+32] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m2, [tlq+28] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m1, m3, [tlq+30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + cmp hd, 8 + jl .w16_filter_5tap_h4 + punpckhwd m3, m3 + je .w16_filter_5tap_h8 + vpblendd m4, m3, [tlq+36], 0x7f ; 4 5 6 7 8 9 a b c d e f g h h h + vpblendd m3, [tlq+34], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g h h + movzx r3d, word [tlq+62] + movzx r2d, word [tlq+60] + pavgw m2, m4 + sub r2d, r3d + paddw m1, m3 + lea r2d, [r2+r3*8+4] + paddw m1, m2 + shr r2d, 3 + psrlw m1, 2 + mov [rsp+66], r3w + mov [rsp+64], r2w + mov tlq, rsp + mov r3d, 33 + cmp hd, 16 + cmovg maxbased, r3d + jmp .w16_filter_end2 +.w16_filter_5tap_h8: + vpblendd xm4, xm3, [tlq+36], 0x07 ; 4 5 6 7 8 9 9 9 + vpblendd xm3, [tlq+34], 0x07 ; 3 4 5 6 7 8 9 9 + pavgw xm2, xm4 + paddw xm1, xm3 + paddw xm1, xm2 + psrlw xm1, 2 + jmp .w16_filter_end2 +.w16_filter_5tap_h4: + pshuflw xm4, xm3, q3332 ; 4 5 5 5 + pshuflw xm3, xm3, q3321 ; 3 4 5 5 + pavgw xm2, xm4 + paddw xm1, xm3 + paddw xm1, xm2 + psrlw xm1, 2 + jmp .w16_filter_end2 +.w16_filter_3tap: + vpbroadcastd m3, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m4, [base+z_filter_k-4+r5*4+12*0] + pmullw m0, m3, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + movu m2, [tlq+32] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pmullw m1, m4 + pmullw m3, m2 + paddw m0, m1 + cmp hd, 8 + je .w16_filter_3tap_h8 + jl .w16_filter_3tap_h4 + punpckhwd m2, m2 + vpblendd m2, [tlq+34], 0x7f ; 2 3 4 5 6 7 8 9 a b c d e f g g + jmp .w16_filter_end +.w16_filter_3tap_h4: + pshuflw xm2, xm2, q3321 ; 2 3 4 4 _ _ _ _ + jmp .w16_filter_end +.w16_filter_3tap_h8: + psrldq xm2, 2 + pshufhw xm2, xm2, q2210 ; 2 3 4 5 6 7 8 8 +.w16_filter_end: + paddw m2, [tlq+30] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + pmullw m2, m4 + psrlw m0, 3 + pxor m1, m1 + paddw m2, m3 + psrlw m2, 3 + pavgw m0, m1 + pavgw m1, m2 +.w16_filter_end2: + mov tlq, rsp + mova [tlq+ 0], m0 + mova [tlq+32], m1 +.w16_main: + movd xm4, dxd + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m4, xm4 + movd xm0, maxbased + mov r3d, dxd + vpbroadcastw m0, xm0 + paddw m3, m4, [z_base_inc] + psubw m3, m0 +.w16_loop: + lea r5d, [r3+dxq] + shr r3d, 6 + movu m0, [tlq+r3*2] + movu m1, [tlq+r3*2+2] + lea r3d, [r5+dxq] + shr r5d, 6 + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + psraw m2, m3, 15 + paddw m3, m4 + paddw m1, m0 + movu m0, [tlq+r5*2] + vpblendvb m2, m6, m1, m2 + movu m1, [tlq+r5*2+2] + mova [dstq+strideq*0], m2 + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + psraw m2, m3, 15 + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + mova [dstq+strideq*1], m0 + sub hd, 2 + jz .w16_end + lea dstq, [dstq+strideq*2] + cmp r3d, maxbased + jb .w16_loop +.w16_end_loop: + mova [dstq+strideq*0], m6 + mova [dstq+strideq*1], m6 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_end_loop +.w16_end: + RET +.w32: + %assign stack_offset org_stack_offset + ALLOC_STACK -160, 8 + lea maxbased, [hq+31] + mov r3d, 63 + cmp hd, 32 + cmova maxbased, r3d + test angled, 0x400 + jnz .w32_main + vpbroadcastd m2, [pw_3] + mova m0, [tlq-2] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + punpcklwd xm1, xm0, xm0 + vpblendd m1, [tlq-4], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + paddw m0, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + paddw m1, m2 + paddw m0, [tlq+2] ; 2 3 4 5 6 7 8 9 a b c d e f g h + pavgw m1, [tlq+4] ; 3 4 5 6 7 8 9 a b c d e f g h i + mov r3, rsp + paddw m0, m1 + lea r5d, [maxbaseq-31] + psrlw m0, 2 + mova [r3], m0 +.w32_filter_loop: + mova m0, [tlq+30] + paddw m1, m2, [tlq+28] + add tlq, 32 + paddw m0, [tlq+0] + pavgw m1, [tlq+4] + paddw m0, [tlq+2] + add r3, 32 + paddw m0, m1 + psrlw m0, 2 + mova [r3], m0 + sub r5d, 16 + jg .w32_filter_loop + movu m0, [tlq+32] ; 2 3 4 5 6 7 8 9 a b c d e f g h + punpckhwd m1, m0, m0 + paddw m2, [tlq+28] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m0, [tlq+30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + jl .w32_filter_h8 + vpblendd m3, m1, [tlq+36], 0x7f ; 4 5 6 7 8 9 a b c d e f g h h h + vpblendd m1, [tlq+34], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g h h + movzx r5d, word [tlq+62] + movzx r2d, word [tlq+60] + pavgw m2, m3 + sub r2d, r5d + paddw m0, m1 + lea r2d, [r2+r5*8+4] + paddw m0, m2 + shr r2d, 3 + psrlw m0, 2 + mova [r3+32], m0 + mov [r3+66], r5w + mov [r3+64], r2w + mov tlq, rsp + mov r3d, 65 + cmp hd, 64 + cmove maxbased, r3d + jmp .w32_main +.w32_filter_h8: + vpblendd xm3, xm1, [tlq+36], 0x07 ; 4 5 6 7 8 9 9 9 + vpblendd xm1, [tlq+34], 0x07 ; 3 4 5 6 7 8 9 9 + pavgw xm2, xm3 + paddw xm0, xm1 + mov tlq, rsp + paddw xm0, xm2 + psrlw xm0, 2 + mova [r3+32], xm0 +.w32_main: + movd xm4, dxd + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m4, xm4 + movd xm0, maxbased + mov r5d, dxd + vpbroadcastd m7, [pw_m1024] ; -16 * 64 + vpbroadcastw m0, xm0 + paddw m3, m4, [z_base_inc] + psubw m3, m0 +.w32_loop: + mov r3d, r5d + shr r3d, 6 + movu m0, [tlq+r3*2] + movu m1, [tlq+r3*2+2] + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + psraw m1, m3, 15 + vpblendvb m0, m6, m0, m1 + mova [dstq+32*0], m0 + movu m0, [tlq+r3*2+32] + movu m1, [tlq+r3*2+34] + add r5d, dxd + psubw m1, m0 + pmulhrsw m1, m2 + pcmpgtw m2, m7, m3 + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + mova [dstq+32*1], m0 + dec hd + jz .w32_end + add dstq, strideq + cmp r5d, maxbased + jb .w32_loop +.w32_end_loop: + mova [dstq+32*0], m6 + mova [dstq+32*1], m6 + add dstq, strideq + dec hd + jg .w32_end_loop +.w32_end: + RET +.w64: + %assign stack_offset org_stack_offset + ALLOC_STACK -256, 10 + lea maxbased, [hq+63] + test angled, 0x400 + jnz .w64_main + vpbroadcastd m2, [pw_3] + mova m0, [tlq-2] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + punpcklwd xm1, xm0, xm0 + vpblendd m1, [tlq-4], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + paddw m0, [tlq+0] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + paddw m1, m2 + paddw m0, [tlq+2] ; 2 3 4 5 6 7 8 9 a b c d e f g h + pavgw m1, [tlq+4] ; 3 4 5 6 7 8 9 a b c d e f g h i + mov r3, rsp + paddw m0, m1 + lea r5d, [hq+32] + psrlw m0, 2 + mova [r3], m0 +.w64_filter_loop: + mova m0, [tlq+30] + paddw m1, m2, [tlq+28] + add tlq, 32 + paddw m0, [tlq+0] + pavgw m1, [tlq+4] + paddw m0, [tlq+2] + add r3, 32 + paddw m0, m1 + psrlw m0, 2 + mova [r3], m0 + sub r5d, 16 + jg .w64_filter_loop + movu m0, [tlq+32] ; 2 3 4 5 6 7 8 9 a b c d e f g h + punpckhwd m1, m0, m0 + paddw m2, [tlq+28] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m0, [tlq+30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpblendd m3, m1, [tlq+36], 0x7f ; 4 5 6 7 8 9 a b c d e f g h h h + vpblendd m1, [tlq+34], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g h h + pavgw m2, m3 + paddw m0, m1 + paddw m0, m2 + mov tlq, rsp + psrlw m0, 2 + mova [r3+32], m0 +.w64_main: + movd xm4, dxd + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m4, xm4 + movd xm0, maxbased + mov r5d, dxd + vpbroadcastd m7, [pw_m1024] ; -16 * 64 + vpbroadcastw m0, xm0 + paddw m3, m4, [z_base_inc] + paddw m8, m7, m7 ; -32 * 64 + psubw m3, m0 + paddw m9, m8, m7 ; -48 * 64 +.w64_loop: + mov r3d, r5d + shr r3d, 6 + movu m0, [tlq+r3*2] + movu m1, [tlq+r3*2+2] + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + psraw m1, m3, 15 + vpblendvb m0, m6, m0, m1 + mova [dstq+32*0], m0 + movu m0, [tlq+r3*2+32] + movu m1, [tlq+r3*2+34] + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + pcmpgtw m1, m7, m3 + vpblendvb m0, m6, m0, m1 + mova [dstq+32*1], m0 + movu m0, [tlq+r3*2+64] + movu m1, [tlq+r3*2+66] + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + pcmpgtw m1, m8, m3 + vpblendvb m0, m6, m0, m1 + mova [dstq+32*2], m0 + movu m0, [tlq+r3*2+96] + movu m1, [tlq+r3*2+98] + add r5d, dxd + psubw m1, m0 + pmulhrsw m1, m2 + pcmpgtw m2, m9, m3 + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + mova [dstq+32*3], m0 + dec hd + jz .w64_end + add dstq, strideq + cmp r5d, maxbased + jb .w64_loop +.w64_end_loop: + mova [dstq+32*0], m6 + mova [dstq+32*1], m6 + mova [dstq+32*2], m6 + mova [dstq+32*3], m6 + add dstq, strideq + dec hd + jg .w64_end_loop +.w64_end: + RET + +cglobal ipred_z2_16bpc, 3, 12, 12, 352, dst, stride, tl, w, h, angle, dx, dy +%define base r9-z_filter_t0 + lea r9, [ipred_z2_16bpc_avx2_table] + tzcnt wd, wm + movifnidn angled, anglem + movifnidn hd, hm + lea dxq, [dr_intra_derivative-90] + movsxd wq, [r9+wq*4] + mova m1, [tlq- 0] + movzx dyd, angleb + xor angled, 0x400 + mova m2, [tlq- 32] + mov r8, dxq + sub dxq, dyq + mova m3, [tlq- 64] + add wq, r9 + add r9, z_filter_t0-ipred_z2_16bpc_avx2_table + mova m4, [tlq- 96] + and dyd, ~1 + mova m5, [tlq-128] + and dxq, ~1 + movzx dyd, word [r8+dyq] ; angle - 90 + movzx dxd, word [dxq+270] ; 180 - angle + vpbroadcastd m11, [base+pw_62] + mova [rsp+128], m1 + mova [rsp+ 96], m2 + mova [rsp+ 64], m3 + neg dxd + mova [rsp+ 32], m4 + neg dyq + mova [rsp+ 0], m5 + jmp wq +.w4: + vbroadcasti128 m10, [base+z2_x_shuf] + vpbroadcastq m6, [base+z_base_inc+2] + lea r8d, [dxq+(65<<6)] ; xpos + mov r10d, (63-4)<<6 + test angled, 0x400 + jnz .w4_main ; !enable_intra_edge_filter + lea r3d, [hq+2] + add angled, 1022 + shl r3d, 6 + test r3d, angled + jnz .w4_no_upsample_above ; angle >= 130 || h > 8 || (is_sm && h == 8) + movq xm0, [tlq+2] ; 1 2 3 4 + movq xm1, [tlq+0] ; 0 1 2 3 + pshuflw xm2, xm0, q3321 ; 2 3 4 4 + pshuflw xm3, xm1, q2100 ; 0 0 1 2 + vpbroadcastw xm4, r8m ; pixel_max + vbroadcasti128 m10, [base+z_upsample] + paddw xm1, xm0 + paddw xm2, xm3 + lea r8d, [r8+dxq+(1<<6)] + psubw xm2, xm1, xm2 + add dxd, dxd + psraw xm2, 3 + pxor xm3, xm3 + sub r10d, 3<<6 + paddw xm1, xm2 + paddw m6, m6 + pmaxsw xm1, xm3 + sub angled, 1075 ; angle - 53 + pavgw xm1, xm3 + lea r3d, [hq+3] + pminsw xm1, xm4 + xor angled, 0x7f ; 180 - angle + punpcklwd xm1, xm0 + movu [rsp+130], xm1 + call .filter_strength + jmp .w4_filter_left +ALIGN function_align +.filter_strength: + movd xm8, r3d + mov r3d, angled + movd xm7, angled + vpbroadcastb m8, xm8 + shr r3d, 8 ; is_sm << 1 + vpbroadcastb m7, xm7 + pcmpeqb m8, [base+z_filter_wh] + mova xm9, [r9+r3*8] + pand m0, m8, m7 + pcmpgtb m0, m9 + pmovmskb r3d, m0 + ret +ALIGN function_align +.upsample_left: ; h4/h8 + mova xm0, [tlq-16] ; 8 7 6 5 4 3 2 1 + movu xm1, [tlq-14] ; 7 6 5 4 3 2 1 0 + vpbroadcastw xm4, r8m ; pixel_max + cmp hd, 8 + je .upsample_left_h8 + pshufhw xm2, xm0, q2100 ; _ _ _ _ 4 4 3 2 + pshufhw xm3, xm1, q3321 ; _ _ _ _ 2 1 0 0 + jmp .upsample_left_end +.upsample_left_h8: + pblendw xm2, xm0, [tlq-18], 0xfe ; 8 8 7 6 5 4 3 2 + pblendw xm3, xm1, [tlq-12], 0x7f ; 6 5 4 3 2 1 0 0 +.upsample_left_end: + paddw xm1, xm0 + paddw xm2, xm3 + psubw xm2, xm1, xm2 + add dyq, dyq + psraw xm2, 3 + pxor xm3, xm3 + paddw xm1, xm2 + pmaxsw xm1, xm3 + pavgw xm1, xm3 + pminsw xm1, xm4 + punpcklwd xm2, xm0, xm1 + punpckhwd xm0, xm1 + mova [rsp+ 96+gprsize], xm2 + mova [rsp+112+gprsize], xm0 + ret +.w4_no_upsample_above: + lea r3d, [hq+3] + sub angled, 1112 ; angle - 90 + call .filter_strength + test r3d, r3d + jz .w4_no_filter_above + popcnt r3d, r3d + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd xm5, [base+z_filter_k-4+r3*4+12*0] + psrldq xm0, xm1, 2 ; 1 2 3 4 + pshuflw xm2, xm1, q2100 ; 0 0 1 2 + pmullw xm4, xm0 + pshuflw xm3, xm0, q3321 ; 2 3 4 4 + paddw xm1, xm3 + pshuflw xm3, xm0, q3332 ; 3 4 4 4 + pmullw xm1, xm5 + vpbroadcastd xm5, [base+z_filter_k-4+r3*4+12*2] + paddw xm2, xm3 + vpbroadcastd xm3, r6m ; max_width + pmullw xm2, xm5 + packssdw xm3, xm3 + paddw xm1, xm4 + paddw xm1, xm2 + psubw xm3, [base+pw_1to16] + pxor xm4, xm4 + psrlw xm1, 3 + pminsw xm3, xm11 ; clip to byte range since there's no variable word blend + pavgw xm1, xm4 + vpblendvb xm1, xm0, xm3 + movq [rsp+130], xm1 +.w4_no_filter_above: + lea r3d, [hq+2] + add angled, 973 ; angle + 883 + shl r3d, 6 + test r3d, angled + jz .w4_upsample_left ; angle <= 140 || h > 8 || (is_sm && h == 8) + vpbroadcastd xm0, [base+pb_90] + psubb xm0, xm7 ; 180 - angle + pand xm0, xm8 ; reuse from previous filter_strength call + pcmpgtb xm0, xm9 + pmovmskb r3d, xm0 +.w4_filter_left: + test r3d, r3d + jz .w4_main + popcnt r3d, r3d + mova m0, [tlq-32] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + vpbroadcastd m5, r7m ; max_height + cmp r3d, 3 + je .w4_filter_left_s3 + vpbroadcastd m2, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m3, [base+z_filter_k-4+r3*4+12*0] + pmullw m2, m0 + cmp hd, 8 + jl .w4_filter_left_h4 + movu m4, [tlq-34] + punpcklwd m1, m0, m0 + vpblendd m1, m4, 0xee ; 0 0 1 2 3 4 5 6 8 8 9 a b c d e + je .w4_filter_left_end + vpblendd m1, m4, 0x10 ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + jmp .w4_filter_left_end +.w4_upsample_left: + call .upsample_left + mov r11, -16 + vbroadcasti128 m9, [base+z_upsample] + jmp .w4_main_upsample_left +.w4_filter_left_s3: ; can only be h16 + movu m2, [tlq-30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpbroadcastd m4, [base+pw_3] + paddw m1, m0, m2 + punpckhwd m2, m2 + vpblendd m2, [tlq-28], 0x7f ; 2 3 4 5 6 7 8 9 a b c d e f g g + punpcklwd xm3, xm0, xm0 + paddw m2, m4 + vpblendd m4, m3, [tlq-34], 0xfe ; 0 0 1 2 3 4 5 6 8 8 9 a b c d e + vpblendd m3, [tlq-36], 0xfe ; 0 0 0 1 2 3 4 5 6 8 8 9 a b c d + paddw m1, m4 + pavgw m2, m3 + paddw m1, m2 + psrlw m1, 2 + jmp .w4_filter_left_end2 +.w4_filter_left_h4: + pshufhw m1, m0, q2100 ; _ _ _ _ _ _ _ _ _ _ _ _ c c d e +.w4_filter_left_end: + paddw m1, [tlq-30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pmullw m1, m3 + paddw m1, m2 + pxor m2, m2 + psrlw m1, 3 + pavgw m1, m2 +.w4_filter_left_end2: + packssdw m5, m5 + psubw m5, [base+pw_16to1] + pminsw m5, m11 + vpblendvb m1, m0, m5 + mova [rsp+96], m1 +.w4_main: + vbroadcasti128 m9, [base+z2_x_shuf] + mov r11, -8 +.w4_main_upsample_left: + movd xm5, dyd + mova m4, [base+z2_y_shuf_h4] + mov r2d, r8d + movd xm0, dxd + vpbroadcastw m5, xm5 + rorx r5, dyq, 5 + lea r8d, [dyq*3] + pmullw m5, [base+z2_ymul] + rorx r9, dyq, 4 + sar dyd, 6 + vpbroadcastw m0, xm0 + sar r8d, 6 + pand m5, m11 ; frac_y + neg dyd + psllw m5, 9 + add r5d, dyd + add r8d, dyd + add r9d, dyd + paddw m7, m0, m0 + lea dyq, [rsp+dyq*2+126] + vpblendd m0, m7, 0xcc + add dyq, r11 + neg r5d + paddw m1, m0, m7 + neg r8d + vpblendd m0, m1, 0xf0 ; xpos0 xpos1 xpos2 xpos3 + neg r9d + paddw m7, m7 + paddw m6, m0 +.w4_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + movu xm1, [rsp+r2*2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + movu xm3, [rsp+r3*2] + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x2 + vinserti128 m1, [rsp+r2*2], 1 + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x3 + vinserti128 m3, [rsp+r3*2], 1 + pshufb m1, m10 ; a0 a1 a2 a3 A0 A1 A2 A3 + pshufb m3, m10 ; b0 b1 b2 b3 B0 B1 B2 B3 + pand m2, m11, m6 + punpcklqdq m0, m1, m3 + punpckhqdq m1, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + cmp r3d, 64 + jge .w4_toponly + movu xm2, [dyq] + vinserti128 m2, [dyq+r8*2], 1 + movu xm3, [dyq+r5*2] + vinserti128 m3, [dyq+r9*2], 1 + pshufb m2, m9 + pshufb m3, m9 + punpckhwd m1, m2, m3 ; a3 b3 a2 b2 a1 b1 a0 b0 + punpcklwd m2, m3 + psubw m2, m1 + pmulhrsw m2, m5 + psraw m3, m6, 15 ; base_x < topleft + paddw m1, m2 + vpermd m1, m4, m1 ; a0 b0 c0 d0 a1 b1 c1 d1 a2 b2 c2 d2 a3 b3 c3 d3 + vpblendvb m0, m1, m3 +.w4_toponly: + paddw m6, m7 ; xpos += dx + lea r3, [strideq*3] + add dyq, r11 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r3 ], xm1 + sub hd, 4 + jz .w4_end + lea dstq, [dstq+strideq*4] + cmp r2d, r10d + jge .w4_loop +.w4_leftonly_loop: + movu xm1, [dyq] + vinserti128 m1, [dyq+r8*2], 1 + movu xm2, [dyq+r5*2] + vinserti128 m2, [dyq+r9*2], 1 + add dyq, r11 + pshufb m1, m9 + pshufb m2, m9 + punpckhwd m0, m1, m2 + punpcklwd m1, m2 + psubw m1, m0 + pmulhrsw m1, m5 + paddw m0, m1 + vpermd m0, m4, m0 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_leftonly_loop +.w4_end: + RET +.w8: + mov r10d, hd + test angled, 0x400 + jnz .w8_main + lea r3d, [angleq+126] + xor r8d, r8d + mov r3b, hb + cmp r3d, 8 + ja .w8_no_upsample_above ; angle >= 130 || h > 8 || is_sm + movu xm0, [tlq+2] ; 1 2 3 4 5 6 7 8 + mova xm1, [tlq+0] ; 0 1 2 3 4 5 6 7 + pblendw xm2, xm0, [tlq+4], 0x7f ; 2 3 4 5 6 7 8 8 + pblendw xm3, xm1, [tlq-2], 0xfe ; 0 0 1 2 3 4 5 6 + vpbroadcastw xm4, r8m ; pixel_max + paddw xm1, xm0 + paddw xm2, xm3 + not r8d + psubw xm2, xm1, xm2 + add dxd, dxd + psraw xm2, 3 + sub angled, 53 ; angle - 53 + pxor xm3, xm3 + paddw xm2, xm1 + lea r3d, [hq+7] + pmaxsw xm2, xm3 + xor angled, 0x7f ; 180 - angle + pavgw xm2, xm3 + pminsw xm2, xm4 + punpcklwd xm1, xm2, xm0 + punpckhwd xm2, xm0 + movu [rsp+130], xm1 + movu [rsp+146], xm2 + call .filter_strength + jmp .w8_filter_left +.w8_no_upsample_above: + lea r3d, [hq+7] + sub angled, 90 ; angle - 90 + call .filter_strength + test r3d, r3d + jz .w8_no_filter_above + popcnt r3d, r3d + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd xm5, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd xm6, [base+z_filter_k-4+r3*4+12*2] + movu xm0, [tlq+2] ; 1 2 3 4 5 6 7 8 x + pblendw xm2, xm1, [tlq-2], 0xfe ; 0 0 1 2 3 4 5 6 x + pmullw xm4, xm0 + pblendw xm3, xm0, [tlq+4], 0x7f ; 2 3 4 5 6 7 8 8 x + paddw xm1, xm3 + vpblendd xm3, [tlq+6], 0x07 ; 3 4 5 6 7 8 8 8 x + paddw xm2, xm3 + vpbroadcastd xm3, r6m ; max_width + pmullw xm1, xm5 + pmullw xm2, xm6 + packssdw xm3, xm3 + paddw xm1, xm4 + paddw xm1, xm2 + psubw xm3, [base+pw_1to16] + pxor xm4, xm4 + psrlw xm1, 3 + pminsw xm3, xm11 + pavgw xm1, xm4 + vpblendvb xm1, xm0, xm3 + movu [rsp+130], xm1 +.w8_no_filter_above: + lea r3d, [angleq-51] + mov r3b, hb + cmp r3d, 8 + jbe .w8_upsample_left ; angle > 140 && h <= 8 && !is_sm + vpbroadcastd m0, [base+pb_90] + psubb m0, m7 + pand m0, m8 + pcmpgtb m0, m9 + pmovmskb r3d, m0 +.w8_filter_left: + test r3d, r3d + jz .w8_main + popcnt r3d, r3d + cmp r3d, 3 + jne .w8_filter_left_s12 + vpbroadcastd m6, [base+pw_3] + vpbroadcastd m7, [base+pw_16] + cmp hd, 16 ; flags needed for later + jmp .filter_left_s3b +.w8_upsample_left: + call .upsample_left + vbroadcasti128 m7, [base+z2_y_shuf_us] + lea r11, [rsp+118] + mov r8, -8 + jmp .w8_main_upsample_left +.w16_filter_left_s12: + xor r8d, r8d +.w8_filter_left_s12: + mova m0, [tlq-32] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + vpbroadcastd m5, r7m ; max_height + vpbroadcastd m2, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m3, [base+z_filter_k-4+r3*4+12*0] + pmullw m2, m0 + cmp hd, 8 + jl .w8_filter_left_h4 + movu m4, [tlq-34] + punpcklwd m1, m0, m0 + vpblendd m1, m4, 0xee ; 0 0 1 2 3 4 5 6 8 8 9 a b c d e + je .w8_filter_left_end + vpblendd m1, m4, 0x10 ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + jmp .w8_filter_left_end +.w8_filter_left_h4: + pshufhw m1, m0, q2100 ; _ _ _ _ _ _ _ _ _ _ _ _ c c d e +.w8_filter_left_end: + paddw m1, [tlq-30] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pmullw m1, m3 + paddw m1, m2 + pxor m2, m2 + psrlw m1, 3 + pavgw m1, m2 + packssdw m5, m5 + psubw m5, [base+pw_16to1] + pminsw m5, m11 + vpblendvb m1, m0, m5 + mova [rsp+96], m1 + test r8d, r8d + jz .w8_main +; upsample_main + vbroadcasti128 m10, [base+z_upsample] + vbroadcasti128 m7, [base+z2_y_shuf] + lea r5, [rsp+120] + movd xm1, dyd + vbroadcasti128 m4, [base+z_base_inc+2] + movd xm2, dxd + vpbroadcastw m1, xm1 + vpbroadcastw m2, xm2 + mov r7, dstq + paddw m4, m4 + pmullw m0, m1, [base+z2_ymul8] + paddw m5, m2, m2 + psllw xm1, 3 + vpblendd m2, m5, 0xf0 + lea r2d, [dxq+(66<<6)] ; xpos + paddw m4, m2 + pshufd m6, m0, q2020 + psraw xm0, 6 + pxor xm1, xm1 + psubw xm8, xm1, xm0 + pand m6, m11 + punpckhwd xm9, xm8, xm1 + psllw m6, 9 + punpcklwd xm8, xm1 +.w8_upsample_above_loop: + lea r3d, [r2+dxq] + shr r2d, 6 + movu xm1, [rsp+r2*2] + movu xm2, [rsp+r2*2+16] + lea r2d, [r3+dxq] + shr r3d, 6 + vinserti128 m1, [rsp+r3*2], 1 + vinserti128 m2, [rsp+r3*2+16], 1 + pshufb m1, m10 + pshufb m2, m10 + punpcklqdq m0, m1, m2 ; a0 b0 c0 d0 e0 f0 g0 h0 + punpckhqdq m1, m2 + pand m2, m11, m4 + psubw m1, m0 + psllw m2, 9 + pmulhrsw m1, m2 + paddw m0, m1 + cmp r3d, 64 + jge .w8_upsample_above_toponly + mova m1, m5 + vpgatherdq m3, [r5+xm9*2], m5 + mova m5, m1 + vpgatherdq m2, [r5+xm8*2], m1 + pshufb m3, m7 + pshufb m2, m7 + punpckldq m1, m2, m3 + punpckhdq m2, m3 + psubw m2, m1 + pmulhrsw m2, m6 + paddw m1, m2 + vpermq m1, m1, q3120 + psraw m2, m4, 15 + vpblendvb m0, m1, m2 +.w8_upsample_above_toponly: + paddw m4, m5 + sub r5, 4 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + sub hd, 2 + jz .w8_ret + lea dstq, [dstq+strideq*2] + jmp .w8_upsample_above_loop +.w8_main: + vbroadcasti128 m7, [base+z2_y_shuf] + lea r11, [rsp+120] + mov r8, -4 +.w8_main_upsample_left: + movd xm1, dyd + vbroadcasti128 m4, [base+z_base_inc+2] + movd xm2, dxd + vpbroadcastw m1, xm1 + vpbroadcastw m2, xm2 + mov r7, dstq + pmullw m0, m1, [base+z2_ymul8] + paddw m5, m2, m2 + psllw xm1, 3 + vpblendd m2, m5, 0xf0 ; xpos0 xpos1 + lea r9d, [dxq+(65<<6)] ; xpos + paddw m4, m2 + movd [rsp+284], xm1 +.w8_loop0: + mov r2d, r9d + mova [rsp+288], m0 + mov r5, r11 + mova [rsp+320], m4 + pshufd m6, m0, q2020 + psraw xm0, 6 + pxor xm1, xm1 + psubw xm8, xm1, xm0 ; base_y + pand m6, m11 ; frac_y + punpckhwd xm9, xm8, xm1 ; base_y 2 3 6 7 + psllw m6, 9 + punpcklwd xm8, xm1 ; base_y 0 1 4 5 +.w8_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + movu xm0, [rsp+r2*2] + movu xm1, [rsp+r2*2+2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + vinserti128 m0, [rsp+r3*2], 1 + vinserti128 m1, [rsp+r3*2+2], 1 + pand m2, m11, m4 + psubw m1, m0 + psllw m2, 9 + pmulhrsw m1, m2 + paddw m0, m1 + cmp r3d, 64 + jge .w8_toponly + mova m1, m5 + vpgatherdq m3, [r5+xm9*2], m5 + mova m5, m1 + vpgatherdq m2, [r5+xm8*2], m1 + pshufb m3, m7 ; c0 d0 c1 d1 g0 h0 g1 h1 + pshufb m2, m7 ; a0 b0 a1 b1 e0 f0 e1 f1 + punpckldq m1, m2, m3 ; a0 b0 c0 d0 a1 b1 c1 d1 e0 f0 g0 h0 e1 f1 g1 h1 + punpckhdq m2, m3 + psubw m2, m1 + pmulhrsw m2, m6 + paddw m1, m2 + vpermq m1, m1, q3120 + psraw m2, m4, 15 ; base_x < topleft + vpblendvb m0, m1, m2 +.w8_toponly: + paddw m4, m5 ; xpos += dx + add r5, r8 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + sub hd, 2 + jz .w8_end + lea dstq, [dstq+strideq*2] + cmp r2d, (63-8)<<6 + jge .w8_loop +.w8_leftonly_loop: + mova m0, m5 + vpgatherdq m4, [r5+xm9*2], m5 + mova m5, m0 + vpgatherdq m3, [r5+xm8*2], m0 + add r5, r8 + pshufb m2, m4, m7 + pshufb m1, m3, m7 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + psubw m1, m0 + pmulhrsw m1, m6 + paddw m0, m1 + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_leftonly_loop +.w8_end: + sub r10d, 1<<8 + jl .w8_ret + vpbroadcastd m0, [rsp+284] + add r7, 16 + paddw m0, [rsp+288] ; base_y += 8*dy + add r9d, 8<<6 + vpbroadcastd m4, [pw_512] + movzx hd, r10b + paddw m4, [rsp+320] ; base_x += 8*64 + mov dstq, r7 + jmp .w8_loop0 +.w8_ret: + RET +.w16: + movd xm0, [tlq+32] + lea r10d, [hq+(1<<8)] + movd [rsp+160], xm0 + test angled, 0x400 + jnz .w8_main + lea r3d, [hq+15] + sub angled, 90 + call .filter_strength + test r3d, r3d + jz .w16_no_filter_above + popcnt r3d, r3d + vpbroadcastd m4, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m5, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd m6, [base+z_filter_k-4+r3*4+12*2] + movu m0, [tlq+2] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + punpcklwd xm2, xm1, xm1 + vpblendd m2, [tlq-2], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + punpckhwd m3, m0, m0 + pmullw m4, m0 + vpblendd m3, [tlq+4], 0x7f ; 2 3 4 5 6 7 8 9 a b c d e f g g + paddw m1, m3 + vpblendd m3, [tlq+6], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g g g + paddw m2, m3 + vpbroadcastd m3, r6m ; max_width + pmullw m1, m5 + pmullw m2, m6 + packssdw m3, m3 + paddw m1, m4 + paddw m1, m2 + psubw m3, [base+pw_1to16] + pxor m4, m4 + psrlw m1, 3 + pminsw m3, m11 + pavgw m1, m4 + vpblendvb m1, m0, m3 + movu [rsp+130], m1 +.w16_no_filter_above: + vpbroadcastd m0, [base+pb_90] + psubb m0, m7 + pand m0, m8 + pcmpgtb m0, m9 + pmovmskb r3d, m0 + test r3d, r3d + jz .w8_main + popcnt r3d, r3d + cmp r3d, 3 + jne .w16_filter_left_s12 + vpbroadcastd m6, [base+pw_3] + vpbroadcastd m7, [base+pw_16] + cmp hd, 4 + jne .filter_left_s3 + movq xm0, [tlq-8] ; 0 1 2 3 + movq xm1, [tlq-6] ; 1 2 3 4 + vpbroadcastd xm5, r7m ; max_height + movq xm4, [base+pw_16to1+24] ; 4to1 + pshuflw xm2, xm0, q2100 ; 0 0 1 2 + pshuflw xm3, xm1, q3321 ; 2 3 4 4 + paddw xm1, xm0 + paddw xm1, xm2 + pshuflw xm2, xm0, q1000 ; 0 0 0 1 + paddw xm3, xm6 + packssdw xm5, xm5 + pavgw xm2, xm3 + psubw xm5, xm4 + paddw xm1, xm2 + pminsw xm5, xm11 + psrlw xm1, 2 + vpblendvb xm1, xm0, xm5 + movq [rsp+120], xm1 + jmp .w8_main +.w32: + mova m2, [tlq+32] + movd xm0, [tlq+64] + lea r10d, [hq+(3<<8)] + mova [rsp+160], m2 + movd [rsp+192], xm0 + test angled, 0x400 + jnz .w8_main + vpbroadcastd m6, [base+pw_3] + vpbroadcastd m0, r6m ; max_width + vpbroadcastd m7, [base+pw_16] + mov r3d, 32 + packssdw m0, m0 + psubw m0, [base+pw_1to16] + pminsw m8, m0, m11 + psubw m9, m8, m7 +.w32_filter_above: + movu m0, [tlq+2] + punpcklwd xm4, xm1, xm1 + paddw m2, m6, [tlq+6] + paddw m1, m0 + vpblendd m4, [tlq-2], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + paddw m1, [tlq+4] + movu m3, [tlq+r3+2] + paddw m5, m6, [tlq+r3-2] + pavgw m2, m4 + punpckhwd m4, m3, m3 + paddw m1, m2 + vpblendd m2, m4, [tlq+r3+6], 0x7f ; 4 5 6 7 8 9 a b c d e f g h h h + vpblendd m4, [tlq+r3+4], 0x7f ; 3 4 5 6 7 8 9 a b c d e f g h h + pavgw m2, m5 + paddw m5, m3, [tlq+r3] + paddw m4, m5 + psrlw m1, 2 + paddw m2, m4 + vpblendvb m1, m0, m8 + psrlw m2, 2 + vpblendvb m2, m3, m9 + movu [rsp+130], m1 + movu [rsp+r3+130], m2 +.filter_left_s3: + cmp hd, 16 + jl .filter_left_s3_h8 ; h8 +.filter_left_s3b: + mova m0, [tlq-32] ; 2 3 4 5 6 7 8 9 a b c d e f g h + movu m2, [tlq-30] ; 3 4 5 6 7 8 9 a b c d e f g h i + vpbroadcastd m5, r7m ; max_height + paddw m1, m0, m2 + punpckhwd m2, m2 + mov r3d, hd + vpblendd m2, [tlq-28], 0x7f ; 4 5 6 7 8 9 a b c d e f g h i i + packssdw m5, m5 + not r3 + psubw m5, [base+pw_16to1] + paddw m2, m6 + pminsw m8, m11, m5 + je .filter_left_s3_end ; h16 + paddw m1, [tlq-34] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pavgw m2, [tlq-36] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m1, m2 + psrlw m1, 2 + vpblendvb m3, m1, m0, m8 + mova m0, [tlq-64] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m1, m0, [tlq-62] ; 3 4 5 6 7 8 9 a b c d e f g h i + paddw m2, m6, [tlq-60] ; 4 5 6 7 8 9 a b c d e f g h i j + psubw m8, m7 + mova [rsp+96], m3 + jnp .filter_left_s3_end ; h32 + mova m5, [tlq-96] + paddw m1, [tlq-66] + pavgw m2, [tlq-68] + paddw m1, m2 + paddw m4, m5, [tlq-94] + paddw m2, m6, [tlq-92] + psrlw m1, 2 + paddw m4, [tlq- 98] + pavgw m2, [tlq-100] + vpblendvb m3, m1, m0, m8 + mova m0, [tlq-128] + psubw m8, m7 + paddw m4, m2 + paddw m1, m0, [tlq-126] + paddw m2, m6, [tlq-124] + psrlw m4, 2 + mova [rsp+64], m3 + vpblendvb m4, m5, m8 + psubw m8, m7 + mova [rsp+32], m4 +.filter_left_s3_end: + punpcklwd xm3, xm0, xm0 + vpblendd m4, m3, [tlq+r3*2], 0xfe ; 2 2 3 4 5 6 7 8 9 a b c d e f g + vpblendd m3, [tlq+r3*2-2], 0xfe ; 2 2 2 3 4 5 6 7 8 9 a b c d e f + paddw m1, m4 + pavgw m2, m3 + paddw m1, m2 + psrlw m1, 2 + vpblendvb m1, m0, m8 + mova [rsp+r3*2+130], m1 + jmp .w8_main +.filter_left_s3_h8: + mova xm0, [tlq-16] ; 0 1 2 3 4 5 6 7 + movu xm3, [tlq-14] ; 1 2 3 4 5 6 7 8 + pblendw xm2, xm0, [tlq-18], 0xfe ; 0 0 1 2 3 4 5 6 + vpbroadcastd xm5, r7m ; max_height + paddw xm1, xm0, xm3 + pblendw xm3, [tlq-12], 0x7f ; 2 3 4 5 6 7 8 8 + paddw xm1, xm2 + vpblendd xm2, [tlq-20], 0x0e ; 0 0 0 1 2 3 4 5 + paddw xm3, xm6 + packssdw xm5, xm5 + pavgw xm2, xm3 + psubw xm5, [base+pw_16to1+16] ; 8to1 + paddw xm1, xm2 + pminsw xm5, xm11 + psrlw xm1, 2 + vpblendvb xm1, xm0, xm5 + mova [rsp+112], xm1 + jmp .w8_main +.w64: + mova m2, [tlq+ 32] + mova m3, [tlq+ 64] + mova m4, [tlq+ 96] + movd xm0, [tlq+128] + lea r10d, [hq+(7<<8)] + mova [rsp+160], m2 + mova [rsp+192], m3 + mova [rsp+224], m4 + movd [rsp+256], xm0 + test angled, 0x400 + jnz .w8_main + vpbroadcastd m6, [base+pw_3] + movu m0, [tlq+34] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m2, m6, [tlq+30] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m5, m0, [tlq+32] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pavgw m2, [tlq+38] ; 4 5 6 7 8 9 a b c d e f g h h h + paddw m5, [tlq+36] ; 3 4 5 6 7 8 9 a b c d e f g h h + movu m4, [tlq+66] + paddw m3, m6, [tlq+62] + paddw m7, m4, [tlq+64] + pavgw m3, [tlq+70] + paddw m7, [tlq+68] + paddw m2, m5 + vpbroadcastd m5, r6m ; max_width + mov r3d, 96 + packssdw m5, m5 + paddw m3, m7 + psubw m5, [base+pw_1to16] + psrlw m2, 2 + vpbroadcastd m7, [base+pw_16] + psrlw m3, 2 + pminsw m8, m11, m5 + psubw m9, m8, m7 + vpblendvb m2, m0, m9 + psubw m9, m7 + vpblendvb m3, m4, m9 + psubw m9, m7 + movu [rsp+162], m2 + movu [rsp+194], m3 + jmp .w32_filter_above + +cglobal ipred_z3_16bpc, 4, 9, 0, dst, stride, tl, w, h, angle, dy, org_w, maxbase + %assign org_stack_offset stack_offset + lea r6, [ipred_z3_16bpc_avx2_table] + tzcnt hd, hm + movifnidn angled, anglem + lea r7, [dr_intra_derivative+45*2-1] + sub tlq, 2 + movsxd hq, [r6+hq*4] + sub angled, 180 + add hq, r6 + mov dyd, angled + neg dyd + xor angled, 0x400 + or dyq, ~0x7e + movzx dyd, word [r7+dyq] + vpbroadcastd m5, [pw_62] + mov org_wd, wd + jmp hq +.h4: + ALLOC_STACK -64, 7 + lea r7, [strideq*3] + cmp angleb, 40 + jae .h4_no_upsample + lea r4d, [angleq-1024] + sar r4d, 7 + add r4d, wd + jg .h4_no_upsample ; !enable_intra_edge_filter || w > 8 || (w == 8 && is_sm) + mova xm2, [tlq-14] ; 0 1 2 3 4 5 6 7 + pblendw xm1, xm2, [tlq-16], 0xfe ; 0 0 1 2 3 4 5 6 + vpblendd xm0, xm1, [tlq-18], 0x0e ; 0 0 0 1 2 3 4 5 + pshufd xm3, xm1, q0000 + paddw xm1, xm2 + paddw xm0, [tlq-12] ; 1 2 3 4 5 6 7 8 + vpbroadcastw xm4, r8m ; pixel_max + add dyd, dyd + psubw xm0, xm1, xm0 + mova [rsp+ 0], xm3 + movd xm3, dyd + psraw xm0, 3 + neg dyd + paddw xm1, xm0 + pxor xm0, xm0 + lea r2d, [dyq+(16<<6)+63] ; ypos + pmaxsw xm1, xm0 + pavgw xm1, xm0 + vpbroadcastw m3, xm3 + pminsw xm1, xm4 + punpckhwd xm0, xm1, xm2 + punpcklwd xm1, xm2 + paddw m2, m3, m3 + mova [rsp+32], xm0 + punpcklwd m3, m2 + mova [rsp+16], xm1 + paddw m4, m2, m2 + paddw m2, m3 + vpblendd m3, m2, 0xf0 ; ypos0 ypos1 ypos2 ypos3 +.h4_upsample_loop: + lea r4d, [r2+dyq] + shr r2d, 6 + movu xm1, [rsp+r2*2] + lea r2d, [r4+dyq] + shr r4d, 6 + movu xm2, [rsp+r4*2] + lea r4d, [r2+dyq] + shr r2d, 6 + vinserti128 m1, [rsp+r2*2], 1 + lea r2d, [r4+dyq] + shr r4d, 6 + vinserti128 m2, [rsp+r4*2], 1 + psrld m0, m1, 16 + pblendw m0, m2, 0xaa ; a3 b3 a2 b2 a1 b1 a0 b0 c3 d3 c2 d2 c1 d1 c0 d0 + pslld m2, 16 + pblendw m1, m2, 0xaa + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m3, m4 + paddw m1, m0 + vextracti128 xm2, m1, 1 + punpckhdq xm0, xm1, xm2 ; a1 b1 c1 d1 a0 b0 c0 d0 + punpckldq xm1, xm2 ; a3 b3 c3 d3 a2 b2 c2 d2 + movhps [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm0 + movhps [dstq+strideq*2], xm1 + movq [dstq+r7 ], xm1 + add dstq, 8 + sub wd, 4 + jg .h4_upsample_loop + RET +ALIGN function_align +.filter_strength: ; h4/h8/h16 +%define base r4-z_filter_t0 + lea r4, [z_filter_t0] + movd xm0, maxbased + movd xm1, angled + shr angled, 8 ; is_sm << 1 + vpbroadcastb m0, xm0 + vpbroadcastb m1, xm1 + pcmpeqb m0, [base+z_filter_wh] + pand m0, m1 + mova xm1, [r4+angleq*8] + pcmpgtb m0, m1 + pmovmskb r5d, m0 + ret +.h4_no_upsample: + mov maxbased, 7 + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h4_main + lea maxbased, [wq+3] + call .filter_strength + mov maxbased, 7 + test r5d, r5d + jz .h4_main ; filter_strength == 0 + popcnt r5d, r5d + mova xm0, [tlq-14] ; 0 1 2 3 4 5 6 7 + movu xm3, [tlq-12] ; 1 2 3 4 5 6 7 8 + vpbroadcastd xm2, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd xm4, [base+z_filter_k-4+r5*4+12*0] + pmullw xm2, xm0 + pblendw xm0, [tlq-16], 0xfe ; 0 0 1 2 3 4 5 6 + paddw xm1, xm0, xm3 + movd [rsp+12], xm0 + pmullw xm1, xm4 + cmp r5d, 3 + jne .h4_filter_3tap + pblendw xm3, [tlq-10], 0x7f ; 2 3 4 5 6 7 8 8 + vpblendd xm0, [tlq-18], 0x0e ; 0 0 0 1 2 3 4 5 + movzx r4d, word [tlq-14] + movzx r2d, word [tlq-12] + inc maxbased + paddw xm1, xm2 + paddw xm0, xm3 + sub r2d, r4d + paddw xm2, xm0, xm0 + lea r2d, [r2+r4*8+4] + shr r2d, 3 + mov [rsp+14], r2w +.h4_filter_3tap: + pxor xm0, xm0 + paddw xm1, xm2 + lea tlq, [rsp+30] + psrlw xm1, 3 + cmp wd, 8 + sbb maxbased, -1 + pavgw xm0, xm1 + mova [rsp+16], xm0 +.h4_main: + movd xm3, dyd + neg maxbaseq + vbroadcasti128 m1, [z_base_inc] + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m3, xm3 + lea r4d, [maxbaseq+3*64] + neg dyq + movd xm2, r4d + sub tlq, 8 + lea r4, [dyq+63] ; ypos + punpcklwd m1, m1 + paddw m0, m3, m3 + vpbroadcastw m2, xm2 + punpcklwd m3, m0 + paddw m4, m0, m0 + paddw m0, m3 + psubw m2, m1 + vpblendd m3, m0, 0xf0 ; ypos0 ypos1 ypos2 ypos3 + or maxbased, 63 + paddw m3, m2 +.h4_loop: + lea r5, [r4+dyq] + sar r4, 6 ; base0 + movu xm1, [tlq+r4*2] + lea r4, [r5+dyq] + sar r5, 6 ; base1 + movu xm2, [tlq+r5*2] + lea r5, [r4+dyq] + sar r4, 6 ; base2 + vinserti128 m1, [tlq+r4*2], 1 + lea r4, [r5+dyq] + sar r5, 6 ; base3 + vinserti128 m2, [tlq+r5*2], 1 + punpckhwd m0, m1, m2 + punpcklwd m1, m2 + pand m2, m5, m3 + palignr m0, m1, 4 ; a3 b3 a2 b2 a1 b1 a0 b0 c3 d3 c2 d2 c1 d1 c0 d0 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + psraw m2, m3, 15 ; ypos < max_base_y + paddw m3, m4 + paddw m1, m0 + vpblendvb m1, m6, m1, m2 + vextracti128 xm2, m1, 1 + punpckhdq xm0, xm1, xm2 ; a1 b1 c1 d1 a0 b0 c0 d0 + punpckldq xm1, xm2 ; a3 b3 c3 d3 a2 b2 c2 d2 + movhps [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm0 + movhps [dstq+strideq*2], xm1 + movq [dstq+r7 ], xm1 + sub wd, 4 + jz .h4_end + add dstq, 8 + cmp r4d, maxbased + jg .h4_loop +.h4_end_loop: + movq [dstq+strideq*0], xm6 + movq [dstq+strideq*1], xm6 + movq [dstq+strideq*2], xm6 + movq [dstq+r7 ], xm6 + add dstq, 8 + sub wd, 4 + jg .h4_end_loop +.h4_end: + RET +.h8: + lea r4d, [angleq+216] + %assign stack_offset org_stack_offset + ALLOC_STACK -64, 8 + mov r4b, wb + lea r7, [strideq*3] + cmp r4d, 8 + ja .h8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || w > 8 + mova m2, [tlq-30] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw m1, m2, [tlq-32] ; _ 0 1 2 3 4 5 6 7 8 9 a b c d e + movu m0, [tlq-34] ; _ _ 0 1 2 3 4 5 6 7 8 9 a b c d + cmp wd, 8 + je .h8_upsample_w8 + pshufhw xm3, xm2, q1000 + vpblendd m0, m3, 0x0f ; _ _ _ _ 4 4 4 5 6 7 8 9 a b c d +.h8_upsample_w8: + paddw m0, [tlq-28] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpbroadcastw m4, r8m ; pixel_max + add dyd, dyd + psubw m0, m1, m0 + movd xm6, dyd + psraw m0, 3 + neg dyd + paddw m1, m0 + pxor m0, m0 + pmaxsw m1, m0 + lea r4d, [dyq+(16<<6)+63] ; ypos + pavgw m1, m0 + vpbroadcastw m6, xm6 + pminsw m1, m4 + punpckhwd m0, m1, m2 + punpcklwd m1, m2 + vextracti128 [rsp+48], m0, 1 + vextracti128 [rsp+32], m1, 1 + paddw m7, m6, m6 + mova [rsp+16], xm0 + mova [rsp+ 0], xm1 + punpcklwd m6, m7 ; ypos0 ypos1 +.h8_upsample_loop: + lea r2d, [r4+dyq] + shr r4d, 6 ; base0 + movu m1, [rsp+r4*2] + lea r4d, [r2+dyq] + shr r2d, 6 ; base1 + movu m2, [rsp+r2*2] + lea r2d, [r4+dyq] + shr r4d, 6 ; base2 + movu m3, [rsp+r4*2] + lea r4d, [r2+dyq] + shr r2d, 6 ; base3 + movu m4, [rsp+r2*2] + psrld m0, m1, 16 + pblendw m0, m2, 0xaa ; a7 b7 a6 b6 a5 b5 a4 b4 a3 b3 a2 b2 a1 b1 a0 b0 + pslld m2, 16 + pblendw m1, m2, 0xaa + psrld m2, m3, 16 + pblendw m2, m4, 0xaa ; c7 d7 c6 d6 c5 d5 c4 d4 c3 d3 c2 d2 c1 d1 c0 d0 + pslld m4, 16 + pblendw m3, m4, 0xaa + pand m4, m5, m6 + paddw m6, m7 + psllw m4, 9 + psubw m1, m0 + pmulhrsw m1, m4 + pand m4, m5, m6 + psllw m4, 9 + psubw m3, m2 + pmulhrsw m3, m4 + paddw m6, m7 + lea r2, [dstq+strideq*4] + paddw m1, m0 + paddw m3, m2 + punpckhdq m0, m1, m3 ; a5 b5 c5 d5 a4 b4 c4 d4 a1 b1 c1 d1 a0 b0 c0 d0 + punpckldq m1, m3 ; a7 b7 c7 d7 a6 b6 c6 d6 a3 b3 c3 d3 a2 b2 c2 d2 + vextracti128 xm2, m0, 1 + vextracti128 xm3, m1, 1 + movhps [r2 +strideq*0], xm0 + movq [r2 +strideq*1], xm0 + movhps [r2 +strideq*2], xm1 + movq [r2 +r7 ], xm1 + movhps [dstq+strideq*0], xm2 + movq [dstq+strideq*1], xm2 + movhps [dstq+strideq*2], xm3 + movq [dstq+r7 ], xm3 + add dstq, 8 + sub wd, 4 + jg .h8_upsample_loop + RET +.h8_no_intra_edge_filter: + and maxbased, 7 + or maxbased, 8 ; imin(w+7, 15) + jmp .h8_main +.h8_no_upsample: + lea maxbased, [wq+7] + test angled, 0x400 + jnz .h8_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .h8_main + popcnt r5d, r5d + mova m0, [tlq-30] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + movu m3, [tlq-28] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpbroadcastd m2, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m4, [base+z_filter_k-4+r5*4+12*0] + pmullw m2, m0 + cmp wd, 8 + jl .h8_filter_w4 + punpcklwd xm0, xm0 + vpblendd m1, m0, [tlq-32], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + movd [rsp+28], xm0 + paddw m1, m3 + mov r4d, 16 + pmullw m1, m4 + cmovg maxbased, r4d + cmp r5d, 3 + jne .h8_filter_3tap + punpckhwd m3, m3 + vpblendd m0, [tlq-34], 0xfe ; 0 0 0 1 2 3 4 5 6 7 8 9 a b c d + vpblendd m3, [tlq-26], 0x7f ; 2 3 4 5 6 7 8 9 a b c d e f g g + movzx r4d, word [tlq-30] + movzx r2d, word [tlq-28] + inc maxbased + paddw m1, m2 + paddw m0, m3 + sub r2d, r4d + paddw m2, m0, m0 + lea r2d, [r2+r4*8+4] + shr r2d, 3 + mov [rsp+30], r2w + jmp .h8_filter_3tap +.h8_filter_w4: + pshufhw xm1, xm0, q2100 + vinserti128 m1, [tlq-16], 1 ; _ _ _ _ 4 4 5 6 7 8 9 a b c d e + paddw m1, m3 + pmullw m1, m4 +.h8_filter_3tap: + pxor m0, m0 + paddw m1, m2 + lea tlq, [rsp+62] + psrlw m1, 3 + pavgw m0, m1 + mova [rsp+32], m0 +.h8_main: + movd xm4, dyd + neg maxbaseq + vbroadcasti128 m1, [z_base_inc] + vpbroadcastw m7, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m4, xm4 + lea r4d, [maxbaseq+7*64] + neg dyq + movd xm2, r4d + sub tlq, 16 + lea r4, [dyq+63] + paddw m6, m4, m4 + vpbroadcastw m2, xm2 + vpblendd m4, m6, 0xf0 ; ypos0 ypos1 + psubw m2, m1 + or maxbased, 63 + paddw m4, m2 +.h8_loop: + lea r5, [r4+dyq] + sar r4, 6 ; base0 + movu xm0, [tlq+r4*2+2] + movu xm1, [tlq+r4*2] + lea r4, [r5+dyq] + sar r5, 6 ; base1 + vinserti128 m0, [tlq+r5*2+2], 1 + vinserti128 m1, [tlq+r5*2], 1 + lea r5, [r4+dyq] + sar r4, 6 ; base2 + pand m3, m5, m4 + psllw m3, 9 + psubw m1, m0 + pmulhrsw m1, m3 + psraw m3, m4, 15 + paddw m4, m6 + paddw m0, m1 + movu xm1, [tlq+r4*2+2] + movu xm2, [tlq+r4*2] + lea r4, [r5+dyq] + sar r5, 6 ; base3 + vpblendvb m0, m7, m0, m3 + vinserti128 m1, [tlq+r5*2+2], 1 + vinserti128 m2, [tlq+r5*2], 1 + pand m3, m5, m4 + psllw m3, 9 + psubw m2, m1 + pmulhrsw m2, m3 + psraw m3, m4, 15 + paddw m4, m6 + lea r5, [dstq+strideq*4] + paddw m1, m2 + vpblendvb m1, m7, m1, m3 + punpckhwd m2, m0, m1 ; a3 c3 a2 c2 a1 c1 a0 c0 b3 d3 b2 d2 b1 d1 b0 d0 + vextracti128 xm3, m2, 1 + punpcklwd m0, m1 ; a7 c7 a6 c6 a5 c5 a4 c5 b7 d7 b6 d6 b5 d5 b4 d4 + punpckhwd xm1, xm2, xm3 ; a1 b1 c1 d1 a0 b0 c0 d0 + punpcklwd xm2, xm3 ; a3 b3 c3 d3 a2 b2 c2 d2 + vextracti128 xm3, m0, 1 + movhps [dstq+strideq*0], xm1 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movq [dstq+r7 ], xm2 + punpckhwd xm1, xm0, xm3 ; a5 b5 c5 d5 a4 b4 c4 d4 + punpcklwd xm0, xm3 ; a7 b7 c7 d7 a6 b6 c6 d6 + movhps [r5 +strideq*0], xm1 + movq [r5 +strideq*1], xm1 + movhps [r5 +strideq*2], xm0 + movq [r5 +r7 ], xm0 + sub wd, 4 + jz .h8_end + add dstq, 8 + cmp r4d, maxbased + jg .h8_loop + lea r6, [strideq*5] + lea r2, [strideq+r7*2] ; stride*7 + test wd, 4 + jz .h8_end_loop + movq [dstq+strideq*0], xm7 + movq [dstq+strideq*1], xm7 + movq [dstq+strideq*2], xm7 + movq [dstq+r7 ], xm7 + movq [dstq+strideq*4], xm7 + movq [dstq+r6 ], xm7 + movq [dstq+r7*2 ], xm7 + movq [dstq+r2 ], xm7 + add dstq, 8 + sub wd, 4 + jz .h8_end +.h8_end_loop: + mova [dstq+strideq*0], xm7 + mova [dstq+strideq*1], xm7 + mova [dstq+strideq*2], xm7 + mova [dstq+r7 ], xm7 + mova [dstq+strideq*4], xm7 + mova [dstq+r6 ], xm7 + mova [dstq+r7*2 ], xm7 + mova [dstq+r2 ], xm7 + add dstq, 16 + sub wd, 8 + jg .h8_end_loop +.h8_end: + RET +.h16_no_intra_edge_filter: + and maxbased, 15 + or maxbased, 16 ; imin(w+15, 31) + jmp .h16_main +ALIGN function_align +.h16: + %assign stack_offset org_stack_offset + ALLOC_STACK -96, 10 + lea maxbased, [wq+15] + lea r7, [strideq*3] + test angled, 0x400 + jnz .h16_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .h16_main ; filter_strength == 0 + popcnt r5d, r5d + movu m0, [tlq-28] ; 3 4 5 6 7 8 9 a b c d e f g h i + paddw m1, m0, [tlq-32] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpbroadcastd m6, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*0] + pmullw m2, m6, [tlq-30] ; 2 3 4 5 6 7 8 9 a b c d e f g h + pmullw m1, m7 + paddw m1, m2 + cmp wd, 8 + jg .h16_filter_w16 + mova xm3, [tlq-46] ; 0 1 2 3 4 5 6 7 + pmullw xm6, xm3 + jl .h16_filter_w4 + pblendw xm3, [tlq-48], 0xfe ; 0 0 1 2 3 4 5 6 + cmp r5d, 3 + jne .h16_filter_w8_3tap + vpblendd xm4, xm3, [tlq-50], 0x0e ; 0 0 0 1 2 3 4 5 +.h16_filter_w8_5tap: + punpckhwd m0, m0 + vpblendd m0, [tlq-26], 0x7f ; 4 5 6 7 8 9 a b c d e f g h i i + paddw xm4, [tlq-42] ; 2 3 4 5 6 7 8 9 + paddw m0, [tlq-34] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + paddw xm4, xm4 + paddw m0, m0 + paddw xm6, xm4 + paddw m1, m0 +.h16_filter_w8_3tap: + paddw xm3, [tlq-44] ; 1 2 3 4 5 6 7 8 + pmullw xm3, xm7 + pxor m0, m0 + paddw xm3, xm6 + psrlw xm3, 3 + pavgw xm3, xm0 + mova [rsp+48], xm3 + jmp .h16_filter_end +.h16_filter_w4: + pshufhw xm3, xm3, q2100 ; _ _ _ _ 4 4 5 6 + cmp r5d, 3 + jne .h16_filter_w8_3tap + pshufhw xm4, xm3, q2100 ; _ _ _ _ 4 4 4 5 + jmp .h16_filter_w8_5tap +.h16_filter_w16: + mova m3, [tlq-62] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + pmullw m6, m3 + punpcklwd xm3, xm3 + vpblendd m4, m3, [tlq-64], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + paddw m4, [tlq-60] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + mov r4d, 32 + cmp wd, 16 + cmovg maxbased, r4d + movd [rsp+28], xm3 + pmullw m4, m7 + cmp r5d, 3 + jne .h16_filter_w16_3tap + punpckhwd m0, m0 + vpblendd m3, [tlq-66], 0xfe ; 0 0 0 1 2 3 4 5 6 7 8 9 a b c d + vpblendd m0, [tlq-26], 0x7f ; 4 5 6 7 8 9 a b c d e f g h i i + paddw m3, [tlq-58] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m0, [tlq-34] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + movzx r4d, word [tlq-62] + movzx r2d, word [tlq-60] + or maxbased, 1 + paddw m3, m3 + sub r2d, r4d + paddw m0, m0 + lea r2d, [r2+r4*8+4] + paddw m4, m3 + shr r2d, 3 + paddw m1, m0 + mov [rsp+30], r2w +.h16_filter_w16_3tap: + pxor m0, m0 + paddw m4, m6 + psrlw m4, 3 + pavgw m4, m0 + mova [rsp+32], m4 +.h16_filter_end: + psrlw m1, 3 + lea tlq, [rsp+94] + pavgw m1, m0 + mova [rsp+64], m1 +.h16_main: + movd xm8, dyd + neg maxbaseq + vpbroadcastw m9, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m8, xm8 + lea r4d, [maxbaseq+dyq+15*64] + neg dyq + movd xm7, r4d + sub tlq, 32 + lea r4, [dyq+63] + vpbroadcastw m7, xm7 + or maxbased, 63 + psubw m7, [z_base_inc] +.h16_loop: + lea r5, [r4+dyq] + sar r4, 6 ; base0 + movu m0, [tlq+r4*2+2] + movu m2, [tlq+r4*2] + lea r4, [r5+dyq] + sar r5, 6 ; base1 + movu m1, [tlq+r5*2+2] + movu m3, [tlq+r5*2] + lea r5, [r4+dyq] + sar r4, 6 ; base3 + pand m6, m5, m7 + psllw m6, 9 + psubw m2, m0 + pmulhrsw m2, m6 + psraw m6, m7, 15 + paddw m7, m8 + paddw m0, m2 + movu m2, [tlq+r4*2+2] + movu m4, [tlq+r4*2] + lea r4, [r5+dyq] + sar r5, 6 ; base3 + vpblendvb m0, m9, m0, m6 + pand m6, m5, m7 + psllw m6, 9 + psubw m3, m1 + pmulhrsw m3, m6 + psraw m6, m7, 15 + paddw m7, m8 + paddw m1, m3 + vpblendvb m1, m9, m1, m6 + pand m6, m5, m7 + psllw m6, 9 + psubw m4, m2 + pmulhrsw m4, m6 + psraw m6, m7, 15 + paddw m7, m8 + paddw m2, m4 + movu m3, [tlq+r5*2+2] + movu m4, [tlq+r5*2] + vpblendvb m2, m9, m2, m6 + pand m6, m5, m7 + psllw m6, 9 + psubw m4, m3 + pmulhrsw m4, m6 + psraw m6, m7, 15 + paddw m7, m8 + lea r5, [dstq+strideq*4] + paddw m3, m4 + vpblendvb m3, m9, m3, m6 + punpckhwd m4, m0, m1 ; ab bb aa ba a9 b9 a8 b8 a3 b3 a2 b2 a1 b1 a0 b0 + punpcklwd m0, m1 ; af bf ae be ad bd ac bc a7 b7 a6 b6 a5 b5 a4 b4 + punpckhwd m1, m2, m3 ; cb db ca da c9 d9 c8 d8 c3 d3 c2 d2 c1 d1 c0 d0 + punpcklwd m2, m3 ; cf df ce de cd dd cc dc c7 d7 c6 d6 c5 d5 c4 d4 + punpckhdq m3, m4, m1 ; a9 b9 c9 d9 a8 b8 c8 d8 a1 b1 c1 d1 a0 b0 c0 d0 + vextracti128 xm6, m3, 1 + punpckldq m4, m1 ; ab bb cb db aa ba ca da a3 b3 c3 d3 a2 b2 c2 d2 + punpckhdq m1, m0, m2 ; ad bd cd dd ac bc cc dc a5 b5 c5 d5 a4 b4 c4 d4 + punpckldq m0, m2 ; af bf cf df ae be ce de a7 b7 c7 d7 a6 b6 c6 d6 + vextracti128 xm2, m4, 1 + movhps [dstq+strideq*0], xm6 + movq [dstq+strideq*1], xm6 + vextracti128 xm6, m1, 1 + movhps [dstq+strideq*2], xm2 + movq [dstq+r7 ], xm2 + vextracti128 xm2, m0, 1 + movhps [r5 +strideq*0], xm6 + movq [r5 +strideq*1], xm6 + movhps [r5 +strideq*2], xm2 + movq [r5 +r7 ], xm2 + lea r5, [dstq+strideq*8] + movhps [r5 +strideq*0], xm3 + movq [r5 +strideq*1], xm3 + movhps [r5 +strideq*2], xm4 + movq [r5 +r7 ], xm4 + lea r5, [r5+strideq*4] + movhps [r5 +strideq*0], xm1 + movq [r5 +strideq*1], xm1 + movhps [r5 +strideq*2], xm0 + movq [r5 +r7 ], xm0 + sub wd, 4 + jz .h16_end + add dstq, 8 + cmp r4d, maxbased + jg .h16_loop + mov hd, 4 +.h16_end_loop0: + mov r6d, wd + mov r2, dstq + test wb, 4 + jz .h16_end_loop + movq [dstq+strideq*0], xm9 + movq [dstq+strideq*1], xm9 + movq [dstq+strideq*2], xm9 + movq [dstq+r7 ], xm9 + and r6d, 120 + jz .h16_end_w4 + add dstq, 8 +.h16_end_loop: + mova [dstq+strideq*0], xm9 + mova [dstq+strideq*1], xm9 + mova [dstq+strideq*2], xm9 + mova [dstq+r7 ], xm9 + add dstq, 16 + sub r6d, 8 + jg .h16_end_loop +.h16_end_w4: + lea dstq, [r2+strideq*4] + dec hd + jg .h16_end_loop0 +.h16_end: + RET +.h32: + %assign stack_offset org_stack_offset + ALLOC_STACK -160, 9 + lea maxbased, [wq+31] + and maxbased, 31 + or maxbased, 32 ; imin(w+31, 63) + test angled, 0x400 + jnz .h32_main + vpbroadcastd m2, [pw_3] + movu m0, [tlq-28] ; 3 4 5 6 7 8 9 a b c d e f g h i + punpckhwd m1, m0, m0 + vpblendd m1, [tlq-26], 0x7f ; 4 5 6 7 8 9 a b c d e f g h i i + paddw m0, [tlq-30] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m1, m2 + paddw m0, [tlq-32] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pavgw m1, [tlq-34] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + lea r4, [rsp+128] + paddw m0, m1 + lea r5d, [maxbaseq-31] + psrlw m0, 2 + mova [r4], m0 +.h32_filter_loop: + mova m0, [tlq-62] + paddw m1, m2, [tlq-66] + paddw m0, [tlq-64] + pavgw m1, [tlq-58] + paddw m0, [tlq-60] + sub tlq, 32 + sub r4, 32 + paddw m0, m1 + psrlw m0, 2 + mova [r4], m0 + sub r5d, 16 + jg .h32_filter_loop + jl .h32_filter_h8 + mova m0, [tlq-62] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + punpcklwd xm1, xm0, xm0 + paddw m2, [tlq-58] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m0, [tlq-60] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpblendd m3, m1, [tlq-66], 0xfe ; 0 0 0 1 2 3 4 5 6 7 8 9 a b c d + vpblendd m1, [tlq-64], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + movzx r5d, word [tlq-62] + movzx r2d, word [tlq-60] + pavgw m2, m3 + sub r2d, r5d + paddw m0, m1 + lea r2d, [r2+r5*8+4] + paddw m0, m2 + shr r2d, 3 + psrlw m0, 2 + mova [r4-32], m0 + mov [r4-36], r5w + mov [r4-34], r2w + lea tlq, [rsp+158] + mov r4d, 65 + cmp wd, 64 + cmove maxbased, r4d + jmp .h32_main +.h32_filter_h8: + mova xm0, [tlq-46] ; 0 1 2 3 4 5 6 7 + pblendw xm1, xm0, [tlq-48], 0xfe ; 0 0 1 2 3 4 5 6 + paddw xm2, [tlq-42] ; 2 3 4 5 6 7 8 9 + paddw xm0, [tlq-44] ; 1 2 3 4 5 6 7 8 + vpblendd xm3, xm1, [tlq-50], 0x0e ; 0 0 0 1 2 3 4 5 + lea tlq, [rsp+158] + pavgw xm2, xm3 + paddw xm0, xm1 + paddw xm0, xm2 + psrlw xm0, 2 + mova [r4-16], xm0 +.h32_main: + movd xm6, dyd + neg maxbaseq + vpbroadcastw m7, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m6, xm6 + lea r4d, [maxbaseq+dyq+15*64] + neg dyq + movd xm4, r4d + vpbroadcastd m8, [pw_m1024] + lea r4, [dyq+63] + vpbroadcastw m4, xm4 + or maxbased, 63 + psubw m4, [z_base_inc] +.h32_loop: + mov r5, r4 + sar r5, 6 + movu m1, [tlq+r5*2-64] + movu m0, [tlq+r5*2-62] + pand m3, m5, m4 + psllw m3, 9 + psubw m1, m0 + pmulhrsw m1, m3 + pcmpgtw m2, m8, m4 + paddw m0, m1 + vpblendvb m0, m7, m0, m2 + movu m2, [tlq+r5*2-32] + movu m1, [tlq+r5*2-30] + add r4, dyq + sub rsp, 64 + psubw m2, m1 + pmulhrsw m2, m3 + psraw m3, m4, 15 + paddw m4, m6 + mova [rsp+32*0], m0 + paddw m1, m2 + vpblendvb m1, m7, m1, m3 + mova [rsp+32*1], m1 + dec wd + jz .h32_transpose + cmp r4d, maxbased + jg .h32_loop +.h32_end_loop: + sub rsp, 64 + mova [rsp+32*0], m7 + mova [rsp+32*1], m7 + dec wd + jg .h32_end_loop +.h32_transpose: + lea r3, [strideq*3] + lea r4, [strideq*5] + mov r8, dstq + lea r5, [strideq+r3*2] +.h32_transpose_loop0: + lea r6, [rsp+32] + lea r2, [r8+org_wq*2-16] +.h32_transpose_loop: + mova m0, [r6+64*7] + mova m1, [r6+64*6] + mova m2, [r6+64*5] + mova m3, [r6+64*4] + mova m4, [r6+64*3] + mova m5, [r6+64*2] + mova m6, [r6+64*1] + mova m7, [r6+64*0] + punpckhwd m8, m0, m1 ; a3 b3 a2 b2 a1 b1 a0 b0 + punpcklwd m0, m1 ; a7 b7 a6 b6 a5 b5 a4 b4 + punpckhwd m1, m2, m3 ; c3 d3 c2 d2 c1 d1 c0 d0 + punpcklwd m2, m3 ; c7 d7 c6 d6 c5 d5 c4 d4 + punpckhwd m3, m4, m5 ; e3 f3 e2 f2 e1 f1 e0 f0 + punpcklwd m4, m5 ; e7 f7 e6 f6 e5 f5 e4 f4 + punpckhwd m5, m6, m7 ; g3 h3 g2 h2 g1 h1 g0 h0 + punpcklwd m6, m7 ; g7 h7 g6 h6 g5 h5 g4 h4 + lea dstq, [r2+strideq*8] + sub r6, 32 + punpckhdq m7, m8, m1 ; a1 b1 c1 d1 a0 b0 c0 d0 + punpckldq m8, m1 ; a3 b3 c3 d3 a2 b2 c2 d2 + punpckhdq m1, m3, m5 ; e1 f1 g1 h1 e0 f0 g0 h0 + punpckldq m3, m5 ; e3 f3 g3 h3 e2 f2 g2 h2 + punpckhqdq m5, m7, m1 ; 8 0 + vextracti128 [r2 +strideq*0], m5, 1 + punpcklqdq m7, m1 ; 9 1 + mova [dstq+strideq*0], xm5 + punpckhqdq m1, m8, m3 ; 10 2 + vextracti128 [r2 +strideq*1], m7, 1 + punpcklqdq m8, m3 ; 11 3 + mova [dstq+strideq*1], xm7 + punpckhdq m3, m0, m2 ; a5 b5 c5 d5 a4 b4 c4 d4 + vextracti128 [r2 +strideq*2], m1, 1 + punpckldq m0, m2 ; a7 b7 c7 d7 a6 b6 c6 d6 + mova [dstq+strideq*2], xm1 + punpckhdq m2, m4, m6 ; e5 f5 g5 h5 e4 f4 g4 h4 + vextracti128 [r2 +r3 ], m8, 1 + punpckldq m4, m6 ; e7 f7 g7 h7 e6 f6 g6 h6 + mova [dstq+r3 ], xm8 + punpckhqdq m6, m3, m2 ; 12 4 + vextracti128 [r2 +strideq*4], m6, 1 + punpcklqdq m3, m2 ; 13 5 + mova [dstq+strideq*4], xm6 + punpckhqdq m2, m0, m4 ; 14 6 + vextracti128 [r2 +r4 ], m3, 1 + punpcklqdq m0, m4 ; 15 7 + mova [dstq+r4 ], xm3 + vextracti128 [r2 +r3*2 ], m2, 1 + mova [dstq+r3*2 ], xm2 + vextracti128 [r2 +r5 ], m0, 1 + mova [dstq+r5 ], xm0 + lea r2, [dstq+strideq*8] + cmp r6, rsp + jae .h32_transpose_loop + add rsp, 64*8 + sub org_wd, 8 + jg .h32_transpose_loop0 +.h32_end: + RET +.h64: + %assign stack_offset org_stack_offset + ALLOC_STACK -256, 10 + lea maxbased, [wq+63] + test angled, 0x400 + jnz .h64_main + vpbroadcastd m2, [pw_3] + movu m0, [tlq-28] ; 3 4 5 6 7 8 9 a b c d e f g h i + punpckhwd m1, m0, m0 + vpblendd m1, [tlq-26], 0x7f ; 4 5 6 7 8 9 a b c d e f g h i i + paddw m0, [tlq-30] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m1, m2 + paddw m0, [tlq-32] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + pavgw m1, [tlq-34] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + lea r4, [rsp+224] + paddw m0, m1 + lea r5d, [wq+32] + psrlw m0, 2 + mova [r4], m0 +.h64_filter_loop: + mova m0, [tlq-62] + paddw m1, m2, [tlq-66] + paddw m0, [tlq-64] + pavgw m1, [tlq-58] + paddw m0, [tlq-60] + sub tlq, 32 + sub r4, 32 + paddw m0, m1 + psrlw m0, 2 + mova [r4], m0 + sub r5d, 16 + jg .h64_filter_loop + mova m0, [tlq-62] ; 0 1 2 3 4 5 6 7 8 9 a b c d e f + punpcklwd xm1, xm0, xm0 + paddw m2, [tlq-58] ; 2 3 4 5 6 7 8 9 a b c d e f g h + paddw m0, [tlq-60] ; 1 2 3 4 5 6 7 8 9 a b c d e f g + vpblendd m3, m1, [tlq-66], 0xfe ; 0 0 0 1 2 3 4 5 6 7 8 9 a b c d + vpblendd m1, [tlq-64], 0xfe ; 0 0 1 2 3 4 5 6 7 8 9 a b c d e + lea tlq, [rsp+254] + pavgw m2, m3 + paddw m0, m1 + paddw m0, m2 + psrlw m0, 2 + mova [r4-32], m0 +.h64_main: + neg maxbaseq + movd xm4, dyd + vpbroadcastw m6, [tlq+maxbaseq*2] + shl maxbased, 6 + vpbroadcastw m4, xm4 + lea r4d, [maxbaseq+dyq+15*64] + neg dyq + vpbroadcastd m7, [pw_m1024] + movd xm3, r4d + lea r4, [dyq+63] + paddw m8, m7, m7 + vpbroadcastw m3, xm3 + or maxbased, 63 + paddw m9, m8, m7 + psubw m3, [z_base_inc] +.h64_loop: + mov r5, r4 + sar r5, 6 + movu m1, [tlq+r5*2-128] + movu m0, [tlq+r5*2-126] + pand m2, m5, m3 + psllw m2, 9 + psubw m1, m0 + pmulhrsw m1, m2 + sub rsp, 128 + paddw m0, m1 + pcmpgtw m1, m9, m3 + vpblendvb m0, m6, m0, m1 + mova [rsp+32*0], m0 + movu m1, [tlq+r5*2-96] + movu m0, [tlq+r5*2-94] + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + pcmpgtw m1, m8, m3 + vpblendvb m0, m6, m0, m1 + mova [rsp+32*1], m0 + movu m1, [tlq+r5*2-64] + movu m0, [tlq+r5*2-62] + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + pcmpgtw m1, m7, m3 + vpblendvb m0, m6, m0, m1 + mova [rsp+32*2], m0 + movu m1, [tlq+r5*2-32] + movu m0, [tlq+r5*2-30] + psubw m1, m0 + pmulhrsw m1, m2 + add r4, dyq + psraw m2, m3, 15 + paddw m3, m4 + paddw m0, m1 + vpblendvb m0, m6, m0, m2 + mova [rsp+32*3], m0 + dec wd + jz .h64_transpose + cmp r4d, maxbased + jg .h64_loop +.h64_end_loop: + sub rsp, 128 + mova [rsp+32*0], m6 + mova [rsp+32*1], m6 + mova [rsp+32*2], m6 + mova [rsp+32*3], m6 + dec wd + jg .h64_end_loop +.h64_transpose: + lea r2, [strideq*3] + lea r3, [strideq*5] + mov r5, dstq + lea r4, [strideq+r2*2] +.h64_transpose_loop0: + lea r6, [rsp+112] + lea dstq, [r5+org_wq*2-32] +.h64_transpose_loop: + mova xm0, [r6+128*15] + vinserti128 m0, [r6+128* 7], 1 + mova xm1, [r6+128*14] + vinserti128 m1, [r6+128* 6], 1 + mova xm2, [r6+128*13] + vinserti128 m2, [r6+128* 5], 1 + mova xm3, [r6+128*12] + vinserti128 m3, [r6+128* 4], 1 + mova xm4, [r6+128*11] + vinserti128 m4, [r6+128* 3], 1 + mova xm5, [r6+128*10] + vinserti128 m5, [r6+128* 2], 1 + mova xm6, [r6+128* 9] + vinserti128 m6, [r6+128* 1], 1 + mova xm7, [r6+128* 8] + vinserti128 m7, [r6+128* 0], 1 + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m6, m7 + punpcklwd m6, m7 + sub r6, 16 + punpckhdq m7, m8, m1 + punpckldq m8, m1 + punpckhdq m1, m3, m5 + punpckldq m3, m5 + punpckhqdq m5, m7, m1 + punpcklqdq m7, m1 + punpckhqdq m1, m8, m3 + punpcklqdq m8, m3 + punpckhdq m3, m0, m2 + mova [dstq+strideq*0], m5 + punpckldq m0, m2 + mova [dstq+strideq*1], m7 + punpckhdq m2, m4, m6 + mova [dstq+strideq*2], m1 + punpckldq m4, m6 + mova [dstq+r2 ], m8 + punpckhqdq m6, m3, m2 + mova [dstq+strideq*4], m6 + punpcklqdq m3, m2 + mova [dstq+r3 ], m3 + punpckhqdq m2, m0, m4 + mova [dstq+r2*2 ], m2 + punpcklqdq m0, m4 + mova [dstq+r4 ], m0 + lea dstq, [dstq+strideq*8] + cmp r6, rsp + jae .h64_transpose_loop + add rsp, 128*16 + sub org_wd, 16 + jg .h64_transpose_loop0 +.h64_end: + RET + +%macro FILTER_1BLK 5 ; dst, src, tmp, shuf, bdmax +%ifnum %4 + pshufb xm%2, xm%4 +%else + pshufb xm%2, %4 +%endif + vinserti128 m%2, xm%2, 1 + pshufd m%1, m%2, q0000 + pmaddwd m%1, m2 + pshufd m%3, m%2, q1111 + pmaddwd m%3, m3 + paddd m%1, m1 + paddd m%1, m%3 + pshufd m%3, m%2, q2222 + pmaddwd m%3, m4 + paddd m%1, m%3 + pshufd m%3, m%2, q3333 + pmaddwd m%3, m5 + paddd m%1, m%3 + psrad m%1, 4 + packusdw m%1, m%1 + pminsw m%1, m%5 +%endmacro + +%macro FILTER_2BLK 7 ; dst, src, tmp_dst, tmp_src, tmp, shuf, bdmax + pshufb m%2, m%6 + vpermq m%4, m%2, q3232 + vinserti128 m%2, xm%2, 1 + pshufd m%1, m%2, q0000 + pshufd m%3, m%4, q0000 + pmaddwd m%1, m2 + pmaddwd m%3, m2 + paddd m%1, m1 + paddd m%3, m1 + pshufd m%5, m%2, q1111 + pmaddwd m%5, m3 + paddd m%1, m%5 + pshufd m%5, m%4, q1111 + pmaddwd m%5, m3 + paddd m%3, m%5 + pshufd m%5, m%2, q2222 + pmaddwd m%5, m4 + paddd m%1, m%5 + pshufd m%5, m%4, q2222 + pmaddwd m%5, m4 + paddd m%3, m%5 + pshufd m%5, m%2, q3333 + pmaddwd m%5, m5 + paddd m%1, m%5 + pshufd m%5, m%4, q3333 + pmaddwd m%5, m5 + paddd m%3, m%5 + psrad m%1, 4 + psrad m%3, 4 + packusdw m%1, m%3 + pminsw m%1, m%7 +%endmacro + +; The ipred_filter SIMD processes 4x2 blocks in the following order which +; increases parallelism compared to doing things row by row. One redundant +; block is calculated for w8 and w16, two for w32. +; w4 w8 w16 w32 +; 1 1 2 1 2 3 5 1 2 3 5 b c d f +; 2 2 3 2 4 5 7 2 4 5 7 c e f h +; 3 3 4 4 6 7 9 4 6 7 9 e g h j +; ___ 4 ___ 4 5 ___ 6 8 9 a ___ 6 8 9 a g i j k ___ +; 5 8 8 i + +cglobal ipred_filter_16bpc, 3, 9, 0, dst, stride, tl, w, h, filter +%assign org_stack_offset stack_offset +%define base r6-ipred_filter_16bpc_avx2_table + lea r6, [filter_intra_taps] + tzcnt wd, wm +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + shl filterd, 6 + add filterq, r6 + lea r6, [ipred_filter_16bpc_avx2_table] + vbroadcasti128 m0, [tlq-6] + movsxd wq, [r6+wq*4] + vpbroadcastd m1, [base+pd_8] + pmovsxbw m2, [filterq+16*0] + pmovsxbw m3, [filterq+16*1] + pmovsxbw m4, [filterq+16*2] + pmovsxbw m5, [filterq+16*3] + add wq, r6 + mov hd, hm + jmp wq +.w4: + WIN64_SPILL_XMM 10 + mova xm8, [base+filter_shuf2] + vpbroadcastw m9, r8m ; bitdepth_max + lea r7, [6+hq*2] + sub tlq, r7 + jmp .w4_loop_start +.w4_loop: + pinsrq xm0, [tlq+hq*2], 0 + lea dstq, [dstq+strideq*2] +.w4_loop_start: + FILTER_1BLK 6, 0, 7, 8, 9 + vextracti128 xm0, m6, 1 + movq [dstq+strideq*0], xm6 + movq [dstq+strideq*1], xm0 + sub hd, 2 + jg .w4_loop + RET +ALIGN function_align +.w8: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + vbroadcasti128 m14, [base+filter_shuf3] + vpbroadcastw m15, r8m ; bitdepth_max + FILTER_1BLK 10, 0, 7, [base+filter_shuf2], 15 + vpermq m6, m10, q1302 ; ____ ____ | ____ 4321 + pslldq m8, m0, 4 + psrldq m7, m6, 2 + psrldq m0, m6, 10 + punpcklwd m7, m0 + vpblendd m8, m6, 0x33 ; _0__ 4321 | ____ 4321 + vpblendd m8, m7, 0x40 ; _056 4321 | ____ 4321 + vpblendd m8, [tlq-6], 0x30 ; _056 4321 | ____ 4321 + lea r7, [16+hq*2] + sub tlq, r7 + jmp .w8_loop_start +.w8_loop: + vpermq m8, m9, q1302 ; ____ 4321 | ____ 4321 + vpermq m6, m9, q2031 + psrldq m0, m6, 2 + psrldq m6, 10 + punpcklwd m6, m0 + vpblendd m8, m7, 0x80 ; _0__ 4321 | ____ 4321 + vpblendd m8, m6, 0x40 ; _056 4321 | ____ 4321 + mova m10, m9 +.w8_loop_start: + vpblendd m8, [tlq+hq*2], 0x0C ; _056 4321 | _056 4321 + call .main + vpblendd m10, m9, 0xCC + mova [dstq+strideq*0], xm10 + vextracti128 [dstq+strideq*1], m10, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + %assign stack_offset stack_offset - stack_size_padded + ALLOC_STACK 32, 16 + vpbroadcastw m15, r8m ; bitdepth_max + sub hd, 2 + TAIL_CALL .w16_main, 0 +.w16_main: + mova xm10, [base+filter_shuf2] + FILTER_1BLK 13, 0, 6, 10, 15 + vpermq m12, m13, q3120 + mova xm14, [base+filter_shuf3] + vinserti128 m14, [base+filter_shuf1], 1 + vpbroadcastq m0, [tlq+10] + vpblendd m0, [tlq-16], 0x4C ; ___0 4321 | _056 ____ + psrldq m6, m12, 8 + vpblendd m0, m6, 0x03 ; ___0 4321 | _056 4321 + punpcklwd m6, m12 + vpblendd m0, m6, 0x80 ; 56_0 4321 | _056 4321 + FILTER_2BLK 12, 0, 6, 7, 8, 14, 15 + vpblendd m13, m12, 0xCC + vpermq m12, m12, q2031 ; 6___ 5___ + psrldq xm6, xm12, 2 + psrldq xm8, xm12, 12 + vpblendd xm6, xm8, 0x01 + pblendw xm6, [tlq+10], 0xF8 ; 4321 056_ + FILTER_1BLK 11, 6, 8, 10, 15 + vpermq m11, m11, q3120 + pshufd m9, m11, q1032 + movu m8, [tlq+6] ; __43 210_ | ____ ____ + pshufd m8, m8, q3021 ; __0_ 4321 | ____ ____ + pshufhw m8, m8, q3201 ; ___0 4321 | ____ ____ + vpblendd m9, m8, 0x70 ; ___0 4321 | ____ 4321 + mova [dstq+strideq*0], xm13 + vextracti128 [dstq+strideq*1], m13, 1 + lea r7, [20+hq*2] + sub tlq, r7 + vpermq m6, m12, q0123 ; ____ 4321 | ____ 4321 + jmp .w16_loop_start +.w16_loop: + vpermq m13, m13, q3322 + vpermq m11, m9, q2020 + vpermq m9, m9, q1302 + vpermq m6, m12, q0123 + psrldq m7, 4 + vpblendd m13, m10, 0xCC + vpblendd m9, m7, 0x40 + mova m0, [rsp+8] + mova [dstq+strideq*0], xm13 + vextracti128 [dstq+strideq*1], m13, 1 +.w16_loop_start: + mova m13, m12 + vpblendd m0, [tlq+hq*2], 0x0C + psrldq m7, m12, 8 + punpcklwd m7, m12 + vpblendd m0, m6, 0x33 ; ___0 4321 | _056 4321 + vpblendd m0, m7, 0x80 ; 56_0 4321 | _056 4321 + FILTER_2BLK 10, 0, 6, 7, 8, 14, 15 + vpermq m12, m10, q2031 + mova [rsp+8], m0 + psrldq m8, m11, 8 + psrldq xm6, xm12, 2 + psrldq xm7, xm12, 10 + psrldq xm0, xm13, 2 + punpcklwd m8, m11 + punpcklwd xm7, xm6 + vpblendd m8, m9, 0x73 ; 56_0 4321 | ____ 4321 + vpblendd m8, m7, 0x04 ; 56_0 4321 | __56 4321 + vpblendd m8, m0, 0x08 ; 56_0 4321 | _056 4321 + call .main + vpermq m8, m11, q3120 + vpblendd m6, m8, m9, 0xCC + mova [dstq+strideq*0+16], xm6 + vextracti128 [dstq+strideq*1+16], m6, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + vpermq m8, m9, q3120 + vextracti128 xm0, m8, 1 ; 4321 ____ + pshufd xm11, xm11, q1032 + vpblendd xm0, xm11, 0x02 ; 4321 0___ + psrldq xm6, xm8, 2 + psrldq xm7, xm8, 12 + pblendw xm0, xm6, 0x4 ; 4321 05__ + pblendw xm0, xm7, 0x2 ; 4321 056_ + FILTER_1BLK 6, 0, 7, [base+filter_shuf2], 15 + vpermq m12, m13, q1302 + vpblendd m12, m10, 0xCC + vpblendd m9, m6, 0xCC + mova [dstq+strideq*0+ 0], xm12 + mova [dstq+strideq*0+16], xm9 + vextracti128 [dstq+strideq*1+ 0], m12, 1 + vextracti128 [dstq+strideq*1+16], m9, 1 + ret +ALIGN function_align +.w32: + %assign stack_offset org_stack_offset + ALLOC_STACK 64, 16 + vpbroadcastw m15, r8m ; bitdepth_max + sub hd, 2 + lea r3, [dstq+32] + lea r5d, [hd*2+20] + call .w16_main + mov dstq, r3 + lea tlq, [tlq+r5+32] + sub r5d, 20 + shr r5d, 1 + sub r5d, 2 + lea r4, [dstq+strideq*2-2] +DEFINE_ARGS dst, stride, tl, stride3, left, h + lea stride3q, [strideq*3] + movu m8, [tlq-6] ; 4321 0___ + mova xm10, [base+filter_shuf2] + pinsrw xm0, xm8, [dstq+strideq*0-2], 2 + pinsrw xm0, xm0, [dstq+strideq*1-2], 1 ; 4321 056_ + pinsrw xm9, [leftq+strideq*0], 5 + pinsrw xm9, [leftq+strideq*1], 4 + FILTER_1BLK 13, 0, 6, 10, 15 + vpermq m12, m13, q3120 + mova xm14, [base+filter_shuf3] + vinserti128 m14, [base+filter_shuf1], 1 + psrldq m6, m12, 8 + punpcklwd m7, m6, m12 + vpblendd m0, m6, 0x03 ; ___0 ____ | _0__ 4321 + vpblendd m0, m7, 0x80 ; 56_0 ____ | _0__ 4321 + vpblendd m0, m8, 0x30 ; 56_0 4321 | _0__ 4321 + vpblendd m0, m9, 0x04 ; 56_0 4321 | _056 4321 + FILTER_2BLK 12, 0, 6, 7, 8, 14, 15 + vpblendd m13, m12, 0xCC + pinsrw xm9, [leftq+strideq*2], 3 + pinsrw xm9, [leftq+stride3q ], 2 + lea leftq, [leftq+strideq*4] + pinsrw xm9, [leftq+strideq*0], 1 + pinsrw xm9, [leftq+strideq*1], 0 + movq [rsp+32], xm9 + mov r7d, 1 + pslldq m8, m9, 4 + vpblendd m0, m8, 0x0C ; ___0 ____ | _056 ____ + vpermq m12, m12, q2031 ; 6___ 5___ + psrldq xm6, xm12, 2 + psrldq xm7, xm12, 12 + vpblendd xm6, xm7, 0x01 ; ____ _56_ + pblendw xm6, [tlq+10], 0xF8 ; 4321 056_ + FILTER_1BLK 11, 6, 7, 10, 15 + vpermq m11, m11, q3120 + pshufd m9, m11, q1032 + vbroadcasti128 m8, [tlq+22] ; __43 210_ | ____ ____ + pshufd m8, m8, q3021 ; __0_ 4321 | ____ ____ + pshufhw m8, m8, q3201 ; ___0 4321 | ____ ____ + vpblendd m9, m8, 0x70 ; ___0 4321 | ____ 4321 + mova [dstq+strideq*0], xm13 + vextracti128 [dstq+strideq*1], m13, 1 + vpermq m6, m12, q0123 ; ____ 4321 | ____ 4321 + jmp .w32_loop_start +.w32_loop_last: + mova m0, [rsp+0] + jmp .w32_loop +.w32_loop_left: + mova m0, [rsp+0] + vpblendd m0, [rsp+32+r7*4-12], 0x0C + dec r7d + jg .w32_loop + cmp hd, 2 + je .w32_loop + pinsrw xm6, [rsp+32], 6 + pinsrw xm6, [leftq+strideq*2], 5 + pinsrw xm6, [leftq+stride3q ], 4 + lea leftq, [leftq+strideq*4] + pinsrw xm6, [leftq+strideq*0], 3 + pinsrw xm6, [leftq+strideq*1], 2 + pinsrw xm6, [leftq+strideq*2], 1 + pinsrw xm6, [leftq+stride3q ], 0 + lea leftq, [leftq+strideq*4] + movu [rsp+36], xm6 + pinsrw xm6, [leftq+strideq*0], 1 + pinsrw xm6, [leftq+strideq*1], 0 + movd [rsp+32], xm6 + mov r7d, 4 +.w32_loop: + vpermq m13, m13, q3322 + vpermq m11, m9, q2020 + vpermq m9, m9, q1302 + vpermq m6, m12, q0123 + psrldq m7, 4 + vpblendd m13, m10, 0xCC + vpblendd m9, m7, 0x40 ; ___0 4321 | ____ 4321 + mova [dstq+strideq*0], xm13 + vextracti128 [dstq+strideq*1], m13, 1 +.w32_loop_start: + mova m13, m12 + psrldq m7, m12, 8 + punpcklwd m7, m12 + vpblendd m0, m6, 0x33 ; ___0 4321 | _056 4321 + vpblendd m0, m7, 0x80 ; 56_0 4321 | _056 4321 + FILTER_2BLK 10, 0, 6, 7, 8, 14, 15 + vpermq m12, m10, q2031 + mova [rsp+0], m0 + psrldq m8, m11, 8 + psrldq xm6, xm12, 2 + psrldq xm7, xm12, 10 + psrldq xm0, xm13, 2 + punpcklwd m8, m11 + punpcklwd xm7, xm6 + vpblendd m8, m9, 0x73 ; 56_0 4321 | ____ 4321 + vpblendd m8, m7, 0x04 ; 56_0 4321 | __56 4321 + vpblendd m8, m0, 0x08 ; 56_0 4321 | _056 4321 + call .main + vpermq m8, m11, q3120 + vpblendd m6, m8, m9, 0xCC + mova [dstq+strideq*0+16], xm6 + vextracti128 [dstq+strideq*1+16], m6, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop_left + jz .w32_loop_last + vpermq m8, m9, q3120 + vextracti128 xm0, m8, 1 ; 4321 ____ + pshufd xm11, xm11, q1032 + vpblendd xm0, xm11, 0x02 ; 4321 0___ + psrldq xm6, xm8, 2 + psrldq xm7, xm8, 12 + pblendw xm0, xm6, 0x4 ; 4321 05__ + pblendw xm0, xm7, 0x2 ; 4321 056_ + FILTER_1BLK 6, 0, 7, [base+filter_shuf2], 15 + vpermq m12, m13, q1302 + vpblendd m12, m10, 0xCC + vpblendd m9, m6, 0xCC + mova [dstq+strideq*0+ 0], xm12 + mova [dstq+strideq*0+16], xm9 + vextracti128 [dstq+strideq*1+ 0], m12, 1 + vextracti128 [dstq+strideq*1+16], m9, 1 + RET +.main: + FILTER_2BLK 9, 8, 6, 7, 0, 14, 15 + ret + +%if WIN64 +DECLARE_REG_TMP 5 +%else +DECLARE_REG_TMP 7 +%endif + +%macro IPRED_CFL 1 ; ac in, unpacked pixels out + psignw m3, m%1, m1 + pabsw m%1, m%1 + pmulhrsw m%1, m2 + psignw m%1, m3 + paddw m%1, m0 +%endmacro + +cglobal ipred_cfl_top_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + add tlq, 2 + movd xm4, wd + pxor m6, m6 + vpbroadcastw m7, r7m + pavgw xm4, xm6 + tzcnt wd, wd + movd xm5, wd + movu m0, [tlq] + lea t0, [ipred_cfl_left_16bpc_avx2_table] + movsxd r6, [t0+wq*4] + add r6, t0 + add t0, ipred_cfl_splat_16bpc_avx2_table-ipred_cfl_left_16bpc_avx2_table + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 + +cglobal ipred_cfl_left_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + mov hd, hm ; zero upper half + sub tlq, hq + movd xm4, hd + sub tlq, hq + pxor m6, m6 + vpbroadcastw m7, r7m + pavgw xm4, xm6 + tzcnt r6d, hd + movd xm5, r6d + movu m0, [tlq] + lea t0, [ipred_cfl_left_16bpc_avx2_table] + movsxd r6, [t0+r6*4] + add r6, t0 + add t0, ipred_cfl_splat_16bpc_avx2_table-ipred_cfl_left_16bpc_avx2_table + tzcnt wd, wd + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h32: + paddw m0, [tlq+32] +.h16: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 +.h8: + psrldq xm1, xm0, 8 + paddw xm0, xm1 +.h4: + punpcklwd xm0, xm6 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + paddd xm0, xm4 + psrld xm0, xm5 + vpbroadcastw m0, xm0 + jmp wq + +cglobal ipred_cfl_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + movifnidn wd, wm + tzcnt r6d, hd + lea t0d, [wq+hq] + movd xm4, t0d + tzcnt t0d, t0d + movd xm5, t0d + lea t0, [ipred_cfl_16bpc_avx2_table] + tzcnt wd, wd + movsxd r6, [t0+r6*4] + movsxd wq, [t0+wq*4+4*4] + psrlw xm4, 1 + pxor m6, m6 + vpbroadcastw m7, r7m + add r6, t0 + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h4: + movq xm0, [tlq-8] + jmp wq +.w4: + movq xm1, [tlq+2] + paddw m0, m4 + paddw m0, m1 + psrlq m1, m0, 32 + paddw m0, m1 + psrld m1, m0, 16 + paddw m0, m1 + cmp hd, 4 + jg .w4_mul + psrlw xm0, 3 + jmp .w4_end +.w4_mul: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 + lea r2d, [hq*2] + mov r6d, 0xAAAB6667 + shrx r6d, r6d, r2d + punpckhwd xm1, xm0, xm6 + punpcklwd xm0, xm6 + paddd xm0, xm1 + movd xm1, r6d + psrld xm0, 2 + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w4_end: + vpbroadcastw m0, xm0 +.s4: + vpbroadcastw m1, alpham + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s4_loop: + mova m4, [acq] + IPRED_CFL 4 + pmaxsw m4, m6 + pminsw m4, m7 + vextracti128 xm5, m4, 1 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm5 + movhps [dstq+strideq*1], xm4 + movhps [dstq+r6 ], xm5 + lea dstq, [dstq+strideq*4] + add acq, 32 + sub hd, 4 + jg .s4_loop + RET +ALIGN function_align +.h8: + mova xm0, [tlq-16] + jmp wq +.w8: + vextracti128 xm1, m0, 1 + paddw xm0, [tlq+2] + paddw xm0, xm4 + paddw xm0, xm1 + psrld xm1, xm0, 16 + paddw xm0, xm1 + pblendw xm0, xm6, 0xAA + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 8 + je .w8_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w8_end: + vpbroadcastw m0, xm0 +.s8: + vpbroadcastw m1, alpham + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s8_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + pmaxsw m4, m6 + pmaxsw m5, m6 + pminsw m4, m7 + pminsw m5, m7 + mova [dstq+strideq*0], xm4 + mova [dstq+strideq*2], xm5 + vextracti128 [dstq+strideq*1], m4, 1 + vextracti128 [dstq+r6 ], m5, 1 + lea dstq, [dstq+strideq*4] + add acq, 64 + sub hd, 4 + jg .s8_loop + RET +ALIGN function_align +.h16: + mova m0, [tlq-32] + jmp wq +.w16: + paddw m0, [tlq+2] + vextracti128 xm1, m0, 1 + paddw xm0, xm4 + paddw xm0, xm1 + punpckhwd xm1, xm0, xm6 + punpcklwd xm0, xm6 + paddd xm0, xm1 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 16 + je .w16_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + test hb, 8|32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w16_end: + vpbroadcastw m0, xm0 +.s16: + vpbroadcastw m1, alpham + pabsw m2, m1 + psllw m2, 9 +.s16_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + pmaxsw m4, m6 + pmaxsw m5, m6 + pminsw m4, m7 + pminsw m5, m7 + mova [dstq+strideq*0], m4 + mova [dstq+strideq*1], m5 + lea dstq, [dstq+strideq*2] + add acq, 64 + sub hd, 2 + jg .s16_loop + RET +ALIGN function_align +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-32] + jmp wq +.w32: + paddw m0, [tlq+ 2] + paddw m0, [tlq+34] + vextracti128 xm1, m0, 1 + paddw xm0, xm4 + paddw xm0, xm1 + punpcklwd xm1, xm0, xm6 + punpckhwd xm0, xm6 + paddd xm0, xm1 + psrlq xm1, xm0, 32 + paddd xm0, xm1 + psrldq xm1, xm0, 8 + paddd xm0, xm1 + psrld xm0, xm5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x6667AAAB + shrx r6d, r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 + psrlw xm0, 1 +.w32_end: + vpbroadcastw m0, xm0 +.s32: + vpbroadcastw m1, alpham + pabsw m2, m1 + psllw m2, 9 +.s32_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + pmaxsw m4, m6 + pmaxsw m5, m6 + pminsw m4, m7 + pminsw m5, m7 + mova [dstq+32*0], m4 + mova [dstq+32*1], m5 + add dstq, strideq + add acq, 64 + dec hd + jg .s32_loop + RET + +cglobal ipred_cfl_128_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + mov r6d, r7m + shr r6d, 11 + lea t0, [ipred_cfl_splat_16bpc_avx2_table] + tzcnt wd, wd + movifnidn hd, hm + movsxd wq, [t0+wq*4] + vpbroadcastd m0, [t0-ipred_cfl_splat_16bpc_avx2_table+pw_512+r6*4] + pxor m6, m6 + vpbroadcastw m7, r7m + add wq, t0 + movifnidn acq, acmp + jmp wq + +cglobal ipred_cfl_ac_420_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm + vpbroadcastd m5, [pw_2] + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + jg .w16 + je .w8 +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + mova xm0, [ypxq+strideq*2] + mova xm1, [ypxq+r3 ] + vinserti128 m0, [ypxq+strideq*0], 1 + vinserti128 m1, [ypxq+strideq*1], 1 + lea ypxq, [ypxq+strideq*4] + pmaddwd m0, m5 + pmaddwd m1, m5 + paddd m0, m1 + vextracti128 xm1, m0, 1 + paddd m4, m0 + packssdw xm1, xm0 + mova [acq], xm1 + add acq, 16 + sub hd, 2 + jg .w4_loop + test hpadd, hpadd + jz .dc + vpermq m1, m1, q1111 + pslld xm0, 2 +.w4_hpad_loop: + mova [acq], m1 + paddd m4, m0 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .dc +.w8: + mov r5, acq + test wpadd, wpadd + jnz .w8_wpad1 +.w8_loop: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd m0, m1 + vextracti128 xm1, m0, 1 + paddd m4, m0 + packssdw xm1, xm0, xm1 + mova [acq], xm1 + add acq, 16 + dec hd + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz .dc + vinserti128 m1, xm1, 1 + pslld m0, 2 + jmp .hpad +.w8_wpad1: + pmaddwd xm0, xm5, [ypxq+strideq*0] + pmaddwd xm3, xm5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd xm0, xm3 + pshufd xm3, xm0, q3333 + packssdw xm1, xm0, xm3 + paddd xm0, xm3 + paddd xm4, xm0 + mova [acq], xm1 + add acq, 16 + dec hd + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad: + mova m0, [ypxq+strideq*0+ 0] + mova m1, [ypxq+strideq*1+ 0] + cmp wpadd, 2 + jl .w16_wpad1 + je .w16_wpad2 + vpbroadcastd m2, [ypxq+strideq*0+12] + vpbroadcastd m3, [ypxq+strideq*1+12] + vpblendd m0, m2, 0xf0 + vpblendd m1, m3, 0xf0 + jmp .w16_wpad_end +.w16_wpad2: + vpbroadcastd m2, [ypxq+strideq*0+28] + vpbroadcastd m3, [ypxq+strideq*1+28] + jmp .w16_wpad_end +.w16_wpad1: + vpbroadcastd m2, [ypxq+strideq*0+44] + vpbroadcastd m3, [ypxq+strideq*1+44] + vinserti128 m2, [ypxq+strideq*0+32], 0 + vinserti128 m3, [ypxq+strideq*1+32], 0 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + REPX {pmaddwd x, m5}, m0, m1, m2, m3 + paddd m0, m1 + paddd m2, m3 + packssdw m1, m0, m2 + paddd m0, m2 + vpermq m1, m1, q3120 + paddd m4, m0 + mova [acq], m1 + add acq, 32 + dec hd + jg .w16_wpad + jmp .w16_hpad +.w16: + mov r5, acq + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + pmaddwd m0, m5, [ypxq+strideq*0+ 0] + pmaddwd m2, m5, [ypxq+strideq*0+32] + pmaddwd m1, m5, [ypxq+strideq*1+ 0] + pmaddwd m3, m5, [ypxq+strideq*1+32] + lea ypxq, [ypxq+strideq*2] + paddd m0, m1 + paddd m2, m3 + packssdw m1, m0, m2 + paddd m0, m2 + vpermq m1, m1, q3120 + paddd m4, m0 + mova [acq], m1 + add acq, 32 + dec hd + jg .w16_loop +.w16_hpad: + add hpadd, hpadd + jz .dc + paddd m0, m0 +.hpad: + mova [acq+32*0], m1 + paddd m4, m0 + mova [acq+32*1], m1 + add acq, 32*2 + sub hpadd, 4 + jg .hpad +.dc: + vextracti128 xm1, m4, 1 + sub r5, acq ; -w*h*2 + tzcnt r1d, r5d + paddd xm4, xm1 + sub r1d, 2 + punpckhqdq xm1, xm4, xm4 + movd xm0, r1d + paddd xm1, xm4 + pshuflw xm4, xm1, q1032 + paddd xm1, xm4 + psrld xm1, xm0 + pxor xm0, xm0 + pavgw xm1, xm0 + vpbroadcastw m1, xm1 +.dc_loop: + mova m0, [acq+r5] + psubw m0, m1 + mova [acq+r5], m0 + add r5, 32 + jl .dc_loop + RET + +cglobal ipred_cfl_ac_422_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm + vpbroadcastd m5, [pw_4] + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + jg .w16 + je .w8 +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + mova xm0, [ypxq+strideq*0] + mova xm1, [ypxq+strideq*1] + vinserti128 m0, [ypxq+strideq*2], 1 + vinserti128 m1, [ypxq+r3 ], 1 + lea ypxq, [ypxq+strideq*4] + pmaddwd m0, m5 + pmaddwd m1, m5 + paddd m4, m0 + packssdw m0, m1 + paddd m4, m1 + mova [acq], m0 + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vextracti128 xm1, m1, 1 + vpermq m0, m0, q3333 + pslld xm1, 2 +.w4_hpad_loop: + mova [acq], m0 + paddd m4, m1 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc +.w8: + mov r5, acq + test wpadd, wpadd + jnz .w8_wpad1 +.w8_loop: + pmaddwd m1, m5, [ypxq+strideq*0] + pmaddwd m0, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd m4, m1 + packssdw m1, m0 + paddd m4, m0 + vpermq m2, m1, q3120 + mova [acq], m2 + add acq, 32 + sub hd, 2 + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vpermq m1, m1, q3131 + pslld m0, 2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w8_wpad1: + vpbroadcastd m1, [ypxq+strideq*0+12] + vpbroadcastd m0, [ypxq+strideq*1+12] + vinserti128 m1, [ypxq+strideq*0+ 0], 0 + vinserti128 m0, [ypxq+strideq*1+ 0], 0 + lea ypxq, [ypxq+strideq*2] + pmaddwd m1, m5 + pmaddwd m0, m5 + paddd m4, m1 + packssdw m1, m0 + paddd m4, m0 + vpermq m2, m1, q3120 + mova [acq], m2 + add acq, 32 + sub hd, 2 + jg .w8_wpad1 + jmp .w8_hpad +.w16: + mov r5, acq + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + pmaddwd m2, m5, [ypxq+strideq*0+ 0] + pmaddwd m1, m5, [ypxq+strideq*0+32] + pmaddwd m0, m5, [ypxq+strideq*1+ 0] + pmaddwd m3, m5, [ypxq+strideq*1+32] + lea ypxq, [ypxq+strideq*2] + paddd m4, m2 + packssdw m2, m1 + paddd m4, m1 + packssdw m1, m0, m3 + paddd m0, m3 + vpermq m2, m2, q3120 + paddd m4, m0 + vpermq m1, m1, q3120 + mova [acq+32*0], m2 + mova [acq+32*1], m1 + add acq, 32*2 + sub hd, 2 + jg .w16_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).w16_hpad +.w16_wpad: + mova m2, [ypxq+strideq*0+ 0] + mova m0, [ypxq+strideq*1+ 0] + cmp wpadd, 2 + jl .w16_wpad1 + je .w16_wpad2 + vpbroadcastd m1, [ypxq+strideq*0+12] + vpbroadcastd m3, [ypxq+strideq*1+12] + vpblendd m2, m1, 0xf0 + vpblendd m0, m3, 0xf0 + jmp .w16_wpad_end +.w16_wpad2: + vpbroadcastd m1, [ypxq+strideq*0+28] + vpbroadcastd m3, [ypxq+strideq*1+28] + jmp .w16_wpad_end +.w16_wpad1: + vpbroadcastd m1, [ypxq+strideq*0+44] + vpbroadcastd m3, [ypxq+strideq*1+44] + vinserti128 m1, [ypxq+strideq*0+32], 0 + vinserti128 m3, [ypxq+strideq*1+32], 0 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + REPX {pmaddwd x, m5}, m2, m0, m1, m3 + paddd m4, m2 + packssdw m2, m1 + paddd m4, m1 + packssdw m1, m0, m3 + paddd m0, m3 + vpermq m2, m2, q3120 + paddd m4, m0 + vpermq m1, m1, q3120 + mova [acq+32*0], m2 + mova [acq+32*1], m1 + add acq, 32*2 + sub hd, 2 + jg .w16_wpad + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).w16_hpad + +cglobal ipred_cfl_ac_444_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h + lea r6, [ipred_cfl_ac_444_16bpc_avx2_table] + tzcnt wd, wm + movifnidn hpadd, hpadm + vpbroadcastd m5, [pw_1] + movsxd wq, [r6+wq*4] + shl hpadd, 2 + add wq, r6 + mov hd, hm + pxor m4, m4 + sub hd, hpadd + jmp wq +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + movq xm0, [ypxq+strideq*0] + movhps xm0, [ypxq+strideq*1] + vpbroadcastq m1, [ypxq+strideq*2] + vpbroadcastq m2, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + vpblendd m0, m1, 0x30 + vpblendd m0, m2, 0xc0 + psllw m0, 3 + pmaddwd m1, m0, m5 + mova [acq], m0 + add acq, 32 + paddd m4, m1 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vpermq m0, m0, q3333 + paddd m1, m1 + mova [acq+32*0], m0 + vpermq m1, m1, q3333 + mova [acq+32*1], m0 + add acq, 32*2 + paddd m4, m1 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc +.w8: + lea r3, [strideq*3] + mov r5, acq +.w8_loop: + mova xm2, [ypxq+strideq*0] + vinserti128 m2, [ypxq+strideq*1], 1 + mova xm1, [ypxq+strideq*2] + vinserti128 m1, [ypxq+r3 ], 1 + lea ypxq, [ypxq+strideq*4] + psllw m2, 3 + psllw m1, 3 + mova [acq+32*0], m2 + pmaddwd m2, m5 + mova [acq+32*1], m1 + pmaddwd m0, m1, m5 + add acq, 32*2 + paddd m4, m2 + paddd m4, m0 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vperm2i128 m1, m1, 0x11 + pslld m0, 2 + pxor m2, m2 + vpblendd m0, m2, 0x0f + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w16_wpad2: + vpbroadcastw m3, [ypxq+strideq*0+14] + vpbroadcastw m0, [ypxq+strideq*1+14] + vpblendd m2, m3, 0xf0 + vpblendd m1, m0, 0xf0 + jmp .w16_wpad_end +.w16: + mov r5, acq +.w16_loop: + mova m2, [ypxq+strideq*0] + mova m1, [ypxq+strideq*1] + test wpadd, wpadd + jnz .w16_wpad2 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + psllw m2, 3 + psllw m1, 3 + mova [acq+32*0], m2 + pmaddwd m2, m5 + mova [acq+32*1], m1 + pmaddwd m0, m1, m5 + add acq, 32*2 + paddd m4, m2 + paddd m4, m0 + sub hd, 2 + jg .w16_loop + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + paddd m0, m0 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w32: + mov r5, acq + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + mova m0, [ypxq+ 0] + mova m1, [ypxq+32] + add ypxq, strideq + psllw m0, 3 + psllw m1, 3 + pmaddwd m2, m0, m5 + mova [acq+32*0], m0 + pmaddwd m3, m1, m5 + mova [acq+32*1], m1 + add acq, 32*2 + paddd m2, m3 + paddd m4, m2 + dec hd + jg .w32_loop +.w32_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + paddd m2, m2 +.w32_hpad_loop: + mova [acq+32*0], m0 + mova [acq+32*1], m1 + paddd m4, m2 + mova [acq+32*2], m0 + mova [acq+32*3], m1 + add acq, 32*4 + sub hpadd, 2 + jg .w32_hpad_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc +.w32_wpad: + mova m0, [ypxq+ 0] + cmp wpadd, 4 + jl .w32_wpad2 + je .w32_wpad4 + vpbroadcastw m1, [ypxq+14] + vpblendd m0, m1, 0xf0 + jmp .w32_wpad_end +.w32_wpad4: + vpbroadcastw m1, [ypxq+30] + jmp .w32_wpad_end +.w32_wpad2: + vpbroadcastw m1, [ypxq+46] + vinserti128 m1, [ypxq+32], 0 +.w32_wpad_end: + add ypxq, strideq + psllw m0, 3 + psllw m1, 3 + pmaddwd m2, m0, m5 + mova [acq+32*0], m0 + pmaddwd m3, m1, m5 + mova [acq+32*1], m1 + add acq, 32*2 + paddd m2, m3 + paddd m4, m2 + dec hd + jg .w32_wpad + jmp .w32_hpad + +cglobal pal_pred_16bpc, 4, 6, 5, dst, stride, pal, idx, w, h + vbroadcasti128 m3, [palq] + lea r2, [pal_pred_16bpc_avx2_table] + tzcnt wd, wm + vbroadcasti128 m4, [pal_pred_shuf] + movifnidn hd, hm + movsxd wq, [r2+wq*4] + pshufb m3, m4 + punpckhqdq m4, m3, m3 + add wq, r2 +DEFINE_ARGS dst, stride, stride3, idx, w, h + lea stride3q, [strideq*3] + jmp wq +.w4: + mova xm2, [idxq] + add idxq, 16 + pshufb xm1, xm3, xm2 + pshufb xm2, xm4, xm2 + punpcklbw xm0, xm1, xm2 + punpckhbw xm1, xm2 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+strideq*1], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + movu m2, [idxq] ; only 16-byte alignment + add idxq, 32 + pshufb m1, m3, m2 + pshufb m2, m4, m2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + vextracti128 [dstq+strideq*2], m0, 1 + vextracti128 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +.w16: + vpermq m2, [idxq+ 0], q3120 + vpermq m5, [idxq+32], q3120 + add idxq, 64 + pshufb m1, m3, m2 + pshufb m2, m4, m2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + pshufb m1, m3, m5 + pshufb m2, m4, m5 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +.w32: + vpermq m2, [idxq+ 0], q3120 + vpermq m5, [idxq+32], q3120 + add idxq, 64 + pshufb m1, m3, m2 + pshufb m2, m4, m2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*0+ 0], m0 + mova [dstq+strideq*0+32], m1 + pshufb m1, m3, m5 + pshufb m2, m4, m5 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*1+ 0], m0 + mova [dstq+strideq*1+32], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32 + RET +.w64: + vpermq m2, [idxq+ 0], q3120 + vpermq m5, [idxq+32], q3120 + add idxq, 64 + pshufb m1, m3, m2 + pshufb m2, m4, m2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+ 0], m0 + mova [dstq+32], m1 + pshufb m1, m3, m5 + pshufb m2, m4, m5 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+64], m0 + mova [dstq+96], m1 + add dstq, strideq + dec hd + jg .w64 + RET + +%endif diff --git a/third_party/dav1d/src/x86/ipred16_avx512.asm b/third_party/dav1d/src/x86/ipred16_avx512.asm new file mode 100644 index 0000000000..1a307adc98 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred16_avx512.asm @@ -0,0 +1,833 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +ipred_shuf: db 14, 15, 14, 15, 0, 1, 2, 3, 6, 7, 6, 7, 0, 1, 2, 3 + db 10, 11, 10, 11, 8, 9, 10, 11, 2, 3, 2, 3, 8, 9, 10, 11 + db 12, 13, 12, 13, 4, 5, 6, 7, 4, 5, 4, 5, 4, 5, 6, 7 + db 8, 9, 8, 9, 12, 13, 14, 15, 0, 1, 0, 1, 12, 13, 14, 15 +smooth_perm: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30 + db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62 + db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94 + db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126 +pal_pred_perm: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39 + db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47 + db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55 + db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63 +filter_permA: times 4 db 6, 7, 8, 9, 14, 15, 4, 5 + times 4 db 10, 11, 12, 13, 2, 3, -1, -1 +filter_permB: times 4 db 22, 23, 24, 25, 30, 31, 6, 7 + times 4 db 26, 27, 28, 29, 14, 15, -1, -1 +filter_permC: dd 8 ; dq 8, 10, 1, 11, 0, 9 +pw_1: times 2 dw 1 + dd 10 +filter_rnd: dd 32 + dd 1 + dd 8 + dd 11 +filter_shift: times 2 dw 6 + dd 0 + times 2 dw 4 + dd 9 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +JMP_TABLE ipred_paeth_16bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_16bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_h_16bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_v_16bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE pal_pred_16bpc, avx512icl, w4, w8, w16, w32, w64 + +cextern smooth_weights_1d_16bpc +cextern smooth_weights_2d_16bpc +cextern filter_intra_taps + +SECTION .text + +%macro PAETH 3 ; top, signed_ldiff, ldiff + paddw m0, m%2, m2 + psubw m1, m0, m3 ; tldiff + psubw m0, m%1 ; tdiff + pabsw m1, m1 + pabsw m0, m0 + pcmpgtw k1, m0, m1 + pminsw m0, m1 + pcmpgtw k2, m%3, m0 + vpblendmw m0{k1}, m%1, m3 + vpblendmw m0{k2}, m2, m0 +%endmacro + +INIT_ZMM avx512icl +cglobal ipred_paeth_16bpc, 3, 7, 10, dst, stride, tl, w, h +%define base r6-ipred_paeth_16bpc_avx512icl_table + lea r6, [ipred_paeth_16bpc_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastw m3, [tlq] ; topleft + add wq, r6 + jmp wq +.w4: + vpbroadcastq m4, [tlq+2] ; top + movsldup m7, [base+ipred_shuf] + lea r6, [strideq*3] + psubw m5, m4, m3 + pabsw m6, m5 +.w4_loop: + sub tlq, 16 + vbroadcasti32x4 m2, [tlq] + pshufb m2, m7 ; left + PAETH 4, 5, 6 + vextracti32x4 xm1, m0, 2 + vextracti32x4 xm8, ym0, 1 + vextracti32x4 xm9, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm8 + movq [dstq+r6 ], xm9 + sub hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm8 + movhps [dstq+r6 ], xm9 + lea dstq, [dstq+strideq*4] + jg .w4_loop +.w4_end: + RET +.w8: + vbroadcasti32x4 m4, [tlq+2] + movsldup m7, [base+ipred_shuf] + lea r6, [strideq*3] + psubw m5, m4, m3 + pabsw m6, m5 +.w8_loop: + sub tlq, 8 + vpbroadcastq m2, [tlq] + pshufb m2, m7 + PAETH 4, 5, 6 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+r6 ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +.w16: + vbroadcasti32x8 m4, [tlq+2] + movsldup m7, [base+ipred_shuf] + psubw m5, m4, m3 + pabsw m6, m5 +.w16_loop: + sub tlq, 4 + vpbroadcastd m2, [tlq] + pshufb m2, m7 + PAETH 4, 5, 6 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + RET +.w32: + movu m4, [tlq+2] + psubw m5, m4, m3 + pabsw m6, m5 +.w32_loop: + sub tlq, 2 + vpbroadcastw m2, [tlq] + PAETH 4, 5, 6 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w32_loop + RET +.w64: + movu m4, [tlq+ 2] + movu m7, [tlq+66] + psubw m5, m4, m3 + psubw m8, m7, m3 + pabsw m6, m5 + pabsw m9, m8 +.w64_loop: + sub tlq, 2 + vpbroadcastw m2, [tlq] + PAETH 4, 5, 6 + mova [dstq+64*0], m0 + PAETH 7, 8, 9 + mova [dstq+64*1], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_smooth_v_16bpc, 3, 7, 7, dst, stride, tl, w, h, weights, stride3 +%define base r6-$$ + lea r6, [$$] + tzcnt wd, wm + mov hd, hm + movsxd wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq*4] + lea weightsq, [base+smooth_weights_1d_16bpc+hq*4] + neg hq + vpbroadcastw m6, [tlq+hq*2] ; bottom + lea wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq] + lea stride3q, [strideq*3] + jmp wq +.w4: + vpbroadcastq m5, [tlq+2] ; top + movsldup m4, [ipred_shuf] + psubw m5, m6 ; top - bottom +.w4_loop: + vbroadcasti32x4 m3, [weightsq+hq*2] + pshufb m3, m4 + pmulhrsw m3, m5 + paddw m3, m6 + vextracti32x4 xm0, m3, 3 + vextracti32x4 xm1, ym3, 1 + vextracti32x4 xm2, m3, 2 + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + add hq, 8 + jg .end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + jl .w4_loop +.end: + RET +.w8: + vbroadcasti32x4 m5, [tlq+2] ; top + movsldup m4, [ipred_shuf] + psubw m5, m6 ; top - bottom +.w8_loop: + vpbroadcastq m0, [weightsq+hq*2] + pshufb m0, m4 + pmulhrsw m0, m5 + paddw m0, m6 + vextracti32x4 [dstq+strideq*0], m0, 3 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + mova [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w8_loop + RET +.w16: + vbroadcasti32x8 m5, [tlq+2] ; top + movsldup m4, [ipred_shuf] + psubw m5, m6 ; top - bottom +.w16_loop: + vpbroadcastd m0, [weightsq+hq*2+0] + vpbroadcastd m1, [weightsq+hq*2+4] + pshufb m0, m4 + pshufb m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + paddw m0, m6 + paddw m1, m6 + vextracti32x8 [dstq+strideq*0], m0, 1 + mova [dstq+strideq*1], ym0 + vextracti32x8 [dstq+strideq*2], m1, 1 + mova [dstq+stride3q ], ym1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w16_loop + RET +.w32: + movu m5, [tlq+2] + psubw m5, m6 +.w32_loop: + vpbroadcastw m0, [weightsq+hq*2+0] + vpbroadcastw m1, [weightsq+hq*2+2] + vpbroadcastw m2, [weightsq+hq*2+4] + vpbroadcastw m3, [weightsq+hq*2+6] + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + REPX {paddw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w32_loop + RET +.w64: + movu m4, [tlq+ 2] + movu m5, [tlq+66] + psubw m4, m6 + psubw m5, m6 +.w64_loop: + vpbroadcastw m1, [weightsq+hq*2+0] + vpbroadcastw m3, [weightsq+hq*2+2] + pmulhrsw m0, m4, m1 + pmulhrsw m1, m5 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m5 + REPX {paddw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*0+64*1], m1 + mova [dstq+strideq*1+64*0], m2 + mova [dstq+strideq*1+64*1], m3 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w64_loop + RET + +cglobal ipred_smooth_h_16bpc, 3, 7, 7, dst, stride, tl, w, h, stride3 + lea r6, [$$] + mov wd, wm + movifnidn hd, hm + vpbroadcastw m6, [tlq+wq*2] ; right + tzcnt wd, wd + add hd, hd + movsxd wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq*4] + sub tlq, hq + lea stride3q, [strideq*3] + lea wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq] + jmp wq +.w4: + movsldup m4, [base+ipred_shuf] + vpbroadcastq m5, [base+smooth_weights_1d_16bpc+4*2] +.w4_loop: + vbroadcasti32x4 m0, [tlq+hq-16] ; left + pshufb m0, m4 + psubw m0, m6 ; left - right + pmulhrsw m0, m5 + paddw m0, m6 + vextracti32x4 xm1, m0, 2 + vextracti32x4 xm2, ym0, 1 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + sub hd, 8*2 + jl .end + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + jg .w4_loop +.end: + RET +.w8: + movsldup m4, [base+ipred_shuf] + vbroadcasti32x4 m5, [base+smooth_weights_1d_16bpc+8*2] +.w8_loop: + vpbroadcastq m0, [tlq+hq-8] ; left + pshufb m0, m4 + psubw m0, m6 ; left - right + pmulhrsw m0, m5 + paddw m0, m6 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 + jg .w8_loop + RET +.w16: + movsldup m4, [base+ipred_shuf] + vbroadcasti32x8 m5, [base+smooth_weights_1d_16bpc+16*2] +.w16_loop: + vpbroadcastd m0, [tlq+hq-4] + vpbroadcastd m1, [tlq+hq-8] + pshufb m0, m4 + pshufb m1, m4 + psubw m0, m6 + psubw m1, m6 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + paddw m0, m6 + paddw m1, m6 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 + jg .w16_loop + RET +.w32: + movu m5, [base+smooth_weights_1d_16bpc+32*2] +.w32_loop: + vpbroadcastq m3, [tlq+hq-8] + punpcklwd m3, m3 + psubw m3, m6 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + REPX {paddw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 + jg .w32_loop + RET +.w64: + movu m4, [base+smooth_weights_1d_16bpc+64*2] + movu m5, [base+smooth_weights_1d_16bpc+64*3] +.w64_loop: + vpbroadcastw m1, [tlq+hq-2] + vpbroadcastw m3, [tlq+hq-4] + psubw m1, m6 + psubw m3, m6 + pmulhrsw m0, m4, m1 + pmulhrsw m1, m5 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m5 + REPX {paddw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*0+64*1], m1 + mova [dstq+strideq*1+64*0], m2 + mova [dstq+strideq*1+64*1], m3 + lea dstq, [dstq+strideq*2] + sub hq, 2*2 + jg .w64_loop + RET + +cglobal ipred_smooth_16bpc, 3, 7, 16, dst, stride, tl, w, h, v_weights, stride3 + lea r6, [$$] + mov wd, wm + movifnidn hd, hm + vpbroadcastw m13, [tlq+wq*2] ; right + tzcnt wd, wd + add hd, hd + movsxd wq, [base+ipred_smooth_16bpc_avx512icl_table+wq*4] + mov r5d, 0x55555555 + sub tlq, hq + mova m14, [base+smooth_perm] + kmovd k1, r5d + vpbroadcastw m0, [tlq] ; bottom + mov r5, 0x3333333333333333 + pxor m15, m15 + lea wq, [base+ipred_smooth_16bpc_avx512icl_table+wq] + kmovq k2, r5 + lea v_weightsq, [base+smooth_weights_2d_16bpc+hq*2] + jmp wq +.w4: + vpbroadcastq m5, [tlq+hq+2] + movshdup m3, [base+ipred_shuf] + movsldup m4, [base+ipred_shuf] + vbroadcasti32x4 m6, [base+smooth_weights_2d_16bpc+4*4] + lea stride3q, [strideq*3] + punpcklwd m5, m0 ; top, bottom +.w4_loop: + vbroadcasti32x4 m0, [v_weightsq] + vpbroadcastq m2, [tlq+hq-8] + mova m1, m13 + pshufb m0, m3 + pmaddwd m0, m5 + pshufb m1{k2}, m2, m4 ; left, right + vpdpwssd m0, m1, m6 + vpermb m0, m14, m0 + pavgw ym0, ym15 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + add v_weightsq, 4*4 + sub hd, 4*2 + jg .w4_loop + RET +.w8: + vbroadcasti32x4 ym5, [tlq+hq+2] + movshdup m6, [base+ipred_shuf] + movsldup m7, [base+ipred_shuf] + pmovzxwd m5, ym5 + vbroadcasti32x8 m8, [base+smooth_weights_2d_16bpc+8*4] + lea stride3q, [strideq*3] + vpblendmw m5{k1}, m0, m5 ; top, bottom +.w8_loop: + vpbroadcastq m0, [v_weightsq+0] + vpbroadcastq m1, [v_weightsq+8] + vpbroadcastd m3, [tlq+hq-4] + vpbroadcastd m4, [tlq+hq-8] + pshufb m0, m6 + pmaddwd m0, m5 + pshufb m1, m6 + pmaddwd m1, m5 + mova m2, m13 + pshufb m2{k2}, m3, m7 ; left, right + mova m3, m13 + pshufb m3{k2}, m4, m7 + vpdpwssd m0, m2, m8 + vpdpwssd m1, m3, m8 + add v_weightsq, 4*4 + vpermt2b m0, m14, m1 + pavgw m0, m15 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 + jg .w8_loop + RET +.w16: + pmovzxwd m5, [tlq+hq+2] + mova m6, [base+smooth_weights_2d_16bpc+16*4] + vpblendmw m5{k1}, m0, m5 ; top, bottom +.w16_loop: + vpbroadcastd m0, [v_weightsq+0] + vpbroadcastd m1, [v_weightsq+4] + pmaddwd m0, m5 + pmaddwd m1, m5 + mova m2, m13 + vpbroadcastw m2{k1}, [tlq+hq-2] ; left, right + mova m3, m13 + vpbroadcastw m3{k1}, [tlq+hq-4] + vpdpwssd m0, m2, m6 + vpdpwssd m1, m3, m6 + add v_weightsq, 2*4 + vpermt2b m0, m14, m1 + pavgw m0, m15 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hq, 2*2 + jg .w16_loop + RET +.w32: + pmovzxwd m5, [tlq+hq+ 2] + pmovzxwd m6, [tlq+hq+34] + mova m7, [base+smooth_weights_2d_16bpc+32*4] + mova m8, [base+smooth_weights_2d_16bpc+32*6] + vpblendmw m5{k1}, m0, m5 ; top, bottom + vpblendmw m6{k1}, m0, m6 +.w32_loop: + vpbroadcastd m2, [v_weightsq+0] + vpbroadcastd m3, [v_weightsq+4] + pmaddwd m0, m5, m2 + pmaddwd m2, m6 + pmaddwd m1, m5, m3 + pmaddwd m3, m6 + mova m4, m13 + vpbroadcastw m4{k1}, [tlq+hq-2] ; left, right + vpdpwssd m0, m4, m7 + vpdpwssd m2, m4, m8 + mova m4, m13 + vpbroadcastw m4{k1}, [tlq+hq-4] + vpdpwssd m1, m4, m7 + vpdpwssd m3, m4, m8 + add v_weightsq, 2*4 + vpermt2b m0, m14, m2 + vpermt2b m1, m14, m3 + pavgw m0, m15 + pavgw m1, m15 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hq, 2*2 + jg .w32_loop + RET +.w64: + pmovzxwd m5, [tlq+hq+ 2] + pmovzxwd m6, [tlq+hq+34] + pmovzxwd m7, [tlq+hq+66] + pmovzxwd m8, [tlq+hq+98] + mova m9, [base+smooth_weights_2d_16bpc+64*4] + vpblendmw m5{k1}, m0, m5 ; top, bottom + mova m10, [base+smooth_weights_2d_16bpc+64*5] + vpblendmw m6{k1}, m0, m6 + mova m11, [base+smooth_weights_2d_16bpc+64*6] + vpblendmw m7{k1}, m0, m7 + mova m12, [base+smooth_weights_2d_16bpc+64*7] + vpblendmw m8{k1}, m0, m8 +.w64_loop: + vpbroadcastd m3, [v_weightsq] + mova m4, m13 + vpbroadcastw m4{k1}, [tlq+hq-2] ; left, right + pmaddwd m0, m5, m3 + pmaddwd m2, m6, m3 + pmaddwd m1, m7, m3 + pmaddwd m3, m8 + vpdpwssd m0, m4, m9 + vpdpwssd m2, m4, m10 + vpdpwssd m1, m4, m11 + vpdpwssd m3, m4, m12 + add v_weightsq, 1*4 + vpermt2b m0, m14, m2 + vpermt2b m1, m14, m3 + pavgw m0, m15 + pavgw m1, m15 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, strideq + sub hd, 1*2 + jg .w64_loop + RET + +cglobal pal_pred_16bpc, 4, 7, 4, dst, stride, pal, idx, w, h, stride3 + lea r6, [pal_pred_16bpc_avx512icl_table] + tzcnt wd, wm + mova m2, [pal_pred_perm] + movsxd wq, [r6+wq*4] + mova xm3, [palq] + movifnidn hd, hm + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.w4: + pmovzxbw ym0, [idxq] + add idxq, 16 + vpermw ym0, ym0, ym3 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + pmovzxbw m0, [idxq] + add idxq, 32 + vpermw m0, m0, m3 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +.w16: + vpermb m1, m2, [idxq] + add idxq, 64 + vpermw m0, m1, m3 + psrlw m1, 8 + vpermw m1, m1, m3 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +.w32: + vpermb m1, m2, [idxq] + add idxq, 64 + vpermw m0, m1, m3 + psrlw m1, 8 + vpermw m1, m1, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32 + RET +.w64: + vpermb m1, m2, [idxq] + add idxq, 64 + vpermw m0, m1, m3 + psrlw m1, 8 + vpermw m1, m1, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, strideq + dec hd + jg .w64 + RET + +; The ipred_filter SIMD processes 4x2 blocks in the following order which +; increases parallelism compared to doing things row by row. +; w4 w8 w16 w32 +; 1 1 2 1 2 5 6 1 2 5 6 9 a d e +; 2 2 3 2 3 6 7 2 3 6 7 a b e f +; 3 3 4 3 4 7 8 3 4 7 8 b c f g +; 4 4 5 4 5 8 9 4 5 8 9 c d g h + +cglobal ipred_filter_16bpc, 4, 7, 14, dst, stride, tl, w, h, filter, top +%define base r6-$$ + lea r6, [$$] +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + shl filterd, 6 + movifnidn hd, hm + movu xm0, [tlq-6] + pmovsxbw m7, [base+filter_intra_taps+filterq+32*0] + pmovsxbw m8, [base+filter_intra_taps+filterq+32*1] + mov r5d, r8m ; bitdepth_max + movsldup m9, [base+filter_permA] + movshdup m10, [base+filter_permA] + shr r5d, 11 ; is_12bpc + jnz .12bpc + psllw m7, 2 ; upshift multipliers so that packusdw + psllw m8, 2 ; will perform clipping for free +.12bpc: + vpbroadcastd m5, [base+filter_rnd+r5*8] + vpbroadcastd m6, [base+filter_shift+r5*8] + sub wd, 8 + jl .w4 +.w8: + call .main4 + movsldup m11, [filter_permB] + lea r5d, [hq*2+2] + movshdup m12, [filter_permB] + lea topq, [tlq+2] + mova m13, [filter_permC] + sub hd, 4 + vinserti32x4 ym0, [topq], 1 ; a0 b0 t0 t1 + sub tlq, r5 +%if WIN64 + push r7 + push r8 +%endif + mov r7, dstq + mov r8d, hd +.w8_loop: + movlps xm4, xm0, [tlq+hq*2] + call .main8 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jge .w8_loop + test wd, wd + jz .end + mov r2d, 0x0d + kmovb k1, r2d + lea r2, [strideq*3] +.w16: + movd xmm0, [r7+strideq*1+12] + vpblendd xmm0, [topq+8], 0x0e ; t1 t2 + pinsrw xm4, xmm0, [r7+strideq*0+14], 2 + call .main8 + add r7, 16 + vinserti32x4 ym0, [topq+16], 1 ; a2 b2 t2 t3 + mov hd, r8d + mov dstq, r7 + add topq, 16 +.w16_loop: + movd xmm1, [dstq+strideq*2-4] + punpcklwd xm4, xmm1, xmm0 + movd xmm0, [dstq+r2-4] + shufps xm4{k1}, xmm0, xm0, q3210 + call .main8 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jge .w16_loop + sub wd, 8 + jg .w16 +.end: + vpermb m2, m11, m0 + mova ym1, ym5 + vpdpwssd m1, m2, m7 + vpermb m2, m12, m0 + vpdpwssd m1, m2, m8 +%if WIN64 + pop r8 + pop r7 +%endif + vextracti32x8 ym2, m1, 1 + paddd ym1, ym2 + packusdw ym1, ym1 + vpsrlvw ym1, ym6 + vpermt2q m0, m13, m1 + vextracti32x4 [dstq+strideq*0], m0, 2 + vextracti32x4 [dstq+strideq*1], ym0, 1 + RET +.w4_loop: + movlps xm0, [tlq-10] + lea dstq, [dstq+strideq*2] + sub tlq, 4 +.w4: + call .main4 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + sub hd, 2 + jg .w4_loop + RET +ALIGN function_align +.main4: + vpermb m2, m9, m0 + mova ym1, ym5 + vpdpwssd m1, m2, m7 + vpermb m0, m10, m0 + vpdpwssd m1, m0, m8 + vextracti32x8 ym0, m1, 1 + paddd ym0, ym1 + vextracti32x4 xm1, ym0, 1 + packusdw xm0, xm1 ; clip + vpsrlvw xm0, xm6 + ret +ALIGN function_align +.main8: + vpermb m3, m11, m0 + mova ym2, ym5 + vpdpwssd m2, m3, m7 + vpermb m3, m9, m4 + mova ym1, ym5 + vpdpwssd m1, m3, m7 + vpermb m3, m12, m0 + vpdpwssd m2, m3, m8 + vpermb m3, m10, m4 + vpdpwssd m1, m3, m8 + vextracti32x8 ym4, m2, 1 + vextracti32x8 ym3, m1, 1 + paddd ym2, ym4 + paddd ym1, ym3 + packusdw ym1, ym2 ; clip + vpsrlvw ym1, ym6 + vpermt2q m0, m13, m1 ; c0 d0 b0 b1 a0 a1 + vextracti32x4 [dstq+strideq*0], m0, 2 + vextracti32x4 [dstq+strideq*1], ym0, 1 + ret + +%endif diff --git a/third_party/dav1d/src/x86/ipred16_sse.asm b/third_party/dav1d/src/x86/ipred16_sse.asm new file mode 100644 index 0000000000..07ea9567e1 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred16_sse.asm @@ -0,0 +1,1923 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +filter_shuf: db 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 2, 3, -1, -1 +pal_pred_shuf: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 + +pb_0_1: times 4 db 0, 1 +pb_2_3: times 4 db 2, 3 +pw_1: times 4 dw 1 +pw_2: times 4 dw 2 +pw_4: times 4 dw 4 +pw_512: times 4 dw 512 +pw_2048: times 4 dw 2048 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_16bpc_ssse3_table (ipred_dc_16bpc_ssse3_table + 10*4) +%define ipred_dc_128_16bpc_ssse3_table (ipred_dc_16bpc_ssse3_table + 15*4) +%define ipred_cfl_splat_16bpc_ssse3_table (ipred_cfl_16bpc_ssse3_table + 8*4) + +JMP_TABLE ipred_dc_left_16bpc, ssse3, h4, h8, h16, h32, h64 +JMP_TABLE ipred_dc_16bpc, ssse3, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4, \ + s4-15*4, s8-15*4, s16c-15*4, s32c-15*4, s64-15*4 +JMP_TABLE ipred_h_16bpc, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_cfl_16bpc, ssse3, h4, h8, h16, h32, w4, w8, w16, w32, \ + s4-8*4, s8-8*4, s16-8*4, s32-8*4 +JMP_TABLE ipred_cfl_left_16bpc, ssse3, h4, h8, h16, h32 +JMP_TABLE ipred_cfl_ac_444_16bpc, ssse3, w4, w8, w16, w32 +JMP_TABLE pal_pred_16bpc, ssse3, w4, w8, w16, w32, w64 + +cextern smooth_weights_1d_16bpc +cextern smooth_weights_2d_16bpc +cextern filter_intra_taps + +SECTION .text + +INIT_XMM ssse3 +cglobal ipred_dc_top_16bpc, 3, 7, 6, dst, stride, tl, w, h + LEA r5, ipred_dc_left_16bpc_ssse3_table + movd m4, wm + tzcnt wd, wm + add tlq, 2 + movifnidn hd, hm + pxor m3, m3 + pavgw m4, m3 + movd m5, wd + movu m0, [tlq] + movsxd r6, [r5+wq*4] + add r6, r5 + add r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +cglobal ipred_dc_left_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_left_16bpc_ssse3_table + mov hd, hm + movd m4, hm + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + pxor m3, m3 + sub tlq, hq + pavgw m4, m3 + movd m5, r6d + movu m0, [tlq] + movsxd r6, [r5+r6*4] + add r6, r5 + add r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + movu m2, [tlq+112] + movu m1, [tlq+ 96] + paddw m0, m2 + movu m2, [tlq+ 80] + paddw m1, m2 + movu m2, [tlq+ 64] + paddw m0, m2 + paddw m0, m1 +.h32: + movu m1, [tlq+ 48] + movu m2, [tlq+ 32] + paddw m1, m2 + paddw m0, m1 +.h16: + movu m1, [tlq+ 16] + paddw m0, m1 +.h8: + movhlps m1, m0 + paddw m0, m1 +.h4: + punpcklwd m0, m3 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + lea stride3q, [strideq*3] + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + jmp wq + +cglobal ipred_dc_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd m4, r5d + tzcnt r5d, r5d + movd m5, r5d + LEA r5, ipred_dc_16bpc_ssse3_table + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+5*4] + pxor m3, m3 + psrlw m4, 1 + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movq m0, [tlq-8] + jmp wq +.w4: + movq m1, [tlq+2] + paddw m1, m0 + punpckhwd m0, m3 + punpcklwd m1, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + cmp hd, 4 + jg .w4_mul + psrlw m0, 3 + jmp .w4_end +.w4_mul: + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 16 + cmove r2d, r3d + psrld m0, 2 + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w4_end: + pshuflw m0, m0, q0000 +.s4: + movq [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +.h8: + mova m0, [tlq-16] + jmp wq +.w8: + movu m1, [tlq+2] + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 8 + je .w8_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 32 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w8_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s8: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +.h16: + mova m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w16: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + paddw m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 16 + je .w16_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + test hd, 8|32 + cmovz r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w16_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s16c: + mova m1, m0 +.s16: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*2+16*0], m0 + mova [dstq+strideq*2+16*1], m1 + mova [dstq+stride3q +16*0], m0 + mova [dstq+stride3q +16*1], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-48] + paddw m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w32: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + movu m2, [tlq+34] + paddw m0, m2 + movu m2, [tlq+50] + paddw m1, m2 + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 32 + je .w32_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 8 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w32_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s32c: + mova m1, m0 + mova m2, m0 + mova m3, m0 +.s32: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + mova [dstq+strideq*0+16*2], m2 + mova [dstq+strideq*0+16*3], m3 + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*1+16*2], m2 + mova [dstq+strideq*1+16*3], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s32 + RET +.h64: + mova m0, [tlq-128] + mova m1, [tlq-112] + paddw m0, [tlq- 96] + paddw m1, [tlq- 80] + paddw m0, [tlq- 64] + paddw m1, [tlq- 48] + paddw m0, [tlq- 32] + paddw m1, [tlq- 16] + paddw m0, m1 + jmp wq +.w64: + movu m1, [tlq+ 2] + movu m2, [tlq+ 18] + paddw m1, m2 + movu m2, [tlq+ 34] + paddw m0, m2 + movu m2, [tlq+ 50] + paddw m1, m2 + movu m2, [tlq+ 66] + paddw m0, m2 + movu m2, [tlq+ 82] + paddw m1, m2 + movu m2, [tlq+ 98] + paddw m0, m2 + movu m2, [tlq+114] + paddw m1, m2 + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 64 + je .w64_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 16 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w64_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s64: + mova [dstq+16*0], m0 + mova [dstq+16*1], m0 + mova [dstq+16*2], m0 + mova [dstq+16*3], m0 + mova [dstq+16*4], m0 + mova [dstq+16*5], m0 + mova [dstq+16*6], m0 + mova [dstq+16*7], m0 + add dstq, strideq + dec hd + jg .s64 + RET + +cglobal ipred_dc_128_16bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 + mov r6d, r8m + LEA r5, ipred_dc_128_16bpc_ssse3_table + tzcnt wd, wm + shr r6d, 11 + movifnidn hd, hm + movsxd wq, [r5+wq*4] + movddup m0, [r5-ipred_dc_128_16bpc_ssse3_table+pw_512+r6*8] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_v_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_splat_16bpc_ssse3_table + movifnidn hd, hm + movu m0, [tlq+ 2] + movu m1, [tlq+ 18] + movu m2, [tlq+ 34] + movu m3, [tlq+ 50] + cmp wd, 64 + je .w64 + tzcnt wd, wd + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w64: + WIN64_SPILL_XMM 8 + movu m4, [tlq+ 66] + movu m5, [tlq+ 82] + movu m6, [tlq+ 98] + movu m7, [tlq+114] +.w64_loop: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + mova [dstq+16*4], m4 + mova [dstq+16*5], m5 + mova [dstq+16*6], m6 + mova [dstq+16*7], m7 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_h_16bpc, 3, 6, 4, dst, stride, tl, w, h, stride3 +%define base r5-ipred_h_16bpc_ssse3_table + tzcnt wd, wm + LEA r5, ipred_h_16bpc_ssse3_table + movifnidn hd, hm + movsxd wq, [r5+wq*4] + movddup m2, [base+pb_0_1] + movddup m3, [base+pb_2_3] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w4: + sub tlq, 8 + movq m3, [tlq] + pshuflw m0, m3, q3333 + pshuflw m1, m3, q2222 + pshuflw m2, m3, q1111 + pshuflw m3, m3, q0000 + movq [dstq+strideq*0], m0 + movq [dstq+strideq*1], m1 + movq [dstq+strideq*2], m2 + movq [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + sub tlq, 8 + movq m3, [tlq] + punpcklwd m3, m3 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +.w16: + sub tlq, 4 + movd m1, [tlq] + pshufb m0, m1, m3 + pshufb m1, m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m0 + mova [dstq+strideq*1+16*0], m1 + mova [dstq+strideq*1+16*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16 + RET +.w32: + sub tlq, 4 + movd m1, [tlq] + pshufb m0, m1, m3 + pshufb m1, m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m0 + mova [dstq+strideq*0+16*2], m0 + mova [dstq+strideq*0+16*3], m0 + mova [dstq+strideq*1+16*0], m1 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*1+16*2], m1 + mova [dstq+strideq*1+16*3], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32 + RET +.w64: + sub tlq, 2 + movd m0, [tlq] + pshufb m0, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m0 + mova [dstq+16*2], m0 + mova [dstq+16*3], m0 + mova [dstq+16*4], m0 + mova [dstq+16*5], m0 + mova [dstq+16*6], m0 + mova [dstq+16*7], m0 + add dstq, strideq + dec hd + jg .w64 + RET + +cglobal ipred_paeth_16bpc, 4, 6, 8, dst, stride, tl, w, h, left +%define base r5-ipred_paeth_16bpc_ssse3_table + movifnidn hd, hm + pshuflw m4, [tlq], q0000 + mov leftq, tlq + add hd, hd + punpcklqdq m4, m4 ; topleft + sub leftq, hq + and wd, ~7 + jnz .w8 + movddup m5, [tlq+2] ; top + psubw m6, m5, m4 + pabsw m7, m6 +.w4_loop: + movd m1, [leftq+hq-4] + punpcklwd m1, m1 + punpckldq m1, m1 ; left +%macro PAETH 0 + paddw m0, m6, m1 + psubw m2, m4, m0 ; tldiff + psubw m0, m5 ; tdiff + pabsw m2, m2 + pabsw m0, m0 + pminsw m2, m0 + pcmpeqw m0, m2 + pand m3, m5, m0 + pandn m0, m4 + por m0, m3 + pcmpgtw m3, m7, m2 + pand m0, m3 + pandn m3, m1 + por m0, m3 +%endmacro + PAETH + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2*2 + jg .w4_loop + RET +.w8: +%if ARCH_X86_32 + PUSH r6 + %define r7d hm + %assign regs_used 7 +%elif WIN64 + movaps r4m, m8 + PUSH r7 + %assign regs_used 8 +%endif +%if ARCH_X86_64 + movddup m8, [pb_0_1] +%endif + lea tlq, [tlq+wq*2+2] + neg wq + mov r7d, hd +.w8_loop0: + movu m5, [tlq+wq*2] + mov r6, dstq + add dstq, 16 + psubw m6, m5, m4 + pabsw m7, m6 +.w8_loop: + movd m1, [leftq+hq-2] +%if ARCH_X86_64 + pshufb m1, m8 +%else + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 +%endif + PAETH + mova [r6], m0 + add r6, strideq + sub hd, 1*2 + jg .w8_loop + mov hd, r7d + add wq, 8 + jl .w8_loop0 +%if WIN64 + movaps m8, r4m +%endif + RET + +%if ARCH_X86_64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 4 +%endif + +cglobal ipred_smooth_v_16bpc, 4, 6, 6, dst, stride, tl, w, h, weights + LEA weightsq, smooth_weights_1d_16bpc + mov hd, hm + lea weightsq, [weightsq+hq*4] + neg hq + movd m5, [tlq+hq*2] ; bottom + pshuflw m5, m5, q0000 + punpcklqdq m5, m5 + cmp wd, 4 + jne .w8 + movddup m4, [tlq+2] ; top + lea r3, [strideq*3] + psubw m4, m5 ; top - bottom +.w4_loop: + movq m1, [weightsq+hq*2] + punpcklwd m1, m1 + pshufd m0, m1, q1100 + punpckhdq m1, m1 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r3 ], m1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w4_loop + RET +.w8: +%if ARCH_X86_32 + PUSH r6 + %assign regs_used 7 + mov hm, hq + %define hq hm +%elif WIN64 + PUSH r7 + %assign regs_used 8 +%endif +.w8_loop0: + mov t0, hq + movu m4, [tlq+2] + add tlq, 16 + mov r6, dstq + add dstq, 16 + psubw m4, m5 +.w8_loop: + movq m3, [weightsq+t0*2] + punpcklwd m3, m3 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [r6+strideq*0], m0 + mova [r6+strideq*1], m1 + lea r6, [r6+strideq*2] + mova [r6+strideq*0], m2 + mova [r6+strideq*1], m3 + lea r6, [r6+strideq*2] + add t0, 4 + jl .w8_loop + sub wd, 8 + jg .w8_loop0 + RET + +cglobal ipred_smooth_h_16bpc, 3, 6, 6, dst, stride, tl, w, h, weights + LEA weightsq, smooth_weights_1d_16bpc + mov wd, wm + movifnidn hd, hm + movd m5, [tlq+wq*2] ; right + sub tlq, 8 + add hd, hd + pshuflw m5, m5, q0000 + sub tlq, hq + punpcklqdq m5, m5 + cmp wd, 4 + jne .w8 + movddup m4, [weightsq+4*2] + lea r3, [strideq*3] +.w4_loop: + movq m1, [tlq+hq] ; left + punpcklwd m1, m1 + psubw m1, m5 ; left - right + pshufd m0, m1, q3322 + punpckldq m1, m1 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + movhps [dstq+strideq*2], m1 + movq [dstq+r3 ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 + jg .w4_loop + RET +.w8: + lea weightsq, [weightsq+wq*4] + neg wq +%if ARCH_X86_32 + PUSH r6 + %assign regs_used 7 + %define hd hm +%elif WIN64 + PUSH r7 + %assign regs_used 8 +%endif +.w8_loop0: + mov t0d, hd + mova m4, [weightsq+wq*2] + mov r6, dstq + add dstq, 16 +.w8_loop: + movq m3, [tlq+t0*(1+ARCH_X86_32)] + punpcklwd m3, m3 + psubw m3, m5 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [r6+strideq*0], m0 + mova [r6+strideq*1], m1 + lea r6, [r6+strideq*2] + mova [r6+strideq*0], m2 + mova [r6+strideq*1], m3 + lea r6, [r6+strideq*2] + sub t0d, 4*(1+ARCH_X86_64) + jg .w8_loop + add wq, 8 + jl .w8_loop0 + RET + +%if ARCH_X86_64 +DECLARE_REG_TMP 10 +%else +DECLARE_REG_TMP 3 +%endif + +cglobal ipred_smooth_16bpc, 3, 7, 8, dst, stride, tl, w, h, \ + h_weights, v_weights, top + LEA h_weightsq, smooth_weights_2d_16bpc + mov wd, wm + mov hd, hm + movd m7, [tlq+wq*2] ; right + lea v_weightsq, [h_weightsq+hq*8] + neg hq + movd m6, [tlq+hq*2] ; bottom + pshuflw m7, m7, q0000 + pshuflw m6, m6, q0000 + cmp wd, 4 + jne .w8 + movq m4, [tlq+2] ; top + mova m5, [h_weightsq+4*4] + punpcklwd m4, m6 ; top, bottom + pxor m6, m6 +.w4_loop: + movq m1, [v_weightsq+hq*4] + sub tlq, 4 + movd m3, [tlq] ; left + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + pmaddwd m0, m4 + punpcklwd m3, m7 ; left, right + pmaddwd m1, m4 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + pmaddwd m2, m5 + pmaddwd m3, m5 + paddd m0, m2 + paddd m1, m3 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pavgw m0, m6 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w4_loop + RET +.w8: +%if ARCH_X86_32 + lea h_weightsq, [h_weightsq+wq*4] + mov t0, tlq + mov r1m, tlq + mov r2m, hq + %define m8 [h_weightsq+16*0] + %define m9 [h_weightsq+16*1] +%else +%if WIN64 + movaps r4m, m8 + movaps r6m, m9 + PUSH r7 + PUSH r8 +%endif + PUSH r9 + PUSH r10 + %assign regs_used 11 + lea h_weightsq, [h_weightsq+wq*8] + lea topq, [tlq+wq*2] + neg wq + mov r8, tlq + mov r9, hq +%endif + punpcklqdq m6, m6 +.w8_loop0: +%if ARCH_X86_32 + movu m5, [t0+2] + add t0, 16 + mov r0m, t0 +%else + movu m5, [topq+wq*2+2] + mova m8, [h_weightsq+wq*4+16*0] + mova m9, [h_weightsq+wq*4+16*1] +%endif + mov t0, dstq + add dstq, 16 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 +.w8_loop: + movd m1, [v_weightsq+hq*4] + sub tlq, 2 + movd m3, [tlq] ; left + pshufd m1, m1, q0000 + pmaddwd m0, m4, m1 + pshuflw m3, m3, q0000 + pmaddwd m1, m5 + punpcklwd m3, m7 ; left, right + pmaddwd m2, m8, m3 + pmaddwd m3, m9 + paddd m0, m2 + paddd m1, m3 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pxor m1, m1 + pavgw m0, m1 + mova [t0], m0 + add t0, strideq + inc hq + jl .w8_loop +%if ARCH_X86_32 + mov t0, r0m + mov tlq, r1m + add h_weightsq, 16*2 + mov hq, r2m + sub dword wm, 8 + jg .w8_loop0 +%else + mov tlq, r8 + mov hq, r9 + add wq, 8 + jl .w8_loop0 +%endif +%if WIN64 + movaps m8, r4m + movaps m9, r6m +%endif + RET + +%if ARCH_X86_64 +cglobal ipred_filter_16bpc, 4, 7, 16, dst, stride, tl, w, h, filter +%else +cglobal ipred_filter_16bpc, 4, 7, 8, -16*8, dst, stride, tl, w, h, filter +%define m8 [esp+16*0] +%define m9 [esp+16*1] +%define m10 [esp+16*2] +%define m11 [esp+16*3] +%define m12 [esp+16*4] +%define m13 [esp+16*5] +%define m14 [esp+16*6] +%define m15 [esp+16*7] +%endif +%define base r6-$$ + movifnidn hd, hm + movd m6, r8m ; bitdepth_max +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + LEA r6, $$ + shl filterd, 6 + movu m0, [tlq-6] ; __ l1 l0 tl t0 t1 t2 t3 + mova m1, [base+filter_intra_taps+filterq+16*0] + mova m2, [base+filter_intra_taps+filterq+16*1] + mova m3, [base+filter_intra_taps+filterq+16*2] + mova m4, [base+filter_intra_taps+filterq+16*3] + pxor m5, m5 +%if ARCH_X86_64 + punpcklbw m8, m5, m1 ; place 8-bit coefficients in the upper + punpckhbw m9, m5, m1 ; half of each 16-bit word to avoid + punpcklbw m10, m5, m2 ; having to perform sign-extension. + punpckhbw m11, m5, m2 + punpcklbw m12, m5, m3 + punpckhbw m13, m5, m3 + punpcklbw m14, m5, m4 + punpckhbw m15, m5, m4 +%else + punpcklbw m7, m5, m1 + mova m8, m7 + punpckhbw m7, m5, m1 + mova m9, m7 + punpcklbw m7, m5, m2 + mova m10, m7 + punpckhbw m7, m5, m2 + mova m11, m7 + punpcklbw m7, m5, m3 + mova m12, m7 + punpckhbw m7, m5, m3 + mova m13, m7 + punpcklbw m7, m5, m4 + mova m14, m7 + punpckhbw m7, m5, m4 + mova m15, m7 +%endif + mova m7, [base+filter_shuf] + add hd, hd + mov r5, dstq + pshuflw m6, m6, q0000 + mov r6, tlq + punpcklqdq m6, m6 + sub tlq, hq +.left_loop: + pshufb m0, m7 ; tl t0 t1 t2 t3 l0 l1 __ + pshufd m1, m0, q0000 + pmaddwd m2, m8, m1 + pmaddwd m1, m9 + pshufd m4, m0, q1111 + pmaddwd m3, m10, m4 + pmaddwd m4, m11 + paddd m2, m3 + paddd m1, m4 + pshufd m4, m0, q2222 + pmaddwd m3, m12, m4 + pmaddwd m4, m13 + paddd m2, m3 + paddd m1, m4 + pshufd m3, m0, q3333 + pmaddwd m0, m14, m3 + pmaddwd m3, m15 + paddd m0, m2 + paddd m1, m3 + psrad m0, 11 ; x >> 3 + psrad m1, 11 + packssdw m0, m1 + pmaxsw m0, m5 + pavgw m0, m5 ; (x + 8) >> 4 + pminsw m0, m6 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movlps m0, [tlq+hq-10] + lea dstq, [dstq+strideq*2] + sub hd, 2*2 + jg .left_loop + sub wd, 4 + jz .end + sub tld, r6d ; -h*2 + sub r6, r5 ; tl-dst +.right_loop0: + add r5, 8 + mov hd, tld + movu m0, [r5+r6] ; tl t0 t1 t2 t3 __ __ __ + mov dstq, r5 +.right_loop: + pshufd m2, m0, q0000 + pmaddwd m1, m8, m2 + pmaddwd m2, m9 + pshufd m4, m0, q1111 + pmaddwd m3, m10, m4 + pmaddwd m4, m11 + pinsrw m0, [dstq+strideq*0-2], 5 + paddd m1, m3 + paddd m2, m4 + pshufd m0, m0, q2222 + movddup m4, [dstq+strideq*1-8] + pmaddwd m3, m12, m0 + pmaddwd m0, m13 + paddd m1, m3 + paddd m0, m2 + pshuflw m2, m4, q3333 + punpcklwd m2, m5 + pmaddwd m3, m14, m2 + pmaddwd m2, m15 + paddd m1, m3 + paddd m0, m2 + psrad m1, 11 + psrad m0, 11 + packssdw m0, m1 + pmaxsw m0, m5 + pavgw m0, m5 + pminsw m0, m6 + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + palignr m0, m4, 14 + lea dstq, [dstq+strideq*2] + add hd, 2*2 + jl .right_loop + sub wd, 4 + jg .right_loop0 +.end: + RET + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal ipred_cfl_top_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac + LEA t0, ipred_cfl_left_16bpc_ssse3_table + movd m4, wd + tzcnt wd, wd + movifnidn hd, hm + add tlq, 2 + movsxd r6, [t0+wq*4] + movd m5, wd + jmp mangle(private_prefix %+ _ipred_cfl_left_16bpc_ssse3.start) + +cglobal ipred_cfl_left_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + LEA t0, ipred_cfl_left_16bpc_ssse3_table + tzcnt wd, wm + lea r6d, [hq*2] + movd m4, hd + sub tlq, r6 + tzcnt r6d, hd + movd m5, r6d + movsxd r6, [t0+r6*4] +.start: + movd m7, r7m + movu m0, [tlq] + add r6, t0 + add t0, ipred_cfl_splat_16bpc_ssse3_table-ipred_cfl_left_16bpc_ssse3_table + movsxd wq, [t0+wq*4] + pxor m6, m6 + pshuflw m7, m7, q0000 + pcmpeqw m3, m3 + add wq, t0 + movifnidn acq, acmp + pavgw m4, m6 + punpcklqdq m7, m7 + jmp r6 +.h32: + movu m1, [tlq+48] + movu m2, [tlq+32] + paddw m0, m1 + paddw m0, m2 +.h16: + movu m1, [tlq+16] + paddw m0, m1 +.h8: + pshufd m1, m0, q1032 + paddw m0, m1 +.h4: + pmaddwd m0, m3 + psubd m4, m0 + pshuflw m0, m4, q1032 + paddd m0, m4 + psrld m0, m5 + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + jmp wq + +%macro IPRED_CFL 2 ; dst, src + pabsw m%1, m%2 + pmulhrsw m%1, m2 + psignw m%2, m1 + psignw m%1, m%2 + paddw m%1, m0 + pmaxsw m%1, m6 + pminsw m%1, m7 +%endmacro + +cglobal ipred_cfl_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + tzcnt r6d, hd + lea t0d, [wq+hq] + movd m4, t0d + tzcnt t0d, t0d + movd m5, t0d + LEA t0, ipred_cfl_16bpc_ssse3_table + tzcnt wd, wd + movd m7, r7m + movsxd r6, [t0+r6*4] + movsxd wq, [t0+wq*4+4*4] + psrlw m4, 1 + pxor m6, m6 + pshuflw m7, m7, q0000 + add r6, t0 + add wq, t0 + movifnidn acq, acmp + pcmpeqw m3, m3 + punpcklqdq m7, m7 + jmp r6 +.h4: + movq m0, [tlq-8] + jmp wq +.w4: + movq m1, [tlq+2] + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + cmp hd, 4 + jg .w4_mul + psrld m0, 3 + jmp .w4_end +.w4_mul: + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 16 + cmove r6d, r2d + movd m1, r6d + psrld m0, 2 + pmulhuw m0, m1 + psrlw m0, 1 +.w4_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s4: + movd m1, alpham + lea r6, [strideq*3] + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s4_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + movq [dstq+strideq*0], m3 + movhps [dstq+strideq*1], m3 + movq [dstq+strideq*2], m4 + movhps [dstq+r6 ], m4 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4_loop + RET +.h8: + mova m0, [tlq-16] + jmp wq +.w8: + movu m1, [tlq+2] + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 8 + je .w8_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 32 + cmove r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w8_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s8: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s8_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+strideq*0], m3 + mova [dstq+strideq*1], m4 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s8_loop + RET +.h16: + mova m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w16: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 16 + je .w16_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + test hd, 8|32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w16_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s16: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s16_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*0], m3 + mova [dstq+16*1], m4 + add dstq, strideq + dec hd + jg .s16_loop + RET +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-48] + paddw m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w32: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + movu m2, [tlq+34] + paddw m1, m2 + movu m2, [tlq+50] + paddw m1, m2 + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 32 + je .w32_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 8 + cmove r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w32_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s32: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s32_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*0], m3 + mova [dstq+16*1], m4 + mova m4, [acq+16*2] + mova m5, [acq+16*3] + add acq, 16*4 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, strideq + dec hd + jg .s32_loop + RET + +cglobal ipred_cfl_128_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac + tzcnt wd, wm + LEA t0, ipred_cfl_splat_16bpc_ssse3_table + mov r6d, r7m + movifnidn hd, hm + shr r6d, 11 + movd m7, r7m + movsxd wq, [t0+wq*4] + movddup m0, [t0-ipred_cfl_splat_16bpc_ssse3_table+pw_512+r6*8] + pshuflw m7, m7, q0000 + pxor m6, m6 + add wq, t0 + movifnidn acq, acmp + punpcklqdq m7, m7 + jmp wq + +cglobal ipred_cfl_ac_420_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm +%if ARCH_X86_32 && PIC + pcmpeqw m5, m5 + pabsw m5, m5 + paddw m5, m5 +%else + movddup m5, [pw_2] +%endif + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + mov r5, acq + jg .w16 + je .w8 + lea r3, [strideq*3] +.w4_loop: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + pmaddwd m2, m5, [ypxq+strideq*2] + pmaddwd m3, m5, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + paddd m0, m1 + paddd m2, m3 + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + sub hd, 2 + jg .w4_loop + test hpadd, hpadd + jz .dc + punpckhqdq m0, m0 + pslld m2, 2 +.w4_hpad: + mova [acq+16*0], m0 + paddd m4, m2 + mova [acq+16*1], m0 + add acq, 16*2 + sub hpadd, 4 + jg .w4_hpad + jmp .dc +.w8: +%if ARCH_X86_32 + cmp dword wpadm, 0 +%else + test wpadd, wpadd +%endif + jnz .w8_wpad1 +.w8_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m2, m5, [ypxq+strideq*1+16*0] + pmaddwd m1, m5, [ypxq+strideq*0+16*1] + pmaddwd m3, m5, [ypxq+strideq*1+16*1] + lea ypxq, [ypxq+strideq*2] + paddd m0, m2 + paddd m1, m3 + paddd m2, m0, m1 + packssdw m0, m1 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + dec hd + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz .dc + pslld m2, 2 + mova m1, m0 + jmp .hpad +.w8_wpad1: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd m0, m1 + pshufd m1, m0, q3333 + paddd m2, m0, m1 + packssdw m0, m1 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + dec hd + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad3: + pshufd m3, m0, q3333 + mova m1, m3 + mova m2, m3 + jmp .w16_wpad_end +.w16_wpad2: + pshufd m1, m3, q3333 + mova m2, m1 + jmp .w16_wpad_end +.w16_wpad1: + pshufd m2, m1, q3333 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + WIN64_SPILL_XMM 7 +.w16_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m6, m5, [ypxq+strideq*1+16*0] + paddd m0, m6 + cmp wpadd, 2 + jg .w16_wpad3 + pmaddwd m3, m5, [ypxq+strideq*0+16*1] + pmaddwd m6, m5, [ypxq+strideq*1+16*1] + paddd m3, m6 + je .w16_wpad2 + pmaddwd m1, m5, [ypxq+strideq*0+16*2] + pmaddwd m6, m5, [ypxq+strideq*1+16*2] + paddd m1, m6 + jp .w16_wpad1 + pmaddwd m2, m5, [ypxq+strideq*0+16*3] + pmaddwd m6, m5, [ypxq+strideq*1+16*3] + paddd m2, m6 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + paddd m6, m0, m3 + packssdw m0, m3 + paddd m6, m1 + mova [acq+16*0], m0 + packssdw m1, m2 + paddd m2, m6 + mova [acq+16*1], m1 + add acq, 16*2 + paddd m4, m2 + dec hd + jg .w16_loop + WIN64_RESTORE_XMM + add hpadd, hpadd + jz .dc + paddd m2, m2 +.hpad: + mova [acq+16*0], m0 + mova [acq+16*1], m1 + paddd m4, m2 + mova [acq+16*2], m0 + mova [acq+16*3], m1 + add acq, 16*4 + sub hpadd, 4 + jg .hpad +.dc: + sub r5, acq ; -w*h*2 + pshufd m2, m4, q1032 + tzcnt r1d, r5d + paddd m2, m4 + sub r1d, 2 + pshufd m4, m2, q2301 + movd m0, r1d + paddd m2, m4 + psrld m2, m0 + pxor m0, m0 + pavgw m2, m0 + packssdw m2, m2 +.dc_loop: + mova m0, [acq+r5+16*0] + mova m1, [acq+r5+16*1] + psubw m0, m2 + psubw m1, m2 + mova [acq+r5+16*0], m0 + mova [acq+r5+16*1], m1 + add r5, 16*2 + jl .dc_loop + RET + +cglobal ipred_cfl_ac_422_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm +%if ARCH_X86_32 && PIC + pcmpeqw m5, m5 + pabsw m5, m5 + psllw m5, 2 +%else + movddup m5, [pw_4] +%endif + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + mov r5, acq + jg .w16 + je .w8 + lea r3, [strideq*3] +.w4_loop: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m3, m5, [ypxq+strideq*1] + pmaddwd m1, m5, [ypxq+strideq*2] + pmaddwd m2, m5, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + paddd m4, m0 + packssdw m0, m3 + paddd m3, m1 + packssdw m1, m2 + paddd m4, m2 + paddd m4, m3 + mova [acq+16*0], m0 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + punpckhqdq m1, m1 + pslld m2, 3 + mova [acq+16*0], m1 + mova [acq+16*1], m1 + paddd m4, m2 + mova [acq+16*2], m1 + mova [acq+16*3], m1 + add acq, 16*4 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w8: +%if ARCH_X86_32 + cmp dword wpadm, 0 +%else + test wpadd, wpadd +%endif + jnz .w8_wpad1 +.w8_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m2, m5, [ypxq+strideq*0+16*1] + pmaddwd m1, m5, [ypxq+strideq*1+16*0] + pmaddwd m3, m5, [ypxq+strideq*1+16*1] + lea ypxq, [ypxq+strideq*2] + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + mova [acq+16*0], m0 + paddd m2, m1, m3 + packssdw m1, m3 + paddd m4, m2 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 2 + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + pslld m2, 2 + mova m0, m1 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w8_wpad1: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + pshufd m2, m0, q3333 + pshufd m3, m1, q3333 + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + paddd m2, m1, m3 + packssdw m1, m3 + paddd m4, m2 + mova [acq+16*0], m0 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 2 + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad3: + pshufd m3, m0, q3333 + mova m1, m3 + mova m2, m3 + jmp .w16_wpad_end +.w16_wpad2: + pshufd m1, m3, q3333 + mova m2, m1 + jmp .w16_wpad_end +.w16_wpad1: + pshufd m2, m1, q3333 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + WIN64_SPILL_XMM 7 +.w16_loop: + pmaddwd m0, m5, [ypxq+16*0] + cmp wpadd, 2 + jg .w16_wpad3 + pmaddwd m3, m5, [ypxq+16*1] + je .w16_wpad2 + pmaddwd m1, m5, [ypxq+16*2] + jp .w16_wpad1 + pmaddwd m2, m5, [ypxq+16*3] +.w16_wpad_end: + add ypxq, strideq + paddd m6, m0, m3 + packssdw m0, m3 + mova [acq+16*0], m0 + paddd m6, m1 + packssdw m1, m2 + paddd m2, m6 + mova [acq+16*1], m1 + add acq, 16*2 + paddd m4, m2 + dec hd + jg .w16_loop + WIN64_RESTORE_XMM + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + paddd m2, m2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad + +cglobal ipred_cfl_ac_444_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h +%define base r6-ipred_cfl_ac_444_16bpc_ssse3_table + LEA r6, ipred_cfl_ac_444_16bpc_ssse3_table + tzcnt wd, wm + movifnidn hpadd, hpadm + pxor m4, m4 + movsxd wq, [r6+wq*4] + movddup m5, [base+pw_1] + add wq, r6 + mov hd, hm + shl hpadd, 2 + sub hd, hpadd + jmp wq +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + movq m0, [ypxq+strideq*0] + movhps m0, [ypxq+strideq*1] + movq m1, [ypxq+strideq*2] + movhps m1, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + psllw m0, 3 + psllw m1, 3 + mova [acq+16*0], m0 + pmaddwd m0, m5 + mova [acq+16*1], m1 + pmaddwd m2, m5, m1 + add acq, 16*2 + paddd m4, m0 + paddd m4, m2 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + punpckhqdq m1, m1 + mova [acq+16*0], m1 + pslld m2, 2 + mova [acq+16*1], m1 + punpckhqdq m2, m2 + mova [acq+16*2], m1 + paddd m4, m2 + mova [acq+16*3], m1 + add acq, 16*4 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w8: + mov r5, acq +.w8_loop: + mova m0, [ypxq+strideq*0] + mova m1, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + psllw m0, 3 + psllw m1, 3 + mova [acq+16*0], m0 + pmaddwd m0, m5 + mova [acq+16*1], m1 + pmaddwd m2, m5, m1 + add acq, 16*2 + paddd m4, m0 + paddd m4, m2 + sub hd, 2 + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + pslld m2, 2 + mova m0, m1 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w16_wpad2: + pshufhw m3, m2, q3333 + pshufhw m1, m0, q3333 + punpckhqdq m3, m3 + punpckhqdq m1, m1 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + mov r5, acq +.w16_loop: + mova m2, [ypxq+strideq*0+16*0] + mova m0, [ypxq+strideq*1+16*0] + psllw m2, 3 + psllw m0, 3 + test wpadd, wpadd + jnz .w16_wpad2 + mova m3, [ypxq+strideq*0+16*1] + mova m1, [ypxq+strideq*1+16*1] + psllw m3, 3 + psllw m1, 3 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + mova [acq+16*0], m2 + pmaddwd m2, m5 + mova [acq+16*1], m3 + pmaddwd m3, m5 + paddd m4, m2 + pmaddwd m2, m5, m0 + mova [acq+16*2], m0 + paddd m4, m3 + pmaddwd m3, m5, m1 + mova [acq+16*3], m1 + add acq, 16*4 + paddd m2, m3 + paddd m4, m2 + sub hd, 2 + jg .w16_loop + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + paddd m2, m2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w32_wpad6: + pshufhw m1, m0, q3333 + punpckhqdq m1, m1 + mova m2, m1 + mova m3, m1 + jmp .w32_wpad_end +.w32_wpad4: + pshufhw m2, m1, q3333 + punpckhqdq m2, m2 + mova m3, m2 + jmp .w32_wpad_end +.w32_wpad2: + pshufhw m3, m2, q3333 + punpckhqdq m3, m3 + jmp .w32_wpad_end +.w32: + movifnidn wpadd, wpadm + mov r5, acq + WIN64_SPILL_XMM 8 +.w32_loop: + mova m0, [ypxq+16*0] + psllw m0, 3 + cmp wpadd, 4 + jg .w32_wpad6 + mova m1, [ypxq+16*1] + psllw m1, 3 + je .w32_wpad4 + mova m2, [ypxq+16*2] + psllw m2, 3 + jnp .w32_wpad2 + mova m3, [ypxq+16*3] + psllw m3, 3 +.w32_wpad_end: + add ypxq, strideq + pmaddwd m6, m5, m0 + mova [acq+16*0], m0 + pmaddwd m7, m5, m1 + mova [acq+16*1], m1 + paddd m6, m7 + pmaddwd m7, m5, m2 + mova [acq+16*2], m2 + paddd m6, m7 + pmaddwd m7, m5, m3 + mova [acq+16*3], m3 + add acq, 16*4 + paddd m6, m7 + paddd m4, m6 + dec hd + jg .w32_loop +%if WIN64 + mova m5, m6 + WIN64_RESTORE_XMM + SWAP 5, 6 +%endif + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w32_hpad_loop: + mova [acq+16*0], m0 + mova [acq+16*1], m1 + paddd m4, m6 + mova [acq+16*2], m2 + mova [acq+16*3], m3 + add acq, 16*4 + dec hpadd + jg .w32_hpad_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + +cglobal pal_pred_16bpc, 4, 5, 5, dst, stride, pal, idx, w, h +%define base r2-pal_pred_16bpc_ssse3_table +%if ARCH_X86_32 + %define hd r2d +%endif + mova m3, [palq] + LEA r2, pal_pred_16bpc_ssse3_table + tzcnt wd, wm + pshufb m3, [base+pal_pred_shuf] + movsxd wq, [r2+wq*4] + pshufd m4, m3, q1032 + add wq, r2 + movifnidn hd, hm + jmp wq +.w4: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 4 + jg .w4 + RET +.w8: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8 + RET +.w16: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, strideq + dec hd + jg .w16 + RET +.w32: + mova m0, [idxq+16*0] + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova m2, [idxq+16*1] + add idxq, 16*2 + mova [dstq+16*0], m0 + pshufb m0, m3, m2 + mova [dstq+16*1], m1 + pshufb m1, m4, m2 + punpcklbw m2, m0, m1 + punpckhbw m0, m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m0 + add dstq, strideq + dec hd + jg .w32 + RET +.w64: + mova m0, [idxq+16*0] + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova m2, [idxq+16*1] + mova [dstq+16*0], m0 + pshufb m0, m3, m2 + mova [dstq+16*1], m1 + pshufb m1, m4, m2 + punpcklbw m2, m0, m1 + punpckhbw m0, m1 + mova m1, [idxq+16*2] + mova [dstq+16*2], m2 + pshufb m2, m3, m1 + mova [dstq+16*3], m0 + pshufb m0, m4, m1 + punpcklbw m1, m2, m0 + punpckhbw m2, m0 + mova m0, [idxq+16*3] + add idxq, 16*4 + mova [dstq+16*4], m1 + pshufb m1, m3, m0 + mova [dstq+16*5], m2 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+16*6], m0 + mova [dstq+16*7], m1 + add dstq, strideq + dec hd + jg .w64 + RET diff --git a/third_party/dav1d/src/x86/ipred_avx2.asm b/third_party/dav1d/src/x86/ipred_avx2.asm new file mode 100644 index 0000000000..dd188a7f37 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred_avx2.asm @@ -0,0 +1,5387 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +%macro SMOOTH_WEIGHT_TABLE 1-* + %rep %0 + db %1-128, 127-%1 + %rotate 1 + %endrep +%endmacro + +; sm_weights[], but modified to precalculate x and 256-x with offsets to +; enable efficient use of pmaddubsw (which requires signed values) +smooth_weights: SMOOTH_WEIGHT_TABLE \ + 0, 0, 255, 128, 255, 149, 85, 64, \ + 255, 197, 146, 105, 73, 50, 37, 32, \ + 255, 225, 196, 170, 145, 123, 102, 84, \ + 68, 54, 43, 33, 26, 20, 17, 16, \ + 255, 240, 225, 210, 196, 182, 169, 157, \ + 145, 133, 122, 111, 101, 92, 83, 74, \ + 66, 59, 52, 45, 39, 34, 29, 25, \ + 21, 17, 14, 12, 10, 9, 8, 8, \ + 255, 248, 240, 233, 225, 218, 210, 203, \ + 196, 189, 182, 176, 169, 163, 156, 150, \ + 144, 138, 133, 127, 121, 116, 111, 106, \ + 101, 96, 91, 86, 82, 77, 73, 69, \ + 65, 61, 57, 54, 50, 47, 44, 41, \ + 38, 35, 32, 29, 27, 25, 22, 20, \ + 18, 16, 15, 13, 12, 10, 9, 8, \ + 7, 6, 6, 5, 5, 4, 4, 4 + +pb_1to32: db 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + db 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +pb_32to1: db 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17 +pb_16to1: db 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 +z_filter_wh: db 7, 7, 11, 11, 15, 15, 19, 19, 19, 23, 23, 23, 31, 31, 31, 39 + db 39, 39, 47, 47, 47, 63, 63, 63, 79, 79, 79, -1 +z_filter_k: db 0, 16, 0, 16, 0, 20, 0, 20, 8, 16, 8, 16 + db 32, 16, 32, 16, 24, 20, 24, 20, 16, 16, 16, 16 + db 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0 +z_filter_s: db 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7 + db 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15 + db 15, 15, 15, 15, 15, 15, 15, 15 ; should be in one cache line +pb_128: times 4 db 128 ; those are just placed here for alignment. +pb_36_m4: times 2 db 36, -4 +z3_shuf: db 8, 7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0 +z_filter_t0: db 55,127, 39,127, 39,127, 7, 15, 31, 7, 15, 31, 0, 3, 31, 0 +z_filter_t1: db 39, 63, 19, 47, 19, 47, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0 +z_upsample1: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +z_upsample2: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 8 +z2_upsample: db 7, 6, 15, 14, 5, 4, 13, 12, 3, 2, 11, 10, 1, 0, 9, 8 +z1_shuf_w4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 +z2_shuf_h2: db 3, 2, 7, 6, 11, 10, 15, 14, 2, 1, 6, 5, 10, 9, 14, 13 +z2_shuf_h4: db 7, 6, 15, 14, 6, 5, 14, 13, 5, 4, 13, 12, 4, 3, 12, 11 +z3_shuf_w4: db 4, 3, 3, 2, 2, 1, 1, 0, 12, 11, 11, 10, 10, 9, 9, 8 +z_transpose4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 +z_base_inc: dw 0*64, 1*64, 2*64, 3*64, 4*64, 5*64, 6*64, 7*64 + dw 16*64, 17*64, 18*64, 19*64, 20*64, 21*64, 22*64, 23*64 +z2_base_inc: dw 1*64, 2*64, 3*64, 4*64, 5*64, 6*64, 7*64, 8*64 + dw 9*64, 10*64, 11*64, 12*64, 13*64, 14*64, 15*64, 16*64 +z2_ymul: dw 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +z2_y_shuf_h4: db 90, 90, 90, 90, 14, 14, 14, 14, 27, 27, 27, 27, 31, 31, 31, 31 ; 2, 6, 3, 7 + db 32, 32, 32, 32, 12, 12, 12, 12, 1, 0, 1, 0, 5, -1, -1, -1 ; 0, 4, 1, 5 +; vpermd indices in bits 4..6 of filter_shuf1: 0, 2, 6, 4, 1, 3, 7, 5 +filter_shuf1: db 10, 4, 10, 4, 37, 6, 5, 6,103, 9, 7, 9, 72, -1, 8, -1 + db 16, 4, 0, 4, 53, 6, 5, 6,119, 11, 7, 11, 95, -1, 15, -1 +filter_shuf2: db 3, 4, 3, 4, 5, 6, 5, 6, 7, 2, 7, 2, 1, -1, 1, -1 +filter_shuf3: db 3, 4, 3, 4, 5, 6, 5, 6, 7, 11, 7, 11; 15, -1, 15, -1 +pb_127_m127: times 2 db 127, -127 +ipred_v_shuf: db 0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13 + db 2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15 +ipred_h_shuf: db 7, 7, 7, 7, 3, 3, 3, 3, 5, 5, 5, 5, 1, 1, 1, 1 + db 6, 6, 6, 6, 2, 2, 2, 2, 4, 4, 4, 4; 0, 0, 0, 0 +pw_64: times 2 dw 64 + +cfl_ac_444_w16_pad1_shuffle: db 0, -1, 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1 + times 9 db 7, -1 +cfl_ac_w16_pad_shuffle: ; w=16, w_pad=1 + db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ; w=8, w_pad=1 as well as second half of previous one +cfl_ac_w8_pad1_shuffle: db 0, 1, 2, 3, 4, 5 + times 5 db 6, 7 + ; w=16,w_pad=2 + db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + times 8 db 14, 15 + ; w=16,w_pad=3 + db 0, 1, 2, 3, 4, 5 + times 13 db 6, 7 +pb_15to0: db 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 + +%define pb_0to15 cfl_ac_w16_pad_shuffle +%define pb_1 (ipred_h_shuf+12) +%define pb_2 (ipred_h_shuf+20) +%define pb_3 (ipred_h_shuf+ 4) +%define pb_4 (ipred_h_shuf+24) +%define pb_5 (ipred_h_shuf+ 8) +%define pb_7 (ipred_h_shuf+ 0) +%define pb_8 (z_upsample2 +12) +%define pb_12 (z2_y_shuf_h4+20) +%define pb_14 (z2_y_shuf_h4+ 4) +%define pb_15 (z_filter_s +32) +%define pb_27 (z2_y_shuf_h4+ 8) +%define pb_31 (z2_y_shuf_h4+12) +%define pb_32 (z2_y_shuf_h4+16) +%define pb_90 (z2_y_shuf_h4+ 0) +%define pw_1 (z2_y_shuf_h4+24) +%define pw_8 (z_filter_k +32) + +pw_62: times 2 dw 62 +pw_128: times 2 dw 128 +pw_255: times 2 dw 255 +pw_512: times 2 dw 512 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_avx2_table (ipred_dc_avx2_table + 10*4) +%define ipred_cfl_splat_avx2_table (ipred_cfl_avx2_table + 8*4) + +JMP_TABLE ipred_smooth, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_v, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_h, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_paeth, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_filter, avx2, w4, w8, w16, w32 +JMP_TABLE ipred_dc, avx2, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4 +JMP_TABLE ipred_dc_left, avx2, h4, h8, h16, h32, h64 +JMP_TABLE ipred_h, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z1, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z2, avx2, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z3, avx2, h4, h8, h16, h32, h64 +JMP_TABLE ipred_cfl, avx2, h4, h8, h16, h32, w4, w8, w16, w32, \ + s4-8*4, s8-8*4, s16-8*4, s32-8*4 +JMP_TABLE ipred_cfl_left, avx2, h4, h8, h16, h32 +JMP_TABLE ipred_cfl_ac_420, avx2, w16_pad1, w16_pad2, w16_pad3 +JMP_TABLE ipred_cfl_ac_422, avx2, w16_pad1, w16_pad2, w16_pad3 +JMP_TABLE ipred_cfl_ac_444, avx2, w32_pad1, w32_pad2, w32_pad3, w4, w8, w16, w32 +JMP_TABLE pal_pred, avx2, w4, w8, w16, w32, w64 + +cextern dr_intra_derivative +cextern filter_intra_taps + +SECTION .text + +INIT_YMM avx2 +cglobal ipred_dc_top_8bpc, 3, 7, 6, dst, stride, tl, w, h + lea r5, [ipred_dc_left_avx2_table] + tzcnt wd, wm + inc tlq + movu m0, [tlq] + movifnidn hd, hm + mov r6d, 0x8000 + shrx r6d, r6d, wd + movd xm3, r6d + movsxd r6, [r5+wq*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, r5 + add r5, ipred_dc_splat_avx2_table-ipred_dc_left_avx2_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +cglobal ipred_dc_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + mov hd, hm ; zero upper half + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + movu m0, [tlq] + mov r5d, 0x8000 + shrx r5d, r5d, r6d + movd xm3, r5d + lea r5, [ipred_dc_left_avx2_table] + movsxd r6, [r5+r6*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, r5 + add r5, ipred_dc_splat_avx2_table-ipred_dc_left_avx2_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + movu m1, [tlq+32] ; unaligned when jumping here from dc_top + pmaddubsw m1, m2 + paddw m0, m1 +.h32: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 +.h16: + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 +.h8: + psrlq xm1, xm0, 32 + paddw xm0, xm1 +.h4: + pmaddwd xm0, xm2 + pmulhrsw xm0, xm3 + lea stride3q, [strideq*3] + vpbroadcastb m0, xm0 + mova m1, m0 + jmp wq + +cglobal ipred_dc_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + movifnidn wd, wm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd xm4, r5d + tzcnt r5d, r5d + movd xm5, r5d + lea r5, [ipred_dc_avx2_table] + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+5*4] + pcmpeqd m3, m3 + psrlw xm4, 1 + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movd xm0, [tlq-4] + pmaddubsw xm0, xm3 + jmp wq +.w4: + movd xm1, [tlq+1] + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm1 + pmaddwd xm0, xm3 + cmp hd, 4 + jg .w4_mul + psrlw xm0, 3 + jmp .w4_end +.w4_mul: + punpckhqdq xm1, xm0, xm0 + lea r2d, [hq*2] + mov r6d, 0x55563334 + paddw xm0, xm1 + shrx r6d, r6d, r2d + psrlq xm1, xm0, 32 + paddw xm0, xm1 + movd xm1, r6d + psrlw xm0, 2 + pmulhuw xm0, xm1 +.w4_end: + vpbroadcastb xm0, xm0 +.s4: + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm0 + movd [dstq+strideq*2], xm0 + movd [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +ALIGN function_align +.h8: + movq xm0, [tlq-8] + pmaddubsw xm0, xm3 + jmp wq +.w8: + movq xm1, [tlq+1] + vextracti128 xm2, m0, 1 + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm2 + punpckhqdq xm2, xm0, xm0 + paddw xm0, xm2 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 8 + je .w8_end + mov r6d, 0x5556 + mov r2d, 0x3334 + cmp hd, 32 + cmove r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w8_end: + vpbroadcastb xm0, xm0 +.s8: + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm0 + movq [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +ALIGN function_align +.h16: + mova xm0, [tlq-16] + pmaddubsw xm0, xm3 + jmp wq +.w16: + movu xm1, [tlq+1] + vextracti128 xm2, m0, 1 + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm2 + paddw xm0, xm1 + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 16 + je .w16_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hb, 8|32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w16_end: + vpbroadcastb xm0, xm0 +.s16: + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm0 + mova [dstq+strideq*2], xm0 + mova [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +ALIGN function_align +.h32: + mova m0, [tlq-32] + pmaddubsw m0, m3 + jmp wq +.w32: + movu m1, [tlq+1] + pmaddubsw m1, m3 + paddw m0, m1 + vextracti128 xm1, m0, 1 + psubw xm0, xm4 + paddw xm0, xm1 + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x33345556 + shrx r6d, r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w32_end: + vpbroadcastb m0, xm0 +.s32: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s32 + RET +ALIGN function_align +.h64: + mova m0, [tlq-64] + mova m1, [tlq-32] + pmaddubsw m0, m3 + pmaddubsw m1, m3 + paddw m0, m1 + jmp wq +.w64: + movu m1, [tlq+ 1] + movu m2, [tlq+33] + pmaddubsw m1, m3 + pmaddubsw m2, m3 + paddw m0, m1 + paddw m0, m2 + vextracti128 xm1, m0, 1 + psubw xm0, xm4 + paddw xm0, xm1 + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 64 + je .w64_end + mov r6d, 0x33345556 + shrx r6d, r6d, hd + movd xm1, r6d + pmulhuw xm0, xm1 +.w64_end: + vpbroadcastb m0, xm0 + mova m1, m0 +.s64: + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m0 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*2+32*0], m0 + mova [dstq+strideq*2+32*1], m1 + mova [dstq+stride3q +32*0], m0 + mova [dstq+stride3q +32*1], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s64 + RET + +cglobal ipred_dc_128_8bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 + lea r5, [ipred_dc_splat_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m0, [r5-ipred_dc_splat_avx2_table+pb_128] + mova m1, m0 + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_v_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + lea r5, [ipred_dc_splat_avx2_table] + tzcnt wd, wm + movu m0, [tlq+ 1] + movu m1, [tlq+33] + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +%macro IPRED_H 2 ; w, store_type + vpbroadcastb m0, [tlq-1] + vpbroadcastb m1, [tlq-2] + vpbroadcastb m2, [tlq-3] + sub tlq, 4 + vpbroadcastb m3, [tlq+0] + mov%2 [dstq+strideq*0], m0 + mov%2 [dstq+strideq*1], m1 + mov%2 [dstq+strideq*2], m2 + mov%2 [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w%1 + RET +ALIGN function_align +%endmacro + +INIT_XMM avx2 +cglobal ipred_h_8bpc, 3, 6, 4, dst, stride, tl, w, h, stride3 + lea r5, [ipred_h_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w4: + IPRED_H 4, d +.w8: + IPRED_H 8, q +.w16: + IPRED_H 16, a +INIT_YMM avx2 +.w32: + IPRED_H 32, a +.w64: + vpbroadcastb m0, [tlq-1] + vpbroadcastb m1, [tlq-2] + vpbroadcastb m2, [tlq-3] + sub tlq, 4 + vpbroadcastb m3, [tlq+0] + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m0 + mova [dstq+strideq*1+32*0], m1 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*2+32*0], m2 + mova [dstq+strideq*2+32*1], m2 + mova [dstq+stride3q +32*0], m3 + mova [dstq+stride3q +32*1], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w64 + RET + +%macro PAETH 2 ; top, ldiff + pavgb m1, m%1, m3 ; Calculating tldiff normally requires + pxor m0, m%1, m3 ; 10-bit intermediates, but we can do it + pand m0, m4 ; in 8-bit with some tricks which avoids + psubusb m2, m5, m1 ; having to unpack everything to 16-bit. + psubb m1, m0 + psubusb m1, m5 + por m1, m2 + paddusb m1, m1 + por m1, m0 ; min(tldiff, 255) + psubusb m2, m5, m3 + psubusb m0, m3, m5 + por m2, m0 ; tdiff + pminub m2, m%2 + pcmpeqb m0, m%2, m2 ; ldiff <= tdiff + vpblendvb m0, m%1, m3, m0 + pminub m1, m2 + pcmpeqb m1, m2 ; ldiff <= tldiff || tdiff <= tldiff + vpblendvb m0, m5, m0, m1 +%endmacro + +cglobal ipred_paeth_8bpc, 3, 6, 9, dst, stride, tl, w, h +%define base r5-ipred_paeth_avx2_table + lea r5, [ipred_paeth_avx2_table] + tzcnt wd, wm + vpbroadcastb m5, [tlq] ; topleft + movifnidn hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m4, [base+pb_1] + add wq, r5 + jmp wq +.w4: + vpbroadcastd m6, [tlq+1] ; top + mova m8, [base+ipred_h_shuf] + lea r3, [strideq*3] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 ; ldiff +.w4_loop: + sub tlq, 8 + vpbroadcastq m3, [tlq] + pshufb m3, m8 ; left + PAETH 6, 7 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r3 ], xm1, 2 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+r3 ], xm1, 3 + lea dstq, [dstq+strideq*4] + sub hd, 8 + jg .w4_loop +.ret: + RET +ALIGN function_align +.w8: + vpbroadcastq m6, [tlq+1] + mova m8, [base+ipred_h_shuf] + lea r3, [strideq*3] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w8_loop: + sub tlq, 4 + vpbroadcastd m3, [tlq] + pshufb m3, m8 + PAETH 6, 7 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +ALIGN function_align +.w16: + vbroadcasti128 m6, [tlq+1] + mova xm8, xm4 ; lower half = 1, upper half = 0 + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w16_loop: + sub tlq, 2 + vpbroadcastd m3, [tlq] + pshufb m3, m8 + PAETH 6, 7 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + RET +ALIGN function_align +.w32: + movu m6, [tlq+1] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w32_loop: + dec tlq + vpbroadcastb m3, [tlq] + PAETH 6, 7 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w32_loop + RET +ALIGN function_align +.w64: + movu m6, [tlq+ 1] + movu m7, [tlq+33] +%if WIN64 + movaps r4m, xmm9 +%endif + psubusb m8, m5, m6 + psubusb m0, m6, m5 + psubusb m9, m5, m7 + psubusb m1, m7, m5 + por m8, m0 + por m9, m1 +.w64_loop: + dec tlq + vpbroadcastb m3, [tlq] + PAETH 6, 8 + mova [dstq+32*0], m0 + PAETH 7, 9 + mova [dstq+32*1], m0 + add dstq, strideq + dec hd + jg .w64_loop +%if WIN64 + movaps xmm9, r4m +%endif + RET + +%macro SMOOTH 6 ; src[1-2], mul[1-2], add[1-2] + ; w * a = (w - 128) * a + 128 * a + ; (256 - w) * b = (127 - w) * b + 129 * b + pmaddubsw m0, m%3, m%1 + pmaddubsw m1, m%4, m%2 + paddw m0, m%5 + paddw m1, m%6 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 +%endmacro + +cglobal ipred_smooth_v_8bpc, 3, 7, 0, dst, stride, tl, w, h, weights +%define base r6-ipred_smooth_v_avx2_table + lea r6, [ipred_smooth_v_avx2_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastd m0, [base+pb_127_m127] + vpbroadcastd m1, [base+pw_128] + lea weightsq, [base+smooth_weights+hq*4] + neg hq + vpbroadcastb m5, [tlq+hq] ; bottom + add wq, r6 + jmp wq +.w4: + vpbroadcastd m2, [tlq+1] + punpcklbw m2, m5 ; top, bottom + mova m5, [base+ipred_v_shuf] + lea r3, [strideq*3] + punpckldq m4, m5, m5 + punpckhdq m5, m5 + pmaddubsw m3, m2, m0 + paddw m1, m2 ; 1 * top + 256 * bottom + 128, overflow is ok + paddw m3, m1 ; 128 * top + 129 * bottom + 128 +.w4_loop: + vbroadcasti128 m1, [weightsq+hq*2] + pshufb m0, m1, m4 + pshufb m1, m5 + SMOOTH 0, 1, 2, 2, 3, 3 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 1 + pextrd [dstq+r3 ], xm1, 1 + cmp hd, -4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm1, 2 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+r3 ], xm1, 3 + lea dstq, [dstq+strideq*4] + add hq, 8 + jl .w4_loop +.ret: + RET +ALIGN function_align +.w8: + vpbroadcastq m2, [tlq+1] + punpcklbw m2, m5 + mova m5, [base+ipred_v_shuf] + lea r3, [strideq*3] + pshufd m4, m5, q0000 + pshufd m5, m5, q1111 + pmaddubsw m3, m2, m0 + paddw m1, m2 + paddw m3, m1 +.w8_loop: + vpbroadcastq m1, [weightsq+hq*2] + pshufb m0, m1, m4 + pshufb m1, m5 + SMOOTH 0, 1, 2, 2, 3, 3 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w8_loop + RET +ALIGN function_align +.w16: + WIN64_SPILL_XMM 7 + vbroadcasti128 m3, [tlq+1] + mova m6, [base+ipred_v_shuf] + punpcklbw m2, m3, m5 + punpckhbw m3, m5 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 +.w16_loop: + vpbroadcastd m1, [weightsq+hq*2] + pshufb m1, m6 + SMOOTH 1, 1, 2, 3, 4, 5 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w16_loop + RET +ALIGN function_align +.w32: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 6 + movu m3, [tlq+1] + punpcklbw m2, m3, m5 + punpckhbw m3, m5 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 +.w32_loop: + vpbroadcastw m1, [weightsq+hq*2] + SMOOTH 1, 1, 2, 3, 4, 5 + mova [dstq], m0 + add dstq, strideq + inc hq + jl .w32_loop + RET +ALIGN function_align +.w64: + WIN64_SPILL_XMM 11 + movu m4, [tlq+ 1] + movu m8, [tlq+33] + punpcklbw m3, m4, m5 + punpckhbw m4, m5 + punpcklbw m7, m8, m5 + punpckhbw m8, m5 + pmaddubsw m5, m3, m0 + pmaddubsw m6, m4, m0 + pmaddubsw m9, m7, m0 + pmaddubsw m10, m8, m0 + paddw m2, m1, m3 + paddw m5, m2 + paddw m2, m1, m4 + paddw m6, m2 + paddw m0, m1, m7 + paddw m9, m0 + paddw m1, m8 + paddw m10, m1 +.w64_loop: + vpbroadcastw m2, [weightsq+hq*2] + SMOOTH 2, 2, 3, 4, 5, 6 + mova [dstq+32*0], m0 + SMOOTH 2, 2, 7, 8, 9, 10 + mova [dstq+32*1], m0 + add dstq, strideq + inc hq + jl .w64_loop + RET + +%macro SETUP_STACK_FRAME 3 ; stack_size, regs_used, xmm_regs_used + %assign stack_offset 0 + %assign stack_size_padded 0 + %assign regs_used %2 + %xdefine rstk rsp + SETUP_STACK_POINTER %1 + %if regs_used != %2 && WIN64 + PUSH r%2 + %endif + ALLOC_STACK %1, %3 +%endmacro + +cglobal ipred_smooth_h_8bpc, 3, 7, 0, dst, stride, tl, w, h +%define base r6-ipred_smooth_h_avx2_table + lea r6, [ipred_smooth_h_avx2_table] + mov wd, wm + vpbroadcastb m3, [tlq+wq] ; right + tzcnt wd, wd + mov hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastd m4, [base+pb_127_m127] + vpbroadcastd m5, [base+pw_128] + add wq, r6 + jmp wq +.w4: + WIN64_SPILL_XMM 8 + vpbroadcastq m6, [base+smooth_weights+4*2] + mova m7, [base+ipred_h_shuf] + sub tlq, 8 + sub tlq, hq + lea r3, [strideq*3] +.w4_loop: + vpbroadcastq m2, [tlq+hq] + pshufb m2, m7 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m6 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + pmaddubsw m2, m6 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r3 ], xm1, 2 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+r3 ], xm1, 3 + lea dstq, [dstq+strideq*4] + sub hd, 8 + jg .w4_loop +.ret: + RET +ALIGN function_align +.w8: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 8 + vbroadcasti128 m6, [base+smooth_weights+8*2] + mova m7, [base+ipred_h_shuf] + sub tlq, 4 + lea r3, [strideq*3] + sub tlq, hq +.w8_loop: + vpbroadcastd m2, [tlq+hq] + pshufb m2, m7 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 + paddw m0, m1 + pmaddubsw m1, m6 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + pmaddubsw m2, m6 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +ALIGN function_align +.w16: + SETUP_STACK_FRAME 32*4, 7, 8 + lea r3, [rsp+64*2-4] + call .prep ; only worthwhile for for w16 and above + sub tlq, 2 + vpbroadcastd xm6, [base+pb_1] + mova xm7, [base+ipred_v_shuf+16] + vinserti128 m7, [base+ipred_v_shuf+ 0], 1 + vbroadcasti128 m4, [base+smooth_weights+16*2] + vbroadcasti128 m5, [base+smooth_weights+16*3] +.w16_loop: + vpbroadcastd m1, [tlq+hq] + vpbroadcastd m2, [r3+hq*2] + pshufb m1, m6 + punpcklbw m1, m3 + pshufb m2, m7 + SMOOTH 4, 5, 1, 1, 2, 2 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + RET +ALIGN function_align +.w32: + SETUP_STACK_FRAME 32*4, 7, 6 + lea r3, [rsp+64*2-2] + call .prep + dec tlq + mova xm4, [base+smooth_weights+16*4] + vinserti128 m4, [base+smooth_weights+16*6], 1 + mova xm5, [base+smooth_weights+16*5] + vinserti128 m5, [base+smooth_weights+16*7], 1 +.w32_loop: + vpbroadcastb m1, [tlq+hq] + punpcklbw m1, m3 + vpbroadcastw m2, [r3+hq*2] + SMOOTH 4, 5, 1, 1, 2, 2 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w32_loop + RET +ALIGN function_align +.w64: + SETUP_STACK_FRAME 32*4, 7, 9 + lea r3, [rsp+64*2-2] + call .prep + add r6, smooth_weights+16*15-ipred_smooth_h_avx2_table + dec tlq + mova xm5, [r6-16*7] + vinserti128 m5, [r6-16*5], 1 + mova xm6, [r6-16*6] + vinserti128 m6, [r6-16*4], 1 + mova xm7, [r6-16*3] + vinserti128 m7, [r6-16*1], 1 + mova xm8, [r6-16*2] + vinserti128 m8, [r6-16*0], 1 +.w64_loop: + vpbroadcastb m2, [tlq+hq] + punpcklbw m2, m3 + vpbroadcastw m4, [r3+hq*2] + SMOOTH 5, 6, 2, 2, 4, 4 + mova [dstq+32*0], m0 + SMOOTH 7, 8, 2, 2, 4, 4 + mova [dstq+32*1], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +ALIGN function_align +.prep: + vpermq m2, [tlq-32*1], q3120 + punpckhbw m1, m2, m3 + punpcklbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m1, m5 ; 1 * left + 256 * right + 128 + paddw m0, m1 ; 128 * left + 129 * right + 128 + pmaddubsw m1, m2, m4 + paddw m2, m5 + paddw m1, m2 + vpermq m2, [tlq-32*2], q3120 + mova [rsp+gprsize+32*3], m0 + mova [rsp+gprsize+32*2], m1 + punpckhbw m1, m2, m3 + punpcklbw m2, m3 + pmaddubsw m0, m1, m4 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m2, m5 + paddw m1, m2 + mova [rsp+gprsize+32*1], m0 + mova [rsp+gprsize+32*0], m1 + sub r3, hq + sub tlq, hq + sub r3, hq + ret + +%macro SMOOTH_2D_END 6 ; src[1-2], mul[1-2], add[1-2] + pmaddubsw m0, m%3, m%1 + pmaddubsw m1, m%4, m%2 +%ifnum %5 + paddw m0, m%5 +%else + paddw m0, %5 +%endif +%ifnum %6 + paddw m1, m%6 +%else + paddw m1, %6 +%endif + pavgw m0, m2 + pavgw m1, m3 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 +%endmacro + +cglobal ipred_smooth_8bpc, 3, 7, 0, dst, stride, tl, w, h, v_weights +%define base r6-ipred_smooth_avx2_table + lea r6, [ipred_smooth_avx2_table] + mov wd, wm + vpbroadcastb m4, [tlq+wq] ; right + tzcnt wd, wd + mov hd, hm + mov r5, tlq + sub r5, hq + movsxd wq, [r6+wq*4] + vpbroadcastd m5, [base+pb_127_m127] + vpbroadcastb m0, [r5] ; bottom + vpbroadcastd m3, [base+pw_255] + add wq, r6 + lea v_weightsq, [base+smooth_weights+hq*2] + jmp wq +.w4: + WIN64_SPILL_XMM 12 + mova m10, [base+ipred_h_shuf] + vpbroadcastq m11, [base+smooth_weights+4*2] + mova m7, [base+ipred_v_shuf] + vpbroadcastd m8, [tlq+1] + sub tlq, 8 + lea r3, [strideq*3] + sub tlq, hq + punpcklbw m8, m0 ; top, bottom + pshufd m6, m7, q2200 + pshufd m7, m7, q3311 + pmaddubsw m9, m8, m5 + paddw m3, m8 ; 1 * top + 255 * bottom + 255 + paddw m9, m3 ; 128 * top + 129 * bottom + 255 +.w4_loop: + vpbroadcastq m1, [tlq+hq] + pshufb m1, m10 + punpcklbw m0, m1, m4 ; left, right + punpckhbw m1, m4 + pmaddubsw m2, m0, m5 ; 127 * left - 127 * right + pmaddubsw m3, m1, m5 + paddw m2, m0 ; 128 * left + 129 * right + paddw m3, m1 + pmaddubsw m0, m11 + pmaddubsw m1, m11 + paddw m2, m0 + paddw m3, m1 + vbroadcasti128 m1, [v_weightsq] + add v_weightsq, 16 + pshufb m0, m1, m6 + pshufb m1, m7 + SMOOTH_2D_END 0, 1, 8, 8, 9, 9 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r3 ], xm1, 2 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+r3 ], xm1, 3 + lea dstq, [dstq+strideq*4] + sub hd, 8 + jg .w4_loop +.ret: + RET +ALIGN function_align +.w8: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 12 + mova m10, [base+ipred_h_shuf] + vbroadcasti128 m11, [base+smooth_weights+8*2] + mova m7, [base+ipred_v_shuf] + vpbroadcastq m8, [tlq+1] + sub tlq, 4 + lea r3, [strideq*3] + sub tlq, hq + punpcklbw m8, m0 + pshufd m6, m7, q0000 + pshufd m7, m7, q1111 + pmaddubsw m9, m8, m5 + paddw m3, m8 + paddw m9, m3 +.w8_loop: + vpbroadcastd m1, [tlq+hq] + pshufb m1, m10 + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + pmaddubsw m2, m0, m5 + pmaddubsw m3, m1, m5 + paddw m2, m0 + paddw m3, m1 + pmaddubsw m0, m11 + pmaddubsw m1, m11 + paddw m2, m0 + paddw m3, m1 + vpbroadcastq m1, [v_weightsq] + add v_weightsq, 8 + pshufb m0, m1, m6 + pshufb m1, m7 + SMOOTH_2D_END 0, 1, 8, 8, 9, 9 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r3 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +ALIGN function_align +.w16: + SETUP_STACK_FRAME 32*4, 7, 14 + vbroadcasti128 m11, [tlq+1] + lea r3, [rsp+64*2-4] + punpcklbw m10, m11, m0 ; top, bottom + punpckhbw m11, m0 + call .prep_v + sub tlq, 2 + pmaddubsw m12, m10, m5 + pmaddubsw m13, m11, m5 + vpbroadcastd xm5, [base+pb_1] + mova m9, [base+ipred_v_shuf] + vbroadcasti128 m6, [base+smooth_weights+16*2] + vbroadcasti128 m7, [base+smooth_weights+16*3] + vperm2i128 m8, m9, m9, 0x01 + paddw m0, m10, m3 + paddw m3, m11 + paddw m12, m0 + paddw m13, m3 +.w16_loop: + vpbroadcastd m3, [tlq+hq] + vpbroadcastd m0, [r3+hq*2] + vpbroadcastd m1, [v_weightsq] + add v_weightsq, 4 + pshufb m3, m5 + punpcklbw m3, m4 ; left, right + pmaddubsw m2, m3, m6 + pmaddubsw m3, m7 + pshufb m0, m8 + pshufb m1, m9 + paddw m2, m0 + paddw m3, m0 + SMOOTH_2D_END 1, 1, 10, 11, 12, 13 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + RET +ALIGN function_align +.w32: + SETUP_STACK_FRAME 32*4, 7, 11 + movu m8, [tlq+1] + lea r3, [rsp+64*2-2] + punpcklbw m7, m8, m0 + punpckhbw m8, m0 + call .prep_v + dec tlq + pmaddubsw m9, m7, m5 + pmaddubsw m10, m8, m5 + mova xm5, [base+smooth_weights+16*4] + vinserti128 m5, [base+smooth_weights+16*6], 1 + mova xm6, [base+smooth_weights+16*5] + vinserti128 m6, [base+smooth_weights+16*7], 1 + paddw m0, m7, m3 + paddw m3, m8 + paddw m9, m0 + paddw m10, m3 +.w32_loop: + vpbroadcastb m3, [tlq+hq] + punpcklbw m3, m4 + vpbroadcastw m0, [r3+hq*2] + vpbroadcastw m1, [v_weightsq] + add v_weightsq, 2 + pmaddubsw m2, m3, m5 + pmaddubsw m3, m6 + paddw m2, m0 + paddw m3, m0 + SMOOTH_2D_END 1, 1, 7, 8, 9, 10 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w32_loop + RET +ALIGN function_align +.w64: + SETUP_STACK_FRAME 32*8, 7, 16 + movu m13, [tlq+1 ] + movu m15, [tlq+33] + add r6, smooth_weights+16*15-ipred_smooth_avx2_table + lea r3, [rsp+64*2-2] + punpcklbw m12, m13, m0 + punpckhbw m13, m0 + punpcklbw m14, m15, m0 + punpckhbw m15, m0 + call .prep_v + dec tlq + pmaddubsw m0, m12, m5 + pmaddubsw m1, m13, m5 + pmaddubsw m2, m14, m5 + pmaddubsw m5, m15, m5 + mova xm8, [r6-16*7] + vinserti128 m8, [r6-16*5], 1 + mova xm9, [r6-16*6] + vinserti128 m9, [r6-16*4], 1 + mova xm10, [r6-16*3] + vinserti128 m10, [r6-16*1], 1 + mova xm11, [r6-16*2] + vinserti128 m11, [r6-16*0], 1 + lea r6, [rsp+32*4] + paddw m0, m3 + paddw m1, m3 + paddw m2, m3 + paddw m3, m5 + paddw m0, m12 + paddw m1, m13 + paddw m2, m14 + paddw m3, m15 + mova [r6+32*0], m0 + mova [r6+32*1], m1 + mova [r6+32*2], m2 + mova [r6+32*3], m3 +.w64_loop: + vpbroadcastb m5, [tlq+hq] + punpcklbw m5, m4 + vpbroadcastw m6, [r3+hq*2] + vpbroadcastw m7, [v_weightsq] + add v_weightsq, 2 + pmaddubsw m2, m5, m8 + pmaddubsw m3, m5, m9 + paddw m2, m6 + paddw m3, m6 + SMOOTH_2D_END 7, 7, 12, 13, [r6+32*0], [r6+32*1] + mova [dstq+32*0], m0 + pmaddubsw m2, m5, m10 + pmaddubsw m3, m5, m11 + paddw m2, m6 + paddw m3, m6 + SMOOTH_2D_END 7, 7, 14, 15, [r6+32*2], [r6+32*3] + mova [dstq+32*1], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +ALIGN function_align +.prep_v: + vpermq m2, [tlq-32*1], q3120 + punpckhbw m1, m2, m4 + punpcklbw m2, m4 + pmaddubsw m0, m1, m5 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m2, m5 + paddw m1, m2 + vpermq m2, [tlq-32*2], q3120 + mova [rsp+gprsize+32*3], m0 + mova [rsp+gprsize+32*2], m1 + punpckhbw m1, m2, m4 + punpcklbw m2, m4 + pmaddubsw m0, m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m5 + paddw m1, m2 + mova [rsp+gprsize+32*1], m0 + mova [rsp+gprsize+32*0], m1 + sub r3, hq + sub tlq, hq + sub r3, hq + ret + +cglobal ipred_z1_8bpc, 3, 8, 0, dst, stride, tl, w, h, angle, dx, maxbase + %assign org_stack_offset stack_offset + lea r6, [ipred_z1_avx2_table] + tzcnt wd, wm + movifnidn angled, anglem + movifnidn hd, hm + lea r7, [dr_intra_derivative] + inc tlq + movsxd wq, [r6+wq*4] + add wq, r6 + mov dxd, angled + and dxd, 0x7e + add angled, 165 ; ~90 + movzx dxd, word [r7+dxq] + xor angled, 0x4ff ; d = 90 - angle + vpbroadcastd m3, [pw_512] + vpbroadcastd m4, [pw_62] + vpbroadcastd m5, [pw_64] + jmp wq +.w4: + cmp angleb, 40 + jae .w4_no_upsample + lea r3d, [angleq-1024] + sar r3d, 7 + add r3d, hd + jg .w4_no_upsample ; !enable_intra_edge_filter || h > 8 || (h == 8 && is_sm) + ALLOC_STACK -32, 8 + mova xm1, [tlq-1] + pshufb xm0, xm1, [z_upsample1] + pshufb xm1, [z_upsample2] + vpbroadcastd xm2, [pb_36_m4] ; upshifted by 2 to be able to reuse + add dxd, dxd ; pw_512 (which is already in m3) + pmaddubsw xm0, xm2 ; for rounding instead of pw_2048 + pextrd [rsp+16], xm1, 3 ; top[max_base_x] + pmaddubsw xm1, xm2 + movd xm7, dxd + mov r3d, dxd ; xpos + vpbroadcastw m7, xm7 + paddw xm1, xm0 + movq xm0, [tlq] + pmulhrsw xm1, xm3 + pslldq m6, m7, 8 + paddw xm2, xm7, xm7 + lea r2, [strideq*3] + paddw m6, m7 + packuswb xm1, xm1 + paddw m6, m2 ; xpos2 xpos3 xpos0 xpos1 + punpcklbw xm0, xm1 + psllw m7, 2 + mova [rsp], xm0 +.w4_upsample_loop: + lea r5d, [r3+dxq] + shr r3d, 6 ; base0 + vpbroadcastq m1, [rsp+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + vpbroadcastq m2, [rsp+r5] + lea r5d, [r3+dxq] + shr r3d, 6 ; base2 + movq xm0, [rsp+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base3 + movhps xm0, [rsp+r5] + vpblendd m1, m2, 0xc0 + pand m2, m4, m6 ; frac + vpblendd m0, m1, 0xf0 + psubw m1, m5, m2 ; 64-frac + psllw m2, 8 + por m1, m2 ; 64-frac, frac + pmaddubsw m0, m1 + paddw m6, m7 ; xpos += dx + pmulhrsw m0, m3 + packuswb m0, m0 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*2], xm0 + pextrd [dstq+r2 ], xm0, 1 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_upsample_loop + RET +ALIGN function_align +.filter_strength: ; w4/w8/w16 + ; The C version uses a lot of branches, but we can do all the comparisons + ; in parallel and use popcnt to get the final filter strength value. +%define base r3-z_filter_t0 + lea r3, [z_filter_t0] + movd xm0, maxbased + movd xm2, angled + shr angled, 8 ; is_sm << 1 + vpbroadcastb m0, xm0 + vpbroadcastb m2, xm2 + pcmpeqb m1, m0, [base+z_filter_wh] + pand m1, m2 + mova xm2, [r3+angleq*8] ; upper ymm half zero in both cases + pcmpgtb m1, m2 + pmovmskb r5d, m1 + ret +.w4_no_upsample: + %assign stack_offset org_stack_offset + ALLOC_STACK -16, 11 + mov maxbased, 7 + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w4_main + lea maxbased, [hq+3] + call .filter_strength + mov maxbased, 7 + test r5d, r5d + jz .w4_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastd m7, [base+pb_8] + vbroadcasti128 m2, [tlq-1] + pminub m1, m7, [base+z_filter_s] + vpbroadcastd m8, [base+z_filter_k-4+r5*4+12*0] + pminub m7, [base+z_filter_s+8] + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m10, [base+z_filter_k-4+r5*4+12*2] + pshufb m0, m2, m1 + shufps m1, m7, q2121 + pmaddubsw m0, m8 + pshufb m1, m2, m1 + pmaddubsw m1, m9 + pshufb m2, m7 + pmaddubsw m2, m10 + paddw m0, m1 + paddw m0, m2 + pmulhrsw m0, m3 + mov r3d, 9 + mov tlq, rsp + cmp hd, 4 + cmovne maxbased, r3d + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + mova [tlq], xm0 +.w4_main: + movd xm6, dxd + vpbroadcastq m0, [z_base_inc] ; base_inc << 6 + vpbroadcastb m7, [tlq+maxbaseq] + shl maxbased, 6 + vpbroadcastw m6, xm6 + mov r3d, dxd ; xpos + movd xm9, maxbased + vpbroadcastw m9, xm9 + vbroadcasti128 m8, [z1_shuf_w4] + psrlw m7, 8 ; top[max_base_x] + paddw m10, m6, m6 + psubw m9, m0 ; max_base_x + vpblendd m6, m10, 0xcc + mova xm0, xm10 + paddw m6, m0 ; xpos2 xpos3 xpos0 xpos1 + paddw m10, m10 +.w4_loop: + lea r5d, [r3+dxq] + shr r3d, 6 ; base0 + vpbroadcastq m1, [tlq+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + vpbroadcastq m2, [tlq+r5] + lea r5d, [r3+dxq] + shr r3d, 6 ; base2 + movq xm0, [tlq+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base3 + movhps xm0, [tlq+r5] + vpblendd m1, m2, 0xc0 + pand m2, m4, m6 ; frac + vpblendd m0, m1, 0xf0 + psubw m1, m5, m2 ; 64-frac + psllw m2, 8 + pshufb m0, m8 + por m1, m2 ; 64-frac, frac + pmaddubsw m0, m1 + pcmpgtw m1, m9, m6 ; base < max_base_x + pmulhrsw m0, m3 + paddw m6, m10 ; xpos += dx + lea r5, [dstq+strideq*2] + vpblendvb m0, m7, m0, m1 + packuswb m0, m0 + vextracti128 xm1, m0, 1 + movd [r5 +strideq*0], xm0 + pextrd [r5 +strideq*1], xm0, 1 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + sub hd, 4 + jz .w4_end + lea dstq, [dstq+strideq*4] + cmp r3d, maxbased + jb .w4_loop + packuswb xm7, xm7 + lea r6, [strideq*3] +.w4_end_loop: + movd [dstq+strideq*0], xm7 + movd [dstq+strideq*1], xm7 + movd [dstq+strideq*2], xm7 + movd [dstq+r6 ], xm7 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_end_loop +.w4_end: + RET +ALIGN function_align +.w8: + lea r3d, [angleq+216] + mov r3b, hb + cmp r3d, 8 + ja .w8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || h > 8 + %assign stack_offset org_stack_offset + ALLOC_STACK -32, 8 + movu xm2, [z_filter_s+6] + mova xm0, [tlq-1] + movd xm6, hd + vinserti128 m0, [tlq+7], 1 + vpbroadcastb xm6, xm6 + vbroadcasti128 m1, [z_upsample1] + pminub xm6, xm2 + vpbroadcastd m7, [pb_36_m4] + vinserti128 m2, xm6, 1 + add dxd, dxd + pshufb m1, m0, m1 + pshufb m2, m0, m2 + movd xm6, dxd + pmaddubsw m1, m7 + pmaddubsw m2, m7 + vpbroadcastw m6, xm6 + mov r3d, dxd + psrldq m0, 1 + lea r2, [strideq*3] + paddw m7, m6, m6 + paddw m1, m2 + vpblendd m6, m7, 0xf0 + pmulhrsw m1, m3 + pslldq m2, m7, 8 + paddw m7, m7 + paddw m6, m2 + packuswb m1, m1 + punpcklbw m0, m1 + mova [rsp], m0 +.w8_upsample_loop: + lea r5d, [r3+dxq] + shr r3d, 6 ; base0 + movu xm0, [rsp+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + vinserti128 m0, [rsp+r5], 1 + lea r5d, [r3+dxq] + shr r3d, 6 ; base2 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + punpcklqdq m1, m2, m2 ; frac0 frac1 + pmaddubsw m0, m1 + movu xm1, [rsp+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base3 + vinserti128 m1, [rsp+r5], 1 + punpckhqdq m2, m2 ; frac2 frac3 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + paddw m6, m7 + pmulhrsw m1, m3 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*2], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+r2 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_upsample_loop + RET +.w8_no_intra_edge_filter: + and maxbased, 7 + or maxbased, 8 ; imin(h+7, 15) + jmp .w8_main +.w8_no_upsample: + %assign stack_offset org_stack_offset + ALLOC_STACK -32, 10 + lea maxbased, [hq+7] + test angled, 0x400 + jnz .w8_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .w8_main ; filter_strength == 0 + popcnt r5d, r5d + movu xm2, [tlq] + pminub xm1, xm0, [base+z_filter_s+14] + vinserti128 m2, [tlq-1], 1 + vinserti128 m1, [base+z_filter_s+ 0], 1 + vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*0] + pminub xm0, [base+z_filter_s+22] + vinserti128 m0, [base+z_filter_s+ 8], 1 + pshufb m6, m2, m1 + pmaddubsw m6, m7 + vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*1] + movzx r3d, byte [tlq+15] + shufps m1, m0, q2121 + pshufb m1, m2, m1 + pmaddubsw m1, m7 + paddw m1, m6 + sub r5d, 3 + jnz .w8_3tap + ; filter_strength == 3 uses a 5-tap filter instead of a 3-tap one, + ; which also results in an awkward edge case where out[w*2] is + ; slightly different from out[max_base_x] when h > w. + vpbroadcastd m7, [z_filter_k+4*8] + movzx r2d, byte [tlq+14] + pshufb m2, m0 + pmaddubsw m2, m7 + sub r2d, r3d + lea r2d, [r2+r3*8+4] + shr r2d, 3 ; (tlq[w*2-2] + tlq[w*2-1]*7 + 4) >> 3 + mov [rsp+16], r2b + paddw m1, m2 +.w8_3tap: + pmulhrsw m1, m3 + sar r5d, 1 + mov tlq, rsp + add r5d, 17 ; w*2 + (filter_strength == 3) + cmp hd, 16 + cmovns maxbased, r5d + mov [tlq+r5], r3b + vextracti128 xm0, m1, 1 + packuswb xm0, xm1 + mova [tlq], xm0 +.w8_main: + movd xm2, dxd + vbroadcasti128 m0, [z_base_inc] + vpbroadcastw m2, xm2 + vpbroadcastb m7, [tlq+maxbaseq] + shl maxbased, 6 + movd xm9, maxbased + vbroadcasti128 m8, [z_filter_s+2] + vpbroadcastw m9, xm9 + psrlw m7, 8 + psubw m9, m0 + mov r3d, dxd + paddw m6, m2, m2 + vpblendd m2, m6, 0xf0 +.w8_loop: + lea r5d, [r3+dxq] + shr r3d, 6 + pand m0, m4, m2 + psubw m1, m5, m0 + psllw m0, 8 + por m1, m0 + movu xm0, [tlq+r3] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + vinserti128 m0, [tlq+r5], 1 + pshufb m0, m8 + pmaddubsw m0, m1 + pcmpgtw m1, m9, m2 + paddw m2, m6 + pmulhrsw m0, m3 + vpblendvb m0, m7, m0, m1 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + sub hd, 2 + jz .w8_end + lea dstq, [dstq+strideq*2] + cmp r3d, maxbased + jb .w8_loop + packuswb xm7, xm7 +.w8_end_loop: + movq [dstq+strideq*0], xm7 + movq [dstq+strideq*1], xm7 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_end_loop +.w8_end: + RET +.w16_no_intra_edge_filter: + and maxbased, 15 + or maxbased, 16 ; imin(h+15, 31) + jmp .w16_main +ALIGN function_align +.w16: + %assign stack_offset org_stack_offset + ALLOC_STACK -64, 12 + lea maxbased, [hq+15] + test angled, 0x400 + jnz .w16_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .w16_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastd m1, [base+pb_12] + vbroadcasti128 m6, [base+z_filter_s+8] + vinserti128 m2, m6, [base+z_filter_s], 0 + vinserti128 m6, [base+z_filter_s+16], 1 + mova xm10, [tlq-1] + vinserti128 m10, [tlq+3], 1 + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*0] + vbroadcasti128 m7, [base+z_filter_s+14] + vinserti128 m8, m7, [base+z_filter_s+6], 0 + vinserti128 m7, [base+z_filter_s+22], 1 + psubw m0, m1 + movu xm11, [tlq+12] + vinserti128 m11, [tlq+16], 1 + pminub m8, m0 + pminub m7, m0 + pshufb m0, m10, m2 + shufps m2, m6, q2121 + pmaddubsw m0, m9 + pshufb m1, m11, m8 + shufps m8, m7, q2121 + pmaddubsw m1, m9 + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*1] + movzx r3d, byte [tlq+31] + pshufb m2, m10, m2 + pmaddubsw m2, m9 + pshufb m8, m11, m8 + pmaddubsw m8, m9 + paddw m0, m2 + paddw m1, m8 + sub r5d, 3 + jnz .w16_3tap + vpbroadcastd m9, [z_filter_k+4*8] + movzx r2d, byte [tlq+30] + pshufb m10, m6 + pmaddubsw m10, m9 + pshufb m11, m7 + pmaddubsw m11, m9 + sub r2d, r3d + lea r2d, [r2+r3*8+4] + shr r2d, 3 + mov [rsp+32], r2b + paddw m0, m10 + paddw m1, m11 +.w16_3tap: + pmulhrsw m0, m3 + pmulhrsw m1, m3 + sar r5d, 1 + mov tlq, rsp + add r5d, 33 + cmp hd, 32 + cmovns maxbased, r5d + mov [tlq+r5], r3b + packuswb m0, m1 + vpermq m0, m0, q3120 + mova [tlq], m0 +.w16_main: + movd xm6, dxd + vbroadcasti128 m0, [z_base_inc] + vpbroadcastb m7, [tlq+maxbaseq] + shl maxbased, 6 + vpbroadcastw m6, xm6 + movd xm9, maxbased + vbroadcasti128 m8, [z_filter_s+2] + vpbroadcastw m9, xm9 + mov r3d, dxd + psubw m9, m0 + paddw m11, m6, m6 + psubw m10, m9, m3 ; 64*8 + vpblendd m6, m11, 0xf0 +.w16_loop: + lea r5d, [r3+dxq] + shr r3d, 6 ; base0 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + movu xm0, [tlq+r3+0] + movu xm1, [tlq+r3+8] + lea r3d, [r5+dxq] + shr r5d, 6 ; base1 + vinserti128 m0, [tlq+r5+0], 1 + vinserti128 m1, [tlq+r5+8], 1 + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + pcmpgtw m1, m9, m6 + pcmpgtw m2, m10, m6 + packsswb m1, m2 + paddw m6, m11 + vpblendvb m0, m7, m0, m1 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + sub hd, 2 + jz .w16_end + lea dstq, [dstq+strideq*2] + cmp r3d, maxbased + jb .w16_loop +.w16_end_loop: + mova [dstq+strideq*0], xm7 + mova [dstq+strideq*1], xm7 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_end_loop +.w16_end: + RET +ALIGN function_align +.w32: + %assign stack_offset org_stack_offset + ALLOC_STACK -96, 15 + lea r3d, [hq+31] + mov maxbased, 63 + cmp hd, 32 + cmovs maxbased, r3d + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w32_main + vbroadcasti128 m0, [pb_0to15] + sub r3d, 29 ; h+2 + movu xm13, [tlq+29] ; 32-39 + movd xm1, r3d + movu xm14, [tlq+37] ; 40-47 + sub r3d, 8 ; h-6 + vinserti128 m14, [tlq+51], 1 ; 56-63 + vpbroadcastb xm1, xm1 + mova xm11, [tlq- 1] ; 0- 7 + vinserti128 m11, [tlq+13], 1 ; 16-23 + movd xm2, r3d + movu xm12, [tlq+ 5] ; 8-15 + vinserti128 m12, [tlq+19], 1 ; 24-31 + pminub xm1, xm0 ; clip 32x8 + mova m7, [z_filter_s+0] + pshufb xm13, xm1 + vpbroadcastd m1, [pb_12] + vpbroadcastb xm2, xm2 + vinserti128 m13, [tlq+43], 1 ; 48-55 + vinserti128 m8, m7, [z_filter_s+4], 1 + vpblendd m2, m1, 0xf0 + vinserti128 m7, [z_filter_s+12], 0 + pminub m2, m0 ; clip 32x16 and 32x(32|64) + vpbroadcastd m9, [z_filter_k+4*2+12*0] + pshufb m14, m2 + pshufb m0, m11, m8 + shufps m8, m7, q1021 + pmaddubsw m0, m9 + pshufb m2, m12, m8 + pmaddubsw m2, m9 + pshufb m1, m13, m8 + pmaddubsw m1, m9 + pshufb m6, m14, m8 + pmaddubsw m6, m9 + vpbroadcastd m9, [z_filter_k+4*2+12*1] + pshufb m10, m11, m8 + shufps m8, m7, q2121 + pmaddubsw m10, m9 + paddw m0, m10 + pshufb m10, m12, m8 + pmaddubsw m10, m9 + paddw m2, m10 + pshufb m10, m13, m8 + pmaddubsw m10, m9 + paddw m1, m10 + pshufb m10, m14, m8 + pmaddubsw m10, m9 + paddw m6, m10 + vpbroadcastd m9, [z_filter_k+4*2+12*2] + pshufb m11, m8 + pmaddubsw m11, m9 + pshufb m12, m7 + pmaddubsw m12, m9 + movzx r3d, byte [tlq+63] + movzx r2d, byte [tlq+62] + paddw m0, m11 + paddw m2, m12 + pshufb m13, m7 + pmaddubsw m13, m9 + pshufb m14, m7 + pmaddubsw m14, m9 + paddw m1, m13 + paddw m6, m14 + sub r2d, r3d + lea r2d, [r2+r3*8+4] ; edge case for 32x64 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + shr r2d, 3 + mov [rsp+64], r2b + mov tlq, rsp + mov [tlq+65], r3b + mov r3d, 65 + cmp hd, 64 + cmove maxbased, r3d + packuswb m0, m2 + packuswb m1, m6 + mova [tlq+ 0], m0 + mova [tlq+32], m1 +.w32_main: + movd xm6, dxd + vpbroadcastb m7, [tlq+maxbaseq] + shl maxbased, 6 + vpbroadcastw m6, xm6 + movd xm9, maxbased + vbroadcasti128 m8, [z_filter_s+2] + vpbroadcastw m9, xm9 + mov r5d, dxd + psubw m9, [z_base_inc] + mova m11, m6 + psubw m10, m9, m3 ; 64*8 +.w32_loop: + mov r3d, r5d + shr r3d, 6 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + movu m0, [tlq+r3+0] + movu m1, [tlq+r3+8] + add r5d, dxd + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + pcmpgtw m1, m9, m6 + pcmpgtw m2, m10, m6 + packsswb m1, m2 + paddw m6, m11 + vpblendvb m0, m7, m0, m1 + mova [dstq], m0 + dec hd + jz .w32_end + add dstq, strideq + cmp r5d, maxbased + jb .w32_loop + test hb, 1 + jz .w32_end_loop + mova [dstq], m7 + add dstq, strideq + dec hd + jz .w32_end +.w32_end_loop: + mova [dstq+strideq*0], m7 + mova [dstq+strideq*1], m7 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_end_loop +.w32_end: + RET +ALIGN function_align +.w64: + %assign stack_offset org_stack_offset + ALLOC_STACK -128, 16 + lea maxbased, [hq+63] + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w64_main + mova xm11, [tlq- 1] ; 0- 7 + vinserti128 m11, [tlq+13], 1 ; 16-23 + movu xm12, [tlq+ 5] ; 8-15 + vinserti128 m12, [tlq+19], 1 ; 24-31 + mova m7, [z_filter_s+0] + vinserti128 m8, m7, [z_filter_s+4], 1 + vinserti128 m7, [z_filter_s+12], 0 + vpbroadcastd m9, [z_filter_k+4*2+12*0] + movu xm13, [tlq+29] ; 32-39 + vinserti128 m13, [tlq+43], 1 ; 48-55 + movu xm14, [tlq+37] ; 40-47 + vinserti128 m14, [tlq+51], 1 ; 56-63 + pshufb m0, m11, m8 + shufps m8, m7, q1021 + pmaddubsw m0, m9 + pshufb m2, m12, m8 + pmaddubsw m2, m9 + pshufb m1, m13, m8 + pmaddubsw m1, m9 + pshufb m6, m14, m8 + pmaddubsw m6, m9 + vpbroadcastd m9, [z_filter_k+4*2+12*1] + pshufb m10, m11, m8 + shufps m15, m8, m7, q2121 + pmaddubsw m10, m9 + paddw m0, m10 + pshufb m10, m12, m15 + pmaddubsw m10, m9 + paddw m2, m10 + pshufb m10, m13, m15 + pmaddubsw m10, m9 + paddw m1, m10 + pshufb m10, m14, m15 + pmaddubsw m10, m9 + paddw m6, m10 + vpbroadcastd m10, [z_filter_k+4*2+12*2] + pshufb m11, m15 + pmaddubsw m11, m10 + pshufb m12, m7 + pmaddubsw m12, m10 + pshufb m13, m7 + pmaddubsw m13, m10 + pshufb m14, m7 + pmaddubsw m14, m10 + paddw m0, m11 + paddw m2, m12 + paddw m1, m13 + paddw m6, m14 + movu xm11, [tlq+ 61] ; 64- 71 + vinserti128 m11, [tlq+ 75], 1 ; 80- 87 + movu xm12, [tlq+ 69] ; 72- 79 + vinserti128 m12, [tlq+ 83], 1 ; 88- 95 + movu xm13, [tlq+ 93] ; 96-103 + vinserti128 m13, [tlq+107], 1 ; 112-119 + movu xm14, [tlq+101] ; 104-111 + vinserti128 m14, [tlq+115], 1 ; 120-127 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + lea r3d, [hq-20] + mov tlq, rsp + packuswb m0, m2 + packuswb m1, m6 + vpbroadcastd xm2, [pb_14] + vbroadcasti128 m6, [pb_0to15] + mova [tlq+32*0], m0 + mova [tlq+32*1], m1 + movd xm0, r3d + vpbroadcastd m1, [pb_12] + vpbroadcastb m0, xm0 + paddb m0, m2 + pminub m0, m6 ; clip 64x16 and 64x32 + pshufb m12, m0 + pminub m1, m6 ; clip 64x64 + pshufb m14, m1 + pshufb m0, m11, m7 + pmaddubsw m0, m10 + pshufb m2, m12, m7 + pmaddubsw m2, m10 + pshufb m1, m13, m7 + pmaddubsw m1, m10 + pshufb m6, m14, m7 + pmaddubsw m6, m10 + pshufb m7, m11, m15 + pmaddubsw m7, m9 + pshufb m10, m12, m15 + pmaddubsw m10, m9 + paddw m0, m7 + pshufb m7, m13, m15 + pmaddubsw m7, m9 + paddw m2, m10 + pshufb m10, m14, m15 + pmaddubsw m10, m9 + paddw m1, m7 + paddw m6, m10 + vpbroadcastd m9, [z_filter_k+4*2+12*0] + pshufb m11, m8 + pmaddubsw m11, m9 + pshufb m12, m8 + pmaddubsw m12, m9 + pshufb m13, m8 + pmaddubsw m13, m9 + pshufb m14, m8 + pmaddubsw m14, m9 + paddw m0, m11 + paddw m2, m12 + paddw m1, m13 + paddw m6, m14 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + packuswb m0, m2 + packuswb m1, m6 + mova [tlq+32*2], m0 + mova [tlq+32*3], m1 +.w64_main: + movd xm12, dxd + vpbroadcastb m7, [tlq+maxbaseq] + lea r3d, [dxq-64] + shl maxbased, 6 + vpbroadcastw m12, xm12 + sub r3d, maxbased + vbroadcasti128 m8, [z_filter_s+2] + movd xm6, r3d + mov r5d, dxd + mova m10, [pb_1to32] + vpbroadcastd m11, [pb_32] + vpbroadcastw m6, xm6 +.w64_loop: + mov r3d, r5d + shr r3d, 6 + movu m0, [tlq+r3+ 0] + movu m1, [tlq+r3+ 8] + pand m2, m4, m6 + psubw m9, m5, m2 + psllw m2, 8 + por m9, m2 + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m9 + pmaddubsw m1, m9 + psraw m2, m6, 6 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packsswb m2, m2 + paddb m2, m10 + packuswb m0, m1 + vpblendvb m0, m7, m0, m2 + mova [dstq+ 0], m0 + movu m0, [tlq+r3+32] + movu m1, [tlq+r3+40] + add r5d, dxd + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m9 + pmaddubsw m1, m9 + paddb m2, m11 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + paddw m6, m12 + packuswb m0, m1 + vpblendvb m0, m7, m0, m2 + mova [dstq+32], m0 + dec hd + jz .w64_end + add dstq, strideq + cmp r5d, maxbased + jb .w64_loop +.w64_end_loop: + mova [dstq+ 0], m7 + mova [dstq+32], m7 + add dstq, strideq + dec hd + jg .w64_end_loop +.w64_end: + RET + +cglobal ipred_z2_8bpc, 3, 10, 16, 224, dst, stride, tl, w, h, angle, dx, dy +%define base r9-z_filter_t0 + lea r9, [ipred_z2_avx2_table] + tzcnt wd, wm + movifnidn angled, anglem + movifnidn hd, hm + lea dxq, [dr_intra_derivative-90] + movsxd wq, [r9+wq*4] + movzx dyd, angleb + xor angled, 0x400 + mov r8, dxq + sub dxq, dyq + add wq, r9 + add r9, z_filter_t0-ipred_z2_avx2_table + mova m2, [tlq-64] + mova m0, [tlq-32] + mova m1, [tlq] + and dyd, ~1 + and dxq, ~1 + movzx dyd, word [r8+dyq] ; angle - 90 + movzx dxd, word [dxq+270] ; 180 - angle + vpbroadcastd m13, [base+pw_512] + vpbroadcastd m14, [base+pw_62] + vpbroadcastd m15, [base+pw_64] + mova [rsp+ 0], m2 + mova [rsp+32], m0 + mova [rsp+64], m1 + neg dxd + neg dyd + jmp wq +.w4: + vpbroadcastq m6, [base+z2_base_inc] ; base_inc << 6 + vbroadcasti128 m10, [base+z1_shuf_w4] + vbroadcasti128 m11, [base+z2_shuf_h4] + lea r2d, [dxq+(65<<6)] ; xpos + movd xm5, dyd + mov r8d, (63-4)<<6 + mov dyq, -4 + pshuflw xm5, xm5, q0000 + pmullw xm5, [base+z2_ymul] + test angled, 0x400 + jnz .w4_main ; !enable_intra_edge_filter + lea r3d, [hq+2] + add angled, 1022 + shl r3d, 6 + test r3d, angled + jnz .w4_no_upsample_above ; angle >= 130 || h > 8 || (is_sm && h == 8) + vpbroadcastd xm3, [base+pb_4] + call .upsample_above + sub angled, 1075 ; angle - 53 + lea r3d, [hq+3] + xor angled, 0x7f ; 180 - angle + call .filter_strength + jmp .w4_filter_left +ALIGN function_align +.filter_strength: + movd xm8, r3d + mov r3d, angled + movd xm7, angled + vpbroadcastb m8, xm8 + shr r3d, 8 ; is_sm << 1 + vpbroadcastb m7, xm7 + pcmpeqb m8, [base+z_filter_wh] + mova xm9, [r9+r3*8] + pand m0, m8, m7 + pcmpgtb m0, m9 + pmovmskb r3d, m0 + ret +ALIGN function_align +.upsample_above: ; w4/w8 + pshufb xm2, xm1, [base+z_upsample1-2] + pminub xm3, [base+z_filter_s+4] + vpbroadcastd xm4, [base+pb_36_m4] + vbroadcasti128 m10, [base+pb_0to15] + pshufb xm3, xm1, xm3 + pmaddubsw xm2, xm4 + pmaddubsw xm3, xm4 + lea r2d, [r2+dxq+(1<<6)] + add dxd, dxd + paddw xm2, xm3 + pmulhrsw xm2, xm13 + sub r8d, 3<<6 + paddw m6, m6 + packuswb xm2, xm2 + punpcklbw xm1, xm2 + mova [rsp+gprsize+64], xm1 + ret +ALIGN function_align +.upsample_left: ; h4/h8 + mov r3d, hd + and r3d, 4 + movd xm2, [rsp+gprsize+64] + movddup xm0, [rsp+gprsize+56] + movd xm1, r3d + palignr xm2, xm0, 1 + vpbroadcastb xm1, xm1 + pshufb xm2, [base+z_filter_s+18] + vpbroadcastd xm3, [base+pb_36_m4] + pmaxub xm1, [base+z_upsample1-2] + pshufb xm1, xm0, xm1 + pmaddubsw xm2, xm3 + pmaddubsw xm1, xm3 + paddw xm5, xm5 + add dyq, dyq + paddw xm1, xm2 + pmulhrsw xm1, xm13 + vbroadcasti128 m11, [base+z2_upsample] + paddw xm5, xm15 + packuswb xm1, xm1 + punpcklbw xm0, xm1 + mova [rsp+gprsize+48], xm0 + ret +.w4_no_upsample_above: + lea r3d, [hq+3] + sub angled, 1112 ; angle - 90 + call .filter_strength + test r3d, r3d + jz .w4_no_filter_above + popcnt r3d, r3d + vpbroadcastd xm2, [base+pb_4] + pminub xm2, [base+z_filter_s] + vpbroadcastd xm0, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] + pshufb xm3, xm1, xm2 ; 00 01 12 23 + pshufd xm2, xm2, q0321 + pmaddubsw xm0, xm3, xm0 + pshufb xm2, xm1, xm2 ; 12 23 34 44 + pmaddubsw xm2, xm4 + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*2] + punpckhqdq xm3, xm3 ; 34 44 44 44 + pmaddubsw xm3, xm4 + movd xm4, r6m ; max_width + pminsw xm4, xm15 + vpbroadcastb xm4, xm4 + paddw xm0, xm2 + paddw xm0, xm3 + pmulhrsw xm0, xm13 + psubb xm4, [base+pb_1to32] + psrlq xm1, 8 + packuswb xm0, xm0 + vpblendvb xm0, xm1, xm4 + movd [rsp+65], xm0 +.w4_no_filter_above: + lea r3d, [hq+2] + add angled, 973 ; angle + 883 + shl r3d, 6 + test r3d, angled + jz .w4_upsample_left ; angle <= 140 || h > 8 || (is_sm && h == 8) + vpbroadcastd xm0, [base+pb_90] + psubb xm0, xm7 ; 180 - angle + pand xm0, xm8 ; reuse from previous filter_strength call + pcmpgtb xm0, xm9 + pmovmskb r3d, xm0 +.w4_filter_left: + test r3d, r3d + jz .w4_main + popcnt r3d, r3d + mov r5d, 10 + cmp hd, 16 + movu xm2, [rsp+49] + vinserti128 m2, [rsp+43], 1 + cmovs r5d, hd + xor r5d, 15 ; h == 16 ? 5 : 15 - h + movd xm0, r5d + vbroadcasti128 m1, [base+z_filter_s+12] + vbroadcasti128 m4, [base+z_filter_s+16] + vinserti128 m3, m1, [z_filter_s+8], 1 ; 56 67 78 89 9a ab bc cd 55 55 56 67 78 89 9a ab + vpblendd m1, m4, 0x0f ; 78 89 9a ab bc cd de ef 56 67 78 89 9a ab bc cd + vinserti128 m4, [base+z_filter_s+20], 0 ; 9a ab bc cd de ef ff ff 78 89 9a ab bc cd de ef + vpbroadcastb m0, xm0 + pmaxub m0, m3 + vpbroadcastd m3, [base+z_filter_k-4+r3*4+12*0] + pshufb m0, m2, m0 + pmaddubsw m0, m3 + vpbroadcastd m3, [base+z_filter_k-4+r3*4+12*1] + pshufb m1, m2, m1 + pmaddubsw m1, m3 + vpbroadcastd m3, [base+z_filter_k-4+r3*4+12*2] + pshufb m2, m4 + pmaddubsw m2, m3 + movd xm4, r7m ; max_height + pminsw xm4, xm15 + vpbroadcastb xm4, xm4 + psubb xm4, [base+pb_16to1] + paddw m1, m0 + paddw m1, m2 + pmulhrsw m1, m13 + vextracti128 xm0, m1, 1 + packuswb xm0, xm1 + vpblendvb xm0, [rsp+48], xm4 + mova [rsp+48], xm0 + jmp .w4_main +.w4_upsample_left: + call .upsample_left +.w4_main: + movd xm0, dxd + mova m12, [base+z2_y_shuf_h4] + lea r5, [rsp+56] ; left-7 + vpbroadcastw m0, xm0 + lea r9, [strideq*3] + psraw xm1, xm5, 6 + pand xm5, xm14 ; frac_y + pxor xm2, xm2 + paddw m7, m0, m0 + psubw xm4, xm2, xm1 ; base_y + vpblendd m0, m7, 0xcc + mova xm1, xm7 + punpcklwd xm4, xm2 + paddw m0, m1 ; xpos2 xpos3 xpos0 xpos1 + psubw xm1, xm15, xm5 ; 64-frac_y + psllw xm5, 8 + paddw m7, m7 + paddw m6, m0 + por xm5, xm1 ; 64-frac_y, frac_y + vpbroadcastq m5, xm5 +.w4_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + vpbroadcastq m1, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + vpbroadcastq m2, [rsp+r3] + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x2 + movq xm0, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x3 + movhps xm0, [rsp+r3] + vpblendd m1, m2, 0xc0 + pand m2, m14, m6 ; frac_x + vpblendd m0, m1, 0xf0 + psubw m1, m15, m2 ; 64-frac_x + psllw m2, 8 + pshufb m0, m10 + por m1, m2 ; 64-frac_x, frac_x + pmaddubsw m0, m1 + cmp r3d, 64 + jge .w4_toponly + mova m1, m7 ; arbitrary negative value + vpgatherdq m3, [r5+xm4], m1 + pshufb m1, m3, m11 + vpermd m1, m12, m1 + pmaddubsw m1, m5 + psraw m2, m6, 15 ; base_x < topleft + vpblendvb m0, m1, m2 +.w4_toponly: + pmulhrsw m0, m13 + paddw m6, m7 ; xpos += dx + add r5, dyq + packuswb m0, m0 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*2], xm0 + pextrd [dstq+r9 ], xm0, 1 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + sub hd, 4 + jz .w4_end + lea dstq, [dstq+strideq*4] + cmp r2d, r8d + jge .w4_loop +.w4_leftonly_loop: + mova m1, m7 + vpgatherdq m2, [r5+xm4], m1 + add r5, dyq + pshufb m0, m2, m11 + vpermd m0, m12, m0 + pmaddubsw m0, m5 + pmulhrsw m0, m13 + packuswb m0, m0 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*2], xm0 + pextrd [dstq+r9 ], xm0, 1 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_leftonly_loop +.w4_end: + RET +.w8: + vbroadcasti128 m6, [base+z2_base_inc] ; base_inc << 6 + movd xm5, dyd + vbroadcasti128 m10, [base+z_filter_s+2] + vbroadcasti128 m11, [base+z2_shuf_h4] + lea r2d, [dxq+(65<<6)] ; xpos + vpbroadcastw xm5, xm5 + mov r8d, (63-8)<<6 + mov dyq, -4 + pmullw xm5, [base+z2_ymul] + test angled, 0x400 + jnz .w8_main + lea r3d, [angleq+126] + mov r3b, hb + cmp r3d, 8 + ja .w8_no_upsample_above ; angle >= 130 || h > 8 || is_sm + vpbroadcastd xm3, [base+pb_8] + movhps [rsp+80], xm1 + call .upsample_above + sub angled, 53 ; angle - 53 + lea r3d, [hq+7] + xor angled, 0x7f ; 180 - angle + call .filter_strength + jmp .w8_filter_left +.w8_no_upsample_above: + lea r3d, [hq+7] + sub angled, 90 ; angle - 90 + call .filter_strength + test r3d, r3d + jz .w8_no_filter_above + popcnt r3d, r3d + vpbroadcastd xm3, [base+pb_8] + pminub xm3, [base+z_filter_s+8] + vpbroadcastd xm0, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] + pshufb xm2, xm1, [base+z_filter_s] ; 00 01 12 23 34 45 56 67 + pmaddubsw xm0, xm2, xm0 + pshufb xm3, xm1, xm3 ; 34 45 56 67 78 88 88 88 + shufps xm2, xm3, q2121 ; 12 23 34 45 56 67 78 88 + pmaddubsw xm2, xm4 + vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*2] + pmaddubsw xm3, xm4 + movd xm4, r6m ; max_width + pminuw xm4, xm15 + vpbroadcastb xm4, xm4 + paddw xm0, xm2 + paddw xm0, xm3 + pmulhrsw xm0, xm13 + psubb xm4, [base+pb_1to32] + psrldq xm1, 1 + packuswb xm0, xm0 + vpblendvb xm0, xm1, xm4 + movq [rsp+65], xm0 +.w8_no_filter_above: + lea r3d, [angleq-51] + mov r3b, hb + cmp r3d, 8 + jbe .w8_upsample_left ; angle > 140 && h <= 8 && !is_sm + vpbroadcastd m0, [base+pb_90] + psubb m0, m7 + pand m0, m8 + pcmpgtb m0, m9 + pmovmskb r3d, m0 +.w8_filter_left: + test r3d, r3d + jz .w8_main + popcnt r3d, r3d + vpbroadcastd m7, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd m8, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m9, [base+z_filter_k-4+r3*4+12*2] + cmp hd, 32 + jne .w8_filter_left_h16 + movu xm2, [rsp+27] + vinserti128 m2, [rsp+35], 1 + vpbroadcastd xm0, [base+pb_5] + vbroadcasti128 m3, [base+z_filter_s+ 8] + vbroadcasti128 m1, [base+z_filter_s+12] + vbroadcasti128 m4, [base+z_filter_s+16] + pmaxub m3, m0 + pshufb m3, m2, m3 + pmaddubsw m3, m7 + pshufb m1, m2, m1 + pmaddubsw m1, m8 + pshufb m2, m4 + pmaddubsw m2, m9 + paddw m3, m1 + paddw m3, m2 + pmulhrsw m3, m13 + jmp .w8_filter_left_top16 +.w8_filter_left_h16: + mov r5d, 10 + cmp hd, 16 + cmovs r5d, hd + xor r5d, 15 ; h == 16 ? 5 : 15 - h + movd xm0, r5d + vpbroadcastb m0, xm0 +.w8_filter_left_top16: + vbroadcasti128 m1, [base+z_filter_s+12] + vinserti128 m2, m1, [base+z_filter_s+8], 1 ; 56 67 78 89 9a ab bc cd 55 55 56 67 78 89 9a ab + vbroadcasti128 m4, [base+z_filter_s+16] + vpblendd m1, m4, 0x0f ; 78 89 9a ab bc cd de ef 56 67 78 89 9a ab bc cd + vinserti128 m4, [base+z_filter_s+20], 0 ; 9a ab bc cd de ef ff ff 78 89 9a ab bc cd de ef + pmaxub m0, m2 + movu xm2, [rsp+49] + vinserti128 m2, [rsp+43], 1 + pshufb m0, m2, m0 + pmaddubsw m0, m7 + movd xm7, r7m ; max_height + pshufb m1, m2, m1 + pmaddubsw m1, m8 + pshufb m2, m4 + pmaddubsw m2, m9 + pminsw xm7, xm15 + paddw m1, m0 + vpbroadcastb m7, xm7 + paddw m1, m2 + pmulhrsw m1, m13 + psubb m7, [base+pb_32to1] + packuswb m3, m1 + vpermq m3, m3, q1320 + vpblendvb m3, [rsp+32], m7 + mova [rsp+32], m3 + jmp .w8_main +.w8_upsample_left: + call .upsample_left +.w8_main: + movd xm3, dxd + lea r5, [rsp+56] ; left-7 + pshufd xm1, xm5, q3120 + pand xm5, xm14 + vpbroadcastw m3, xm3 + pxor xm0, xm0 + psubw xm2, xm15, xm5 + psraw xm1, 6 + lea r9, [strideq*3] + paddw m7, m3, m3 + psubw xm9, xm0, xm1 ; base_y + psllw xm5, 8 + punpcklwd xm8, xm9, xm0 ; base_y 0, 1, 4, 5 + vpblendd m3, m7, 0xf0 ; xpos0 xpos1 + por xm5, xm2 ; 64-frac_y, frac_y + punpckhwd xm9, xm0 ; base_y 2, 3, 6, 7 + paddw m6, m3 + vinserti128 m12, m5, xm5, 1 +.w8_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + movu xm0, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + vinserti128 m0, [rsp+r3], 1 + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x2 + movu xm1, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x3 + vinserti128 m1, [rsp+r3], 1 + pand m2, m14, m6 + paddsw m4, m6, m7 + psubw m5, m15, m2 + psllw m2, 8 + pshufb m0, m10 + por m2, m5 + pmaddubsw m0, m2 + pand m2, m14, m4 + psubw m5, m15, m2 + psllw m2, 8 + pshufb m1, m10 + por m2, m5 + pmaddubsw m1, m2 + cmp r3d, 64 + jge .w8_toponly + mova m5, m7 + vpgatherdq m3, [r5+xm9], m7 + mova m7, m5 + vpgatherdq m2, [r5+xm8], m5 + pshufb m3, m11 + pshufb m2, m11 + punpckldq m5, m2, m3 ; a0 b0 c0 d0 a1 b1 c1 d1 e0 f0 g0 h0 e1 f1 g1 h1 + punpckhdq m2, m3 ; a2 b2 c2 d2 a3 b3 c3 d3 e2 f2 g2 h2 e3 f3 g3 h3 + vpermq m5, m5, q3120 ; y0 y1 + vpermq m2, m2, q3120 ; y2 y3 + pmaddubsw m5, m12 + pmaddubsw m2, m12 + psraw m6, 15 ; base_x < topleft + vpblendvb m0, m5, m6 + psraw m3, m4, 15 + vpblendvb m1, m2, m3 +.w8_toponly: + pmulhrsw m0, m13 + pmulhrsw m1, m13 + paddw m6, m4, m7 ; xpos += dx + add r5, dyq + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*2], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+r9 ], xm1 + sub hd, 4 + jz .w8_end + lea dstq, [dstq+strideq*4] + cmp r2d, r8d + jge .w8_loop +.w8_leftonly_loop: + mova m0, m7 + vpgatherdq m5, [r5+xm9], m7 + mova m7, m0 + vpgatherdq m3, [r5+xm8], m0 + add r5, dyq + pshufb m2, m5, m11 + pshufb m1, m3, m11 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + vpermq m0, m0, q3120 + vpermq m1, m1, q3120 + pmaddubsw m0, m12 + pmaddubsw m1, m12 + pmulhrsw m0, m13 + pmulhrsw m1, m13 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*2], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+r9 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_leftonly_loop +.w8_end: + RET +.w16: + mov r8d, hd + test angled, 0x400 + jnz .w16_main + lea r3d, [hq+15] + sub angled, 90 + call .filter_strength + test r3d, r3d + jz .w16_no_filter_above + popcnt r3d, r3d + vbroadcasti128 m6, [tlq+1] + mova xm2, [base+z_filter_s] + vinserti128 m2, [base+z_filter_s+14], 1 ; 00 01 12 23 34 45 56 67 67 78 89 9a ab bc cd de + movu xm3, [base+z_filter_s+8] + vinserti128 m3, [base+z_filter_s+22], 1 ; 34 45 56 67 78 89 9a ab ab bc cd de ef ff ff ff + vpblendd m1, m6, 0xf0 + vpbroadcastd m0, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd m4, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m5, [base+z_filter_k-4+r3*4+12*2] + pshufb m2, m1, m2 + pshufb m1, m3 + pmaddubsw m0, m2, m0 + shufps m2, m1, q2121 ; 12 23 34 45 56 67 78 89 89 9a ab bc cd de ef ff + pmaddubsw m2, m4 + pmaddubsw m1, m5 + movd xm4, r6m ; max_width + pminsw xm4, xm15 + vpbroadcastb xm4, xm4 + paddw m0, m2 + paddw m0, m1 + pmulhrsw m0, m13 + psubb xm4, [base+pb_1to32] + vextracti128 xm2, m0, 1 + packuswb xm0, xm2 + vpblendvb xm0, xm6, xm4 + movu [rsp+65], xm0 +.w16_no_filter_above: + vpbroadcastd m0, [base+pb_90] + psubb m0, m7 + pand m0, m8 + pcmpgtb m0, m9 + pmovmskb r3d, m0 + test r3d, r3d + jz .w16_main + popcnt r3d, r3d + vpbroadcastd m7, [base+z_filter_k-4+r3*4+12*0] + vpbroadcastd m8, [base+z_filter_k-4+r3*4+12*1] + vpbroadcastd m9, [base+z_filter_k-4+r3*4+12*2] +.w16_filter_left: + movd xm6, r7m ; max_height + pminsw xm6, xm15 + vpbroadcastb m6, xm6 + cmp hd, 32 + jl .w16_filter_left_h16 + vpbroadcastd xm0, [base+pb_5] + vbroadcasti128 m10, [base+z_filter_s+ 8] + vbroadcasti128 m11, [base+z_filter_s+12] + vbroadcasti128 m12, [base+z_filter_s+16] + je .w16_filter_left_h32 + movu m3, [tlq-69] + movu m5, [tlq-61] + pmaxub m1, m10, m0 + pshufb m1, m3, m1 + pmaddubsw m1, m7 + pshufb m2, m3, m11 + pmaddubsw m2, m8 + pshufb m3, m12 + pmaddubsw m3, m9 + paddw m1, m2 + pshufb m2, m5, m10 + pmaddubsw m2, m7 + pshufb m4, m5, m11 + pmaddubsw m4, m8 + pshufb m5, m12 + pmaddubsw m5, m9 + paddw m1, m3 + vpbroadcastd m3, [base+pb_32] + paddb m3, [base+pb_32to1] + paddw m2, m4 + paddw m2, m5 + pmulhrsw m1, m13 + pmulhrsw m2, m13 + psubb m3, m6, m3 + packuswb m1, m2 + vpblendvb m1, [tlq-64], m3 + mova [rsp], m1 + jmp .w16_filter_left_top32 +.w16_filter_left_h32: + pmaxub m10, m0 +.w16_filter_left_top32: + movu xm2, [tlq-37] + vinserti128 m2, [tlq-29], 1 + pshufb m3, m2, m10 + pshufb m1, m2, m11 + pshufb m2, m12 + pmaddubsw m3, m7 + pmaddubsw m1, m8 + pmaddubsw m2, m9 + paddw m3, m1 + paddw m3, m2 + pmulhrsw m3, m13 + jmp .w16_filter_left_top16 +.w16_filter_left_h16: + mov r5d, 10 + cmp hd, 16 + cmovs r5d, hd + xor r5d, 15 ; h == 16 ? 5 : 15 - h + movd xm0, r5d + vpbroadcastb m0, xm0 +.w16_filter_left_top16: + movu xm2, [tlq-15] + vinserti128 m2, [tlq-21], 1 + vbroadcasti128 m1, [base+z_filter_s+12] + vbroadcasti128 m4, [base+z_filter_s+16] + vinserti128 m5, m1, [base+z_filter_s+8], 1 ; 56 67 78 89 9a ab bc cd 34 45 56 67 78 89 9a ab + vpblendd m1, m4, 0x0f ; 78 89 9a ab bc cd de ef 56 67 78 89 9a ab bc cd + vinserti128 m4, [base+z_filter_s+20], 0 ; 9a ab bc cd de ef ff ff 78 89 9a ab bc cd de ef + pmaxub m0, m5 + pshufb m0, m2, m0 + pmaddubsw m0, m7 + pshufb m1, m2, m1 + pmaddubsw m1, m8 + pshufb m2, m4 + pmaddubsw m2, m9 + psubb m6, [base+pb_32to1] + paddw m1, m0 + paddw m1, m2 + pmulhrsw m1, m13 + packuswb m3, m1 + vpermq m3, m3, q1320 + vpblendvb m3, [tlq-32], m6 + mova [rsp+32], m3 +.w16_main: + movd xm1, dyd + vbroadcasti128 m10, [base+z_filter_s+2] + movd xm7, dxd + vbroadcasti128 m11, [base+z2_shuf_h2] + vpbroadcastw m1, xm1 + vpbroadcastw m7, xm7 + mov r7, dstq + pmullw m0, m1, [base+z2_ymul] + psllw xm1, 4 + paddw m6, m7, [base+z2_base_inc] + lea r9d, [dxq+(65<<6)] ; xpos + movd [rsp+156], xm1 +.w16_loop0: + mov r2d, r9d + mova [rsp+160], m0 + lea r5, [rsp+60] ; left-3 + mova [rsp+192], m6 + pxor m1, m1 + psraw m2, m0, 6 + pand m0, m14 + psubw m9, m1, m2 ; base_y + psubw m12, m15, m0 + punpcklwd m8, m9, m1 ; base_y 0, 1, 2, 3, 8, 9, 10, 11 + psllw m0, 8 + punpckhwd m9, m1 ; base_y 4, 5, 6, 7, 12, 13, 14, 15 + por m12, m0 ; 64-frac_y, frac_y +.w16_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + movu xm0, [rsp+r2] + vinserti128 m0, [rsp+r2+8], 1 + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + movu xm1, [rsp+r3] + vinserti128 m1, [rsp+r3+8], 1 + pand m2, m14, m6 + paddsw m5, m6, m7 + psubw m3, m15, m2 + psllw m2, 8 + pshufb m0, m10 + por m2, m3 + pmaddubsw m0, m2 + pand m2, m14, m5 + psubw m3, m15, m2 + psllw m2, 8 + pshufb m1, m10 + por m2, m3 + pmaddubsw m1, m2 + cmp r3d, 64 + jge .w16_toponly + punpckhwd m2, m5, m5 ; mask out unnecessary loads + vpgatherdd m4, [r5+m9], m2 + punpcklwd m2, m5, m5 + vpgatherdd m3, [r5+m8], m2 + pshufb m4, m11 ; e0 f0 g0 h0 e1 f1 g1 h1 m0 n0 o0 p0 m1 n1 o1 p1 + pshufb m3, m11 ; a0 b0 c0 d0 a1 b1 c1 d1 i0 j0 k0 l0 i1 j1 k1 l1 + punpcklqdq m2, m3, m4 ; y0 + punpckhqdq m3, m4 ; y1 + pmaddubsw m2, m12 + pmaddubsw m3, m12 + psraw m6, 15 ; base_x < topleft + vpblendvb m0, m2, m6 + psraw m6, m5, 15 + vpblendvb m1, m3, m6 +.w16_toponly: + pmulhrsw m0, m13 + pmulhrsw m1, m13 + paddw m6, m5, m7 ; xpos += dx + sub r5, 2 + packuswb m0, m1 + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + sub hd, 2 + jz .w16_end + lea dstq, [dstq+strideq*2] + cmp r2d, (63-16)<<6 + jge .w16_loop +.w16_leftonly_loop: + mova m0, m7 + vpgatherdd m4, [r5+m9], m7 + mova m7, m0 + vpgatherdd m3, [r5+m8], m0 + sub r5, 2 + pshufb m2, m4, m11 + pshufb m1, m3, m11 + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + pmaddubsw m0, m12 + pmaddubsw m1, m12 + pmulhrsw m0, m13 + pmulhrsw m1, m13 + packuswb m0, m1 + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_leftonly_loop +.w16_end: + sub r8d, 1<<8 + jl .w16_ret + vpbroadcastd m0, [rsp+156] + paddw m0, [rsp+160] ; base_y += 16*dy + paddw m6, m13, [rsp+192] + add r7, 16 + add r9d, 16<<6 + movzx hd, r8b + mov dstq, r7 + paddw m6, m13 ; base_x += 16*64 + jmp .w16_loop0 +.w16_ret: + RET +.w32: + mova m2, [tlq+32] + lea r8d, [hq+(1<<8)] + mova [rsp+96], m2 + test angled, 0x400 + jnz .w16_main + vpbroadcastd m7, [base+z_filter_k+4*2+12*0] + vpbroadcastd m8, [base+z_filter_k+4*2+12*1] + vpbroadcastd m9, [base+z_filter_k+4*2+12*2] + mova xm5, [base+z_filter_s] + vinserti128 m5, [base+z_filter_s+10], 1 ; 00 01 12 23 34 45 56 67 45 56 67 78 89 9a ab bc + vinserti128 m1, [tlq+11], 1 + movu xm6, [base+z_filter_s+12] + vinserti128 m6, [base+z_filter_s+22], 1 ; 56 67 78 89 9a ab bc cd ab bc cd de ef ff ff ff + movu xm3, [tlq+ 6] + vinserti128 m3, [tlq+17], 1 + movd xm0, r6m ; max_width + pminsw xm0, xm15 + vpbroadcastb m10, xm0 +.w32_filter_above: + pshufb m0, m1, m5 + shufps m4, m5, m6, q1021 ; 12 23 34 45 56 67 78 89 67 78 89 9a ab bc cd de + pmaddubsw m0, m7 + pshufb m2, m1, m4 + shufps m5, m6, q2132 ; 34 45 56 67 78 89 9a ab 89 9a ab bc cd de ef ff + pmaddubsw m2, m8 + pshufb m1, m5 + pmaddubsw m1, m9 + paddw m0, m2 + paddw m0, m1 + pshufb m1, m3, m4 + pmaddubsw m1, m7 + pshufb m2, m3, m5 + pmaddubsw m2, m8 + pshufb m3, m6 + pmaddubsw m3, m9 + paddw m1, m2 + paddw m1, m3 + pmulhrsw m0, m13 + pmulhrsw m1, m13 + psubb m10, [base+pb_1to32] + packuswb m0, m1 + vpblendvb m0, [tlq+1], m10 + movu [rsp+65], m0 + jmp .w16_filter_left +.w64: + mova m2, [tlq+32] + mov r3d, [tlq+64] + lea r8d, [hq+(3<<8)] + mova [rsp+ 96], m2 + mov [rsp+128], r3d + test angled, 0x400 + jnz .w16_main + vpbroadcastd m7, [base+z_filter_k+4*2+12*0] + vpbroadcastd m8, [base+z_filter_k+4*2+12*1] + vpbroadcastd m9, [base+z_filter_k+4*2+12*2] + movu xm6, [base+z_filter_s+ 4] + vinserti128 m6, [base+z_filter_s+10], 1 ; 12 23 34 45 56 67 78 89 45 56 67 78 89 9a ab bc + movu xm3, [tlq+30] + vinserti128 m3, [tlq+43], 1 + movu xm5, [base+z_filter_s+16] + vinserti128 m5, [base+z_filter_s+22], 1 ; 78 89 9a ab bc cd de ef ab bc cd de ef ff ff ff + pshufb m0, m3, m6 + shufps m4, m6, m5, q1021 ; 34 45 56 67 78 89 9a ab 67 78 89 9a ab bc cd de + pmaddubsw m0, m7 + pshufb m2, m3, m4 + shufps m6, m5, q2132 ; 56 67 78 89 9a ab bc cd 89 9a ab bc cd de ef ff + pmaddubsw m2, m8 + pshufb m3, m6 + pmaddubsw m3, m9 + paddw m0, m2 + paddw m0, m3 + movu xm2, [tlq+36] + vinserti128 m2, [tlq+49], 1 + pshufb m4, m2, m4 + pmaddubsw m4, m7 + pshufb m3, m2, m6 + pmaddubsw m3, m8 + pshufb m2, m5 + pmaddubsw m2, m9 + movd xm5, r6m ; max_width + pminsw xm5, xm15 + vpbroadcastb m10, xm5 + paddw m3, m4 + paddw m2, m3 + vpbroadcastd m3, [base+pb_32] + pmulhrsw m0, m13 + pmulhrsw m2, m13 + mova xm5, [base+z_filter_s] + vinserti128 m5, [base+z_filter_s+6], 1 + psubb m3, m10, m3 + psubb m3, [base+pb_1to32] + vinserti128 m1, [tlq+13], 1 + packuswb m0, m2 + vpblendvb m0, [tlq+33], m3 + movu xm3, [tlq+ 6] + vinserti128 m3, [tlq+19], 1 + movu [rsp+97], m0 + jmp .w32_filter_above + +cglobal ipred_z3_8bpc, 4, 9, 0, dst, stride, tl, w, h, angle, dy, org_w, maxbase + %assign org_stack_offset stack_offset + lea r6, [ipred_z3_avx2_table] + tzcnt hd, hm + movifnidn angled, anglem + lea r7, [dr_intra_derivative+45*2-1] + dec tlq + movsxd hq, [r6+hq*4] + sub angled, 180 + add hq, r6 + mov dyd, angled + neg dyd + xor angled, 0x400 + or dyq, ~0x7e + movzx dyd, word [r7+dyq] + vpbroadcastd m3, [pw_512] + vpbroadcastd m4, [pw_62] + vpbroadcastd m5, [pw_64] + mov org_wd, wd + jmp hq +.h4: + lea r7, [strideq*3] + cmp angleb, 40 + jae .h4_no_upsample + lea r4d, [angleq-1024] + sar r4d, 7 + add r4d, wd + jg .h4_no_upsample ; !enable_intra_edge_filter || w > 8 || (w == 8 && is_sm) + ALLOC_STACK -32, 9 + movu xm8, [tlq-7] + pshufb xm0, xm8, [z_upsample1-4] + vpbroadcastb xm2, xm8 + pshufb xm1, xm8, [z_filter_s+2] + mova [rsp+16], xm2 ; top[max_base_y] + vpbroadcastd xm2, [pb_36_m4] + add dyd, dyd + pmaddubsw xm0, xm2 + pmaddubsw xm1, xm2 + movd xm7, dyd + mov r2d, dyd + vpbroadcastw m7, xm7 + paddw xm1, xm0 + pmulhrsw xm1, xm3 + pslldq m6, m7, 8 + paddw xm2, xm7, xm7 + paddw m6, m7 + packuswb xm1, xm1 + paddw m6, m2 + punpcklbw xm1, xm8 + mova xm8, [z_transpose4] + psllw m7, 2 + pshufb xm1, [pb_15to0] + mova [rsp], xm1 +.h4_upsample_loop: + lea r4d, [r2+dyq] + shr r2d, 6 + vpbroadcastq m1, [rsp+r2] + lea r2d, [r4+dyq] + shr r4d, 6 + vpbroadcastq m2, [rsp+r4] + lea r4d, [r2+dyq] + shr r2d, 6 + movq xm0, [rsp+r2] + lea r2d, [r4+dyq] + shr r4d, 6 + movhps xm0, [rsp+r4] + vpblendd m1, m2, 0xc0 + pand m2, m4, m6 + vpblendd m0, m1, 0xf0 + psubw m1, m5, m2 + psllw m2, 8 + por m1, m2 + pmaddubsw m0, m1 + paddw m6, m7 + pmulhrsw m0, m3 + vextracti128 xm1, m0, 1 + packuswb xm1, xm0 + pshufb xm1, xm8 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+r7 ], xm1, 3 + add dstq, 4 + sub wd, 4 + jg .h4_upsample_loop + RET +ALIGN function_align +.filter_strength: ; h4/h8/h16 +%define base r4-z_filter_t0 + lea r4, [z_filter_t0] + movd xm0, maxbased + movd xm2, angled + shr angled, 8 ; is_sm << 1 + vpbroadcastb m0, xm0 + vpbroadcastb m2, xm2 + pcmpeqb m1, m0, [base+z_filter_wh] + pand m1, m2 + mova xm2, [r4+angleq*8] + pcmpgtb m1, m2 + pmovmskb r5d, m1 + ret +.h4_no_upsample: + %assign stack_offset org_stack_offset + ALLOC_STACK -16, 12 + mov maxbased, 7 + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h4_main + lea maxbased, [wq+3] + call .filter_strength + mov maxbased, 7 + test r5d, r5d + jz .h4_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastd m7, [base+pb_7] + vbroadcasti128 m2, [tlq-14] + pmaxub m1, m7, [base+z_filter_s-4] + vpbroadcastd m8, [base+z_filter_k-4+r5*4+12*0] + pmaxub m7, [base+z_filter_s+4] + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*1] + vpbroadcastd m10, [base+z_filter_k-4+r5*4+12*2] + pshufb m0, m2, m1 + shufps m1, m7, q2121 + pmaddubsw m0, m8 + pshufb m1, m2, m1 + pmaddubsw m1, m9 + pshufb m2, m7 + pmaddubsw m2, m10 + paddw m0, m1 + paddw m0, m2 + pmulhrsw m0, m3 + mov r4d, 9 + lea tlq, [rsp+15] + cmp wd, 4 + cmovne maxbased, r4d + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + mova [rsp], xm0 +.h4_main: + movd xm6, dyd + vpbroadcastq m0, [z_base_inc] ; base_inc << 6 + mov r4, tlq + sub tlq, 4 + neg dyq + vpbroadcastw m6, xm6 + sub r4, maxbaseq + shl maxbased, 6 + vpbroadcastb m7, [r4] + lea r4, [dyq+63] ; ypos + movd xm9, maxbased + not maxbased + vbroadcasti128 m8, [z3_shuf_w4] + add maxbased, 64 + vpbroadcastw m9, xm9 + psrlw m7, 8 ; top[max_base_y] + paddw m10, m6, m6 + psubw m9, m0 ; max_base_y + vpblendd m6, m10, 0xcc + mova xm0, xm10 + paddw m6, m0 ; ypos2 ypos3 ypos0 ypos1 + paddw m10, m10 + mova xm11, [z_transpose4] +.h4_loop: + lea r5, [r4+dyq] + sar r4, 6 ; base0 + vpbroadcastq m1, [tlq+r4] + lea r4, [r5+dyq] + sar r5, 6 ; base1 + vpbroadcastq m2, [tlq+r5] + lea r5, [r4+dyq] + sar r4, 6 ; base2 + movq xm0, [tlq+r4] + lea r4, [r5+dyq] + sar r5, 6 ; base3 + movhps xm0, [tlq+r5] + vpblendd m1, m2, 0xc0 + pand m2, m4, m6 ; frac + vpblendd m0, m1, 0xf0 + psubw m1, m5, m2 ; 64-frac + psllw m2, 8 + pshufb m0, m8 + por m1, m2 ; 64-frac, frac + pmaddubsw m0, m1 + pcmpgtw m1, m9, m6 ; base < max_base_y + pmulhrsw m0, m3 + paddw m6, m10 ; ypos += dy + vpblendvb m0, m7, m0, m1 + vextracti128 xm1, m0, 1 + packuswb xm1, xm0 + pshufb xm1, xm11 ; transpose + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+r7 ], xm1, 3 + sub wd, 4 + jz .h4_end + add dstq, 4 + cmp r4d, maxbased + jg .h4_loop + packuswb xm7, xm7 +.h4_end_loop: + movd [dstq+strideq*0], xm7 + movd [dstq+strideq*1], xm7 + movd [dstq+strideq*2], xm7 + movd [dstq+r7 ], xm7 + add dstq, 4 + sub wd, 4 + jg .h4_end_loop +.h4_end: + RET +ALIGN function_align +.h8: + lea r4d, [angleq+216] + mov r4b, wb + cmp r4d, 8 + ja .h8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || w > 8 + %assign stack_offset org_stack_offset + ALLOC_STACK -32, 8 + and r4d, 4 + mova xm0, [tlq-15] + vinserti128 m0, [tlq- 9], 1 + movd xm1, r4d + movu xm2, [z_filter_s+2] + vinserti128 m2, [z_filter_s+6], 1 + vpbroadcastb xm1, xm1 ; w & 4 + vpbroadcastd m7, [pb_36_m4] + pmaxub xm1, [z_upsample1-4] ; clip 4x8 + vinserti128 m1, [z_upsample1], 1 + add dyd, dyd + pshufb m1, m0, m1 + pshufb m2, m0, m2 + vinserti128 m0, [tlq-7], 1 + movd xm6, dyd + pmaddubsw m1, m7 + pmaddubsw m2, m7 + vpbroadcastw m6, xm6 + mov r2d, dyd + lea r5, [strideq*3] + paddw m7, m6, m6 + paddw m1, m2 + vpblendd m6, m7, 0xf0 + pmulhrsw m1, m3 + pslldq m2, m7, 8 + paddw m7, m7 + paddw m6, m2 + vbroadcasti128 m2, [pb_15to0] + packuswb m1, m1 + punpcklbw m1, m0 + pshufb m1, m2 + vextracti128 [rsp+ 0], m1, 1 + mova [rsp+16], xm1 +.h8_upsample_loop: + lea r4d, [r2+dyq] + shr r2d, 6 ; base0 + movu xm0, [rsp+r2] + lea r2d, [r4+dyq] + shr r4d, 6 ; base1 + vinserti128 m0, [rsp+r4], 1 + lea r4d, [r2+dyq] + shr r2d, 6 ; base2 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + punpcklqdq m1, m2, m2 ; frac0 frac1 + pmaddubsw m0, m1 + movu xm1, [rsp+r2] + lea r2d, [r4+dyq] + shr r4d, 6 ; base3 + vinserti128 m1, [rsp+r4], 1 + punpckhqdq m2, m2 ; frac2 frac3 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + paddw m6, m7 + pmulhrsw m1, m3 + lea r4, [dstq+strideq*4] + psllw m1, 8 + por m0, m1 + vextracti128 xm1, m0, 1 + punpcklbw xm2, xm0, xm1 + punpckhbw xm0, xm1 + movd [dstq+strideq*0], xm2 + pextrd [dstq+strideq*1], xm2, 1 + pextrd [dstq+strideq*2], xm2, 2 + pextrd [dstq+r5 ], xm2, 3 + movd [r4 +strideq*0], xm0 + pextrd [r4 +strideq*1], xm0, 1 + pextrd [r4 +strideq*2], xm0, 2 + pextrd [r4 +r5 ], xm0, 3 + add dstq, 4 + sub wd, 4 + jg .h8_upsample_loop + RET +.h8_no_intra_edge_filter: + and maxbased, 7 + or maxbased, 8 ; imin(w+7, 15) + jmp .h8_main +.h8_no_upsample: + %assign stack_offset org_stack_offset + ALLOC_STACK -32, 10 + lea maxbased, [wq+7] + test angled, 0x400 + jnz .h8_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .h8_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastd xm6, [base+pb_15] + pcmpeqb xm1, xm1 + psubusb xm6, xm0 + psubb xm6, xm1 ; w == 4 ? 5 : 1 + movu xm2, [tlq-16] + pmaxub xm1, xm6, [base+z_filter_s] + vinserti128 m2, [tlq-14], 1 + vinserti128 m1, [base+z_filter_s+12], 1 + vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*0] + pmaxub xm6, [base+z_filter_s+ 8] + vinserti128 m6, [base+z_filter_s+20], 1 + pshufb m0, m2, m1 + pmaddubsw m0, m7 + vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*1] + movzx r4d, byte [tlq-15] + shufps m1, m6, q2121 + pshufb m1, m2, m1 + pmaddubsw m1, m7 + paddw m0, m1 + sub r5d, 3 + jnz .h8_3tap + vpbroadcastd m7, [z_filter_k+4*8] + movzx r2d, byte [tlq-14] + pshufb m2, m6 + pmaddubsw m2, m7 + sub r2d, r4d + lea r2d, [r2+r4*8+4] + shr r2d, 3 + mov [rsp+15], r2b + paddw m0, m2 +.h8_3tap: + pmulhrsw m0, m3 + sar r5d, 1 + lea tlq, [rsp+31] + add r5d, 17 + cmp wd, 16 + cmovns maxbased, r5d + neg r5 + mov [tlq+r5], r4b + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + mova [tlq-15], xm0 +.h8_main: + movd xm2, dyd + vbroadcasti128 m0, [z_base_inc] + mov r4, tlq + sub tlq, 8 + neg dyq + vpbroadcastw m2, xm2 + sub r4, maxbaseq + shl maxbased, 6 + vpbroadcastb m7, [r4] + lea r4, [dyq+63] + movd xm9, maxbased + not maxbased + vbroadcasti128 m8, [z3_shuf] + add maxbased, 64 + vpbroadcastw m9, xm9 + psrlw m7, 8 + psubw m9, m0 + paddw m6, m2, m2 + vpblendd m2, m6, 0x0f +.h8_loop: + lea r5, [r4+dyq] + sar r4, 6 + pand m0, m4, m2 + psubw m1, m5, m0 + psllw m0, 8 + por m1, m0 + vbroadcasti128 m0, [tlq+r4] + lea r4, [r5+dyq] + sar r5, 6 + vinserti128 m0, [tlq+r5], 0 + sub rsp, 8*2 + pshufb m0, m8 + pmaddubsw m0, m1 + pcmpgtw m1, m9, m2 + paddw m2, m6 + pmulhrsw m0, m3 + vpblendvb m0, m7, m0, m1 + vextracti128 xm1, m0, 1 + psllw xm0, 8 + por xm0, xm1 ; interleave rows (partial transpose) + mova [rsp], xm0 + sub wd, 2 + jz .h8_transpose + cmp r4d, maxbased + jg .h8_loop + packuswb xm0, xm7, xm7 +.h8_end_loop: + sub rsp, 8*2 + mova [rsp], xm0 + sub wd, 2 + jg .h8_end_loop +.h8_transpose: + mova xm2, [rsp+16*1] + sub org_wd, 8 + lea r2, [strideq*3] + lea r6, [dstq+org_wq] + cmovns dstq, r6 + punpcklwd xm1, xm2, xm0 + punpckhwd xm2, xm0 + lea r6, [dstq+strideq*4] + jge .h8_w8 + add rsp, 16*2 + movd [dstq+strideq*0], xm1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+r2 ], xm1, 3 + movd [r6 +strideq*0], xm2 + pextrd [r6 +strideq*1], xm2, 1 + pextrd [r6 +strideq*2], xm2, 2 + pextrd [r6 +r2 ], xm2, 3 + jmp .h8_end +.h8_w8_loop: + mova xm0, [rsp+16*0] + mova xm2, [rsp+16*1] + punpcklwd xm1, xm2, xm0 + punpckhwd xm2, xm0 +.h8_w8: ; w8/w16/w32 + mova xm0, [rsp+16*2] + mova xm4, [rsp+16*3] + add rsp, 16*4 + punpcklwd xm3, xm4, xm0 + punpckhwd xm4, xm0 + punpckldq xm0, xm3, xm1 + punpckhdq xm3, xm1 + punpckldq xm1, xm4, xm2 + punpckhdq xm4, xm2 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm3 + movhps [dstq+r2 ], xm3 + movq [r6 +strideq*0], xm1 + movhps [r6 +strideq*1], xm1 + movq [r6 +strideq*2], xm4 + movhps [r6 +r2 ], xm4 + sub dstq, 8 + sub r6, 8 + sub org_wd, 8 + jge .h8_w8_loop +.h8_end: + RET +.h16_no_intra_edge_filter: + and maxbased, 15 + or maxbased, 16 ; imin(w+15, 31) + jmp .h16_main +ALIGN function_align +.h16: + %assign stack_offset org_stack_offset + ALLOC_STACK -64, 12 + lea maxbased, [wq+15] + test angled, 0x400 + jnz .h16_no_intra_edge_filter + call .filter_strength + test r5d, r5d + jz .h16_main ; filter_strength == 0 + popcnt r5d, r5d + vpbroadcastd m11, [base+pb_27] + vpbroadcastd m1, [base+pb_1] + vbroadcasti128 m6, [base+z_filter_s+12] + vinserti128 m2, m6, [base+z_filter_s+4], 0 + vinserti128 m6, [base+z_filter_s+20], 1 + movu xm10, [tlq-18] + vinserti128 m10, [tlq-14], 1 + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*0] + vbroadcasti128 m7, [base+z_filter_s+8] + vinserti128 m8, m7, [base+z_filter_s+0], 0 + vinserti128 m7, [base+z_filter_s+16], 1 + psubusb m11, m0 + por m1, m11 + movu xm11, [tlq-32] + vinserti128 m11, [tlq-28], 1 + pmaxub m8, m1 + pmaxub m7, m1 + pshufb m0, m10, m2 + shufps m2, m6, q2121 + pmaddubsw m0, m9 + pshufb m1, m11, m8 + shufps m8, m7, q2121 + pmaddubsw m1, m9 + vpbroadcastd m9, [base+z_filter_k-4+r5*4+12*1] + movzx r4d, byte [tlq-31] + pshufb m2, m10, m2 + pmaddubsw m2, m9 + pshufb m8, m11, m8 + pmaddubsw m8, m9 + paddw m0, m2 + paddw m1, m8 + sub r5d, 3 + jnz .h16_3tap + vpbroadcastd m9, [z_filter_k+4*8] + movzx r2d, byte [tlq-30] + pshufb m10, m6 + pmaddubsw m10, m9 + pshufb m11, m7 + pmaddubsw m11, m9 + sub r2d, r4d + lea r2d, [r2+r4*8+4] + shr r2d, 3 + mov [rsp+31], r2b + paddw m0, m10 + paddw m1, m11 +.h16_3tap: + pmulhrsw m0, m3 + pmulhrsw m1, m3 + sar r5d, 1 + lea tlq, [rsp+63] + add r5d, 33 + cmp wd, 32 + cmovns maxbased, r5d + neg r5 + mov [tlq+r5], r4b + packuswb m0, m1 + vpermq m0, m0, q2031 + mova [tlq-31], m0 +.h16_main: + movd xm6, dyd + vbroadcasti128 m0, [z_base_inc] + mov r4, tlq + sub tlq, 8 + neg dyq + vpbroadcastw m6, xm6 + sub r4, maxbaseq + shl maxbased, 6 + vpbroadcastb m7, [r4] + lea r4, [dyq+63] + movd xm9, maxbased + not maxbased + vbroadcasti128 m8, [z3_shuf] + add maxbased, 64 + vpbroadcastw m9, xm9 + psubw m9, m0 + paddw m11, m6, m6 + psubw m10, m9, m3 ; 64*8 + vpblendd m6, m11, 0xf0 +.h16_loop: + lea r5, [r4+dyq] + sar r4, 6 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + movu xm0, [tlq+r4-0] + movu xm1, [tlq+r4-8] + lea r4, [r5+dyq] + sar r5, 6 + vinserti128 m0, [tlq+r5-0], 1 + vinserti128 m1, [tlq+r5-8], 1 + sub rsp, 32 + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + pcmpgtw m1, m9, m6 + pcmpgtw m2, m10, m6 + packsswb m1, m2 + paddw m6, m11 + vpblendvb m0, m7, m0, m1 + vpermq m0, m0, q3120 + mova [rsp], m0 + sub wd, 2 + jz .h16_transpose + cmp r4d, maxbased + jg .h16_loop + mova m0, m7 +.h16_end_loop: + sub rsp, 32 + mova [rsp], m7 + sub wd, 2 + jg .h16_end_loop +.h16_transpose: + mova m2, [rsp+32*1] + sub org_wd, 8 + lea r2, [strideq*3] + lea r6, [dstq+org_wq] + cmovns dstq, r6 + punpcklbw m1, m2, m0 + punpckhbw m2, m0 + lea r3, [strideq*5] + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + lea r4, [strideq+r2*2] ; stride*7 + jge .h16_w8 + add rsp, 32*2 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + vextracti128 xm0, m0, 1 + movd [dstq+strideq*4], xm1 + pextrd [dstq+r3 ], xm1, 1 + pextrd [dstq+r2*2 ], xm1, 2 + pextrd [dstq+r4 ], xm1, 3 + lea dstq, [dstq+strideq*8] + vextracti128 xm1, m1, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + movd [dstq+strideq*4], xm1 + pextrd [dstq+r3 ], xm1, 1 + pextrd [dstq+r2*2 ], xm1, 2 + pextrd [dstq+r4 ], xm1, 3 + jmp .h16_end +.h16_w8_loop: + mova m0, [rsp+32*0] + mova m2, [rsp+32*1] + punpcklbw m1, m2, m0 + punpckhbw m2, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 +.h16_w8: + mova m2, [rsp+32*2] + mova m4, [rsp+32*3] + lea r6, [dstq+strideq*8] + add rsp, 32*4 + punpcklbw m3, m4, m2 + punpckhbw m4, m2 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + punpckldq m4, m2, m0 + punpckhdq m2, m0 + punpckldq m0, m3, m1 + punpckhdq m3, m1 + movq [dstq+strideq*0], xm4 + movhps [dstq+strideq*1], xm4 + vextracti128 xm4, m4, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+r2 ], xm2 + vextracti128 xm2, m2, 1 + movq [dstq+strideq*4], xm0 + movhps [dstq+r3 ], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+r2*2 ], xm3 + movhps [dstq+r4 ], xm3 + vextracti128 xm3, m3, 1 + movq [r6+strideq*0], xm4 + movhps [r6+strideq*1], xm4 + movq [r6+strideq*2], xm2 + movhps [r6+r2 ], xm2 + movq [r6+strideq*4], xm0 + movhps [r6+r3 ], xm0 + movq [r6+r2*2 ], xm3 + movhps [r6+r4 ], xm3 + sub dstq, 8 + sub org_wd, 8 + jge .h16_w8_loop +.h16_end: + RET +ALIGN function_align +.h32: + %assign stack_offset org_stack_offset + ALLOC_STACK -96, 15 + lea maxbased, [wq+31] + and maxbased, 31 + or maxbased, 32 ; imin(w+31, 63) + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h32_main + vbroadcasti128 m0, [pb_0to15] + mov r4d, 21 + mov r5d, 3 + movu xm11, [tlq-66] ; 56-63 + vinserti128 m11, [tlq-52], 1 ; 40-47 + sub r4d, wd ; 21-w + cmovns r5d, r4d + movu xm12, [tlq-58] ; 48-55 + vinserti128 m12, [tlq-44], 1 ; 32-39 + sub r4d, 8 ; 13-w + movd xm1, r5d + movu xm13, [tlq-34] ; 24-31 + vinserti128 m13, [tlq-20], 1 ; 8-15 + movd xm2, r4d + vpbroadcastb m1, xm1 + movu xm14, [tlq-28] ; 16-23 + vinserti128 m14, [tlq-14], 1 ; 0- 7 + vpbroadcastb m2, xm2 + pmaxsb m1, m0 ; clip 16x32 and (32|64)x32 + movu m7, [z_filter_s+4] + pshufb m11, m1 + vinserti128 m8, m7, [z_filter_s+8], 1 + vinserti128 m7, [z_filter_s+16], 0 + pmaxsb m2, m0 ; clip 8x32 + vpbroadcastd m9, [z_filter_k+4*2+12*0] + pshufb m12, m2 + pshufb m0, m11, m8 + pmaddubsw m0, m9 + pshufb m2, m12, m8 + pmaddubsw m2, m9 + pshufb m1, m13, m8 + pmaddubsw m1, m9 + shufps m8, m7, q1021 + pshufb m6, m14, m8 + pmaddubsw m6, m9 + vpbroadcastd m9, [z_filter_k+4*2+12*1] + pshufb m10, m11, m8 + pmaddubsw m10, m9 + paddw m0, m10 + pshufb m10, m12, m8 + pmaddubsw m10, m9 + paddw m2, m10 + pshufb m10, m13, m8 + pmaddubsw m10, m9 + shufps m8, m7, q2121 + paddw m1, m10 + pshufb m10, m14, m8 + pmaddubsw m10, m9 + paddw m6, m10 + vpbroadcastd m9, [z_filter_k+4*2+12*2] + pshufb m11, m8 + pmaddubsw m11, m9 + pshufb m12, m8 + pmaddubsw m12, m9 + movzx r4d, byte [tlq-63] + movzx r2d, byte [tlq-62] + paddw m0, m11 + paddw m2, m12 + pshufb m13, m8 + pmaddubsw m13, m9 + pshufb m14, m7 + pmaddubsw m14, m9 + paddw m1, m13 + paddw m6, m14 + sub r2d, r4d + lea r2d, [r2+r4*8+4] ; edge case for 64x32 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + shr r2d, 3 + mov [rsp+31], r2b + lea tlq, [rsp+95] + mov [tlq-65], r4b + mov r4d, 65 + cmp wd, 64 + cmove maxbased, r4d + packuswb m0, m2 + packuswb m1, m6 + mova [tlq-63], m0 + mova [tlq-31], m1 +.h32_main: + movd xm6, dyd + mov r4, tlq + sub tlq, 8 + neg dyq + vpbroadcastw m6, xm6 + sub r4, maxbaseq + shl maxbased, 6 + vpbroadcastb m7, [r4] + lea r4, [dyq+63] + movd xm9, maxbased + not maxbased + vbroadcasti128 m8, [z3_shuf] + add maxbased, 64 + vpbroadcastw m9, xm9 + psubw m9, [z_base_inc] + mova m11, m6 + psubw m10, m9, m3 ; 64*8 +.h32_loop: + mov r5, r4 + sar r5, 6 + pand m1, m4, m6 + psubw m2, m5, m1 + psllw m1, 8 + por m2, m1 + movu xm0, [tlq+r5- 0] + vinserti128 m0, [tlq+r5-16], 1 + movu xm1, [tlq+r5- 8] + vinserti128 m1, [tlq+r5-24], 1 + sub rsp, 32 + add r4, dyq + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + pcmpgtw m1, m9, m6 + pcmpgtw m2, m10, m6 + packsswb m1, m2 + paddw m6, m11 + vpblendvb m0, m7, m0, m1 + mova [rsp], m0 + dec wd + jz .h32_transpose + cmp r4d, maxbased + jg .h32_loop +.h32_end_loop: + sub rsp, 32 + mova [rsp], m7 + dec wd + jg .h32_end_loop +.h32_transpose: + lea dstq, [dstq+org_wq-8] + lea r2, [strideq*3] + lea r3, [strideq*5] + lea r4, [strideq+r2*2] ; stride*7 +.h32_w8_loop: + mova m7, [rsp+32*0] + mova m6, [rsp+32*1] + mova m5, [rsp+32*2] + mova m4, [rsp+32*3] + mova m3, [rsp+32*4] + mova m2, [rsp+32*5] + mova m1, [rsp+32*6] + mova m0, [rsp+32*7] + lea r6, [dstq+strideq*8] + add rsp, 32*8 + punpcklbw m8, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + punpcklbw m3, m4, m5 + punpckhbw m4, m5 + punpcklbw m5, m6, m7 + punpckhbw m6, m7 + punpcklwd m7, m8, m1 + punpckhwd m8, m1 + punpcklwd m1, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m6 + punpckhwd m4, m6 + punpckldq m6, m7, m2 + punpckhdq m7, m2 + punpckldq m2, m8, m3 + punpckhdq m8, m3 + punpckldq m3, m1, m5 + punpckhdq m1, m5 + punpckldq m5, m0, m4 + punpckhdq m0, m4 + movq [dstq+strideq*0], xm6 + movhps [dstq+strideq*1], xm6 + vextracti128 xm6, m6, 1 + movq [dstq+strideq*2], xm7 + movhps [dstq+r2 ], xm7 + vextracti128 xm7, m7, 1 + movq [dstq+strideq*4], xm2 + movhps [dstq+r3 ], xm2 + vextracti128 xm2, m2, 1 + movq [dstq+r2*2 ], xm8 + movhps [dstq+r4 ], xm8 + vextracti128 xm8, m8, 1 + movq [r6+strideq*0], xm3 + movhps [r6+strideq*1], xm3 + vextracti128 xm3, m3, 1 + movq [r6+strideq*2], xm1 + movhps [r6+r2 ], xm1 + vextracti128 xm1, m1, 1 + movq [r6+strideq*4], xm5 + movhps [r6+r3 ], xm5 + vextracti128 xm5, m5, 1 + movq [r6+r2*2 ], xm0 + movhps [r6+r4 ], xm0 + lea r6, [r6+strideq*8] + vextracti128 xm0, m0, 1 + movq [r6+strideq*0], xm6 + movhps [r6+strideq*1], xm6 + movq [r6+strideq*2], xm7 + movhps [r6+r2 ], xm7 + movq [r6+strideq*4], xm2 + movhps [r6+r3 ], xm2 + movq [r6+r2*2 ], xm8 + movhps [r6+r4 ], xm8 + lea r6, [r6+strideq*8] + movq [r6+strideq*0], xm3 + movhps [r6+strideq*1], xm3 + movq [r6+strideq*2], xm1 + movhps [r6+r2 ], xm1 + movq [r6+strideq*4], xm5 + movhps [r6+r3 ], xm5 + movq [r6+r2*2 ], xm0 + movhps [r6+r4 ], xm0 + sub dstq, 8 + sub org_wd, 8 + jg .h32_w8_loop + RET +ALIGN function_align +.h64: + %assign stack_offset org_stack_offset + ALLOC_STACK -128, 16 + lea maxbased, [wq+63] + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h64_main + mov r4d, 21 + vpbroadcastb xm11, [tlq-127] + vpblendd xm11, [tlq-130], 0x0e ; 120-127 + sub r4d, wd ; 21-w + mov r5d, 3 + vinserti128 m11, [tlq-116], 1 ; 104-111 + movu m7, [z_filter_s+4] + cmp wd, 32 + cmove r4d, r5d + vinserti128 m8, m7, [z_filter_s+8], 1 + vbroadcasti128 m6, [pb_0to15] + movd xm1, r4d + vpbroadcastd m9, [z_filter_k+4*2+12*0] + movu xm12, [tlq-122] ; 112-119 + vinserti128 m12, [tlq-108], 1 ; 96-103 + vpbroadcastb m1, xm1 + movu xm13, [tlq- 98] ; 88- 95 + vinserti128 m13, [tlq- 84], 1 ; 72- 79 + movu xm14, [tlq- 90] ; 80- 87 + vinserti128 m14, [tlq- 76], 1 ; 64- 71 + vinserti128 m7, [z_filter_s+16], 0 + pshufb m0, m11, m8 + pmaddubsw m0, m9 + pshufb m2, m12, m8 + pmaddubsw m2, m9 + pmaxsb m1, m6 ; clip (16|32)x64 + pshufb m13, m1 + pshufb m1, m13, m8 + pmaddubsw m1, m9 + pshufb m6, m14, m8 + pmaddubsw m6, m9 + vpbroadcastd m9, [z_filter_k+4*2+12*1] + shufps m15, m8, m7, q1021 + pshufb m10, m11, m15 + pmaddubsw m10, m9 + paddw m0, m10 + pshufb m10, m12, m15 + pmaddubsw m10, m9 + paddw m2, m10 + pshufb m10, m13, m15 + pmaddubsw m10, m9 + paddw m1, m10 + pshufb m10, m14, m15 + pmaddubsw m10, m9 + paddw m6, m10 + vpbroadcastd m9, [z_filter_k+4*2+12*2] + shufps m10, m8, m7, q2132 + pshufb m11, m10 + pmaddubsw m11, m9 + pshufb m12, m10 + pmaddubsw m12, m9 + pshufb m13, m10 + pmaddubsw m13, m9 + pshufb m14, m10 + pmaddubsw m14, m9 + paddw m0, m11 + paddw m2, m12 + paddw m1, m13 + paddw m6, m14 + movu xm11, [tlq-66] ; 56-63 + vinserti128 m11, [tlq-52], 1 ; 40-47 + movu xm12, [tlq-58] ; 48-55 + vinserti128 m12, [tlq-44], 1 ; 32-39 + movu xm13, [tlq-34] ; 24-31 + vinserti128 m13, [tlq-20], 1 ; 8-15 + movu xm14, [tlq-28] ; 16-23 + vinserti128 m14, [tlq-14], 1 ; 0- 7 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + lea tlq, [rsp+127] + packuswb m0, m2 + packuswb m1, m6 + mova [tlq-127], m0 + mova [tlq- 95], m1 + pshufb m0, m11, m10 + pmaddubsw m0, m9 + pshufb m2, m12, m10 + pmaddubsw m2, m9 + pshufb m1, m13, m10 + pmaddubsw m1, m9 + pshufb m6, m14, m7 + pmaddubsw m6, m9 + vpbroadcastd m9, [z_filter_k+4*2+12*1] + pshufb m7, m11, m15 + pmaddubsw m7, m9 + paddw m0, m7 + pshufb m7, m12, m15 + pmaddubsw m7, m9 + paddw m2, m7 + pshufb m7, m13, m15 + pmaddubsw m7, m9 + paddw m1, m7 + pshufb m7, m14, m10 + pmaddubsw m7, m9 + paddw m6, m7 + vpbroadcastd m9, [z_filter_k+4*2+12*0] + pshufb m11, m8 + pmaddubsw m11, m9 + pshufb m12, m8 + pmaddubsw m12, m9 + pshufb m13, m8 + pmaddubsw m13, m9 + pshufb m14, m15 + pmaddubsw m14, m9 + paddw m0, m11 + paddw m2, m12 + paddw m1, m13 + paddw m6, m14 + pmulhrsw m0, m3 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + pmulhrsw m6, m3 + packuswb m0, m2 + packuswb m1, m6 + mova [tlq-63], m0 + mova [tlq-31], m1 +.h64_main: + movd xm12, dyd + neg maxbaseq + vbroadcasti128 m8, [z3_shuf] + vpbroadcastb m7, [tlq+maxbaseq] + shl maxbased, 6 + vpbroadcastw m12, xm12 + lea r5d, [dyq+maxbaseq-64] + neg dyq + or maxbased, 63 + lea r4, [dyq+63] + movd xm6, r5d + mova xm10, [pb_1to32+16] + vinserti128 m10, [pb_1to32], 1 + vpbroadcastd m11, [pb_32] + vpbroadcastw m6, xm6 +.h64_loop: + mov r5, r4 + sar r5, 6 + movu m0, [tlq+r5-24] + movu m1, [tlq+r5-32] + pand m2, m4, m6 + psubw m9, m5, m2 + psllw m2, 8 + por m9, m2 + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m9 + pmaddubsw m1, m9 + psraw m2, m6, 6 + sub rsp, 64 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packsswb m2, m2 + paddb m2, m10 + packuswb m0, m1 + vpblendvb m0, m7, m0, m2 + mova [rsp+32], m0 + movu m0, [tlq+r5-56] + movu m1, [tlq+r5-64] + add r4, dyq + pshufb m0, m8 + pshufb m1, m8 + pmaddubsw m0, m9 + pmaddubsw m1, m9 + paddb m2, m11 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + paddw m6, m12 + packuswb m0, m1 + vpblendvb m0, m7, m0, m2 + mova [rsp], m0 + dec wd + jz .h64_transpose + cmp r4d, maxbased + jg .h64_loop +.h64_end_loop: + sub rsp, 64 + mova [rsp+32], m7 + mova [rsp+ 0], m7 + dec wd + jg .h64_end_loop +.h64_transpose: + lea r2, [strideq*3] + lea r3, [strideq*5] + imul r5, strideq, -8 + lea dstq, [dstq+org_wq-16] + lea r4, [strideq+r2*2] ; stride*7 +.h64_transpose_loop0: + lea r6, [rsp+16*3] +.h64_transpose_loop: + mova xm0, [r6+64*15] + vinserti128 m0, [r6+64* 7], 1 + mova xm1, [r6+64*14] + vinserti128 m1, [r6+64* 6], 1 + mova xm2, [r6+64*13] + vinserti128 m2, [r6+64* 5], 1 + mova xm3, [r6+64*12] + vinserti128 m3, [r6+64* 4], 1 + mova xm4, [r6+64*11] + vinserti128 m4, [r6+64* 3], 1 + mova xm5, [r6+64*10] + vinserti128 m5, [r6+64* 2], 1 + mova xm6, [r6+64* 9] + vinserti128 m6, [r6+64* 1], 1 + mova xm7, [r6+64* 8] + vinserti128 m7, [r6+64* 0], 1 + sub r6, 16 + punpcklbw m8, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + punpcklbw m3, m4, m5 + punpckhbw m4, m5 + punpcklbw m5, m6, m7 + punpckhbw m6, m7 + punpcklwd m7, m8, m1 + punpckhwd m8, m1 + punpcklwd m1, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m6 + punpckhwd m4, m6 + punpckldq m6, m7, m2 + punpckhdq m7, m2 + punpckldq m2, m8, m3 + punpckhdq m8, m3 + punpckldq m3, m1, m5 + punpckhdq m1, m5 + punpckldq m5, m0, m4 + punpckhdq m0, m4 + vpermq m6, m6, q3120 + vpermq m7, m7, q3120 + vpermq m2, m2, q3120 + vpermq m8, m8, q3120 + vpermq m3, m3, q3120 + vpermq m1, m1, q3120 + vpermq m5, m5, q3120 + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm6 + vextracti128 [dstq+strideq*1], m6, 1 + mova [dstq+strideq*2], xm7 + vextracti128 [dstq+r2 ], m7, 1 + mova [dstq+strideq*4], xm2 + vextracti128 [dstq+r3 ], m2, 1 + mova [dstq+r2*2 ], xm8 + vextracti128 [dstq+r4 ], m8, 1 + sub dstq, r5 + mova [dstq+strideq*0], xm3 + vextracti128 [dstq+strideq*1], m3, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+r2 ], m1, 1 + mova [dstq+strideq*4], xm5 + vextracti128 [dstq+r3 ], m5, 1 + mova [dstq+r2*2 ], xm0 + vextracti128 [dstq+r4 ], m0, 1 + sub dstq, r5 + cmp r6, rsp + jae .h64_transpose_loop + add rsp, 64*16 + lea dstq, [dstq+r5*8-16] + sub org_wd, 16 + jg .h64_transpose_loop0 +.h64_end: + RET + +%macro FILTER_XMM 4 ; dst, src, tmp, shuf +%ifnum %4 + pshufb xm%2, xm%4 +%else + pshufb xm%2, %4 +%endif + pshufd xm%1, xm%2, q0000 ; p0 p1 + pmaddubsw xm%1, xm2 + pshufd xm%3, xm%2, q1111 ; p2 p3 + pmaddubsw xm%3, xm3 + paddw xm%1, xm1 + paddw xm%1, xm%3 + pshufd xm%3, xm%2, q2222 ; p4 p5 + pmaddubsw xm%3, xm4 + paddw xm%1, xm%3 + pshufd xm%3, xm%2, q3333 ; p6 __ + pmaddubsw xm%3, xm5 + paddw xm%1, xm%3 + psraw xm%1, 4 + packuswb xm%1, xm%1 +%endmacro + +%macro FILTER_YMM 4 ; dst, src, tmp, shuf + pshufb m%2, m%4 + pshufd m%1, m%2, q0000 + pmaddubsw m%1, m2 + pshufd m%3, m%2, q1111 + pmaddubsw m%3, m3 + paddw m%1, m1 + paddw m%1, m%3 + pshufd m%3, m%2, q2222 + pmaddubsw m%3, m4 + paddw m%1, m%3 + pshufd m%3, m%2, q3333 + pmaddubsw m%3, m5 + paddw m%1, m%3 + psraw m%1, 4 + vperm2i128 m%3, m%1, m%1, 0x01 + packuswb m%1, m%3 +%endmacro + +; The ipred_filter SIMD processes 4x2 blocks in the following order which +; increases parallelism compared to doing things row by row. One redundant +; block is calculated for w8 and w16, two for w32. +; w4 w8 w16 w32 +; 1 1 2 1 2 3 5 1 2 3 5 b c d f +; 2 2 3 2 4 5 7 2 4 5 7 c e f h +; 3 3 4 4 6 7 9 4 6 7 9 e g h j +; ___ 4 ___ 4 5 ___ 6 8 9 a ___ 6 8 9 a g i j k ___ +; 5 8 8 i + +cglobal ipred_filter_8bpc, 3, 7, 0, dst, stride, tl, w, h, filter +%define base r6-ipred_filter_avx2_table + lea r6, [filter_intra_taps] + tzcnt wd, wm +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + shl filterd, 6 + add filterq, r6 + lea r6, [ipred_filter_avx2_table] + movq xm0, [tlq-3] ; _ 6 5 0 1 2 3 4 + movsxd wq, [r6+wq*4] + vpbroadcastd m1, [base+pw_8] + vbroadcasti128 m2, [filterq+16*0] + vbroadcasti128 m3, [filterq+16*1] + vbroadcasti128 m4, [filterq+16*2] + vbroadcasti128 m5, [filterq+16*3] + add wq, r6 + mov hd, hm + jmp wq +.w4: + WIN64_SPILL_XMM 9 + mova xm8, [base+filter_shuf2] + sub tlq, 3 + sub tlq, hq + jmp .w4_loop_start +.w4_loop: + pinsrd xm0, xm6, [tlq+hq], 0 + lea dstq, [dstq+strideq*2] +.w4_loop_start: + FILTER_XMM 6, 0, 7, 8 + movd [dstq+strideq*0], xm6 + pextrd [dstq+strideq*1], xm6, 1 + sub hd, 2 + jg .w4_loop + RET +ALIGN function_align +.w8: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 10 + mova m8, [base+filter_shuf1] + FILTER_XMM 7, 0, 6, [base+filter_shuf2] + vpbroadcastd m0, [tlq+4] + vpbroadcastd m6, [tlq+5] + sub tlq, 4 + sub tlq, hq + vpbroadcastq m7, xm7 + vpblendd m7, m6, 0x20 +.w8_loop: + vpbroadcastd xm6, [tlq+hq] + palignr m6, m0, 12 + vpblendd m0, m6, m7, 0xeb ; _ _ _ _ 1 2 3 4 6 5 0 _ _ _ _ _ + ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + mova xm6, xm7 + call .main + vpblendd xm6, xm7, 0x0c + pshufd xm6, xm6, q3120 + movq [dstq+strideq*0], xm6 + movhps [dstq+strideq*1], xm6 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: +%if WIN64 + %assign stack_offset stack_offset - stack_size_padded + %assign xmm_regs_used 15 + %assign stack_size_padded 0x98 + SUB rsp, stack_size_padded +%endif + sub hd, 2 + TAIL_CALL .w16_main, 0 +.w16_main: +%if WIN64 + movaps [rsp+0xa8], xmm6 + movaps [rsp+0xb8], xmm7 + movaps [rsp+0x28], xmm8 + movaps [rsp+0x38], xmm9 + movaps [rsp+0x48], xmm10 + movaps [rsp+0x58], xmm11 + movaps [rsp+0x68], xmm12 + movaps [rsp+0x78], xmm13 + movaps [rsp+0x88], xmm14 +%endif + FILTER_XMM 12, 0, 7, [base+filter_shuf2] + vpbroadcastd m0, [tlq+5] + vpblendd m0, [tlq-12], 0x14 + mova m8, [base+filter_shuf1] + vpbroadcastq m7, xm12 + vpblendd m0, m7, 0xc2 ; _ _ _ _ 1 2 3 4 6 5 0 _ _ _ _ _ + ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + call .main ; c0 d0 a1 b1 a1 b1 c0 d0 + movlps xm9, xm7, [tlq+5] ; _ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + vinserti128 m14, m8, [base+filter_shuf3], 0 + vpblendd xm12, xm7, 0x0c ; a0 b0 a1 b1 + FILTER_XMM 6, 9, 10, 14 + vpbroadcastq m6, xm6 ; a2 b2 __ __ __ __ a2 b2 + vpbroadcastd m9, [tlq+13] + vpbroadcastd m10, [tlq+12] + psrld m11, m8, 4 + vpblendd m6, m9, 0x20 ; top + sub tlq, 6 + sub tlq, hq +.w16_loop: + vpbroadcastd xm9, [tlq+hq] + palignr m9, m0, 12 + vpblendd m0, m9, m7, 0xe2 ; _ _ _ _ 1 2 3 4 6 5 0 _ _ _ _ _ + ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + mova xm13, xm7 + call .main ; e0 f0 c1 d1 c1 d1 e0 f0 + vpblendd m9, m12, m10, 0xf0 + vpblendd m12, m6, 0xc0 + pshufd m9, m9, q3333 + vpblendd m9, m6, 0xee + vpblendd m10, m9, m7, 0x0c ; _ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + FILTER_YMM 6, 10, 9, 14 ; c2 d2 a3 b3 a3 b3 c2 d2 + vpblendd m12, m6, 0x30 ; a0 b0 a1 b1 a3 b3 a2 b2 + vpermd m9, m11, m12 ; a0 a1 a2 a3 b0 b1 b2 b3 + vpblendd xm12, xm13, xm7, 0x0c ; c0 d0 c1 d1 + mova [dstq+strideq*0], xm9 + vextracti128 [dstq+strideq*1], m9, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + vpblendd xm7, xm6, xm10, 0x04 ; _ _ _ 5 _ _ _ 6 0 _ _ _ 1 2 3 4 + pshufd xm7, xm7, q1032 ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + FILTER_XMM 0, 7, 9, [base+filter_shuf1+16] + vpblendd xm6, xm0, 0x0c ; c2 d2 c3 d3 + shufps xm0, xm12, xm6, q2020 ; c0 c1 c2 c3 + shufps xm6, xm12, xm6, q3131 ; d0 d1 d2 d3 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm6 + ret +ALIGN function_align +.w32: + sub rsp, stack_size_padded + sub hd, 2 + lea r3, [dstq+16] + lea r5d, [hq-2] + call .w16_main + add tlq, r5 + mov dstq, r3 + lea r3, [strideq-4] + lea r4, [r3+strideq*2] + movq xm0, [tlq+21] + pinsrd xm0, [dstq-4], 2 + pinsrd xm0, [dstq+r3*1], 3 + FILTER_XMM 12, 0, 7, 14 ; a0 b0 a0 b0 + movq xm7, [dstq+r3*2] + pinsrd xm7, [dstq+r4], 2 + palignr xm7, xm0, 12 ; 0 _ _ _ _ _ _ _ _ _ _ 5 _ _ _ 6 + vpbroadcastd m0, [tlq+28] + vpbroadcastd m9, [tlq+29] + vbroadcasti128 m8, [base+filter_shuf1+16] + vpblendd m0, m9, 0x20 + vpblendd m0, m7, 0x0f + vpbroadcastq m7, xm12 + vpblendd m0, m7, 0xc2 ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + call .main ; c0 d0 a1 b1 a1 b1 c0 d0 + add r3, 2 + lea r4, [r4+strideq*2] + movlps xm9, xm7, [tlq+29] ; _ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + vpblendd xm12, xm7, 0x0c ; a0 b0 a1 b1 + FILTER_XMM 6, 9, 10, 14 + vpbroadcastq m6, xm6 ; a2 b2 __ __ __ __ a2 b2 + vpbroadcastd m9, [tlq+37] + vpbroadcastd m10, [tlq+36] + vpblendd m6, m9, 0x20 ; top +.w32_loop: + movq xm9, [dstq+r3*4] + pinsrd xm9, [dstq+r4], 2 +.w32_loop_last: + palignr m9, m0, 12 + vpblendd m0, m9, m7, 0xe2 ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + mova xm13, xm7 ; c0 d0 + call .main ; e0 f0 c1 d1 c1 d1 e0 f0 + vpblendd m9, m12, m10, 0xf0 + vpblendd m12, m6, 0xc0 + pshufd m9, m9, q3333 + vpblendd m9, m6, 0xee + vpblendd m10, m9, m7, 0x0c ; _ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + FILTER_YMM 6, 10, 9, 14 ; c2 d2 a3 b3 a3 b3 c2 d2 + vpblendd m12, m6, 0x30 ; a0 b0 a1 b1 a3 b3 a2 b2 + vpermd m9, m11, m12 ; a0 a1 a2 a3 b0 b1 b2 b3 + vpblendd xm12, xm13, xm7, 0x0c ; c0 d0 c1 d1 + mova [dstq+strideq*0], xm9 + vextracti128 [dstq+strideq*1], m9, 1 + lea dstq, [dstq+strideq*2] + sub r5d, 2 + jg .w32_loop + jz .w32_loop_last + vpblendd xm7, xm6, xm10, 0x04 ; _ _ _ 5 _ _ _ 6 0 _ _ _ 1 2 3 4 + pshufd xm7, xm7, q1032 ; 0 _ _ _ 1 2 3 4 _ _ _ 5 _ _ _ 6 + FILTER_XMM 0, 7, 9, [base+filter_shuf1+16] + vpblendd xm6, xm0, 0x0c ; c2 d2 c3 d3 + shufps xm0, xm12, xm6, q2020 ; c0 c1 c2 c3 + shufps xm6, xm12, xm6, q3131 ; d0 d1 d2 d3 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm6 + RET +ALIGN function_align +.main: + FILTER_YMM 7, 0, 9, 8 + ret + +%if WIN64 +DECLARE_REG_TMP 5 +%else +DECLARE_REG_TMP 7 +%endif + +%macro IPRED_CFL 1 ; ac in, unpacked pixels out + psignw m3, m%1, m1 + pabsw m%1, m%1 + pmulhrsw m%1, m2 + psignw m%1, m3 + paddw m%1, m0 +%endmacro + +cglobal ipred_cfl_top_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + lea t0, [ipred_cfl_left_avx2_table] + tzcnt wd, wm + inc tlq + movu m0, [tlq] + movifnidn hd, hm + mov r6d, 0x8000 + shrx r6d, r6d, wd + movd xm3, r6d + movsxd r6, [t0+wq*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, t0 + add t0, ipred_cfl_splat_avx2_table-ipred_cfl_left_avx2_table + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 + +cglobal ipred_cfl_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + mov hd, hm ; zero upper half + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + movu m0, [tlq] + mov t0d, 0x8000 + shrx t0d, t0d, r6d + movd xm3, t0d + lea t0, [ipred_cfl_left_avx2_table] + movsxd r6, [t0+r6*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, t0 + add t0, ipred_cfl_splat_avx2_table-ipred_cfl_left_avx2_table + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h32: + vextracti128 xm1, m0, 1 + paddw xm0, xm1 +.h16: + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 +.h8: + psrlq xm1, xm0, 32 + paddw xm0, xm1 +.h4: + pmaddwd xm0, xm2 + pmulhrsw xm0, xm3 + vpbroadcastw m0, xm0 + jmp wq + +cglobal ipred_cfl_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + movifnidn wd, wm + tzcnt r6d, hd + lea t0d, [wq+hq] + movd xm4, t0d + tzcnt t0d, t0d + movd xm5, t0d + lea t0, [ipred_cfl_avx2_table] + tzcnt wd, wd + movsxd r6, [t0+r6*4] + movsxd wq, [t0+wq*4+4*4] + pcmpeqd m3, m3 + psrlw xm4, 1 + add r6, t0 + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h4: + movd xm0, [tlq-4] + pmaddubsw xm0, xm3 + jmp wq +.w4: + movd xm1, [tlq+1] + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm1 + pmaddwd xm0, xm3 + cmp hd, 4 + jg .w4_mul + psrlw xm0, 3 + jmp .w4_end +.w4_mul: + punpckhqdq xm1, xm0, xm0 + lea r2d, [hq*2] + mov r6d, 0x55563334 + paddw xm0, xm1 + shrx r6d, r6d, r2d + psrlq xm1, xm0, 32 + paddw xm0, xm1 + movd xm1, r6d + psrlw xm0, 2 + pmulhuw xm0, xm1 +.w4_end: + vpbroadcastw m0, xm0 +.s4: + vpbroadcastw m1, alpham + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s4_loop: + mova m4, [acq] + IPRED_CFL 4 + packuswb m4, m4 + vextracti128 xm5, m4, 1 + movd [dstq+strideq*0], xm4 + pextrd [dstq+strideq*1], xm4, 1 + movd [dstq+strideq*2], xm5 + pextrd [dstq+r6 ], xm5, 1 + lea dstq, [dstq+strideq*4] + add acq, 32 + sub hd, 4 + jg .s4_loop + RET +ALIGN function_align +.h8: + movq xm0, [tlq-8] + pmaddubsw xm0, xm3 + jmp wq +.w8: + movq xm1, [tlq+1] + vextracti128 xm2, m0, 1 + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm2 + punpckhqdq xm2, xm0, xm0 + paddw xm0, xm2 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 8 + je .w8_end + mov r6d, 0x5556 + mov r2d, 0x3334 + cmp hd, 32 + cmove r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w8_end: + vpbroadcastw m0, xm0 +.s8: + vpbroadcastw m1, alpham + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s8_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + vextracti128 xm5, m4, 1 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*1], xm5 + movhps [dstq+strideq*2], xm4 + movhps [dstq+r6 ], xm5 + lea dstq, [dstq+strideq*4] + add acq, 64 + sub hd, 4 + jg .s8_loop + RET +ALIGN function_align +.h16: + mova xm0, [tlq-16] + pmaddubsw xm0, xm3 + jmp wq +.w16: + movu xm1, [tlq+1] + vextracti128 xm2, m0, 1 + pmaddubsw xm1, xm3 + psubw xm0, xm4 + paddw xm0, xm2 + paddw xm0, xm1 + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 16 + je .w16_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hb, 8|32 + cmovz r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w16_end: + vpbroadcastw m0, xm0 +.s16: + vpbroadcastw m1, alpham + pabsw m2, m1 + psllw m2, 9 +.s16_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + vpermq m4, m4, q3120 + mova [dstq+strideq*0], xm4 + vextracti128 [dstq+strideq*1], m4, 1 + lea dstq, [dstq+strideq*2] + add acq, 64 + sub hd, 2 + jg .s16_loop + RET +ALIGN function_align +.h32: + mova m0, [tlq-32] + pmaddubsw m0, m3 + jmp wq +.w32: + movu m1, [tlq+1] + pmaddubsw m1, m3 + paddw m0, m1 + vextracti128 xm1, m0, 1 + psubw xm0, xm4 + paddw xm0, xm1 + punpckhqdq xm1, xm0, xm0 + paddw xm0, xm1 + psrlq xm1, xm0, 32 + paddw xm0, xm1 + pmaddwd xm0, xm3 + psrlw xm0, xm5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x33345556 + shrx r6d, r6d, r2d + movd xm1, r6d + pmulhuw xm0, xm1 +.w32_end: + vpbroadcastw m0, xm0 +.s32: + vpbroadcastw m1, alpham + pabsw m2, m1 + psllw m2, 9 +.s32_loop: + mova m4, [acq] + mova m5, [acq+32] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + vpermq m4, m4, q3120 + mova [dstq], m4 + add dstq, strideq + add acq, 64 + dec hd + jg .s32_loop + RET + +cglobal ipred_cfl_128_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + lea t0, [ipred_cfl_splat_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [t0+wq*4] + vpbroadcastd m0, [t0-ipred_cfl_splat_avx2_table+pw_128] + add wq, t0 + movifnidn acq, acmp + jmp wq + +cglobal ipred_cfl_ac_420_8bpc, 4, 9, 5, ac, y, stride, wpad, hpad, w, h, sz, ac_bak + movifnidn hpadd, hpadm + movifnidn wd, wm + mov hd, hm + mov szd, wd + mov ac_bakq, acq + imul szd, hd + shl hpadd, 2 + sub hd, hpadd + vpbroadcastd m2, [pb_2] + pxor m4, m4 + cmp wd, 8 + jg .w16 + je .w8 + ; fall-through + + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, sz, ac_bak +.w4: + lea stride3q, [strideq*3] +.w4_loop: + movq xm0, [yq] + movq xm1, [yq+strideq] + movhps xm0, [yq+strideq*2] + movhps xm1, [yq+stride3q] + pmaddubsw xm0, xm2 + pmaddubsw xm1, xm2 + paddw xm0, xm1 + mova [acq], xm0 + paddw xm4, xm0 + lea yq, [yq+strideq*4] + add acq, 16 + sub hd, 2 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg + vpermq m0, m0, q1111 +.w4_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .calc_avg + +.w8: + lea stride3q, [strideq*3] + test wpadd, wpadd + jnz .w8_wpad +.w8_loop: + mova xm0, [yq] + mova xm1, [yq+strideq] + vinserti128 m0, [yq+strideq*2], 1 + vinserti128 m1, [yq+stride3q], 1 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 2 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg + jmp .w8_hpad +.w8_wpad: + vbroadcasti128 m3, [cfl_ac_w8_pad1_shuffle] +.w8_wpad_loop: + movq xm0, [yq] + movq xm1, [yq+strideq] + vinserti128 m0, [yq+strideq*2], 1 + vinserti128 m1, [yq+stride3q], 1 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + pshufb m0, m3 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 2 + jg .w8_wpad_loop + test hpadd, hpadd + jz .calc_avg +.w8_hpad: + vpermq m0, m0, q3232 +.w8_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 2 + jg .w8_hpad_loop + jmp .calc_avg + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + mova m0, [yq] + mova m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jg .w16_loop + test hpadd, hpadd + jz .calc_avg + jmp .w16_hpad_loop +.w16_wpad: + DEFINE_ARGS ac, y, stride, wpad, hpad, iptr, h, sz, ac_bak + lea iptrq, [ipred_cfl_ac_420_avx2_table] + shl wpadd, 2 + mova m3, [iptrq+cfl_ac_w16_pad_shuffle- \ + ipred_cfl_ac_420_avx2_table+wpadq*8-32] + movsxd wpadq, [iptrq+wpadq+4] + add iptrq, wpadq + jmp iptrq +.w16_pad3: + vpbroadcastq m0, [yq] + vpbroadcastq m1, [yq+strideq] + jmp .w16_wpad_end +.w16_pad2: + vbroadcasti128 m0, [yq] + vbroadcasti128 m1, [yq+strideq] + jmp .w16_wpad_end +.w16_pad1: + mova m0, [yq] + mova m1, [yq+strideq] + ; fall-through +.w16_wpad_end: + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + pshufb m0, m3 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jz .w16_wpad_done + jmp iptrq +.w16_wpad_done: + test hpadd, hpadd + jz .calc_avg +.w16_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 32 + dec hpadd + jg .w16_hpad_loop + ; fall-through + +.calc_avg: + vpbroadcastd m2, [pw_1] + pmaddwd m0, m4, m2 + vextracti128 xm1, m0, 1 + tzcnt r1d, szd + paddd xm0, xm1 + movd xm2, r1d + movd xm3, szd + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 + psrad xm3, 1 + psrlq xm1, xm0, 32 + paddd xm0, xm3 + paddd xm0, xm1 + psrad xm0, xm2 + vpbroadcastw m0, xm0 +.sub_loop: + mova m1, [ac_bakq] + psubw m1, m0 + mova [ac_bakq], m1 + add ac_bakq, 32 + sub szd, 16 + jg .sub_loop + RET + +cglobal ipred_cfl_ac_422_8bpc, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak + movifnidn hpadd, hpadm + movifnidn wd, wm + mov hd, hm + mov szd, wd + mov ac_bakq, acq + imul szd, hd + shl hpadd, 2 + sub hd, hpadd + vpbroadcastd m2, [pb_4] + pxor m4, m4 + pxor m5, m5 + cmp wd, 8 + jg .w16 + je .w8 + ; fall-through + + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, sz, ac_bak +.w4: + lea stride3q, [strideq*3] +.w4_loop: + movq xm1, [yq] + movhps xm1, [yq+strideq] + movq xm0, [yq+strideq*2] + movhps xm0, [yq+stride3q] + pmaddubsw xm0, xm2 + pmaddubsw xm1, xm2 + mova [acq], xm1 + mova [acq+16], xm0 + paddw xm4, xm0 + paddw xm5, xm1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg + vpermq m0, m0, q1111 +.w4_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .calc_avg + +.w8: + lea stride3q, [strideq*3] + test wpadd, wpadd + jnz .w8_wpad +.w8_loop: + mova xm1, [yq] + vinserti128 m1, [yq+strideq], 1 + mova xm0, [yq+strideq*2] + vinserti128 m0, [yq+stride3q], 1 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq], m1 + mova [acq+32], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg + jmp .w8_hpad +.w8_wpad: + vbroadcasti128 m3, [cfl_ac_w8_pad1_shuffle] +.w8_wpad_loop: + movq xm1, [yq] + vinserti128 m1, [yq+strideq], 1 + movq xm0, [yq+strideq*2] + vinserti128 m0, [yq+stride3q], 1 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pshufb m0, m3 + pshufb m1, m3 + mova [acq], m1 + mova [acq+32], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_wpad_loop + test hpadd, hpadd + jz .calc_avg +.w8_hpad: + vpermq m0, m0, q3232 +.w8_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 2 + jg .w8_hpad_loop + jmp .calc_avg + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + mova m1, [yq] + mova m0, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq], m1 + mova [acq+32], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg + jmp .w16_hpad_loop +.w16_wpad: + DEFINE_ARGS ac, y, stride, wpad, hpad, iptr, h, sz, ac_bak + lea iptrq, [ipred_cfl_ac_422_avx2_table] + shl wpadd, 2 + mova m3, [iptrq+cfl_ac_w16_pad_shuffle- \ + ipred_cfl_ac_422_avx2_table+wpadq*8-32] + movsxd wpadq, [iptrq+wpadq+4] + add iptrq, wpadq + jmp iptrq +.w16_pad3: + vpbroadcastq m1, [yq] + vpbroadcastq m0, [yq+strideq] + jmp .w16_wpad_end +.w16_pad2: + vbroadcasti128 m1, [yq] + vbroadcasti128 m0, [yq+strideq] + jmp .w16_wpad_end +.w16_pad1: + mova m1, [yq] + mova m0, [yq+strideq] + ; fall-through +.w16_wpad_end: + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pshufb m0, m3 + pshufb m1, m3 + mova [acq], m1 + mova [acq+32], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jz .w16_wpad_done + jmp iptrq +.w16_wpad_done: + test hpadd, hpadd + jz .calc_avg +.w16_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddw m4, m0 + paddw m5, m0 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop + ; fall-through + +.calc_avg: + vpbroadcastd m2, [pw_1] + pmaddwd m5, m5, m2 + pmaddwd m0, m4, m2 + paddd m0, m5 + vextracti128 xm1, m0, 1 + tzcnt r1d, szd + paddd xm0, xm1 + movd xm2, r1d + movd xm3, szd + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 + psrad xm3, 1 + psrlq xm1, xm0, 32 + paddd xm0, xm3 + paddd xm0, xm1 + psrad xm0, xm2 + vpbroadcastw m0, xm0 +.sub_loop: + mova m1, [ac_bakq] + psubw m1, m0 + mova [ac_bakq], m1 + add ac_bakq, 32 + sub szd, 16 + jg .sub_loop + RET + +cglobal ipred_cfl_ac_444_8bpc, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak + movifnidn hpadd, hpadm + movifnidn wd, wm + mov hd, hm + mov szd, wd + imul szd, hd + shl hpadd, 2 + sub hd, hpadd + pxor m4, m4 + vpbroadcastd m5, [pw_1] + tzcnt r8d, wd + lea r5, [ipred_cfl_ac_444_avx2_table] + movsxd r8, [r5+r8*4+12] + add r5, r8 + + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, sz, ac_bak + mov ac_bakq, acq + jmp r5 + +.w4: + lea stride3q, [strideq*3] + pxor xm2, xm2 +.w4_loop: + movd xm1, [yq] + movd xm0, [yq+strideq*2] + pinsrd xm1, [yq+strideq], 1 + pinsrd xm0, [yq+stride3q], 1 + punpcklbw xm1, xm2 + punpcklbw xm0, xm2 + psllw xm1, 3 + psllw xm0, 3 + mova [acq], xm1 + mova [acq+16], xm0 + paddw xm1, xm0 + paddw xm4, xm1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_mul + pshufd xm0, xm0, q3232 + paddw xm1, xm0, xm0 +.w4_hpad_loop: + mova [acq], xm0 + mova [acq+16], xm0 + paddw xm4, xm1 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .calc_avg_mul + +.w8: + lea stride3q, [strideq*3] + pxor m2, m2 +.w8_loop: + movq xm1, [yq] + movq xm0, [yq+strideq*2] + vinserti128 m1, [yq+strideq], 1 + vinserti128 m0, [yq+stride3q], 1 + punpcklbw m1, m2 + punpcklbw m0, m2 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + paddw m4, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_mul + vpermq m0, m0, q3232 + paddw m1, m0, m0 +.w8_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddw m4, m1 + add acq, 64 + sub hpadd, 4 + jg .w8_hpad_loop + jmp .calc_avg_mul + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+strideq] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg + jmp .w16_hpad +.w16_wpad: + mova m3, [cfl_ac_444_w16_pad1_shuffle] +.w16_wpad_loop: + vpbroadcastq m1, [yq] + vpbroadcastq m0, [yq+strideq] + pshufb m1, m3 + pshufb m0, m3 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_wpad_loop + test hpadd, hpadd + jz .calc_avg +.w16_hpad: + paddw m1, m0, m0 + pmaddwd m1, m5 +.w16_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddd m4, m1 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop + jmp .calc_avg + +.w32: + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+16] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jg .w32_loop + test hpadd, hpadd + jz .calc_avg + jmp .w32_hpad_loop +.w32_wpad: + DEFINE_ARGS ac, y, stride, wpad, hpad, iptr, h, sz, ac_bak + lea iptrq, [ipred_cfl_ac_444_avx2_table] + add wpadd, wpadd + mova m3, [iptrq+cfl_ac_444_w16_pad1_shuffle-ipred_cfl_ac_444_avx2_table] + movsxd wpadq, [iptrq+wpadq+4] + add iptrq, wpadq + jmp iptrq +.w32_pad3: + vpbroadcastq m1, [yq] + pshufb m1, m3 + vpermq m0, m1, q3232 + jmp .w32_wpad_end +.w32_pad2: + pmovzxbw m1, [yq] + pshufhw m0, m1, q3333 + vpermq m0, m0, q3333 + jmp .w32_wpad_end +.w32_pad1: + pmovzxbw m1, [yq] + vpbroadcastq m0, [yq+16] + pshufb m0, m3 + ; fall-through +.w32_wpad_end: + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jz .w32_wpad_done + jmp iptrq +.w32_wpad_done: + test hpadd, hpadd + jz .calc_avg +.w32_hpad_loop: + mova [acq], m1 + mova [acq+32], m0 + paddd m4, m2 + add acq, 64 + dec hpadd + jg .w32_hpad_loop + jmp .calc_avg + +.calc_avg_mul: + pmaddwd m4, m5 +.calc_avg: + vextracti128 xm1, m4, 1 + tzcnt r1d, szd + paddd xm0, xm4, xm1 + movd xm2, r1d + movd xm3, szd + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 + psrad xm3, 1 + psrlq xm1, xm0, 32 + paddd xm0, xm3 + paddd xm0, xm1 + psrad xm0, xm2 + vpbroadcastw m0, xm0 +.sub_loop: + mova m1, [ac_bakq] + psubw m1, m0 + mova [ac_bakq], m1 + add ac_bakq, 32 + sub szd, 16 + jg .sub_loop + RET + +cglobal pal_pred_8bpc, 4, 6, 5, dst, stride, pal, idx, w, h + vbroadcasti128 m4, [palq] + lea r2, [pal_pred_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r2+wq*4] + packuswb m4, m4 + add wq, r2 + lea r2, [strideq*3] + jmp wq +.w4: + pshufb xm0, xm4, [idxq] + add idxq, 16 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +ALIGN function_align +.w8: + pshufb xm0, xm4, [idxq+16*0] + pshufb xm1, xm4, [idxq+16*1] + add idxq, 16*2 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r2 ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +ALIGN function_align +.w16: + pshufb m0, m4, [idxq+32*0] + pshufb m1, m4, [idxq+32*1] + add idxq, 32*2 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+r2 ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +ALIGN function_align +.w32: + pshufb m0, m4, [idxq+32*0] + pshufb m1, m4, [idxq+32*1] + pshufb m2, m4, [idxq+32*2] + pshufb m3, m4, [idxq+32*3] + add idxq, 32*4 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r2 ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w32 + RET +ALIGN function_align +.w64: + pshufb m0, m4, [idxq+32*0] + pshufb m1, m4, [idxq+32*1] + pshufb m2, m4, [idxq+32*2] + pshufb m3, m4, [idxq+32*3] + add idxq, 32*4 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w64 + RET + +%endif diff --git a/third_party/dav1d/src/x86/ipred_avx512.asm b/third_party/dav1d/src/x86/ipred_avx512.asm new file mode 100644 index 0000000000..38c86b54f5 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred_avx512.asm @@ -0,0 +1,1432 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +%macro SMOOTH_WEIGHT_TABLE 1-* + %rep %0 + db %1-128, 127-%1 + %rotate 1 + %endrep +%endmacro + +smooth_weights: SMOOTH_WEIGHT_TABLE \ + 0, 0, 255, 128, 255, 149, 85, 64, \ + 255, 197, 146, 105, 73, 50, 37, 32, \ + 255, 225, 196, 170, 145, 123, 102, 84, \ + 68, 54, 43, 33, 26, 20, 17, 16, \ + 255, 240, 225, 210, 196, 182, 169, 157, \ + 145, 133, 122, 111, 101, 92, 83, 74, \ + 66, 59, 52, 45, 39, 34, 29, 25, \ + 21, 17, 14, 12, 10, 9, 8, 8, \ + 255, 248, 240, 233, 225, 218, 210, 203, \ + 196, 189, 182, 176, 169, 163, 156, 150, \ + 144, 138, 133, 127, 121, 116, 111, 106, \ + 101, 96, 91, 86, 82, 77, 73, 69, \ + 65, 61, 57, 54, 50, 47, 44, 41, \ + 38, 35, 32, 29, 27, 25, 22, 20, \ + 18, 16, 15, 13, 12, 10, 9, 8, \ + 7, 6, 6, 5, 5, 4, 4, 4 + +; dav1d_filter_intra_taps[], reordered for VNNI: p1 p2 p3 p4, p6 p5 p0 __ +filter_taps: db 10, 0, 0, 0, 2, 10, 0, 0, 1, 1, 10, 0, 1, 1, 2, 10 + db 6, 0, 0, 0, 2, 6, 0, 0, 2, 2, 6, 0, 1, 2, 2, 6 + db 0, 12, -6, 0, 0, 9, -5, 0, 0, 7, -3, 0, 0, 5, -3, 0 + db 12, 2, -4, 0, 9, 2, -3, 0, 7, 2, -3, 0, 5, 3, -3, 0 + db 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16 + db 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16 + db 0, 10,-10, 0, 0, 6, -6, 0, 0, 4, -4, 0, 0, 2, -2, 0 + db 10, 0,-10, 0, 6, 0, -6, 0, 4, 0, -4, 0, 2, 0, -2, 0 + db 8, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8 + db 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4 + db 0, 16, -8, 0, 0, 16, -8, 0, 0, 16, -8, 0, 0, 16, -8, 0 + db 16, 0, -4, 0, 16, 0, -4, 0, 16, 0, -4, 0, 16, 0, -4, 0 + db 8, 0, 0, 0, 3, 8, 0, 0, 2, 3, 8, 0, 1, 2, 3, 8 + db 4, 0, 0, 0, 3, 4, 0, 0, 2, 3, 4, 0, 2, 2, 3, 4 + db 0, 10, -2, 0, 0, 6, -1, 0, 0, 4, -1, 0, 0, 2, 0, 0 + db 10, 3, -1, 0, 6, 4, -1, 0, 4, 4, -1, 0, 3, 3, -1, 0 + db 14, 0, 0, 0, 0, 14, 0, 0, 0, 0, 14, 0, 0, 0, 0, 14 + db 12, 0, 0, 0, 1, 12, 0, 0, 0, 0, 12, 0, 0, 0, 1, 12 + db 0, 14,-12, 0, 0, 12,-10, 0, 0, 11, -9, 0, 0, 10, -8, 0 + db 14, 0,-10, 0, 12, 0, -9, 0, 11, 1, -8, 0, 9, 1, -7, 0 +filter_perm: db 0, 1, 2, 3, 24, 25, 26, 27, 4, 5, 6, 7, 28, 29, 30, 31 + db 15, 11, 7, 3, 15, 11, 7, 3, 15, 11, 7, 3, 15, 11, 7,131 + db 31, 27, 23, 19, 31, 27, 23, 19, 31, 27, 23, 19, 31, 27, 23,147 + db 47, 43, 39, 35, 47, 43, 39, 35, 47, 43, 39, 35, 47, 43, 39,163 +filter_end: dd 2, 3, 16, 17, -1, -1, 20, 21, 0, 6, 24, 30, 1, 7, 25, 31 +smooth_shuf: db 7, 7, 7, 7, 0, 1, 0, 1, 3, 3, 3, 3, 8, 9, 8, 9 + db 5, 5, 5, 5, 4, 5, 4, 5, 1, 1, 1, 1, 12, 13, 12, 13 + db 6, 6, 6, 6, 2, 3, 2, 3, 2, 2, 2, 2, 10, 11, 10, 11 + db 4, 4, 4, 4, 6, 7, 6, 7, 0, 0, 0, 0, 14, 15, 14, 15 +smooth_endA: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 + db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95 + db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127 +smooth_endB: db 1, 3, 5, 7, 9, 11, 13, 15, 65, 67, 69, 71, 73, 75, 77, 79 + db 17, 19, 21, 23, 25, 27, 29, 31, 81, 83, 85, 87, 89, 91, 93, 95 + db 33, 35, 37, 39, 41, 43, 45, 47, 97, 99,101,103,105,107,109,111 + db 49, 51, 53, 55, 57, 59, 61, 63,113,115,117,119,121,123,125,127 +ipred_h_shuf: db 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4 + db 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 + +pb_127_m127: times 2 db 127, -127 +pb_128: times 4 db 128 +pw_128: times 2 dw 128 +pw_255: times 2 dw 255 + +%define pb_1 (ipred_h_shuf+24) +%define pb_2 (ipred_h_shuf+20) +%define pb_3 (ipred_h_shuf+16) +%define pd_8 (filter_taps+128) + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_8bpc_avx512icl_table (ipred_dc_8bpc_avx512icl_table + 10*4) + +JMP_TABLE ipred_h_8bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_paeth_8bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_8bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_v_8bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_h_8bpc, avx512icl, w4, w8, w16, w32, w64 +JMP_TABLE ipred_dc_8bpc, avx512icl, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4 +JMP_TABLE ipred_dc_left_8bpc, avx512icl, h4, h8, h16, h32, h64 +JMP_TABLE pal_pred_8bpc, avx512icl, w4, w8, w16, w32, w64 + +SECTION .text + +INIT_ZMM avx512icl +cglobal ipred_dc_top_8bpc, 3, 7, 5, dst, stride, tl, w, h + lea r5, [ipred_dc_left_8bpc_avx512icl_table] + movd xm0, wm + tzcnt wd, wm + inc tlq + movifnidn hd, hm + movu ym1, [tlq] + movd xmm3, wd + movsxd r6, [r5+wq*4] + vpbroadcastd ym2, [r5-ipred_dc_left_8bpc_avx512icl_table+pb_1] + psrld xm0, 1 + vpdpbusd ym0, ym1, ym2 + add r6, r5 + add r5, ipred_dc_splat_8bpc_avx512icl_table-ipred_dc_left_8bpc_avx512icl_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +cglobal ipred_dc_left_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 + lea r5, [ipred_dc_left_8bpc_avx512icl_table] + mov hd, hm + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + movd xm0, hm + movu ym1, [tlq] + movd xmm3, r6d + movsxd r6, [r5+r6*4] + vpbroadcastd ym2, [r5-ipred_dc_left_8bpc_avx512icl_table+pb_1] + psrld xm0, 1 + vpdpbusd ym0, ym1, ym2 + add r6, r5 + add r5, ipred_dc_splat_8bpc_avx512icl_table-ipred_dc_left_8bpc_avx512icl_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + movu ym1, [tlq+32] ; unaligned when jumping here from dc_top + vpdpbusd ym0, ym1, ym2 +.h32: + vextracti32x4 xm1, ym0, 1 + paddd xm0, xm1 +.h16: + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 +.h8: + psrlq xm1, xm0, 32 + paddd xm0, xm1 +.h4: + vpsrlvd xm0, xmm3 + lea stride3q, [strideq*3] + vpbroadcastb m0, xm0 + jmp wq + +cglobal ipred_dc_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + movifnidn wd, wm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd xm0, r5d + tzcnt r5d, r5d + movd xmm4, r5d + lea r5, [ipred_dc_8bpc_avx512icl_table] + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+5*4] + vpbroadcastd ym3, [r5-ipred_dc_8bpc_avx512icl_table+pb_1] + psrld xm0, 1 + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movd xmm1, [tlq-4] + vpdpbusd xm0, xmm1, xm3 + jmp wq +.w4: + movd xmm1, [tlq+1] + vpdpbusd xm0, xmm1, xm3 + cmp hd, 4 + jg .w4_mul + psrlw xmm0, xm0, 3 + jmp .w4_end +.w4_mul: + punpckhqdq xmm1, xm0, xm0 + lea r2d, [hq*2] + mov r6d, 0x55563334 + paddd xmm1, xm0 + shrx r6d, r6d, r2d + psrlq xmm0, xmm1, 32 + paddd xmm0, xmm1 + movd xmm1, r6d + psrld xmm0, 2 + pmulhuw xmm0, xmm1 +.w4_end: + vpbroadcastb xm0, xmm0 +.s4: + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm0 + movd [dstq+strideq*2], xm0 + movd [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +.h8: + movq xmm1, [tlq-8] + vpdpbusd xm0, xmm1, xm3 + jmp wq +.w8: + movq xmm1, [tlq+1] + vextracti32x4 xm2, ym0, 1 + vpdpbusd xm0, xmm1, xm3 + paddd xmm2, xm2, xm0 + punpckhqdq xmm0, xmm2, xmm2 + paddd xmm0, xmm2 + psrlq xmm1, xmm0, 32 + paddd xmm0, xmm1 + vpsrlvd xmm0, xmm4 + cmp hd, 8 + je .w8_end + mov r6d, 0x5556 + mov r2d, 0x3334 + cmp hd, 32 + cmove r6d, r2d + movd xmm1, r6d + pmulhuw xmm0, xmm1 +.w8_end: + vpbroadcastb xm0, xmm0 +.s8: + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm0 + movq [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +.h16: + mova xmm1, [tlq-16] + vpdpbusd xm0, xmm1, xm3 + jmp wq +.w16: + movu xmm1, [tlq+1] + vextracti32x4 xm2, ym0, 1 + vpdpbusd xm0, xmm1, xm3 + paddd xmm2, xm2, xm0 + punpckhqdq xmm0, xmm2, xmm2 + paddd xmm0, xmm2 + psrlq xmm1, xmm0, 32 + paddd xmm0, xmm1 + vpsrlvd xmm0, xmm4 + cmp hd, 16 + je .w16_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hb, 8|32 + cmovz r6d, r2d + movd xmm1, r6d + pmulhuw xmm0, xmm1 +.w16_end: + vpbroadcastb xm0, xmm0 +.s16: + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm0 + mova [dstq+strideq*2], xm0 + mova [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +.h32: + mova ym1, [tlq-32] + vpdpbusd ym0, ym1, ym3 + jmp wq +.w32: + movu ym1, [tlq+1] + vpdpbusd ym0, ym1, ym3 + vextracti32x4 xm1, ym0, 1 + paddd xmm1, xm1, xm0 + punpckhqdq xmm0, xmm1, xmm1 + paddd xmm0, xmm1 + psrlq xmm1, xmm0, 32 + paddd xmm0, xmm1 + vpsrlvd xmm0, xmm4 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x33345556 + shrx r6d, r6d, r2d + movd xmm1, r6d + pmulhuw xmm0, xmm1 +.w32_end: + vpbroadcastb ym0, xmm0 +.s32: + mova [dstq+strideq*0], ym0 + mova [dstq+strideq*1], ym0 + mova [dstq+strideq*2], ym0 + mova [dstq+stride3q ], ym0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s32 + RET +.h64: + mova ym1, [tlq-64] + mova ym2, [tlq-32] + vpdpbusd ym0, ym1, ym3 + vpdpbusd ym0, ym2, ym3 + jmp wq +.w64: + movu ym1, [tlq+ 1] + movu ym2, [tlq+33] + vpdpbusd ym0, ym1, ym3 + vpdpbusd ym0, ym2, ym3 + vextracti32x4 xm1, ym0, 1 + paddd xmm1, xm1, xm0 + punpckhqdq xmm0, xmm1, xmm1 + paddd xmm0, xmm1 + psrlq xmm1, xmm0, 32 + paddd xmm0, xmm1 + vpsrlvd xmm0, xmm4 + cmp hd, 64 + je .w64_end + mov r6d, 0x33345556 + shrx r6d, r6d, hd + movd xmm1, r6d + pmulhuw xmm0, xmm1 +.w64_end: + vpbroadcastb m0, xmm0 +.s64: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s64 + RET + +cglobal ipred_dc_128_8bpc, 2, 7, 5, dst, stride, tl, w, h, stride3 + lea r5, [ipred_dc_splat_8bpc_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m0, [r5-ipred_dc_splat_8bpc_avx512icl_table+pb_128] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_v_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 + lea r5, [ipred_dc_splat_8bpc_avx512icl_table] + tzcnt wd, wm + movu m0, [tlq+1] + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_h_8bpc, 3, 7, 8, dst, stride, tl, w, h, stride3 +%define base r6-ipred_h_8bpc_avx512icl_table + lea r6, [ipred_h_8bpc_avx512icl_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + lea stride3q, [strideq*3] + sub tlq, hq + add wq, r6 + jmp wq +.w4: + mova xmm1, [base+ipred_h_shuf+16] +.w4_loop: + movd xmm0, [tlq+hq-4] + pshufb xmm0, xmm1 + movd [dstq+strideq*0], xmm0 + pextrd [dstq+strideq*1], xmm0, 1 + pextrd [dstq+strideq*2], xmm0, 2 + pextrd [dstq+stride3q ], xmm0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_loop + RET +.w8: + movsldup xmm2, [base+ipred_h_shuf+16] + movshdup xmm3, [base+ipred_h_shuf+16] +.w8_loop: + movd xmm1, [tlq+hq-4] + pshufb xmm0, xmm1, xmm2 + pshufb xmm1, xmm3 + movq [dstq+strideq*0], xmm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm0 + movhps [dstq+stride3q ], xmm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +.w16: + movsldup m1, [base+smooth_shuf] +.w16_loop: + vpbroadcastd m0, [tlq+hq-4] + pshufb m0, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +.w32: + vpbroadcastd ym3, [base+pb_1] + vpord m2, m3, [base+pb_2] {1to16} +.w32_loop: + vpbroadcastd m1, [tlq+hq-4] + pshufb m0, m1, m2 + pshufb m1, m3 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w32_loop + RET +.w64: + vpbroadcastd m4, [base+pb_3] + vpbroadcastd m5, [base+pb_2] + vpbroadcastd m6, [base+pb_1] + pxor m7, m7 +.w64_loop: + vpbroadcastd m3, [tlq+hq-4] + pshufb m0, m3, m4 + pshufb m1, m3, m5 + pshufb m2, m3, m6 + pshufb m3, m7 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w64_loop + RET + +%macro PAETH 0 + psubusb m1, m5, m4 + psubusb m0, m4, m5 + por m1, m0 ; tdiff + pavgb m2, m6, m4 + vpcmpub k1, m1, m7, 1 ; tdiff < ldiff + vpblendmb m0{k1}, m4, m6 + vpternlogd m4, m6, m8, 0x28 ; (m4 ^ m6) & m8 + psubusb m3, m5, m2 + psubb m2, m4 + psubusb m2, m5 + por m2, m3 + pminub m1, m7 + paddusb m2, m2 + por m2, m4 ; min(tldiff, 255) + vpcmpub k1, m2, m1, 1 ; tldiff < ldiff && tldiff < tdiff + vmovdqu8 m0{k1}, m5 +%endmacro + +cglobal ipred_paeth_8bpc, 3, 7, 10, dst, stride, tl, w, h, top, stride3 + lea r6, [ipred_paeth_8bpc_avx512icl_table] + tzcnt wd, wm + vpbroadcastb m5, [tlq] ; topleft + mov hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastd m8, [r6-ipred_paeth_8bpc_avx512icl_table+pb_1] + lea topq, [tlq+1] + sub tlq, hq + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +INIT_YMM avx512icl +.w4: + vpbroadcastd m6, [topq] + mova m9, [ipred_h_shuf] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 ; ldiff +.w4_loop: + vpbroadcastq m4, [tlq+hq-8] + pshufb m4, m9 ; left + PAETH + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+stride3q ], xm0, 3 + sub hd, 8 + jl .w4_ret + vextracti32x4 xm0, m0, 1 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+stride3q ], xm0, 3 + lea dstq, [dstq+strideq*4] + jg .w4_loop +.w4_ret: + RET +INIT_ZMM avx512icl +.w8: + vpbroadcastq m6, [topq] + movsldup m9, [smooth_shuf] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w8_loop: + vpbroadcastq m4, [tlq+hq-8] + pshufb m4, m9 + PAETH + vextracti32x4 xm1, m0, 2 + vextracti32x4 xm2, ym0, 1 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + sub hd, 8 + jl .w8_ret + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + jg .w8_loop +.w8_ret: + RET +.w16: + vbroadcasti32x4 m6, [topq] + movsldup m9, [smooth_shuf] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w16_loop: + vpbroadcastd m4, [tlq+hq-4] + pshufb m4, m9 + PAETH + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + vbroadcasti32x8 m6, [topq] + mova ym9, ym8 + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w32_loop: + vpbroadcastd m4, [tlq+hq-2] + pshufb m4, m9 + PAETH + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + movu m6, [topq] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w64_loop: + vpbroadcastb m4, [tlq+hq-1] + PAETH + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_smooth_v_8bpc, 3, 7, 7, dst, stride, tl, w, h, weights, stride3 +%define base r6-ipred_smooth_v_8bpc_avx512icl_table + lea r6, [ipred_smooth_v_8bpc_avx512icl_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastd m0, [base+pb_127_m127] + vpbroadcastd m1, [base+pw_128] + lea weightsq, [base+smooth_weights+hq*4] + neg hq + vpbroadcastb m4, [tlq+hq] ; bottom + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.w4: + vpbroadcastd m2, [tlq+1] + movshdup m5, [smooth_shuf] + mova ym6, [smooth_endA] + punpcklbw m2, m4 ; top, bottom + pmaddubsw m3, m2, m0 + paddw m1, m2 ; 1 * top + 256 * bottom + 128, overflow is ok + paddw m3, m1 ; 128 * top + 129 * bottom + 128 +.w4_loop: + vbroadcasti32x4 m0, [weightsq+hq*2] + pshufb m0, m5 + pmaddubsw m0, m2, m0 + paddw m0, m3 + vpermb m0, m6, m0 + vextracti32x4 xm1, ym0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+stride3q ], xm1, 2 + add hq, 8 + jg .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+stride3q ], xm1, 3 + lea dstq, [dstq+strideq*4] + jl .w4_loop +.ret: + RET +.w8: + vpbroadcastq m2, [tlq+1] + movshdup m5, [smooth_shuf] + mova ym6, [smooth_endA] + punpcklbw m2, m4 + pmaddubsw m3, m2, m0 + paddw m1, m2 + paddw m3, m1 +.w8_loop: + vpbroadcastq m0, [weightsq+hq*2] + pshufb m0, m5 + pmaddubsw m0, m2, m0 + paddw m0, m3 + vpermb m0, m6, m0 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w8_loop + RET +.w16: + vbroadcasti32x4 m3, [tlq+1] + movshdup m6, [smooth_shuf] + mova m7, [smooth_endB] + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 +.w16_loop: + vpbroadcastq m1, [weightsq+hq*2] + pshufb m1, m6 + pmaddubsw m0, m2, m1 + pmaddubsw m1, m3, m1 + paddw m0, m4 + paddw m1, m5 + vpermt2b m0, m7, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w16_loop + RET +.w32: + vbroadcasti32x8 m3, [tlq+1] + movshdup m6, [smooth_shuf] + mova m7, [smooth_endB] + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 +.w32_loop: + vpbroadcastd m1, [weightsq+hq*2] + pshufb m1, m6 + pmaddubsw m0, m2, m1 + pmaddubsw m1, m3, m1 + paddw m0, m4 + paddw m1, m5 + vpermt2b m0, m7, m1 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w32_loop + RET +.w64: + movu m3, [tlq+1] + mova m6, [smooth_endB] + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 +.w64_loop: + vpbroadcastw m1, [weightsq+hq*2] + pmaddubsw m0, m2, m1 + pmaddubsw m1, m3, m1 + paddw m0, m4 + paddw m1, m5 + vpermt2b m0, m6, m1 + mova [dstq], m0 + add dstq, strideq + inc hq + jl .w64_loop + RET + +cglobal ipred_smooth_h_8bpc, 4, 7, 11, dst, stride, tl, w, h, stride3 +%define base r5-ipred_smooth_h_8bpc_avx512icl_table + lea r5, [ipred_smooth_h_8bpc_avx512icl_table] + mov r6d, wd + tzcnt wd, wd + vpbroadcastb m4, [tlq+r6] ; right + mov hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m5, [base+pb_127_m127] + vpbroadcastd m6, [base+pw_128] + sub tlq, hq + add wq, r5 + vpmovb2m k1, m6 + lea stride3q, [strideq*3] + jmp wq +.w4: + movsldup m3, [smooth_shuf] + vpbroadcastq m7, [smooth_weights+4*2] + mova ym8, [smooth_endA] +.w4_loop: + vpbroadcastq m0, [tlq+hq-8] + mova m2, m4 + vpshufb m2{k1}, m0, m3 ; left, right + pmaddubsw m0, m2, m5 + pmaddubsw m1, m2, m7 + paddw m2, m6 + paddw m0, m2 + paddw m0, m1 + vpermb m0, m8, m0 + vextracti32x4 xm1, ym0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+stride3q ], xm1, 2 + sub hd, 8 + jl .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+stride3q ], xm1, 3 + lea dstq, [dstq+strideq*4] + jg .w4_loop +.ret: + RET +.w8: + movsldup m3, [smooth_shuf] + vbroadcasti32x4 m7, [smooth_weights+8*2] + mova ym8, [smooth_endA] +.w8_loop: + vpbroadcastd m0, [tlq+hq-4] + mova m2, m4 + vpshufb m2{k1}, m0, m3 + pmaddubsw m0, m2, m5 + pmaddubsw m1, m2, m7 + paddw m2, m6 + paddw m0, m2 + paddw m0, m1 + vpermb m0, m8, m0 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +.w16: + movsldup m7, [smooth_shuf] + vbroadcasti32x4 m8, [smooth_weights+16*2] + vbroadcasti32x4 m9, [smooth_weights+16*3] + mova m10, [smooth_endB] +.w16_loop: + vpbroadcastd m0, [tlq+hq-4] + mova m3, m4 + vpshufb m3{k1}, m0, m7 + pmaddubsw m2, m3, m5 + pmaddubsw m0, m3, m8 + pmaddubsw m1, m3, m9 + paddw m3, m6 + paddw m2, m3 + paddw m0, m2 + paddw m1, m2 + vpermt2b m0, m10, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + mova m10, [smooth_endA] + vpbroadcastd ym7, [pb_1] + vbroadcasti32x8 m8, [smooth_weights+32*2] + vbroadcasti32x8 m9, [smooth_weights+32*3] + vshufi32x4 m10, m10, q3120 +.w32_loop: + vpbroadcastd m0, [tlq+hq-2] + mova m3, m4 + vpshufb m3{k1}, m0, m7 + pmaddubsw m2, m3, m5 + pmaddubsw m0, m3, m8 + pmaddubsw m1, m3, m9 + paddw m3, m6 + paddw m2, m3 + paddw m0, m2 + paddw m1, m2 + vpermt2b m0, m10, m1 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + mova m7, [smooth_weights+64*2] + mova m8, [smooth_weights+64*3] + mova m9, [smooth_endA] +.w64_loop: + mova m3, m4 + vpbroadcastb m3{k1}, [tlq+hq-1] + pmaddubsw m2, m3, m5 + pmaddubsw m0, m3, m7 + pmaddubsw m1, m3, m8 + paddw m3, m6 + paddw m2, m3 + paddw m0, m2 + paddw m1, m2 + vpermt2b m0, m9, m1 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_smooth_8bpc, 4, 7, 16, dst, stride, tl, w, h, v_weights, stride3 +%define base r5-ipred_smooth_8bpc_avx512icl_table + lea r5, [ipred_smooth_8bpc_avx512icl_table] + mov r6d, wd + tzcnt wd, wd + mov hd, hm + vpbroadcastb m6, [tlq+r6] ; right + sub tlq, hq + movsxd wq, [r5+wq*4] + vpbroadcastd m7, [base+pb_127_m127] + vpbroadcastb m0, [tlq] ; bottom + vpbroadcastd m1, [base+pw_255] + add wq, r5 + lea v_weightsq, [base+smooth_weights+hq*2] + vpmovb2m k1, m1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vpbroadcastd m8, [tlq+hq+1] + movsldup m4, [smooth_shuf] + movshdup m5, [smooth_shuf] + vpbroadcastq m9, [smooth_weights+4*2] + mova ym11, [smooth_endA] + + punpcklbw m8, m0 ; top, bottom + pmaddubsw m10, m8, m7 + paddw m1, m8 ; 1 * top + 256 * bottom + 255 + paddw m10, m1 ; 128 * top + 129 * bottom + 255 +.w4_loop: + vpbroadcastq m1, [tlq+hq-8] + vbroadcasti32x4 m0, [v_weightsq] + add v_weightsq, 16 + mova m2, m6 + vpshufb m2{k1}, m1, m4 ; left, right + pmaddubsw m1, m2, m7 ; 127 * left - 127 * right + pshufb m0, m5 + pmaddubsw m0, m8, m0 + paddw m1, m2 ; 128 * left + 129 * right + pmaddubsw m2, m9 + paddw m0, m10 + paddw m1, m2 + pavgw m0, m1 + vpermb m0, m11, m0 + vextracti32x4 xm1, ym0, 1 + movd [dstq+strideq*0], xm0 + movd [dstq+strideq*1], xm1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+stride3q ], xm1, 2 + sub hd, 8 + jl .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 1 + pextrd [dstq+strideq*1], xm1, 1 + pextrd [dstq+strideq*2], xm0, 3 + pextrd [dstq+stride3q ], xm1, 3 + lea dstq, [dstq+strideq*4] + jg .w4_loop +.ret: + RET +.w8: + vpbroadcastq m8, [tlq+hq+1] + movsldup m4, [smooth_shuf] + movshdup m5, [smooth_shuf] + vbroadcasti32x4 m9, [smooth_weights+8*2] + mova ym11, [smooth_endA] + punpcklbw m8, m0 + pmaddubsw m10, m8, m7 + paddw m1, m8 + paddw m10, m1 +.w8_loop: + vpbroadcastd m1, [tlq+hq-4] + vpbroadcastq m0, [v_weightsq] + add v_weightsq, 8 + mova m2, m6 + vpshufb m2{k1}, m1, m4 + pmaddubsw m1, m2, m7 + pshufb m0, m5 + pmaddubsw m0, m8, m0 + paddw m1, m2 + pmaddubsw m2, m9 + paddw m0, m10 + paddw m1, m2 + pavgw m0, m1 + vpermb m0, m11, m0 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8_loop + RET +.w16: + vbroadcasti32x4 m9, [tlq+hq+1] + movsldup m5, [smooth_shuf] + movshdup m10, [smooth_shuf] + vbroadcasti32x4 m11, [smooth_weights+16*2] + vbroadcasti32x4 m12, [smooth_weights+16*3] + mova m15, [smooth_endB] + punpcklbw m8, m9, m0 + punpckhbw m9, m0 + pmaddubsw m13, m8, m7 + pmaddubsw m14, m9, m7 + paddw m0, m1, m8 + paddw m1, m9 + paddw m13, m0 + paddw m14, m1 +.w16_loop: + vpbroadcastd m0, [tlq+hq-4] + vpbroadcastq m1, [v_weightsq] + add v_weightsq, 8 + mova m4, m6 + vpshufb m4{k1}, m0, m5 + pmaddubsw m2, m4, m7 + pshufb m1, m10 + pmaddubsw m0, m8, m1 + pmaddubsw m1, m9, m1 + paddw m2, m4 + pmaddubsw m3, m4, m11 + pmaddubsw m4, m12 + paddw m0, m13 + paddw m1, m14 + paddw m3, m2 + paddw m4, m2 + pavgw m0, m3 + pavgw m1, m4 + vpermt2b m0, m15, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + vbroadcasti32x8 m9, [tlq+hq+1] + movshdup m10, [smooth_shuf] + mova m12, [smooth_weights+32*2] + vpbroadcastd ym5, [pb_1] + mova m15, [smooth_endB] + punpcklbw m8, m9, m0 + punpckhbw m9, m0 + pmaddubsw m13, m8, m7 + pmaddubsw m14, m9, m7 + vshufi32x4 m11, m12, m12, q2020 + vshufi32x4 m12, m12, q3131 + paddw m0, m1, m8 + paddw m1, m9 + paddw m13, m0 + paddw m14, m1 +.w32_loop: + vpbroadcastd m0, [tlq+hq-2] + vpbroadcastd m1, [v_weightsq] + add v_weightsq, 4 + mova m4, m6 + vpshufb m4{k1}, m0, m5 + pmaddubsw m2, m4, m7 + pshufb m1, m10 + pmaddubsw m0, m8, m1 + pmaddubsw m1, m9, m1 + paddw m2, m4 + pmaddubsw m3, m4, m11 + pmaddubsw m4, m12 + paddw m0, m13 + paddw m1, m14 + paddw m3, m2 + paddw m4, m2 + pavgw m0, m3 + pavgw m1, m4 + vpermt2b m0, m15, m1 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + movu m9, [tlq+hq+1] + mova m11, [smooth_weights+64*2] + mova m2, [smooth_weights+64*3] + mova m14, [smooth_endB] + punpcklbw m8, m9, m0 + punpckhbw m9, m0 + pmaddubsw m12, m8, m7 + pmaddubsw m13, m9, m7 + vshufi32x4 m10, m11, m2, q2020 + vshufi32x4 m11, m2, q3131 + paddw m0, m1, m8 + paddw m1, m9 + paddw m12, m0 + paddw m13, m1 +.w64_loop: + mova m4, m6 + vpbroadcastb m4{k1}, [tlq+hq-1] + vpbroadcastw m1, [v_weightsq] + add v_weightsq, 2 + pmaddubsw m2, m4, m7 + pmaddubsw m0, m8, m1 + pmaddubsw m1, m9, m1 + paddw m2, m4 + pmaddubsw m3, m4, m10 + pmaddubsw m4, m11 + paddw m0, m12 + paddw m1, m13 + paddw m3, m2 + paddw m4, m2 + pavgw m0, m3 + pavgw m1, m4 + vpermt2b m0, m14, m1 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal pal_pred_8bpc, 4, 7, 5, dst, stride, pal, idx, w, h, stride3 + lea r6, [pal_pred_8bpc_avx512icl_table] + tzcnt wd, wm + vbroadcasti32x4 m4, [palq] + movifnidn hd, hm + movsxd wq, [r6+wq*4] + packuswb m4, m4 + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.w4: + pshufb xmm0, xm4, [idxq] + add idxq, 16 + movd [dstq+strideq*0], xmm0 + pextrd [dstq+strideq*1], xmm0, 1 + pextrd [dstq+strideq*2], xmm0, 2 + pextrd [dstq+stride3q ], xmm0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + pshufb xmm0, xm4, [idxq+16*0] + pshufb xmm1, xm4, [idxq+16*1] + add idxq, 16*2 + movq [dstq+strideq*0], xmm0 + movhps [dstq+strideq*1], xmm0 + movq [dstq+strideq*2], xmm1 + movhps [dstq+stride3q ], xmm1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +.w16: + pshufb m0, m4, [idxq] + add idxq, 64 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +.w32: + pshufb m0, m4, [idxq+64*0] + pshufb m1, m4, [idxq+64*1] + add idxq, 64*2 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w32 + RET +.w64: + pshufb m0, m4, [idxq+64*0] + pshufb m1, m4, [idxq+64*1] + pshufb m2, m4, [idxq+64*2] + pshufb m3, m4, [idxq+64*3] + add idxq, 64*4 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w64 + RET + +; The ipred_filter code processes 4x2 blocks in the following order +; which increases parallelism compared to doing things row by row. +; Some redundant blocks are calculated for w > 4. +; w4 w8 w16 w32 +; 1 1 2 1 2 3 4 1 2 3 4 9 a b c +; 2 2 3 2 3 4 5 2 3 4 5 a b c d +; 3 3 4 3 4 5 6 3 4 5 6 b c d e +; 4 4 5 4 5 6 7 4 5 6 7 c d e f +; 5 5 6 5 6 7 8 5 6 7 8 d e f g +; 6 6 7 6 7 8 9 6 7 8 9 e f g h +; 7 7 8 7 8 9 a 7 8 9 a f g h i +; ___ 8 ___ 8 9 ___ 8 9 a b ___ 8 9 a b g h i j ___ +; 9 9 a b h i j +; a b i j +; b j + +cglobal ipred_filter_8bpc, 4, 7, 14, dst, stride, tl, w, h, flt +%define base r6-filter_taps + lea r6, [filter_taps] +%ifidn fltd, fltm + movzx fltd, fltb +%else + movzx fltd, byte fltm +%endif + vpbroadcastd xmm2, [tlq+1] ; t0 t0 t0 t0 + movifnidn hd, hm + shl fltd, 6 + vpbroadcastd m6, [base+pd_8] + vpbroadcastd xmm3, [tlq-2] ; l1 l0 tl __ + vbroadcasti32x4 m7, [r6+fltq+16*0] ; p1 p2 p3 p4 + vbroadcasti32x4 m8, [r6+fltq+16*1] + vbroadcasti32x4 m9, [r6+fltq+16*2] ; p6 p5 p0 __ + vbroadcasti32x4 m10, [r6+fltq+16*3] + mova xmm0, xm6 + vpdpbusd xmm0, xmm2, xm7 + mova xmm1, xm6 + vpdpbusd xmm1, xmm2, xm8 + vpdpbusd xmm0, xmm3, xm9 + vpdpbusd xmm1, xmm3, xm10 + packssdw xmm0, xmm1 + cmp wd, 8 + jb .w4 + vpbroadcastd ym2, [tlq+5] + mova m11, [base+filter_perm] + mov r5, 0xffffffffffff000f + psrldq xmm2, 1 ; __ t0 + kmovq k1, r5 ; 0x000f + psraw xm5, xmm0, 4 + packuswb xmm2, xm5 ; __ t0 a0 b0 + pshufd ym2{k1}, ymm2, q3333 ; b0 b0 b0 b0 t1 t1 t1 t1 + je .w8 + kxnorb k3, k3, k3 ; 0x00ff + vpbroadcastd xm3, [tlq-4] + kandnq k2, k3, k1 ; 0xffffffffffff0000 + vpermb ym3{k2}, ym11, ymm2 ; l3 l2 l1 __ b3 a3 t3 __ + mova ym0, ym6 + vpdpbusd ym0, ym2, ym7 + mova ym1, ym6 + vpdpbusd ym1, ym2, ym8 + pshufb ym5{k2}, ym2, ym11 ; a0 b0 __ t0 + vpbroadcastd m2, [tlq+9] + vpdpbusd ym0, ym3, ym9 + vpdpbusd ym1, ym3, ym10 + vpbroadcastd xm3, [tlq-6] ; l5 l4 l3 __ + kunpckbw k4, k1, k3 ; 0x0fff + packssdw ym0, ym1 + psraw ym0, 4 ; a0 d0 a1 b1 + packuswb ym5, ym0 ; a0 b0 c0 d0 __ t1 a1 b1 + pshufd m2{k3}, m5, q3333 ; d0 d0 d0 d0 b1 b1 b1 b1 t2 t2 t2 t2 + vpermb m3{k2}, m11, m5 ; l5 l4 l3 __ d3 c3 b3 __ b7 a7 t7 __ + mova m4, m6 + vpdpbusd m4, m2, m7 + mova m1, m6 + vpdpbusd m1, m2, m8 + psrldq m0, m2, 1 ; __ d0 __ b0 __ t0 + vpbroadcastd m2, [tlq+13] + vpdpbusd m4, m3, m9 + vpdpbusd m1, m3, m10 + mova m12, [base+filter_end] + lea r5d, [hq-6] + mov r6, dstq + cmovp hd, r5d ; w == 16 ? h : h - 6 + packssdw m4, m1 + psraw m4, 4 ; e0 f0 c1 d1 a2 b2 + packuswb m0, m4 ; __ d0 e0 f0 __ b1 c1 d1 __ t2 a2 b2 + pshufd m2{k4}, m0, q3333 ; f0 f0 f0 f0 d1 d1 d1 d1 b2 b2 b2 b2 t3 t3 t3 t3 +.w16_loop: + vpbroadcastd xm3, [tlq-8] + vpermb m3{k2}, m11, m0 ; l7 l6 l5 __ f3 e3 d3 __ d7 c7 b7 __ bb ab tb __ + mova m1, m6 + vpdpbusd m1, m2, m7 + mova m0, m6 + vpdpbusd m0, m2, m8 + sub tlq, 2 + vpdpbusd m1, m3, m9 + vpdpbusd m0, m3, m10 + packssdw m1, m0 + mova m0, m4 + psraw m4, m1, 4 ; g0 h0 e1 f1 c2 d2 a3 b3 + packuswb m0, m4 ; e0 f0 g0 h0 c1 d1 e1 f1 a2 b2 c2 d2 __ __ a3 b3 + pshufd m2, m0, q3333 ; h0 h0 h0 h0 f1 f1 f1 f1 d2 d2 d2 d2 b3 b3 b3 b3 + vpermt2d m5, m12, m0 ; c0 d0 e0 f0 __ __ c1 d1 a0 a1 a2 a3 b0 b1 b2 b3 + vextracti32x4 [dstq+strideq*0], m5, 2 + vextracti32x4 [dstq+strideq*1], m5, 3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + cmp wd, 16 + je .ret + mova xm13, [filter_perm+16] + mova xmm3, [r6+strideq*0] + punpckhdq xmm3, [r6+strideq*1] + vpbroadcastd m2{k1}, [tlq+r5+17] ; t4 t4 t4 t4 f1 f1 f1 f1 d2 d2 d2 d2 b3 b3 b3 b3 + pinsrb xm3, xmm3, [tlq+r5+16], 7 + pshufb xm3, xm13 + vpermb m3{k2}, m11, m0 ; bf af tf __ h3 g3 f3 __ f7 e7 d7 __ db cb bb __ + mova m0, m6 + vpdpbusd m0, m2, m7 + mova m1, m6 + vpdpbusd m1, m2, m8 + kunpckbw k5, k3, k1 ; 0xff0f + lea r3, [strideq*3] + vpdpbusd m0, m3, m9 + vpdpbusd m1, m3, m10 + packssdw m0, m1 + psraw m0, 4 ; a4 b4 g1 h1 e2 f2 c3 d3 + packuswb m4, m0 ; g0 h0 a4 b4 e1 f1 g1 h1 c2 d2 e2 f2 __ __ c3 d3 + vpblendmb m1{k3}, m4, m2 ; __ t4 a4 b4 e1 f1 g1 h1 c2 d2 e2 f2 __ __ c3 d3 + vpbroadcastd ym2, [tlq+r5+21] + pshufd m2{k5}, m4, q3333 ; b4 b4 b4 b4 t5 t5 t5 t5 f2 f2 f2 f2 d3 d3 d3 d3 + vpermt2d m5, m12, m4 ; e0 f0 g0 h0 __ __ e1 f1 c0 c1 c2 c3 d0 d1 d2 d3 + vextracti32x4 [dstq+strideq*0], m5, 2 + vextracti32x4 [dstq+strideq*1], m5, 3 + punpckhqdq xmm3, [r6+r3] + pinsrb xmm3, [r6+strideq*2+15], 11 + pshufb xm3, xmm3, xm13 + vpermb m3{k2}, m11, m1 ; df cf bf __ bj aj tj __ h7 g7 f7 __ fb eb db __ + mova m4, m6 + vpdpbusd m4, m2, m7 + mova m1, m6 + vpdpbusd m1, m2, m8 + kxnord k3, k3, k4 ; 0xfffff0ff + lea r4, [strideq*5] + vpdpbusd m4, m3, m9 + vpdpbusd m1, m3, m10 + packssdw m4, m1 + psraw m4, 4 ; c4 d4 a5 b5 g2 h2 e3 f3 + packuswb m0, m4 ; a4 b4 c4 d4 g1 h1 a5 b5 e2 f2 g2 h2 __ __ e3 f3 + vpblendmw m1{k3}, m2, m0 ; a4 b4 c4 d4 __ t5 a5 b5 e2 f2 g2 h2 __ __ e3 f3 + vpbroadcastd m2, [tlq+r5+25] + pshufd m2{k3}, m0, q3333 ; d4 d4 d4 d4 b5 b5 b5 b5 t6 t6 t6 t6 f3 f3 f3 f3 + vpermt2d m5, m12, m0 ; g0 h0 a4 b4 __ __ g1 h1 e0 e1 e2 e3 f0 f1 f2 f3 + vextracti32x4 [dstq+strideq*2], m5, 2 + vextracti32x4 [dstq+r3 ], m5, 3 + punpckhqdq xmm3, [r6+r4] + pinsrb xmm3, [r6+strideq*4+15], 11 + pshufb xm3, xmm3, xm13 + vpermb m3{k2}, m11, m1 ; ff ef df __ dj cj bj __ bn an tn __ hb hb fb __ + mova m0, m6 + vpdpbusd m0, m2, m7 + mova m1, m6 + vpdpbusd m1, m2, m8 + kunpckwd k1, k1, k2 ; 0x000f0000 + vpdpbusd m0, m3, m9 + vpdpbusd m1, m3, m10 + packssdw m0, m1 + psraw m0, 4 ; e4 f4 c5 d5 a6 b6 g3 h3 + packuswb m4, m0 ; c4 d4 e4 f4 a5 b5 c5 d5 g2 h2 a6 b6 __ __ g3 h3 + vpblendmw m1{k1}, m4, m2 ; c4 d4 e4 f4 a5 b5 c5 d5 __ t6 a6 b6 __ __ g3 h3 + vpbroadcastd m2, [tlq+r5+29] + pshufd m2{k4}, m4, q3333 ; f4 f4 f4 f4 d5 d5 d5 d5 b6 b6 b6 b6 t7 t7 t7 t7 + vpermt2d m5, m12, m4 ; a4 b4 c4 d4 __ __ a5 b5 g0 g1 g2 g3 h0 h1 h2 h3 + vextracti32x4 [dstq+strideq*4], m5, 2 + vextracti32x4 [dstq+r4 ], m5, 3 + lea r0, [strideq+r3*2] +.w32_loop: + punpckhqdq xmm3, [r6+r0] + pinsrb xmm3, [r6+r3*2+15], 11 + pshufb xm3, xmm3, xm13 + vpermb m3{k2}, m11, m1 ; hf gf ff __ fj ej dj __ dn cn bn __ br ar tr __ +.w32_loop_tail: + mova m4, m6 + vpdpbusd m4, m2, m7 + mova m1, m6 + vpdpbusd m1, m2, m8 + vpdpbusd m4, m3, m9 + vpdpbusd m1, m3, m10 + packssdw m4, m1 + mova m1, m0 + psraw m0, m4, 4 ; g4 h4 e5 f5 c6 d6 a7 b7 + packuswb m1, m0 ; e4 f4 g4 h4 c5 d5 e5 f5 a6 b6 c6 d6 __ __ a7 b7 + pshufd m2, m1, q3333 ; h4 h4 h4 h4 f5 f5 f5 f5 d6 d6 d6 d6 b7 b7 b7 b7 + vpermt2d m5, m12, m1 ; c4 d4 e4 f4 __ __ c5 d5 a4 a5 a6 a7 b4 b5 b6 b7 + vextracti32x4 [r6+strideq*0+16], m5, 2 + vextracti32x4 [r6+strideq*1+16], m5, 3 + lea r6, [r6+strideq*2] + sub r5d, 2 + jg .w32_loop + vpermb m3, m11, m1 + cmp r5d, -6 + jg .w32_loop_tail +.ret: + RET +.w8: + vpermb ym3, ym11, ymm2 +.w8_loop: + vpbroadcastd ym3{k1}, [tlq-4] ; l3 l2 l1 __ b3 a3 t3 __ + mova ym0, ym6 + vpdpbusd ym0, ym2, ym7 + mova ym1, ym6 + vpdpbusd ym1, ym2, ym8 + sub tlq, 2 + vpdpbusd ym0, ym3, ym9 + vpdpbusd ym1, ym3, ym10 + mova ym3, ym5 + packssdw ym0, ym1 + psraw ym5, ym0, 4 ; c0 d0 a1 b1 + packuswb ym3, ym5 ; a0 b0 c0 d0 __ __ a1 b1 + pshufd ym2, ym3, q3333 ; d0 d0 d0 d0 b1 b1 b1 b1 + vpermb ym3, ym11, ym3 ; a0 a1 b0 b1 + movq [dstq+strideq*0], xm3 + movhps [dstq+strideq*1], xm3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +.w4_loop: + vpbroadcastd xmm3, [tlq-4] ; l3 l2 l1 __ + mova xmm0, xm6 + vpdpbusd xmm0, xmm2, xm7 + mova xmm1, xm6 + vpdpbusd xmm1, xmm2, xm8 + sub tlq, 2 + vpdpbusd xmm0, xmm3, xm9 + vpdpbusd xmm1, xmm3, xm10 + packssdw xmm0, xmm1 +.w4: + psraw xmm0, 4 ; a0 b0 + packuswb xmm0, xmm0 + movd [dstq+strideq*0], xmm0 + pshufd xmm2, xmm0, q1111 ; b0 b0 b0 b0 + movd [dstq+strideq*1], xmm2 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w4_loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/ipred_sse.asm b/third_party/dav1d/src/x86/ipred_sse.asm new file mode 100644 index 0000000000..67e90b79ae --- /dev/null +++ b/third_party/dav1d/src/x86/ipred_sse.asm @@ -0,0 +1,5409 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +%macro SMOOTH_WEIGHT_TABLE 1-* + %rep %0 + db %1-128, 127-%1 + %rotate 1 + %endrep +%endmacro + +; sm_weights[], but modified to precalculate x and 256-x with offsets to +; enable efficient use of pmaddubsw (which requires signed values) +smooth_weights: SMOOTH_WEIGHT_TABLE \ + 0, 0, 255, 128, 255, 149, 85, 64, \ + 255, 197, 146, 105, 73, 50, 37, 32, \ + 255, 225, 196, 170, 145, 123, 102, 84, \ + 68, 54, 43, 33, 26, 20, 17, 16, \ + 255, 240, 225, 210, 196, 182, 169, 157, \ + 145, 133, 122, 111, 101, 92, 83, 74, \ + 66, 59, 52, 45, 39, 34, 29, 25, \ + 21, 17, 14, 12, 10, 9, 8, 8, \ + 255, 248, 240, 233, 225, 218, 210, 203, \ + 196, 189, 182, 176, 169, 163, 156, 150, \ + 144, 138, 133, 127, 121, 116, 111, 106, \ + 101, 96, 91, 86, 82, 77, 73, 69, \ + 65, 61, 57, 54, 50, 47, 44, 41, \ + 38, 35, 32, 29, 27, 25, 22, 20, \ + 18, 16, 15, 13, 12, 10, 9, 8, \ + 7, 6, 6, 5, 5, 4, 4, 4 + +ipred_v_shuf: db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7 +ipred_h_shuf: db 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 +ipred_paeth_shuf: db 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 +z_upsample1: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +z_upsample2: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 8 +z_transpose4: db 8, 12, 0, 4, 9, 13, 1, 5, 10, 14, 2, 6, 11, 15, 3, 7 +z3_shuf: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +z3_shuf_h4: db 4, 3, 3, 2, 2, 1, 1, 0, 12, 11, 11, 10, 10, 9, 9, 8 +filter_shuf1: db 3, 4, 3, 4, 5, 6, 5, 6, 7, 2, 7, 2, 1, -1, 1, -1 +filter_shuf2: db 3, 4, 3, 4, 5, 6, 5, 6, 7, 11, 7, 11, 15, -1, 15, -1 +z_filter_wh4: db 7, 7, 19, 7, +z_filter_wh8: db 19, 19, 11, 19, 11, 15, 15, 15, 23, 23, 23, 23, 39, 39, 39, 39 +pd_32768: dd 32768 +z3_filter_k_tail: db 64, 0, 64, 0, 64, 0, 56, 8 +z1_shuf_w4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 +pb_0to15: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +pb_15to0: db 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +z_base_inc: dw 0*64, 1*64, 2*64, 3*64, 4*64, 5*64, 6*64, 7*64 +z3_base_inc: dw 7*64, 6*64, 5*64, 4*64, 3*64, 2*64, 1*64, 0*64 +z_filter_wh16: db 19, 19, 19, 23, 23, 23, 31, 31, 31, 47, 47, 47, 79, 79, 79, -1 +z_filter_t_w48: db 55,127, 7,127, 15, 31, 39, 31,127, 39,127, 39, 7, 15, 31, 15 + db 39, 63, 3, 63, 3, 3, 19, 3, 47, 19, 47, 19, 3, 3, 3, 3 +z_filter_t_w16: db 15, 31, 7, 15, 31, 7, 3, 31, 3, 3, 3, 3, 3, 3, 0, 0 +z_filter_s: db 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7 + db 7, 8, 8, 9, 9, 10, 10, 11 +z_filter_k_tail: db 0, 64, 0, 64, 8, 56, 0, 64 +z2_h_shuf: db 7, 6, 15, 14, 6, 5, 14, 13, 5, 4, 13, 12, 4, 3, 12, 11 +z2_upsample: db 7, 6, 15, 14, 5, 4, 13, 12, 3, 2, 11, 10, 1, 0, 9, 8 +z2_dy_offset: dw 88*64, 88*64, 87*64, 87*64 +pw_m1to4: dw -1, -2, -3, -4 +z_filter_k: times 4 db 0, 16 + times 4 db 0, 20 + times 4 db 8, 16 + times 4 db 32, 16 + times 4 db 24, 20 + times 4 db 16, 16 + times 4 db 0, 0 + times 4 db 0, 0 +pw_8: times 8 db 8, 0 +pb_3: times 16 db 3 +pb_16: times 16 db 16 +pw_62: times 8 dw 62 +pw_64: times 8 dw 64 +pw_256: times 8 dw 256 +pw_512: times 8 dw 512 +pw_m256: times 8 dw -256 +pb_2: times 8 db 2 +pb_4: times 8 db 4 +pb_8: times 8 db 8 +pb_128: times 8 db 128 +pb_m16: times 8 db -16 +pw_128: times 4 dw 128 +pw_255: times 4 dw 255 +pb_36_m4: times 4 db 36, -4 +pb_127_m127: times 4 db 127, -127 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_ssse3_table (ipred_dc_ssse3_table + 10*4) +%define ipred_cfl_splat_ssse3_table (ipred_cfl_ssse3_table + 8*4) + +JMP_TABLE ipred_h, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_dc, ssse3, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4 +JMP_TABLE ipred_dc_left, ssse3, h4, h8, h16, h32, h64 +JMP_TABLE ipred_smooth, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_v, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_smooth_h, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_paeth, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z1, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z2, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_z3, ssse3, h4, h8, h16, h32, h64 +JMP_TABLE pal_pred, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_cfl, ssse3, h4, h8, h16, h32, w4, w8, w16, w32, \ + s4-8*4, s8-8*4, s16-8*4, s32-8*4 +JMP_TABLE ipred_cfl_left, ssse3, h4, h8, h16, h32 +JMP_TABLE ipred_filter, ssse3, w4, w8, w16, w32 + +cextern dr_intra_derivative +cextern filter_intra_taps + +SECTION .text + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_h_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +%macro IPRED_SET 3 ; width, stride, stride size pshuflw_imm8 + pshuflw m1, m0, %3 ; extend 8 byte for 2 pos + punpcklqdq m1, m1 + mova [dstq + %2], m1 +%if %1 > 16 + mova [dstq + 16 + %2], m1 +%endif +%if %1 > 32 + mova [dstq + 32 + %2], m1 + mova [dstq + 48 + %2], m1 +%endif +%endmacro + +%macro IPRED_H 1 ; width + sub tlq, 4 + movd m0, [tlq] ; get 4 bytes of topleft data + punpcklbw m0, m0 ; extend 2 byte +%if %1 == 4 + pshuflw m1, m0, q2233 + movd [dstq+strideq*0], m1 + psrlq m1, 32 + movd [dstq+strideq*1], m1 + pshuflw m0, m0, q0011 + movd [dstq+strideq*2], m0 + psrlq m0, 32 + movd [dstq+stride3q ], m0 + +%elif %1 == 8 + punpcklwd m0, m0 + punpckhdq m1, m0, m0 + punpckldq m0, m0 + movq [dstq+strideq*1], m1 + movhps [dstq+strideq*0], m1 + movq [dstq+stride3q ], m0 + movhps [dstq+strideq*2], m0 +%else + IPRED_SET %1, 0, q3333 + IPRED_SET %1, strideq, q2222 + IPRED_SET %1, strideq*2, q1111 + IPRED_SET %1, stride3q, q0000 +%endif + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w%1 + RET +%endmacro + +INIT_XMM ssse3 +cglobal ipred_h_8bpc, 3, 6, 2, dst, stride, tl, w, h, stride3 + LEA r5, ipred_h_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w4: + IPRED_H 4 +.w8: + IPRED_H 8 +.w16: + IPRED_H 16 +.w32: + IPRED_H 32 +.w64: + IPRED_H 64 + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_v_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_v_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_splat_ssse3_table + tzcnt wd, wm + movu m0, [tlq+ 1] + movu m1, [tlq+17] + movu m2, [tlq+33] + movu m3, [tlq+49] + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_dc_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_dc_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + movifnidn wd, wm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd m4, r5d + tzcnt r5d, r5d + movd m5, r5d + LEA r5, ipred_dc_ssse3_table + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+20] + pcmpeqd m3, m3 + psrlw m4, 1 ; dc = (width + height) >> 1; + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movd m0, [tlq-4] + pmaddubsw m0, m3 + jmp wq +.w4: + movd m1, [tlq+1] + pmaddubsw m1, m3 + psubw m0, m4 + paddw m0, m1 + pmaddwd m0, m3 + cmp hd, 4 + jg .w4_mul + psrlw m0, 3 ; dc >>= ctz(width + height); + jmp .w4_end +.w4_mul: + punpckhqdq m1, m0, m0 + paddw m0, m1 + psrlq m1, m0, 32 + paddw m0, m1 + psrlw m0, 2 + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 8 + cmovz r6d, r2d + movd m5, r6d + pmulhuw m0, m5 +.w4_end: + pxor m1, m1 + pshufb m0, m1 +.s4: + movd [dstq+strideq*0], m0 + movd [dstq+strideq*1], m0 + movd [dstq+strideq*2], m0 + movd [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +ALIGN function_align +.h8: + movq m0, [tlq-8] + pmaddubsw m0, m3 + jmp wq +.w8: + movq m1, [tlq+1] + pmaddubsw m1, m3 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + paddw m0, m1 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 8 + je .w8_end + mov r6d, 0x5556 + mov r2d, 0x3334 + cmp hd, 32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w8_end: + pxor m1, m1 + pshufb m0, m1 +.s8: + movq [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +ALIGN function_align +.h16: + mova m0, [tlq-16] + pmaddubsw m0, m3 + jmp wq +.w16: + movu m1, [tlq+1] + pmaddubsw m1, m3 + paddw m0, m1 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 16 + je .w16_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 8|32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w16_end: + pxor m1, m1 + pshufb m0, m1 +.s16: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +ALIGN function_align +.h32: + mova m0, [tlq-32] + pmaddubsw m0, m3 + mova m2, [tlq-16] + pmaddubsw m2, m3 + paddw m0, m2 + jmp wq +.w32: + movu m1, [tlq+1] + pmaddubsw m1, m3 + movu m2, [tlq+17] + pmaddubsw m2, m3 + paddw m1, m2 + paddw m0, m1 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 64|16 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w32_end: + pxor m1, m1 + pshufb m0, m1 + mova m1, m0 +.s32: + mova [dstq], m0 + mova [dstq+16], m1 + mova [dstq+strideq], m0 + mova [dstq+strideq+16], m1 + mova [dstq+strideq*2], m0 + mova [dstq+strideq*2+16], m1 + mova [dstq+stride3q], m0 + mova [dstq+stride3q+16], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s32 + RET +ALIGN function_align +.h64: + mova m0, [tlq-64] + mova m1, [tlq-48] + pmaddubsw m0, m3 + pmaddubsw m1, m3 + paddw m0, m1 + mova m1, [tlq-32] + pmaddubsw m1, m3 + paddw m0, m1 + mova m1, [tlq-16] + pmaddubsw m1, m3 + paddw m0, m1 + jmp wq +.w64: + movu m1, [tlq+ 1] + movu m2, [tlq+17] + pmaddubsw m1, m3 + pmaddubsw m2, m3 + paddw m1, m2 + movu m2, [tlq+33] + pmaddubsw m2, m3 + paddw m1, m2 + movu m2, [tlq+49] + pmaddubsw m2, m3 + paddw m1, m2 + paddw m0, m1 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 64 + je .w64_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w64_end: + pxor m1, m1 + pshufb m0, m1 + mova m1, m0 + mova m2, m0 + mova m3, m0 +.s64: + mova [dstq], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + mova [dstq+strideq], m0 + mova [dstq+strideq+16], m1 + mova [dstq+strideq+32], m2 + mova [dstq+strideq+48], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s64 + RET + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_dc_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_dc_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_left_ssse3_table + mov hd, hm ; zero upper half + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + movu m0, [tlq] + movd m3, [r5-ipred_dc_left_ssse3_table+pd_32768] + movd m2, r6d + psrld m3, m2 + movsxd r6, [r5+r6*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, r5 + add r5, ipred_dc_splat_ssse3_table-ipred_dc_left_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + movu m1, [tlq+48] ; unaligned when jumping here from dc_top + pmaddubsw m1, m2 + paddw m0, m1 + movu m1, [tlq+32] ; unaligned when jumping here from dc_top + pmaddubsw m1, m2 + paddw m0, m1 +.h32: + movu m1, [tlq+16] ; unaligned when jumping here from dc_top + pmaddubsw m1, m2 + paddw m0, m1 +.h16: + pshufd m1, m0, q3232 ; psrlq m1, m0, 16 + paddw m0, m1 +.h8: + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 +.h4: + pmaddwd m0, m2 + pmulhrsw m0, m3 + lea stride3q, [strideq*3] + pxor m1, m1 + pshufb m0, m1 + mova m1, m0 + mova m2, m0 + mova m3, m0 + jmp wq + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_dc_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_dc_128_8bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_splat_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + movddup m0, [r5-ipred_dc_splat_ssse3_table+pb_128] + mova m1, m0 + mova m2, m0 + mova m3, m0 + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_dc_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_dc_top_8bpc, 3, 7, 6, dst, stride, tl, w, h + LEA r5, ipred_dc_left_ssse3_table + tzcnt wd, wm + inc tlq + movu m0, [tlq] + movifnidn hd, hm + movd m3, [r5-ipred_dc_left_ssse3_table+pd_32768] + movd m2, wd + psrld m3, m2 + movsxd r6, [r5+wq*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, r5 + add r5, ipred_dc_splat_ssse3_table-ipred_dc_left_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_smooth_v_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +%macro SMOOTH 6 ; src[1-2], mul[1-2], add[1-2] + ; w * a = (w - 128) * a + 128 * a + ; (256 - w) * b = (127 - w) * b + 129 * b + ; => w * a + (256 - w) * b = [(w - 128) * a + (127 - w) * b] + [128 * a + 129 * b] + pmaddubsw m6, m%3, m%1 + pmaddubsw m0, m%4, m%2 ; (w - 128) * a + (127 - w) * b + paddw m6, m%5 + paddw m0, m%6 ; [(w - 128) * a + (127 - w) * b] + [128 * a + 129 * b + 128] + psrlw m6, 8 + psrlw m0, 8 + packuswb m6, m0 +%endmacro + +cglobal ipred_smooth_v_8bpc, 3, 7, 7, dst, stride, tl, w, h, weights +%define base r6-ipred_smooth_v_ssse3_table + LEA r6, ipred_smooth_v_ssse3_table + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + movddup m0, [base+pb_127_m127] + movddup m1, [base+pw_128] + lea weightsq, [base+smooth_weights+hq*4] + neg hq + movd m5, [tlq+hq] + pxor m2, m2 + pshufb m5, m2 + add wq, r6 + jmp wq +.w4: + movd m2, [tlq+1] + punpckldq m2, m2 + punpcklbw m2, m5 ; top, bottom + lea r3, [strideq*3] + mova m4, [base+ipred_v_shuf] + mova m5, m4 + punpckldq m4, m4 + punpckhdq m5, m5 + pmaddubsw m3, m2, m0 ; m3: 127 * top - 127 * bottom + paddw m1, m2 ; m1: 1 * top + 256 * bottom + 128, overflow is ok + paddw m3, m1 ; m3: 128 * top + 129 * bottom + 128 +.w4_loop: + movu m1, [weightsq+hq*2] + pshufb m0, m1, m4 ;m2, m3, m4 and m5 should be stable in loop + pshufb m1, m5 + SMOOTH 0, 1, 2, 2, 3, 3 + movd [dstq+strideq*0], m6 + pshuflw m1, m6, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m6, m6 + movd [dstq+strideq*2], m6 + psrlq m6, 32 + movd [dstq+r3 ], m6 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w4_loop + RET +ALIGN function_align +.w8: + movq m2, [tlq+1] + punpcklbw m2, m5 + mova m5, [base+ipred_v_shuf] + lea r3, [strideq*3] + pshufd m4, m5, q0000 + pshufd m5, m5, q1111 + pmaddubsw m3, m2, m0 + paddw m1, m2 + paddw m3, m1 ; m3 is output for loop +.w8_loop: + movq m1, [weightsq+hq*2] + pshufb m0, m1, m4 + pshufb m1, m5 + SMOOTH 0, 1, 2, 2, 3, 3 + movq [dstq+strideq*0], m6 + movhps [dstq+strideq*1], m6 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w8_loop + RET +ALIGN function_align +.w16: + movu m3, [tlq+1] + punpcklbw m2, m3, m5 + punpckhbw m3, m5 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 ; m4 and m5 is output for loop +.w16_loop: + movd m1, [weightsq+hq*2] + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + SMOOTH 1, 1, 2, 3, 4, 5 + mova [dstq], m6 + add dstq, strideq + add hq, 1 + jl .w16_loop + RET +ALIGN function_align +.w32: +%if WIN64 + movaps [rsp+24], xmm7 + %define xmm_regs_used 8 +%endif + mova m7, m5 +.w32_loop_init: + mov r3d, 2 +.w32_loop: + movddup m0, [base+pb_127_m127] + movddup m1, [base+pw_128] + movu m3, [tlq+1] + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 + movd m1, [weightsq+hq*2] + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + SMOOTH 1, 1, 2, 3, 4, 5 + mova [dstq], m6 + add tlq, 16 + add dstq, 16 + dec r3d + jg .w32_loop + lea dstq, [dstq-32+strideq] + sub tlq, 32 + add hq, 1 + jl .w32_loop_init + RET +ALIGN function_align +.w64: +%if WIN64 + movaps [rsp+24], xmm7 + %define xmm_regs_used 8 +%endif + mova m7, m5 +.w64_loop_init: + mov r3d, 4 +.w64_loop: + movddup m0, [base+pb_127_m127] + movddup m1, [base+pw_128] + movu m3, [tlq+1] + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + pmaddubsw m4, m2, m0 + pmaddubsw m5, m3, m0 + paddw m0, m1, m2 + paddw m1, m3 + paddw m4, m0 + paddw m5, m1 + movd m1, [weightsq+hq*2] + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + SMOOTH 1, 1, 2, 3, 4, 5 + mova [dstq], m6 + add tlq, 16 + add dstq, 16 + dec r3d + jg .w64_loop + lea dstq, [dstq-64+strideq] + sub tlq, 64 + add hq, 1 + jl .w64_loop_init + RET + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_smooth_h_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +cglobal ipred_smooth_h_8bpc, 3, 7, 8, dst, stride, tl, w, h +%define base r6-ipred_smooth_h_ssse3_table + LEA r6, ipred_smooth_h_ssse3_table + mov wd, wm + movd m3, [tlq+wq] + pxor m1, m1 + pshufb m3, m1 ; right + tzcnt wd, wd + mov hd, hm + movsxd wq, [r6+wq*4] + movddup m4, [base+pb_127_m127] + movddup m5, [base+pw_128] + add wq, r6 + jmp wq +.w4: + movddup m6, [base+smooth_weights+4*2] + mova m7, [base+ipred_h_shuf] + sub tlq, 4 + sub tlq, hq + lea r3, [strideq*3] +.w4_loop: + movd m2, [tlq+hq] ; left + pshufb m2, m7 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m6 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + pmaddubsw m2, m6 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + movd [dstq+strideq*2], m0 + psrlq m0, 32 + movd [dstq+r3 ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_loop + RET +ALIGN function_align +.w8: + mova m6, [base+smooth_weights+8*2] + mova m7, [base+ipred_h_shuf] + sub tlq, 4 + sub tlq, hq + punpckldq m7, m7 +.w8_loop: + movd m2, [tlq+hq] ; left + pshufb m2, m7 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m6 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + pmaddubsw m2, m6 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + mova m6, [base+smooth_weights+16*2] + mova m7, [base+smooth_weights+16*3] + sub tlq, 1 + sub tlq, hq +.w16_loop: + pxor m1, m1 + movd m2, [tlq+hq] ; left + pshufb m2, m1 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m6 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + pmaddubsw m2, m7 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq], m0 + lea dstq, [dstq+strideq] + sub hd, 1 + jg .w16_loop + RET +ALIGN function_align +.w32: + sub tlq, 1 + sub tlq, hq + pxor m6, m6 +.w32_loop_init: + mov r5, 2 + lea r3, [base+smooth_weights+16*4] +.w32_loop: + mova m7, [r3] + add r3, 16 + movd m2, [tlq+hq] ; left + pshufb m2, m6 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m7 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + mova m7, [r3] + add r3, 16 + pmaddubsw m2, m7 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq], m0 + add dstq, 16 + dec r5 + jg .w32_loop + lea dstq, [dstq-32+strideq] + sub hd, 1 + jg .w32_loop_init + RET +ALIGN function_align +.w64: + sub tlq, 1 + sub tlq, hq + pxor m6, m6 +.w64_loop_init: + mov r5, 4 + lea r3, [base+smooth_weights+16*8] +.w64_loop: + mova m7, [r3] + add r3, 16 + movd m2, [tlq+hq] ; left + pshufb m2, m6 + punpcklbw m1, m2, m3 ; left, right + punpckhbw m2, m3 + pmaddubsw m0, m1, m4 ; 127 * left - 127 * right + paddw m0, m1 ; 128 * left + 129 * right + pmaddubsw m1, m7 + paddw m1, m5 + paddw m0, m1 + pmaddubsw m1, m2, m4 + paddw m1, m2 + mova m7, [r3] + add r3, 16 + pmaddubsw m2, m7 + paddw m2, m5 + paddw m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq], m0 + add dstq, 16 + dec r5 + jg .w64_loop + lea dstq, [dstq-64+strideq] + sub hd, 1 + jg .w64_loop_init + RET + +;--------------------------------------------------------------------------------------- +;int dav1d_ipred_smooth_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int a); +;--------------------------------------------------------------------------------------- +%macro SMOOTH_2D_END 7 ; src[1-2], mul[1-2], add[1-2], m3 + pmaddubsw m6, m%3, m%1 + mova m0, m6 + pmaddubsw m6, m%4, m%2 + mova m1, m6 +%ifnum %5 + paddw m0, m%5 +%else + paddw m0, %5 +%endif +%ifnum %6 + paddw m1, m%6 +%else + paddw m1, %6 +%endif +%ifnum %7 +%else + mova m3, %7 +%endif + pavgw m0, m2 + pavgw m1, m3 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 +%endmacro + +%macro SMOOTH_OUTPUT_16B 12 ; m1, [buffer1, buffer2, buffer3, buffer4,] [w1, w2,] m3, m7, [m0, m4, m5] + mova m1, [rsp+16*%1] ; top + punpckhbw m6, m1, m0 ; top, bottom + punpcklbw m1, m0 ; top, bottom + pmaddubsw m2, m1, m5 + mova [rsp+16*%2], m1 + paddw m1, m3 ; 1 * top + 255 * bottom + 255 + paddw m2, m1 ; 128 * top + 129 * bottom + 255 + mova [rsp+16*%3], m2 + pmaddubsw m2, m6, m5 + mova [rsp+16*%4], m6 + paddw m6, m3 ; 1 * top + 255 * bottom + 255 + paddw m2, m6 ; 128 * top + 129 * bottom + 255 + mova [rsp+16*%5], m2 + movd m1, [tlq+hq] ; left + pshufb m1, [base+pb_3] ; topleft[-(1 + y)] + punpcklbw m1, m4 ; left, right + pmaddubsw m2, m1, m5 ; 127 * left - 127 * right + paddw m2, m1 ; 128 * left + 129 * right + mova m3, m2 + pmaddubsw m0, m1, %6 ; weights_hor = &dav1d_sm_weights[width]; + pmaddubsw m1, %7 + paddw m2, m3, m0 + paddw m3, m1 + movd m1, [v_weightsq] ; weights_ver = &dav1d_sm_weights[height]; + mova m7, [rsp+16*%9] + pshufb m1, m7 + mova [rsp+16*%8], m3 + mova m4, [rsp+16*%2] + mova m5, [rsp+16*%3] + mova m3, [rsp+16*%4] + mova m7, [rsp+16*%5] + SMOOTH_2D_END 1, 1, 4, 3, 5, 7, [rsp+16*%8] + mova [dstq], m0 + movddup m3, [base+pw_255] ; recovery + mova m0, [rsp+16*%10] ; recovery + mova m4, [rsp+16*%11] ; recovery + mova m5, [rsp+16*%12] ; recovery +%endmacro + +cglobal ipred_smooth_8bpc, 3, 7, 8, -13*16, dst, stride, tl, w, h, v_weights +%define base r6-ipred_smooth_ssse3_table + mov wd, wm + mov hd, hm + LEA r6, ipred_smooth_ssse3_table + movd m4, [tlq+wq] ; right + pxor m2, m2 + pshufb m4, m2 + tzcnt wd, wd + mov r5, tlq + sub r5, hq + movsxd wq, [r6+wq*4] + movddup m5, [base+pb_127_m127] + movd m0, [r5] + pshufb m0, m2 ; bottom + movddup m3, [base+pw_255] + add wq, r6 + lea v_weightsq, [base+smooth_weights+hq*2] ; weights_ver = &dav1d_sm_weights[height] + jmp wq +.w4: + mova m7, [base+ipred_v_shuf] + movd m1, [tlq+1] ; left + pshufd m1, m1, q0000 + sub tlq, 4 + lea r3, [strideq*3] + sub tlq, hq + punpcklbw m1, m0 ; top, bottom + pshufd m6, m7, q1100 + pshufd m7, m7, q3322 + pmaddubsw m2, m1, m5 + paddw m3, m1 ; 1 * top + 255 * bottom + 255 + paddw m2, m3 ; 128 * top + 129 * bottom + 255 + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + movq m1, [base+smooth_weights+4*2] ; weights_hor = &dav1d_sm_weights[width]; + punpcklqdq m1, m1 + mova [rsp+16*2], m1 + mova [rsp+16*3], m4 + mova [rsp+16*4], m6 + mova [rsp+16*5], m5 +.w4_loop: + movd m1, [tlq+hq] ; left + pshufb m1, [base+ipred_h_shuf] + punpcklbw m0, m1, m4 ; left, right + punpckhbw m1, m4 + pmaddubsw m2, m0, m5 ; 127 * left - 127 * right + pmaddubsw m3, m1, m5 + paddw m2, m0 ; 128 * left + 129 * right + paddw m3, m1 + mova m4, [rsp+16*2] + pmaddubsw m0, m4 + pmaddubsw m1, m4 + paddw m2, m0 + paddw m3, m1 + movq m1, [v_weightsq] ; weights_ver = &dav1d_sm_weights[height]; + add v_weightsq, 8 + pshufb m0, m1, m6 + pshufb m1, m7 + mova m4, [rsp+16*0] + mova m5, [rsp+16*1] + SMOOTH_2D_END 0, 1, 4, 4, 5, 5, 3 + mova m4, [rsp+16*3] + mova m6, [rsp+16*4] + mova m5, [rsp+16*5] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + movd [dstq+strideq*2], m0 + psrlq m0, 32 + movd [dstq+r3 ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_loop + RET +ALIGN function_align +.w8: + mova m7, [base+ipred_v_shuf] + movq m1, [tlq+1] ; left + punpcklqdq m1, m1 + sub tlq, 4 + sub tlq, hq + punpcklbw m1, m0 + pshufd m6, m7, q0000 + pshufd m7, m7, q1111 + pmaddubsw m2, m1, m5 + paddw m3, m1 + paddw m2, m3 + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + mova m1, [base+smooth_weights+8*2] ; weights_hor = &dav1d_sm_weights[width]; + mova [rsp+16*2], m1 + mova [rsp+16*3], m4 + mova [rsp+16*4], m6 + mova [rsp+16*5], m5 +.w8_loop: + movd m1, [tlq+hq] ; left + pshufb m1, [base+ipred_h_shuf] + pshufd m1, m1, q1100 + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + pmaddubsw m2, m0, m5 + pmaddubsw m3, m1, m5 + paddw m2, m0 + paddw m3, m1 + mova m4, [rsp+16*2] + pmaddubsw m0, m4 + pmaddubsw m1, m4 + paddw m2, m0 + paddw m3, m1 + movd m1, [v_weightsq] ; weights_ver = &dav1d_sm_weights[height]; + add v_weightsq, 4 + pshufb m0, m1, m6 + pshufb m1, m7 + mova m4, [rsp+16*0] + mova m5, [rsp+16*1] + SMOOTH_2D_END 0, 1, 4, 4, 5, 5, 3 + mova m4, [rsp+16*3] + mova m6, [rsp+16*4] + mova m5, [rsp+16*5] + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + mova m7, [base+ipred_v_shuf] + movu m1, [tlq+1] ; left + sub tlq, 4 + sub tlq, hq + punpckhbw m6, m1, m0 ; top, bottom + punpcklbw m1, m0 ; top, bottom + pshufd m7, m7, q0000 + mova [rsp+16*2], m7 + pmaddubsw m2, m6, m5 + mova [rsp+16*5], m6 + paddw m6, m3 ; 1 * top + 255 * bottom + 255 + paddw m2, m6 ; 128 * top + 129 * bottom + 255 + mova [rsp+16*6], m2 + pmaddubsw m2, m1, m5 + paddw m3, m1 ; 1 * top + 255 * bottom + 255 + mova [rsp+16*0], m1 + paddw m2, m3 ; 128 * top + 129 * bottom + 255 + mova [rsp+16*1], m2 + mova [rsp+16*3], m4 + mova [rsp+16*4], m5 +.w16_loop: + movd m1, [tlq+hq] ; left + pshufb m1, [base+pb_3] ; topleft[-(1 + y)] + punpcklbw m1, m4 ; left, right + pmaddubsw m2, m1, m5 ; 127 * left - 127 * right + paddw m2, m1 ; 128 * left + 129 * right + mova m0, m1 + mova m3, m2 + pmaddubsw m0, [base+smooth_weights+16*2] ; weights_hor = &dav1d_sm_weights[width]; + pmaddubsw m1, [base+smooth_weights+16*3] + paddw m2, m0 + paddw m3, m1 + movd m1, [v_weightsq] ; weights_ver = &dav1d_sm_weights[height]; + add v_weightsq, 2 + mova m7, [rsp+16*2] + pshufb m1, m7 + mova [rsp+16*7], m3 + mova m4, [rsp+16*0] + mova m5, [rsp+16*1] + mova m3, [rsp+16*5] + mova m7, [rsp+16*6] + SMOOTH_2D_END 1, 1, 4, 3, 5, 7, [rsp+16*7] + mova m4, [rsp+16*3] + mova m5, [rsp+16*4] + mova [dstq], m0 + lea dstq, [dstq+strideq] + sub hd, 1 + jg .w16_loop + RET +ALIGN function_align +.w32: + movu m1, [tlq+1] ; top topleft[1 + x] + movu m2, [tlq+17] ; top + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + sub tlq, 4 + sub tlq, hq + mova m7, [base+ipred_v_shuf] + pshufd m7, m7, q0000 + mova [rsp+16*2], m7 + mova [rsp+16*3], m0 + mova [rsp+16*4], m4 + mova [rsp+16*5], m5 +.w32_loop: + SMOOTH_OUTPUT_16B 0, 6, 7, 8, 9, [base+smooth_weights+16*4], [base+smooth_weights+16*5], 10, 2, 3, 4, 5 + add dstq, 16 + SMOOTH_OUTPUT_16B 1, 6, 7, 8, 9, [base+smooth_weights+16*6], [base+smooth_weights+16*7], 10, 2, 3, 4, 5 + lea dstq, [dstq-16+strideq] + add v_weightsq, 2 + sub hd, 1 + jg .w32_loop + RET +ALIGN function_align +.w64: + movu m1, [tlq+1] ; top topleft[1 + x] + movu m2, [tlq+17] ; top + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + movu m1, [tlq+33] ; top + movu m2, [tlq+49] ; top + mova [rsp+16*11], m1 + mova [rsp+16*12], m2 + sub tlq, 4 + sub tlq, hq + mova m7, [base+ipred_v_shuf] + pshufd m7, m7, q0000 + mova [rsp+16*2], m7 + mova [rsp+16*3], m0 + mova [rsp+16*4], m4 + mova [rsp+16*5], m5 +.w64_loop: + SMOOTH_OUTPUT_16B 0, 6, 7, 8, 9, [base+smooth_weights+16*8], [base+smooth_weights+16*9], 10, 2, 3, 4, 5 + add dstq, 16 + SMOOTH_OUTPUT_16B 1, 6, 7, 8, 9, [base+smooth_weights+16*10], [base+smooth_weights+16*11], 10, 2, 3, 4, 5 + add dstq, 16 + SMOOTH_OUTPUT_16B 11, 6, 7, 8, 9, [base+smooth_weights+16*12], [base+smooth_weights+16*13], 10, 2, 3, 4, 5 + add dstq, 16 + SMOOTH_OUTPUT_16B 12, 6, 7, 8, 9, [base+smooth_weights+16*14], [base+smooth_weights+16*15], 10, 2, 3, 4, 5 + lea dstq, [dstq-48+strideq] + add v_weightsq, 2 + sub hd, 1 + jg .w64_loop + RET + +%if ARCH_X86_64 +cglobal ipred_z1_8bpc, 3, 8, 11, 16*12, dst, stride, tl, w, h, angle, dx + %define base r7-$$ + lea r7, [$$] + mova m8, [base+pw_62] + mova m9, [base+pw_64] + mova m10, [base+pw_512] +%else +cglobal ipred_z1_8bpc, 3, 7, 8, -16*13, dst, _, tl, w, h, angle, dx + %define base r1-$$ + %define m8 [base+pw_62] + %define m9 [base+pw_64] + %define m10 [base+pw_512] + %define strideq r3 + %define stridemp dword [rsp+16*12] + mov stridemp, r1 + LEA r1, $$ +%endif + tzcnt wd, wm + movifnidn angled, anglem + movifnidn hd, hm + inc tlq + movsxd wq, [base+ipred_z1_ssse3_table+wq*4] + mov dxd, angled + and dxd, 0x7e + add angled, 165 ; ~90 + lea wq, [base+wq+ipred_z1_ssse3_table] + movzx dxd, word [base+dr_intra_derivative+dxq] + xor angled, 0x4ff ; d = 90 - angle + jmp wq +.w4: + lea r3d, [angleq+88] + test r3d, 0x480 + jnz .w4_no_upsample ; !enable_intra_edge_filter || angle >= 40 + sar r3d, 9 + add r3d, hd + cmp r3d, 8 + jg .w4_no_upsample ; h > 8 || (w == h && is_sm) + mova m1, [tlq-1] + pshufb m0, m1, [base+z_upsample1] + pshufb m1, [base+z_upsample2] + movddup m2, [base+pb_36_m4] + add dxd, dxd + pmaddubsw m0, m2 + pshufd m7, m1, q3333 + movd [rsp+16], m7 ; top[max_base_x] + pmaddubsw m1, m2 + movd m6, dxd + mov r5d, dxd ; xpos + pshufb m6, [base+pw_256] + paddw m1, m0 + movq m0, [tlq] + pmulhrsw m1, m10 + paddw m7, m6, m6 + punpcklqdq m6, m7 ; xpos0 xpos1 + packuswb m1, m1 + punpcklbw m0, m1 + movifnidn strideq, stridemp + mova [rsp], m0 +.w4_upsample_loop: + lea r2d, [r5+dxq] + shr r5d, 6 ; base0 + movq m0, [rsp+r5] + lea r5d, [r2+dxq] + shr r2d, 6 ; base1 + movhps m0, [rsp+r2] + pand m2, m8, m6 ; frac + psubw m1, m9, m2 ; 64-frac + psllw m2, 8 + por m1, m2 ; 64-frac, frac + pmaddubsw m0, m1 + paddw m6, m7 ; xpos += dx + pmulhrsw m0, m10 + packuswb m0, m0 + movd [dstq+strideq*0], m0 + pshuflw m0, m0, q1032 + movd [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w4_upsample_loop + RET +.w4_no_upsample: + mov r3d, 7 ; max_base + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w4_main + lea r3d, [hq+3] + movd m0, r3d + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + pcmpeqb m1, m0, [base+z_filter_wh4] + pand m1, m2 + pcmpgtb m1, [base+z_filter_t_w48+angleq*8] + pmovmskb r5d, m1 + mov r3d, 7 + test r5d, r5d + jz .w4_main ; filter_strength == 0 + mova m3, [tlq-1] + imul r5d, 0x55555555 + movu m7, [base+z_filter_s+8] + shr r5d, 30 ; filter_strength + movddup m0, [base+pb_8] + pminub m7, m0 + pshufb m0, m3, [base+z_filter_s] + movddup m4, [base+z_filter_k-8+r5*8+24*0] + pshufb m3, m7 + movddup m5, [base+z_filter_k-8+r5*8+24*1] + shufps m2, m0, m3, q2121 + movddup m6, [base+z_filter_k-8+r5*8+24*2] + pmaddubsw m0, m4 + pmaddubsw m1, m2, m4 + pmaddubsw m2, m5 + paddd m5, m6 + pmaddubsw m4, m3, m5 + pmaddubsw m3, m6 + paddw m0, m2 + paddw m1, m4 + paddw m0, m3 + pshufd m1, m1, q3333 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + mov r5d, 9 + mov tlq, rsp + cmp hd, 4 + cmovne r3d, r5d + packuswb m0, m1 + mova [tlq], m0 +.w4_main: + add tlq, r3 + movd m5, dxd + movddup m0, [base+z_base_inc] ; base_inc << 6 + movd m7, [tlq] ; top[max_base_x] + shl r3d, 6 + movd m4, r3d + pshufb m5, [base+pw_256] + mov r5d, dxd ; xpos + pshufb m7, [base+pw_m256] + sub r5, r3 + pshufb m4, [base+pw_256] + mova m3, [base+z1_shuf_w4] + paddw m6, m5, m5 + psubw m4, m0 ; max_base_x + punpcklqdq m5, m6 ; xpos0 xpos1 +.w4_loop: + lea r3, [r5+dxq] + sar r5, 6 ; base0 + movq m0, [tlq+r5] + lea r5, [r3+dxq] + sar r3, 6 ; base1 + movhps m0, [tlq+r3] + pand m2, m8, m5 ; frac + psubw m1, m9, m2 ; 64-frac + psllw m2, 8 + pshufb m0, m3 + por m1, m2 ; 64-frac, frac + pmaddubsw m0, m1 + movifnidn strideq, stridemp + pcmpgtw m1, m4, m5 ; base < max_base_x + pmulhrsw m0, m10 + paddw m5, m6 ; xpos += dx + pand m0, m1 + pandn m1, m7 + por m0, m1 + packuswb m0, m0 + movd [dstq+strideq*0], m0 + pshuflw m0, m0, q1032 + movd [dstq+strideq*1], m0 + sub hd, 2 + jz .w4_end + lea dstq, [dstq+strideq*2] + test r5d, r5d + jl .w4_loop + packuswb m7, m7 +.w4_end_loop: + movd [dstq+strideq*0], m7 + movd [dstq+strideq*1], m7 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w4_end_loop +.w4_end: + RET +.w8: + lea r3d, [angleq+88] + and r3d, ~0x7f + or r3d, hd + cmp r3d, 8 + ja .w8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || h > 8 + mova m5, [base+z_upsample1] + movu m3, [base+z_filter_s+6] + movd m4, hd + mova m0, [tlq-1] + movu m1, [tlq+7] + pxor m7, m7 + pshufb m4, m7 + movddup m7, [base+pb_36_m4] + pminub m4, m3 + add dxd, dxd + pshufb m2, m0, m5 + pmaddubsw m2, m7 + pshufb m0, m3 + pmaddubsw m0, m7 + movd m6, dxd + pshufb m3, m1, m5 + pmaddubsw m3, m7 + pshufb m1, m4 + pmaddubsw m1, m7 + pshufb m6, [base+pw_256] + mov r5d, dxd + paddw m2, m0 + paddw m7, m6, m6 + paddw m3, m1 + punpcklqdq m6, m7 ; xpos0 xpos1 + movu m1, [tlq] + pmulhrsw m2, m10 + pmulhrsw m3, m10 + packuswb m2, m3 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + movifnidn strideq, stridemp + mova [rsp+16*0], m0 + mova [rsp+16*1], m1 +.w8_upsample_loop: + lea r2d, [r5+dxq] + shr r5d, 6 ; base0 + movu m0, [rsp+r5] + lea r5d, [r2+dxq] + shr r2d, 6 ; base1 + movu m1, [rsp+r2] + pand m2, m8, m6 + psubw m3, m9, m2 + psllw m2, 8 + por m3, m2 + punpcklqdq m2, m3, m3 ; frac0 + pmaddubsw m0, m2 + punpckhqdq m3, m3 ; frac1 + pmaddubsw m1, m3 + paddw m6, m7 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_upsample_loop + RET +.w8_no_upsample: + lea r3d, [hq+7] + movd m0, r3d + and r3d, 7 + or r3d, 8 ; imin(h+7, 15) + test angled, 0x400 + jnz .w8_main + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + movu m1, [base+z_filter_wh8] + psrldq m3, [base+z_filter_t_w48+angleq*8], 4 + pcmpeqb m1, m0 + pand m1, m2 + pcmpgtb m1, m3 + pmovmskb r5d, m1 + test r5d, r5d + jz .w8_main ; filter_strength == 0 + movd m3, [tlq-1] + movu m0, [tlq+16*0] + imul r5d, 0x55555555 + movu m1, [tlq+16*1] + shr r5d, 30 ; filter_strength + movd m2, [tlq+r3] + lea tlq, [rsp+16*4] + sub r5, 3 + mova [tlq-16*1], m0 + pxor m7, m7 + mova [tlq+16*0], m1 + pshufb m3, m7 + pshufb m2, m7 + mova [tlq-16*2], m3 + movq [tlq+r3-15], m2 + call .filter_edge + sar r5d, 1 + add r5d, 17 + cmp hd, 8 + cmova r3d, r5d +.w8_main: + add tlq, r3 + movd m5, dxd + movd m7, [tlq] + shl r3d, 6 + movu m3, [base+z_filter_s+2] + movd m4, r3d + pshufb m5, [base+pw_256] + mov r5d, dxd + pshufb m7, [base+pw_m256] + sub r5, r3 + pshufb m4, [base+pw_256] + psubw m4, [base+z_base_inc] + mova m6, m5 +.w8_loop: + mov r3, r5 + sar r3, 6 + movu m0, [tlq+r3] + pand m1, m8, m5 + psubw m2, m9, m1 + psllw m1, 8 + pshufb m0, m3 + por m1, m2 + pmaddubsw m0, m1 + pcmpgtw m1, m4, m5 + paddw m5, m6 + pmulhrsw m0, m10 + pand m0, m1 + pandn m1, m7 + por m0, m1 + packuswb m0, m0 + movq [dstq], m0 + dec hd + jz .w8_end + movifnidn strideq, stridemp + add dstq, strideq + add r5, dxq + jl .w8_loop + packuswb m7, m7 +.w8_end_loop: + movq [dstq], m7 + add dstq, strideq + dec hd + jg .w8_end_loop +.w8_end: + RET +.w16: + lea r3d, [hq+15] + movd m0, r3d + and r3d, 15 + or r3d, 16 ; imin(h+15, 31) + test angled, 0x400 + jnz .w16_main + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + movq m3, [base+z_filter_t_w16+angleq*4] + pcmpeqb m1, m0, [base+z_filter_wh16] + pand m1, m2 + pcmpgtb m1, m3 + pmovmskb r5d, m1 + test r5d, r5d + jz .w16_main ; filter_strength == 0 + movd m4, [tlq-1] + movu m0, [tlq+16*0] + imul r5d, 0x24924924 + movu m1, [tlq+16*1] + shr r5d, 30 + movd m2, [tlq+30] + adc r5, -4 ; filter_strength-3 + movd m3, [tlq+r3] + lea tlq, [rsp+16*4] + mova [tlq-16*1], m0 + pxor m7, m7 + mova [tlq+16*0], m1 + pshufb m4, m7 + movd [rsp], m2 + pshufb m3, m7 + mova [tlq-16*2], m4 + movd [tlq+r3-16], m3 + call .filter_edge + cmp hd, 16 + jle .w16_main + pshuflw m0, [rsp], q0000 + sar r5, 1 + movd m1, [base+z_filter_k_tail+4+r5*4] + lea r3d, [r5+33] + pmaddubsw m0, m1 +%if ARCH_X86_64 + pmulhrsw m0, m10 +%else + pmulhrsw m0, m4 +%endif + packuswb m0, m0 + movd [tlq+32], m0 +.w16_main: + add tlq, r3 + movd m5, dxd + movd m7, [tlq] + movd m4, r3d + shl r3d, 6 + pshufb m5, [base+pw_256] + pxor m6, m6 + pshufb m7, m6 + mov r5d, dxd + pshufb m4, m6 + sub r5, r3 + psubb m4, [base+pb_0to15] + mova m6, m5 +.w16_loop: + mov r3, r5 + sar r3, 6 + movu m1, [tlq+r3+0] + pand m0, m8, m5 + movu m2, [tlq+r3+1] + psubw m3, m9, m0 + psllw m0, 8 + por m3, m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + psrlw m3, m5, 6 + packsswb m3, m3 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + paddw m5, m6 + pcmpgtb m2, m4, m3 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + mova [dstq], m0 + dec hd + jz .w16_end + movifnidn strideq, stridemp + add dstq, strideq + add r5, dxq + jl .w16_loop +.w16_end_loop: + mova [dstq], m7 + add dstq, strideq + dec hd + jg .w16_end_loop +.w16_end: + RET +.w32: + lea r3d, [hq+31] + and r3d, 31 + or r3d, 32 ; imin(h+31, 63) + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w32_main + movd m6, [tlq-1] + movu m0, [tlq+16*0] + movu m1, [tlq+16*1] + movu m2, [tlq+16*2] + movu m3, [tlq+16*3] + movd m4, [tlq+62] + movd m5, [tlq+r3] + lea tlq, [rsp+16*6] + mova [tlq-16*3], m0 + pxor m7, m7 + mova [tlq-16*2], m1 + pshufb m6, m7 + mova [tlq-16*1], m2 + xor r5d, r5d ; filter_strength = 3 + mova [tlq+16*0], m3 + movd [rsp], m4 + pshufb m5, m7 + mova [tlq-16*4], m6 + movd [tlq+r3-48], m5 + call .filter_edge + sub tlq, 16*2 + call .filter_edge + cmp hd, 32 + jle .w32_main + pshuflw m0, [rsp], q0000 + movd m1, [base+z_filter_k_tail+4] + add r3d, 2 + pmaddubsw m0, m1 +%if ARCH_X86_64 + pmulhrsw m0, m10 +%else + pmulhrsw m0, m4 +%endif + packuswb m0, m0 + movd [tlq+64], m0 +.w32_main: + add tlq, r3 + movd m0, r3d + movd m7, [tlq] + shl r3d, 6 + movd m5, dxd + pxor m6, m6 + mov r5d, dxd + pshufb m0, m6 + pshufb m5, [base+pw_256] + sub r5, r3 + pshufb m7, m6 + psubb m0, [base+pb_0to15] + movddup m1, [base+pb_m16] + mova [rsp+16*0], m0 + paddb m0, m1 + mova [rsp+16*1], m0 + mova m6, m5 +.w32_loop: + mov r3, r5 + sar r3, 6 + movu m1, [tlq+r3+16*0+0] + pand m0, m8, m5 + movu m2, [tlq+r3+16*0+1] + psubw m3, m9, m0 + psllw m0, 8 + por m3, m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + psrlw m4, m5, 6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packsswb m4, m4 + pcmpgtb m2, [rsp+16*0], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + movu m1, [tlq+r3+16*1+0] + movu m2, [tlq+r3+16*1+1] + mova [dstq+16*0], m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + paddw m5, m6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pcmpgtb m2, [rsp+16*1], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + mova [dstq+16*1], m0 + dec hd + jz .w32_end + movifnidn strideq, stridemp + add dstq, strideq + add r5, dxq + jl .w32_loop +.w32_end_loop: + mova [dstq+16*0], m7 + mova [dstq+16*1], m7 + add dstq, strideq + dec hd + jg .w32_end_loop +.w32_end: + RET +.w64: + lea r3d, [hq+63] + test angled, 0x400 ; !enable_intra_edge_filter + jnz .w64_main + movd m4, [tlq-1] + movu m0, [tlq+16*0] + movu m1, [tlq+16*1] + movu m2, [tlq+16*2] + movu m3, [tlq+16*3] + mova [rsp+16*3], m0 + pxor m7, m7 + mova [rsp+16*4], m1 + pshufb m4, m7 + mova [rsp+16*5], m2 + mova [rsp+16*6], m3 + mova [rsp+16*2], m4 + movu m0, [tlq+16*4] + movu m1, [tlq+16*5] + movu m2, [tlq+16*6] + movu m3, [tlq+16*7] + movd m4, [tlq+r3] + lea tlq, [rsp+16*10] + mova [tlq-16*3], m0 + xor r5d, r5d ; filter_strength = 3 + mova [tlq-16*2], m1 + pshufb m4, m7 + mova [tlq-16*1], m2 + mova [tlq+16*0], m3 + movd [tlq+r3-16*7], m4 + cmp hd, 64 + jl .w64_filter96 ; skip one call if the last 32 bytes aren't used + call .filter_edge +.w64_filter96: + sub tlq, 16*2 + call .filter_edge + sub tlq, 16*2 + call .filter_edge + sub tlq, 16*2 + call .filter_edge +.w64_main: + add tlq, r3 + movd m0, r3d + movd m7, [tlq] + shl r3d, 6 + movd m5, dxd + pxor m6, m6 + mov r5d, dxd + pshufb m0, m6 + sub r5, r3 + pshufb m5, [base+pw_256] + pshufb m7, m6 + psubb m0, [base+pb_0to15] + movddup m1, [base+pb_m16] + mova [rsp+16*0], m0 + paddb m0, m1 + mova [rsp+16*1], m0 + paddb m0, m1 + mova [rsp+16*2], m0 + paddb m0, m1 + mova [rsp+16*3], m0 + mova m6, m5 +.w64_loop: + mov r3, r5 + sar r3, 6 + movu m1, [tlq+r3+16*0+0] + pand m0, m8, m5 + movu m2, [tlq+r3+16*0+1] + psubw m3, m9, m0 + psllw m0, 8 + por m3, m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + psrlw m4, m5, 6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packsswb m4, m4 + pcmpgtb m2, [rsp+16*0], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + movu m1, [tlq+r3+16*1+0] + movu m2, [tlq+r3+16*1+1] + mova [dstq+16*0], m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pcmpgtb m2, [rsp+16*1], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + movu m1, [tlq+r3+16*2+0] + movu m2, [tlq+r3+16*2+1] + mova [dstq+16*1], m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pcmpgtb m2, [rsp+16*2], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + movu m1, [tlq+r3+16*3+0] + movu m2, [tlq+r3+16*3+1] + mova [dstq+16*2], m0 + punpcklbw m0, m1, m2 + pmaddubsw m0, m3 + punpckhbw m1, m2 + pmaddubsw m1, m3 + paddw m5, m6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pcmpgtb m2, [rsp+16*3], m4 + packuswb m0, m1 + pand m0, m2 + pandn m2, m7 + por m0, m2 + mova [dstq+16*3], m0 + dec hd + jz .w64_end + movifnidn strideq, stridemp + add dstq, strideq + add r5, dxq + jl .w64_loop +.w64_end_loop: + mova [dstq+16*0], m7 + mova [dstq+16*1], m7 + mova [dstq+16*2], m7 + mova [dstq+16*3], m7 + add dstq, strideq + dec hd + jg .w64_end_loop +.w64_end: + RET +ALIGN function_align +.filter_edge: ; 32 pixels/iteration + movddup m7, [base+z_filter_k+8*2+r5*8+24*0] + movu m2, [tlq-18] + movu m1, [tlq-17] + movu m3, [tlq- 2] + movu m4, [tlq- 1] + punpcklbw m0, m2, m1 + pmaddubsw m0, m7 + punpckhbw m2, m1 + pmaddubsw m2, m7 + punpcklbw m1, m3, m4 + pmaddubsw m1, m7 + punpckhbw m3, m4 + pmaddubsw m3, m7 + movddup m7, [base+z_filter_k+8*2+r5*8+24*1] + mova m5, [tlq-16] + movu m6, [tlq-15] + punpcklbw m4, m5, m6 + pmaddubsw m4, m7 + punpckhbw m5, m6 + pmaddubsw m5, m7 + paddw m0, m4 + paddw m2, m5 + mova m5, [tlq+ 0] + movu m6, [tlq+ 1] + punpcklbw m4, m5, m6 + pmaddubsw m4, m7 + punpckhbw m5, m6 + pmaddubsw m5, m7 + paddw m1, m4 + paddw m3, m5 + test r5d, r5d + jnz .filter_end ; 3-tap + movddup m7, [base+z_filter_k+8*8] + movu m5, [tlq-14] + movu m6, [tlq+ 2] + punpcklbw m4, m5, m5 + pmaddubsw m4, m7 + punpckhbw m5, m5 + pmaddubsw m5, m7 + paddw m0, m4 + paddw m2, m5 + punpcklbw m5, m6, m6 + pmaddubsw m5, m7 + punpckhbw m6, m6 + pmaddubsw m6, m7 + paddw m1, m5 + paddw m3, m6 +.filter_end: +%if ARCH_X86_64 + REPX {pmulhrsw x, m10}, m0, m2, m1, m3 +%else + mova m4, m10 + REPX {pmulhrsw x, m4 }, m0, m2, m1, m3 +%endif + packuswb m0, m2 + packuswb m1, m3 + mova [tlq+16*0], m0 + mova [tlq+16*1], m1 + ret + +%if ARCH_X86_64 +cglobal ipred_z2_8bpc, 4, 12, 13, 16*16, dst, stride, tl, w, h, angle, dx, _, dy + %define base r7-$$ + %define maxwm r6m + %define maxhm r7m + lea r7, [$$] + mov hd, hm + mova m8, [base+pw_62] + mova m9, [base+pw_64] + lea r9d, [wq-4] + mova m10, [base+pw_512] + shl r9d, 6 + mova m11, [base+z1_shuf_w4] + or r9d, hd + mova m12, [base+z2_h_shuf] +%else +cglobal ipred_z2_8bpc, 4, 7, 8, -16*20, dst, _, tl, w, h, angle, dx + %define base r1-$$ + %define m8 [base+pw_62] + %define m9 [base+pw_64] + %define m10 [base+pw_512] + %define m11 [rsp+16*16] + %define m12 [rsp+16*17] + %define r8 [rsp+16*6+4*1] + %define r9b byte [rsp+16*18+4*0] + %define r9d dword [rsp+16*18+4*0] + %define r10d dword [rsp+16*18+4*1] + %define r11d dword [rsp+16*18+4*2] + %define maxwm [rsp+16*18+4*3] + %define maxhm [rsp+16*19+4*0] + %define stridemp [rsp+16*19+4*1] + %define strideq r3 + %define dyd r4 + %define dyq r4 + mov stridemp, r1 + mov r1d, r6m + mov r4d, r7m + mov maxwm, r1d + mov maxhm, r4d + LEA r1, $$ + lea hd, [wq-4] + mova m0, [base+z1_shuf_w4] + shl hd, 6 + mova m1, [base+z2_h_shuf] + or hd, hm + mova m11, m0 + mov r9d, hd + mova m12, m1 +%endif + tzcnt wd, wd + movifnidn angled, anglem + movsxd wq, [base+ipred_z2_ssse3_table+wq*4] +%if ARCH_X86_64 + movzx dxd, angleb +%else + movzx dxd, byte anglem +%endif + xor angled, 0x400 + mova m0, [tlq-16*4] + mov dyd, dxd + mova m1, [tlq-16*3] + neg dxq + mova m2, [tlq-16*2] + and dyd, ~1 + mova m3, [tlq-16*1] + and dxq, ~1 + movd m4, [tlq] + movu m5, [tlq+16*0+1] + movu m6, [tlq+16*1+1] + movzx dyd, word [base+dr_intra_derivative+dyq-90] ; angle - 90 + movzx dxd, word [base+dr_intra_derivative+dxq+180] ; 180 - angle + mova [rsp+16*2], m0 + pxor m7, m7 + mova [rsp+16*3], m1 + pshufb m4, m7 + mova [rsp+16*4], m2 + lea wq, [base+ipred_z2_ssse3_table+wq] + mova [rsp+16*5], m3 + neg dxd + mova [rsp+16*6], m4 + or dyd, 4<<16 + mova [rsp+16*7], m4 + mova [rsp+16*8], m5 + mova [rsp+16*9], m6 + movq m0, [base+z_base_inc+2] + movsldup m1, [base+z2_dy_offset] + movq m2, [base+pw_256] ; 4<<6 + movq [rsp+16*14+8*0], m0 + movq [rsp+16*15+8*0], m1 + movq [rsp+16*15+8*1], m2 +%if ARCH_X86_64 + lea r10d, [dxq+(128<<6)] ; xpos +%else + mov [rsp+16*7+4*1], dyd + lea r4d, [dxq+(128<<6)] + mov r10d, r4d + movzx hd, r9b +%endif + mov r11d, (128-4)<<6 + jmp wq +.w4: + test angled, 0x400 + jnz .w4_main + movd m5, [tlq+4] + lea r3d, [hq+2] + add angled, 1022 + pshufb m5, m7 + shl r3d, 6 + movd [rsp+16*8+4], m5 + test r3d, angled + jnz .w4_no_upsample_above ; angle >= 130 || h > 8 || (is_sm && h == 8) + call .upsample_above + sub angled, 1075 ; angle - 53 + lea r3d, [hq+3] + xor angled, 0x7f ; 180 - angle + movd m0, r3d + movd m6, angled + shr angled, 8 ; is_sm << 1 + pshufb m0, m7 + pshufb m6, m7 + pcmpeqb m0, [base+z_filter_wh4] + pand m6, m0 + pcmpgtb m6, [base+z_filter_t_w48+angleq*8] + jmp .w8_filter_left +.upsample_above: ; w4/w8 + movq m3, [rsp+gprsize+16*8-2] + movq m1, [rsp+gprsize+16*8-1] + movq m0, [rsp+gprsize+16*8+0] + movq m4, [rsp+gprsize+16*8+1] + movddup m5, [base+pb_36_m4] + punpcklbw m1, m3 + punpcklbw m2, m0, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 +%if ARCH_X86_64 + mova m11, [base+pb_0to15] + lea r10d, [r10+dxq+(1<<6)] + mov r11d, (128-7)<<6 +%else + mova m3, [base+pb_0to15] + mov r3d, [rsp+gprsize+16*18+4*1] + mov dword [rsp+gprsize+16*18+4*2], (128-7)<<6 + lea r3d, [r3+dxq+(1<<6)] + mov [rsp+gprsize+16*18+4*1], r3d + mova [rsp+gprsize+16*16], m3 +%endif + add dxd, dxd + paddw m1, m2 + pmulhrsw m1, m10 + movq m2, [rsp+gprsize+16*14] + paddw m2, m2 + movq [rsp+gprsize+16*14], m2 + packuswb m1, m1 + punpcklbw m1, m0 + mova [rsp+gprsize+16*8], m1 + ret +.w4_no_upsample_above: + lea r3d, [hq+3] + mov [rsp], angled + sub angled, 1112 ; angle - 90 + movd m0, r3d + mov r3d, 90 + movd m1, angled + sub r3d, angled ; 180 - angle + shr angled, 8 ; is_sm << 1 + movu m3, [base+z_filter_wh4] + mova m4, [base+z_filter_t_w48+angleq*8] + call .w8_filter_top + mov angled, [rsp] + lea r3d, [hq+2] + sub angled, 139 + shl r3d, 6 + test r3d, angled + jnz .w8_filter_left ; angle <= 140 || h > 8 || (is_sm && h == 8) +.upsample_left: ; w4/w8 + neg hq + movd m0, [tlq+hq] + pshufb m0, m7 + movd [rsp+16*6+hq-4], m0 + movq m3, [rsp+16*5+7] + movq m0, [rsp+16*5+8] + movq m2, [rsp+16*5+9] + movq m4, [rsp+16*5+10] + movddup m5, [base+pb_36_m4] + punpcklbw m1, m0, m3 + punpcklbw m2, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + movshdup m3, [base+z2_dy_offset] +%if ARCH_X86_64 + mova m12, [base+z2_upsample] + add dyd, dyd +%else + mova m4, [base+z2_upsample] + shl dword [rsp+16*7+4*1], 1 + mova m12, m4 +%endif + paddw m1, m2 + pmulhrsw m1, m10 + movq [rsp+16*15], m3 + packuswb m1, m1 + punpcklbw m0, m1 + mova [rsp+16*5], m0 +.w4_main: + movd m6, dxd +%if ARCH_X86_64 + movd m3, dyd +%else + movd m3, [rsp+16*7+4*1] +%endif + movddup m0, [rsp+16*14+8*0] + pshufb m6, [base+pw_256] + paddw m7, m6, m6 + movq m5, [base+pw_m1to4] + pshuflw m4, m3, q0000 + punpcklqdq m6, m7 + pmullw m4, m5 + pshuflw m3, m3, q1111 + paddw m6, m0 + pshuflw m0, m4, q3333 + psubw m4, [rsp+16*15] + movq [rsp+16*6+8*1], m3 + movq [rsp+8*1], m0 ; dy*4 +%if ARCH_X86_64 + mov r8, dstq +%endif +.w4_loop0: +%if ARCH_X86_32 + mov r8, dstq +%endif + mova [rsp+16*12], m6 + mov r2d, r10d + movq [rsp+8*0], m4 + pand m0, m4, m8 + psraw m4, 6 + psubw m1, m9, m0 + psllw m0, 8 + por m0, m1 ; 64-frac_y, frac_y + movq [rsp+8*3], m0 + pabsw m4, m4 + movq [rsp+8*2], m4 + movzx hd, r9b +.w4_loop: + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x0 + movq m0, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x1 + movhps m0, [rsp+r3] + lea r3d, [r2+dxq] + shr r2d, 6 ; base_x2 + movq m1, [rsp+r2] + lea r2d, [r3+dxq] + shr r3d, 6 ; base_x3 + movhps m1, [rsp+r3] + pand m2, m8, m6 + paddsw m5, m6, m7 + psubw m3, m9, m2 + psllw m2, 8 + pshufb m0, m11 + por m2, m3 + pmaddubsw m0, m2 + pand m2, m8, m5 + psubw m3, m9, m2 + psllw m2, 8 + pshufb m1, m11 + por m2, m3 + pmaddubsw m1, m2 + cmp r3d, 127 ; topleft + jge .w4_toponly + movzx r3d, byte [rsp+8*2+0] ; base_y0 + movq m3, [rsp+r3] + movzx r3d, byte [rsp+8*2+2] ; base_y1 + movhps m3, [rsp+r3] + movzx r3d, byte [rsp+8*2+4] ; base_y2 + movq m4, [rsp+r3] + movzx r3d, byte [rsp+8*2+6] ; base_y3 + movhps m4, [rsp+r3] + pshufb m3, m12 + pshufb m4, m12 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + movddup m4, [rsp+8*3] + pmaddubsw m2, m4 + pmaddubsw m3, m4 + psraw m6, 15 ; base_x < topleft + pand m2, m6 + pandn m6, m0 + por m0, m2, m6 + psraw m6, m5, 15 + pand m3, m6 + pandn m6, m1 + por m1, m3, m6 +.w4_toponly: + pmulhrsw m0, m10 + pmulhrsw m1, m10 + movifnidn strideq, stridemp + packuswb m0, m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + punpckhqdq m0, m0 + movd [dstq+strideq*0], m0 + psrlq m0, 32 + movd [dstq+strideq*1], m0 + sub hd, 4 + jz .w4_end + movq m4, [rsp+8*2] + movq m3, [rsp+16*6+8*1] + paddw m6, m5, m7 ; xpos += dx + psubw m4, m3 + movq [rsp+8*2], m4 + lea dstq, [dstq+strideq*2] + cmp r2d, r11d + jge .w4_loop + movddup m5, [rsp+8*3] +.w4_leftonly_loop: + movzx r3d, byte [rsp+8*2+0] ; base_y0 + movq m1, [rsp+r3] + movzx r3d, byte [rsp+8*2+2] ; base_y1 + movhps m1, [rsp+r3] + movzx r3d, byte [rsp+8*2+4] ; base_y2 + movq m2, [rsp+r3] + movzx r3d, byte [rsp+8*2+6] ; base_y3 + movhps m2, [rsp+r3] + psubw m4, m3 + pshufb m1, m12 + pshufb m2, m12 + movq [rsp+8*2], m4 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + movifnidn strideq, stridemp + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + punpckhqdq m0, m0 + movd [dstq+strideq*0], m0 + psrlq m0, 32 + movd [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 4 + jg .w4_leftonly_loop +.w4_end: + sub r9d, 1<<8 + jl .w4_ret + movq m4, [rsp+8*1] +%if ARCH_X86_64 + add r8, 4 + mov dstq, r8 +%else + mov dstq, r8 + add dstq, 4 +%endif + paddw m4, [rsp+8*0] ; base_y += 4*dy + movzx r3d, word [rsp+16*15+8*1] + add r10d, r3d + movddup m6, [rsp+16*15+8*1] + paddw m6, [rsp+16*12] ; base_x += (4 << upsample_above) + jmp .w4_loop0 +.w4_ret: + RET +.w8: + test angled, 0x400 + jnz .w4_main + movd m5, [tlq+8] + lea r3d, [angleq+126] + pshufb m5, m7 +%if ARCH_X86_64 + mov r3b, hb +%else + xor r3b, r3b + or r3d, hd +%endif + movd [rsp+16*8+8], m5 + cmp r3d, 8 + ja .w8_no_upsample_above ; angle >= 130 || h > 8 || is_sm + call .upsample_above + sub angled, 53 + lea r3d, [hq+7] + xor angled, 0x7f ; 180 - angle + movu m1, [base+z_filter_wh8] + movd m0, r3d + movd m6, angled + shr angled, 8 ; is_sm << 1 + psrldq m2, [base+z_filter_t_w48+angleq*8], 4 + pshufb m0, m7 + pshufb m6, m7 + pcmpeqb m0, m1 + pand m6, m0 + pcmpgtb m6, m2 +%if ARCH_X86_64 + movq [rsp+16*15+8*1], m10 ; 8<<6 +%else + movq m0, m10 + movq [rsp+16*15+8*1], m0 +%endif + jmp .w8_filter_left +.w8_no_upsample_above: + lea r3d, [hq+7] + mov [rsp], angled + sub angled, 90 + movd m0, r3d + mov r3d, 90 + movd m1, angled + sub r3d, angled ; 180 - angle + shr angled, 8 ; is_sm << 1 + movu m3, [base+z_filter_wh8] + psrldq m4, [base+z_filter_t_w48+angleq*8], 4 + call .w8_filter_top + mov r3d, [rsp] + sub r3d, 141 +%if ARCH_X86_64 + mov r3b, hb +%else + xor r3b, r3b + or r3d, hd +%endif + cmp r3d, 8 + jbe .upsample_left ; angle > 140 && h <= 8 && !is_sm +.w8_filter_left: + pmovmskb r5d, m6 + test r5d, r5d + jz .w4_main + imul r5d, 0x55555555 + mov r3, tlq + shr r5d, 30 + sub r5, 3 ; filter_strength-3 + jmp .filter_left +.w8_filter_top: + movd m6, r3d + REPX {pshufb x, m7}, m0, m1, m6 + pcmpeqb m0, m3 + pand m1, m0 + pand m6, m0 + pcmpgtb m1, m4 + pcmpgtb m6, m4 + pmovmskb r5d, m1 + test r5d, r5d + jz .w8_filter_top_end ; filter_strength == 0 + imul r5d, 0x55555555 + movq m0, [rsp+gprsize+16*8-2] + shr r5d, 30 + movq m1, [rsp+gprsize+16*8-1] + sub r5, 3 ; filter_strength-3 + movddup m7, [base+z_filter_k+8*2+r5*8+24*0] + punpcklbw m0, m1 + pmaddubsw m0, m7 + movq m1, [rsp+gprsize+16*8+0] + movq m2, [rsp+gprsize+16*8+1] + movddup m7, [base+z_filter_k+8*2+r5*8+24*1] + punpcklbw m1, m2 + pmaddubsw m1, m7 + movq m2, [rsp+gprsize+16*8+2] + movddup m7, [base+z_filter_k+8*2+r5*8+24*2] + punpcklbw m2, m2 + pmaddubsw m2, m7 + paddw m0, m1 + paddw m0, m2 +%if ARCH_X86_64 + mov r3d, r7m ; maxw, offset due to call +%else + mov r3d, [rsp+gprsize+16*18+4*3] +%endif + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + movq [rsp+gprsize+16*8], m0 + cmp r3d, 8 + jge .w8_filter_top_end + movq m0, [tlq+r3+1] + movq [rsp+gprsize+r3+16*8], m0 +.w8_filter_top_end: + ret +.w16: + test angled, 0x400 + jnz .w4_main + lea r3d, [hq+15] + sub angled, 90 + movd m0, r3d + mov r3d, 90 + movd m1, angled + sub r3d, angled ; 180 - angle + shr angled, 8 ; is_sm << 1 + movd m6, r3d + REPX {pshufb x, m7}, m0, m1, m6 + movq m3, [base+z_filter_t_w16+angleq*4] + pcmpeqb m0, [base+z_filter_wh16] + pand m1, m0 + pand m6, m0 + pcmpgtb m1, m3 + pcmpgtb m6, m3 + pmovmskb r5d, m1 + mov r3, tlq + test r5d, r5d + jz .w16_filter_left ; filter_strength == 0 + imul r5d, 0x24924924 + pshufb m5, [base+z_filter_t_w16] ; tlq[16] + shr r5d, 30 + adc r5, -4 ; filter_strength-3 + movd [rsp+16*9], m5 + movddup m7, [base+z_filter_k+8*2+r5*8+24*0] + movu m1, [rsp+16*8-2] + movu m2, [rsp+16*8-1] + punpcklbw m0, m1, m2 + pmaddubsw m0, m7 + punpckhbw m1, m2 + pmaddubsw m1, m7 + movddup m7, [base+z_filter_k+8*2+r5*8+24*1] + mova m3, [rsp+16*8+0] + movu m4, [rsp+16*8+1] + punpcklbw m2, m3, m4 + pmaddubsw m2, m7 + punpckhbw m3, m4 + pmaddubsw m3, m7 + paddw m0, m2 + paddw m1, m3 + test r5d, r5d + jnz .w16_filter_end ; 3-tap + movddup m7, [base+z_filter_k+8*8] + movu m3, [rsp+16*8+2] + punpcklbw m2, m3, m3 + pmaddubsw m2, m7 + punpckhbw m3, m3 + pmaddubsw m3, m7 + paddw m0, m2 + paddw m1, m3 +.w16_filter_end: + mov r2d, maxwm + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + mova [rsp+16*8], m0 + cmp r2d, 16 + jge .w16_filter_left + movu m0, [r3+r2+1] + movu [rsp+r2+16*8], m0 +.w16_filter_left: + pmovmskb r5d, m6 + test r5d, r5d + jz .w4_main + imul r5d, 0x24924924 + shr r5d, 30 + adc r5, -4 ; filter_strength-3 + jmp .filter_left +.w32: + test angled, 0x400 + jnz .w4_main + pshufb m6, [base+z_filter_t_w16] ; tlq[32] + mov r3, tlq + lea tlq, [rsp+16*9] + movd [tlq+16*1], m6 + xor r5d, r5d ; filter_strength = 3 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + mova m0, [tlq+16*0] + mova m1, [tlq+16*1] + mov r2d, maxwm + mova [rsp+16*8], m0 + mova [rsp+16*9], m1 + cmp r2d, 32 + jge .filter_left + movu m0, [r3+r2+16*0+1] + movu m1, [r3+r2+16*1+1] + movu [rsp+r2+16*8], m0 + movu [rsp+r2+16*9], m1 + jmp .filter_left +.w64: + movu m0, [tlq+16*2+1] + movu m1, [tlq+16*3+1] + mova [rsp+16*10], m0 + mova [rsp+16*11], m1 + test angled, 0x400 + jnz .w4_main + pshufb m1, [base+z_filter_t_w16] ; tlq[64] + mov r3, tlq + lea tlq, [rsp+16*11] + movd [tlq+16*1], m1 + xor r5d, r5d ; filter_strength = 3 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sub tlq, 16*2 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + mova m0, [tlq+16*0] + mova m1, [tlq+16*1] + mova m2, [tlq+16*2] + mova m3, [tlq+16*3] + mov r2d, maxwm + mova [rsp+16* 8], m0 + mova [rsp+16* 9], m1 + mova [rsp+16*10], m2 + mova [rsp+16*11], m3 + cmp r2d, 64 + jge .filter_left + movu m0, [r3+r2+16*0+1] + movu m1, [r3+r2+16*1+1] + movu [rsp+r2+16* 8], m0 + movu [rsp+r2+16* 9], m1 + cmp r2d, 32 + jge .filter_left + movu m0, [r3+r2+16*2+1] + movu m1, [r3+r2+16*3+1] + movu [rsp+r2+16*10], m0 + movu [rsp+r2+16*11], m1 +.filter_left: + neg hq + movd m0, [r3+hq] + pxor m1, m1 + pshufb m0, m1 + movd [rsp+16*6+hq-4], m0 + lea tlq, [rsp+16*5] + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + cmp hd, -32 + jge .filter_left_end + sub tlq, 16*2 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + mova m0, [tlq+16*0] + mova m1, [tlq+16*1] + mova [rsp+16*2], m0 + mova [rsp+16*3], m1 +.filter_left_end: + mov r2d, maxhm + mova m0, [rsp+16*5] + mova m1, [rsp+16*6] + mova m2, [rsp+16*7] + neg r2 + mova [rsp+16*4], m0 + mova [rsp+16*5], m1 + mova [rsp+16*6], m2 + cmp r2d, hd + jle .w4_main + movu m0, [r3+r2-16*2] + movu m1, [r3+r2-16*1] + movu [rsp+r2+16*4], m0 + movu [rsp+r2+16*5], m1 + cmp r2d, -32 + jle .w4_main + movu m0, [r3+r2-16*4] + movu m1, [r3+r2-16*3] + movu [rsp+r2+16*2], m0 + movu [rsp+r2+16*3], m1 + jmp .w4_main + +%if ARCH_X86_64 +cglobal ipred_z3_8bpc, 4, 9, 11, 16*10, dst, stride, tl, w, h, angle, dy, _, org_w + %define base r7-$$ + lea r7, [$$] + mova m8, [base+pw_62] + mova m9, [base+pw_64] + mova m10, [base+pw_512] + mov org_wd, wd +%else +cglobal ipred_z3_8bpc, 4, 7, 8, -16*10, dst, stride, tl, w, h, angle, dy + %define base r1-$$ + %define m8 [base+pw_62] + %define m9 [base+pw_64] + %define m10 [base+pw_512] + %define org_wd r5 + %define org_wq r5 + mov [dstq+strideq*0], strideq + mov [dstq+strideq*1], wd + LEA r1, $$ +%endif + tzcnt hd, hm + movifnidn angled, anglem + dec tlq + movsxd hq, [base+ipred_z3_ssse3_table+hq*4] + sub angled, 180 + mov dyd, angled + neg dyd + xor angled, 0x400 + or dyq, ~0x7e + lea hq, [base+ipred_z3_ssse3_table+hq] + movzx dyd, word [base+dr_intra_derivative+45*2-1+dyq] + jmp hq +.h4: + lea r4d, [angleq+88] + test r4d, 0x480 + jnz .h4_no_upsample ; !enable_intra_edge_filter || angle >= 40 + sar r4d, 9 + add r4d, wd + cmp r4d, 8 + jg .h4_no_upsample ; w > 8 || (w == 8 && is_sm) + movu m3, [tlq-7] + movu m1, [base+z_upsample1-4] + movu m4, [base+z_filter_s+2] + pshufb m0, m3, m1 + pxor m1, m1 + pshufb m2, m3, m1 + pshufb m1, m3, m4 + mova [rsp+16], m2 ; top[max_base_y] + movddup m2, [base+pb_36_m4] + add dyd, dyd + pmaddubsw m0, m2 + pmaddubsw m1, m2 + movd m5, dyd + mov r5d, dyd + pshufb m5, [base+pw_256] + paddw m0, m1 + pmulhrsw m0, m10 + shl wd, 2 + mov tlq, rsp + sub rsp, wq + packuswb m0, m0 + punpcklbw m0, m3 + paddw m6, m5, m5 + punpcklqdq m5, m6 + pshufb m0, [base+pb_15to0] + mova [tlq], m0 +.h4_upsample_loop: + lea r4d, [r5+dyq] + shr r5d, 6 + movq m0, [tlq+r5] + lea r5d, [r4+dyq] + shr r4d, 6 + movhps m0, [tlq+r4] + pand m2, m8, m5 + psubw m1, m9, m2 + psllw m2, 8 + por m1, m2 + pmaddubsw m0, m1 + paddw m5, m6 + pmulhrsw m0, m10 + packuswb m0, m0 + movq [rsp+wq-8], m0 + sub wd, 8 + jg .h4_upsample_loop + jmp .h4_transpose +.h4_no_upsample: + mov r4d, 7 + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h4_main + lea r4d, [wq+3] + movd m0, r4d + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + pcmpeqb m1, m0, [base+z_filter_wh4] + pand m1, m2 + pcmpgtb m1, [base+z_filter_t_w48+angleq*8] + pmovmskb r5d, m1 + mov r4d, 7 + test r5d, r5d + jz .h4_main ; filter_strength == 0 + movu m2, [tlq-7] + imul r5d, 0x55555555 + movu m3, [base+z_filter_s-2] + shr r5d, 30 ; filter_strength + mova m4, [base+z_upsample2] + movddup m5, [base+z_filter_k-8+r5*8+24*0] + movddup m6, [base+z_filter_k-8+r5*8+24*1] + movddup m7, [base+z_filter_k-8+r5*8+24*2] + pshufb m0, m2, m3 + shufps m3, m4, q2121 + pmaddubsw m1, m0, m5 + pmaddubsw m0, m6 + pshufb m5, m2, m3 + pmaddubsw m3, m5, m6 + pmaddubsw m5, m7 + pshufb m2, m4 + pmaddubsw m2, m7 + paddw m0, m1 + paddw m1, m3 + paddw m0, m5 + paddw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + lea r2d, [r4+2] + cmp wd, 4 + cmovne r4d, r2d + pshufd m0, m0, q0000 + lea tlq, [rsp+15] + packuswb m0, m1 + mova [rsp], m0 +.h4_main: + movd m5, dyd + movddup m0, [base+z_base_inc] ; base_inc << 6 + sub tlq, r4 + shl r4d, 6 + movd m7, [tlq] + movd m4, r4d + pshufb m5, [base+pw_256] + neg dyq + pshufb m7, [base+pw_m256] + mova m3, [base+z3_shuf_h4] + lea r5, [dyq+r4+63] ; ypos + pshufb m4, [base+pw_256] + psubw m4, m0 ; max_base_y + shl wd, 2 + paddw m6, m5, m5 + sub rsp, wq + punpcklqdq m5, m6 +.h4_loop: + lea r4, [r5+dyq] + sar r5, 6 + movq m0, [tlq+r5-4] + lea r5, [r4+dyq] + sar r4, 6 + movhps m0, [tlq+r4-4] + pand m2, m8, m5 + psubw m1, m9, m2 + psllw m2, 8 + pshufb m0, m3 + por m1, m2 + pmaddubsw m0, m1 + pcmpgtw m1, m4, m5 + paddw m5, m6 + pmulhrsw m0, m10 + pand m0, m1 + pandn m1, m7 + por m0, m1 + packuswb m0, m0 + movq [rsp+wq-8], m0 + sub wd, 8 + jz .h4_transpose + test r5d, r5d + jg .h4_loop + packuswb m7, m7 +.h4_end_loop: + movq [rsp+wq-8], m7 + sub wd, 8 + jg .h4_end_loop +.h4_transpose: + mova m1, [base+z_transpose4] +%if ARCH_X86_32 + mov strideq, [dstq] + mov org_wd, [dstq+strideq] +%endif + lea r2, [strideq*3] + lea dstq, [dstq+org_wq-4] +.h4_transpose_loop: + mova m0, [rsp] + add rsp, 16 + pshufb m0, m1 + movd [dstq+strideq*0], m0 + pshuflw m2, m0, q1032 + movd [dstq+strideq*1], m2 + punpckhqdq m0, m0 + movd [dstq+strideq*2], m0 + psrlq m0, 32 + movd [dstq+r2 ], m0 + sub dstq, 4 + sub org_wd, 4 + jg .h4_transpose_loop + RET +.h8: + lea r4d, [angleq+88] + and r4d, ~0x7f + or r4d, wd + cmp r4d, 8 + ja .h8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || w > 8 + mova m4, [tlq-15] + and r4d, 4 + movu m3, [tlq- 9] + movd m1, r4d + movu m2, [base+z_filter_s+2] + pxor m0, m0 + movu m5, [base+z_filter_s+6] + movddup m7, [base+pb_36_m4] + pshufb m1, m0 ; w & 4 + movu m0, [base+z_upsample1-4] + pmaxub m1, m0 ; clip 4x8 + add dyd, dyd + pshufb m0, m4, m1 + pmaddubsw m0, m7 + pshufb m1, m4, m2 + pmaddubsw m1, m7 + pshufb m2, m3, [base+z_upsample1] + pmaddubsw m2, m7 + pshufb m3, m5 + pmaddubsw m3, m7 + movd m5, dyd + neg dyq + paddw m1, m0 + paddw m2, m3 + pmulhrsw m1, m10 + pmulhrsw m2, m10 + shl wd, 3 + lea tlq, [rsp+16] + pshufb m5, [base+pw_256] + sub rsp, wq + packuswb m1, m2 + lea r5, [dyq+63] + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + mova [tlq-16*1], m0 + mova [tlq-16*0], m1 + paddw m6, m5, m5 + punpcklqdq m5, m6 +.h8_upsample_loop: + lea r4, [r5+dyq] + sar r5, 6 + movu m0, [tlq+r5] + lea r5, [r4+dyq] + sar r4, 6 + movu m1, [tlq+r4] + pand m3, m8, m5 + psubw m2, m9, m3 + psllw m2, 8 + por m3, m2 + pshufd m2, m3, q1010 + pmaddubsw m0, m2 + punpckhqdq m3, m3 + pmaddubsw m1, m3 + paddw m5, m6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m1, m0 + mova [rsp+wq-16], m1 + sub wd, 16 + jg .h8_upsample_loop + jmp .h8_transpose +.h8_no_upsample: + lea r4d, [wq+7] + movd m0, r4d + and r4d, 7 + or r4d, 8 ; imin(w+7, 15) + test angled, 0x400 + jnz .h8_main + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + movu m1, [base+z_filter_wh8] + psrldq m3, [base+z_filter_t_w48+angleq*8], 4 + pcmpeqb m1, m0 + pand m1, m2 + pcmpgtb m1, m3 + pmovmskb r5d, m1 + test r5d, r5d + jz .h8_main ; filter_strength == 0 + mova m0, [tlq-15] + imul r5d, 0x55555555 + movd m1, [tlq+1] + neg r4 + movd m2, [tlq+r4] + shr r5d, 30 + pxor m7, m7 + lea tlq, [rsp+16*2] + sub r5, 3 ; filter_strength-3 + mova [tlq+16*0], m0 + pshufb m1, m7 + mova [tlq+16*1], m1 + pshufb m2, m7 + movq [tlq+r4+8], m2 + neg r4d + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sar r5d, 1 + add tlq, 31 + add r5d, 17 + cmp wd, 8 + cmova r4d, r5d +.h8_main: + movd m5, dyd + sub tlq, r4 + shl r4d, 6 + movd m7, [tlq] + movd m4, r4d + pshufb m5, [base+pw_256] + neg dyq + pshufb m7, [base+pw_m256] + mova m3, [base+z3_shuf] + lea r5, [dyq+r4+63] + pshufb m4, [base+pw_256] + psubw m4, [base+z3_base_inc] + shl wd, 3 + mova m6, m5 + sub rsp, wq +.h8_loop: + mov r4, r5 + sar r4, 6 + movu m0, [tlq+r4-8] + pand m2, m8, m5 + psubw m1, m9, m2 + psllw m2, 8 + pshufb m0, m3 + por m1, m2 + pmaddubsw m0, m1 + pcmpgtw m1, m4, m5 + paddw m5, m6 + pmulhrsw m0, m10 + pand m0, m1 + pandn m1, m7 + por m0, m1 + packuswb m0, m0 + movq [rsp+wq-8], m0 + sub wd, 8 + jz .h8_transpose + add r5, dyq + jg .h8_loop + packuswb m7, m7 +.h8_end_loop: + movq [rsp+wq-8], m7 + sub wd, 8 + jg .h8_end_loop +.h8_transpose: +%if ARCH_X86_32 + mov strideq, [dstq] + mov org_wd, [dstq+strideq] +%endif + or r3d, 8 + cmp org_wd, 4 +%if ARCH_X86_64 + jne .end_transpose_main +%else + jne .end_transpose_loop +%endif + mova m1, [rsp+16*1] + mova m0, [rsp+16*0] + lea r2, [strideq*3] + add rsp, 16*2 + punpcklbw m2, m1, m0 + punpckhbw m1, m0 + punpckhbw m0, m1, m2 + punpcklbw m1, m2 +.write_4x8_end: + call .write_4x8 + RET +.write_4x8: + movd [dstq+r2 ], m0 + pshuflw m4, m0, q1032 + movd [dstq+strideq*2], m4 + punpckhqdq m0, m0 + movd [dstq+strideq*1], m0 + psrlq m0, 32 + movd [dstq+strideq*0], m0 + lea dstq, [dstq+strideq*4] + movd [dstq+r2 ], m1 + pshuflw m4, m1, q1032 + movd [dstq+strideq*2], m4 + punpckhqdq m1, m1 + movd [dstq+strideq*1], m1 + psrlq m1, 32 + movd [dstq+strideq*0], m1 + ret +.h16: + lea r4d, [wq+15] + movd m0, r4d + and r4d, 15 + or r4d, 16 ; imin(w+15, 31) + test angled, 0x400 + jnz .h16_main + movd m2, angled + shr angled, 8 ; is_sm << 1 + pxor m1, m1 + pshufb m0, m1 + pshufb m2, m1 + movq m3, [base+z_filter_t_w16+angleq*4] + pcmpeqb m1, m0, [base+z_filter_wh16] + pand m1, m2 + pcmpgtb m1, m3 + pmovmskb r5d, m1 + test r5d, r5d + jz .h16_main ; filter_strength == 0 + mova m0, [tlq-16*2+1] + imul r5d, 0x24924924 + mova m1, [tlq-16*1+1] + neg r4 + movd m2, [tlq-16*0+1] + shr r5d, 30 + movd m3, [tlq+r4] + adc r5, -4 ; filter_strength-3 + pxor m7, m7 + lea tlq, [rsp+16*2] + mova [tlq-16*1], m0 + pshufb m2, m7 + mova [tlq+16*0], m1 + pshufb m3, m7 + mova [tlq+16*1], m2 + movq [tlq+r4+8], m3 + neg r4d + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + add tlq, 31 + cmp wd, 16 + jle .h16_main + pshuflw m0, [tlq-47], q0000 + sar r5, 1 + movq m1, [base+z3_filter_k_tail+r5*4] + lea r4d, [r5+33] + pmaddubsw m0, m1 +%if ARCH_X86_64 + pmulhrsw m0, m10 +%else + pmulhrsw m0, m4 +%endif + packuswb m0, m0 + movd [tlq-35], m0 +.h16_main: + movd m5, dyd + sub tlq, r4 + movd m4, r4d + shl r4d, 6 + movd m7, [tlq] + pxor m6, m6 + pshufb m5, [base+pw_256] + neg dyq + pshufb m7, m6 + mova m3, [base+z3_shuf] + lea r5, [dyq+r4+63] + pshufb m4, m6 + psubb m4, [base+pb_15to0] + shl wd, 4 + mova m6, m5 + sub rsp, wq +.h16_loop: + mov r4, r5 + pand m2, m8, m5 + sar r4, 6 + psubw m1, m9, m2 + psllw m2, 8 + movu m0, [tlq+r4-8*2] + por m2, m1 + movu m1, [tlq+r4-8*1] + pshufb m0, m3 + pmaddubsw m0, m2 + pshufb m1, m3 + pmaddubsw m1, m2 + psrlw m2, m5, 6 + paddw m5, m6 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packsswb m2, m2 + packuswb m0, m1 + pcmpgtb m1, m4, m2 + pand m0, m1 + pandn m1, m7 + por m0, m1 + mova [rsp+wq-16], m0 + sub wd, 16 + jz .h16_transpose + add r5, dyq + jg .h16_loop +.h16_end_loop: + mova [rsp+wq-16], m7 + sub wd, 16 + jg .h16_end_loop +.h16_transpose: +%if ARCH_X86_32 + mov strideq, [dstq] + mov org_wd, [dstq+strideq] +%endif + or r3d, 16 + cmp org_wd, 4 +%if ARCH_X86_64 + jne .end_transpose_main +%else + jne .end_transpose_loop +%endif +.h16_transpose_w4: + mova m2, [rsp+16*3] + mova m4, [rsp+16*2] + mova m3, [rsp+16*1] + mova m0, [rsp+16*0] + lea r2, [strideq*3] + add rsp, 16*4 + punpckhbw m1, m2, m4 + punpcklbw m2, m4 + punpckhbw m4, m3, m0 + punpcklbw m3, m0 + punpckhwd m0, m1, m4 + punpcklwd m1, m4 + call .write_4x8 + lea dstq, [dstq+strideq*4] + punpckhwd m0, m2, m3 + punpcklwd m1, m2, m3 + jmp .write_4x8_end +.h32: + lea r4d, [wq+31] + and r4d, 31 + or r4d, 32 ; imin(w+31, 63) + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h32_main + mova m0, [tlq-16*4+1] + mova m1, [tlq-16*3+1] + mova m2, [tlq-16*2+1] + mova m3, [tlq-16*1+1] + movd m4, [tlq-16*0+1] + neg r4 + movd m5, [tlq+r4] + pxor m7, m7 + lea tlq, [rsp+16*4] + mova [tlq-16*3], m0 + mova [tlq-16*2], m1 + xor r5d, r5d ; filter_strength = 3 + mova [tlq-16*1], m2 + pshufb m4, m7 + mova [tlq+16*0], m3 + pshufb m5, m7 + mova [tlq+16*1], m4 + movq [tlq+r4+8], m5 + neg r4d + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sub tlq, 16*2 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + add tlq, 63 + cmp wd, 32 + jle .h32_main + pshuflw m0, [tlq-79], q0000 + movq m1, [base+z3_filter_k_tail] + add r4d, 2 + pmaddubsw m0, m1 +%if ARCH_X86_64 + pmulhrsw m0, m10 +%else + pmulhrsw m0, m4 +%endif + packuswb m0, m0 + movd [tlq-67], m0 +.h32_main: + movd m5, dyd + sub tlq, r4 + movd m4, r4d + shl r4d, 6 + movd m7, [tlq] + pxor m6, m6 + pshufb m5, [base+pw_256] + neg dyq + pshufb m7, m6 + mova m3, [base+z3_shuf] + lea r5, [dyq+r4+63] + pshufb m4, m6 + psubb m4, [base+pb_15to0] + mova m6, m5 +.h32_loop: + mov r4, r5 + pand m2, m8, m5 + sar r4, 6 + psubw m1, m9, m2 + psllw m2, 8 + movu m0, [tlq+r4-8*4] + por m2, m1 + movu m1, [tlq+r4-8*3] + pshufb m0, m3 + pmaddubsw m0, m2 + pshufb m1, m3 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + sub rsp, 32 + packuswb m0, m1 + mova [rsp+16*0], m0 + movu m0, [tlq+r4-8*2] + movu m1, [tlq+r4-8*1] + pshufb m0, m3 + pshufb m1, m3 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + psrlw m2, m5, 6 + paddw m5, m6 + packsswb m2, m2 + packuswb m0, m1 + pcmpgtb m1, m4, m2 + paddsb m2, [base+pb_16] + pand m0, m1 + pandn m1, m7 + por m0, m1 + pcmpgtb m1, m4, m2 + mova [rsp+16*1], m0 + pand m0, m1, [rsp+16*0] + pandn m1, m7 + por m0, m1 + mova [rsp+16*0], m0 + dec wd + jz .h32_transpose + add r5, dyq + jg .h32_loop +.h32_end_loop: + sub rsp, 32 + mova [rsp+16*1], m7 + mova [rsp+16*0], m7 + dec wd + jg .h32_end_loop +.h32_transpose: + or r3d, 32 + jmp .end_transpose_main +.h64: + lea r4d, [wq+63] + test angled, 0x400 ; !enable_intra_edge_filter + jnz .h64_main + mova m0, [tlq-16*8+1] + mova m1, [tlq-16*7+1] + mova m2, [tlq-16*6+1] + mova m3, [tlq-16*5+1] + mova [rsp+16*1], m0 + mova [rsp+16*2], m1 + mova [rsp+16*3], m2 + mova [rsp+16*4], m3 + mova m0, [tlq-16*4+1] + mova m1, [tlq-16*3+1] + mova m2, [tlq-16*2+1] + mova m3, [tlq-16*1+1] + movd m4, [tlq-16*0+1] + neg r4 + movd m5, [tlq+r4] + pxor m7, m7 + lea tlq, [rsp+16*8] + mova [tlq-16*3], m0 + mova [tlq-16*2], m1 + xor r5d, r5d ; filter_strength = 3 + mova [tlq-16*1], m2 + pshufb m4, m7 + mova [tlq+16*0], m3 + pshufb m5, m7 + mova [tlq+16*1], m4 + movq [tlq+r4+8], m5 + neg r4d + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sub tlq, 16*2 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sub tlq, 16*2 + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge + sub tlq, 16*2 + cmp wd, 64 + jl .h64_filter96 ; skip one call if the last 32 bytes aren't used + call mangle(private_prefix %+ _ipred_z1_8bpc_ssse3).filter_edge +.h64_filter96: + add tlq, 127 +.h64_main: + movd m5, dyd + sub tlq, r4 + movd m4, r4d + shl r4d, 6 + movd m7, [tlq] + pxor m6, m6 + pshufb m5, [base+pw_256] + neg dyq + pshufb m7, m6 + mova m3, [base+z3_shuf] + lea r5, [dyq+r4+63] + pshufb m4, m6 + psubb m4, [base+pb_15to0] + mova m6, m5 +.h64_loop: + mov r4, r5 + pand m2, m8, m5 + sar r4, 6 + psubw m1, m9, m2 + psllw m2, 8 + movu m0, [tlq+r4-8*8] + por m2, m1 + movu m1, [tlq+r4-8*7] + pshufb m0, m3 + pmaddubsw m0, m2 + pshufb m1, m3 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + sub rsp, 64 + packuswb m0, m1 + mova [rsp+16*0], m0 + movu m0, [tlq+r4-8*6] + movu m1, [tlq+r4-8*5] + pshufb m0, m3 + pshufb m1, m3 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + mova [rsp+16*1], m0 + movu m0, [tlq+r4-8*4] + movu m1, [tlq+r4-8*3] + pshufb m0, m3 + pshufb m1, m3 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + packuswb m0, m1 + mova [rsp+16*2], m0 + movu m0, [tlq+r4-8*2] + movu m1, [tlq+r4-8*1] + pshufb m0, m3 + pshufb m1, m3 + pmaddubsw m0, m2 + pmaddubsw m1, m2 + pmulhrsw m0, m10 + pmulhrsw m1, m10 + psrlw m2, m5, 6 + paddw m5, m6 + packsswb m2, m2 + packuswb m0, m1 + pcmpgtb m1, m4, m2 + paddsb m2, [base+pb_16] + pand m0, m1 + pandn m1, m7 + por m0, m1 + pcmpgtb m1, m4, m2 + paddsb m2, [base+pb_16] + mova [rsp+16*3], m0 + pand m0, m1, [rsp+16*2] + pandn m1, m7 + por m0, m1 + pcmpgtb m1, m4, m2 + paddsb m2, [base+pb_16] + mova [rsp+16*2], m0 + pand m0, m1, [rsp+16*1] + pandn m1, m7 + por m0, m1 + pcmpgtb m1, m4, m2 + mova [rsp+16*1], m0 + pand m0, m1, [rsp+16*0] + pandn m1, m7 + por m0, m1 + mova [rsp+16*0], m0 + dec wd + jz .h64_transpose + add r5, dyq + jg .h64_loop +.h64_end_loop: + sub rsp, 64 + mova [rsp+16*3], m7 + mova [rsp+16*2], m7 + mova [rsp+16*1], m7 + mova [rsp+16*0], m7 + dec wd + jg .h64_end_loop +.h64_transpose: + or r3d, 64 +.end_transpose_main: +%if ARCH_X86_64 + lea r5, [r3*3] + lea r7, [strideq*3] +%else + mov strideq, [dstq] + mov org_wd, [dstq+strideq] +%endif +.end_transpose_loop: + lea r4, [rsp+r3-8] + lea r6, [dstq+org_wq-8] +.end_transpose_loop_y: + movq m0, [r4+r3*1] + movq m4, [r4+r3*0] +%if ARCH_X86_64 + movq m1, [r4+r5 ] + movq m5, [r4+r3*2] + lea r2, [r4+r3*4] +%else + lea r2, [r4+r3*2] + movq m1, [r2+r3*1] + movq m5, [r2+r3*0] + lea r2, [r2+r3*2] +%endif + movq m2, [r2+r3*1] + movq m6, [r2+r3*0] +%if ARCH_X86_64 + movq m3, [r2+r5 ] + movq m7, [r2+r3*2] +%else + lea r2, [r2+r3*2] + movq m3, [r2+r3*1] + movq m7, [r2+r3*0] +%endif + sub r4, 8 + punpcklbw m0, m4 + punpcklbw m1, m5 + punpcklbw m2, m6 + punpcklbw m3, m7 + punpckhwd m4, m1, m0 + punpcklwd m1, m0 + punpckhwd m0, m3, m2 + punpcklwd m3, m2 + punpckhdq m2, m3, m1 + punpckldq m3, m1 + punpckldq m1, m0, m4 + punpckhdq m0, m4 + movhps [r6+strideq*0], m0 + movq [r6+strideq*1], m0 +%if ARCH_X86_64 + movhps [r6+strideq*2], m1 + movq [r6+r7 ], m1 + lea r6, [r6+strideq*4] +%else + lea r6, [r6+strideq*2] + movhps [r6+strideq*0], m1 + movq [r6+strideq*1], m1 + lea r6, [r6+strideq*2] +%endif + movhps [r6+strideq*0], m2 + movq [r6+strideq*1], m2 +%if ARCH_X86_64 + movhps [r6+strideq*2], m3 + movq [r6+r7 ], m3 + lea r6, [r6+strideq*4] +%else + lea r6, [r6+strideq*2] + movhps [r6+strideq*0], m3 + movq [r6+strideq*1], m3 + lea r6, [r6+strideq*2] +%endif + cmp r4, rsp + jae .end_transpose_loop_y + lea rsp, [rsp+r3*8] + sub org_wd, 8 + jg .end_transpose_loop + RET + +;--------------------------------------------------------------------------------------- +;int dav1d_pal_pred_ssse3(pixel *dst, const ptrdiff_t stride, const uint16_t *const pal, +; const uint8_t *idx, const int w, const int h); +;--------------------------------------------------------------------------------------- +cglobal pal_pred_8bpc, 4, 6, 5, dst, stride, pal, idx, w, h + mova m4, [palq] + LEA r2, pal_pred_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r2+wq*4] + packuswb m4, m4 + add wq, r2 + lea r2, [strideq*3] + jmp wq +.w4: + pshufb m0, m4, [idxq] + add idxq, 16 + movd [dstq ], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq ], m1 + punpckhqdq m0, m0 + movd [dstq+strideq*2], m0 + psrlq m0, 32 + movd [dstq+r2 ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +ALIGN function_align +.w8: + pshufb m0, m4, [idxq] + pshufb m1, m4, [idxq+16] + add idxq, 32 + movq [dstq ], m0 + movhps [dstq+strideq ], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r2 ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +ALIGN function_align +.w16: + pshufb m0, m4, [idxq] + pshufb m1, m4, [idxq+16] + pshufb m2, m4, [idxq+32] + pshufb m3, m4, [idxq+48] + add idxq, 64 + mova [dstq ], m0 + mova [dstq+strideq ], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r2 ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16 + RET +ALIGN function_align +.w32: + pshufb m0, m4, [idxq] + pshufb m1, m4, [idxq+16] + pshufb m2, m4, [idxq+32] + pshufb m3, m4, [idxq+48] + add idxq, 64 + mova [dstq ], m0 + mova [dstq+16 ], m1 + mova [dstq+strideq ], m2 + mova [dstq+strideq+16], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32 + RET +ALIGN function_align +.w64: + pshufb m0, m4, [idxq] + pshufb m1, m4, [idxq+16] + pshufb m2, m4, [idxq+32] + pshufb m3, m4, [idxq+48] + add idxq, 64 + mova [dstq ], m0 + mova [dstq+16], m1 + mova [dstq+32], m2 + mova [dstq+48], m3 + add dstq, strideq + sub hd, 1 + jg .w64 + RET + +;--------------------------------------------------------------------------------------- +;void dav1d_ipred_cfl_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int16_t *ac, const int alpha); +;--------------------------------------------------------------------------------------- +%macro IPRED_CFL 1 ; ac in, unpacked pixels out + psignw m3, m%1, m1 + pabsw m%1, m%1 + pmulhrsw m%1, m2 + psignw m%1, m3 + paddw m%1, m0 +%endmacro + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal ipred_cfl_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + movifnidn wd, wm + movifnidn hd, hm + tzcnt r6d, hd + lea t0d, [wq+hq] + movd m4, t0d + tzcnt t0d, t0d + movd m5, t0d + LEA t0, ipred_cfl_ssse3_table + tzcnt wd, wd + movsxd r6, [t0+r6*4] + movsxd wq, [t0+wq*4+16] + pcmpeqd m3, m3 + psrlw m4, 1 + add r6, t0 + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h4: + movd m0, [tlq-4] + pmaddubsw m0, m3 + jmp wq +.w4: + movd m1, [tlq+1] + pmaddubsw m1, m3 + psubw m0, m4 + paddw m0, m1 + pmaddwd m0, m3 + cmp hd, 4 + jg .w4_mul + psrlw m0, 3 ; dc >>= ctz(width + height); + jmp .w4_end +.w4_mul: + punpckhqdq m1, m0, m0 + paddw m0, m1 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + psrlw m0, 2 + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 8 + cmovz r6d, r2d + movd m5, r6d + pmulhuw m0, m5 +.w4_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s4: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s4_loop: + mova m4, [acq] + mova m5, [acq+16] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + movd [dstq+strideq*0], m4 + pshuflw m4, m4, q1032 + movd [dstq+strideq*1], m4 + punpckhqdq m4, m4 + movd [dstq+strideq*2], m4 + psrlq m4, 32 + movd [dstq+r6 ], m4 + lea dstq, [dstq+strideq*4] + add acq, 32 + sub hd, 4 + jg .s4_loop + RET +ALIGN function_align +.h8: + movq m0, [tlq-8] + pmaddubsw m0, m3 + jmp wq +.w8: + movq m1, [tlq+1] + pmaddubsw m1, m3 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + paddw m0, m1 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 8 + je .w8_end + mov r6d, 0x5556 + mov r2d, 0x3334 + cmp hd, 32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w8_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s8: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + lea r6, [strideq*3] + pabsw m2, m1 + psllw m2, 9 +.s8_loop: + mova m4, [acq] + mova m5, [acq+16] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + movq [dstq ], m4 + movhps [dstq+strideq ], m4 + mova m4, [acq+32] + mova m5, [acq+48] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + movq [dstq+strideq*2], m4 + movhps [dstq+r6 ], m4 + lea dstq, [dstq+strideq*4] + add acq, 64 + sub hd, 4 + jg .s8_loop + RET +ALIGN function_align +.h16: + mova m0, [tlq-16] + pmaddubsw m0, m3 + jmp wq +.w16: + movu m1, [tlq+1] + pmaddubsw m1, m3 + paddw m0, m1 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 16 + je .w16_end + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 8|32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w16_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s16: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s16_loop: + mova m4, [acq] + mova m5, [acq+16] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + mova [dstq], m4 + mova m4, [acq+32] + mova m5, [acq+48] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + mova [dstq+strideq], m4 + lea dstq, [dstq+strideq*2] + add acq, 64 + sub hd, 2 + jg .s16_loop + RET +ALIGN function_align +.h32: + mova m0, [tlq-32] + pmaddubsw m0, m3 + mova m2, [tlq-16] + pmaddubsw m2, m3 + paddw m0, m2 + jmp wq +.w32: + movu m1, [tlq+1] + pmaddubsw m1, m3 + movu m2, [tlq+17] + pmaddubsw m2, m3 + paddw m1, m2 + paddw m0, m1 + psubw m4, m0 + punpckhqdq m0, m0 + psubw m0, m4 + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 + pmaddwd m0, m3 + psrlw m0, m5 + cmp hd, 32 + je .w32_end + lea r2d, [hq*2] + mov r6d, 0x5556 + mov r2d, 0x3334 + test hd, 64|16 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 +.w32_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s32: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s32_loop: + mova m4, [acq] + mova m5, [acq+16] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + mova [dstq], m4 + mova m4, [acq+32] + mova m5, [acq+48] + IPRED_CFL 4 + IPRED_CFL 5 + packuswb m4, m5 + mova [dstq+16], m4 + add dstq, strideq + add acq, 64 + dec hd + jg .s32_loop + RET + +;--------------------------------------------------------------------------------------- +;void dav1d_ipred_cfl_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int16_t *ac, const int alpha); +;--------------------------------------------------------------------------------------- +cglobal ipred_cfl_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + mov hd, hm ; zero upper half + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + movu m0, [tlq] + mov t0d, 0x8000 + movd m3, t0d + movd m2, r6d + psrld m3, m2 + LEA t0, ipred_cfl_left_ssse3_table + movsxd r6, [t0+r6*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, t0 + add t0, ipred_cfl_splat_ssse3_table-ipred_cfl_left_ssse3_table + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 +.h32: + movu m1, [tlq+16] ; unaligned when jumping here from dc_top + pmaddubsw m1, m2 + paddw m0, m1 +.h16: + pshufd m1, m0, q3232 ; psrlq m1, m0, 16 + paddw m0, m1 +.h8: + pshuflw m1, m0, q1032 ; psrlq m1, m0, 32 + paddw m0, m1 +.h4: + pmaddwd m0, m2 + pmulhrsw m0, m3 + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + jmp wq + +;--------------------------------------------------------------------------------------- +;void dav1d_ipred_cfl_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int16_t *ac, const int alpha); +;--------------------------------------------------------------------------------------- +cglobal ipred_cfl_top_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + LEA t0, ipred_cfl_left_ssse3_table + tzcnt wd, wm + inc tlq + movu m0, [tlq] + movifnidn hd, hm + mov r6d, 0x8000 + movd m3, r6d + movd m2, wd + psrld m3, m2 + movsxd r6, [t0+wq*4] + pcmpeqd m2, m2 + pmaddubsw m0, m2 + add r6, t0 + add t0, ipred_cfl_splat_ssse3_table-ipred_cfl_left_ssse3_table + movsxd wq, [t0+wq*4] + add wq, t0 + movifnidn acq, acmp + jmp r6 + +;--------------------------------------------------------------------------------------- +;void dav1d_ipred_cfl_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, +; const int width, const int height, const int16_t *ac, const int alpha); +;--------------------------------------------------------------------------------------- +cglobal ipred_cfl_128_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha + tzcnt wd, wm + movifnidn hd, hm + LEA r6, ipred_cfl_splat_ssse3_table + movsxd wq, [r6+wq*4] + movddup m0, [r6-ipred_cfl_splat_ssse3_table+pw_128] + add wq, r6 + movifnidn acq, acmp + jmp wq + +%macro RELOAD_ACQ_32 1 + mov acq, ac_bakq ; restore acq +%endmacro + +%if ARCH_X86_64 +cglobal ipred_cfl_ac_420_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak +DECLARE_REG_TMP 7 + movddup m2, [pb_2] +%else +cglobal ipred_cfl_ac_420_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h +DECLARE_REG_TMP 4 +%define ac_bakq acmp + mov t0d, 0x02020202 + movd m2, t0d + pshufd m2, m2, q0000 +%endif + movifnidn wd, wm + mov t0d, hm + mov hd, t0d + imul t0d, wd + movd m5, t0d + movifnidn hpadd, hpadm +%if ARCH_X86_64 + mov ac_bakq, acq +%endif + shl hpadd, 2 + sub hd, hpadd + pxor m4, m4 + cmp wd, 8 + jg .w16 + je .w8 + ; fall-through +%if ARCH_X86_64 + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak +%else + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h +%endif +.w4: + lea stride3q, [strideq*3] +.w4_loop: + movq m0, [yq] + movq m1, [yq+strideq] + movhps m0, [yq+strideq*2] + movhps m1, [yq+stride3q] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*4] + add acq, 16 + sub hd, 2 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_4_8 + punpckhqdq m0, m0 +.w4_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 16 + sub hpadd, 2 + jg .w4_hpad_loop + jmp .calc_avg_4_8 +.w8: + lea stride3q, [strideq*3] + test wpadd, wpadd + jnz .w8_wpad +.w8_loop: + mova m0, [yq] + mova m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + mova m0, [yq+strideq*2] + mova m1, [yq+stride3q] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq+16], m0 + paddw m4, m0 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 2 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_4_8 + jmp .w8_hpad +.w8_wpad: ; wpadd=1 + movddup m0, [yq] + movddup m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + pshufhw m0, m0, q3333 + mova [acq], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 16 + sub hd, 1 + jg .w8_wpad + test hpadd, hpadd + jz .calc_avg_4_8 +.w8_hpad: + mova [acq], m0 + paddw m4, m0 + add acq, 16 + sub hpadd, 1 + jg .w8_hpad + jmp .calc_avg_4_8 +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + mova m0, [yq] + mova m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + mova m6, [yq+16] + mova m1, [yq+strideq+16] + pmaddubsw m6, m2 + pmaddubsw m1, m2 + paddw m6, m1 + mova [acq+16], m6 + paddw m4, m6 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jg .w16_loop + test hpadd, hpadd + jz .calc_avg16 + jmp .w16_hpad_loop +.w16_wpad: + cmp wpadd, 2 + jl .w16_pad1 + je .w16_pad2 +.w16_pad3: + movddup m0, [yq] + movddup m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + pshufhw m0, m0, q3333 + mova [acq], m0 + paddw m4, m0 + mova m6, m0 + punpckhqdq m6, m0, m0 + mova [acq+16], m6 + paddw m4, m6 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jg .w16_pad3 + jmp .w16_wpad_done +.w16_pad2: + mova m0, [yq] + mova m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + pshufhw m6, m0, q3333 + punpckhqdq m6, m6 + mova [acq+16], m6 + paddw m4, m6 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jg .w16_pad2 + jmp .w16_wpad_done +.w16_pad1: + mova m0, [yq] + mova m1, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + paddw m0, m1 + mova [acq], m0 + paddw m4, m0 + movddup m6, [yq+16] + movddup m1, [yq+strideq+16] + pmaddubsw m6, m2 + pmaddubsw m1, m2 + paddw m6, m1 + pshufhw m6, m6, q3333 + mova [acq+16], m6 + paddw m4, m6 + lea yq, [yq+strideq*2] + add acq, 32 + dec hd + jg .w16_pad1 +.w16_wpad_done: + test hpadd, hpadd + jz .calc_avg16 +.w16_hpad_loop: + mova [acq], m0 + paddw m4, m0 + mova [acq+16], m6 + paddw m4, m6 + add acq, 32 + dec hpadd + jg .w16_hpad_loop + jmp .calc_avg16 + +%if ARCH_X86_64 + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak +%else + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h +%endif +.calc_avg_4_8: + psrlw m2, 9 + pmaddwd m4, m2 + jmp .calc_avg +.calc_avg16: + psrld m0, m4, 16 + pslld m4, 16 + psrld m4, 16 + paddd m4, m0 +.calc_avg: + movd szd, m5 + psrad m5, 1 + tzcnt r1d, szd + paddd m4, m5 + movd m1, r1d + pshufd m0, m4, q2301 + paddd m0, m4 + pshufd m4, m0, q1032 + paddd m0, m4 + psrad m0, m1 ; sum >>= log2sz; + packssdw m0, m0 + RELOAD_ACQ_32 acq +.sub_loop: + mova m1, [acq] + psubw m1, m0 ; ac[x] -= sum; + mova [acq], m1 + add acq, 16 + sub szd, 8 + jg .sub_loop + RET + +%if ARCH_X86_64 +cglobal ipred_cfl_ac_422_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak + movddup m2, [pb_4] +%else +cglobal ipred_cfl_ac_422_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h + mov t0d, 0x04040404 + movd m2, t0d + pshufd m2, m2, q0000 +%endif + movifnidn wd, wm + mov t0d, hm + mov hd, t0d + imul t0d, wd + movd m6, t0d + movifnidn hpadd, hpadm +%if ARCH_X86_64 + mov ac_bakq, acq +%endif + shl hpadd, 2 + sub hd, hpadd + pxor m4, m4 + pxor m5, m5 + cmp wd, 8 + jg .w16 + je .w8 + ; fall-through + +%if ARCH_X86_64 + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak +%else + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h +%endif +.w4: + lea stride3q, [strideq*3] +.w4_loop: + movq m1, [yq] + movhps m1, [yq+strideq] + movq m0, [yq+strideq*2] + movhps m0, [yq+stride3q] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq], m1 + mova [acq+16], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_4 + punpckhqdq m0, m0 +.w4_hpad_loop: + mova [acq], m0 + paddw m4, m0 + add acq, 16 + sub hpadd, 2 + jg .w4_hpad_loop + jmp .calc_avg_4 +.w8: + lea stride3q, [strideq*3] + test wpadd, wpadd + jnz .w8_wpad +.w8_loop: + mova m1, [yq] + mova m0, [yq+strideq] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq], m1 + mova [acq+16], m0 + paddw m4, m0 + paddw m5, m1 + mova m1, [yq+strideq*2] + mova m0, [yq+stride3q] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq+32], m1 + mova [acq+48], m0 + paddw m4, m0 + paddw m5, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_8_16 + jmp .w8_hpad +.w8_wpad: + movddup m1, [yq] + pmaddubsw m1, m2 + pshufhw m1, m1, q3333 + mova [acq], m1 + paddw m5, m1 + movddup m0, [yq+strideq] + pmaddubsw m0, m2 + pshufhw m0, m0, q3333 + mova [acq+16], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 32 + sub hd, 2 + jg .w8_wpad + test hpadd, hpadd + jz .calc_avg_8_16 +.w8_hpad: + mova [acq], m0 + paddw m4, m0 + mova [acq+16], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 2 + jg .w8_hpad + jmp .calc_avg_8_16 +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + mova m1, [yq] + mova m0, [yq+16] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq], m1 + mova [acq+16], m0 + paddw m5, m0 + paddw m5, m1 + mova m1, [yq+strideq] + mova m0, [yq+strideq+16] + pmaddubsw m0, m2 + pmaddubsw m1, m2 + mova [acq+32], m1 + mova [acq+48], m0 + paddw m4, m0 + paddw m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg_8_16 + jmp .w16_hpad_loop +.w16_wpad: + cmp wpadd, 2 + jl .w16_pad1 + je .w16_pad2 +.w16_pad3: + movddup m1, [yq] + pmaddubsw m1, m2 + pshufhw m1, m1, q3333 + mova [acq], m1 + paddw m5, m1 + punpckhqdq m1, m1 + mova [acq+16], m1 + paddw m5, m1 + movddup m1, [yq+strideq] + pmaddubsw m1, m2 + pshufhw m1, m1, q3333 + mova [acq+32], m1 + paddw m4, m1 + punpckhqdq m0, m1, m1 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad3 + jmp .w16_wpad_done +.w16_pad2: + mova m1, [yq] + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + pshufhw m1, m1, q3333 + punpckhqdq m1, m1 + mova [acq+16], m1 + paddw m5, m1 + mova m1, [yq+strideq] + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + mova m0, m1 + pshufhw m0, m0, q3333 + punpckhqdq m0, m0 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad2 + jmp .w16_wpad_done +.w16_pad1: + mova m1, [yq] + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + movddup m0, [yq+16] + pmaddubsw m0, m2 + pshufhw m0, m0, q3333 + mova [acq+16], m0 + paddw m5, m0 + mova m1, [yq+strideq] + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + movddup m0, [yq+strideq+16] + pmaddubsw m0, m2 + pshufhw m0, m0, q3333 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad1 +.w16_wpad_done: + test hpadd, hpadd + jz .calc_avg_8_16 +.w16_hpad_loop: + mova [acq], m1 + mova [acq+16], m0 + paddw m4, m1 + paddw m5, m0 + mova [acq+32], m1 + mova [acq+48], m0 + paddw m4, m1 + paddw m5, m0 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop + jmp .calc_avg_8_16 + +%if ARCH_X86_64 + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak +%else + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h +%endif +.calc_avg_4: + psrlw m2, 10 + pmaddwd m5, m2 + pmaddwd m0, m4, m2 + jmp .calc_avg +.calc_avg_8_16: + mova m0, m5 + psrld m5, 16 + pslld m0, 16 + psrld m0, 16 + paddd m5, m0 + mova m0, m4 + psrld m0, 16 + pslld m4, 16 + psrld m4, 16 + paddd m0, m4 +.calc_avg: + paddd m5, m0 + movd szd, m6 + psrad m6, 1 + tzcnt r1d, szd ; const int log2sz = ctz(width) + ctz(height); + paddd m5, m6 + movd m1, r1d + pshufd m0, m5, q2301 + paddd m0, m5 + pshufd m5, m0, q1032 + paddd m0, m5 + psrad m0, m1 ; sum >>= log2sz; + packssdw m0, m0 + RELOAD_ACQ_32 acq ; ac = ac_orig +.sub_loop: + mova m1, [acq] + psubw m1, m0 + mova [acq], m1 + add acq, 16 + sub szd, 8 + jg .sub_loop + RET + +%if ARCH_X86_64 +cglobal ipred_cfl_ac_444_8bpc, 4, 8, 7, -4*16, ac, y, stride, wpad, hpad, w, h, ac_bak + movddup m2, [pb_4] +%else +cglobal ipred_cfl_ac_444_8bpc, 4, 7, 7, -5*16, ac, y, stride, wpad, hpad, w, h +%define ac_bakq [rsp+16*4] + mov t0d, 0x04040404 + movd m2, t0d + pshufd m2, m2, q0000 +%endif + movifnidn wd, wm + movifnidn hpadd, hpadm + movd m0, hpadd + mov t0d, hm + mov hd, t0d + imul t0d, wd + movd m6, t0d + movd hpadd, m0 + mov ac_bakq, acq + shl hpadd, 2 + sub hd, hpadd + pxor m5, m5 + pxor m4, m4 + cmp wd, 16 + jg .w32 + cmp wd, 8 + jg .w16 + je .w8 + ; fall-through + +%if ARCH_X86_64 + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak +%else + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h +%endif +.w4: + lea stride3q, [strideq*3] +.w4_loop: + movd m1, [yq] + movd m3, [yq+strideq] + punpckldq m1, m3 + punpcklbw m1, m1 + movd m0, [yq+strideq*2] + movd m3, [yq+stride3q] + punpckldq m0, m3 + punpcklbw m0, m0 + pmaddubsw m1, m2 + pmaddubsw m0, m2 + mova [acq], m1 + mova [acq+16], m0 + paddw m5, m0 + paddw m5, m1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_4 + punpckhqdq m0, m0 +.w4_hpad_loop: + mova [acq], m0 + paddw m5, m0 + add acq, 16 + sub hpadd, 2 + jg .w4_hpad_loop +.calc_avg_4: + psrlw m2, 10 + pmaddwd m5, m2 + jmp .calc_avg + +.w8: + lea stride3q, [strideq*3] + test wpadd, wpadd + jnz .w8_wpad +.w8_loop: + movq m1, [yq] + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + movq m0, [yq+strideq] + punpcklbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0 + movq m1, [yq+strideq*2] + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + movq m0, [yq+stride3q] + punpcklbw m0, m0 + pmaddubsw m0, m2 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_8_16 + jmp .w8_hpad +.w8_wpad: + movd m1, [yq] + punpcklbw m1, m1 + punpcklqdq m1, m1 + pmaddubsw m1, m2 + pshufhw m1, m1, q3333 + mova [acq], m1 + paddw m5, m1 + movd m0, [yq+strideq] + punpcklbw m0, m0 + punpcklqdq m0, m0 + pmaddubsw m0, m2 + pshufhw m0, m0, q3333 + mova [acq+16], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 32 + sub hd, 2 + jg .w8_wpad + test hpadd, hpadd + jz .calc_avg_8_16 +.w8_hpad: + mova [acq], m0 + paddw m5, m0 + mova [acq+16], m0 + paddw m4, m0 + add acq, 32 + sub hpadd, 2 + jg .w8_hpad + jmp .calc_avg_8_16 + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0 + mova m0, [yq+strideq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg_8_16 + jmp .w16_hpad_loop +.w16_wpad: + cmp wpadd, 2 + jl .w16_pad1 + je .w16_pad2 +.w16_pad3: + movd m1, [yq] + punpcklbw m1, m1 + punpcklqdq m1, m1 + pshufhw m1, m1, q3333 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + punpckhqdq m1, m1 + mova [acq+16], m1 + paddw m5, m1 + movd m1, [yq+strideq] + punpcklbw m1, m1 + punpcklqdq m1, m1 + pshufhw m1, m1, q3333 + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + punpckhqdq m0, m1, m1 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad3 + jmp .w16_wpad_done +.w16_pad2: + movq m1, [yq] + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + pshufhw m1, m1, q3333 + punpckhqdq m1, m1 + mova [acq+16], m1 + paddw m5, m1 + movq m1, [yq+strideq] + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + mova m0, m1 + pshufhw m0, m0, q3333 + punpckhqdq m0, m0 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad2 + jmp .w16_wpad_done +.w16_pad1: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1 + punpckhbw m0, m0 + punpcklqdq m0, m0 + pshufhw m0, m0, q3333 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0 + mova m0, [yq+strideq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq+32], m1 + paddw m4, m1 + punpckhbw m0, m0 + punpcklqdq m0, m0 + pshufhw m0, m0, q3333 + pmaddubsw m0, m2 + mova [acq+48], m0 + paddw m4, m0 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_pad1 +.w16_wpad_done: + test hpadd, hpadd + jz .calc_avg_8_16 +.w16_hpad_loop: + mova [acq], m1 + mova [acq+16], m0 + paddw m4, m1 + paddw m5, m0 + mova [acq+32], m1 + mova [acq+48], m0 + paddw m4, m1 + paddw m5, m0 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop +.calc_avg_8_16: + mova m0, m5 + psrld m5, 16 + pslld m0, 16 + psrld m0, 16 + paddd m5, m0 + mova m0, m4 + psrld m0, 16 + pslld m4, 16 + psrld m4, 16 + paddd m0, m4 + paddd m5, m0 + jmp .calc_avg + +.w32: + pxor m0, m0 + mova [rsp ], m0 + mova [rsp+16], m0 + mova [rsp+32], m0 + mova [rsp+48], m0 + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m4, [yq+16] + mova m3, m4 + punpcklbw m3, m3 + pmaddubsw m3, m2 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + punpckhbw m4, m4 + pmaddubsw m4, m2 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_loop + test hpadd, hpadd + jz .calc_avg_32 + jmp .w32_hpad_loop +.w32_wpad: + cmp wpadd, 2 + jl .w32_pad1 + je .w32_pad2 + cmp wpadd, 4 + jl .w32_pad3 + je .w32_pad4 + cmp wpadd, 6 + jl .w32_pad5 + je .w32_pad6 +.w32_pad7: + movd m1, [yq] + punpcklbw m1, m1 + punpcklqdq m1, m1 + pshufhw m1, m1, q3333 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + mova m0, m1 + punpckhqdq m0, m0 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m3, m0 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + mova m4, m3 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad7 + jmp .w32_wpad_done +.w32_pad6: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + pshufhw m0, m1, q3333 + punpckhqdq m0, m0 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m3, m0 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + mova m4, m3 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad6 + jmp .w32_wpad_done +.w32_pad5: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + mova m5, [rsp] + paddw m5, m1 + mova [rsp ], m5 + punpckhbw m0, m0 + punpcklqdq m0, m0 + pshufhw m0, m0, q3333 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m3, m0 + punpckhqdq m3, m3 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + mova m4, m3 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad5 + jmp .w32_wpad_done +.w32_pad4: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m3, m0 + pshufhw m3, m3, q3333 + punpckhqdq m3, m3 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + mova m4, m3 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad4 + jmp .w32_wpad_done +.w32_pad3: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + movd m3, [yq+16] + punpcklbw m3, m3 + punpcklqdq m3, m3 + pshufhw m3, m3, q3333 + pmaddubsw m3, m2 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + mova m4, m3 + punpckhqdq m4, m4 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad3 + jmp .w32_wpad_done +.w32_pad2: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m3, [yq+16] + punpcklbw m3, m3 + pmaddubsw m3, m2 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + pshufhw m4, m3, q3333 + punpckhqdq m4, m4 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad2 + jmp .w32_wpad_done +.w32_pad1: + mova m0, [yq] + mova m1, m0 + punpcklbw m1, m1 + pmaddubsw m1, m2 + mova [acq], m1 + paddw m5, m1, [rsp] + mova [rsp ], m5 + punpckhbw m0, m0 + pmaddubsw m0, m2 + mova [acq+16], m0 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova m4, [yq+16] + mova m3, m4 + punpcklbw m3, m3 + pmaddubsw m3, m2 + mova [acq+32], m3 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + punpckhbw m4, m4 + punpcklqdq m4, m4 + pshufhw m4, m4, q3333 + pmaddubsw m4, m2 + mova [acq+48], m4 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + lea yq, [yq+strideq] + add acq, 64 + sub hd, 1 + jg .w32_pad1 +.w32_wpad_done: + test hpadd, hpadd + jz .calc_avg_32 +.w32_hpad_loop: + mova [acq], m1 + mova [acq+16], m0 + paddw m5, m1, [rsp] + mova [rsp ], m5 + paddw m5, m0, [rsp+16] + mova [rsp+16], m5 + mova [acq+32], m3 + mova [acq+48], m4 + paddw m5, m3, [rsp+32] + mova [rsp+32], m5 + paddw m5, m4, [rsp+48] + mova [rsp+48], m5 + add acq, 64 + sub hpadd, 1 + jg .w32_hpad_loop + +%if ARCH_X86_64 + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak +%else + DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h +%endif + +.calc_avg_32: + mova m5, [rsp] + mova m0, m5 + psrld m5, 16 + pslld m0, 16 + psrld m0, 16 + paddd m5, m0 + mova m0, [rsp+16] + mova m3, m0 + psrld m0, 16 + pslld m3, 16 + psrld m3, 16 + paddd m0, m3 + paddd m5, m0 + mova m0, [rsp+32] + mova m3, m0 + psrld m0, 16 + pslld m3, 16 + psrld m3, 16 + paddd m0, m3 + mova m1, [rsp+48] + mova m3, m1 + psrld m1, 16 + pslld m3, 16 + psrld m3, 16 + paddd m1, m3 + paddd m1, m0 + paddd m5, m1 +.calc_avg: + movd szd, m6 + psrad m6, 1 + tzcnt r1d, szd ; const int log2sz = ctz(width) + ctz(height); + paddd m5, m6 + movd m1, r1d + pshufd m0, m5, q2301 + paddd m0, m5 + pshufd m5, m0, q1032 + paddd m0, m5 + psrad m0, m1 ; sum >>= log2sz; + packssdw m0, m0 + RELOAD_ACQ_32 acq ; ac = ac_orig +.sub_loop: + mova m1, [acq] + psubw m1, m0 + mova [acq], m1 + add acq, 16 + sub szd, 8 + jg .sub_loop + RET + +; %1 simd register that hold the mask and will hold the result +; %2 simd register that holds the "true" values +; %3 location of the "false" values (simd register/memory) +%macro BLEND 3 ; mask, true, false + pand %2, %1 + pandn %1, %3 + por %1, %2 +%endmacro + +%macro PAETH 2 ; top, ldiff + pavgb m1, m%1, m3 + pxor m0, m%1, m3 + pand m0, m4 + psubusb m2, m5, m1 + psubb m1, m0 + psubusb m1, m5 + por m1, m2 + paddusb m1, m1 + por m1, m0 ; min(tldiff, 255) + psubusb m2, m5, m3 + psubusb m0, m3, m5 + por m2, m0 ; tdiff +%ifnum %2 + pminub m2, m%2 + pcmpeqb m0, m%2, m2 ; ldiff <= tdiff +%else + mova m0, %2 + pminub m2, m0 + pcmpeqb m0, m2 +%endif + pminub m1, m2 + pcmpeqb m1, m2 ; ldiff <= tldiff && tdiff <= tldiff + mova m2, m3 + BLEND m0, m2, m%1 + BLEND m1, m0, m5 +%endmacro + +cglobal ipred_paeth_8bpc, 3, 6, 8, -7*16, dst, stride, tl, w, h +%define base r5-ipred_paeth_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + pxor m0, m0 + movd m5, [tlq] + pshufb m5, m0 + LEA r5, ipred_paeth_ssse3_table + movsxd wq, [r5+wq*4] + movddup m4, [base+ipred_paeth_shuf] + add wq, r5 + jmp wq +.w4: + movd m6, [tlq+1] ; top + pshufd m6, m6, q0000 + lea r3, [strideq*3] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 ; ldiff +.w4_loop: + sub tlq, 4 + movd m3, [tlq] + mova m1, [base+ipred_h_shuf] + pshufb m3, m1 ; left + PAETH 6, 7 + movd [dstq ], m1 + pshuflw m0, m1, q1032 + movd [dstq+strideq ], m0 + punpckhqdq m1, m1 + movd [dstq+strideq*2], m1 + psrlq m1, 32 + movd [dstq+r3 ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4_loop + RET +ALIGN function_align +.w8: + movddup m6, [tlq+1] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w8_loop: + sub tlq, 2 + movd m3, [tlq] + pshufb m3, [base+ipred_paeth_shuf] + PAETH 6, 7 + movq [dstq ], m1 + movhps [dstq+strideq], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + movu m6, [tlq+1] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 +.w16_loop: + sub tlq, 1 + movd m3, [tlq] + pxor m1, m1 + pshufb m3, m1 + PAETH 6, 7 + mova [dstq], m1 + add dstq, strideq + sub hd, 1 + jg .w16_loop + RET +ALIGN function_align +.w32: + movu m6, [tlq+1] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp ], m6 + mova [rsp+16], m7 + movu m6, [tlq+17] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp+32], m6 +.w32_loop: + dec tlq + movd m3, [tlq] + pxor m1, m1 + pshufb m3, m1 + mova m6, [rsp] + PAETH 6, [rsp+16] + mova [dstq ], m1 + mova m6, [rsp+32] + PAETH 6, 7 + mova [dstq+16], m1 + add dstq, strideq + dec hd + jg .w32_loop + RET +ALIGN function_align +.w64: + movu m6, [tlq+1] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp ], m6 + mova [rsp+16], m7 + movu m6, [tlq+17] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp+32], m6 + mova [rsp+48], m7 + movu m6, [tlq+33] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp+64], m6 + mova [rsp+80], m7 + movu m6, [tlq+49] + psubusb m7, m5, m6 + psubusb m0, m6, m5 + por m7, m0 + mova [rsp+96], m6 +.w64_loop: + dec tlq + movd m3, [tlq] + pxor m1, m1 + pshufb m3, m1 + mova m6, [rsp] + PAETH 6, [rsp+16] + mova [dstq ], m1 + mova m6, [rsp+32] + PAETH 6, [rsp+48] + mova [dstq+16], m1 + mova m6, [rsp+64] + PAETH 6, [rsp+80] + mova [dstq+32], m1 + mova m6, [rsp+96] + PAETH 6, 7 + mova [dstq+48], m1 + add dstq, strideq + dec hd + jg .w64_loop + RET + + +%macro FILTER 4 ;dst, src, tmp, shuf +%ifnum %4 + pshufb m%2, m%4 +%else + pshufb m%2, %4 +%endif + pshufd m%1, m%2, q0000 ;p0 p1 + pmaddubsw m%1, m2 + pshufd m%3, m%2, q1111 ;p2 p3 + pmaddubsw m%3, m3 + paddw m%1, [base+pw_8] + paddw m%1, m%3 + pshufd m%3, m%2, q2222 ;p4 p5 + pmaddubsw m%3, m4 + paddw m%1, m%3 + pshufd m%3, m%2, q3333 ;p6 __ + pmaddubsw m%3, m5 + paddw m%1, m%3 + psraw m%1, 4 + packuswb m%1, m%1 +%endmacro + +cglobal ipred_filter_8bpc, 3, 7, 8, dst, stride, tl, w, h, filter +%define base r6-$$ + LEA r6, $$ + tzcnt wd, wm +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + shl filterd, 6 + lea filterq, [base+filter_intra_taps+filterq] + movq m0, [tlq-3] ;_ 6 5 0 1 2 3 4 + movsxd wq, [base+ipred_filter_ssse3_table+wq*4] + mova m2, [filterq+16*0] + mova m3, [filterq+16*1] + mova m4, [filterq+16*2] + mova m5, [filterq+16*3] + lea wq, [base+ipred_filter_ssse3_table+wq] + mov hd, hm + jmp wq +.w4: + mova m1, [base+filter_shuf1] + sub tlq, 3 + sub tlq, hq + jmp .w4_loop_start +.w4_loop: + movd m0, [tlq+hq] + punpckldq m0, m6 + lea dstq, [dstq+strideq*2] +.w4_loop_start: + FILTER 6, 0, 7, 1 + movd [dstq+strideq*0], m6 + pshuflw m6, m6, q1032 + movd [dstq+strideq*1], m6 + sub hd, 2 + jg .w4_loop + RET + +ALIGN function_align +.w8: + movq m6, [tlq+1] ;_ _ _ 0 1 2 3 4 + sub tlq, 5 + sub tlq, hq + +.w8_loop: + FILTER 7, 0, 1, [base+filter_shuf1] + punpcklqdq m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + FILTER 0, 6, 1, [base+filter_shuf2] + + punpckldq m6, m7, m0 + movq [dstq+strideq*0], m6 + punpckhqdq m6, m6 + movq [dstq+strideq*1], m6 + + movd m0, [tlq+hq] ;_ 6 5 0 + punpckldq m0, m6 ;_ 6 5 0 1 2 3 4 + + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET + +ALIGN function_align +.w16: + movu m6, [tlq+1] ;top row + sub tlq, 5 + sub tlq, hq + +.w16_loop: + FILTER 7, 0, 1, [base+filter_shuf1] + punpcklqdq m0, m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+strideq*0], m7 + psrlq m7, 32 + palignr m7, m6, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m7, m6 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+4+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + + FILTER 7, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+8+strideq*0], m7 + psrlq m7, 32 + palignr m7, m6, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + movd [dstq+12+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + mova [dstq+strideq*1], m6 + + movd m0, [tlq+hq] ;_ 6 5 0 + punpckldq m0, m6 ;_ 6 5 0 1 2 3 4 + + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16_loop + RET + +ALIGN function_align +.w32: + movu m6, [tlq+1] ;top row + lea filterq, [tlq+17] + sub tlq, 5 + sub tlq, hq + +.w32_loop: + FILTER 7, 0, 1, [base+filter_shuf1] + punpcklqdq m0, m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+strideq*0], m7 + psrlq m7, 32 + palignr m7, m6, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m7, m6 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+4+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + + FILTER 7, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+8+strideq*0], m7 + psrlq m7, 32 + palignr m7, m6, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + movu m1, [filterq] + punpckldq m0, m7, m1 ;_ _ _ 0 1 2 3 4 _ _ _ _ _ _ _ _ + punpcklqdq m0, m6 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+12+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + mova [dstq+strideq*1], m6 + + mova m6, m1 + + FILTER 7, 0, 6, [base+filter_shuf2] + punpcklqdq m0, m1, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+16+strideq*0], m7 + psrlq m7, 32 + palignr m7, m1, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m7, m6 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+20+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + + FILTER 7, 0, 1, [base+filter_shuf2] + punpcklqdq m0, m6, m7 ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6 + movd [dstq+24+strideq*0], m7 + psrlq m7, 32 + palignr m7, m6, 4 + + FILTER 6, 0, 1, [base+filter_shuf2] + movd [dstq+28+strideq*0], m6 + psrlq m6, 32 + palignr m6, m7, 4 + mova [dstq+16+strideq*1], m6 + + mova m6, [dstq+strideq*1] + movd m0, [tlq+hq] ;_ 6 5 0 + punpckldq m0, m6 ;_ 6 5 0 1 2 3 4 + lea filterq, [dstq+16+strideq*1] + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET diff --git a/third_party/dav1d/src/x86/itx.h b/third_party/dav1d/src/x86/itx.h new file mode 100644 index 0000000000..478eb6c6b6 --- /dev/null +++ b/third_party/dav1d/src/x86/itx.h @@ -0,0 +1,363 @@ +/* + * Copyright © 2018-2023, VideoLAN and dav1d authors + * Copyright © 2018-2023, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/itx.h" + +#define BF_BPC(x, bits, suffix) x##_##bits##bpc_##suffix + +#define decl_itx2_fns(w, h, opt) \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) + +#define decl_itx12_fns(w, h, opt) \ +decl_itx2_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) + +#define decl_itx16_fns(w, h, opt) \ +decl_itx12_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) + +#define decl_itx17_fns(w, h, opt) \ +decl_itx16_fns(w, h, opt); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) + +#define decl_itx_fns(ext) \ +decl_itx17_fns( 4, 4, ext); \ +decl_itx16_fns( 4, 8, ext); \ +decl_itx16_fns( 4, 16, ext); \ +decl_itx16_fns( 8, 4, ext); \ +decl_itx16_fns( 8, 8, ext); \ +decl_itx16_fns( 8, 16, ext); \ +decl_itx2_fns ( 8, 32, ext); \ +decl_itx16_fns(16, 4, ext); \ +decl_itx16_fns(16, 8, ext); \ +decl_itx12_fns(16, 16, ext); \ +decl_itx2_fns (16, 32, ext); \ +decl_itx2_fns (32, 8, ext); \ +decl_itx2_fns (32, 16, ext); \ +decl_itx2_fns (32, 32, ext); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_16x64, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_32x64, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x16, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, ext)) + + +#define decl_itx2_bpc_fns(w, h, bpc, opt) \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_identity_identity_##w##x##h, bpc, opt)) + +#define decl_itx12_bpc_fns(w, h, bpc, opt) \ +decl_itx2_bpc_fns(w, h, bpc, opt); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_adst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_identity_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_adst_dct_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_adst_adst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_identity_dct_##w##x##h, bpc, opt)) + +#define decl_itx16_bpc_fns(w, h, bpc, opt) \ +decl_itx12_bpc_fns(w, h, bpc, opt); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_adst_identity_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_identity_adst_##w##x##h, bpc, opt)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, bpc, opt)) + +#define decl_itx_bpc_fns(bpc, ext) \ +decl_itx16_bpc_fns( 4, 4, bpc, ext); \ +decl_itx16_bpc_fns( 4, 8, bpc, ext); \ +decl_itx16_bpc_fns( 4, 16, bpc, ext); \ +decl_itx16_bpc_fns( 8, 4, bpc, ext); \ +decl_itx16_bpc_fns( 8, 8, bpc, ext); \ +decl_itx16_bpc_fns( 8, 16, bpc, ext); \ +decl_itx2_bpc_fns ( 8, 32, bpc, ext); \ +decl_itx16_bpc_fns(16, 4, bpc, ext); \ +decl_itx16_bpc_fns(16, 8, bpc, ext); \ +decl_itx12_bpc_fns(16, 16, bpc, ext); \ +decl_itx2_bpc_fns (16, 32, bpc, ext); \ +decl_itx2_bpc_fns (32, 8, bpc, ext); \ +decl_itx2_bpc_fns (32, 16, bpc, ext); \ +decl_itx2_bpc_fns (32, 32, bpc, ext); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_16x64, bpc, ext)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_32x64, bpc, ext)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_64x16, bpc, ext)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_64x32, bpc, ext)); \ +decl_itx_fn(BF_BPC(dav1d_inv_txfm_add_dct_dct_64x64, bpc, ext)) + +decl_itx_fns(avx512icl); +decl_itx_bpc_fns(10, avx512icl); +decl_itx_fns(avx2); +decl_itx_bpc_fns(10, avx2); +decl_itx_bpc_fns(12, avx2); +decl_itx_fns(sse4); +decl_itx_fns(ssse3); +decl_itx_fn(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_avx2); +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_4x4, sse2)); + +static ALWAYS_INLINE void itx_dsp_init_x86(Dav1dInvTxfmDSPContext *const c, const int bpc) { +#define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ + c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ + BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) + +#define assign_itx1_fn(pfx, w, h, ext) \ + assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) + +#define assign_itx2_fn(pfx, w, h, ext) \ + assign_itx1_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, identity_identity, IDTX, ext) + +#define assign_itx12_fn(pfx, w, h, ext) \ + assign_itx2_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, dct_adst, ADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, ext); \ + assign_itx_fn(pfx, w, h, dct_identity, H_DCT, ext); \ + assign_itx_fn(pfx, w, h, adst_dct, DCT_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_adst, ADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_dct, V_DCT, ext) + +#define assign_itx16_fn(pfx, w, h, ext) \ + assign_itx12_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, adst_identity, H_ADST, ext); \ + assign_itx_fn(pfx, w, h, flipadst_identity, H_FLIPADST, ext); \ + assign_itx_fn(pfx, w, h, identity_adst, V_ADST, ext); \ + assign_itx_fn(pfx, w, h, identity_flipadst, V_FLIPADST, ext) + +#define assign_itx17_fn(pfx, w, h, ext) \ + assign_itx16_fn(pfx, w, h, ext); \ + assign_itx_fn(pfx, w, h, wht_wht, WHT_WHT, ext) + + +#define assign_itx_bpc_fn(pfx, w, h, type, type_enum, bpc, ext) \ + c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ + BF_BPC(dav1d_inv_txfm_add_##type##_##w##x##h, bpc, ext) + +#define assign_itx1_bpc_fn(pfx, w, h, bpc, ext) \ + assign_itx_bpc_fn(pfx, w, h, dct_dct, DCT_DCT, bpc, ext) + +#define assign_itx2_bpc_fn(pfx, w, h, bpc, ext) \ + assign_itx1_bpc_fn(pfx, w, h, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, identity_identity, IDTX, bpc, ext) + +#define assign_itx12_bpc_fn(pfx, w, h, bpc, ext) \ + assign_itx2_bpc_fn(pfx, w, h, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, dct_adst, ADST_DCT, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, dct_flipadst, FLIPADST_DCT, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, dct_identity, H_DCT, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, adst_dct, DCT_ADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, adst_adst, ADST_ADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, adst_flipadst, FLIPADST_ADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, flipadst_dct, DCT_FLIPADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, flipadst_adst, ADST_FLIPADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, flipadst_flipadst, FLIPADST_FLIPADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, identity_dct, V_DCT, bpc, ext) + +#define assign_itx16_bpc_fn(pfx, w, h, bpc, ext) \ + assign_itx12_bpc_fn(pfx, w, h, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, adst_identity, H_ADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, flipadst_identity, H_FLIPADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, identity_adst, V_ADST, bpc, ext); \ + assign_itx_bpc_fn(pfx, w, h, identity_flipadst, V_FLIPADST, bpc, ext) + + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; + + assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, sse2); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + +#if BITDEPTH == 8 + assign_itx16_fn(, 4, 4, ssse3); + assign_itx16_fn(R, 4, 8, ssse3); + assign_itx16_fn(R, 8, 4, ssse3); + assign_itx16_fn(, 8, 8, ssse3); + assign_itx16_fn(R, 4, 16, ssse3); + assign_itx16_fn(R, 16, 4, ssse3); + assign_itx16_fn(R, 8, 16, ssse3); + assign_itx16_fn(R, 16, 8, ssse3); + assign_itx12_fn(, 16, 16, ssse3); + assign_itx2_fn (R, 8, 32, ssse3); + assign_itx2_fn (R, 32, 8, ssse3); + assign_itx2_fn (R, 16, 32, ssse3); + assign_itx2_fn (R, 32, 16, ssse3); + assign_itx2_fn (, 32, 32, ssse3); + assign_itx1_fn (R, 16, 64, ssse3); + assign_itx1_fn (R, 32, 64, ssse3); + assign_itx1_fn (R, 64, 16, ssse3); + assign_itx1_fn (R, 64, 32, ssse3); + assign_itx1_fn ( , 64, 64, ssse3); +#endif + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; + +#if BITDEPTH == 16 + if (bpc == 10) { + assign_itx16_fn(, 4, 4, sse4); + assign_itx16_fn(R, 4, 8, sse4); + assign_itx16_fn(R, 4, 16, sse4); + assign_itx16_fn(R, 8, 4, sse4); + assign_itx16_fn(, 8, 8, sse4); + assign_itx16_fn(R, 8, 16, sse4); + assign_itx16_fn(R, 16, 4, sse4); + assign_itx16_fn(R, 16, 8, sse4); + assign_itx12_fn(, 16, 16, sse4); + assign_itx2_fn (R, 8, 32, sse4); + assign_itx2_fn (R, 32, 8, sse4); + assign_itx2_fn (R, 16, 32, sse4); + assign_itx2_fn (R, 32, 16, sse4); + assign_itx2_fn (, 32, 32, sse4); + assign_itx1_fn (R, 16, 64, sse4); + assign_itx1_fn (R, 32, 64, sse4); + assign_itx1_fn (R, 64, 16, sse4); + assign_itx1_fn (R, 64, 32, sse4); + assign_itx1_fn (, 64, 64, sse4); + } +#endif + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, avx2); + +#if BITDEPTH == 8 + assign_itx16_fn( , 4, 4, avx2); + assign_itx16_fn(R, 4, 8, avx2); + assign_itx16_fn(R, 4, 16, avx2); + assign_itx16_fn(R, 8, 4, avx2); + assign_itx16_fn( , 8, 8, avx2); + assign_itx16_fn(R, 8, 16, avx2); + assign_itx2_fn (R, 8, 32, avx2); + assign_itx16_fn(R, 16, 4, avx2); + assign_itx16_fn(R, 16, 8, avx2); + assign_itx12_fn( , 16, 16, avx2); + assign_itx2_fn (R, 16, 32, avx2); + assign_itx1_fn (R, 16, 64, avx2); + assign_itx2_fn (R, 32, 8, avx2); + assign_itx2_fn (R, 32, 16, avx2); + assign_itx2_fn ( , 32, 32, avx2); + assign_itx1_fn (R, 32, 64, avx2); + assign_itx1_fn (R, 64, 16, avx2); + assign_itx1_fn (R, 64, 32, avx2); + assign_itx1_fn ( , 64, 64, avx2); +#else + if (bpc == 10) { + assign_itx16_bpc_fn( , 4, 4, 10, avx2); + assign_itx16_bpc_fn(R, 4, 8, 10, avx2); + assign_itx16_bpc_fn(R, 4, 16, 10, avx2); + assign_itx16_bpc_fn(R, 8, 4, 10, avx2); + assign_itx16_bpc_fn( , 8, 8, 10, avx2); + assign_itx16_bpc_fn(R, 8, 16, 10, avx2); + assign_itx2_bpc_fn (R, 8, 32, 10, avx2); + assign_itx16_bpc_fn(R, 16, 4, 10, avx2); + assign_itx16_bpc_fn(R, 16, 8, 10, avx2); + assign_itx12_bpc_fn( , 16, 16, 10, avx2); + assign_itx2_bpc_fn (R, 16, 32, 10, avx2); + assign_itx1_bpc_fn (R, 16, 64, 10, avx2); + assign_itx2_bpc_fn (R, 32, 8, 10, avx2); + assign_itx2_bpc_fn (R, 32, 16, 10, avx2); + assign_itx2_bpc_fn ( , 32, 32, 10, avx2); + assign_itx1_bpc_fn (R, 32, 64, 10, avx2); + assign_itx1_bpc_fn (R, 64, 16, 10, avx2); + assign_itx1_bpc_fn (R, 64, 32, 10, avx2); + assign_itx1_bpc_fn ( , 64, 64, 10, avx2); + } else { + assign_itx16_bpc_fn( , 4, 4, 12, avx2); + assign_itx16_bpc_fn(R, 4, 8, 12, avx2); + assign_itx16_bpc_fn(R, 4, 16, 12, avx2); + assign_itx16_bpc_fn(R, 8, 4, 12, avx2); + assign_itx16_bpc_fn( , 8, 8, 12, avx2); + assign_itx16_bpc_fn(R, 8, 16, 12, avx2); + assign_itx2_bpc_fn (R, 8, 32, 12, avx2); + assign_itx16_bpc_fn(R, 16, 4, 12, avx2); + assign_itx16_bpc_fn(R, 16, 8, 12, avx2); + assign_itx12_bpc_fn( , 16, 16, 12, avx2); + assign_itx2_bpc_fn (R, 32, 8, 12, avx2); + assign_itx_bpc_fn(R, 16, 32, identity_identity, IDTX, 12, avx2); + assign_itx_bpc_fn(R, 32, 16, identity_identity, IDTX, 12, avx2); + assign_itx_bpc_fn( , 32, 32, identity_identity, IDTX, 12, avx2); + } +#endif + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + +#if BITDEPTH == 8 + assign_itx16_fn( , 4, 4, avx512icl); // no wht + assign_itx16_fn(R, 4, 8, avx512icl); + assign_itx16_fn(R, 4, 16, avx512icl); + assign_itx16_fn(R, 8, 4, avx512icl); + assign_itx16_fn( , 8, 8, avx512icl); + assign_itx16_fn(R, 8, 16, avx512icl); + assign_itx2_fn (R, 8, 32, avx512icl); + assign_itx16_fn(R, 16, 4, avx512icl); + assign_itx16_fn(R, 16, 8, avx512icl); + assign_itx12_fn( , 16, 16, avx512icl); + assign_itx2_fn (R, 16, 32, avx512icl); + assign_itx1_fn (R, 16, 64, avx512icl); + assign_itx2_fn (R, 32, 8, avx512icl); + assign_itx2_fn (R, 32, 16, avx512icl); + assign_itx2_fn ( , 32, 32, avx512icl); + assign_itx1_fn (R, 32, 64, avx512icl); + assign_itx1_fn (R, 64, 16, avx512icl); + assign_itx1_fn (R, 64, 32, avx512icl); + assign_itx1_fn ( , 64, 64, avx512icl); +#else + if (bpc == 10) { + assign_itx16_bpc_fn( , 8, 8, 10, avx512icl); + assign_itx16_bpc_fn(R, 8, 16, 10, avx512icl); + assign_itx2_bpc_fn (R, 8, 32, 10, avx512icl); + assign_itx16_bpc_fn(R, 16, 8, 10, avx512icl); + assign_itx12_bpc_fn( , 16, 16, 10, avx512icl); + assign_itx2_bpc_fn (R, 16, 32, 10, avx512icl); + assign_itx2_bpc_fn (R, 32, 8, 10, avx512icl); + assign_itx2_bpc_fn (R, 32, 16, 10, avx512icl); + assign_itx2_bpc_fn ( , 32, 32, 10, avx512icl); + assign_itx1_bpc_fn (R, 16, 64, 10, avx512icl); + } +#endif +#endif +} diff --git a/third_party/dav1d/src/x86/itx16_avx2.asm b/third_party/dav1d/src/x86/itx16_avx2.asm new file mode 100644 index 0000000000..2315ec1e47 --- /dev/null +++ b/third_party/dav1d/src/x86/itx16_avx2.asm @@ -0,0 +1,8599 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; Copyright © 2021, Matthias Dressel +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 +itx4_shuf: dd 0x50401600, 0xd0c09284, 0x70603422, 0xf0e0b0a6 + dd 0x50401701, 0xd0c09385, 0x70603523, 0xf0e0b1a7 +idct4_12_shuf: dd 0, 2, 4, 6, 1, 3, 5, 7 +idct4_12_shuf2: dd 2, 0, 6, 4, 3, 1, 7, 5 +iadst8_12_shuf: dd 0, 4, 1, 5, 2, 6, 3, 7 +idct16_12_shuf: dd 0, 4, 1, 5, 3, 7, 2, 6 +iadst16_12_shuf: dd 3, 7, 0, 4, 2, 6, 1, 5 +pw_2048_m2048: dw 2048, 2048, 2048, 2048, -2048, -2048, -2048, -2048 +idct4_shuf: db 0, 1, 4, 5, 12, 13, 8, 9, 2, 3, 6, 7, 14, 15, 10, 11 +idct32_shuf: db 0, 1, 8, 9, 4, 5, 12, 13, 2, 3, 10, 11, 6, 7, 14, 15 + +%macro COEF_PAIR 2-3 0 +pd_%1_%2: dd %1, %1, %2, %2 +%define pd_%1 (pd_%1_%2 + 4*0) +%define pd_%2 (pd_%1_%2 + 4*2) +%if %3 +dd -%2, -%2 +%define pd_%2_m%2 pd_%2 +%endif +%endmacro + +COEF_PAIR 201, 995 +COEF_PAIR 401, 1931 +COEF_PAIR 799, 3406 +COEF_PAIR 1380, 601 +COEF_PAIR 1751, 2440 +COEF_PAIR 2598, 1189 +COEF_PAIR 2751, 2106 +COEF_PAIR 2896, 1567, 1 +COEF_PAIR 2896, 3784, 1 +COEF_PAIR 3035, 3513 +COEF_PAIR 3166, 3920 +COEF_PAIR 3703, 3290 +COEF_PAIR 3857, 4052 +COEF_PAIR 4017, 2276 +COEF_PAIR 4076, 3612 +COEF_PAIR 4091, 3973 + +pd_8: dd 8 +pd_m601: dd -601 +pd_m1189: dd -1189 +pd_m1380: dd -1380 +pd_m2106: dd -2106 +pd_m2598: dd -2598 +pd_m2751: dd -2751 +pd_m3344: dd -3344 +pd_1024: dd 1024 +pd_1321: dd 1321 +pd_1448: dd 1448 +pd_1697: dd 1697 +pd_2482: dd 2482 +pd_3072: dd 3072 ; 1024 + 2048 +pd_3803: dd 3803 +pd_5119: dd 5119 ; 1024 + 4096 - 1 +pd_5120: dd 5120 ; 1024 + 4096 +pd_5793: dd 5793 +pd_6144: dd 6144 ; 2048 + 4096 +pd_17408: dd 17408 ; 1024 + 16384 + +pixel_10bpc_max: times 2 dw 0x03ff +pixel_12bpc_max: times 2 dw 0x0fff +dconly_10bpc: times 2 dw 0x7c00 +dconly_12bpc: times 2 dw 0x7000 +clip_18b_min: dd -0x20000 +clip_18b_max: dd 0x1ffff +clip_20b_min: dd -0x80000 +clip_20b_max: dd 0x7ffff + +idct64_mul_16bpc: +dd 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474, 401, 4076, 799, 4017 +dd -700, 4036, 2359, 3349, -2191, 3461, 897, 3996, -2598, -3166, -4017, -799 +dd 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092, 1931, 3612, 3406, 2276 +dd -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889, -1189, -3920, -2276, -3406 + +cextern deint_shuf +cextern idct64_mul +cextern pw_1697x8 +cextern pw_1697x16 +cextern pw_1567_3784 +cextern pw_m1567_m3784 +cextern pw_m3784_1567 +cextern pw_2896_2896 +cextern pw_m2896_2896 +cextern pw_5 +cextern pw_2048 +cextern pw_4096 +cextern pw_8192 +cextern pw_16384 +cextern pw_2896x8 +cextern pd_2048 + +cextern idct_4x8_internal_8bpc_avx2.main +cextern idct_4x16_internal_8bpc_avx2.main +cextern idct_8x8_internal_8bpc_avx2.main +cextern idct_8x16_internal_8bpc_avx2.main +cextern idct_16x4_internal_8bpc_avx2.main +cextern idct_16x8_internal_8bpc_avx2.main +cextern idct_16x16_internal_8bpc_avx2.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx2.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx2.main_fast +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx2.main_oddhalf +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx2.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx2.main_part1 +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx2.main_part2_internal + +cextern iadst_4x4_internal_8bpc_avx2.main +cextern iadst_4x8_internal_8bpc_avx2.main_pass2 +cextern iadst_4x16_internal_8bpc_avx2.main2 +cextern iadst_8x4_internal_8bpc_avx2.main +cextern iadst_8x8_internal_8bpc_avx2.main_pass2 +cextern iadst_8x16_internal_8bpc_avx2.main +cextern iadst_8x16_internal_8bpc_avx2.main_pass2_end +cextern iadst_16x4_internal_8bpc_avx2.main +cextern iadst_16x8_internal_8bpc_avx2.main +cextern iadst_16x8_internal_8bpc_avx2.main_pass2_end +cextern iadst_16x16_internal_8bpc_avx2.main +cextern iadst_16x16_internal_8bpc_avx2.main_pass2_end + +SECTION .text + +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +%macro WRAP_XMM 1+ + INIT_XMM cpuname + %1 + INIT_YMM cpuname +%endmacro + +%macro IWHT4_1D_PACKED 0 + ; m0 = in0 in2, m1 = in1 in3 + psubd m2, m0, m1 ; t2 + paddd xm0, xm1 ; t0 + vpermq m2, m2, q3322 + vpermq m0, m0, q1100 + vpermq m1, m1, q3120 + psubd m3, m0, m2 + psrad m3, 1 + psubd m3, m1 ; t1 t3 + psubd m0, m3 ; ____ out0 + paddd m2, m3 ; out3 ____ +%endmacro + +INIT_YMM avx2 +cglobal inv_txfm_add_wht_wht_4x4_16bpc, 3, 7, 6, dst, stride, c, eob, bdmax + mova xm0, [cq+16*0] + vinserti128 m0, [cq+16*2], 1 + mova xm1, [cq+16*1] + vinserti128 m1, [cq+16*3], 1 + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + lea r6, [dstq+strideq*2] + psrad m0, 2 + psrad m1, 2 + IWHT4_1D_PACKED + punpckhdq m0, m3 + punpckldq m3, m2 + punpckhqdq m1, m0, m3 + punpcklqdq m0, m3 + IWHT4_1D_PACKED + vpblendd m0, m2, 0x33 + packssdw m0, m3 + vextracti128 xm2, m0, 1 + punpckhdq xm1, xm0, xm2 ; out2 out1 + punpckldq xm0, xm2 ; out3 out0 + movq xm2, [r6 +strideq*1] + movhps xm2, [dstq+strideq*0] + movq xm3, [r6 +strideq*0] + movhps xm3, [dstq+strideq*1] +%ifidn bdmaxd, bdmaxm + movd xm5, bdmaxd + vpbroadcastw xm5, xm5 +%else ; win64: load from stack + vpbroadcastw xm5, bdmaxm +%endif + paddsw xm0, xm2 + paddsw xm1, xm3 + pmaxsw xm0, xm4 + pmaxsw xm1, xm4 + pminsw xm0, xm5 + pminsw xm1, xm5 + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movq [r6 +strideq*0], xm1 + movq [r6 +strideq*1], xm0 + RET + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +; flags: 1 = packed, 2 = inv_dst2 +; skip round/shift if rnd is not a number +%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], flags +%if %8 < 32 + pmulld m%4, m%1, m%8 + pmulld m%3, m%2, m%8 +%else +%if %9 & 1 + vbroadcasti128 m%3, [pd_%8] +%else + vpbroadcastd m%3, [pd_%8] +%endif + pmulld m%4, m%1, m%3 + pmulld m%3, m%2 +%endif +%if %7 < 32 + pmulld m%1, m%7 + pmulld m%2, m%7 +%else +%if %9 & 1 + vbroadcasti128 m%5, [pd_%7] +%else + vpbroadcastd m%5, [pd_%7] +%endif + pmulld m%1, m%5 + pmulld m%2, m%5 +%endif +%if %9 & 2 + psubd m%4, m%6, m%4 + psubd m%2, m%4, m%2 +%else +%ifnum %6 + paddd m%4, m%6 +%endif + paddd m%2, m%4 +%endif +%ifnum %6 + paddd m%1, m%6 +%endif + psubd m%1, m%3 +%ifnum %6 + psrad m%2, 12 + psrad m%1, 12 +%endif +%endmacro + +%macro INV_TXFM_FN 4-5 10 ; type1, type2, eob_offset, size, bitdepth +cglobal inv_txfm_add_%1_%2_%4_%5bpc, 4, 5, 0, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%4_internal_%5bpc) + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [m(i%2_%4_internal_%5bpc).pass2] +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else +%if %3 + add eobd, %3 +%endif + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 4x4, %3 +%ifidn %1_%2, dct_dct + vpbroadcastd xm2, [dconly_%3bpc] +%if %3 = 10 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 4 +.dconly2: + add r6d, 128 + sar r6d, 8 +.dconly3: + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + movd xm0, r6d + paddsw xm0, xm2 + vpbroadcastw xm0, xm0 +.dconly_loop: + movq xm1, [dstq+strideq*0] + movhps xm1, [dstq+strideq*1] + paddsw xm1, xm0 + psubusw xm1, xm2 + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + WRAP_XMM RET +%else + jmp m(inv_txfm_add_dct_dct_4x4_10bpc).dconly +%endif +%endif +%endmacro + +%macro IDCT4_1D_PACKED 6 ; dst/src[1-2], tmp[1-3], rnd + ITX_MULSUB_2D %1, %2, %3, %4, %5, %6, 2896_1567, 2896_3784, 1 + punpckhqdq m%3, m%2, m%1 ; t3 t2 + punpcklqdq m%2, m%1 ; t0 t1 + paddd m%1, m%2, m%3 ; out0 out1 + psubd m%2, m%3 ; out3 out2 +%endmacro + +%macro IDCT4_1D_PACKED_WORD 6 ; dst/src[1-2], tmp[1-3], rnd + vpbroadcastd m%5, [pw_m3784_1567] + punpckhwd m%3, m%2, m%1 + vpbroadcastd m%4, [pw_1567_3784] + punpcklwd m%2, m%1 + vpbroadcastd m%1, [pw_m2896_2896] + pmaddwd m%5, m%3 + pmaddwd m%3, m%4 + vpbroadcastd m%4, [pw_2896_2896] + pmaddwd m%1, m%2 + pmaddwd m%2, m%4 + REPX {paddd x, m%6}, m%5, m%3, m%1, m%2 + REPX {psrad x, 12 }, m%5, m%3, m%1, m%2 + packssdw m%3, m%5 ; t3 t2 + packssdw m%2, m%1 ; t0 t1 + paddsw m%1, m%2, m%3 ; out0 out1 + psubsw m%2, m%3 ; out3 out2 +%endmacro + +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, identity +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst + +cglobal idct_4x4_internal_10bpc, 0, 7, 6, dst, stride, c, eob, tx2 + call .main + vbroadcasti128 m2, [idct4_shuf] + packssdw m0, m1 + pshufb m0, m2 + jmp tx2q +.pass2: + vextracti128 xm1, m0, 1 + WRAP_XMM IDCT4_1D_PACKED_WORD 0, 1, 2, 3, 4, 5 + packssdw xm5, xm5 ; pw_2048 + pmulhrsw xm0, xm5 + pmulhrsw xm1, xm5 + movq xm2, [dstq+strideq*0] + movhps xm2, [dstq+strideq*1] + lea r6, [dstq+strideq*2] + movq xm3, [r6 +strideq*1] + movhps xm3, [r6 +strideq*0] + vpbroadcastd xm5, [pixel_10bpc_max] + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + paddw xm0, xm2 + paddw xm1, xm3 + pmaxsw xm0, xm4 + pmaxsw xm1, xm4 + pminsw xm0, xm5 + pminsw xm1, xm5 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movhps [r6 +strideq*0], xm1 + movq [r6 +strideq*1], xm1 + RET +ALIGN function_align +.main: + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m5, [pd_2048] +.main2: + IDCT4_1D_PACKED 0, 1, 2, 3, 4, 5 + ret + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +%macro IADST4_1D 0 + vpbroadcastd m5, [pd_1321] + vpbroadcastd m7, [pd_2482] + pmulld m4, m0, m5 ; 1321*in0 + pmulld m6, m3, m7 ; 2482*in3 + paddd m4, m6 ; 1321*in0 + 2482*in3 + pmulld m6, m0, m7 ; 2482*in0 + paddd m0, m3 ; in0 + in3 + paddd m7, m5 ; pd_3803 + pmulld m5, m2 ; 1321*in2 + pmulld m3, m7 ; 3803*in3 + pmulld m7, m2 ; 3803*in2 + psubd m2, m0 ; in2 - in0 - in3 + vpbroadcastd m0, [pd_m3344] + pmulld m1, m0 ; -t3 + pmulld m2, m0 ; out2 (unrounded) + psubd m6, m5 ; 2482*in0 - 1321*in2 + paddd m4, m7 ; t0 + psubd m6, m3 ; t1 + paddd m3, m4, m6 + psubd m4, m1 ; out0 (unrounded) + psubd m6, m1 ; out1 (unrounded) + paddd m3, m1 ; out3 (unrounded) +%endmacro + +cglobal iadst_4x4_internal_10bpc, 0, 7, 6, dst, stride, c, eob, tx2 + call .main + vinserti128 m0, m4, xm6, 1 + vinserti128 m1, m2, xm3, 1 +.pass1_end: + vpbroadcastd m5, [pd_2048] + mova m2, [itx4_shuf] + paddd m0, m5 + paddd m1, m5 + psrad m0, 12 + psrad m1, 12 + packssdw m0, m1 + vpermd m0, m2, m0 + psrld m2, 4 + pshufb m0, m2 +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + jmp tx2q +.pass2: + lea r6, [deint_shuf+128] + vextracti128 xm1, m0, 1 + call m(iadst_4x4_internal_8bpc).main +.end: + vpbroadcastd xm4, [pw_2048] + movq xm2, [dstq+strideq*0] + movhps xm2, [dstq+strideq*1] + lea r6, [dstq+strideq*2] + movq xm3, [r6 +strideq*0] + movhps xm3, [r6 +strideq*1] + vpbroadcastd xm5, [pixel_10bpc_max] + pmulhrsw xm0, xm4 + pmulhrsw xm1, xm4 + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + paddw xm0, xm2 + paddw xm1, xm3 + pmaxsw xm0, xm4 + pmaxsw xm1, xm4 + pminsw xm0, xm5 + pminsw xm1, xm5 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [r6 +strideq*0], xm1 + movhps [r6 +strideq*1], xm1 + RET +ALIGN function_align +.main: + mova xm0, [cq+16*0] + mova xm1, [cq+16*1] + mova xm2, [cq+16*2] + mova xm3, [cq+16*3] +%if WIN64 + movaps [rsp+16], xmm6 + movaps [rsp+32], xmm7 +%endif +.main2: + WRAP_XMM IADST4_1D + ret + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_10bpc, 0, 7, 6, dst, stride, c, eob, tx2 + call m(iadst_4x4_internal_10bpc).main + vinserti128 m0, m3, xm2, 1 + vinserti128 m1, m6, xm4, 1 + jmp m(iadst_4x4_internal_10bpc).pass1_end +.pass2: + lea r6, [deint_shuf+128] + vextracti128 xm1, m0, 1 + call m(iadst_4x4_internal_8bpc).main + vpbroadcastd xm4, [pw_2048] + movq xm3, [dstq+strideq*1] + movhps xm3, [dstq+strideq*0] + lea r6, [dstq+strideq*2] + movq xm2, [r6 +strideq*1] + movhps xm2, [r6 +strideq*0] + vpbroadcastd xm5, [pixel_10bpc_max] + pmulhrsw xm0, xm4 + pmulhrsw xm1, xm4 + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + paddw xm0, xm2 + paddw xm1, xm3 + pmaxsw xm0, xm4 + pmaxsw xm1, xm4 + pminsw xm0, xm5 + pminsw xm1, xm5 + movhps [dstq+strideq*0], xm1 + movq [dstq+strideq*1], xm1 + movhps [r6 +strideq*0], xm0 + movq [r6 +strideq*1], xm0 + RET + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_10bpc, 0, 7, 6, dst, stride, c, eob, tx2 + vpbroadcastd m1, [pd_5793] + pmulld m0, m1, [cq+32*0] + pmulld m1, [cq+32*1] + vpbroadcastd m5, [pd_2048] + mova m3, [itx4_shuf] + paddd m0, m5 + paddd m1, m5 + psrad m0, 12 + psrad m1, 12 + packssdw m0, m1 + vpermd m0, m3, m0 + psrld m3, 4 + pshufb m0, m3 + jmp tx2q +.pass2: + vpbroadcastd m1, [pw_1697x8] + movq xm2, [dstq+strideq*0] + movhps xm2, [dstq+strideq*1] + lea r6, [dstq+strideq*2] + pmulhrsw m1, m0 + paddsw m0, m1 + movq xm3, [r6 +strideq*0] + movhps xm3, [r6 +strideq*1] + vpbroadcastd xm4, [pixel_10bpc_max] + packssdw m5, m5 ; pw_2048 + pmulhrsw m0, m5 + pxor m5, m5 + mova [cq+32*0], m5 + mova [cq+32*1], m5 + vextracti128 xm1, m0, 1 + paddw xm0, xm2 + paddw xm1, xm3 + pmaxsw xm0, xm5 + pmaxsw xm1, xm5 + pminsw xm0, xm4 + pminsw xm1, xm4 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [r6 +strideq*0], xm1 + movhps [r6 +strideq*1], xm1 + RET + +INV_TXFM_4X4_FN dct, dct, 12 +INV_TXFM_4X4_FN dct, identity, 12 +INV_TXFM_4X4_FN dct, adst, 12 +INV_TXFM_4X4_FN dct, flipadst, 12 + +cglobal idct_4x4_internal_12bpc, 0, 7, 8, dst, stride, c, eob, tx2 + call m(idct_4x4_internal_10bpc).main + mova m3, [idct4_12_shuf] + mova m4, [idct4_12_shuf2] + vpermd m2, m4, m1 + vpermd m1, m3, m0 + jmp m(iadst_4x4_internal_12bpc).pass1_end2 +.pass2: + vpbroadcastd m5, [pd_2048] + vpermq m0, m0, q3120 + vpermq m1, m1, q3120 + call m(idct_4x4_internal_10bpc).main2 + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + jmp m(iadst_4x4_internal_12bpc).end + +INV_TXFM_4X4_FN adst, dct, 12 +INV_TXFM_4X4_FN adst, adst, 12 +INV_TXFM_4X4_FN adst, flipadst, 12 +INV_TXFM_4X4_FN adst, identity, 12 + +cglobal iadst_4x4_internal_12bpc, 0, 7, 8, dst, stride, c, eob, tx2 + call m(iadst_4x4_internal_10bpc).main + vinserti128 m1, m4, xm6, 1 + vinserti128 m2, xm3, 1 +.pass1_end: + mova m3, [itx4_shuf] + vpbroadcastd m5, [pd_1024] + psrad m1, 1 + psrad m2, 1 + vpermd m1, m3, m1 + vpermd m2, m3, m2 + paddd m1, m5 + paddd m2, m5 + psrad m1, 11 + psrad m2, 11 +.pass1_end2: + vpbroadcastd m3, [clip_18b_min] + vpbroadcastd m4, [clip_18b_max] + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + pmaxsd m0, m3 + pmaxsd m1, m3 + pminsd m0, m4 + pminsd m1, m4 + jmp tx2q +.pass2: + call .main_pass2 + vinserti128 m0, m4, xm6, 1 + vinserti128 m1, m2, xm3, 1 +.pass2_end: + vpbroadcastd m5, [pd_2048] + paddd m0, m5 + paddd m1, m5 + psrad m0, 12 + psrad m1, 12 +.end: +%if WIN64 + WIN64_RESTORE_XMM_INTERNAL + %assign xmm_regs_used 6 +%endif +.end2: + vpbroadcastd m4, [pw_16384] + movq xm2, [dstq+strideq*0] + movq xm3, [dstq+strideq*1] + lea r6, [dstq+strideq*2] + movhps xm2, [r6 +strideq*0] ; dst0 dst2 + movhps xm3, [r6 +strideq*1] ; dst1 dst3 + vpbroadcastd m5, [pixel_12bpc_max] + vinserti128 m2, xm3, 1 + psrad m0, 3 + psrad m1, 3 + packssdw m0, m1 ; t0 t2 t1 t3 + pmulhrsw m0, m4 + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + paddw m0, m2 ; out0 out2 out1 out3 + pmaxsw m0, m4 + pminsw m0, m5 + vextracti128 xm1, m0, 1 ; out1 out3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [r6 +strideq*0], xm0 + movhps [r6 +strideq*1], xm1 + RET +.main_pass2: + vextracti128 xm3, m1, 1 + mova xm2, xm1 + vextracti128 xm1, m0, 1 + jmp m(iadst_4x4_internal_10bpc).main2 + +INV_TXFM_4X4_FN flipadst, dct, 12 +INV_TXFM_4X4_FN flipadst, adst, 12 +INV_TXFM_4X4_FN flipadst, flipadst, 12 +INV_TXFM_4X4_FN flipadst, identity, 12 + +cglobal iflipadst_4x4_internal_12bpc, 0, 7, 8, dst, stride, c, eob, tx2 + call m(iadst_4x4_internal_10bpc).main + vinserti128 m1, m3, xm2, 1 + vinserti128 m2, m6, xm4, 1 + jmp m(iadst_4x4_internal_12bpc).pass1_end +.pass2: + call m(iadst_4x4_internal_12bpc).main_pass2 + vinserti128 m0, m3, xm2, 1 + vinserti128 m1, m6, xm4, 1 + jmp m(iadst_4x4_internal_12bpc).pass2_end + +INV_TXFM_4X4_FN identity, dct, 12 +INV_TXFM_4X4_FN identity, adst, 12 +INV_TXFM_4X4_FN identity, flipadst, 12 +INV_TXFM_4X4_FN identity, identity, 12 + +cglobal iidentity_4x4_internal_12bpc, 0, 7, 8, dst, stride, c, eob, tx2 + mova m2, [itx4_shuf] + vpbroadcastd m3, [pd_1697] + vpermd m0, m2, [cq+32*0] + vpermd m2, m2, [cq+32*1] + vpbroadcastd m5, [pd_2048] + pmulld m1, m3, m0 + pmulld m3, m2 + paddd m1, m5 + paddd m3, m5 + psrad m1, 12 + psrad m3, 12 + paddd m1, m0 + paddd m2, m3 + jmp m(iadst_4x4_internal_12bpc).pass1_end2 +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + vpbroadcastd m3, [pd_5793] + vpbroadcastd m5, [pd_2048] + pmulld m0, m3 + pmulld m1, m3 + paddd m0, m5 ; 2048 + paddd m1, m5 + psrad m0, 12 + psrad m1, 12 + jmp m(iadst_4x4_internal_12bpc).end + +%macro INV_TXFM_4X8_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 4x8, %3 +%ifidn %1_%2, dct_dct + vpbroadcastd xm2, [dconly_%3bpc] +%if %3 = 10 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 8 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + jmp m(inv_txfm_add_dct_dct_4x4_10bpc).dconly2 +%else + jmp m(inv_txfm_add_dct_dct_4x8_10bpc).dconly +%endif +%endif +%endmacro + +%macro IDCT4_1D 8 ; src[1-4], tmp[1-3], rnd + ITX_MULSUB_2D %2, %4, %5, %6, %7, %8, 1567, 3784 ; t2, t3 + vpbroadcastd m%5, [pd_2896] + pmulld m%1, m%5 + pmulld m%3, m%5 + paddd m%1, m%8 + paddd m%5, m%1, m%3 + psubd m%1, m%3 + psrad m%5, 12 ; t0 + psrad m%1, 12 ; t1 + psubd m%3, m%1, m%2 + paddd m%2, m%1 + paddd m%1, m%5, m%4 + psubd m%4, m%5, m%4 +%endmacro + +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, identity +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst + +cglobal idct_4x8_internal_10bpc, 0, 7, 8, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m3, [pd_2896] + pmulld m0, m3, [cq+32*0] + pmulld m1, m3, [cq+32*1] + pmulld m2, m3, [cq+32*2] + pmulld m3, m3, [cq+32*3] + vpbroadcastd m7, [pd_2048] + REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + IDCT4_1D 0, 1, 2, 3, 4, 5, 6, 7 + jmp tx2q +.pass2: + packssdw m0, m2 + packssdw m1, m3 + lea r6, [deint_shuf+128] + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhdq m1, m0, m2 ; 2 3 + punpckldq m0, m2 ; 0 1 + vextracti128 xm2, m0, 1 ; 4 5 + vextracti128 xm3, m1, 1 ; 6 7 + call m(idct_4x8_internal_8bpc).main + vpbroadcastd xm4, [pw_2048] + REPX {pmulhrsw x, xm4}, xm0, xm1, xm2, xm3 + lea r3, [strideq*3] + lea r6, [dstq+strideq*4] + movq xm4, [dstq+strideq*0] + movhps xm4, [dstq+strideq*1] + movq xm5, [dstq+r3 ] + movhps xm5, [dstq+strideq*2] + movq xm6, [r6 +strideq*0] + movhps xm6, [r6 +strideq*1] + movq xm7, [r6 +r3 ] + movhps xm7, [r6 +strideq*2] + paddw xm0, xm4 ; 0 1 + paddw xm1, xm5 ; 3 2 + paddw xm2, xm6 ; 4 5 + paddw xm3, xm7 ; 7 6 + vpbroadcastd xm5, [pixel_10bpc_max] + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + REPX {pmaxsw x, xm4}, xm0, xm1, xm2, xm3 + REPX {pminsw x, xm5}, xm0, xm1, xm2, xm3 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movhps [dstq+strideq*2], xm1 + movq [dstq+r3 ], xm1 + movq [r6 +strideq*0], xm2 + movhps [r6 +strideq*1], xm2 + movhps [r6 +strideq*2], xm3 + movq [r6 +r3 ], xm3 + RET + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity + +cglobal iadst_4x8_internal_10bpc, 0, 7, 8, dst, stride, c, eob, tx2 + call m(iadst_8x4_internal_10bpc).main + vpbroadcastd m5, [pd_2048] + paddd m0, m5, m4 + paddd m1, m5, m6 + paddd m2, m5 + paddd m3, m5 +.pass1_end: + REPX {psrad x, 12}, m0, m1, m2, m3 + jmp tx2q +.pass2: + call .pass2_main + mova xm4, [pw_2048_m2048] + REPX {pmulhrsw x, xm4}, xm0, xm1, xm2, xm3 +.end: + lea r3, [strideq*3] + lea r6, [dstq+strideq*4] + movq xm4, [dstq+strideq*0] + movhps xm4, [dstq+strideq*1] + movq xm5, [dstq+strideq*2] + movhps xm5, [dstq+r3 ] + movq xm6, [r6 +strideq*0] + movhps xm6, [r6 +strideq*1] + movq xm7, [r6 +strideq*2] + movhps xm7, [r6 +r3 ] + paddw xm0, xm4 ; 0 1 + paddw xm1, xm5 ; 2 3 + paddw xm2, xm6 ; 4 5 + paddw xm3, xm7 ; 6 7 + vpbroadcastd xm5, [pixel_10bpc_max] + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + REPX {pmaxsw x, xm4}, xm0, xm1, xm2, xm3 + REPX {pminsw x, xm5}, xm0, xm1, xm2, xm3 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r3 ], xm1 + movq [r6 +strideq*0], xm2 + movhps [r6 +strideq*1], xm2 + movq [r6 +strideq*2], xm3 + movhps [r6 +r3 ], xm3 + RET +ALIGN function_align +.pass2_main: + packssdw m0, m2 + packssdw m1, m3 + lea r6, [deint_shuf+128] + punpcklwd m4, m0, m1 + punpckhwd m0, m1 + punpckhdq m5, m4, m0 + punpckldq m4, m0 + vextracti128 xm2, m4, 1 ; 4 5 + vextracti128 xm3, m5, 1 ; 6 7 + pshufd xm4, xm4, q1032 ; 1 0 + pshufd xm5, xm5, q1032 ; 3 2 + jmp m(iadst_4x8_internal_8bpc).main_pass2 +ALIGN function_align +.main: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] +.main2: + vbroadcasti128 m0, [cq+16*0] + vbroadcasti128 m2, [cq+16*2] + vbroadcasti128 m3, [cq+16*5] + vbroadcasti128 m1, [cq+16*7] + vpbroadcastd m6, [pd_2896] + shufpd m0, m2, 0x0c ; 0 2 + shufpd m1, m3, 0x0c ; 7 5 + vbroadcasti128 m2, [cq+16*4] + vbroadcasti128 m4, [cq+16*6] + vbroadcasti128 m5, [cq+16*1] + vbroadcasti128 m3, [cq+16*3] + vpbroadcastd m7, [pd_2048] + shufpd m2, m4, 0x0c ; 4 6 + shufpd m3, m5, 0x0c ; 3 1 + REPX {pmulld x, m6}, m0, m1, m2, m3 + REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 +.main3: + ITX_MULSUB_2D 1, 0, 4, 5, 6, 7, 401_1931, 4076_3612, 1 + ITX_MULSUB_2D 3, 2, 4, 5, 6, 7, 3166_3920, 2598_1189, 1 + psubd m4, m0, m2 ; t4 t6 + paddd m0, m2 ; t0 t2 + psubd m2, m1, m3 ; t5 t7 + paddd m1, m3 ; t1 t3 + REPX {pmaxsd x, m8}, m4, m2, m0, m1 + REPX {pminsd x, m9}, m4, m2, m0, m1 + pxor m5, m5 + psubd m5, m4 + vpblendd m4, m2, 0xcc ; t4 t7 + vpblendd m2, m5, 0xcc ; t5 -t6 + ITX_MULSUB_2D 4, 2, 3, 5, 6, 7, 1567, 3784 + vpbroadcastd m5, [pd_2896] + vbroadcasti128 m6, [pw_2048_m2048] ; + + - - + punpckhqdq m3, m0, m1 + punpcklqdq m0, m1 + psubd m1, m0, m3 ; t2 t3 + paddd m0, m3 ; out0 -out7 + punpckhqdq m3, m4, m2 ; t7a t6a + punpcklqdq m4, m2 ; t5a t4a + psubd m2, m4, m3 ; t7 t6 + paddd m4, m3 ; out6 -out1 + REPX {pmaxsd x, m8}, m1, m2 + REPX {pminsd x, m9}, m1, m2 + vpblendd m3, m1, m2, 0xcc + shufpd m1, m2, 0x05 + pmulld m3, m5 + pmulld m5, m1 + psignd m0, m6 ; out0 out7 + psignd m4, m6 ; out6 out1 + paddd m3, m7 + psubd m2, m3, m5 + paddd m5, m3 + psrad m2, 12 ; out4 -out5 + psrad m5, 12 ; -out3 out2 + ret + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity + +cglobal iflipadst_4x8_internal_10bpc, 0, 7, 8, dst, stride, c, eob, tx2 + call m(iadst_8x4_internal_10bpc).main + vpbroadcastd m5, [pd_2048] + paddd m0, m5, m3 + paddd m1, m5, m2 + paddd m2, m5, m6 + paddd m3, m5, m4 + jmp m(iadst_4x8_internal_10bpc).pass1_end +.pass2: + call m(iadst_4x8_internal_10bpc).pass2_main + mova xm4, [pw_2048_m2048] + REPX {pmulhrsw x, xm4}, xm3, xm2, xm1, xm0 + lea r3, [strideq*3] + lea r6, [dstq+strideq*4] + movq xm4, [dstq+strideq*1] + movhps xm4, [dstq+strideq*0] + movq xm5, [dstq+r3 ] + movhps xm5, [dstq+strideq*2] + movq xm6, [r6 +strideq*1] + movhps xm6, [r6 +strideq*0] + movq xm7, [r6 +r3 ] + movhps xm7, [r6 +strideq*2] + paddw xm3, xm4 ; 1 0 + paddw xm2, xm5 ; 3 2 + paddw xm1, xm6 ; 5 4 + paddw xm0, xm7 ; 7 6 + vpbroadcastd xm5, [pixel_10bpc_max] + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + REPX {pmaxsw x, xm4}, xm3, xm2, xm1, xm0 + REPX {pminsw x, xm5}, xm3, xm2, xm1, xm0 + movhps [dstq+strideq*0], xm3 + movq [dstq+strideq*1], xm3 + movhps [dstq+strideq*2], xm2 + movq [dstq+r3 ], xm2 + movhps [r6 +strideq*0], xm1 + movq [r6 +strideq*1], xm1 + movhps [r6 +strideq*2], xm0 + movq [r6 +r3 ], xm0 + RET + +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity + +cglobal iidentity_4x8_internal_10bpc, 0, 7, 8, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m3, [pd_2896] + pmulld m0, m3, [cq+32*0] + pmulld m1, m3, [cq+32*1] + pmulld m2, m3, [cq+32*2] + pmulld m3, [cq+32*3] + vpbroadcastd m5, [pd_2048] + vpbroadcastd m4, [pd_5793] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + REPX {pmulld x, m4}, m0, m1, m2, m3 + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + jmp tx2q +.pass2: + vpbroadcastd m6, [pixel_10bpc_max] + call .pass2_end + RET +ALIGN function_align +.pass2_end: + vpbroadcastd m4, [pw_4096] + packssdw m0, m2 + packssdw m1, m3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmulhrsw m2, m4 + pmulhrsw m0, m4 + punpckhdq m1, m0, m2 ; 2 3 6 7 + punpckldq m0, m2 ; 0 1 4 5 + lea r3, [strideq*3] + lea r6, [dstq+strideq*4] + movq xm2, [dstq+strideq*0] + movhps xm2, [dstq+strideq*1] + vpbroadcastq m4, [r6 +strideq*0] + vpbroadcastq m5, [r6 +strideq*1] + movq xm3, [dstq+strideq*2] + movhps xm3, [dstq+r3 ] + vpblendd m2, m4, 0x30 + vpblendd m2, m5, 0xc0 + vpbroadcastq m4, [r6 +strideq*2] + vpbroadcastq m5, [r6 +r3 ] + vpblendd m3, m4, 0x30 + vpblendd m3, m5, 0xc0 + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + paddw m0, m2 ; out0 out1 out4 out5 + paddw m1, m3 ; out2 out3 out6 out7 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m6 + pminsw m1, m6 + vextracti128 xm2, m0, 1 ; out4 out5 + vextracti128 xm3, m1, 1 ; out6 out7 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r3 ], xm1 + movq [r6 +strideq*0], xm2 + movhps [r6 +strideq*1], xm2 + movq [r6 +strideq*2], xm3 + movhps [r6 +r3 ], xm3 + ret + +INV_TXFM_4X8_FN dct, dct, 12 +INV_TXFM_4X8_FN dct, identity, 12 +INV_TXFM_4X8_FN dct, adst, 12 +INV_TXFM_4X8_FN dct, flipadst, 12 + +cglobal idct_4x8_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + jmp m(idct_4x8_internal_10bpc).pass1 +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + ; transpose & interleave + pshufd m0, m0, q1320 + pshufd m1, m1, q1320 + pshufd m2, m2, q1320 + pshufd m3, m3, q1320 + punpckldq m4, m0, m1 + punpckhdq m0, m1 + punpckldq m5, m2, m3 + punpckhdq m2, m3 + vpermq m0, m0, q3102 + vpermq m2, m2, q3102 + vperm2i128 m1, m0, m2, 0x31 ; 1 5 (interleaved) + vperm2i128 m3, m0, m2, 0x20 ; 7 3 (interleaved) + vperm2i128 m0, m4, m5, 0x20 ; 0 2 (interleaved) + vperm2i128 m2, m4, m5, 0x31 ; 4 6 (interleaved) + vpbroadcastd m7, [pd_2048] + call m(idct_8x4_internal_10bpc).main + psubd m3, m0, m4 ; out7 out6 + paddd m0, m4 ; out0 out1 + paddd m1, m2, m5 ; out3 out2 + psubd m2, m5 ; out4 out5 + pshufd m1, m1, q1032 + pshufd m3, m3, q1032 + jmp m(iadst_4x8_internal_12bpc).end + +INV_TXFM_4X8_FN adst, dct, 12 +INV_TXFM_4X8_FN adst, adst, 12 +INV_TXFM_4X8_FN adst, flipadst, 12 +INV_TXFM_4X8_FN adst, identity, 12 + +cglobal iadst_4x8_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + call m(iadst_8x4_internal_10bpc).main + psrad m0, m4, 1 + psrad m1, m6, 1 + psrad m2, 1 + psrad m3, 1 +.pass1_end: + vpbroadcastd m5, [pd_1024] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 11}, m0, m1, m2, m3 + jmp tx2q +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + call .pass2_main + vpblendd m3, m0, m4, 0x33 ; out6 out7 + vpblendd m0, m4, 0xcc ; out0 out1 + pshufd m1, m5, q1032 + psignd m2, m6 ; out4 out5 + psignd m1, m6 ; out2 out3 +.end: + vpbroadcastd m4, [pw_16384] + REPX {psrad x, 3}, m0, m1, m2, m3 + packssdw m0, m2 ; 0 1 4 5 (interleaved) + packssdw m1, m3 ; 2 3 6 7 (interleaved) + mova m2, [iadst8_12_shuf] + vpermd m0, m2, m0 ; 0 1 4 5 + vpermd m1, m2, m1 ; 2 3 6 7 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + lea r3, [strideq*3] + lea r6, [dstq+strideq*4] + movq xm4, [dstq+strideq*0] + movhps xm4, [dstq+strideq*1] + movq xm5, [dstq+strideq*2] + movhps xm5, [dstq+r3 ] + movq xm6, [r6 +strideq*0] + movhps xm6, [r6 +strideq*1] + vinserti128 m4, xm6, 1 + movq xm7, [r6 +strideq*2] + movhps xm7, [r6 +r3 ] + vinserti128 m5, xm7, 1 + paddw m0, m4 ; 0 1 4 5 + paddw m1, m5 ; 2 3 6 7 + vpbroadcastd m5, [pixel_12bpc_max] + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + REPX {pmaxsw x, m4}, m0, m1 + REPX {pminsw x, m5}, m0, m1 + vextracti128 xm2, m0, 1 ; out4 out5 + vextracti128 xm3, m1, 1 ; out6 out7 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movhps [dstq+r3 ], xm1 + movq [r6 +strideq*0], xm2 + movhps [r6 +strideq*1], xm2 + movq [r6 +strideq*2], xm3 + movhps [r6 +r3 ], xm3 + RET +ALIGN function_align +.pass2_main: + ; transpose & interleave + pshufd m0, m0, q1320 + pshufd m1, m1, q1320 + pshufd m2, m2, q1320 + pshufd m3, m3, q1320 + punpckldq m4, m0, m1 + punpckhdq m0, m1 + punpckldq m5, m2, m3 + punpckhdq m2, m3 + vperm2i128 m1, m0, m2, 0x31 ; 7 5 (interleaved) + vperm2i128 m3, m0, m2, 0x20 ; 3 1 (interleaved) + vperm2i128 m0, m4, m5, 0x20 ; 0 2 (interleaved) + vperm2i128 m2, m4, m5, 0x31 ; 4 6 (interleaved) + vpbroadcastd m7, [pd_2048] + jmp m(iadst_4x8_internal_10bpc).main3 + +INV_TXFM_4X8_FN flipadst, dct, 12 +INV_TXFM_4X8_FN flipadst, adst, 12 +INV_TXFM_4X8_FN flipadst, flipadst, 12 +INV_TXFM_4X8_FN flipadst, identity, 12 + +cglobal iflipadst_4x8_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + call m(iadst_8x4_internal_10bpc).main + psrad m0, m3, 1 + psrad m1, m2, 1 + psrad m2, m6, 1 + psrad m3, m4, 1 + jmp m(iadst_4x8_internal_12bpc).pass1_end +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + call m(iadst_4x8_internal_12bpc).pass2_main + shufpd m3, m4, m0, 0x05 ; out1 out0 + shufpd m0, m4, 0x05 ; out7 out6 + psignd m2, m6 + pshufd m6, m6, q1032 + pshufd m1, m2, q1032 ; out5 out4 + psignd m2, m5, m6 ; out3 out2 + jmp m(iadst_4x8_internal_12bpc).end + +INV_TXFM_4X8_FN identity, dct, 12 +INV_TXFM_4X8_FN identity, adst, 12 +INV_TXFM_4X8_FN identity, flipadst, 12 +INV_TXFM_4X8_FN identity, identity, 12 + +cglobal iidentity_4x8_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + jmp m(iidentity_4x8_internal_10bpc).pass1 +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + ; m2 = in4 in5 + ; m3 = in6 in7 + vpbroadcastd m6, [pixel_12bpc_max] + call m(iidentity_4x8_internal_10bpc).pass2_end + RET + +%macro INV_TXFM_4X16_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 4x16, %3 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + vpbroadcastd xm2, [dconly_%3bpc] + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 384 + sar r6d, 9 + jmp m(inv_txfm_add_dct_dct_4x4_10bpc).dconly3 +%endif +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, identity +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst + +cglobal idct_4x16_internal_10bpc, 0, 7, 11, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m10, [pd_3072] + mova m1, [cq+32*2] + mova m3, [cq+32*6] + mova m5, [cq+32*3] + mova m7, [cq+32*7] + call .pass1_main + pmulld m0, m6, [cq+32*0] + pmulld m2, m6, [cq+32*4] + pmulld m4, m6, [cq+32*1] + pmulld m6, [cq+32*5] + call .pass1_main2 + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7 + jmp tx2q +.pass2: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + lea r6, [deint_shuf+128] + punpcklwd m4, m2, m3 + punpckhwd m2, m3 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhdq m1, m0, m4 ; 2 3 + punpckldq m0, m4 ; 0 1 + punpckldq m4, m5, m2 ; 8 9 + punpckhdq m5, m2 ; a b + vextracti128 xm2, m0, 1 ; 4 5 + vextracti128 xm3, m1, 1 ; 6 7 + vextracti128 xm6, m4, 1 ; c d + vextracti128 xm7, m5, 1 ; e f + call m(idct_4x16_internal_8bpc).main + vpbroadcastd m9, [pw_2048] + vinserti128 m0, m0, xm1, 1 ; 0 1 3 2 + vinserti128 m1, m2, xm3, 1 ; 4 5 7 6 + vinserti128 m2, m4, xm5, 1 ; 8 9 b a + vinserti128 m3, m6, xm7, 1 ; c d f e + vpbroadcastd m8, [pixel_10bpc_max] + call .pass2_end + RET +ALIGN function_align +.pass1_main: + vpbroadcastd m4, [pd_3784] + vpbroadcastd m8, [pd_1567] + vpbroadcastd m9, [pd_2048] + vpbroadcastd m6, [pd_1448] + ITX_MULSUB_2D 1, 3, 0, 2, _, 9, 8, 4 ; t2l, t3l + ITX_MULSUB_2D 5, 7, 4, 2, _, 9, 8, 4 ; t2h, t3h + ret +ALIGN function_align +.pass1_main2: + paddd m0, m10 + paddd m4, m10 + paddd m8, m0, m2 + psubd m0, m2 + paddd m9, m4, m6 + psubd m4, m6 + REPX {psrad x, 11}, m8, m0, m9, m4 ; t0l, t1l, t0h, t1h + psubd m2, m0, m1 + paddd m1, m0 + psubd m6, m4, m5 + paddd m5, m4 + paddd m0, m8, m3 + psubd m3, m8, m3 + paddd m4, m9, m7 + psubd m7, m9, m7 + ret +ALIGN function_align +.pass2_end: + lea r6, [strideq*3] + pxor m7, m7 + pmulhrsw m0, m9 + call .write_4x4 + pmulhrsw m0, m1, m9 + call .write_4x4 + pmulhrsw m0, m2, m9 + call .write_4x4 + pmulhrsw m0, m3, m9 + call .write_4x4 + ret +ALIGN function_align +.write_4x4: + movq xm4, [dstq+strideq*0] + movhps xm4, [dstq+strideq*1] + vpbroadcastq m5, [dstq+strideq*2] + vpbroadcastq m6, [dstq+r6 ] + mova [cq+32*0], m7 + mova [cq+32*1], m7 + add cq, 32*2 + vpblendd m4, m5, 0xc0 + vpblendd m4, m6, 0x30 + paddw m4, m0 + pmaxsw m4, m7 + pminsw m4, m8 + vextracti128 xm5, m4, 1 + movq [dstq+strideq*0], xm4 + movhps [dstq+strideq*1], xm4 + movhps [dstq+strideq*2], xm5 + movq [dstq+r6 ], xm5 + lea dstq, [dstq+strideq*4] + ret + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity + +cglobal iadst_4x16_internal_10bpc, 0, 7, 11, dst, stride, c, eob, tx2 + call m(iadst_16x4_internal_10bpc).main + vpbroadcastd m6, [pd_6144] + call m(iadst_16x4_internal_10bpc).main_end + psrad m0, m4, 13 + psrad m1, m5, 13 + psrad m2, 13 + psrad m3, 13 + psrad m4, m8, 13 + psrad m5, m9, 13 + psrad m6, 13 + psrad m7, 13 + jmp tx2q +.pass2: + call .pass2_main + vpbroadcastd m5, [pw_2048] + vpbroadcastd m8, [pixel_10bpc_max] + lea r6, [strideq*3] + vpblendd m4, m3, m0, 0xcc ; -out3 out0 out2 -out1 + pshufd m2, m2, q1032 ; -out11 out8 out10 -out9 + vpblendd m3, m0, 0x33 ; -out15 out12 out14 -out13 + pxor m7, m7 + psubw m9, m7, m5 + vpblendd m9, m5, 0x3c ; -2048 2048 2048 -2048 + pmulhrsw m0, m4, m9 + call .write_4x4 + pmulhrsw m0, m1, m9 + call .write_4x4 + pmulhrsw m0, m2, m9 + call .write_4x4 + pmulhrsw m0, m3, m9 + call .write_4x4 + RET +ALIGN function_align +.write_4x4: + movq xm4, [dstq+r6 ] + movhps xm4, [dstq+strideq*0] + vpbroadcastq m5, [dstq+strideq*1] + vpbroadcastq m6, [dstq+strideq*2] + mova [cq+32*0], m7 + mova [cq+32*1], m7 + add cq, 32*2 + vpblendd m4, m5, 0xc0 + vpblendd m4, m6, 0x30 + paddw m4, m0 + pmaxsw m4, m7 + pminsw m4, m8 + vextracti128 xm5, m4, 1 + movhps [dstq+strideq*0], xm4 + movhps [dstq+strideq*1], xm5 + movq [dstq+strideq*2], xm5 + movq [dstq+r6 ], xm4 + lea dstq, [dstq+strideq*4] + ret +ALIGN function_align +.pass2_main: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + lea r6, [deint_shuf+128] + punpcklwd m4, m2, m3 + punpckhwd m2, m3 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhdq m1, m0, m4 + punpckldq m0, m4 + punpckldq m4, m5, m2 + punpckhdq m5, m2 + vpblendd m3, m0, m1, 0x33 + vpblendd m0, m1, 0xcc + shufpd m2, m5, m4, 0x05 + shufpd m4, m5, 0x05 + vperm2i128 m1, m0, m3, 0x31 ; 4 7 6 5 + vinserti128 m0, xm3, 1 ; 0 3 2 1 + vperm2i128 m3, m2, m4, 0x31 ; c f e d ; ???? + vinserti128 m2, xm4, 1 ; b 8 9 a + call m(iadst_4x16_internal_8bpc).main2 + vpbroadcastd m5, [pw_2896x8] + paddsw m1, m2, m4 + psubsw m2, m4 + pmulhrsw m1, m5 ; -out7 out4 out6 -out5 + pmulhrsw m2, m5 ; out8 -out11 -out9 out10 + ret +ALIGN function_align +.main: + vbroadcasti128 m0, [cq+16* 0] + vbroadcasti128 m4, [cq+16* 2] + vbroadcasti128 m1, [cq+16*15] + vbroadcasti128 m5, [cq+16*13] + vbroadcasti128 m2, [cq+16* 4] + vbroadcasti128 m6, [cq+16* 6] + vbroadcasti128 m3, [cq+16*11] + vbroadcasti128 m7, [cq+16* 9] + shufpd m0, m4, 0x0c ; 0 2 + shufpd m1, m5, 0x0c ; 15 13 + shufpd m2, m6, 0x0c ; 4 6 + shufpd m3, m7, 0x0c ; 11 9 + vbroadcasti128 m4, [cq+16* 8] + vbroadcasti128 m6, [cq+16*10] + vbroadcasti128 m5, [cq+16* 7] + vbroadcasti128 m7, [cq+16* 5] + shufpd m4, m6, 0x0c ; 8 10 + shufpd m5, m7, 0x0c ; 7 5 + vbroadcasti128 m6, [cq+16*12] + vbroadcasti128 m7, [cq+16*14] + shufpd m6, m7, 0x0c ; 12 14 + vbroadcasti128 m7, [cq+16* 3] + vbroadcasti128 m8, [cq+16* 1] + shufpd m7, m8, 0x0c ; 3 1 +.main2: + ; expects: m12 = clip_min m13 = clip_max + vpbroadcastd m11, [pd_2048] + ITX_MULSUB_2D 1, 0, 8, 9, 10, 11, 201_995, 4091_3973, 1 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 11, 1751_2440, 3703_3290, 1 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 11, 3035_3513, 2751_2106, 1 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 11, 3857_4052, 1380_601, 1 + psubd m8, m0, m4 ; t8a t10a + paddd m0, m4 ; t0a t2a + psubd m4, m1, m5 ; t9a t11a + paddd m1, m5 ; t1a t3a + psubd m5, m2, m6 ; t12a t14a + paddd m2, m6 ; t4a t6a + psubd m6, m3, m7 ; t13a t15a + paddd m3, m7 ; t5a t7a + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m8 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m8 + ITX_MULSUB_2D 8, 4, 7, 9, 10, 11, 799_3406, 4017_2276, 1 + ITX_MULSUB_2D 6, 5, 7, 9, 10, 11, 4017_2276, 10, 1 + psubd m7, m0, m2 ; t4 t6 + paddd m0, m2 ; t0 t2 + psubd m2, m1, m3 ; t5 t7 + paddd m1, m3 ; t1 t3 + psubd m3, m4, m6 ; t12a t14a + paddd m4, m6 ; t8a t10a + psubd m6, m8, m5 ; t13a t15a + paddd m8, m5 ; t9a t11a + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m6, m7, m8 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m6, m7, m8 + punpcklqdq m5, m3, m7 ; t12a t4 + punpckhqdq m3, m7 ; t14a t6 + punpckhqdq m7, m6, m2 ; t15a t7 + punpcklqdq m6, m2 ; t13a t5 + ITX_MULSUB_2D 7, 3, 2, 9, 10, 11, 3784, 1567 + ITX_MULSUB_2D 5, 6, 2, 9, 10, 11, 1567, 10 + vpbroadcastd m10, [pd_2896] + vbroadcasti128 m9, [pw_2048_m2048] ; + + - - + punpckhqdq m2, m4, m0 ; t10a t2 + punpcklqdq m4, m0 ; t8a t0 + punpckhqdq m0, m8, m1 ; t11a t3 + punpcklqdq m8, m1 ; t9a t1 + paddd m1, m6, m7 ; out2 -out3 + psubd m6, m7 ; t14a t6 + paddd m7, m5, m3 ; -out13 out12 + psubd m5, m3 ; t15a t7 + psubd m3, m8, m0 ; t11 t3a + paddd m8, m0 ; out14 -out15 + paddd m0, m4, m2 ; -out1 out0 + psubd m4, m2 ; t10 t2a + REPX {pmaxsd x, m12}, m6, m5, m3, m4 + REPX {pminsd x, m13}, m6, m5, m3, m4 + REPX {pmulld x, m10}, m6, m5, m3, m4 + paddd m6, m11 + paddd m4, m11 + paddd m2, m6, m5 ; -out5 out4 + psubd m6, m5 ; out10 -out11 + psubd m5, m4, m3 ; -out9 out8 + paddd m3, m4 ; out6 -out7 + REPX {psrad x, 12}, m2, m3, m5, m6 + REPX {psignd x, m9}, m1, m8, m3, m6 + pshufd m9, m9, q1032 + REPX {psignd x, m9}, m0, m7, m2, m5 + ret + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity + +cglobal iflipadst_4x16_internal_10bpc, 0, 7, 11, dst, stride, c, eob, tx2 +.pass1: + call m(iadst_16x4_internal_10bpc).main + vpbroadcastd m6, [pd_6144] + call m(iadst_16x4_internal_10bpc).main_end + psrad m0, m3, 13 + psrad m1, m2, 13 + psrad m2, m5, 13 + psrad m3, m4, 13 + psrad m4, m7, 13 + psrad m5, m6, 13 + psrad m6, m9, 13 + psrad m7, m8, 13 + jmp tx2q +.pass2: + call m(iadst_4x16_internal_10bpc).pass2_main + vpbroadcastd m5, [pw_2048] + vpbroadcastd m8, [pixel_10bpc_max] + lea r6, [strideq*3] + vpblendd m4, m3, m0, 0x33 ; -out0 out3 out1 -out2 + pshufd m2, m2, q1032 ; -out11 out8 out10 -out9 + vpblendd m3, m0, 0xcc ; -out12 out15 out13 -out14 + pxor m7, m7 + psubw m9, m7, m5 + vpblendd m9, m5, 0x3c ; -2048 2048 2048 -2048 + pmulhrsw m0, m4, m9 + call .write_4x4 + pmulhrsw m0, m2, m9 + call .write_4x4 + pmulhrsw m0, m1, m9 + call .write_4x4 + pmulhrsw m0, m3, m9 + call .write_4x4 + RET +ALIGN function_align +.write_4x4: + movq xm4, [dstq+strideq*0] + movhps xm4, [dstq+r6 ] + vpbroadcastq m5, [dstq+strideq*1] + vpbroadcastq m6, [dstq+strideq*2] + mova [cq+32*0], m7 + mova [cq+32*1], m7 + add cq, 32*2 + vpblendd m4, m5, 0x30 + vpblendd m4, m6, 0xc0 + paddw m4, m0 + pmaxsw m4, m7 + pminsw m4, m8 + vextracti128 xm5, m4, 1 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*1], xm5 + movhps [dstq+strideq*2], xm5 + movhps [dstq+r6 ], xm4 + lea dstq, [dstq+strideq*4] + ret + +INV_TXFM_4X16_FN identity, dct +INV_TXFM_4X16_FN identity, adst +INV_TXFM_4X16_FN identity, flipadst +INV_TXFM_4X16_FN identity, identity + +cglobal iidentity_4x16_internal_10bpc, 0, 7, 11, dst, stride, c, eob, tx2 + vpbroadcastd m7, [pd_5793] + pmulld m0, m7, [cq+32*0] + pmulld m4, m7, [cq+32*1] + pmulld m1, m7, [cq+32*2] + pmulld m5, m7, [cq+32*3] + pmulld m2, m7, [cq+32*4] + pmulld m6, m7, [cq+32*5] + pmulld m3, m7, [cq+32*6] + pmulld m7, [cq+32*7] + vpbroadcastd m8, [pd_6144] + REPX {paddd x, m8}, m0, m4, m1, m5, m2, m6, m3, m7 + REPX {psrad x, 13}, m0, m4, m1, m5, m2, m6, m3, m7 + jmp tx2q +.pass2: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + vpbroadcastd m7, [pw_1697x16] + vpbroadcastd m8, [pw_2048] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + REPX {paddsw x, x}, m0, m1, m2, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + vpbroadcastd m4, [pixel_10bpc_max] + call .pass2_end + RET +ALIGN function_align +.pass2_end: + punpckhwd m7, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + lea r6, [strideq*5] + pxor m3, m3 + punpckhdq m5, m0, m2 ; 2 3 6 7 + punpckldq m0, m2 ; 0 1 4 5 + punpckldq m6, m7, m1 ; 8 9 c d + punpckhdq m7, m1 ; a b e f + pmulhrsw m0, m8 + call .write_2x4x2 + pmulhrsw m0, m5, m8 + call .write_2x4x2 + pmulhrsw m0, m6, m8 + lea dstq, [dstq+strideq*4] + call .write_2x4x2 + pmulhrsw m0, m7, m8 + call .write_2x4x2 + ret +ALIGN function_align +.write_2x4x2: + movq xm1, [dstq+strideq*0] + movhps xm1, [dstq+strideq*1] + vpbroadcastq m2, [dstq+strideq*4] + vpblendd m1, m2, 0x30 + vpbroadcastq m2, [dstq+r6 ] + vpblendd m1, m2, 0xc0 + mova [cq+32*0], m3 + mova [cq+32*1], m3 + add cq, 32*2 + paddw m1, m0 + pmaxsw m1, m3 + pminsw m1, m4 + vextracti128 xm2, m1, 1 + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + movq [dstq+strideq*4], xm2 + movhps [dstq+r6 ], xm2 + lea dstq, [dstq+strideq*2] + ret + +INV_TXFM_4X16_FN dct, dct, 12 +INV_TXFM_4X16_FN dct, identity, 12 +INV_TXFM_4X16_FN dct, adst, 12 +INV_TXFM_4X16_FN dct, flipadst, 12 + +cglobal idct_4x16_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + jmp m(idct_4x16_internal_10bpc).pass1 +.pass2: + punpckldq m8, m0, m1 + punpckhdq m0, m1 + punpckldq m9, m2, m3 + punpckhdq m2, m3 + punpckldq m1, m4, m5 + punpckhdq m4, m5 + punpckldq m3, m6, m7 + punpckhdq m6, m7 + punpcklqdq m5, m0, m2 ; 2 6 + punpckhqdq m12, m0, m2 ; 3 7 + punpcklqdq m0, m8, m9 ; 0 4 + punpckhqdq m10, m8, m9 ; 1 5 + punpcklqdq m2, m1, m3 ; 8 12 + punpckhqdq m13, m1, m3 ; 9 13 + punpcklqdq m9, m4, m6 ; 10 14 + punpckhqdq m4, m6 ; 11 15 + vperm2i128 m1, m5, m9, 0x20 ; 2 10 + vperm2i128 m3, m9, m5, 0x31 ; 14 6 + vpermq m11, m4, q1302 ; 15 11 + ; interleave + REPX {vpermq x, x, q3120}, m0, m1, m2, m3, m10 + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3, m10, m11, m12, m13 + REPX {pminsd x, m9}, m0, m1, m2, m3, m10, m11, m12, m13 + call m(idct_16x4_internal_10bpc).pass1_main + vpermq m6, m12, q1302 ; 7 3 + vpermq m5, m13, q3120 ; 9 13 + call m(idct_16x4_internal_10bpc).pass1_main2 + call m(idct_16x4_internal_10bpc).pass1_main3 + REPX {psrad x, 3}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m1, m2, m3 + packssdw m2, m4, m5 + packssdw m3, m6, m7 + mova m4, [idct16_12_shuf] + REPX {vpermd x, m4, x}, m0, m1, m2, m3 + vpbroadcastd m9, [pw_16384] + vpbroadcastd m8, [pixel_12bpc_max] + call m(idct_4x16_internal_10bpc).pass2_end + RET + +INV_TXFM_4X16_FN adst, dct, 12 +INV_TXFM_4X16_FN adst, adst, 12 +INV_TXFM_4X16_FN adst, flipadst, 12 +INV_TXFM_4X16_FN adst, identity, 12 + +cglobal iadst_4x16_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + call .main_pass1 + psrad m0, m4, 12 + psrad m1, m5, 12 + psrad m2, 12 + psrad m3, 12 + psrad m4, m8, 12 + psrad m5, m9, 12 + psrad m6, 12 + psrad m7, 12 + jmp tx2q +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call .transpose_16x4 + call m(iadst_4x16_internal_10bpc).main2 + pshufd m4, m5, q1032 + psrad m5, m6, 3 + pshufd m6, m7, q1032 + psrad m7, m8, 3 + REPX {pshufd x, x, q1032}, m0, m2 + REPX {psrad x, 3}, m0, m1, m2, m3, m4, m6 +.pass2_end: + packssdw m0, m1 + packssdw m1, m2, m3 + packssdw m2, m4, m5 + packssdw m3, m6, m7 + mova m4, [iadst16_12_shuf] + REPX {vpermd x, m4, x}, m0, m1, m2, m3 + vpbroadcastd m9, [pw_16384] + vpbroadcastd m8, [pixel_12bpc_max] + lea r6, [strideq*3] + pxor m7, m7 + pmulhrsw m0, m9 + call m(iadst_4x16_internal_10bpc).write_4x4 + pmulhrsw m0, m9, m1 + call m(iadst_4x16_internal_10bpc).write_4x4 + pmulhrsw m0, m9, m2 + call m(iadst_4x16_internal_10bpc).write_4x4 + pmulhrsw m0, m9, m3 + call m(iadst_4x16_internal_10bpc).write_4x4 + RET +ALIGN function_align +.transpose_16x4: + ; transpose & interleave + punpckldq m8, m0, m1 + punpckhdq m0, m1 + punpckldq m9, m2, m3 + punpckhdq m2, m3 + punpckldq m1, m4, m5 + punpckhdq m4, m5 + punpckldq m3, m6, m7 + punpckhdq m6, m7 + punpcklqdq m10, m8, m0 + punpckhqdq m0, m8 + punpcklqdq m11, m9, m2 + punpckhqdq m2, m9 + punpcklqdq m8, m1, m4 + punpckhqdq m4, m1 + punpcklqdq m9, m3, m6 + punpckhqdq m6, m3 + vperm2i128 m5, m0, m2, 0x31 ; 7 5 + vperm2i128 m7, m0, m2, 0x20 ; 3 1 + vperm2i128 m0, m10, m11, 0x20 ; 0 2 + vperm2i128 m2, m10, m11, 0x31 ; 4 6 + vperm2i128 m1, m4, m6, 0x31 ; 15 13 + vperm2i128 m3, m4, m6, 0x20 ; 11 9 + vperm2i128 m4, m8, m9, 0x20 ; 8 10 + vperm2i128 m6, m8, m9, 0x31 ; 12 14 + ret +ALIGN function_align +.main_pass1: + call m(iadst_16x4_internal_10bpc).main + vpbroadcastd m6, [pd_3072] + paddd m10, m4, m5 + psubd m4, m3 + psubd m5, m3 + paddd m3, m10 + psubd m8, m7, m1 + paddd m7, m9 + psubd m9, m1 + paddd m7, m1 + REPX {psrad x, 1 }, m4, m5, m2, m3, m8, m9, m0, m7 + REPX {paddd x, m6}, m4, m5, m2, m3, m8, m9, m7 + paddd m6, m0 + ret + +INV_TXFM_4X16_FN flipadst, dct, 12 +INV_TXFM_4X16_FN flipadst, adst, 12 +INV_TXFM_4X16_FN flipadst, flipadst, 12 +INV_TXFM_4X16_FN flipadst, identity, 12 + +cglobal iflipadst_4x16_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + call m(iadst_4x16_internal_12bpc).main_pass1 + psrad m0, m3, 12 + psrad m1, m2, 12 + psrad m2, m5, 12 + psrad m3, m4, 12 + psrad m4, m7, 12 + psrad m5, m6, 12 + psrad m6, m9, 12 + psrad m7, m8, 12 + jmp tx2q +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(iadst_4x16_internal_12bpc).transpose_16x4 + call m(iadst_4x16_internal_10bpc).main2 + pshufd m4, m3, q1032 + psrad m3, m5, 3 + psrad m5, m2, 3 + pshufd m2, m6, q1032 + pshufd m6, m1, q1032 + psrad m1, m7, 3 + psrad m7, m0, 3 + pshufd m0, m8, q1032 + REPX {psrad x, 3}, m0, m2, m4, m6 + jmp m(iadst_4x16_internal_12bpc).pass2_end + +INV_TXFM_4X16_FN identity, dct, 12 +INV_TXFM_4X16_FN identity, adst, 12 +INV_TXFM_4X16_FN identity, flipadst, 12 +INV_TXFM_4X16_FN identity, identity, 12 + +cglobal iidentity_4x16_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m8, [pd_1697] + mova m0, [cq+32*0] + mova m4, [cq+32*1] + mova m1, [cq+32*2] + mova m5, [cq+32*3] + vpbroadcastd m9, [pd_6144] + pmulld m2, m8, m0 + pmulld m6, m8, m4 + pmulld m3, m8, m1 + pmulld m7, m8, m5 + mova m10, [cq+32*4] + mova m11, [cq+32*5] + mova m12, [cq+32*6] + mova m13, [cq+32*7] + REPX {paddd x, m9}, m2, m6, m3, m7 + REPX {psrad x, 12}, m2, m6, m3, m7 + paddd m0, m2 + pmulld m2, m8, m10 + paddd m4, m6 + pmulld m6, m8, m11 + paddd m1, m3 + pmulld m3, m8, m12 + paddd m5, m7 + pmulld m7, m8, m13 + REPX {psrad x, 1 }, m0, m4, m1, m5 + REPX {paddd x, m9}, m2, m6, m3, m7 + REPX {psrad x, 12}, m2, m6, m3, m7 + paddd m2, m10 + paddd m6, m11 + paddd m3, m12 + paddd m7, m13 + REPX {psrad x, 1 }, m2, m6, m3, m7 + jmp tx2q +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m8, [pd_5793] + vpbroadcastd m9, [pd_1024] + REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddd x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + vpbroadcastd m8, [pw_16384] + vpbroadcastd m4, [pixel_12bpc_max] + call m(iidentity_4x16_internal_10bpc).pass2_end + RET + +%macro INV_TXFM_8X4_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 8x4, %3 +%ifidn %1_%2, dct_dct + vpbroadcastd m2, [dconly_%3bpc] +%if %3 = 10 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 4 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 128 + sar r6d, 8 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 +%else + jmp m(inv_txfm_add_dct_dct_8x4_10bpc).dconly +%endif +%endif +%endmacro + +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, identity +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst + +cglobal idct_8x4_internal_10bpc, 0, 7, 10, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] +.pass1: + vbroadcasti128 m1, [cq+16*1] + vbroadcasti128 m0, [cq+16*5] + vbroadcasti128 m2, [cq+16*3] + vbroadcasti128 m3, [cq+16*7] + vpbroadcastd m6, [pd_2896] + shufpd m1, m0, 0x0c ; 1 5 + shufpd m3, m2, 0x0c ; 7 3 + vbroadcasti128 m0, [cq+16*0] + vbroadcasti128 m4, [cq+16*2] + vbroadcasti128 m2, [cq+16*4] + vbroadcasti128 m5, [cq+16*6] + vpbroadcastd m7, [pd_2048] + shufpd m0, m4, 0x0c ; 0 2 + shufpd m2, m5, 0x0c ; 4 6 + REPX {pmulld x, m6}, m1, m3, m0, m2 + REPX {paddd x, m7}, m1, m3, m0, m2 + REPX {psrad x, 12}, m1, m3, m0, m2 + call .main + psubd m3, m0, m4 ; out7 out6 (interleaved) + paddd m0, m4 ; out0 out1 (interleaved) + paddd m1, m2, m5 ; out3 out2 (interleaved) + psubd m2, m5 ; out4 out5 (interleaved) + pshufd m1, m1, q1032 + pshufd m3, m3, q1032 + jmp tx2q +.pass2: + vbroadcasti128 m4, [deint_shuf] + packssdw m0, m1 + packssdw m2, m3 + vperm2i128 m1, m0, m2, 0x31 + vinserti128 m0, xm2, 1 + pshufb m0, m4 + pshufb m1, m4 + IDCT4_1D_PACKED_WORD 0, 1, 2, 3, 4, 7 + vpermq m0, m0, q3120 ; out0 out1 + vpermq m2, m1, q2031 ; out2 out3 + jmp m(iadst_8x4_internal_10bpc).end +ALIGN function_align +.main: + ITX_MULSUB_2D 1, 3, 4, 5, 6, 7, 799_3406, 4017_2276, 1 + IDCT4_1D_PACKED 0, 2, 4, 5, 6, 7 + vpbroadcastd m6, [pd_2896] + punpcklqdq m4, m1, m3 ; t4a t7a + punpckhqdq m1, m3 ; t5a t6a + psubd m3, m4, m1 ; t5a t6a + paddd m4, m1 ; t4 t7 + REPX {pmaxsd x, m8}, m3, m4, m0, m2 + REPX {pminsd x, m9}, m3, m4, m0, m2 + pmulld m3, m6 + pshufd m1, m3, q1032 + paddd m3, m7 + psubd m5, m3, m1 + paddd m1, m3 + psrad m5, 12 + psrad m1, 12 + vpblendd m5, m4, 0x33 ; t4 t5 + punpckhqdq m4, m1 ; t7 t6 + ret + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_10bpc, 0, 7, 10, dst, stride, c, eob, tx2 + call m(iadst_4x8_internal_10bpc).main + vpblendd m3, m0, m4, 0x33 ; out6 out7 + vpblendd m0, m4, 0xcc ; out0 out1 + pshufd m1, m5, q1032 + psignd m2, m6 ; out4 out5 + psignd m1, m6 ; out2 out3 + jmp tx2q +.pass2: + call .pass2_main + vpermq m0, m0, q3120 ; out0 out1 + vpermq m2, m1, q3120 ; out2 out3 +.end: + vpbroadcastd m1, [pw_2048] + pmulhrsw m0, m1 + pmulhrsw m1, m2 + vpbroadcastd m5, [pixel_10bpc_max] +.end2: + mova xm2, [dstq+strideq*0] + vinserti128 m2, [dstq+strideq*1], 1 + lea r6, [dstq+strideq*2] + mova xm3, [r6 +strideq*0] + vinserti128 m3, [r6 +strideq*1], 1 + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [r6 +strideq*0], xm1 + vextracti128 [r6 +strideq*1], m1, 1 + RET +ALIGN function_align +.pass2_main: + vbroadcasti128 m4, [deint_shuf] + packssdw m0, m1 + packssdw m2, m3 + lea r6, [deint_shuf+128] + vperm2i128 m1, m0, m2, 0x31 + vinserti128 m0, xm2, 1 + pshufb m0, m4 + pshufb m1, m4 + jmp m(iadst_8x4_internal_8bpc).main +ALIGN function_align +.main: + vpbroadcastd m1, [pd_2896] + pmulld m0, m1, [cq+32*0] + pmulld m3, m1, [cq+32*3] + pmulld m2, m1, [cq+32*2] + pmulld m1, [cq+32*1] + vpbroadcastd m4, [pd_2048] + REPX {paddd x, m4}, m0, m3, m2, m1 + REPX {psrad x, 12}, m0, m3, m2, m1 +.main2: + IADST4_1D + ret + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_10bpc, 0, 5, 10, dst, stride, c, eob, tx2 + call m(iadst_4x8_internal_10bpc).main + shufpd m3, m4, m0, 0x05 + shufpd m0, m4, 0x05 + psignd m2, m6 + pshufd m6, m6, q1032 + pshufd m1, m2, q1032 + psignd m2, m5, m6 + jmp tx2q +.pass2: + call m(iadst_8x4_internal_10bpc).pass2_main + vpermq m2, m0, q2031 + vpermq m0, m1, q2031 + jmp m(iadst_8x4_internal_10bpc).end + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_10bpc, 0, 7, 10, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m4, [pd_2896] + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpermq m2, [cq+32*2], q3120 + vpermq m3, [cq+32*3], q3120 + vpbroadcastd m7, [pd_2048] + REPX {pmulld x, m4}, m0, m1, m2, m3 + REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + REPX {paddd x, x }, m0, m1, m2, m3 + jmp tx2q +.pass2: + vpbroadcastd m5, [pixel_10bpc_max] + vpbroadcastd m4, [pw_1697x8] + packssdw m0, m1 + packssdw m2, m3 + pmulhrsw m1, m4, m0 + pmulhrsw m4, m2 + paddsw m0, m1 + paddsw m2, m4 + packssdw m7, m7 ; pw_2048 +.pass2_end: + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + lea r6, [dstq+strideq*2] + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmulhrsw m2, m7 + pmulhrsw m0, m7 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + mova xm2, [dstq+strideq*0] + vinserti128 m2, [r6 +strideq*0], 1 + mova xm3, [dstq+strideq*1] + vinserti128 m3, [r6 +strideq*1], 1 + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + vextracti128 [r6 +strideq*0], m0, 1 + vextracti128 [r6 +strideq*1], m1, 1 + RET + +INV_TXFM_8X4_FN dct, dct, 12 +INV_TXFM_8X4_FN dct, identity, 12 +INV_TXFM_8X4_FN dct, adst, 12 +INV_TXFM_8X4_FN dct, flipadst, 12 + +cglobal idct_8x4_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_20b_min] + vpbroadcastd m9, [clip_20b_max] + jmp m(idct_8x4_internal_10bpc).pass1 +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + call m(iadst_8x4_internal_12bpc).transpose_4x8 + IDCT4_1D 0, 1, 2, 3, 4, 5, 6, 7 + jmp m(iadst_8x4_internal_12bpc).end + +INV_TXFM_8X4_FN adst, dct, 12 +INV_TXFM_8X4_FN adst, adst, 12 +INV_TXFM_8X4_FN adst, flipadst, 12 +INV_TXFM_8X4_FN adst, identity, 12 + +cglobal iadst_8x4_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_20b_min] + vpbroadcastd m9, [clip_20b_max] + call m(iadst_4x8_internal_10bpc).main2 + vpblendd m3, m0, m4, 0x33 ; out6 out7 + vpblendd m0, m4, 0xcc ; out0 out1 + pshufd m1, m5, q1032 + psignd m2, m6 ; out4 out5 + psignd m1, m6 ; out2 out3 + jmp tx2q +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + call .pass2_main + vpbroadcastd m5, [pd_2048] + paddd m0, m5, m4 + paddd m1, m5, m6 + paddd m2, m5 + paddd m3, m5 +.pass2_end: + REPX {psrad x, 12}, m0, m1, m2, m3 +.end: + vpbroadcastd m4, [pw_16384] + REPX {psrad x, 3}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + pmulhrsw m0, m4 + pmulhrsw m1, m2, m4 + vpermq m0, m0, q3120 ; out0 out1 + vpermq m1, m1, q3120 ; out2 out3 + vpbroadcastd m5, [pixel_12bpc_max] + jmp m(iadst_8x4_internal_10bpc).end2 +ALIGN function_align +.pass2_main: + call .transpose_4x8 + jmp m(iadst_8x4_internal_10bpc).main2 +ALIGN function_align +.transpose_4x8: + ; deinterleave + pshufd m0, m0, q3120 + pshufd m1, m1, q3120 + pshufd m2, m2, q3120 + pshufd m3, m3, q3120 + ; transpose + punpcklqdq m4, m0, m1 + punpckhqdq m0, m1 + punpcklqdq m5, m2, m3 + punpckhqdq m2, m3 + vperm2i128 m1, m0, m2, 0x20 ; out1 + vperm2i128 m3, m0, m2, 0x31 ; out3 + vperm2i128 m2, m4, m5, 0x31 ; out2 + vperm2i128 m0, m4, m5, 0x20 ; out0 + ret + +INV_TXFM_8X4_FN flipadst, dct, 12 +INV_TXFM_8X4_FN flipadst, adst, 12 +INV_TXFM_8X4_FN flipadst, flipadst, 12 +INV_TXFM_8X4_FN flipadst, identity, 12 + +cglobal iflipadst_8x4_internal_12bpc, 0, 5, 10, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_20b_min] + vpbroadcastd m9, [clip_20b_max] + call m(iadst_4x8_internal_10bpc).main2 + shufpd m3, m4, m0, 0x05 + shufpd m0, m4, 0x05 + psignd m2, m6 + pshufd m6, m6, q1032 + pshufd m1, m2, q1032 + psignd m2, m5, m6 + jmp tx2q +.pass2: + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + call m(iadst_8x4_internal_12bpc).pass2_main + vpbroadcastd m5, [pd_2048] + paddd m0, m5, m3 + paddd m1, m5, m2 + paddd m3, m5, m4 + paddd m2, m5, m6 + jmp m(iadst_8x4_internal_12bpc).pass2_end + +INV_TXFM_8X4_FN identity, dct, 12 +INV_TXFM_8X4_FN identity, adst, 12 +INV_TXFM_8X4_FN identity, flipadst, 12 +INV_TXFM_8X4_FN identity, identity, 12 + +cglobal iidentity_8x4_internal_12bpc, 0, 7, 10, dst, stride, c, eob, tx2 + jmp m(iidentity_8x4_internal_10bpc).pass1 +.pass2: + ; m0 = in0 in1 (interleaved) + ; m1 = in2 in3 (interleaved) + ; m2 = in4 in5 (interleaved) + ; m3 = in6 in7 (interleaved) + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] + REPX {pmaxsd x, m8}, m0, m1, m2, m3 + REPX {pminsd x, m9}, m0, m1, m2, m3 + vpbroadcastd m4, [pd_5793] + REPX {pmulld x, m4}, m0, m1, m2, m3 + REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {psrad x, 15}, m0, m1, m2, m3 + vpbroadcastd m5, [pixel_12bpc_max] + vpbroadcastd m7, [pw_16384] + packssdw m0, m1 + packssdw m2, m3 + jmp m(iidentity_8x4_internal_10bpc).pass2_end + +%macro INV_TXFM_8X8_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 8x8, %3 +%ifidn %1_%2, dct_dct + vpbroadcastd m2, [dconly_%3bpc] +%if %3 = 10 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 8 +.dconly2: + add r6d, 384 + sar r6d, 9 +.dconly3: + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + movd xm0, r6d + paddsw xm0, xm2 + vpbroadcastw m0, xm0 +.dconly_loop: + mova xm1, [dstq+strideq*0] + vinserti128 m1, [dstq+strideq*1], 1 + paddsw m1, m0 + psubusw m1, m2 + mova [dstq+strideq*0], xm1 + vextracti128 [dstq+strideq*1], m1, 1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%else + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly +%endif +%endif +%endmacro + +%macro IADST8_1D 14 ; src[1-8], tmp[1-3], pd_2048, clip[1-2] + ITX_MULSUB_2D %8, %1, %9, %10, %11, %12, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D %2, %7, %9, %10, %11, %12, 3920, 1189 ; t7a, t6a + ITX_MULSUB_2D %6, %3, %9, %10, %11, %12, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D %4, %5, %9, %10, %11, %12, 3166, 2598 ; t5a, t4a + psubd m%9, m%3, m%7 ; t6 + paddd m%3, m%7 ; t2 + psubd m%7, m%1, m%5 ; t4 + paddd m%1, m%5 ; t0 + psubd m%5, m%6, m%2 ; t7 + paddd m%6, m%2 ; t3 + psubd m%2, m%8, m%4 ; t5 + paddd m%8, m%4 ; t1 + REPX {pmaxsd x, m%13}, m%7, m%2, m%9, m%5, m%3, m%1, m%6, m%8 + REPX {pminsd x, m%14}, m%7, m%2, m%9, m%5, m%3, m%1, m%6, m%8 + ITX_MULSUB_2D %7, %2, %4, %10, %11, %12, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2D %5, %9, %4, %10, %11, %12, 3784, %11 ; t6a, t7a + psubd m%10, m%7, m%9 ; t7 + paddd m%7, m%9 ; out6 + vpbroadcastd m%9, [pd_1448] + psubd m%4, m%8, m%6 ; t3 + paddd m%8, m%6 ; -out7 + psubd m%6, m%1, m%3 ; t2 + paddd m%1, m%3 ; out0 + psubd m%3, m%2, m%5 ; t6 + paddd m%2, m%5 ; -out1 + REPX {pmaxsd x, m%13}, m%6, m%4, m%3, m%10 + REPX {pminsd x, m%14}, m%6, m%4, m%3, m%10 + REPX {pmulld x, m%9 }, m%6, m%4, m%3, m%10 + psubd m%5, m%6, m%4 ; (t2 - t3) * 1448 + paddd m%4, m%6 ; (t2 + t3) * 1448 + psubd m%6, m%3, m%10 ; (t6 - t7) * 1448 + paddd m%3, m%10 ; (t6 + t7) * 1448 +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, identity +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst + +cglobal idct_8x8_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + mova m4, [cq+32*4] + mova m5, [cq+32*5] + mova m6, [cq+32*6] + mova m7, [cq+32*7] + vpbroadcastd m11, [pd_2048] + call .main + call .round_shift1 + jmp tx2q +.pass2: + call .transpose_8x8_packed + call m(idct_8x8_internal_8bpc).main + vpbroadcastd m12, [pw_2048] + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call .write_8x4_start + pmulhrsw m0, m2, m12 + pmulhrsw m1, m3, m12 + call .write_8x4 + RET +ALIGN function_align +.write_8x4_start: + vpbroadcastd m11, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m10, m10 +.write_8x4: + mova xm8, [dstq+strideq*0] + vinserti128 m8, [dstq+strideq*1], 1 + mova xm9, [dstq+strideq*2] + vinserti128 m9, [dstq+r6 ], 1 + mova [cq+32*0], m10 + mova [cq+32*1], m10 + mova [cq+32*2], m10 + mova [cq+32*3], m10 + add cq, 32*4 + paddw m0, m8 + paddw m1, m9 + pmaxsw m0, m10 + pmaxsw m1, m10 + pminsw m0, m11 + pminsw m1, m11 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+r6 ], m1, 1 + lea dstq, [dstq+strideq*4] + ret +ALIGN function_align +.transpose_8x8_packed: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + lea r6, [deint_shuf+128] + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckhdq m2, m4, m1 + punpckldq m4, m1 + vinserti128 m1, m3, xm2, 1 + vperm2i128 m3, m2, 0x31 + vperm2i128 m2, m0, m4, 0x31 + vinserti128 m0, xm4, 1 + ret +ALIGN function_align +.main_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.main: + ITX_MULSUB_2D 5, 3, 8, 9, 10, 11, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 8, 9, 10, 11, 799, 4017 ; t4a t7a + ITX_MULSUB_2D 2, 6, 8, 9, 10, 11, 1567, 3784 ; t2 t3 + paddd m8, m1, m5 ; t4 + psubd m1, m5 ; t5a + paddd m9, m7, m3 ; t7 + psubd m7, m3 ; t6a + vpbroadcastd m3, [pd_2896] + REPX {pmaxsd x, m12}, m1, m8, m7, m9 + REPX {pminsd x, m13}, m1, m8, m7, m9 + REPX {pmulld x, m3 }, m0, m4, m7, m1 + paddd m0, m11 + paddd m7, m11 + psubd m5, m0, m4 + paddd m0, m4 + psubd m4, m7, m1 + paddd m7, m1 + REPX {psrad x, 12 }, m5, m0, m4, m7 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + paddd m6, m5, m2 ; dct4 out1 + psubd m5, m2 ; dct4 out2 + REPX {pmaxsd x, m12}, m0, m6, m5, m3 + REPX {pminsd x, m13}, m0, m6, m5, m3 + ret +ALIGN function_align +.round_shift1: + pcmpeqd m1, m1 + REPX {psubd x, m1}, m0, m6, m5, m3 + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + psubd m7, m0, m9 ; out7 + paddd m0, m9 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 + REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity + +cglobal iadst_8x8_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + call .main + call .main_end + jmp tx2q +.pass2: + call m(idct_8x8_internal_10bpc).transpose_8x8_packed + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call m(iadst_8x8_internal_8bpc).main_pass2 + vpbroadcastd m5, [pw_2048] + vpbroadcastd xm12, [pw_4096] + psubw m12, m5 + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m0, m2, m12 + pmulhrsw m1, m3, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +ALIGN function_align +.main: + mova m0, [cq+32*0] + mova m7, [cq+32*7] + mova m1, [cq+32*1] + mova m6, [cq+32*6] + mova m2, [cq+32*2] + mova m5, [cq+32*5] + mova m3, [cq+32*3] + mova m4, [cq+32*4] + vpbroadcastd m11, [pd_2048] +.main2: + IADST8_1D 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 + psrld m8, 10 ; pd_1 + vpbroadcastd m9, [pd_3072] + ret +ALIGN function_align +.main_end: + paddd m0, m8 + psubd m1, m8, m1 + paddd m6, m8 + psubd m7, m8, m7 + REPX {psrad x, 1 }, m0, m1, m6, m7 + ; (1 + ((x + 1024) >> 11)) >> 1 = (3072 + x) >> 12 + ; (1 - ((x + 1024) >> 11)) >> 1 = (3071 - x) >> 12 + psubd m8, m9, m8 ; pd_3071 + paddd m2, m9 + psubd m3, m8, m3 + paddd m4, m9 + psubd m5, m8, m5 + REPX {psrad x, 12}, m2, m3, m4, m5 + ret + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity + +cglobal iflipadst_8x8_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + call m(iadst_8x8_internal_10bpc).main + call .main_end + jmp tx2q +.pass2: + call m(idct_8x8_internal_10bpc).transpose_8x8_packed + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call m(iadst_8x8_internal_8bpc).main_pass2 + vpbroadcastd m12, [pw_2048] + vpbroadcastd xm5, [pw_4096] + psubw m12, m5 + vpermq m8, m3, q2031 + vpermq m9, m2, q2031 + vpermq m2, m1, q2031 + vpermq m3, m0, q2031 + pmulhrsw m0, m8, m12 + pmulhrsw m1, m9, m12 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m0, m2, m12 + pmulhrsw m1, m3, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +ALIGN function_align +.main_end: + paddd m10, m8, m0 + psubd m0, m8, m7 + psubd m7, m8, m1 + paddd m1, m8, m6 + psrad m0, 1 + psrad m1, 1 + psrad m6, m7, 1 + psrad m7, m10, 1 + psubd m8, m9, m8 ; pd_6143 + psubd m10, m8, m5 + paddd m5, m9, m2 + psubd m2, m8, m3 + paddd m3, m9, m4 + psrad m4, m2, 12 + psrad m2, m10, 12 + psrad m3, 12 + psrad m5, 12 + ret + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 +.pass1: + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + mova m4, [cq+32*4] + mova m5, [cq+32*5] + mova m6, [cq+32*6] + mova m7, [cq+32*7] + jmp tx2q +.pass2: + packssdw m3, m7 + vpbroadcastd m7, [pixel_10bpc_max] +.pass2_main: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + vpbroadcastd m12, [pw_4096] + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m1 + punpckhdq m4, m1 + punpckhqdq m1, m0, m2 ; 1 5 + punpcklqdq m0, m2 ; 0 4 + punpcklqdq m2, m3, m4 ; 2 6 + punpckhqdq m3, m4 ; 3 7 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call .write_2x8x2_start + pmulhrsw m0, m2, m12 + pmulhrsw m1, m3, m12 + call .write_2x8x2_zero + RET +.write_2x8x2_start: + lea r6, [strideq*5] + pxor m6, m6 +.write_2x8x2_zero: + mova [cq+32*0], m6 + mova [cq+32*1], m6 + mova [cq+32*2], m6 + mova [cq+32*3], m6 + add cq, 32*4 +.write_2x8x2: + mova xm4, [dstq+strideq*0] + vinserti128 m4, [dstq+strideq*4], 1 + mova xm5, [dstq+strideq*1] + vinserti128 m5, [dstq+r6 ], 1 + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m6 + pmaxsw m1, m6 + pminsw m0, m7 + pminsw m1, m7 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + vextracti128 [dstq+strideq*4], m0, 1 + vextracti128 [dstq+r6 ], m1, 1 + lea dstq, [dstq+strideq*2] + ret + +%macro TRANSPOSE_8X8_DWORD 12 ; src/dst[1-8], tmp[1-4] + punpckldq m%9, m%1, m%2 ; aibj emfn + punpckhdq m%1, m%2 ; ckdl gohp + punpckldq m%10, m%3, m%4 ; qyrz uCvD + punpckhdq m%3, m%4 ; sAtB wExF + punpckldq m%11, m%5, m%6 ; GOHP KSLT + punpckhdq m%5, m%6 ; IQJR MUNV + punpckldq m%12, m%7, m%8 ; WeXf aibj + punpckhdq m%7, m%8 ; YgZh ckdl + punpcklqdq m%2, m%9, m%10 ; aiqy emuC + punpckhqdq m%9, m%10 ; bjrz fnvD + punpcklqdq m%4, m%1, m%3 ; cksA gowE + punpckhqdq m%10, m%1, m%3 ; dltB hpxF + punpcklqdq m%6, m%11, m%12 ; GOWe KSai + punpckhqdq m%11, m%12 ; HPXf LTbj + punpcklqdq m%8, m%5, m%7 ; IQYg MUck + punpckhqdq m%12, m%5, m%7 ; JRZh NVdl + vperm2i128 m%1, m%2, m%6, 0x20 ; out0 + vperm2i128 m%5, m%2, m%6, 0x31 ; out4 + vperm2i128 m%2, m%9, m%11, 0x20 ; out1 + vperm2i128 m%6, m%9, m%11, 0x31 ; out5 + vperm2i128 m%3, m%4, m%8, 0x20 ; out2 + vperm2i128 m%7, m%4, m%8, 0x31 ; out6 + vperm2i128 m%4, m%10, m%12, 0x20 ; out3 + vperm2i128 m%8, m%10, m%12, 0x31 ; out7 +%endmacro + +INV_TXFM_8X8_FN dct, dct, 12 +INV_TXFM_8X8_FN dct, identity, 12 +INV_TXFM_8X8_FN dct, adst, 12 +INV_TXFM_8X8_FN dct, flipadst, 12 + +cglobal idct_8x8_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(idct_8x8_internal_10bpc).pass1 +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call .transpose_8x8 + vpbroadcastd m11, [pd_2048] + call m(idct_8x8_internal_10bpc).main + call .round_shift4 + jmp m(iadst_8x8_internal_12bpc).pass2_end +ALIGN function_align +.write_8x4_start: + vpbroadcastd m11, [pixel_12bpc_max] + lea r6, [strideq*3] + pxor m10, m10 + ret +ALIGN function_align +.transpose_8x8: + TRANSPOSE_8X8_DWORD 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + ret +ALIGN function_align +.round_shift4: + vpbroadcastd m1, [pd_8] + REPX {paddd x, m1}, m0, m6, m5, m3 + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + psubd m7, m0, m9 ; out7 + paddd m0, m9 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 + REPX {psrad x, 4}, m0, m1, m2, m3, m4, m5, m6, m7 + ret + +INV_TXFM_8X8_FN adst, dct, 12 +INV_TXFM_8X8_FN adst, adst, 12 +INV_TXFM_8X8_FN adst, flipadst, 12 +INV_TXFM_8X8_FN adst, identity, 12 + +cglobal iadst_8x8_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iadst_8x8_internal_10bpc).pass1 +.pass2: + call .pass2_main +.pass2_end: + packssdw m0, m1 + packssdw m1, m2, m3 + REPX {vpermq x, x, q3120}, m0, m1 + call m(idct_8x8_internal_12bpc).write_8x4_start + call m(idct_8x8_internal_10bpc).write_8x4 + packssdw m0, m4, m5 + packssdw m1, m6, m7 + REPX {vpermq x, x, q3120}, m0, m1 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +ALIGN function_align +.pass2_main: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_12bpc).transpose_8x8 + vpbroadcastd m11, [pd_2048] +.pass2_main2: + call m(iadst_8x8_internal_10bpc).main2 + pslld m9, m8, 3 ; pd_8 + paddd m0, m9 + psubd m1, m9, m1 ; 8+x + paddd m6, m9 + psubd m7, m9, m7 + REPX {psrad x, 4}, m0, m1, m6, m7 + vpbroadcastd m9, [pd_17408] + psubd m8, m9, m8 ; 17407 + paddd m2, m9 + psubd m3, m8, m3 + paddd m4, m9 + psubd m5, m8, m5 + REPX {psrad x, 15}, m2, m3, m4, m5 + ret + +INV_TXFM_8X8_FN flipadst, dct, 12 +INV_TXFM_8X8_FN flipadst, adst, 12 +INV_TXFM_8X8_FN flipadst, flipadst, 12 +INV_TXFM_8X8_FN flipadst, identity, 12 + +cglobal iflipadst_8x8_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iflipadst_8x8_internal_10bpc).pass1 +.pass2: + call m(iadst_8x8_internal_12bpc).pass2_main + packssdw m7, m7, m6 + packssdw m6, m1, m0 + packssdw m1, m5, m4 + vpermq m0, m7, q3120 + vpermq m1, m1, q3120 + call m(idct_8x8_internal_12bpc).write_8x4_start + call m(idct_8x8_internal_10bpc).write_8x4 + packssdw m0, m3, m2 + vpermq m0, m0, q3120 + vpermq m1, m6, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + RET + +INV_TXFM_8X8_FN identity, dct, 12 +INV_TXFM_8X8_FN identity, adst, 12 +INV_TXFM_8X8_FN identity, flipadst, 12 +INV_TXFM_8X8_FN identity, identity, 12 + +cglobal iidentity_8x8_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + jmp m(iidentity_8x8_internal_10bpc).pass1 +.pass2: + packssdw m3, m7 + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(iidentity_8x8_internal_10bpc).pass2_main + +%macro INV_TXFM_8X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth + INV_TXFM_FN %1, %2, %3, 8x16, %4 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_%4bpc] + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 +%endif +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, identity, 35 +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst + +cglobal idct_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + vpbroadcastd m14, [pd_2896] + vpbroadcastd m11, [pd_2048] + cmp eobd, 43 + jl .fast + add cq, 32 + call .pass1_main + sub cq, 32 + mova [cq+32* 1], m0 + mova [cq+32* 3], m1 + mova [cq+32* 5], m2 + mova [cq+32* 7], m3 + mova [cq+32* 9], m4 + mova [cq+32*11], m5 + mova [cq+32*13], m6 + mova m15, m7 + call .pass1_main + mova m8, [cq+32* 1] + mova m9, [cq+32* 3] + mova m10, [cq+32* 5] + mova m11, [cq+32* 7] + mova m12, [cq+32* 9] + mova m13, [cq+32*11] + mova m14, [cq+32*13] + jmp tx2q +.fast: + call .pass1_main + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call .transpose + call m(idct_8x16_internal_8bpc).main + vpbroadcastd m12, [pw_2048] + REPX {vpermq x, x, q3120}, m0, m2, m4, m6 + REPX {vpermq x, x, q2031}, m1, m3, m5, m7 +.end: + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m0, m2, m12 + pmulhrsw m1, m3, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m0, m4, m12 + pmulhrsw m1, m5, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m0, m6, m12 + pmulhrsw m1, m7, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +ALIGN function_align +.transpose: + packssdw m0, m8 + packssdw m1, m9 + packssdw m2, m10 + packssdw m3, m11 + packssdw m4, m12 + packssdw m5, m13 + packssdw m6, m14 + packssdw m7, m15 + lea r6, [deint_shuf+128] + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpcklwd m3, m4, m5 + punpckhwd m4, m5 + punpckhwd m5, m6, m7 + punpcklwd m6, m7 + punpckhdq m7, m3, m6 + punpckldq m3, m6 + punpckhdq m6, m4, m5 + punpckldq m4, m5 + punpckhdq m5, m8, m1 + punpckldq m8, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + vperm2i128 m2, m0, m3, 0x31 + vinserti128 m0, xm3, 1 + vperm2i128 m3, m1, m7, 0x31 + vinserti128 m1, xm7, 1 + vperm2i128 m7, m5, m6, 0x31 + vinserti128 m5, xm6, 1 + vperm2i128 m6, m8, m4, 0x31 + vinserti128 m4, m8, xm4, 1 + ret +ALIGN function_align +.pass1_main: + pmulld m0, m14, [cq+32* 0] + pmulld m1, m14, [cq+32* 2] + pmulld m2, m14, [cq+32* 4] + pmulld m3, m14, [cq+32* 6] + pmulld m4, m14, [cq+32* 8] + pmulld m5, m14, [cq+32*10] + pmulld m6, m14, [cq+32*12] + pmulld m7, m14, [cq+32*14] + call m(idct_8x8_internal_10bpc).main_rect2 + jmp m(idct_8x8_internal_10bpc).round_shift1 +ALIGN function_align +.main_evenhalf: + paddd m1, m6, m7 ; idct8 out1 + psubd m6, m7 ; idct8 out6 + psubd m7, m0, m9 ; idct8 out7 + paddd m0, m9 ; idct8 out0 + paddd m2, m5, m4 ; idct8 out2 + psubd m5, m4 ; idct8 out5 + psubd m4, m3, m8 ; idct8 out4 + paddd m3, m8 ; idct8 out3 + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + ret +.main_oddhalf_fast_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_oddhalf_fast: ; lower half zero + vpbroadcastd m7, [pd_4076] + vpbroadcastd m8, [pd_401] + vpbroadcastd m6, [pd_m1189] + vpbroadcastd m9, [pd_3920] + vpbroadcastd m5, [pd_3612] + vpbroadcastd m10, [pd_1931] + vpbroadcastd m4, [pd_m2598] + vpbroadcastd m15, [pd_3166] + pmulld m7, m0 + pmulld m0, m8 + pmulld m6, m1 + pmulld m1, m9 + pmulld m5, m2 + pmulld m2, m10 + pmulld m4, m3 + pmulld m3, m15 + jmp .main_oddhalf_fast2 +.main_oddhalf_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.main_oddhalf: + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3920, 1189 ; t11a, t12a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3166, 2598 ; t9a, t14a +.main_oddhalf_fast2: + REPX {paddd x, m11}, m0, m7, m6, m1, m2, m5, m4, m3 + REPX {psrad x, 12 }, m0, m4, m6, m2, m1, m5, m7, m3 + psubd m8, m0, m4 ; t9 + paddd m0, m4 ; t8 + psubd m4, m6, m2 ; t10 + paddd m2, m6 ; t11 + psubd m6, m1, m5 ; t13 + paddd m5, m1 ; t12 + psubd m1, m7, m3 ; t14 + paddd m7, m3 ; t15 + REPX {pmaxsd x, m12}, m8, m1, m4, m6, m0, m2, m5, m7 + REPX {pminsd x, m13}, m8, m1, m4, m6, m0, m2, m5, m7 + vpbroadcastd m15, [pd_3784] + vpbroadcastd m10, [pd_1567] + ITX_MULSUB_2D 1, 8, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 4, 3, 9, _, 11, 10, 15, 2 + psubd m3, m1, m4 ; t10 + paddd m1, m4 ; t9 + psubd m4, m0, m2 ; t11a + paddd m0, m2 ; t8a + psubd m2, m8, m6 ; t13 + paddd m6, m8 ; t14 + psubd m8, m7, m5 ; t12a + paddd m7, m5 ; t15a + REPX {pmaxsd x, m12}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pminsd x, m13}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pmulld x, m14}, m2, m8, m3, m4 + paddd m2, m11 + paddd m8, m11 + paddd m5, m2, m3 ; t13a + psubd m2, m3 ; t10a + psubd m3, m8, m4 ; t11 + paddd m4, m8 ; t12 + REPX {psrad x, 12}, m5, m2, m3, m4 + mova [r6-32*4], m7 + mova [r6-32*3], m6 + mova [r6-32*2], m5 + mova [r6-32*1], m4 + mova [r6+32*0], m3 + mova [r6+32*1], m2 + mova [r6+32*2], m1 + mova [r6+32*3], m0 + ret + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity, 35 + +cglobal iadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + vpbroadcastd m14, [pd_2896] + vpbroadcastd m11, [pd_2048] + cmp eobd, 43 + jl .fast + add cq, 32 + call .pass1_main + call m(iadst_8x8_internal_10bpc).main_end + sub cq, 32 + mova [cq+32* 1], m0 + mova [cq+32* 3], m1 + mova [cq+32* 5], m2 + mova [cq+32* 7], m3 + mova [cq+32* 9], m4 + mova [cq+32*11], m5 + mova [cq+32*13], m6 + mova m15, m7 + call .pass1_main + call m(iadst_8x8_internal_10bpc).main_end + mova m8, [cq+32* 1] + mova m9, [cq+32* 3] + mova m10, [cq+32* 5] + mova m11, [cq+32* 7] + mova m12, [cq+32* 9] + mova m13, [cq+32*11] + mova m14, [cq+32*13] + jmp tx2q +.fast: + call .pass1_main + call m(iadst_8x8_internal_10bpc).main_end + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call m(idct_8x16_internal_10bpc).transpose + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end + vpbroadcastd m8, [pw_2048] + vpbroadcastd xm12, [pw_4096] + REPX {vpermq x, x, q2031}, m0, m1, m2, m3 + REPX {vpermq x, x, q3120}, m4, m5, m6, m7 + psubw m12, m8 + jmp m(idct_8x16_internal_10bpc).end +ALIGN function_align +.pass1_main: + pmulld m0, m14, [cq+32* 0] + pmulld m7, m14, [cq+32*14] + pmulld m1, m14, [cq+32* 2] + pmulld m6, m14, [cq+32*12] + pmulld m2, m14, [cq+32* 4] + pmulld m5, m14, [cq+32*10] + pmulld m3, m14, [cq+32* 6] + pmulld m4, m14, [cq+32* 8] + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + jmp m(iadst_8x8_internal_10bpc).main2 + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity, 35 + +cglobal iflipadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + vpbroadcastd m14, [pd_2896] + vpbroadcastd m11, [pd_2048] + cmp eobd, 43 + jl .fast + add cq, 32 + call m(iadst_8x16_internal_10bpc).pass1_main + call m(iflipadst_8x8_internal_10bpc).main_end + sub cq, 32 + mova [cq+32* 1], m0 + mova [cq+32* 3], m1 + mova [cq+32* 5], m2 + mova [cq+32* 7], m3 + mova [cq+32* 9], m4 + mova [cq+32*11], m5 + mova [cq+32*13], m6 + mova m15, m7 + call m(iadst_8x16_internal_10bpc).pass1_main + call m(iflipadst_8x8_internal_10bpc).main_end + mova m8, [cq+32* 1] + mova m9, [cq+32* 3] + mova m10, [cq+32* 5] + mova m11, [cq+32* 7] + mova m12, [cq+32* 9] + mova m13, [cq+32*11] + mova m14, [cq+32*13] + jmp tx2q +.fast: + call m(iadst_8x16_internal_10bpc).pass1_main + call m(iflipadst_8x8_internal_10bpc).main_end + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call m(idct_8x16_internal_10bpc).transpose + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end + vpbroadcastd m12, [pw_2048] + vpbroadcastd xm13, [pw_4096] + mova m11, m0 + vpermq m0, m7, q2031 + mova m10, m1 + vpermq m1, m6, q2031 + mova m9, m2 + vpermq m2, m5, q2031 + mova m8, m3 + vpermq m3, m4, q2031 + vpermq m4, m8, q3120 + vpermq m5, m9, q3120 + vpermq m6, m10, q3120 + vpermq m7, m11, q3120 + psubw m12, m13 + jmp m(idct_8x16_internal_10bpc).end + +INV_TXFM_8X16_FN identity, dct +INV_TXFM_8X16_FN identity, adst +INV_TXFM_8X16_FN identity, flipadst +INV_TXFM_8X16_FN identity, identity + +%macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16384] + pmulhrsw m%2, m%3, m%1 +%if %0 == 4 ; if downshifting by 1 +%ifnum %4 + pmulhrsw m%2, m%4 +%else ; without rounding + psraw m%2, 1 +%endif +%else + paddsw m%1, m%1 +%endif + paddsw m%1, m%2 +%endmacro + +cglobal iidentity_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m15, [pd_2896] + pmulld m0, m15, [cq+32* 0] + pmulld m8, m15, [cq+32* 1] + pmulld m1, m15, [cq+32* 2] + pmulld m9, m15, [cq+32* 3] + pmulld m2, m15, [cq+32* 4] + pmulld m10, m15, [cq+32* 5] + pmulld m3, m15, [cq+32* 6] + pmulld m11, m15, [cq+32* 7] + pmulld m4, m15, [cq+32* 8] + pmulld m12, m15, [cq+32* 9] + pmulld m5, m15, [cq+32*10] + pmulld m13, m15, [cq+32*11] + pmulld m6, m15, [cq+32*12] + pmulld m14, m15, [cq+32*13] + pmulld m7, m15, [cq+32*14] + pmulld m15, [cq+32*15] + mova [cq], m7 + vpbroadcastd m7, [pd_2048] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + paddd m7, [cq] + REPX {psrad x, 12}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + packssdw m0, m8 + packssdw m1, m9 + packssdw m2, m10 + packssdw m3, m11 + packssdw m4, m12 + packssdw m5, m13 + packssdw m6, m14 + packssdw m13, m7, m15 + vpbroadcastd m8, [pw_1697x16] + REPX {IDTX16 x, 9, 8}, 0, 1, 2, 3, 4, 5, 6, 13 + vpbroadcastd m7, [pixel_10bpc_max] + vpbroadcastd m12, [pw_2048] + call .pass2_end + RET +ALIGN function_align +.pass2_end: + punpckhwd m9, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m13 + punpcklwd m6, m13 + punpckhwd m13, m4, m5 + punpcklwd m4, m5 + punpcklwd m5, m2, m3 + punpckhwd m2, m3 + punpckhdq m3, m0, m5 + punpckldq m0, m5 + punpckhdq m11, m9, m2 + punpckldq m9, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckldq m6, m13, m1 + punpckhdq m13, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m8, m9, m6 + punpckhqdq m9, m6 + punpcklqdq m10, m11, m13 + punpckhqdq m11, m13 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(iidentity_8x8_internal_10bpc).write_2x8x2_start + pmulhrsw m0, m12, m2 + pmulhrsw m1, m12, m3 + call m(iidentity_8x8_internal_10bpc).write_2x8x2_zero + pmulhrsw m0, m12, m8 + pmulhrsw m1, m12, m9 + lea dstq, [dstq+strideq*4] + call m(iidentity_8x8_internal_10bpc).write_2x8x2_zero + pmulhrsw m0, m12, m10 + pmulhrsw m1, m12, m11 + call m(iidentity_8x8_internal_10bpc).write_2x8x2_zero + ret + +INV_TXFM_8X16_FN dct, dct, 0, 12 +INV_TXFM_8X16_FN dct, identity, 35, 12 +INV_TXFM_8X16_FN dct, adst, 0, 12 +INV_TXFM_8X16_FN dct, flipadst, 0, 12 + +cglobal idct_8x16_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(idct_8x16_internal_10bpc).pass1 +.pass2: + lea r6, [rsp+32*4] + call .transpose + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + mova [cq+32* 8], m0 + mova [cq+32*10], m2 + mova [cq+32*12], m4 + mova [cq+32*14], m6 + pmaxsd m0, m12, [cq+32* 1] + pmaxsd m4, m12, m1 + pmaxsd m1, m12, [cq+32* 3] + pmaxsd m2, m12, [cq+32* 5] + pmaxsd m6, m12, m5 + pmaxsd m5, m12, m3 + pmaxsd m3, m12, [cq+32* 7] + pmaxsd m7, m12 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m11, [pd_2048] + vpbroadcastd m14, [pd_2896] + call m(idct_8x16_internal_10bpc).main_oddhalf + pmaxsd m0, m12, [cq+32* 0] + pmaxsd m1, m12, [cq+32* 2] + pmaxsd m2, m12, [cq+32* 4] + pmaxsd m3, m12, [cq+32* 6] + pmaxsd m4, m12, [cq+32* 8] + pmaxsd m5, m12, [cq+32*10] + pmaxsd m6, m12, [cq+32*12] + pmaxsd m7, m12, [cq+32*14] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + vpbroadcastd m11, [pd_8] + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_16x8_internal_10bpc).pass1_rotations + REPX {psrad x, 4}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 +.end: + packssdw m0, m1 + packssdw m1, m2, m3 + packssdw m2, m4, m5 + packssdw m3, m6, m7 + packssdw m4, m8, m9 + packssdw m5, m10, m11 + packssdw m6, m12, m13 + packssdw m7, m14, m15 + vpermq m0, m0, q3120 + vpermq m1, m1, q3120 + call m(idct_8x8_internal_12bpc).write_8x4_start + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m2, q3120 + vpermq m1, m3, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m4, q3120 + vpermq m1, m5, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m6, q3120 + vpermq m1, m7, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +ALIGN function_align +.transpose: + mova [cq+32* 8], m8 + mova [cq+32* 9], m9 + mova [cq+32*10], m10 + mova [cq+32*11], m11 + call m(idct_8x8_internal_12bpc).transpose_8x8 + mova [cq+32* 0], m0 + mova [cq+32* 1], m1 + mova [cq+32* 2], m2 + mova [cq+32* 3], m3 + mova [cq+32* 4], m4 + mova [cq+32* 5], m5 + mova [cq+32* 6], m6 + mova [cq+32* 7], m7 + mova m0, [cq+32* 8] + mova m1, [cq+32* 9] + mova m2, [cq+32*10] + mova m3, [cq+32*11] + mova m4, m12 + mova m5, m13 + mova m6, m14 + mova m7, m15 + jmp m(idct_8x8_internal_12bpc).transpose_8x8 + +INV_TXFM_8X16_FN adst, dct, 0, 12 +INV_TXFM_8X16_FN adst, adst, 0, 12 +INV_TXFM_8X16_FN adst, flipadst, 0, 12 +INV_TXFM_8X16_FN adst, identity, 35, 12 + +cglobal iadst_8x16_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iadst_8x16_internal_10bpc).pass1 +.pass2: + lea r6, [rsp+32*4] + call .pass2_main + call m(iadst_16x8_internal_10bpc).pass1_rotations +.pass2_end: + REPX {psrad x, 4 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 15}, m4, m5, m6, m7, m8, m9, m10, m11 + jmp m(idct_8x16_internal_12bpc).end +ALIGN function_align +.pass2_main: + call m(idct_8x16_internal_12bpc).transpose + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] + mova [cq+32* 8], m0 + mova [cq+32*11], m3 + mova [cq+32*12], m4 + mova [cq+32*15], m7 + pmaxsd m0, m13, [cq+32* 2] ; 2 + pmaxsd m3, m13, m1 ; 9 + pmaxsd m1, m13, m5 ; 13 + pmaxsd m4, m13, m2 ; 10 + pmaxsd m2, m13, [cq+32* 6] ; 6 + pmaxsd m5, m13, [cq+32* 5] ; 5 + pmaxsd m6, m13, m6 ; 14 + pmaxsd m7, m13, [cq+32* 1] ; 1 + REPX {pminsd x, m14}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m12, [pd_2048] + vpbroadcastd m15, [pd_2896] + call m(iadst_16x8_internal_10bpc).main_part1 + pmaxsd m0, m13, [cq+32* 0] ; 0 + pmaxsd m1, m13, [cq+32*15] ; 15 + pmaxsd m2, m13, [cq+32* 4] ; 4 + pmaxsd m3, m13, [cq+32*11] ; 11 + pmaxsd m4, m13, [cq+32* 8] ; 8 + pmaxsd m5, m13, [cq+32* 7] ; 7 + pmaxsd m6, m13, [cq+32*12] ; 12 + pmaxsd m7, m13, [cq+32* 3] ; 3 + REPX {pminsd x, m14}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(iadst_16x8_internal_10bpc).main_part2 + vpbroadcastd m14, [pd_17408] + psrld m15, 11 ; pd_1 + psubd m13, m14, m15 ; pd_17407 + pslld m15, 3 ; pd_8 + ret + +INV_TXFM_8X16_FN flipadst, dct, 0, 12 +INV_TXFM_8X16_FN flipadst, adst, 0, 12 +INV_TXFM_8X16_FN flipadst, flipadst, 0, 12 +INV_TXFM_8X16_FN flipadst, identity, 35, 12 + +cglobal iflipadst_8x16_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iflipadst_8x16_internal_10bpc).pass1 +.pass2: + lea r6, [rsp+32*4] + call m(iadst_8x16_internal_12bpc).pass2_main + call m(iflipadst_16x8_internal_10bpc).pass1_rotations + jmp m(iadst_8x16_internal_12bpc).pass2_end + +INV_TXFM_8X16_FN identity, dct, 0, 12 +INV_TXFM_8X16_FN identity, adst, 0, 12 +INV_TXFM_8X16_FN identity, flipadst, 0, 12 +INV_TXFM_8X16_FN identity, identity, 0, 12 + +cglobal iidentity_8x16_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + jmp m(iidentity_8x16_internal_10bpc).pass1 +.pass2: + call .pass2_main + packssdw m0, m8 + packssdw m1, m9 + packssdw m2, m10 + packssdw m3, m11 + packssdw m4, m12 + packssdw m5, m13 + packssdw m6, m14 + packssdw m13, m7, m15 + vpbroadcastd m7, [pixel_12bpc_max] + vpbroadcastd m12, [pw_16384] + call m(iidentity_8x16_internal_10bpc).pass2_end + RET +ALIGN function_align +.pass2_main: + mova [cq], m7 + vpbroadcastd m7, [clip_18b_min] + REPX {pmaxsd x, m7}, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + pmaxsd m7, [cq] + mova [cq], m15 + vpbroadcastd m15, [clip_18b_max] + REPX {pminsd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + pminsd m15, [cq] + mova [cq], m7 + vpbroadcastd m7, [pd_5793] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + pmulld m7, [cq] + mova [cq], m15 + vpbroadcastd m15, [pd_1024] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [cq] + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ret + +%macro INV_TXFM_16X4_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 16x4, %3 +%ifidn %1_%2, dct_dct + vpbroadcastd m3, [dconly_%3bpc] +%if %3 = 10 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 4 +.dconly2: + add r6d, 384 + sar r6d, 9 +.dconly3: + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + movd xm0, r6d + paddsw xm0, xm3 + vpbroadcastw m0, xm0 +.dconly_loop: + paddsw m1, m0, [dstq+strideq*0] + paddsw m2, m0, [dstq+strideq*1] + psubusw m1, m3 + psubusw m2, m3 + mova [dstq+strideq*0], m1 + mova [dstq+strideq*1], m2 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%else + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly +%endif +%endif +%endmacro + +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, identity +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst + +cglobal idct_16x4_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_18b_min] + vpbroadcastd m9, [clip_18b_max] +.pass1: + vbroadcasti128 m0, [cq+16* 0] + vbroadcasti128 m4, [cq+16* 4] + vbroadcasti128 m1, [cq+16* 2] + vbroadcasti128 m7, [cq+16* 6] + vbroadcasti128 m5, [cq+16*10] + vbroadcasti128 m2, [cq+16* 8] + vbroadcasti128 m6, [cq+16*12] + vbroadcasti128 m3, [cq+16*14] + shufpd m0, m4, 0x0c ; 0 4 + shufpd m1, m5, 0x0c ; 2 10 + shufpd m2, m6, 0x0c ; 8 12 + shufpd m3, m7, 0x0c ; 14 6 + call .pass1_main + vbroadcasti128 m10, [cq+16* 1] + vbroadcasti128 m4, [cq+16* 5] + vbroadcasti128 m11, [cq+16*15] + vbroadcasti128 m5, [cq+16*11] + shufpd m10, m4, 0x0c ; 1 5 + shufpd m11, m5, 0x0c ; 15 11 + vbroadcasti128 m5, [cq+16* 9] + vbroadcasti128 m4, [cq+16*13] + shufpd m5, m4, 0x0c ; 9 13 + vbroadcasti128 m6, [cq+16* 7] + vbroadcasti128 m4, [cq+16* 3] + shufpd m6, m4, 0x0c ; 7 3 + call .pass1_main2 + pcmpeqd m4, m4 + REPX {psubd x, m4}, m0, m1, m2, m3 + call .pass1_main3 + REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 + jmp tx2q +.pass2: + call .transpose_4x16_packed + lea r6, [deint_shuf+128] + call m(idct_16x4_internal_8bpc).main +.end: + vpbroadcastd m4, [pw_2048] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + vpbroadcastd m5, [pixel_10bpc_max] +.end2: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] +.end3: + lea r6, [dstq+strideq*2] + paddw m2, [r6 +strideq*0] + paddw m3, [r6 +strideq*1] + pxor m4, m4 + REPX {mova [cq+32*x], m4}, 0, 1, 2, 3, 4, 5, 6, 7 + REPX {pmaxsw x, m4}, m0, m1, m2, m3 + REPX {pminsw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [r6 +strideq*0], m2 + mova [r6 +strideq*1], m3 + RET +ALIGN function_align +.pass1_main: + vpbroadcastd m7, [pd_2048] + call m(idct_8x4_internal_10bpc).main + psubd m3, m0, m4 ; idct8 out7 out6 + paddd m0, m4 ; idct8 out0 out1 + paddd m1, m2, m5 ; idct8 out3 out2 + psubd m2, m5 ; idct8 out4 out5 + ret +ALIGN function_align +.pass1_main2: + ITX_MULSUB_2D 10, 11, 4, 12, 13, 7, 401_1931, 4076_3612, 1 + ITX_MULSUB_2D 5, 6, 4, 12, 13, 7, 3166_3920, 2598_1189, 1 + vbroadcasti128 m12, [pd_3784_m3784] + psubd m4, m10, m5 + paddd m10, m5 ; t8 t11 + psignd m4, m12 ; t9 t10 + psubd m5, m11, m6 + paddd m11, m6 ; t15 t12 + psignd m5, m12 ; t14 t13 + vpbroadcastd m6, [pd_1567] + vpbroadcastd m13, [pd_3784] + REPX {pmaxsd x, m8}, m5, m4 + REPX {pminsd x, m9}, m5, m4 + pmulld m12, m5 + pmulld m5, m6 + vbroadcasti128 m6, [pd_1567_m1567] + pmulld m13, m4 + pmulld m4, m6 + REPX {pmaxsd x, m8}, m10, m11, m0, m1 + REPX {pminsd x, m9}, m10, m11, m0, m1 + paddd m12, m7 + paddd m5, m7 + paddd m4, m12 + psubd m5, m13 + psrad m4, 12 ; t14a t10a + psrad m5, 12 ; t9a t13a + vpbroadcastd m12, [pd_2896] + punpckhqdq m6, m11, m5 + punpcklqdq m11, m4 + punpckhqdq m4, m10, m4 + punpcklqdq m10, m5 + psubd m5, m11, m6 ; t12a t13 + paddd m11, m6 ; t15a t14 + psubd m6, m10, m4 ; t11a t10 + paddd m10, m4 ; t8a t9 + REPX {pmaxsd x, m8}, m5, m6 + REPX {pminsd x, m9}, m5, m6 + pmulld m5, m12 + pmulld m6, m12 + REPX {pmaxsd x, m8}, m2, m3, m11, m10 + REPX {pminsd x, m9}, m2, m3, m11, m10 + ret +ALIGN function_align +.pass1_main3: + paddd m5, m7 + psubd m4, m5, m6 + paddd m5, m6 + psrad m4, 12 ; t11 t10a + psrad m5, 12 ; t12 t13a + psubd m7, m0, m11 ; out15 out14 + paddd m0, m11 ; out0 out1 + psubd m6, m1, m5 ; out12 out13 + paddd m1, m5 ; out3 out2 + psubd m5, m2, m4 ; out11 out10 + paddd m2, m4 ; out4 out5 + psubd m4, m3, m10 ; out8 out9 + paddd m3, m10 ; out7 out6 + REPX {pshufd x, x, q1032}, m1, m3, m5, m7 + ret +ALIGN function_align +.transpose_4x16_packed: + vbroadcasti128 m8, [deint_shuf] + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + REPX {pshufb x, m8}, m0, m2, m4, m6 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpckhqdq m2, m4, m6 + punpcklqdq m4, m6 + vperm2i128 m3, m1, m2, 0x31 + vinserti128 m1, xm2, 1 + vperm2i128 m2, m0, m4, 0x31 + vinserti128 m0, xm4, 1 + ret + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + call m(iadst_4x16_internal_10bpc).main + psrad m11, 11 ; pd_1 + REPX {paddd x, m11}, m0, m1, m2, m3 + paddd m4, m5, m11 + paddd m5, m6, m11 + paddd m6, m7, m11 + paddd m7, m8, m11 +.pass1_end: + REPX {pshufd x, x, q1032}, m0, m2, m4, m6 + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7 + jmp tx2q +.pass2: + call m(idct_16x4_internal_10bpc).transpose_4x16_packed + lea r6, [deint_shuf+128] + call m(iadst_16x4_internal_8bpc).main + jmp m(idct_16x4_internal_10bpc).end +ALIGN function_align +.main: + vpbroadcastd m6, [pd_1321] + mova m0, [cq+32*0] + mova m1, [cq+32*1] + vpbroadcastd m7, [pd_2482] + mova m2, [cq+32*6] + mova m3, [cq+32*7] + pmulld m4, m0, m6 + pmulld m5, m1, m6 ; 1321*in0 + pmulld m9, m2, m7 + pmulld m8, m3, m7 ; 2482*in3 + paddd m4, m9 + paddd m8, m5 ; 1321*in0 + 2482*in3 + pmulld m5, m0, m7 + pmulld m9, m1, m7 ; 2482*in0 + paddd m0, m2 + paddd m1, m3 ; in0 + in3 + paddd m7, m6 ; pd_3803 + pmulld m2, m7 + pmulld m3, m7 ; 3803*in3 + psubd m5, m2 + psubd m9, m3 ; 2482*in0 - 3803*in3 + mova m2, [cq+32*4] + pmulld m10, m7, m2 + pmulld m3, m6, m2 + psubd m2, m0 + mova m0, [cq+32*5] + pmulld m7, m0 ; 3803*in2 + pmulld m6, m0 ; 1321*in2 + psubd m0, m1 ; in2 - in0 - in3 + vpbroadcastd m1, [pd_m3344] + paddd m4, m10 + paddd m7, m8 ; t0 + psubd m5, m3 + psubd m9, m6 ; t1 + pmulld m2, m1 + pmulld m0, m1 ; t2 + pmulld m3, m1, [cq+32*2] + pmulld m1, [cq+32*3] ; -t3 + ret +ALIGN function_align +.main_end: + ; expects: m6 = rnd + paddd m5, m6 + paddd m9, m6 + paddd m10, m4, m5 + paddd m4, m6 + paddd m8, m7, m6 + paddd m7, m9 + psubd m4, m3 ; out0 (unshifted) + psubd m5, m3 ; out1 (unshifted) + paddd m2, m6 ; out2 (unshifted) + paddd m3, m10 ; out3 (unshifted) + psubd m8, m1 ; out4 (unshifted) + psubd m9, m1 ; out5 (unshifted) + paddd m6, m0 ; out6 (unshifted) + paddd m7, m1 ; out7 (unshifted) + ret + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + call m(iadst_4x16_internal_10bpc).main + psrad m11, 11 ; pd_1 + paddd m4, m3, m11 + paddd m3, m5, m11 + paddd m5, m2, m11 + paddd m2, m6, m11 + paddd m6, m1, m11 + paddd m1, m7, m11 + paddd m7, m0, m11 + paddd m0, m8, m11 + jmp m(iadst_16x4_internal_10bpc).pass1_end +.pass2: + call m(idct_16x4_internal_10bpc).transpose_4x16_packed + lea r6, [deint_shuf+128] + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m4, [pw_2048] + pmulhrsw m5, m3, m4 + pmulhrsw m6, m2, m4 + pmulhrsw m2, m1, m4 + pmulhrsw m3, m0, m4 + paddw m0, m5, [dstq+strideq*0] + paddw m1, m6, [dstq+strideq*1] + vpbroadcastd m5, [pixel_10bpc_max] + jmp m(idct_16x4_internal_10bpc).end3 + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_10bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m8, [pd_5793] + vpermq m0, [cq+32*0], q3120 ; 0 1 + vpermq m1, [cq+32*1], q3120 ; 2 3 + vpermq m2, [cq+32*2], q3120 ; 4 5 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m4, [cq+32*4], q3120 ; 8 9 + vpermq m5, [cq+32*5], q3120 ; a b + vpermq m6, [cq+32*6], q3120 ; c d + vpermq m7, [cq+32*7], q3120 ; e f + vpbroadcastd m9, [pd_3072] + REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddd x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12}, m0, m1, m2, m3, m4, m5, m6, m7 + jmp tx2q +.pass2: + call m(idct_16x4_internal_10bpc).transpose_4x16_packed + vpbroadcastd m7, [pw_1697x8] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(idct_16x4_internal_10bpc).end + +INV_TXFM_16X4_FN dct, dct, 12 +INV_TXFM_16X4_FN dct, identity, 12 +INV_TXFM_16X4_FN dct, adst, 12 +INV_TXFM_16X4_FN dct, flipadst, 12 + +cglobal idct_16x4_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m8, [clip_20b_min] + vpbroadcastd m9, [clip_20b_max] + jmp m(idct_16x4_internal_10bpc).pass1 +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + ; deinterleave + REPX {pshufd x, x, q3120}, m0, m1, m2, m3, m4, m5, m6, m7 + ; transpose + punpcklqdq m8, m0, m1 + punpckhqdq m0, m1 + punpcklqdq m9, m2, m3 + punpckhqdq m2, m3 + punpcklqdq m10, m4, m5 + punpckhqdq m4, m5 + punpcklqdq m11, m6, m7 + punpckhqdq m6, m7 + vperm2i128 m3, m0, m2, 0x31 ; out6 + vperm2i128 m1, m0, m2, 0x20 ; out2 + vperm2i128 m7, m4, m6, 0x31 ; out7 + vperm2i128 m5, m4, m6, 0x20 ; out3 + vperm2i128 m13, m10, m11, 0x31 ; out5 + vperm2i128 m12, m10, m11, 0x20 ; out1 + vperm2i128 m11, m8, m9, 0x31 ; out4 + vperm2i128 m10, m8, m9, 0x20 ; out0 + call m(idct_4x16_internal_10bpc).pass1_main + pmulld m0, m6, m10 + pmulld m2, m6, m11 + pmulld m4, m6, m12 + pmulld m6, m13 + vpbroadcastd m10, [pd_17408] + call m(idct_4x16_internal_10bpc).pass1_main2 + REPX {psrad x, 4}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + vpbroadcastd m5, [pixel_12bpc_max] + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 + jmp m(idct_16x4_internal_10bpc).end2 + +INV_TXFM_16X4_FN adst, dct, 12 +INV_TXFM_16X4_FN adst, adst, 12 +INV_TXFM_16X4_FN adst, flipadst, 12 +INV_TXFM_16X4_FN adst, identity, 12 + +cglobal iadst_16x4_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iadst_16x4_internal_10bpc).pass1 +.pass2: + call .pass2_main + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + jmp m(idct_16x4_internal_10bpc).end2 +ALIGN function_align +.pass2_main: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m6, m7 + pmaxsd m8, m4, m12 + pmaxsd m9, m5, m12 + REPX {pminsd x, m13}, m0, m1, m2, m3 + call m(iadst_8x4_internal_12bpc).transpose_4x8 + mova [cq+32*0], m0 + mova [cq+32*2], m1 + mova [cq+32*4], m2 + mova [cq+32*6], m3 + pminsd m0, m8, m13 + pminsd m1, m9, m13 + pminsd m2, m6, m13 + pminsd m3, m7, m13 + call m(iadst_8x4_internal_12bpc).transpose_4x8 + mova [cq+32*1], m0 + mova [cq+32*3], m1 + mova [cq+32*5], m2 + mova [cq+32*7], m3 + call m(iadst_16x4_internal_10bpc).main + vpbroadcastd m6, [pd_2048] + call m(iadst_16x4_internal_10bpc).main_end + psrad m0, m4, 15 + psrad m1, m5, 15 + psrad m2, 15 + psrad m3, 15 + psrad m4, m8, 15 + psrad m5, m9, 15 + psrad m6, 15 + psrad m7, 15 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + vpbroadcastd m4, [pw_16384] + vpbroadcastd m5, [pixel_12bpc_max] + ret + +INV_TXFM_16X4_FN flipadst, dct, 12 +INV_TXFM_16X4_FN flipadst, adst, 12 +INV_TXFM_16X4_FN flipadst, flipadst, 12 +INV_TXFM_16X4_FN flipadst, identity, 12 + +cglobal iflipadst_16x4_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(iflipadst_16x4_internal_10bpc).pass1 +.pass2: + call m(iadst_16x4_internal_12bpc).pass2_main + vpermq m7, m0, q3120 + vpermq m6, m1, q3120 + vpermq m1, m2, q3120 + vpermq m0, m3, q3120 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pmulhrsw m2, m6, m4 + pmulhrsw m3, m7, m4 + jmp m(idct_16x4_internal_10bpc).end2 + +INV_TXFM_16X4_FN identity, dct, 12 +INV_TXFM_16X4_FN identity, adst, 12 +INV_TXFM_16X4_FN identity, flipadst, 12 +INV_TXFM_16X4_FN identity, identity, 12 + +cglobal iidentity_16x4_internal_12bpc, 0, 7, 14, dst, stride, c, eob, tx2 + vpbroadcastd m8, [pd_1697] + vpermq m0, [cq+32*0], q3120 ; 0 1 + vpermq m1, [cq+32*1], q3120 ; 2 3 + vpermq m2, [cq+32*2], q3120 ; 4 5 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpbroadcastd m9, [pd_3072] + pmulld m4, m8, m0 + pmulld m5, m8, m1 + pmulld m6, m8, m2 + pmulld m7, m8, m3 + vpermq m10, [cq+32*4], q3120 ; 8 9 + vpermq m11, [cq+32*5], q3120 ; a b + vpermq m12, [cq+32*6], q3120 ; c d + vpermq m13, [cq+32*7], q3120 ; e f + REPX {paddd x, m9}, m4, m5, m6, m7 + REPX {psrad x, 12}, m4, m5, m6, m7 + paddd m0, m4 + pmulld m4, m8, m10 + paddd m1, m5 + pmulld m5, m8, m11 + paddd m2, m6 + pmulld m6, m8, m12 + paddd m3, m7 + pmulld m7, m8, m13 + REPX {paddd x, m9}, m4, m5, m6, m7 + REPX {psrad x, 12}, m4, m5, m6, m7 + paddd m4, m10 + paddd m5, m11 + paddd m6, m12 + paddd m7, m13 + jmp tx2q +.pass2: + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m8, [pd_5793] + vpbroadcastd m9, [pd_2048] + REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddd x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 15}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_16x4_internal_10bpc).transpose_4x16_packed + vpbroadcastd m4, [pw_16384] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + vpbroadcastd m5, [pixel_12bpc_max] + jmp m(idct_16x4_internal_10bpc).end2 + +%macro INV_TXFM_16X8_FN 2-3 10 ; type1, type2, bitdepth + INV_TXFM_FN %1, %2, 0, 16x8, %3 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_%3bpc] + mov [cq], eobd ; 0 + or r3d, 8 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 +%endif +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, identity +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst + +cglobal idct_16x8_internal_10bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + vpbroadcastd m14, [pd_2896] + pmulld m0, m14, [cq+32* 1] + pmulld m1, m14, [cq+32* 3] + pmulld m2, m14, [cq+32* 5] + pmulld m3, m14, [cq+32* 7] + pmulld m4, m14, [cq+32* 9] + pmulld m5, m14, [cq+32*11] + pmulld m6, m14, [cq+32*13] + pmulld m7, m14, [cq+32*15] + vpbroadcastd m11, [pd_2048] + lea r6, [rsp+32*4] + call m(idct_8x16_internal_10bpc).main_oddhalf_rect2 + pmulld m0, m14, [cq+32* 0] + pmulld m1, m14, [cq+32* 2] + pmulld m2, m14, [cq+32* 4] + pmulld m3, m14, [cq+32* 6] + pmulld m4, m14, [cq+32* 8] + pmulld m5, m14, [cq+32*10] + pmulld m6, m14, [cq+32*12] + pmulld m7, m14, [cq+32*14] + call m(idct_8x8_internal_10bpc).main_rect2 + call m(idct_8x16_internal_10bpc).main_evenhalf + psrld m11, 11 ; pd_1 + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + call .pass1_rotations + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call .transpose + call m(idct_16x8_internal_8bpc).main + vpbroadcastd m10, [pw_2048] +.end: + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pmulhrsw m2, m10 + pmulhrsw m3, m10 + call .write_16x4_start +.end2: + pmulhrsw m0, m4, m10 + pmulhrsw m1, m5, m10 + pmulhrsw m2, m6, m10 + pmulhrsw m3, m7, m10 + call .write_16x4_zero + RET +ALIGN function_align +.pass1_rotations: + mova m14, [r6-32*4] + mova m13, [r6-32*3] + mova m12, [r6-32*2] + mova m11, [r6-32*1] + mova m10, [r6+32*0] + mova m9, [r6+32*1] + mova m8, [r6+32*2] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r6+32*3] ; out8 + paddd m7, [r6+32*3] ; out7 + ret +ALIGN function_align +.transpose: + lea r6, [deint_shuf+128] +.transpose2: + packssdw m0, m8 + packssdw m1, m9 + packssdw m2, m10 + packssdw m3, m11 + packssdw m4, m12 + packssdw m5, m13 + packssdw m6, m14 + packssdw m7, m15 +.transpose3: + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + punpckhwd m3, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m6, m7 + punpcklwd m6, m7 + punpckhdq m7, m4, m6 + punpckldq m4, m6 + punpckldq m6, m8, m2 + punpckhdq m8, m2 + punpckhdq m2, m0, m1 + punpckldq m0, m1 + punpckhdq m1, m3, m5 + punpckldq m3, m5 + punpcklqdq m5, m6, m3 + punpckhqdq m6, m3 + punpckhqdq m3, m2, m7 + punpcklqdq m2, m7 + punpcklqdq m7, m8, m1 + punpckhqdq m8, m1 + punpckhqdq m1, m0, m4 + punpcklqdq m0, m4 + vperm2i128 m4, m0, m5, 0x31 + vinserti128 m0, xm5, 1 + vperm2i128 m5, m1, m6, 0x31 + vinserti128 m1, xm6, 1 + vperm2i128 m6, m2, m7, 0x31 + vinserti128 m2, xm7, 1 + vperm2i128 m7, m3, m8, 0x31 + vinserti128 m3, xm8, 1 + ret +ALIGN function_align +.write_16x4_start: + vpbroadcastd m9, [pixel_10bpc_max] + lea r3, [strideq*3] + pxor m8, m8 +.write_16x4_zero: + REPX {mova [cq+32*x], m8}, 0, 1, 2, 3, 4, 5, 6, 7 + add cq, 32*8 +.write_16x4: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3 ] + REPX {pmaxsw x, m8}, m0, m1, m2, m3 + REPX {pminsw x, m9}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + lea dstq, [dstq+strideq*4] + ret + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity + +cglobal iadst_16x8_internal_10bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] +.pass1: + lea r6, [rsp+32*4] + call .main + vpbroadcastd m14, [pd_3072] + psrld m15, 11 ; pd_1 + psubd m13, m14, m15 ; pd_3071 + call .pass1_rotations +.pass1_end: + REPX {psrad x, 1 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 12}, m4, m5, m6, m7, m8, m9, m10, m11 + jmp tx2q +.pass2: + call m(idct_16x8_internal_10bpc).transpose + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + vpbroadcastd m10, [pw_2048] + pxor m11, m11 + psubw m11, m10 + pmulhrsw m0, m10 + pmulhrsw m1, m11 + pmulhrsw m2, m10 + pmulhrsw m3, m11 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m4, m10 + pmulhrsw m1, m5, m11 + pmulhrsw m2, m6, m10 + pmulhrsw m3, m7, m11 + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET +ALIGN function_align +.pass1_rotations: + paddd m0, m15 + psubd m1, m15, m1 + paddd m2, m15 + psubd m3, m15, m3 + paddd m4, m14 + psubd m5, m13, m5 + paddd m6, m14 + psubd m7, m13, m7 + paddd m8, m14, m9 + psubd m9, m13, m10 + paddd m10, m14, m11 + psubd m11, m13, m12 + paddd m12, m15, [r6-32*1] + psubd m13, m15, [r6-32*2] + paddd m14, m15, [r6-32*3] + psubd m15, [r6-32*4] + ret +ALIGN function_align +.main: + ; expects: m13 = clip_min m14 = clip_max + vpbroadcastd m15, [pd_2896] + pmulld m0, m15, [cq+32* 2] + pmulld m1, m15, [cq+32*13] + pmulld m2, m15, [cq+32* 6] + pmulld m3, m15, [cq+32* 9] + pmulld m4, m15, [cq+32*10] + pmulld m5, m15, [cq+32* 5] + pmulld m6, m15, [cq+32*14] + pmulld m7, m15, [cq+32* 1] + vpbroadcastd m12, [pd_2048] + REPX {paddd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + call .main_part1 + pmulld m0, m15, [cq+32* 0] + pmulld m1, m15, [cq+32*15] + pmulld m2, m15, [cq+32* 4] + pmulld m3, m15, [cq+32*11] + pmulld m4, m15, [cq+32* 8] + pmulld m5, m15, [cq+32* 7] + pmulld m6, m15, [cq+32*12] + pmulld m7, m15, [cq+32* 3] + REPX {paddd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.main_part2: + ITX_MULSUB_2D 1, 0, 8, 9, 10, 12, 201, 4091 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 12, 1751, 3703 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 12, 3035, 2751 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 12, 3857, 1380 + psubd m8, m0, m4 ; t8a + paddd m0, m4 ; t0a + psubd m4, m1, m5 ; t9a + paddd m1, m5 ; t1a + psubd m5, m2, m6 ; t12a + paddd m2, m6 ; t4a + psubd m6, m3, m7 ; t13a + paddd m7, m3 ; t5a + REPX {pmaxsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m14}, m8, m4, m5, m6, m0, m1, m2, m7 + vpbroadcastd m11, [pd_4017] + vpbroadcastd m10, [pd_799] + ITX_MULSUB_2D 8, 4, 3, 9, _, 12, 10, 11 + ITX_MULSUB_2D 6, 5, 3, 9, _, 12, 11, 10 + psubd m3, m0, m2 ; t4 + paddd m0, m2 ; t0 + psubd m2, m1, m7 ; t5 + paddd m1, m7 ; t1 + psubd m7, m4, m6 ; t12a + paddd m4, m6 ; t8a + psubd m6, m8, m5 ; t13a + paddd m5, m8 ; t9a + REPX {pmaxsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m14}, m3, m2, m7, m6, m0, m1, m4, m5 + vpbroadcastd m11, [pd_3784] + vpbroadcastd m10, [pd_1567] + ITX_MULSUB_2D 3, 2, 8, 9, _, 12, 10, 11 + ITX_MULSUB_2D 7, 6, 8, 9, _, 12, 10, 11 + pminsd m10, m14, [r6-32*4] ; t2 + pminsd m8, m14, [r6-32*3] ; t3 + psubd m9, m0, m10 ; t2a + paddd m0, m10 ; out0 + psubd m10, m1, m8 ; t3a + paddd m1, m8 ; -out15 + pmaxsd m9, m13 + pmaxsd m10, m13 + pminsd m9, m14 + pminsd m10, m14 + mova [r6-32*4], m1 + mova m11, [r6-32*1] ; t7a + mova m1, [r6-32*2] ; t6a + psubd m8, m3, m11 ; t7 + paddd m11, m3 ; out12 + paddd m3, m2, m1 ; -out3 + psubd m2, m1 ; t6 + pmaxsd m8, m13 + pmaxsd m2, m13 + pminsd m8, m14 + pminsd m2, m14 + mova [r6-32*1], m11 + mova [r6-32*3], m2 + mova m1, [r6+32*3] ; t15 + mova m2, [r6+32*2] ; t14 + paddd m12, m7, m1 ; -out13 + psubd m7, m1 ; t15a + psubd m11, m6, m2 ; t14a + paddd m2, m6 ; out2 + pmaxsd m7, m13 + pmaxsd m11, m13 + pminsd m7, m14 + pminsd m11, m14 + mova [r6-32*2], m12 + pminsd m1, m14, [r6+32*0] ; t10a + pminsd m12, m14, [r6+32*1] ; t11a + psubd m6, m4, m1 ; t10 + paddd m1, m4 ; -out1 + psubd m4, m5, m12 ; t11 + paddd m5, m12 ; out14 + vpbroadcastd m12, [pd_1448] + pmaxsd m6, m13 + pmaxsd m4, m13 + pminsd m6, m14 + pminsd m4, m14 + REPX {pmulld x, m12}, m9, m10, m8, m7, m11, m6, m4 + pmulld m12, [r6-32*3] ; t6 + mova [r6-32*3], m5 + paddd m5, m11, m7 ; -out5 (unshifted) + psubd m11, m7 ; out10 (unshifted) + paddd m7, m9, m10 ; -out7 (unshifted) + psubd m9, m10 ; out8 (unshifted) + psubd m10, m6, m4 ; -out9 (unshifted) + paddd m6, m4 ; out6 (unshifted) + paddd m4, m12, m8 ; out4 (unshifted) + psubd m12, m8 ; -out11 (unshifted) + ret +.main_part1: + ITX_MULSUB_2D 1, 0, 8, 9, 10, 12, 995, 3973 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 12, 2440, 3290 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 12, 3513, 2106 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 12, 4052, 601 + psubd m8, m0, m4 ; t10a + paddd m0, m4 ; t2a + psubd m4, m1, m5 ; t11a + paddd m1, m5 ; t3a + psubd m5, m2, m6 ; t14a + paddd m2, m6 ; t6a + psubd m6, m3, m7 ; t15a + paddd m7, m3 ; t7a + REPX {pmaxsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m14}, m8, m4, m5, m6, m0, m1, m2, m7 + vpbroadcastd m11, [pd_2276] + vpbroadcastd m10, [pd_3406] + ITX_MULSUB_2D 8, 4, 3, 9, _, 12, 10, 11 + ITX_MULSUB_2D 6, 5, 3, 9, _, 12, 11, 10 + psubd m3, m0, m2 ; t6 + paddd m0, m2 ; t2 + psubd m2, m1, m7 ; t7 + paddd m1, m7 ; t3 + psubd m7, m4, m6 ; t14a + paddd m4, m6 ; t10a + psubd m6, m8, m5 ; t15a + paddd m5, m8 ; t11a + REPX {pmaxsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m14}, m3, m2, m7, m6 ; clip the rest later + vpbroadcastd m11, [pd_1567] + vpbroadcastd m10, [pd_3784] + ITX_MULSUB_2D 2, 3, 8, 9, _, 12, 10, 11 + ITX_MULSUB_2D 6, 7, 8, 9, _, 12, 10, 11 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6-32*2], m2 + mova [r6-32*1], m3 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + ret + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity + +cglobal iflipadst_16x8_internal_10bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] +.pass1: + lea r6, [rsp+32*4] + call m(iadst_16x8_internal_10bpc).main + vpbroadcastd m14, [pd_3072] + psrld m15, 11 + psubd m13, m14, m15 + call .pass1_rotations + jmp m(iadst_16x8_internal_10bpc).pass1_end +.pass2: + call m(idct_16x8_internal_10bpc).transpose + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + vpbroadcastd m10, [pw_2048] + pxor m11, m11 + psubw m11, m10 + mova m12, m0 + pmulhrsw m0, m7, m11 + mova m7, m1 + pmulhrsw m1, m6, m10 + mova m6, m2 + pmulhrsw m2, m5, m11 + mova m5, m3 + pmulhrsw m3, m4, m10 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m5, m11 + pmulhrsw m1, m6, m10 + pmulhrsw m2, m7, m11 + pmulhrsw m3, m12, m10 + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET +ALIGN function_align +.pass1_rotations: + psubd m8, m13, m7 + paddd m7, m14, m9 + paddd m9, m14, m6 + psubd m6, m13, m10 + psubd m10, m13, m5 + paddd m5, m14, m11 + paddd m11, m14, m4 + psubd m4, m13, m12 + psubd m12, m15, m3 + paddd m3, m15, [r6-32*1] + paddd m13, m15, m2 + psubd m2, m15, [r6-32*2] + psubd m14, m15, m1 + mova m1, m15 + paddd m15, m0 + psubd m0, m1, [r6-32*4] + paddd m1, [r6-32*3] + ret + +INV_TXFM_16X8_FN identity, dct +INV_TXFM_16X8_FN identity, adst +INV_TXFM_16X8_FN identity, flipadst +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_10bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 +.pass1: + vpbroadcastd m15, [pd_2896] + pmulld m0, m15, [cq+32* 0] + pmulld m1, m15, [cq+32* 1] + pmulld m2, m15, [cq+32* 2] + pmulld m3, m15, [cq+32* 3] + pmulld m4, m15, [cq+32* 4] + pmulld m5, m15, [cq+32* 5] + pmulld m6, m15, [cq+32* 6] + pmulld m7, m15, [cq+32* 7] + pmulld m8, m15, [cq+32* 8] + pmulld m9, m15, [cq+32* 9] + pmulld m10, m15, [cq+32*10] + pmulld m11, m15, [cq+32*11] + pmulld m12, m15, [cq+32*12] + pmulld m13, m15, [cq+32*13] + pmulld m14, m15, [cq+32*14] + pmulld m15, [cq+32*15] + mova [rsp], m7 + vpbroadcastd m7, [pd_2048] + REPX {paddd x, m7 }, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + paddd m7, [rsp] + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + mova [rsp], m15 + vpbroadcastd m15, [pd_5793] + REPX {pmulld x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + pmulld m15, [rsp] + mova [rsp], m7 + vpbroadcastd m7, [pd_3072] + REPX {paddd x, m7 }, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + paddd m7, [rsp] + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call m(idct_16x8_internal_10bpc).transpose + vpbroadcastd m10, [pw_4096] + jmp m(idct_16x8_internal_10bpc).end + +INV_TXFM_16X8_FN dct, dct, 12 +INV_TXFM_16X8_FN dct, identity, 12 +INV_TXFM_16X8_FN dct, adst, 12 +INV_TXFM_16X8_FN dct, flipadst, 12 + +cglobal idct_16x8_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(idct_16x8_internal_10bpc).pass1 +.pass2: + call .pass2_main + RET +ALIGN function_align +.pass2_main: + call m(idct_8x16_internal_12bpc).transpose + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m11, [pd_2048] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x8_internal_12bpc).round_shift4 + mova [cq+32* 8], m0 + mova [cq+32* 9], m1 + mova [cq+32*10], m2 + mova [cq+32*11], m3 + mova [cq+32*12], m4 + mova [cq+32*13], m5 + mova [cq+32*14], m6 + mova [cq+32*15], m7 + pmaxsd m0, m12, [cq+32*0] + pmaxsd m1, m12, [cq+32*1] + pmaxsd m2, m12, [cq+32*2] + pmaxsd m3, m12, [cq+32*3] + pmaxsd m4, m12, [cq+32*4] + pmaxsd m5, m12, [cq+32*5] + pmaxsd m6, m12, [cq+32*6] + pmaxsd m7, m12, [cq+32*7] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x8_internal_12bpc).round_shift4 +.end: + packssdw m0, [cq+32* 8] + packssdw m1, [cq+32* 9] + packssdw m2, [cq+32*10] + packssdw m3, [cq+32*11] + packssdw m4, [cq+32*12] + packssdw m5, [cq+32*13] + packssdw m6, [cq+32*14] + packssdw m7, [cq+32*15] + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 + call .write_16x4_start + call m(idct_16x8_internal_10bpc).write_16x4_zero + vpermq m0, m4, q3120 + vpermq m1, m5, q3120 + vpermq m2, m6, q3120 + vpermq m3, m7, q3120 + jmp m(idct_16x8_internal_10bpc).write_16x4_zero +ALIGN function_align +.write_16x4_start: + vpbroadcastd m9, [pixel_12bpc_max] + lea r3, [strideq*3] + pxor m8, m8 + ret + +INV_TXFM_16X8_FN adst, dct, 12 +INV_TXFM_16X8_FN adst, adst, 12 +INV_TXFM_16X8_FN adst, flipadst, 12 +INV_TXFM_16X8_FN adst, identity, 12 + +cglobal iadst_16x8_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_20b_min] + vpbroadcastd m14, [clip_20b_max] + jmp m(iadst_16x8_internal_10bpc).pass1 +.pass2: + call .pass2_main + call m(idct_16x8_internal_12bpc).end + RET +ALIGN function_align +.pass2_main: + call m(idct_8x16_internal_12bpc).transpose + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m11, [pd_2048] + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(iadst_8x8_internal_12bpc).pass2_main2 + mova [cq+32* 8], m0 + mova [cq+32* 9], m1 + mova [cq+32*10], m2 + mova [cq+32*11], m3 + mova [cq+32*12], m4 + mova [cq+32*13], m5 + mova [cq+32*14], m6 + mova [cq+32*15], m7 + pmaxsd m0, m12, [cq+32*0] + pmaxsd m1, m12, [cq+32*1] + pmaxsd m2, m12, [cq+32*2] + pmaxsd m3, m12, [cq+32*3] + pmaxsd m4, m12, [cq+32*4] + pmaxsd m5, m12, [cq+32*5] + pmaxsd m6, m12, [cq+32*6] + pmaxsd m7, m12, [cq+32*7] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(iadst_8x8_internal_12bpc).pass2_main2 + ret + +INV_TXFM_16X8_FN flipadst, dct, 12 +INV_TXFM_16X8_FN flipadst, adst, 12 +INV_TXFM_16X8_FN flipadst, flipadst, 12 +INV_TXFM_16X8_FN flipadst, identity, 12 + +cglobal iflipadst_16x8_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_20b_min] + vpbroadcastd m14, [clip_20b_max] + jmp m(iflipadst_16x8_internal_10bpc).pass1 +.pass2: + call m(iadst_16x8_internal_12bpc).pass2_main + packssdw m13, m0, [cq+32* 8] + packssdw m12, m1, [cq+32* 9] + packssdw m11, m2, [cq+32*10] + packssdw m10, m3, [cq+32*11] + packssdw m3, m4, [cq+32*12] + packssdw m2, m5, [cq+32*13] + packssdw m1, m6, [cq+32*14] + packssdw m0, m7, [cq+32*15] + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 + call m(idct_16x8_internal_12bpc).write_16x4_start + call m(idct_16x8_internal_10bpc).write_16x4_zero + vpermq m0, m10, q3120 + vpermq m1, m11, q3120 + vpermq m2, m12, q3120 + vpermq m3, m13, q3120 + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET + +INV_TXFM_16X8_FN identity, dct, 12 +INV_TXFM_16X8_FN identity, adst, 12 +INV_TXFM_16X8_FN identity, flipadst, 12 +INV_TXFM_16X8_FN identity, identity, 12 + +cglobal iidentity_16x8_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 + jmp m(iidentity_16x8_internal_10bpc).pass1 +.pass2: + call m(idct_16x8_internal_10bpc).transpose2 + vpbroadcastd m10, [pw_4096] + pmulhrsw m0, m10 + pmulhrsw m1, m10 + pmulhrsw m2, m10 + pmulhrsw m3, m10 + call m(idct_16x8_internal_12bpc).write_16x4_start + call m(idct_16x8_internal_10bpc).write_16x4_zero + jmp m(idct_16x8_internal_10bpc).end2 + +%macro INV_TXFM_16X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth + INV_TXFM_FN %1, %2, %3, 16x16, %4 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_%4bpc] + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, identity, 28 +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst + +cglobal idct_16x16_internal_10bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] +.pass1: + vpbroadcastd m11, [pd_2048] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*4] + sub eobd, 36 + jl .fast + add cq, 32 + call .main + sub cq, 32 + mova m10, [r6-32*4] + mova m9, [r6-32*3] + mova m8, [r6-32*2] + psubd m15, m0, m10 ; out15 + paddd m0, m10 ; out0 + psubd m10, m1, m9 ; out14 + paddd m1, m9 ; out1 + psubd m9, m2, m8 ; out13 + paddd m2, m8 ; out2 + REPX {psrad x, 2}, m0, m1, m2 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6-32*2], m2 + mova m2, [r6-32*1] + mova m1, [r6+32*0] + mova m0, [r6+32*1] + REPX {psrad x, 2}, m9, m10, m15 + psubd m8, m3, m2 ; out12 + paddd m3, m2 ; out3 + psubd m2, m4, m1 ; out11 + paddd m4, m1 ; out4 + psubd m1, m5, m0 ; out10 + paddd m5, m0 ; out5 + REPX {psrad x, 2}, m3, m4, m5 + mova [r6-32*1], m3 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova m4, [r6+32*2] + mova m3, [r6+32*3] + REPX {psrad x, 2}, m1, m2, m8 + psubd m5, m6, m4 ; out9 + paddd m6, m4 ; out6 + psubd m4, m7, m3 ; out8 + paddd m7, m3 ; out7 + REPX {psrad x, 2}, m6, m7, m4, m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + add r6, 32*8 + mova [r6-32*4], m4 + mova [r6-32*3], m5 + mova [r6-32*2], m1 + mova [r6-32*1], m2 + mova [r6+32*0], m8 + mova [r6+32*1], m9 + mova [r6+32*2], m10 + mova [r6+32*3], m15 +.fast: + add r6, 32*8 + call .main + mova m14, [r6-32*4] + mova m13, [r6-32*3] + mova m12, [r6-32*2] + mova m11, [r6-32*1] + mova m10, [r6+32*0] + mova m9, [r6+32*1] + mova m8, [r6+32*2] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r6+32*3] ; out8 + paddd m7, [r6+32*3] ; out7 + sub r6, 32*8 + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call .transpose + lea r6, [pw_5+128] + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] +.end: + call .write_16x16 + RET +ALIGN function_align +.write_16x16: + mova [rsp+gprsize+32*0], m8 + mova [rsp+gprsize+32*1], m9 + mova [rsp+gprsize+32*2], m12 + vpbroadcastd m12, [pw_2048] + pmulhrsw m0, m12 + pmulhrsw m1, m12 + pmulhrsw m2, m12 + pmulhrsw m3, m12 + call m(idct_16x8_internal_10bpc).write_16x4_start +.write_16x16_2: + pmulhrsw m0, m12, m4 + pmulhrsw m1, m12, m5 + pmulhrsw m2, m12, m6 + pmulhrsw m3, m12, m7 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m12, [rsp+gprsize+32*0] + pmulhrsw m1, m12, [rsp+gprsize+32*1] + pmulhrsw m2, m12, m10 + pmulhrsw m3, m12, m11 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m12, [rsp+gprsize+32*2] + pmulhrsw m1, m12, m13 + pmulhrsw m2, m12, m14 + pmulhrsw m3, m12, m15 + jmp m(idct_16x8_internal_10bpc).write_16x4_zero +ALIGN function_align +.transpose: + test eobd, eobd + jl .transpose_fast + packssdw m8, [r6-32*4] + packssdw m9, [r6-32*3] + packssdw m10, [r6-32*2] + packssdw m11, [r6-32*1] + packssdw m12, [r6+32*0] + packssdw m13, [r6+32*1] + packssdw m14, [r6+32*2] + packssdw m15, [r6+32*3] + sub r6, 32*8 + packssdw m0, [r6-32*4] + packssdw m1, [r6-32*3] + packssdw m2, [r6-32*2] + packssdw m3, [r6-32*1] + packssdw m4, [r6+32*0] + packssdw m5, [r6+32*1] + packssdw m6, [r6+32*2] + packssdw m7, [r6+32*3] + mova [r6], m8 + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + punpckhwd m3, m6, m7 + punpcklwd m6, m7 + punpcklwd m7, m4, m5 + punpckhwd m4, m5 + punpckldq m5, m8, m2 + punpckhdq m8, m2 + punpckhdq m2, m0, m1 + punpckldq m0, m1 + punpckhdq m1, m7, m6 + punpckldq m7, m6 + punpckhdq m6, m4, m3 + punpckldq m4, m3 + punpckhqdq m3, m2, m1 + punpcklqdq m2, m1 + punpckhqdq m1, m0, m7 + punpcklqdq m0, m7 + punpcklqdq m7, m8, m6 + punpckhqdq m8, m6 + punpckhqdq m6, m5, m4 + punpcklqdq m5, m4 + mova m4, [r6] + mova [r6], m8 + punpcklwd m8, m4, m9 + punpckhwd m4, m9 + punpcklwd m9, m10, m11 + punpckhwd m10, m11 + punpckhwd m11, m14, m15 + punpcklwd m14, m15 + punpckhwd m15, m12, m13 + punpcklwd m12, m13 + punpckldq m13, m4, m10 + punpckhdq m4, m10 + punpckhdq m10, m8, m9 + punpckldq m8, m9 + punpckhdq m9, m12, m14 + punpckldq m12, m14 + punpckhdq m14, m15, m11 + punpckldq m15, m11 + punpckhqdq m11, m10, m9 + punpcklqdq m10, m9 + punpckhqdq m9, m8, m12 + punpcklqdq m8, m12 + punpcklqdq m12, m13, m15 + punpckhqdq m13, m15 + punpckhqdq m15, m4, m14 + punpcklqdq m14, m4, m14 + vperm2i128 m4, m0, m8, 0x31 + vinserti128 m0, xm8, 1 + vinserti128 m8, m5, xm12, 1 + vperm2i128 m12, m5, 0x13 + vperm2i128 m5, m1, m9, 0x31 + vinserti128 m1, xm9, 1 + vinserti128 m9, m6, xm13, 1 + vperm2i128 m13, m6, 0x13 + vperm2i128 m6, m2, m10, 0x31 + vinserti128 m2, xm10, 1 + vinserti128 m10, m7, xm14, 1 + vperm2i128 m14, m7, 0x13 + vperm2i128 m7, m3, m11, 0x31 + vinserti128 m3, xm11, 1 + mova xm11, [r6] + vinserti128 m11, xm15, 1 + vinserti128 m15, [r6+16], 0 + ret +.transpose_fast: + call m(idct_16x8_internal_10bpc).transpose2 + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + ret +ALIGN function_align +.main: + mova m0, [cq+64* 1] + mova m1, [cq+64* 3] + mova m2, [cq+64* 5] + mova m3, [cq+64* 7] + mova m4, [cq+64* 9] + mova m5, [cq+64*11] + mova m6, [cq+64*13] + mova m7, [cq+64*15] + call m(idct_8x16_internal_10bpc).main_oddhalf + mova m0, [cq+64* 0] + mova m1, [cq+64* 2] + mova m2, [cq+64* 4] + mova m3, [cq+64* 6] + mova m4, [cq+64* 8] + mova m5, [cq+64*10] + mova m6, [cq+64*12] + mova m7, [cq+64*14] + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + psrld m10, m11, 10 ; pd_2 + REPX {paddd x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + ret + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_10bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] +.pass1: + vpbroadcastd m15, [pd_2896] + lea r6, [rsp+32*4] + sub eobd, 36 + jl .fast + add cq, 32 + call .main + sub cq, 32 + vpbroadcastd m8, [pd_5120] + paddd m4, m8 + paddd m6, m8 + paddd m9, m8 + paddd m11, m8 + vpbroadcastd m8, [pd_5119] + psubd m5, m8, m5 + psubd m7, m8, m7 + psubd m10, m8, m10 + psubd m12, m8, m12 + REPX {psrad x, 13}, m4, m5, m6, m7, m9, m10, m11, m12 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + psrld m4, m15, 10 ; pd_2 + paddd m0, m4 + psubd m1, m4, m1 + paddd m2, m4 + psubd m3, m4, m3 + psubd m7, m4, [r6-32*4] + paddd m6, m4, [r6-32*3] + psubd m5, m4, [r6-32*2] + paddd m4, [r6-32*1] + REPX {psrad x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6-32*2], m2 + mova [r6-32*1], m3 + add r6, 32*8 + mova [r6-32*4], m9 + mova [r6-32*3], m10 + mova [r6-32*2], m11 + mova [r6-32*1], m12 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 +.fast: + add r6, 32*8 + call .main + vpbroadcastd m14, [pd_5120] + vpbroadcastd m13, [pd_5119] + psrld m15, 10 ; pd_2 + paddd m0, m15 + psubd m1, m15, m1 + paddd m2, m15 + psubd m3, m15, m3 + paddd m4, m14 + psubd m5, m13, m5 + paddd m6, m14 + psubd m7, m13, m7 + paddd m8, m14, m9 + psubd m9, m13, m10 + paddd m10, m14, m11 + psubd m11, m13, m12 + paddd m12, m15, [r6-32*1] + psubd m13, m15, [r6-32*2] + paddd m14, m15, [r6-32*3] + psubd m15, [r6-32*4] +.pass1_end: + REPX {psrad x, 2 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 13}, m4, m5, m6, m7, m8, m9, m10, m11 + sub r6, 32*8 + jmp tx2q +.pass2: + call m(idct_16x16_internal_10bpc).transpose + lea r6, [pw_5+128] + mova [rsp], m15 + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end + mova [rsp+32*0], m8 + mova [rsp+32*2], m12 + mova [rsp+32*3], m13 + vpbroadcastd m12, [pw_2048] + pxor m13, m13 + psubw m13, m12 + pmulhrsw m0, m12 + pmulhrsw m1, m13, [rsp+32*1] + mova [rsp+32*1], m9 + pmulhrsw m2, m12 + pmulhrsw m3, m13 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m12, m4 + pmulhrsw m1, m13, m5 + pmulhrsw m2, m12, m6 + pmulhrsw m3, m13, m7 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m12, [rsp+32*0] + pmulhrsw m1, m13, [rsp+32*1] + pmulhrsw m2, m12, m10 + pmulhrsw m3, m13, m11 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m12, [rsp+32*2] + pmulhrsw m1, m13, [rsp+32*3] + pmulhrsw m2, m12, m14 + pmulhrsw m3, m13, m15 + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET +ALIGN function_align +.main: + mova m0, [cq+64* 2] + mova m1, [cq+64*13] + mova m2, [cq+64* 6] + mova m3, [cq+64* 9] + mova m4, [cq+64*10] + mova m5, [cq+64* 5] + mova m6, [cq+64*14] + mova m7, [cq+64* 1] + vpbroadcastd m12, [pd_2048] + call m(iadst_16x8_internal_10bpc).main_part1 + mova m0, [cq+64* 0] + mova m1, [cq+64*15] + mova m2, [cq+64* 4] + mova m3, [cq+64*11] + mova m4, [cq+64* 8] + mova m5, [cq+64* 7] + mova m6, [cq+64*12] + mova m7, [cq+64* 3] + jmp m(iadst_16x8_internal_10bpc).main_part2 + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_10bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] +.pass1: + vpbroadcastd m15, [pd_2896] + lea r6, [rsp+32*4] + sub eobd, 36 + jl .fast + add cq, 32 + call m(iadst_16x16_internal_10bpc).main + sub cq, 32 + vpbroadcastd m8, [pd_5120] + paddd m11, m8 + paddd m9, m8 + paddd m6, m8 + paddd m4, m8 + vpbroadcastd m8, [pd_5119] + psubd m12, m8, m12 + psubd m10, m8, m10 + psubd m7, m8, m7 + psubd m5, m8, m5 + REPX {psrad x, 13}, m12, m11, m10, m9, m7, m6, m5, m4 + mova [r6+32*0], m12 + mova [r6+32*1], m11 + mova [r6+32*2], m10 + mova [r6+32*3], m9 + psrld m9, m15, 10 ; pd_2 + psubd m3, m9, m3 + paddd m2, m9 + psubd m1, m9, m1 + paddd m0, m9 + psubd m12, m9, [r6-32*4] + paddd m11, m9, [r6-32*3] + psubd m10, m9, [r6-32*2] + paddd m9, [r6-32*1] + REPX {psrad x, 2 }, m12, m11, m10, m9, m3, m2, m1, m0 + mova [r6-32*4], m12 + mova [r6-32*3], m11 + mova [r6-32*2], m10 + mova [r6-32*1], m9 + add r6, 32*8 + mova [r6-32*4], m7 + mova [r6-32*3], m6 + mova [r6-32*2], m5 + mova [r6-32*1], m4 + mova [r6+32*0], m3 + mova [r6+32*1], m2 + mova [r6+32*2], m1 + mova [r6+32*3], m0 +.fast: + add r6, 32*8 + call m(iadst_16x16_internal_10bpc).main + vpbroadcastd m14, [pd_5120] + vpbroadcastd m13, [pd_5119] + psrld m15, 10 ; pd_2 + psubd m8, m13, m7 + paddd m7, m14, m9 + paddd m9, m14, m6 + psubd m6, m13, m10 + psubd m10, m13, m5 + paddd m5, m14, m11 + paddd m11, m14, m4 + psubd m4, m13, m12 + psubd m12, m15, m3 + paddd m3, m15, [r6-32*1] + paddd m13, m15, m2 + psubd m2, m15, [r6-32*2] + psubd m14, m15, m1 + mova m1, m15 + paddd m15, m0 + psubd m0, m1, [r6-32*4] + paddd m1, [r6-32*3] + jmp m(iadst_16x16_internal_10bpc).pass1_end +.pass2: + call m(idct_16x16_internal_10bpc).transpose + lea r6, [pw_5+128] + mova [rsp], m15 + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end + mova [rsp+32*3], m3 + mova [rsp+32*2], m2 + mova [rsp+32*0], m0 + mova m2, m13 + mova m3, m12 + vpbroadcastd m12, [pw_2048] + pxor m13, m13 + psubw m13, m12 + pmulhrsw m0, m13, m15 + pmulhrsw m1, m12, m14 + pmulhrsw m2, m13 + pmulhrsw m3, m12 + mova m14, m8 + mova m15, m9 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m13, m11 + pmulhrsw m1, m12, m10 + pmulhrsw m2, m13, m15 + pmulhrsw m3, m12, m14 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m13, m7 + pmulhrsw m1, m12, m6 + pmulhrsw m2, m13, m5 + pmulhrsw m3, m12, m4 + call m(idct_16x8_internal_10bpc).write_16x4_zero + pmulhrsw m0, m13, [rsp+32*3] + pmulhrsw m1, m12, [rsp+32*2] + pmulhrsw m2, m13, [rsp+32*1] + pmulhrsw m3, m12, [rsp+32*0] + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET + +INV_TXFM_16X16_FN identity, dct, -92 +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_10bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m15, [pd_5793] + vpbroadcastd m7, [pd_5120] + lea r6, [rsp+32*4] + sub eobd, 36 + jl .fast + mov r3, -32*8*4 +.righthalf: + pmulld m0, m15, [cq+r3+32*33] + pmulld m1, m15, [cq+r3+32*35] + pmulld m2, m15, [cq+r3+32*37] + pmulld m3, m15, [cq+r3+32*39] + add r6, 32*4 + REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {psrad x, 13}, m0, m1, m2, m3 + mova [r6+32*0], m0 + mova [r6+32*1], m1 + mova [r6+32*2], m2 + mova [r6+32*3], m3 + add r3, 32*8 + jl .righthalf +.fast: + pmulld m0, m15, [cq+64* 0] + pmulld m1, m15, [cq+64* 1] + pmulld m2, m15, [cq+64* 2] + pmulld m3, m15, [cq+64* 3] + pmulld m4, m15, [cq+64* 4] + pmulld m5, m15, [cq+64* 5] + pmulld m6, m15, [cq+64* 6] + pmulld m8, m15, [cq+64* 7] + mova [cq], m8 + pmulld m8, m15, [cq+64* 8] + pmulld m9, m15, [cq+64* 9] + pmulld m10, m15, [cq+64*10] + pmulld m11, m15, [cq+64*11] + pmulld m12, m15, [cq+64*12] + pmulld m13, m15, [cq+64*13] + pmulld m14, m15, [cq+64*14] + pmulld m15, [cq+64*15] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6, \ + m8, m9, m10, m11, m12, m13, m14, m15 + paddd m7, [cq] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp tx2q +.pass2: + call m(idct_16x16_internal_10bpc).transpose + + mova [cq+32*0], m15 + mova [cq+32*1], m0 + vpbroadcastd m15, [pw_1697x16] + + REPX {IDTX16 x, 0, 15}, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14 + mova m0, [cq+32*1] + mova [cq+32*1], m1 + IDTX16 0, 1, 15 + mova m1, [cq+32*0] + pmulhrsw m15, m1 + paddsw m1, m1 + paddsw m15, m1 + mova m1, [cq+32*1] + jmp m(idct_16x16_internal_10bpc).end + +INV_TXFM_16X16_FN dct, dct, 0, 12 +INV_TXFM_16X16_FN dct, identity, 28, 12 +INV_TXFM_16X16_FN dct, adst, 0, 12 +INV_TXFM_16X16_FN dct, flipadst, 0, 12 + +cglobal idct_16x16_internal_12bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + jmp m(idct_16x16_internal_10bpc).pass1 +.pass2: + mova [cq+32* 8], m8 + mova [cq+32* 9], m9 + mova [cq+32*10], m10 + mova [cq+32*11], m11 + mova [cq+32*12], m12 + mova [cq+32*13], m13 + mova [cq+32*14], m14 + mova [cq+32*15], m15 + call .pass2_main + packssdw m0, m1 + packssdw m1, m2, m3 + packssdw m2, m4, m5 + packssdw m3, m6, m7 + packssdw m4, m8, m9 + packssdw m5, m10, m11 + packssdw m6, m12, m13 + packssdw m7, m14, m15 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6-32*2], m2 + mova [r6-32*1], m3 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + mova m0, [cq+32* 8] + mova m1, [cq+32* 9] + mova m2, [cq+32*10] + mova m3, [cq+32*11] + mova m4, [cq+32*12] + mova m5, [cq+32*13] + mova m6, [cq+32*14] + mova m7, [cq+32*15] + mov r5, r6 + add r6, 32*16 + call .pass2_main + jmp m(iadst_16x16_internal_12bpc).end +ALIGN function_align +.write_16x16: + mova [rsp+gprsize+32*0], m8 + mova [rsp+gprsize+32*1], m9 + mova [rsp+gprsize+32*2], m12 + vpbroadcastd m12, [pw_16384] + pmulhrsw m0, m12 + pmulhrsw m1, m12 + pmulhrsw m2, m12 + pmulhrsw m3, m12 + call m(idct_16x8_internal_12bpc).write_16x4_start + call m(idct_16x8_internal_10bpc).write_16x4_zero + jmp m(idct_16x16_internal_10bpc).write_16x16_2 +ALIGN function_align +.pass2_main: + call m(idct_8x8_internal_12bpc).transpose_8x8 + mova [cq+32* 0], m0 + mova [cq+32* 1], m2 + mova [cq+32* 2], m4 + mova [cq+32* 3], m6 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + pmaxsd m0, m12, m1 + pmaxsd m1, m12, m3 + pmaxsd m2, m12, m5 + pmaxsd m3, m12, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3 + test eobd, eobd + jge .pass2_slow + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + jmp .pass2_fast +.pass2_slow: + sub r6, 32*8 + mova m8, [r6-32*4] + mova m4, [r6-32*3] + mova m10, [r6-32*2] + mova m5, [r6-32*1] + mova m12, [r6+32*0] + mova m6, [r6+32*1] + mova m14, [r6+32*2] + mova m7, [r6+32*3] + TRANSPOSE_8X8_DWORD 8, 4, 10, 5, 12, 6, 14, 7, 9, 11, 13, 15 + mova [cq+32* 4], m8 + mova [cq+32* 5], m10 + mova [cq+32* 6], m12 + mova [cq+32* 7], m14 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + REPX {pmaxsd x, m12}, m4, m5, m6, m7 + REPX {pminsd x, m13}, m4, m5, m6, m7 +.pass2_fast: + vpbroadcastd m11, [pd_2048] + vpbroadcastd m14, [pd_2896] + call m(idct_8x16_internal_10bpc).main_oddhalf + pmaxsd m0, m12, [cq+32* 0] + pmaxsd m1, m12, [cq+32* 1] + pmaxsd m2, m12, [cq+32* 2] + pmaxsd m3, m12, [cq+32* 3] + REPX {pminsd x, m13}, m0, m1, m2, m3 + test eobd, eobd + jge .pass2_slow2 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + jmp .pass2_fast2 +.pass2_slow2: + pmaxsd m4, m12, [cq+32* 4] + pmaxsd m5, m12, [cq+32* 5] + pmaxsd m6, m12, [cq+32* 6] + pmaxsd m7, m12, [cq+32* 7] + REPX {pminsd x, m13}, m4, m5, m6, m7 +.pass2_fast2: + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + psrad m11, 8 ; pd_8 + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_16x8_internal_10bpc).pass1_rotations + REPX {psrad x, 4}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ret + +INV_TXFM_16X16_FN adst, dct, 0, 12 +INV_TXFM_16X16_FN adst, adst, 0, 12 +INV_TXFM_16X16_FN adst, flipadst, 0, 12 + +cglobal iadst_16x16_internal_12bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_20b_min] + vpbroadcastd m14, [clip_20b_max] + jmp m(iadst_16x16_internal_10bpc).pass1 +.pass2: + call .pass2_part1 + call m(iadst_16x8_internal_10bpc).pass1_rotations + call .pass2_part2 + call m(iadst_16x8_internal_10bpc).pass1_rotations +.pass2_part3: + REPX {psrad x, 4 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 15}, m4, m5, m6, m7, m8, m9, m10, m11 +.end: + packssdw m15, m14 + packssdw m14, m13, m12 + packssdw m13, m11, m10 + packssdw m12, m9, m8 + packssdw m11, m7, m6 + packssdw m10, m5, m4 + packssdw m7, m3, m2 + packssdw m6, m1, m0 + vpblendd m0, m6, [r5-32*4], 0x33 + vpblendd m1, m6, [r5-32*4], 0xcc + vpblendd m2, m7, [r5-32*3], 0x33 + vpblendd m3, m7, [r5-32*3], 0xcc + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + call m(idct_16x8_internal_12bpc).write_16x4_start + call m(idct_16x8_internal_10bpc).write_16x4_zero + vpblendd m0, m10, [r5-32*2], 0x33 + vpblendd m1, m10, [r5-32*2], 0xcc + vpblendd m2, m11, [r5-32*1], 0x33 + vpblendd m3, m11, [r5-32*1], 0xcc + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + call m(idct_16x8_internal_10bpc).write_16x4_zero + vpblendd m0, m12, [r5+32*0], 0x33 + vpblendd m1, m12, [r5+32*0], 0xcc + vpblendd m2, m13, [r5+32*1], 0x33 + vpblendd m3, m13, [r5+32*1], 0xcc + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + call m(idct_16x8_internal_10bpc).write_16x4_zero + vpblendd m0, m14, [r5+32*2], 0x33 + vpblendd m1, m14, [r5+32*2], 0xcc + vpblendd m2, m15, [r5+32*3], 0x33 + vpblendd m3, m15, [r5+32*3], 0xcc + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + call m(idct_16x8_internal_10bpc).write_16x4_zero + RET +ALIGN function_align +.pass2_part1: + mova [cq+32* 8], m8 + mova [cq+32* 9], m9 + mova [cq+32*10], m10 + mova [cq+32*11], m11 + mova [cq+32*12], m12 + mova [cq+32*13], m13 + mova [cq+32*14], m14 + mova [cq+32*15], m15 +.pass2_main: + call m(idct_8x8_internal_12bpc).transpose_8x8 + mova [cq+32* 0], m0 + mova [cq+32* 1], m3 + mova [cq+32* 2], m4 + mova [cq+32* 3], m7 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] + pmaxsd m0, m13, m2 + pmaxsd m2, m13, m6 + pmaxsd m5, m13, m5 + pmaxsd m7, m13, m1 + REPX {pminsd x, m14}, m0, m2, m5, m7 + test eobd, eobd + jge .pass2_slow + pxor m1, m1 + REPX {mova x, m1}, m3, m4, m6 + jmp .pass2_fast +.pass2_slow: + sub r6, 32*8 + mova m8, [r6-32*4] + mova m3, [r6-32*3] + mova m4, [r6-32*2] + mova m11, [r6-32*1] + mova m12, [r6+32*0] + mova m1, [r6+32*1] + mova m6, [r6+32*2] + mova m15, [r6+32*3] + TRANSPOSE_8X8_DWORD 8, 3, 4, 11, 12, 1, 6, 15, 13, 9, 10, 14 + mova [cq+32* 4], m8 + mova [cq+32* 5], m11 + mova [cq+32* 6], m12 + mova [cq+32* 7], m15 + vpbroadcastd m13, [clip_18b_min] + vpbroadcastd m14, [clip_18b_max] + REPX {pmaxsd x, m13}, m1, m3, m4, m6 + REPX {pminsd x, m14}, m1, m3, m4, m6 +.pass2_fast: + vpbroadcastd m12, [pd_2048] + vpbroadcastd m15, [pd_2896] + call m(iadst_16x8_internal_10bpc).main_part1 + pmaxsd m0, m13, [cq+32* 0] ; 0 + pmaxsd m7, m13, [cq+32* 1] ; 3 + pmaxsd m2, m13, [cq+32* 2] ; 4 + pmaxsd m5, m13, [cq+32* 3] ; 7 + REPX {pminsd x, m14}, m0, m2, m5, m7 + test eobd, eobd + jge .pass2_slow2 + pxor m1, m1 + REPX {mova x, m1}, m3, m4, m6 + jmp .pass2_fast2 +.pass2_slow2: + pmaxsd m4, m13, [cq+32* 4] ; 8 + pmaxsd m3, m13, [cq+32* 5] ; 11 + pmaxsd m6, m13, [cq+32* 6] ; 12 + pmaxsd m1, m13, [cq+32* 7] ; 15 + REPX {pminsd x, m14}, m1, m3, m4, m6 +.pass2_fast2: + call m(iadst_16x8_internal_10bpc).main_part2 + vpbroadcastd m14, [pd_17408] + psrld m15, 11 ; pd_1 + psubd m13, m14, m15 ; pd_17407 + pslld m15, 3 ; pd_8 + ret +ALIGN function_align +.pass2_part2: + REPX {psrad x, 4 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 15}, m4, m5, m6, m7, m8, m9, m10, m11 + packssdw m0, m1 + packssdw m1, m2, m3 + packssdw m2, m4, m5 + packssdw m3, m6, m7 + packssdw m4, m8, m9 + packssdw m5, m10, m11 + packssdw m6, m12, m13 + packssdw m7, m14, m15 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6-32*2], m2 + mova [r6-32*1], m3 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + mova m0, [cq+32* 8] + mova m1, [cq+32* 9] + mova m2, [cq+32*10] + mova m3, [cq+32*11] + mova m4, [cq+32*12] + mova m5, [cq+32*13] + mova m6, [cq+32*14] + mova m7, [cq+32*15] + mov r5, r6 + add r6, 32*16 + jmp .pass2_main + +INV_TXFM_16X16_FN flipadst, dct, 0, 12 +INV_TXFM_16X16_FN flipadst, adst, 0, 12 +INV_TXFM_16X16_FN flipadst, flipadst, 0, 12 + +cglobal iflipadst_16x16_internal_12bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m13, [clip_20b_min] + vpbroadcastd m14, [clip_20b_max] + jmp m(iflipadst_16x16_internal_10bpc).pass1 +.pass2: + call m(iadst_16x16_internal_12bpc).pass2_part1 + call m(iflipadst_16x8_internal_10bpc).pass1_rotations + call m(iadst_16x16_internal_12bpc).pass2_part2 + call m(iflipadst_16x8_internal_10bpc).pass1_rotations + jmp m(iadst_16x16_internal_12bpc).pass2_part3 + +INV_TXFM_16X16_FN identity, dct, -92, 12 +INV_TXFM_16X16_FN identity, identity, 0, 12 + +%macro IDTX16_12BPC 1 ; src + pmulld m6, m7, m%1 + paddd m6, m15 + psrad m6, 12 + paddd m6, m%1 + psrad m%1, m6, 1 +%endmacro + +cglobal iidentity_16x16_internal_12bpc, 0, 7, 16, 32*24, dst, stride, c, eob, tx2 + vpbroadcastd m7, [pd_1697] + vpbroadcastd m15, [pd_5120] + lea r6, [rsp+32*4] + sub eobd, 36 + jl .fast + mov r3, -32*8*4 +.righthalf: + mova m10, [cq+r3+32*33] + mova m11, [cq+r3+32*35] + mova m12, [cq+r3+32*37] + mova m13, [cq+r3+32*39] + add r6, 32*4 + pmulld m0, m7, m10 + pmulld m1, m7, m11 + pmulld m2, m7, m12 + pmulld m3, m7, m13 + REPX {paddd x, m15}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 + paddd m0, m10 + paddd m1, m11 + paddd m2, m12 + paddd m3, m13 + REPX {psrad x, 1 }, m0, m1, m2, m3 + mova [r6+32*0], m0 + mova [r6+32*1], m1 + mova [r6+32*2], m2 + mova [r6+32*3], m3 + add r3, 32*8 + jl .righthalf +.fast: + mova m0, [cq+64* 0] + mova m1, [cq+64* 1] + mova m2, [cq+64* 2] + mova m3, [cq+64* 3] + mova m4, [cq+64* 4] + mova m5, [cq+64* 5] + mova m8, [cq+64* 6] + mova m9, [cq+64* 7] + REPX {IDTX16_12BPC x}, 0, 1, 2, 3, 4, 5, 8, 9 + mova [cq+64*0], m8 + mova [cq+64*1], m9 + mova m8, [cq+64* 8] + mova m9, [cq+64* 9] + mova m10, [cq+64*10] + mova m11, [cq+64*11] + mova m12, [cq+64*12] + mova m13, [cq+64*13] + mova m14, [cq+64*14] + REPX {IDTX16_12BPC x}, 8, 9, 10, 11, 12, 13, 14 + mova m6, [cq+64*15] + pmulld m7, m6 + paddd m7, m15 + psrad m7, 12 + paddd m7, m6 + mova m6, [cq+64*0] + psrad m15, m7, 1 + mova m7, [cq+64*1] + jmp tx2q +.pass2: + call m(iidentity_8x16_internal_12bpc).pass2_main + call m(idct_16x16_internal_10bpc).transpose_fast + test eobd, eobd + jl .pass2_fast + mova [cq+32* 8], m0 + mova [cq+32* 9], m1 + mova [cq+32*10], m2 + mova [cq+32*11], m3 + mova [cq+32*12], m4 + mova [cq+32*13], m5 + mova [cq+32*14], m6 + mova [cq+32*15], m7 + mova m8, [r6-32*4] + mova m9, [r6-32*3] + mova m10, [r6-32*2] + mova m11, [r6-32*1] + mova m12, [r6+32*0] + mova m13, [r6+32*1] + mova m14, [r6+32*2] + mova m15, [r6+32*3] + sub r6, 32*8 + mova m0, [r6-32*4] + mova m1, [r6-32*3] + mova m2, [r6-32*2] + mova m3, [r6-32*1] + mova m4, [r6+32*0] + mova m5, [r6+32*1] + mova m6, [r6+32*2] + mova m7, [r6+32*3] + call m(iidentity_8x16_internal_12bpc).pass2_main + call m(idct_16x8_internal_10bpc).transpose2 + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 + mova m12, m4 + mova m13, m5 + mova m14, m6 + mova m15, m7 + mova m0, [cq+32* 8] + mova m1, [cq+32* 9] + mova m2, [cq+32*10] + mova m3, [cq+32*11] + mova m4, [cq+32*12] + mova m5, [cq+32*13] + mova m6, [cq+32*14] + mova m7, [cq+32*15] +.pass2_fast: + call m(idct_16x16_internal_12bpc).write_16x16 + RET + +%macro IDCT32_END 6-7 1 ; in/out1, out2, tmp[1-3], shift, pack + mova m%4, [r6+32*(%1-4)] + mova m%2, [r5+32*(3-%1)] + mova m%5, [r4+32*(%1-4)] + psubd m%3, m%1, m%4 ; idct16 out15 - n + paddd m%1, m%4 ; idct16 out0 + n + pmaxsd m%1, m12 + pmaxsd m%3, m12 + pminsd m%1, m13 + pminsd m%3, m13 + paddd m%1, m11 + paddd m%3, m11 + psubd m%4, m%1, m%2 ; out31 - n + paddd m%1, m%2 ; out0 + n + paddd m%2, m%3, m%5 ; out15 - n + psubd m%3, m%5 ; out16 + n + REPX {psrad x, %6}, m%1, m%3, m%2, m%4 +%if %7 & 1 + packssdw m%1, m%3 ; out0 + n, out16 + n + packssdw m%2, m%4 ; out15 - n, out31 - n +%endif +%endmacro + +cglobal inv_txfm_add_dct_dct_8x32_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 7, 16, 32*12, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vbroadcasti128 m14, [idct32_shuf] + mov r4, cq + call .pass1_main + mova [rsp+32*0], m2 + mova [rsp+32*1], m3 + cmp eobd, 43 + jge .eob43 + pxor m4, m4 + REPX {mova x, m4}, [rsp+32*2], m2, m3, m11 + jmp .pass1_end_fast +.eob43: + lea r6, [rsp+32*8] + mova [r6-32*4], m0 + mova [r6-32*3], m1 + call .pass1_main + mova [rsp+32*2], m2 + cmp eobd, 107 + jge .eob107 + mova m11, m3 + mova m2, m0 + mova m3, m1 + mova m0, [r6-32*4] + mova m1, [r6-32*3] + pxor m4, m4 +.pass1_end_fast: + vpbroadcastd m10, [pw_2048] + lea r6, [deint_shuf+128] + REPX {mova x, m4}, m5, m6, m7 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast + jmp .end +.eob107: + mova [rsp+32*3], m3 + mova [r6-32*2], m0 + mova [r6-32*1], m1 + call .pass1_main + cmp eobd, 171 + jge .eob171 + pshufd m12, m2, q1032 + pshufd m13, m3, q1032 + mova m4, m0 + mova m5, m1 + pxor m6, m6 + REPX {mova x, m6}, m7, m14, m15 + jmp .pass1_end +.eob171: + mova [r6+32*0], m0 + mova [r6+32*1], m1 + mova [r6+32*2], m2 + mova [r6+32*3], m3 + call .pass1_main + pshufd m12, [r6+32*2], q1032 ; out19 out17 + pshufd m13, [r6+32*3], q1032 ; out23 out21 + mova m4, [r6+32*0] ; out16 out18 + mova m5, [r6+32*1] ; out20 out22 + pshufd m14, m2, q1032 ; out27 out25 + pshufd m15, m3, q1032 ; out31 out29 + mova m6, m0 ; out24 out26 + mova m7, m1 ; out28 out30 +.pass1_end: + mova m0, [r6-32*4] ; out0 out2 + mova m1, [r6-32*3] ; out4 out6 + mova m2, [r6-32*2] ; out8 out10 + mova m3, [r6-32*1] ; out12 out14 + lea r6, [deint_shuf+128] + mova m11, [rsp+32*3] ; out13 out15 + vpbroadcastd m10, [pw_2048] + call m(inv_txfm_add_dct_dct_8x32_8bpc).main +.end: ; [rsp+0*32] = m12 + vpbroadcastd m12, [pw_2048] + mov cq, r4 + mova [rsp+32*1], m8 + mova [rsp+32*2], m9 + mova [rsp+32*3], m10 + mova [rsp+32*4], m11 + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4_start + vpermq m0, m2, q3120 + vpermq m1, m3, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m4, q3120 + vpermq m1, m5, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m6, q3120 + vpermq m1, m7, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [rsp+32*1], q3120 + vpermq m1, [rsp+32*2], q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [rsp+32*3], q3120 + vpermq m1, [rsp+32*4], q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [rsp+32*0], q3120 + vpermq m1, m13, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m14, q3120 + vpermq m1, m15, q2031 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 +ALIGN function_align +.pass1_main_part1: + mova m0, [cq+128*0] + mova m1, [cq+128*1] + mova m2, [cq+128*2] + mova m3, [cq+128*3] + mova m4, [cq+128*4] + mova m5, [cq+128*5] + mova m6, [cq+128*6] + mova m7, [cq+128*7] + call m(idct_8x8_internal_10bpc).main + psrld m1, m11, 10 ; pd_2 + REPX {paddd x, m1}, m0, m6, m5, m3 + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + psubd m7, m0, m9 ; out7 + paddd m0, m9 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 + REPX {psrad x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + ret +ALIGN function_align +.pass1_main: + call .pass1_main_part1 + add cq, 32 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + pshufb m0, m14 + pshufb m2, m14 + pshufb m4, m14 + pshufb m6, m14 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + vperm2i128 m1, m0, m2, 0x31 ; 4 6 + vinserti128 m0, xm2, 1 ; 0 2 + vinserti128 m2, m3, xm4, 1 ; 1 3 + vperm2i128 m3, m4, 0x31 ; 5 7 + ret +.main_oddhalf_part1_fast_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_oddhalf_part1_fast: ; lower half zero + vpbroadcastd m7, [pd_4091] + vpbroadcastd m8, [pd_201] + vpbroadcastd m6, [pd_m1380] + vpbroadcastd m9, [pd_3857] + vpbroadcastd m5, [pd_3703] + vpbroadcastd m10, [pd_1751] + vpbroadcastd m4, [pd_m2751] + vpbroadcastd m15, [pd_3035] + pmulld m7, m0 + pmulld m0, m8 + pmulld m6, m1 + pmulld m1, m9 + pmulld m5, m2 + pmulld m2, m10 + pmulld m4, m3 + pmulld m3, m15 + jmp .main_oddhalf_part1_fast2 +.main_oddhalf_part1_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.main_oddhalf_part1: ; in1, in7, in9, in15, in17, in23, in25, in31 + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3035, 2751 ; t17a, t30a +.main_oddhalf_part1_fast2: + REPX {paddd x, m11}, m0, m7, m6, m1, m2, m5, m4, m3 + REPX {psrad x, 12 }, m0, m4, m6, m2, m1, m5, m7, m3 + psubd m8, m0, m4 ; t17 + paddd m0, m4 ; t16 + psubd m4, m6, m2 ; t18 + paddd m6, m2 ; t19 + psubd m2, m1, m5 ; t29 + paddd m1, m5 ; t28 + psubd m5, m7, m3 ; t30 + paddd m7, m3 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + vpbroadcastd m15, [pd_4017] + vpbroadcastd m10, [pd_799] + ITX_MULSUB_2D 5, 8, 3, 9, _, 11, 10, 15 ; t17a, t30a + ITX_MULSUB_2D 2, 4, 3, 9, _, 11, 10, 15, 2 ; t29a, t18a + psubd m3, m0, m6 ; t19a + paddd m0, m6 ; t16a + psubd m6, m7, m1 ; t28a + paddd m7, m1 ; t31a + psubd m1, m5, m4 ; t18 + paddd m5, m4 ; t17 + psubd m4, m8, m2 ; t29 + paddd m8, m2 ; t30 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + vpbroadcastd m15, [pd_3784] + vpbroadcastd m10, [pd_1567] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15 ; t18a, t29a + ITX_MULSUB_2D 6, 3, 2, 9, _, 11, 10, 15 ; t19, t28 + mova [r6-32*4], m0 + mova [r6-32*3], m5 + mova [r6-32*2], m4 + mova [r6-32*1], m6 + mova [r6+32*0], m3 + mova [r6+32*1], m1 + mova [r6+32*2], m8 + mova [r6+32*3], m7 + ret +.main_oddhalf_part2_fast_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_oddhalf_part2_fast: ; lower half zero + vpbroadcastd m7, [pd_m601] + vpbroadcastd m8, [pd_4052] + vpbroadcastd m6, [pd_3973] + vpbroadcastd m9, [pd_995] + vpbroadcastd m5, [pd_m2106] + vpbroadcastd m10, [pd_3513] + vpbroadcastd m4, [pd_3290] + vpbroadcastd m15, [pd_2440] + pmulld m7, m0 + pmulld m0, m8 + pmulld m6, m1 + pmulld m1, m9 + pmulld m5, m2 + pmulld m2, m10 + pmulld m4, m3 + pmulld m3, m15 + jmp .main_oddhalf_part2_fast2 +.main_oddhalf_part2_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.main_oddhalf_part2: ; in3, in5, in11, in13, in19, in21, in27, in29 + ITX_MULSUB_2D 7, 0, 8, 9, 10, _, 4052, 601 ; t23a, t24a + ITX_MULSUB_2D 1, 6, 8, 9, 10, _, 995, 3973 ; t20a, t27a + ITX_MULSUB_2D 5, 2, 8, 9, 10, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 3, 4, 8, 9, 10, _, 2440, 3290 ; t22a, t25a +.main_oddhalf_part2_fast2: + REPX {paddd x, m11}, m0, m7, m6, m1, m2, m5, m4, m3 + REPX {psrad x, 12 }, m0, m4, m6, m2, m1, m5, m7, m3 + psubd m8, m0, m4 ; t25 + paddd m0, m4 ; t24 + psubd m4, m6, m2 ; t26 + paddd m6, m2 ; t27 + psubd m2, m1, m5 ; t21 + paddd m1, m5 ; t20 + psubd m5, m7, m3 ; t22 + paddd m7, m3 ; t23 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + vpbroadcastd m15, [pd_2276] + vpbroadcastd m10, [pd_3406] + ITX_MULSUB_2D 4, 2, 3, 9, _, 11, 10, 15 ; t21a, t26a + ITX_MULSUB_2D 8, 5, 3, 9, _, 11, 10, 15, 2 ; t25a, t22a + psubd m3, m0, m6 ; t27a + paddd m0, m6 ; t24a + psubd m6, m7, m1 ; t20a + paddd m7, m1 ; t23a + psubd m1, m5, m4 ; t21 + paddd m5, m4 ; t22 + psubd m4, m8, m2 ; t26 + paddd m8, m2 ; t25 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + vpbroadcastd m15, [pd_3784] + vpbroadcastd m10, [pd_1567] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15, 2 ; t26a, t21a + ITX_MULSUB_2D 3, 6, 2, 9, _, 11, 10, 15, 2 ; t27, t20 + mova m9, [r6-32*4] ; t16a + mova m10, [r6-32*3] ; t17 + psubd m2, m9, m7 ; t23 + paddd m9, m7 ; t16 + psubd m7, m10, m5 ; t22a + paddd m10, m5 ; t17a + REPX {pmaxsd x, m12}, m9, m10, m2, m7 + REPX {pminsd x, m13}, m9, m10, m2, m7 + mova [r6-32*4], m9 + mova [r6-32*3], m10 + mova m9, [r6-32*2] ; t18a + mova m10, [r6-32*1] ; t19 + psubd m5, m9, m1 ; t21 + paddd m9, m1 ; t18 + psubd m1, m10, m6 ; t20a + paddd m10, m6 ; t19a + REPX {pmaxsd x, m12}, m9, m10, m5, m1 + REPX {pminsd x, m13}, m9, m10, m5, m1 + mova [r6-32*2], m9 + mova [r6-32*1], m10 + mova m9, [r6+32*0] ; t28 + mova m10, [r6+32*1] ; t29a + psubd m6, m9, m3 ; t27a + paddd m9, m3 ; t28a + psubd m3, m10, m4 ; t26 + paddd m10, m4 ; t29 + REPX {pmaxsd x, m12}, m9, m10, m6, m3 + REPX {pminsd x, m13}, m9, m10, m6, m3 + REPX {pmulld x, m14}, m6, m3, m1, m5 + paddd m6, m11 + paddd m3, m11 + psubd m4, m6, m1 ; t20 + paddd m6, m1 ; t27 + psubd m1, m3, m5 ; t21a + paddd m3, m5 ; t26a + REPX {psrad x, 12 }, m4, m1, m3, m6 + mova [r6+32*0], m4 + mova [r6+32*1], m1 + mova m4, [r6+32*2] ; t30 + mova m1, [r6+32*3] ; t31a + psubd m5, m4, m8 ; t25a + paddd m4, m8 ; t30a + psubd m8, m1, m0 ; t24 + paddd m1, m0 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m1 + REPX {pminsd x, m13}, m8, m5, m4, m1 + REPX {pmulld x, m14}, m5, m8, m7, m2 + paddd m5, m11 + paddd m8, m11 + psubd m0, m5, m7 ; t22 + paddd m5, m7 ; t25 + psubd m7, m8, m2 ; t23a + paddd m2, m8 ; t24a + REPX {psrad x, 12 }, m0, m7, m2, m5 + mova [r6+32*2], m0 + mova [r6+32*3], m7 + mov r4, r6 + add r6, 32*8 + mova [r6-32*4], m2 + mova [r6-32*3], m5 + mova [r6-32*2], m3 + mova [r6-32*1], m6 + mova [r6+32*0], m9 + mova [r6+32*1], m10 + mova [r6+32*2], m4 + mova [r6+32*3], m1 + mov r5, r6 + add r6, 32*8 + ret +ALIGN function_align +.main_end: + psrld m11, 10 ; pd_2 + IDCT32_END 0, 15, 8, 9, 10, 2 + IDCT32_END 1, 14, 8, 9, 10, 2 + punpckhwd m8, m0, m1 ; 16 17 + punpcklwd m0, m1 ; 0 1 + punpcklwd m1, m14, m15 ; 14 15 + punpckhwd m14, m15 ; 30 31 + mova [r5+32*3], m8 + mova [r5+32*2], m14 + IDCT32_END 2, 15, 8, 9, 10, 2 + IDCT32_END 3, 14, 8, 9, 10, 2 + punpckhwd m8, m2, m3 ; 18 19 + punpcklwd m2, m3 ; 2 3 + punpcklwd m3, m14, m15 ; 12 13 + punpckhwd m14, m15 ; 28 29 + mova [r5+32*1], m8 + mova [r5+32*0], m14 + IDCT32_END 4, 15, 8, 9, 10, 2 + IDCT32_END 5, 14, 8, 9, 10, 2 + punpckhwd m8, m4, m5 ; 20 21 + punpcklwd m4, m5 ; 4 5 + punpcklwd m5, m14, m15 ; 10 11 + punpckhwd m14, m15 ; 26 27 + mova [r5-32*1], m8 + mova [r5-32*2], m14 + IDCT32_END 6, 15, 8, 9, 10, 2 + IDCT32_END 7, 14, 8, 9, 10, 2 + punpckhwd m8, m6, m7 ; 22 23 + punpcklwd m6, m7 ; 6 7 + punpcklwd m7, m14, m15 ; 8 9 + punpckhwd m14, m15 ; 24 25 + mova [r5-32*3], m8 + mova [r5-32*4], m14 +.transpose: + punpckhdq m15, m3, m1 + punpckldq m3, m1 + punpckhdq m1, m4, m6 + punpckldq m4, m6 + punpckhdq m6, m0, m2 + punpckldq m0, m2 + punpckhdq m2, m7, m5 + punpckldq m7, m5 + punpcklqdq m5, m2, m15 + punpckhqdq m2, m15 + punpckhqdq m15, m7, m3 + punpcklqdq m7, m3 + punpckhqdq m3, m6, m1 + punpcklqdq m6, m1 + punpckhqdq m1, m0, m4 + punpcklqdq m0, m4 + vperm2i128 m4, m0, m7, 0x31 + vinserti128 m0, xm7, 1 + vperm2i128 m7, m3, m2, 0x31 + vinserti128 m3, xm2, 1 + vinserti128 m2, m6, xm5, 1 + vperm2i128 m6, m5, 0x31 + vperm2i128 m5, m1, m15, 0x31 + vinserti128 m1, xm15, 1 + ret + +cglobal inv_txfm_add_identity_identity_8x32_10bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: + vpbroadcastd m5, [pw_5] + pxor m6, m6 + mov r6d, eobd + add eobb, 21 + cmovc eobd, r6d ; 43, 107, 171 -> 64, 128, 192 + lea r6, [strideq*3] + lea r5, [strideq*5] + lea r4, [strideq+r6*2] ; strideq*7 +.loop: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {paddsw x, m5}, m0, m1, m2, m3 + REPX {psraw x, 3 }, m0, m1, m2, m3 + call .main_zero + add cq, 32 + lea dstq, [dstq+strideq*8] + sub eobd, 64 + jge .loop + RET +ALIGN function_align +.main_zero: + REPX {mova [cq+128*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 +.main: + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m4 + punpcklwd m0, m4 + punpckhwd m4, m2, m1 + punpcklwd m2, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + mova xm4, [dstq+strideq*0] + vinserti128 m4, [dstq+strideq*4], 1 + paddw m0, m4 + mova xm4, [dstq+strideq*1] + vinserti128 m4, [dstq+r5 ], 1 + paddw m1, m4 + mova xm4, [dstq+strideq*2] + vinserti128 m4, [dstq+r6*2 ], 1 + paddw m2, m4 + mova xm4, [dstq+r6 ] + vinserti128 m4, [dstq+r4 ], 1 + paddw m3, m4 + REPX {pmaxsw x, m6}, m0, m1, m2, m3 + REPX {pminsw x, m7}, m0, m1, m2, m3 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*4], m0, 1 + mova [dstq+strideq*1], xm1 + vextracti128 [dstq+r5 ], m1, 1 + mova [dstq+strideq*2], xm2 + vextracti128 [dstq+r6*2 ], m2, 1 + mova [dstq+r6 ], xm3 + vextracti128 [dstq+r4 ], m3, 1 + ret + +cglobal inv_txfm_add_dct_dct_8x32_12bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + mov r4, cq + lea r6, [rsp+32*4] + call .pass1_main + cmp eobd, 43 + jge .eob43 + jmp .pass2_fast +.eob43: + call .pass1_main + cmp eobd, 107 + jge .eob107 +.pass2_fast: + mov cq, r4 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + pmaxsd m0, m12, [cq+128*1+ 0] + pmaxsd m1, m12, [cq+128*7+ 0] + pmaxsd m2, m12, [cq+128*1+32] + pmaxsd m3, m12, [cq+128*7+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_fast + pmaxsd m0, m12, [cq+128*3+ 0] + pmaxsd m1, m12, [cq+128*5+ 0] + pmaxsd m2, m12, [cq+128*3+32] + pmaxsd m3, m12, [cq+128*5+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_fast + pmaxsd m0, m12, [cq+128*2+ 0] + pmaxsd m1, m12, [cq+128*6+ 0] + pmaxsd m2, m12, [cq+128*2+32] + pmaxsd m3, m12, [cq+128*6+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + call m(idct_8x16_internal_10bpc).main_oddhalf_fast + pmaxsd m0, m12, [cq+128*0+ 0] + pmaxsd m1, m12, [cq+128*4+ 0] + pmaxsd m2, m12, [cq+128*0+32] + pmaxsd m3, m12, [cq+128*4+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + jmp .pass2_end +.eob107: + call .pass1_main + cmp eobd, 171 + jge .eob171 + jmp .pass2 +.eob171: + call .pass1_main +.pass2: + mov cq, r4 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + pmaxsd m0, m12, [cq+128*1+ 0] + pmaxsd m1, m12, [cq+128*7+ 0] + pmaxsd m2, m12, [cq+128*1+32] + pmaxsd m3, m12, [cq+128*7+32] + pmaxsd m4, m12, [cq+128*1+64] + pmaxsd m5, m12, [cq+128*7+64] + pmaxsd m6, m12, [cq+128*1+96] + pmaxsd m7, m12, [cq+128*7+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1 + pmaxsd m0, m12, [cq+128*3+ 0] + pmaxsd m1, m12, [cq+128*5+ 0] + pmaxsd m2, m12, [cq+128*3+32] + pmaxsd m3, m12, [cq+128*5+32] + pmaxsd m4, m12, [cq+128*3+64] + pmaxsd m5, m12, [cq+128*5+64] + pmaxsd m6, m12, [cq+128*3+96] + pmaxsd m7, m12, [cq+128*5+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2 + pmaxsd m0, m12, [cq+128*2+ 0] + pmaxsd m1, m12, [cq+128*6+ 0] + pmaxsd m2, m12, [cq+128*2+32] + pmaxsd m3, m12, [cq+128*6+32] + pmaxsd m4, m12, [cq+128*2+64] + pmaxsd m5, m12, [cq+128*6+64] + pmaxsd m6, m12, [cq+128*2+96] + pmaxsd m7, m12, [cq+128*6+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x16_internal_10bpc).main_oddhalf + pmaxsd m0, m12, [cq+128*0+ 0] + pmaxsd m1, m12, [cq+128*4+ 0] + pmaxsd m2, m12, [cq+128*0+32] + pmaxsd m3, m12, [cq+128*4+32] + pmaxsd m4, m12, [cq+128*0+64] + pmaxsd m5, m12, [cq+128*4+64] + pmaxsd m6, m12, [cq+128*0+96] + pmaxsd m7, m12, [cq+128*4+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf +.pass2_end: + psrld m11, 8 ; pd_8 + IDCT32_END 0, 15, 8, 9, 10, 4 + IDCT32_END 1, 14, 8, 9, 10, 4 + punpckhqdq m8, m0, m1 ; 16 17 (interleaved) + punpcklqdq m0, m1 ; 0 1 (interleaved) + punpcklqdq m1, m14, m15 ; 14 15 (interleaved) + punpckhqdq m14, m15 ; 30 31 (interleaved) + mova [r5+32*3], m8 + mova [r5+32*2], m14 + IDCT32_END 2, 15, 8, 9, 10, 4 + IDCT32_END 3, 14, 8, 9, 10, 4 + punpckhqdq m8, m2, m3 ; 18 19 (interleaved) + punpcklqdq m2, m3 ; 2 3 (interleaved) + punpcklqdq m3, m14, m15 ; 12 13 (interleaved) + punpckhqdq m14, m15 ; 28 29 (interleaved) + mova [r5+32*1], m8 + mova [r5+32*0], m14 + IDCT32_END 4, 15, 8, 9, 10, 4 + IDCT32_END 5, 14, 8, 9, 10, 4 + punpckhqdq m8, m4, m5 ; 20 21 (interleaved) + punpcklqdq m4, m5 ; 4 5 (interleaved) + punpcklqdq m5, m14, m15 ; 10 11 (interleaved) + punpckhqdq m14, m15 ; 26 27 (interleaved) + mova [r5-32*1], m8 + mova [r5-32*2], m14 + IDCT32_END 6, 15, 8, 9, 10, 4 + IDCT32_END 7, 14, 8, 9, 10, 4 + punpckhqdq m8, m6, m7 ; 22 23 (interleaved) + punpcklqdq m6, m7 ; 6 7 (interleaved) + punpcklqdq m7, m14, m15 ; 8 9 (interleaved) + punpckhqdq m14, m15 ; 24 25 (interleaved) + mova [r5-32*3], m8 + mova [r5-32*4], m14 + mova m15, m1 +.end: + vpermq m0, m0, q3120 + vpermq m1, m2, q3120 + call m(idct_8x8_internal_12bpc).write_8x4_start + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m4, q3120 + vpermq m1, m6, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m7, q3120 + vpermq m1, m5, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m3, q3120 + vpermq m1, m15, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5+32*3], q3120 + vpermq m1, [r5+32*1], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5-32*1], q3120 + vpermq m1, [r5-32*3], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5-32*4], q3120 + vpermq m1, [r5-32*2], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5+32*0], q3120 + vpermq m1, [r5+32*2], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_12bpc] + mov [cq], eobd ; 0 + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 +ALIGN function_align +.pass1_main: + call m(inv_txfm_add_dct_dct_8x32_10bpc).pass1_main_part1 + TRANSPOSE_8X8_DWORD 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15 + mova [cq+128*0], m0 + mova [cq+128*1], m1 + mova [cq+128*2], m2 + mova [cq+128*3], m3 + mova [cq+128*4], m4 + mova [cq+128*5], m5 + mova [cq+128*6], m6 + mova [cq+128*7], m7 + add cq, 32 + ret +ALIGN function_align +.main_end: + psrld m11, 10 ; pd_2 + IDCT32_END 0, 15, 8, 9, 10, 2, 0 + mova [cq+32*16], m8 + mova [cq+32*31], m9 + IDCT32_END 1, 14, 8, 9, 10, 2, 0 + mova [cq+32*17], m8 + mova [cq+32*30], m9 + mova [cq+32*14], m14 + IDCT32_END 2, 14, 8, 9, 10, 2, 0 + mova [cq+32*18], m8 + mova [cq+32*29], m9 + mova [cq+32*13], m14 + IDCT32_END 3, 14, 8, 9, 10, 2, 0 + mova [cq+32*19], m8 + mova [cq+32*28], m9 + mova [cq+32*12], m14 + IDCT32_END 4, 14, 8, 9, 10, 2, 0 + mova [cq+32*20], m8 + mova [cq+32*27], m9 + mova [cq+32* 0], m0 + mova [cq+32* 1], m1 + mova [cq+32* 2], m2 + IDCT32_END 5, 10, 0, 1, 2, 2, 0 + mova [cq+32*21], m0 + mova [cq+32*26], m1 + IDCT32_END 6, 9, 0, 1, 2, 2, 0 + mova [cq+32*22], m0 + mova [cq+32*25], m1 + IDCT32_END 7, 8, 0, 1, 2, 2, 0 + mova [cq+32*23], m0 + mova [cq+32*24], m1 + mova m0, [cq+32* 0] + mova m1, [cq+32* 1] + mova m2, [cq+32* 2] + mova m11, m14 + mova m12, [cq+32*12] + mova m13, [cq+32*13] + mova m14, [cq+32*14] + ret + +cglobal inv_txfm_add_identity_identity_8x32_12bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_8x32_10bpc).pass1 + +cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jnz .full + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 8 +.dconly: + add r6d, 640 + sar r6d, 10 +.dconly2: + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + movd xm0, r6d + paddsw xm0, xm3 + vpbroadcastw m0, xm0 +.dconly_loop: + paddsw m1, m0, [dstq+32*0] + paddsw m2, m0, [dstq+32*1] + psubusw m1, m3 + psubusw m2, m3 + mova [dstq+32*0], m1 + mova [dstq+32*1], m2 + add dstq, strideq + dec r3d + jg .dconly_loop + RET +.full: + PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob + lea r6, [rsp+32*4] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + call .pass1 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_end + lea r6, [deint_shuf+128] + vpbroadcastd m11, [pw_2048] + mov r4, dstq + call .pass2 + mova m0, [r5+32*3] ; 16 17 + mova m1, [r5+32*2] ; 30 31 + mova m2, [r5+32*1] ; 18 19 + mova m3, [r5+32*0] ; 28 29 + mova m4, [r5-32*1] ; 20 21 + mova m5, [r5-32*2] ; 26 27 + mova m6, [r5-32*3] ; 22 23 + mova m7, [r5-32*4] ; 24 25 + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + lea dstq, [r4+32] + call .pass2 + RET +ALIGN function_align +.pass2: + call m(idct_16x8_internal_8bpc).main + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m11, m4 + pmulhrsw m1, m11, m5 + pmulhrsw m2, m11, m6 + pmulhrsw m3, m11, m7 + jmp m(idct_16x8_internal_10bpc).write_16x4_zero +ALIGN function_align +.pass1: + mova m0, [cq+32* 1] + mova m1, [cq+32* 7] + mova m2, [cq+32* 9] + mova m3, [cq+32*15] + mova m4, [cq+32*17] + mova m5, [cq+32*23] + mova m6, [cq+32*25] + mova m7, [cq+32*31] + vpbroadcastd m11, [pd_2048] + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1 + mova m0, [cq+32* 3] + mova m1, [cq+32* 5] + mova m2, [cq+32*11] + mova m3, [cq+32*13] + mova m4, [cq+32*19] + mova m5, [cq+32*21] + mova m6, [cq+32*27] + mova m7, [cq+32*29] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2 + mova m0, [cq+32* 2] + mova m1, [cq+32* 6] + mova m2, [cq+32*10] + mova m3, [cq+32*14] + mova m4, [cq+32*18] + mova m5, [cq+32*22] + mova m6, [cq+32*26] + mova m7, [cq+32*30] + call m(idct_8x16_internal_10bpc).main_oddhalf + mova m0, [cq+32* 0] + mova m1, [cq+32* 4] + mova m2, [cq+32* 8] + mova m3, [cq+32*12] + mova m4, [cq+32*16] + mova m5, [cq+32*20] + mova m6, [cq+32*24] + mova m7, [cq+32*28] + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + ret + +cglobal inv_txfm_add_identity_identity_32x8_10bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: + vpbroadcastd m5, [pw_4096] + pxor m6, m6 + mov r6d, eobd + add eobb, 21 + cmovc eobd, r6d + lea r6, [strideq*3] + lea r5, [strideq*5] + lea r4, [strideq+r6*2] ; strideq*7 +.loop: + mova m0, [cq+32*0] + packssdw m0, [cq+32*1] + mova m1, [cq+32*2] + packssdw m1, [cq+32*3] + REPX {mova [cq+32*x], m6}, 0, 1, 2, 3 + add cq, 32*8 + mova m2, [cq-32*4] + packssdw m2, [cq-32*3] + mova m3, [cq-32*2] + packssdw m3, [cq-32*1] + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + REPX {mova [cq+32*x], m6}, -4, -3, -2, -1 + call m(inv_txfm_add_identity_identity_8x32_10bpc).main + add dstq, 16 + sub eobd, 64 + jge .loop + RET + +cglobal inv_txfm_add_dct_dct_32x8_12bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jnz .full + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_12bpc] + mov [cq], eobd ; 0 + or r3d, 8 + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly +.full: + PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob + lea r6, [rsp+32*4] + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + call m(inv_txfm_add_dct_dct_32x8_10bpc).pass1 + call m(inv_txfm_add_dct_dct_8x32_12bpc).main_end + mov r4, dstq + call m(idct_16x8_internal_12bpc).pass2_main + mova m0, [cq+32* 0] ; 16 + mova m1, [cq+32* 1] ; 17 + mova m2, [cq+32* 2] ; 18 + mova m3, [cq+32* 3] ; 19 + mova m4, [cq+32* 4] ; 20 + mova m5, [cq+32* 5] ; 21 + mova m6, [cq+32* 6] ; 22 + mova m7, [cq+32* 7] ; 23 + mova m8, [cq+32* 8] ; 24 + mova m9, [cq+32* 9] ; 25 + mova m10, [cq+32*10] ; 26 + mova m11, [cq+32*11] ; 27 + mova m12, [cq+32*12] ; 28 + mova m13, [cq+32*13] ; 29 + mova m14, [cq+32*14] ; 30 + mova m15, [cq+32*15] ; 31 + lea dstq, [r4+32] + call m(idct_16x8_internal_12bpc).pass2_main + RET + +cglobal inv_txfm_add_identity_identity_32x8_12bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_32x8_10bpc).pass1 + +%macro IDCT32_PASS2_END 6 ; coefs[1-2], tmp[1-2], offset[1-2] + mova m%4, [%2] + paddsw m%3, m%1, m%4 + psubsw m%1, m%4 +%if %1 == 0 + pxor m6, m6 +%endif + pmulhrsw m%3, m15 + pmulhrsw m%1, m15 + paddw m%3, [dstq+%5] + paddw m%1, [r2+%6] + pmaxsw m%3, m6 + pmaxsw m%1, m6 + pminsw m%3, m7 + pminsw m%1, m7 + mova [dstq+%5], m%3 + mova [r2+%6], m%1 +%endmacro + +cglobal inv_txfm_add_dct_dct_16x32_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 8, 16, 32*36, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*16] + lea r4, [r6+32*8] + lea r5, [r6+32*16] + call .main + sub eobd, 44 + jge .eob44 + vperm2i128 m2, m0, m3, 0x31 ; 5 + vinserti128 m0, xm3, 1 ; 1 + vperm2i128 m3, m1, m4, 0x31 ; 7 + vinserti128 m1, xm4, 1 ; 3 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + REPX {mova [r6+32*x], m4}, 0, 1, 2, 3 + jmp .fast +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 32 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 +.eob44: + mova [r4+16*0], xm0 + mova [r4+16*1], xm3 + mova [r4+16*2], xm1 + mova [r4+16*3], xm4 + vextracti128 [r4+16*4], m0, 1 + vextracti128 [r4+16*5], m3, 1 + vextracti128 [r4+16*6], m1, 1 + vextracti128 [r4+16*7], m4, 1 + call .main + sub eobd, 107 + jge .eob151 + vperm2i128 m7, m1, m4, 0x31 ; 15 + vinserti128 m5, m1, xm4, 1 ; 11 + vperm2i128 m6, m0, m3, 0x31 ; 13 + vinserti128 m4, m0, xm3, 1 ; 9 + mova m0, [r4+32*0] + mova m1, [r4+32*1] + mova m2, [r4+32*2] + mova m3, [r4+32*3] +.fast: + lea r6, [pw_5+128] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + jmp .idct16 +.eob151: + mova [r4-16*8], xm0 + mova [r4-16*7], xm3 + mova [r4-16*6], xm1 + mova [r4-16*5], xm4 + vextracti128 [r4-16*4], m0, 1 + vextracti128 [r4-16*3], m3, 1 + vextracti128 [r4-16*2], m1, 1 + vextracti128 [r4-16*1], m4, 1 + call .main + sub eobd, 128 + jge .eob279 + vperm2i128 m10, m0, m3, 0x31 ; 21 + vinserti128 m8, m0, xm3, 1 ; 17 + vperm2i128 m11, m1, m4, 0x31 ; 23 + vinserti128 m9, m1, xm4, 1 ; 19 + pxor m12, m12 + REPX {mova x, m12}, m13, m14, m15 + REPX {mova [r6+32*x], m12}, 0, 1, 2, 3 + jmp .full +.eob279: + mova [r5+16*0], xm0 + mova [r5+16*1], xm3 + mova [r5+16*2], xm1 + mova [r5+16*3], xm4 + vextracti128 [r5+16*4], m0, 1 + vextracti128 [r5+16*5], m3, 1 + vextracti128 [r5+16*6], m1, 1 + vextracti128 [r5+16*7], m4, 1 + call .main + vperm2i128 m14, m0, m3, 0x31 ; 29 + vinserti128 m12, m0, xm3, 1 ; 25 + vperm2i128 m15, m1, m4, 0x31 ; 31 + vinserti128 m13, m1, xm4, 1 ; 27 + mova m8, [r5+32*0] + mova m9, [r5+32*1] + mova m10, [r5+32*2] + mova m11, [r5+32*3] +.full: + mova m0, [r4+32*0] + mova m1, [r4+32*1] + mova m2, [r4+32*2] + mova m3, [r4+32*3] + mova m4, [r4-32*4] + mova m5, [r4-32*3] + mova m6, [r4-32*2] + mova m7, [r4-32*1] + lea r6, [pw_5 + 128] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + lea r3, [rsp+32*8] + mova m8, [r3+32*0] + mova m9, [r3+32*1] + mova m10, [r3+32*2] + mova m11, [r3+32*3] + mova m12, [r3-32*4] + mova m13, [r3-32*3] + mova m14, [r3-32*2] + mova m15, [r3-32*1] +.idct16: + lea r3, [rsp+32*16] + mova m0, [r3+32*0] + mova m1, [r3+32*1] + mova m2, [r3+32*2] + mova m3, [r3+32*3] + mova m4, [r3-32*4] + mova m5, [r3-32*3] + mova m6, [r3-32*2] + mova m7, [r3-32*1] + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq + call .pass2_end + RET +ALIGN function_align +.main: + pmulld m0, m14, [cq+128* 1] + pmulld m1, m14, [cq+128* 3] + pmulld m2, m14, [cq+128* 5] + pmulld m3, m14, [cq+128* 7] + pmulld m4, m14, [cq+128* 9] + pmulld m5, m14, [cq+128*11] + pmulld m6, m14, [cq+128*13] + pmulld m7, m14, [cq+128*15] + call m(idct_8x16_internal_10bpc).main_oddhalf_rect2 + pmulld m0, m14, [cq+128* 0] + pmulld m1, m14, [cq+128* 2] + pmulld m2, m14, [cq+128* 4] + pmulld m3, m14, [cq+128* 6] + pmulld m4, m14, [cq+128* 8] + pmulld m5, m14, [cq+128*10] + pmulld m6, m14, [cq+128*12] + pmulld m7, m14, [cq+128*14] + call m(idct_8x8_internal_10bpc).main_rect2 + call m(idct_8x16_internal_10bpc).main_evenhalf + psrld m15, m11, 11 ; pd_1 + mova m8, [r6-32*4] + mova m9, [r6-32*3] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m10, m0, m8 ; out15 + paddd m0, m8 ; out0 + mova m8, [r6-32*2] + paddd m15, m1, m9 ; out1 + psubd m1, m9 ; out14 + mova m9, [r6-32*1] + REPX {psrad x, 1}, m0, m15, m10, m1 + packssdw m0, m15 + packssdw m1, m10 + psubd m10, m2, m8 ; out13 + paddd m2, m8 ; out2 + mova m8, [r6+32*0] + paddd m15, m3, m9 ; out3 + psubd m3, m9 ; out12 + mova m9, [r6+32*1] + REPX {psrad x, 1}, m2, m15, m10, m3 + packssdw m2, m15 + packssdw m3, m10 + psubd m10, m4, m8 ; out11 + paddd m4, m8 ; out4 + mova m8, [r6+32*2] + paddd m15, m5, m9 ; out5 + psubd m5, m9 ; out10 + mova m9, [r6+32*3] + REPX {psrad x, 1}, m4, m10, m15, m5 + packssdw m4, m15 + packssdw m5, m10 + psubd m10, m6, m8 ; out9 + paddd m6, m8 ; out6 + paddd m15, m7, m9 ; out7 + psubd m7, m9 ; out8 + REPX {psrad x, 1}, m6, m10, m15, m7 + packssdw m6, m15 + packssdw m7, m10 + punpckhwd m8, m0, m2 + punpcklwd m0, m2 + punpckhwd m2, m3, m1 + punpcklwd m3, m1 + punpckhwd m1, m4, m6 + punpcklwd m4, m6 + punpcklwd m6, m7, m5 + punpckhwd m7, m5 + pxor m5, m5 + mov r7d, 128*13 +.main_zero_loop: + mova [cq+r7-128*1], m5 + mova [cq+r7+128*0], m5 + mova [cq+r7+128*1], m5 + mova [cq+r7+128*2], m5 + sub r7d, 128*4 + jg .main_zero_loop + add cq, 32 + punpcklwd m5, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m1 + punpckhwd m4, m1 + punpckhwd m1, m0, m8 + punpcklwd m0, m8 + punpckhwd m8, m6, m7 + punpcklwd m6, m7 + punpcklqdq m7, m1, m4 + punpckhqdq m1, m4 + punpckhqdq m4, m8, m3 + punpcklqdq m8, m3 + punpckhqdq m3, m6, m5 + punpcklqdq m6, m5 + punpcklqdq m5, m0, m2 + punpckhqdq m0, m2 + mova [r6+16*0], xm5 + mova [r6+16*1], xm6 + mova [r6+16*2], xm7 + mova [r6+16*3], xm8 + vextracti128 [r6+16*4], m5, 1 + vextracti128 [r6+16*5], m6, 1 + vextracti128 [r6+16*6], m7, 1 + vextracti128 [r6+16*7], m8, 1 + sub r6, 32*4 + ret +ALIGN function_align +.pass2_end: + mova [rsp+gprsize+32*0], m6 + mova [rsp+gprsize+32*2], m7 + mova [rsp+gprsize+32*3], m15 + vpbroadcastd m15, [pw_2048] + vpbroadcastd m7, [pixel_10bpc_max] + IDCT32_PASS2_END 0, r5+32*3, 1, 6, strideq*0, r3*4 + IDCT32_PASS2_END 4, r5-32*1, 0, 1, strideq*4, strideq*8 + IDCT32_PASS2_END 8, r4+32*3, 0, 4, strideq*8, strideq*4 + IDCT32_PASS2_END 12, r4-32*1, 0, 4, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + mova m1, [rsp+gprsize+32*1] + IDCT32_PASS2_END 1, r5+32*2, 0, 4, strideq*0, r3*4 + IDCT32_PASS2_END 5, r5-32*2, 0, 4, strideq*4, strideq*8 + IDCT32_PASS2_END 9, r4+32*2, 0, 4, strideq*8, strideq*4 + IDCT32_PASS2_END 13, r4-32*2, 0, 4, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + mova m1, [rsp+gprsize+32*0] + IDCT32_PASS2_END 2, r5+32*1, 0, 4, strideq*0, r3*4 + IDCT32_PASS2_END 1, r5-32*3, 0, 4, strideq*4, strideq*8 + IDCT32_PASS2_END 10, r4+32*1, 0, 4, strideq*8, strideq*4 + IDCT32_PASS2_END 14, r4-32*3, 0, 4, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + mova m1, [rsp+gprsize+32*2] + mova m2, [rsp+gprsize+32*3] + IDCT32_PASS2_END 3, r5+32*0, 0, 4, strideq*0, r3*4 + IDCT32_PASS2_END 1, r5-32*4, 0, 4, strideq*4, strideq*8 + IDCT32_PASS2_END 11, r4+32*0, 0, 4, strideq*8, strideq*4 + IDCT32_PASS2_END 2, r4-32*4, 0, 4, r3*4, strideq*0 + ret + +cglobal inv_txfm_add_identity_identity_16x32_10bpc, 4, 7, 12, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: + vpbroadcastd m8, [pw_2896x8] + vpbroadcastd m9, [pw_1697x16] + vpbroadcastd m11, [pw_8192] + lea r6, [strideq*5] + pxor m6, m6 + paddw m10, m11, m11 ; pw_16384 + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + add cq, 128*8 + lea dstq, [r5+16] + call .main + sub cq, 128*8-32 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 107 ; eob < 143 + jl .ret + add cq, 128*8 + lea dstq, [r5+16] + call .main + sub cq, 128*8-32 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 271 + jl .ret + add cq, 128*8 + lea dstq, [r5+16] + call .main + sub cq, 128*8-32 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 399 + jl .ret + add cq, 128*8 + lea dstq, [r5+16] + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 + REPX {IDTX16 x, 4, 9, 10}, 0, 1, 2, 3 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 + REPX {mova [cq+128*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 +.main2: + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m4 + punpcklwd m0, m4 + punpcklwd m4, m2, m1 + punpckhwd m2, m1 + punpckhqdq m1, m0, m4 + punpcklqdq m0, m4 + call m(iidentity_8x8_internal_10bpc).write_2x8x2 + punpcklqdq m0, m3, m2 + punpckhqdq m1, m3, m2 + jmp m(iidentity_8x8_internal_10bpc).write_2x8x2 + +cglobal inv_txfm_add_identity_identity_16x32_12bpc, 4, 7, 12, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).pass1 + +cglobal inv_txfm_add_dct_dct_32x16_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 8, 16, 32*40, dst, stride, c, eob +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + lea r6, [rsp+32*4] + call .main + cmp eobd, 36 + jge .full + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, [rsp] + lea r6, [pw_5+128] + mov r7, dstq + call m(idct_16x16_internal_8bpc).main + call .write_16x16 + mova m0, [r5+32*3] + mova m1, [r5+32*2] + mova m2, [r5+32*1] + mova m3, [r5+32*0] + mova m4, [r5-32*1] + mova m5, [r5-32*2] + mova m6, [r5-32*3] + mova m7, [r5-32*4] + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, [rsp] + jmp .end +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 +.full: + add cq, 32 + mova [r4+32*3], m0 + mova [r4+32*2], m1 + mova [r4+32*1], m2 + mova [r4+32*0], m3 + mova [r4-32*1], m4 + mova [r4-32*2], m5 + mova [r4-32*3], m6 + mova [r4-32*4], m7 + call .main + sub r4, 32*16 ; topleft 16x8 + call .transpose_16x16 + lea r6, [pw_5+128] + mov r7, dstq + call m(idct_16x16_internal_8bpc).main + call .write_16x16 + mova m0, [r5+32*3] + mova m1, [r5+32*2] + mova m2, [r5+32*1] + mova m3, [r5+32*0] + mova m4, [r5-32*1] + mova m5, [r5-32*2] + mova m6, [r5-32*3] + mova m7, [r5-32*4] + add r4, 32*8 ; bottomleft 16x8 + call .transpose_16x16 +.end: + lea dstq, [r7+32] + call m(idct_16x16_internal_8bpc).main + call .write_16x16 + RET +ALIGN function_align +.transpose_16x16: + punpckhdq m8, m3, m1 + punpckldq m3, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckhdq m2, m7, m5 + punpckldq m7, m5 + punpckhdq m5, m4, m6 + punpckldq m4, m6 + punpckhqdq m6, m0, m4 + punpcklqdq m0, m4 + punpckhqdq m4, m1, m5 + punpcklqdq m1, m5 + punpckhqdq m5, m7, m3 + punpcklqdq m7, m3 + punpckhqdq m3, m2, m8 + punpcklqdq m2, m8 + vinserti128 m8, m0, xm7, 1 + vperm2i128 m12, m0, m7, 0x31 + vinserti128 m9, m6, xm5, 1 + vperm2i128 m13, m6, m5, 0x31 + vinserti128 m10, m1, xm2, 1 + vperm2i128 m14, m1, m2, 0x31 + vinserti128 m11, m4, xm3, 1 + vperm2i128 m15, m4, m3, 0x31 + mova m0, [r4+32*3] + mova m1, [r4+32*2] + mova m2, [r4+32*1] + mova m3, [r4+32*0] + mova m4, [r4-32*1] + mova m5, [r4-32*2] + mova m6, [r4-32*3] + mova m7, [r4-32*4] + mova [rsp+gprsize], m15 + jmp m(inv_txfm_add_dct_dct_8x32_10bpc).transpose +ALIGN function_align +.main: + vpbroadcastd m14, [pd_2896] + vpbroadcastd m11, [pd_2048] + pmulld m0, m14, [cq+64* 1] + pmulld m1, m14, [cq+64* 7] + pmulld m2, m14, [cq+64* 9] + pmulld m3, m14, [cq+64*15] + pmulld m4, m14, [cq+64*17] + pmulld m5, m14, [cq+64*23] + pmulld m6, m14, [cq+64*25] + pmulld m7, m14, [cq+64*31] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_rect2 + pmulld m0, m14, [cq+64* 3] + pmulld m1, m14, [cq+64* 5] + pmulld m2, m14, [cq+64*11] + pmulld m3, m14, [cq+64*13] + pmulld m4, m14, [cq+64*19] + pmulld m5, m14, [cq+64*21] + pmulld m6, m14, [cq+64*27] + pmulld m7, m14, [cq+64*29] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_rect2 + pmulld m0, m14, [cq+64* 2] + pmulld m1, m14, [cq+64* 6] + pmulld m2, m14, [cq+64*10] + pmulld m3, m14, [cq+64*14] + pmulld m4, m14, [cq+64*18] + pmulld m5, m14, [cq+64*22] + pmulld m6, m14, [cq+64*26] + pmulld m7, m14, [cq+64*30] + call m(idct_8x16_internal_10bpc).main_oddhalf_rect2 + pmulld m0, m14, [cq+64* 0] + pmulld m1, m14, [cq+64* 4] + pmulld m2, m14, [cq+64* 8] + pmulld m3, m14, [cq+64*12] + pmulld m4, m14, [cq+64*16] + pmulld m5, m14, [cq+64*20] + pmulld m6, m14, [cq+64*24] + pmulld m7, m14, [cq+64*28] + call m(idct_8x8_internal_10bpc).main_rect2 + call m(idct_8x16_internal_10bpc).main_evenhalf + pxor m8, m8 + mov r7d, 64*30 +.main_zero_loop: + mova [cq+r7-64*2], m8 + mova [cq+r7-64*1], m8 + mova [cq+r7+64*0], m8 + mova [cq+r7+64*1], m8 + sub r7d, 64*4 + jg .main_zero_loop +.main_end: + psrld m11, 11 ; pd_1 + IDCT32_END 0, 15, 8, 9, 10, 1 + IDCT32_END 1, 14, 8, 9, 10, 1 + punpckhwd m8, m0, m1 ; 16 17 + punpcklwd m0, m1 ; 0 1 + punpcklwd m1, m14, m15 ; 14 15 + punpckhwd m14, m15 ; 30 31 + mova [r5+32*3], m8 + mova [r5+32*2], m14 + IDCT32_END 2, 15, 8, 9, 10, 1 + IDCT32_END 3, 14, 8, 9, 10, 1 + punpckhwd m8, m2, m3 ; 18 19 + punpcklwd m2, m3 ; 2 3 + punpcklwd m3, m14, m15 ; 12 13 + punpckhwd m14, m15 ; 28 29 + mova [r5+32*1], m8 + mova [r5+32*0], m14 + IDCT32_END 4, 15, 8, 9, 10, 1 + IDCT32_END 5, 14, 8, 9, 10, 1 + punpckhwd m8, m4, m5 ; 20 21 + punpcklwd m4, m5 ; 4 5 + punpcklwd m5, m14, m15 ; 10 11 + punpckhwd m14, m15 ; 26 27 + mova [r5-32*1], m8 + mova [r5-32*2], m14 + IDCT32_END 6, 15, 8, 9, 10, 1 + IDCT32_END 7, 14, 8, 9, 10, 1 + punpckhwd m8, m6, m7 ; 22 23 + punpcklwd m6, m7 ; 6 7 + punpcklwd m7, m14, m15 ; 8 9 + punpckhwd m14, m15 ; 24 25 + mova [r5-32*3], m8 + mova [r5-32*4], m14 + ret +ALIGN function_align +.write_16x16: + mova m1, [rsp+gprsize+32*1] + mova [rsp+gprsize+32*0], m8 + mova [rsp+gprsize+32*1], m9 + mova [rsp+gprsize+32*2], m12 + vpbroadcastd m12, [pw_2048] + vpbroadcastd m9, [pixel_10bpc_max] + lea r3, [strideq*3] + pxor m8, m8 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + pmulhrsw m2, m12 + pmulhrsw m3, m12 + call m(idct_16x8_internal_10bpc).write_16x4 + pmulhrsw m0, m12, m4 + pmulhrsw m1, m12, m5 + pmulhrsw m2, m12, m6 + pmulhrsw m3, m12, m7 + call m(idct_16x8_internal_10bpc).write_16x4 + pmulhrsw m0, m12, [rsp+gprsize+32*0] + pmulhrsw m1, m12, [rsp+gprsize+32*1] + pmulhrsw m2, m12, m10 + pmulhrsw m3, m12, m11 + call m(idct_16x8_internal_10bpc).write_16x4 + pmulhrsw m0, m12, [rsp+gprsize+32*2] + pmulhrsw m1, m12, m13 + pmulhrsw m2, m12, m14 + pmulhrsw m3, m12, m15 + jmp m(idct_16x8_internal_10bpc).write_16x4 + +cglobal inv_txfm_add_identity_identity_32x16_10bpc, 4, 7, 11, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: + vpbroadcastd m8, [pw_2896x8] + vpbroadcastd m9, [pw_1697x16] + vpbroadcastd m10, [pw_4096] + lea r6, [strideq*5] + pxor m6, m6 + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + add cq, 32 + lea dstq, [dstq+strideq*4] + call .main + add cq, 64*8-32 + lea dstq, [r5+16*1] + call .main + sub eobd, 107 ; eob < 143 + jl .ret + add cq, 32 + lea dstq, [dstq+strideq*4] + call .main + add cq, 64*8-32 + lea dstq, [r5+16*2] + call .main + sub eobd, 128 ; eob < 271 + jl .ret + add cq, 32 + lea dstq, [dstq+strideq*4] + call .main + add cq, 64*8-32 + lea dstq, [r5+16*3] + call .main + sub eobd, 128 ; eob < 399 + jl .ret + add cq, 32 + lea dstq, [dstq+strideq*4] + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+64*0] + packssdw m0, [cq+64*1] + mova m1, [cq+64*2] + packssdw m1, [cq+64*3] + mova m2, [cq+64*4] + packssdw m2, [cq+64*5] + mova m3, [cq+64*6] + packssdw m3, [cq+64*7] + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 + REPX {paddsw x, x }, m0, m1, m2, m3 + REPX {IDTX16 x, 4, 9, _ }, 0, 1, 2, 3 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3 + REPX {mova [cq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 + +cglobal inv_txfm_add_identity_identity_32x16_12bpc, 4, 7, 11, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_32x16_10bpc).pass1 + +cglobal inv_txfm_add_dct_dct_32x32_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 8, 16, 32*83, dst, stride, c, eob +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + lea r6, [rsp+32*7] + call .main + cmp eobd, 36 + jl .fast + call .main + cmp eobd, 136 + jl .fast + call .main + cmp eobd, 300 + jl .fast + call .main + jmp .pass2 +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly +.fast: + lea r4, [rsp+32*71] + pxor m0, m0 +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + cmp r6, r4 + jl .fast_loop +.pass2: + lea r3, [rsp+32*3] + mov r4, r6 + lea r5, [r6+32*8] + lea r6, [pw_5+128] + call .pass2_oddhalf + call .pass2_evenhalf + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq + call m(inv_txfm_add_dct_dct_16x32_10bpc).pass2_end + sub dstq, r3 + lea r2, [r2+r3+32] + add dstq, 32 + lea r3, [rsp+32*11] + call .pass2_oddhalf + call .pass2_evenhalf + lea r3, [strideq*3] + call m(inv_txfm_add_dct_dct_16x32_10bpc).pass2_end + RET +ALIGN function_align +.main: + mova m0, [cq+128* 1] + mova m1, [cq+128* 7] + mova m2, [cq+128* 9] + mova m3, [cq+128*15] + mova m4, [cq+128*17] + mova m5, [cq+128*23] + mova m6, [cq+128*25] + mova m7, [cq+128*31] + vpbroadcastd m11, [pd_2048] + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1 + mova m0, [cq+128* 3] + mova m1, [cq+128* 5] + mova m2, [cq+128*11] + mova m3, [cq+128*13] + mova m4, [cq+128*19] + mova m5, [cq+128*21] + mova m6, [cq+128*27] + mova m7, [cq+128*29] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2 + mova m0, [cq+128* 2] + mova m1, [cq+128* 6] + mova m2, [cq+128*10] + mova m3, [cq+128*14] + mova m4, [cq+128*18] + mova m5, [cq+128*22] + mova m6, [cq+128*26] + mova m7, [cq+128*30] + call m(idct_8x16_internal_10bpc).main_oddhalf + mova m0, [cq+128* 0] + mova m1, [cq+128* 4] + mova m2, [cq+128* 8] + mova m3, [cq+128*12] + mova m4, [cq+128*16] + mova m5, [cq+128*20] + mova m6, [cq+128*24] + mova m7, [cq+128*28] + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_end + pxor m15, m15 + mov r7d, 128*29 +.main_zero_loop: + mova [cq+r7-128*1], m15 + mova [cq+r7+128*0], m15 + mova [cq+r7+128*1], m15 + mova [cq+r7+128*2], m15 + sub r7d, 128*4 + jg .main_zero_loop + add cq, 32 + mova [r4-32*4], m0 + mova [r4-32*3], m1 + mova [r4-32*2], m2 + mova [r4-32*1], m3 + mova [r4+32*0], m4 + mova [r4+32*1], m5 + mova [r4+32*2], m6 + mova [r4+32*3], m7 + mova m0, [r5+32*3] + mova m1, [r5+32*2] + mova m2, [r5+32*1] + mova m3, [r5+32*0] + mova m4, [r5-32*1] + mova m5, [r5-32*2] + mova m6, [r5-32*3] + mova m7, [r5-32*4] + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + mova [r5-32*4], m0 + mova [r5-32*3], m1 + mova [r5-32*2], m2 + mova [r5-32*1], m3 + mova [r5+32*0], m4 + mova [r5+32*1], m5 + mova [r5+32*2], m6 + mova [r5+32*3], m7 + ret +ALIGN function_align +.pass2_oddhalf: + mova m0, [r3+32* 1] ; 1 + mova m1, [r3+32* 3] ; 3 + mova m2, [r3+32* 5] ; 5 + mova m3, [r3+32* 7] ; 7 + mova m4, [r3+32*17] ; 9 + mova m5, [r3+32*19] ; 11 + mova m6, [r3+32*21] ; 13 + mova m7, [r3+32*23] ; 15 + mova m8, [r3+32*33] ; 17 + mova m9, [r3+32*35] ; 19 + mova m10, [r3+32*37] ; 21 + mova m11, [r3+32*39] ; 23 + mova m12, [r3+32*49] ; 25 + mova m13, [r3+32*51] ; 27 + mova m14, [r3+32*53] ; 29 + mova m15, [r3+32*55] ; 31 + jmp m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf +ALIGN function_align +.pass2_evenhalf: + mova m0, [r3+32* 0] ; 0 + mova m1, [r3+32* 2] ; 2 + mova m2, [r3+32* 4] ; 4 + mova m3, [r3+32* 6] ; 6 + mova m4, [r3+32*16] ; 8 + mova m5, [r3+32*18] ; 10 + mova m6, [r3+32*20] ; 12 + mova m7, [r3+32*22] ; 14 + mova m8, [r3+32*32] ; 16 + mova m9, [r3+32*34] ; 18 + mova m10, [r3+32*36] ; 20 + mova m11, [r3+32*38] ; 22 + mova m12, [r3+32*48] ; 24 + mova m13, [r3+32*50] ; 26 + mova m14, [r3+32*52] ; 28 + mova m15, [r3+32*54] ; 30 + mova [rsp+gprsize], m15 + jmp m(idct_16x16_internal_8bpc).main + +cglobal inv_txfm_add_identity_identity_32x32_10bpc, 4, 8, 8, dst, stride, c, eob +%undef cmp + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: + vpbroadcastd m5, [pw_8192] + pxor m6, m6 + lea r6, [strideq*3] + lea r5, [strideq*5] + lea r4, [strideq+r6*2] ; strideq*7 + call .main ; 0 + cmp eobd, 36 + jl .ret + add cq, 128*8 ; 0 1 + mov r7, dstq ; 1 + add dstq, 16 + call .main + call .main2 + cmp eobd, 136 + jl .ret + add cq, 128*16-32 ; 0 1 2 + lea dstq, [r7+16*2] ; 1 2 + call .main ; 2 + call .main2 + call .main2 + cmp eobd, 300 + jl .ret + add cq, 128*24-64 ; 0 1 2 3 + add r7, 16*3 ; 1 2 3 + mov dstq, r7 ; 2 3 + call .main ; 3 + call .main2 + call .main2 + call .main2 + cmp eobd, 535 + jl .ret + add cq, 128*24-64 ; 0 1 2 3 + lea dstq, [r7+strideq*8] ; 1 2 3 4 + mov r7, dstq ; 2 3 4 + call .main ; 3 4 + call .main2 + call .main2 + cmp eobd, 755 + jl .ret + add cq, 128*16-32 ; 0 1 2 3 + lea dstq, [r7+strideq*8] ; 1 2 3 4 + call .main ; 2 3 4 5 + call .main2 ; 3 4 5 + cmp eobd, 911 + jl .ret + add cq, 128*8 ; 0 1 2 3 + add dstq, 16 ; 1 2 3 4 + call .main ; 2 3 4 5 +.ret: ; 3 4 5 6 + RET +ALIGN function_align +.main2: + sub cq, 128*8-32 + lea dstq, [dstq+strideq*8-16] +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + jmp m(inv_txfm_add_identity_identity_8x32_10bpc).main_zero + +cglobal inv_txfm_add_identity_identity_32x32_12bpc, 4, 8, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_32x32_10bpc).pass1 + +%macro IDCT64_PART2_END 6-10 ; out, src[1-2], tmp[1-3], (offset[1-4]) +%if %1 & 1 + mova m%5, [r5-32*(51-%1)] ; idct16 out 0+n + mova m%4, [r4-32*(14+%1)] ; idct32 out31-n +%else + mova m%5, [r4-32*(45-%1)] + mova m%4, [r5-32*(20+%1)] +%endif + paddsw m%6, m%5, m%4 ; idct32 out 0+n + psubsw m%5, m%4 ; idct32 out31-n + paddsw m%4, m%5, m%3 ; out31-n + psubsw m%5, m%3 ; out32+n + paddsw m%3, m%6, m%2 ; out 0+n + psubsw m%6, m%2 ; out63-n + REPX {pmulhrsw x, m14}, m%5, m%6, m%4, m%3 +%if %1 & 1 + %define %%d0 r2 + %define %%d1 dstq +%else + %define %%d0 dstq + %define %%d1 r2 +%endif + paddw m%3, [%%d0+%7 ] + paddw m%4, [%%d1+%8 ] + paddw m%5, [%%d0+%9 ] + paddw m%6, [%%d1+%10] + pxor m%2, m%2 + REPX {pmaxsw x, m%2}, m%3, m%4, m%5, m%6 + vpbroadcastd m%2, [pixel_10bpc_max] + REPX {pminsw x, m%2}, m%3, m%4, m%5, m%6 + mova [%%d0+%7 ], m%3 + mova [%%d1+%8 ], m%4 + mova [%%d0+%9 ], m%5 + mova [%%d1+%10], m%6 +%endmacro + +cglobal inv_txfm_add_dct_dct_16x64_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 10, 16, 32*98, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*6] + call .main + sub eobd, 44 + jl .fast + call .main + sub eobd, 107 + jl .fast + call .main + sub eobd, 128 + jl .fast + call .main + jmp .pass2 +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 64 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 +.fast: + lea r4, [rsp+32*38] + pxor m0, m0 +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + cmp r6, r4 + jl .fast_loop +.pass2: + lea r6, [pw_5+128] + mova m0, [rsp+32* 2] ; in0 + mova m1, [rsp+32* 6] ; in4 + mova m2, [rsp+32*10] ; in8 + mova m3, [rsp+32*14] ; in12 + mova m4, [rsp+32*18] ; in16 + mova m5, [rsp+32*22] ; in20 + mova m6, [rsp+32*26] ; in24 + mova m7, [rsp+32*30] ; in28 + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + lea r4, [rsp+32*38] + mova [r4-32*4], m0 + mova [r4-32*3], m1 + mova [r4-32*2], m2 + mova [r4-32*1], m3 + mova [r4+32*0], m4 + mova [r4+32*1], m5 + mova [r4+32*2], m6 + mova [r4+32*3], m7 + add r4, 32*8 + mova [r4-32*4], m8 + mova [r4-32*3], m9 + mova [r4-32*2], m10 + mova [r4-32*1], m11 + mova [r4+32*0], m12 + mova [r4+32*1], m13 + mova [r4+32*2], m14 + mova [r4+32*3], m15 + mova m0, [rsp+32* 4] ; in2 + mova m1, [rsp+32* 8] ; in6 + mova m2, [rsp+32*12] ; in10 + mova m3, [rsp+32*16] ; in14 + mova m4, [rsp+32*20] ; in18 + mova m5, [rsp+32*24] ; in22 + mova m6, [rsp+32*28] ; in26 + mova m7, [rsp+32*32] ; in30 + lea r5, [r4+32*16] + add r4, 32*8 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova m0, [rsp+32* 3] ; in1 + mova m1, [rsp+32*33] ; in31 + mova m2, [rsp+32*19] ; in17 + mova m3, [rsp+32*17] ; in15 + mova m4, [rsp+32*11] ; in9 + mova m5, [rsp+32*25] ; in23 + mova m6, [rsp+32*27] ; in25 + mova m7, [rsp+32* 9] ; in7 + lea r6, [idct64_mul - 8] + add r4, 32*16 + add r5, 32*32 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + mova m0, [rsp+32* 7] ; in5 + mova m1, [rsp+32*29] ; in27 + mova m2, [rsp+32*23] ; in21 + mova m3, [rsp+32*13] ; in11 + mova m4, [rsp+32*15] ; in13 + mova m5, [rsp+32*21] ; in19 + mova m6, [rsp+32*31] ; in29 + mova m7, [rsp+32* 5] ; in3 + add r6, 8 + add r4, 32*8 + sub r5, 32*8 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + lea r8, [strideq*4] + lea r9, [strideq*5] + lea r3, [r9+strideq*1] ; stride*6 + lea r7, [r9+strideq*2] ; stride*7 + call .main_part2_pass2 + RET +ALIGN function_align +.main: + mova m0, [cq+128* 1] + mova m1, [cq+128* 3] + mova m2, [cq+128* 5] + mova m3, [cq+128* 7] + mova m4, [cq+128* 9] + mova m5, [cq+128*11] + mova m6, [cq+128*13] + mova m7, [cq+128*15] + call m(idct_8x16_internal_10bpc).main_oddhalf + mova m0, [cq+128* 0] + mova m1, [cq+128* 2] + mova m2, [cq+128* 4] + mova m3, [cq+128* 6] + mova m4, [cq+128* 8] + mova m5, [cq+128*10] + mova m6, [cq+128*12] + mova m7, [cq+128*14] + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + pxor m15, m15 + mov r7d, 128*13 +.main_zero_loop: + mova [cq+r7-128*1], m15 + mova [cq+r7+128*0], m15 + mova [cq+r7+128*1], m15 + mova [cq+r7+128*2], m15 + sub r7d, 128*4 + jg .main_zero_loop + add cq, 32 + psrld m15, m11, 10 ; pd_2 + mova m8, [r6-32*4] + mova m9, [r6+32*3] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m10, m0, m8 ; out15 + paddd m0, m8 ; out0 + mova m8, [r6-32*3] + psubd m15, m7, m9 ; out8 + paddd m7, m9 ; out7 + mova m9, [r6+32*2] + REPX {psrad x, 2}, m0, m15, m10, m7 + packssdw m0, m15 + packssdw m7, m10 + psubd m10, m1, m8 ; out14 + paddd m1, m8 ; out1 + mova m8, [r6-32*2] + psubd m15, m6, m9 ; out9 + paddd m6, m9 ; out6 + mova m9, [r6+32*1] + REPX {psrad x, 2}, m1, m15, m10, m6 + packssdw m1, m15 + packssdw m6, m10 + psubd m10, m2, m8 ; out13 + paddd m2, m8 ; out2 + mova m8, [r6-32*1] + psubd m15, m5, m9 ; out10 + paddd m5, m9 ; out5 + mova m9, [r6+32*0] + REPX {psrad x, 2}, m2, m15, m10, m5 + packssdw m2, m15 + packssdw m5, m10 + psubd m10, m3, m8 ; out12 + paddd m3, m8 ; out3 + psubd m15, m4, m9 ; out11 + paddd m4, m9 ; out4 + REPX {psrad x, 2}, m3, m15, m10, m4 + packssdw m3, m15 + packssdw m4, m10 + call m(idct_16x8_internal_10bpc).transpose3 + mova [r6-32*4], m0 + mova [r6-32*3], m1 + mova [r6-32*2], m2 + mova [r6-32*1], m3 + mova [r6+32*0], m4 + mova [r6+32*1], m5 + mova [r6+32*2], m6 + mova [r6+32*3], m7 + add r6, 32*8 + ret +.main_part2_pass2: + vpbroadcastd m11, [pw_1567_3784] + vpbroadcastd m12, [pw_m3784_1567] + vpbroadcastd m13, [pw_2896_2896] + lea r6, [pw_5+128] + lea r2, [dstq+r7] +.main_part2_pass2_loop: + vpbroadcastd m14, [pw_m2896_2896] + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_internal + vpbroadcastd m14, [pw_2048] + IDCT64_PART2_END 0, 7, 0, 6, 9, 10, strideq*0, r3*4, r8*8, r7*8 + IDCT64_PART2_END 7, 8, 5, 0, 6, 7, strideq*0, r3*4, r8*8, r7*8 + IDCT64_PART2_END 8, 2, 1, 0, 6, 7, strideq*8, r8*4, r9*8, r3*8 + IDCT64_PART2_END 15, 3, 4, 0, 6, 7, strideq*8, r8*4, r9*8, r3*8 + add dstq, strideq + sub r2, strideq + cmp r4, r5 + jne .main_part2_pass2_loop + ret +ALIGN function_align +.main_part1_rect2: + REPX {paddd x, m11}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_part1: ; idct64 steps 1-5 + ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + vpbroadcastd m7, [r5+4*0] + vpbroadcastd m8, [r5+4*1] + vpbroadcastd m6, [r5+4*2] + vpbroadcastd m9, [r5+4*3] + vpbroadcastd m5, [r5+4*4] + vpbroadcastd m10, [r5+4*5] + vpbroadcastd m4, [r5+4*6] + vpbroadcastd m15, [r5+4*7] + pmulld m7, m0 ; t63a + pmulld m0, m8 ; t32a + pmulld m6, m1 ; t62a + pmulld m1, m9 ; t33a + pmulld m5, m2 ; t61a + pmulld m2, m10 ; t34a + pmulld m4, m3 ; t60a + pmulld m3, m15 ; t35a + vpbroadcastd m10, [r5+4*8] + vpbroadcastd m15, [r5+4*9] + REPX {paddd x, m11}, m7, m0, m6, m1, m5, m2, m4, m3 + REPX {psrad x, 12 }, m0, m1, m7, m6, m2, m3, m5, m4 + psubd m8, m0, m1 ; t33 + paddd m0, m1 ; t32 + psubd m1, m7, m6 ; t62 + paddd m7, m6 ; t63 + psubd m6, m3, m2 ; t34 + paddd m3, m2 ; t35 + psubd m2, m4, m5 ; t61 + paddd m4, m5 ; t60 + REPX {pmaxsd x, m12}, m8, m1, m6, m2 + REPX {pminsd x, m13}, m8, m1, m6, m2 + ITX_MULSUB_2D 1, 8, 5, 9, _, 11, 10, 15 ; t33a, t62a + ITX_MULSUB_2D 2, 6, 5, 9, _, 11, 10, 15, 2 ; t61a, t34a + REPX {pmaxsd x, m12}, m0, m3, m7, m4 + REPX {pminsd x, m13}, m0, m3, m7, m4 + vpbroadcastd m10, [r5+4*10] + vpbroadcastd m15, [r5+4*11] + psubd m5, m0, m3 ; t35a + paddd m0, m3 ; t32a + psubd m3, m7, m4 ; t60a + paddd m7, m4 ; t63a + psubd m4, m1, m6 ; t34 + paddd m1, m6 ; t33 + psubd m6, m8, m2 ; t61 + paddd m8, m2 ; t62 + REPX {pmaxsd x, m12}, m5, m3, m4, m6 + REPX {pminsd x, m13}, m5, m3, m4, m6 + ITX_MULSUB_2D 3, 5, 2, 9, _, 11, 10, 15 ; t35, t60 + ITX_MULSUB_2D 6, 4, 2, 9, _, 11, 10, 15 ; t34a, t61a + REPX {pmaxsd x, m12}, m0, m7, m1, m8 + REPX {pminsd x, m13}, m0, m7, m1, m8 + add r5, 4*12 + mova [r6-32*4], m0 + mova [r6+32*3], m7 + mova [r6-32*3], m1 + mova [r6+32*2], m8 + mova [r6-32*2], m6 + mova [r6+32*1], m4 + mova [r6-32*1], m3 + mova [r6+32*0], m5 + add r6, 32*8 + ret +.main_part2: ; idct64 steps 6-9 + lea r5, [r6+32*3] + sub r6, 32*4 + vpbroadcastd m10, [pd_1567] + vpbroadcastd m15, [pd_3784] +.main_part2_loop: + mova m0, [r6-32*32] ; t32a + mova m1, [r5-32*24] ; t39a + mova m2, [r5-32*32] ; t63a + mova m3, [r6-32*24] ; t56a + mova m4, [r6-32*16] ; t40a + mova m5, [r5-32* 8] ; t47a + mova m6, [r5-32*16] ; t55a + mova m7, [r6-32* 8] ; t48a + psubd m8, m0, m1 ; t39 + paddd m0, m1 ; t32 + psubd m1, m2, m3 ; t56 + paddd m2, m3 ; t63 + psubd m3, m5, m4 ; t40 + paddd m5, m4 ; t47 + psubd m4, m7, m6 ; t55 + paddd m7, m6 ; t48 + REPX {pmaxsd x, m12}, m8, m1, m3, m4 + REPX {pminsd x, m13}, m8, m1, m3, m4 + ITX_MULSUB_2D 1, 8, 6, 9, _, 11, 10, 15 ; t39a, t56a + ITX_MULSUB_2D 4, 3, 6, 9, _, 11, 10, 15, 2 ; t55a, t40a + REPX {pmaxsd x, m12}, m0, m2, m5, m7 + REPX {pminsd x, m13}, m0, m5, m2, m7 + psubd m6, m2, m7 ; t48a + paddd m2, m7 ; t63a + psubd m7, m0, m5 ; t47a + paddd m0, m5 ; t32a + psubd m5, m8, m4 ; t55 + paddd m8, m4 ; t56 + psubd m4, m1, m3 ; t40 + paddd m1, m3 ; t39 + REPX {pmaxsd x, m12}, m6, m7, m5, m4 + REPX {pminsd x, m13}, m6, m7, m5, m4 + REPX {pmulld x, m14}, m6, m7, m5, m4 + REPX {pmaxsd x, m12}, m2, m0, m8, m1 + REPX {pminsd x, m13}, m2, m0, m8, m1 + paddd m6, m11 + paddd m5, m11 + psubd m3, m6, m7 ; t47 + paddd m6, m7 ; t48 + psubd m7, m5, m4 ; t40a + paddd m5, m4 ; t55a + REPX {psrad x, 12}, m3, m6, m7, m5 + mova [r5-32* 8], m2 + mova [r6-32*32], m0 + mova [r6-32* 8], m8 + mova [r5-32*32], m1 + mova [r5-32*24], m3 + mova [r6-32*16], m6 + mova [r6-32*24], m7 + mova [r5-32*16], m5 + add r6, 32 + sub r5, 32 + cmp r6, r5 + jl .main_part2_loop + ret + +cglobal inv_txfm_add_dct_dct_32x64_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 11, 16, 32*134, dst, stride, c, eob +%undef cmp + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + lea r6, [rsp+32*6] + call .main + cmp eobd, 36 + jl .fast + call .main + cmp eobd, 136 + jl .fast + call .main + cmp eobd, 300 + jl .fast + call .main + jmp .pass2 +.dconly: + imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] + mov [cq], eobd ; 0 + or r3d, 64 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 +.fast: + lea r4, [rsp+32*70] + pxor m0, m0 +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + cmp r6, r4 + jl .fast_loop +.pass2: + lea r6, [pw_5 + 128] + mov r10, rsp + lea r8, [strideq*4] + lea r9, [strideq*5] + lea r3, [r9+strideq*1] ; stride*6 + lea r7, [r9+strideq*2] ; stride*7 +.pass2_loop: + mova m0, [r10+32* 2] ; in0 + mova m1, [r10+32* 6] ; in4 + mova m2, [r10+32*18] ; in8 + mova m3, [r10+32*22] ; in12 + mova m4, [r10+32*34] ; in16 + mova m5, [r10+32*38] ; in20 + mova m6, [r10+32*50] ; in24 + mova m7, [r10+32*54] ; in28 + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + lea r4, [rsp+32*70] + mova [r4-32*4], m0 + mova [r4-32*3], m1 + mova [r4-32*2], m2 + mova [r4-32*1], m3 + mova [r4+32*0], m4 + mova [r4+32*1], m5 + mova [r4+32*2], m6 + mova [r4+32*3], m7 + add r4, 32*8 + mova [r4-32*4], m8 + mova [r4-32*3], m9 + mova [r4-32*2], m10 + mova [r4-32*1], m11 + mova [r4+32*0], m12 + mova [r4+32*1], m13 + mova [r4+32*2], m14 + mova [r4+32*3], m15 + mova m0, [r10+32* 4] ; in2 + mova m1, [r10+32* 8] ; in6 + mova m2, [r10+32*20] ; in10 + mova m3, [r10+32*24] ; in14 + mova m4, [r10+32*36] ; in18 + mova m5, [r10+32*40] ; in22 + mova m6, [r10+32*52] ; in26 + mova m7, [r10+32*56] ; in30 + lea r5, [r4+32*16] + add r4, 32*8 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova m0, [r10+32* 3] ; in1 + mova m1, [r10+32*57] ; in31 + mova m2, [r10+32*35] ; in17 + mova m3, [r10+32*25] ; in15 + mova m4, [r10+32*19] ; in9 + mova m5, [r10+32*41] ; in23 + mova m6, [r10+32*51] ; in25 + mova m7, [r10+32* 9] ; in7 + lea r6, [idct64_mul - 8] + add r4, 32*16 + add r5, 32*32 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + mova m0, [r10+32* 7] ; in5 + mova m1, [r10+32*53] ; in27 + mova m2, [r10+32*39] ; in21 + mova m3, [r10+32*21] ; in11 + mova m4, [r10+32*23] ; in13 + mova m5, [r10+32*37] ; in19 + mova m6, [r10+32*55] ; in29 + mova m7, [r10+32* 5] ; in3 + add r6, 8 + add r4, 32*8 + sub r5, 32*8 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part2_pass2 + add r10, 32*8 + sub r4, 32*98 ; rsp+32*16 + sub dstq, r8 + add dstq, 32 + cmp r10, r4 + jl .pass2_loop + RET +ALIGN function_align +.main: + vpbroadcastd m14, [pd_2896] + vpbroadcastd m11, [pd_2048] + pmulld m0, m14, [cq+128* 1] + pmulld m1, m14, [cq+128* 7] + pmulld m2, m14, [cq+128* 9] + pmulld m3, m14, [cq+128*15] + pmulld m4, m14, [cq+128*17] + pmulld m5, m14, [cq+128*23] + pmulld m6, m14, [cq+128*25] + pmulld m7, m14, [cq+128*31] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_rect2 + pmulld m0, m14, [cq+128* 3] + pmulld m1, m14, [cq+128* 5] + pmulld m2, m14, [cq+128*11] + pmulld m3, m14, [cq+128*13] + pmulld m4, m14, [cq+128*19] + pmulld m5, m14, [cq+128*21] + pmulld m6, m14, [cq+128*27] + pmulld m7, m14, [cq+128*29] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_rect2 + pmulld m0, m14, [cq+128* 2] + pmulld m1, m14, [cq+128* 6] + pmulld m2, m14, [cq+128*10] + pmulld m3, m14, [cq+128*14] + pmulld m4, m14, [cq+128*18] + pmulld m5, m14, [cq+128*22] + pmulld m6, m14, [cq+128*26] + pmulld m7, m14, [cq+128*30] + call m(idct_8x16_internal_10bpc).main_oddhalf_rect2 + pmulld m0, m14, [cq+128* 0] + pmulld m1, m14, [cq+128* 4] + pmulld m2, m14, [cq+128* 8] + pmulld m3, m14, [cq+128*12] + pmulld m4, m14, [cq+128*16] + pmulld m5, m14, [cq+128*20] + pmulld m6, m14, [cq+128*24] + pmulld m7, m14, [cq+128*28] + pxor m15, m15 + mov r7d, 128*29 +.main_zero_loop: + mova [cq+r7-128*1], m15 + mova [cq+r7+128*0], m15 + mova [cq+r7+128*1], m15 + mova [cq+r7+128*2], m15 + sub r7d, 128*4 + jg .main_zero_loop + add cq, 32 + call m(idct_8x8_internal_10bpc).main_rect2 + call m(idct_8x16_internal_10bpc).main_evenhalf + call m(inv_txfm_add_dct_dct_32x16_10bpc).main_end + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + mova [r4-32*4], m0 + mova [r4-32*3], m1 + mova [r4-32*2], m2 + mova [r4-32*1], m3 + mova [r4+32*0], m4 + mova [r4+32*1], m5 + mova [r4+32*2], m6 + mova [r4+32*3], m7 + mova m0, [r5+32*3] + mova m1, [r5+32*2] + mova m2, [r5+32*1] + mova m3, [r5+32*0] + mova m4, [r5-32*1] + mova m5, [r5-32*2] + mova m6, [r5-32*3] + mova m7, [r5-32*4] + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + mova [r5-32*4], m0 + mova [r5-32*3], m1 + mova [r5-32*2], m2 + mova [r5-32*1], m3 + mova [r5+32*0], m4 + mova [r5+32*1], m5 + mova [r5+32*2], m6 + mova [r5+32*3], m7 + ret + +cglobal inv_txfm_add_dct_dct_64x16_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jnz .normal + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 16 +.dconly: + add r6d, 640 + sar r6d, 10 +.dconly2: + vpbroadcastd m5, [dconly_10bpc] + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + movd xm0, r6d + paddsw xm0, xm5 + vpbroadcastw m0, xm0 +.dconly_loop: + paddsw m1, m0, [dstq+32*0] + paddsw m2, m0, [dstq+32*1] + paddsw m3, m0, [dstq+32*2] + paddsw m4, m0, [dstq+32*3] + REPX {psubusw x, m5}, m1, m2, m3, m4 + mova [dstq+32*0], m1 + mova [dstq+32*1], m2 + mova [dstq+32*2], m3 + mova [dstq+32*3], m4 + add dstq, strideq + dec r3d + jg .dconly_loop + RET +.normal: + PROLOGUE 0, 8, 16, 32*96, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*4] + call .main + call .shift_transpose + cmp eobd, 36 + jl .fast + call .main + call .shift_transpose + jmp .pass2 +.fast: + pxor m0, m0 + mov r3d, 4 +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + dec r3d + jg .fast_loop +.pass2: + lea r7, [r6-32*64] + lea r4, [r6-32*32] + lea r6, [pw_5+128] + mov r5, dstq +.pass2_loop: + mova m0, [r7-32*4] + mova m1, [r7-32*3] + mova m2, [r7-32*2] + mova m3, [r7-32*1] + mova m4, [r7+32*0] + mova m5, [r7+32*1] + mova m6, [r7+32*2] + mova m7, [r7+32*3] + add r7, 32*32 + mova m8, [r7-32*4] + mova m9, [r7-32*3] + mova m10, [r7-32*2] + mova m11, [r7-32*1] + mova m12, [r7+32*0] + mova m13, [r7+32*1] + mova m14, [r7+32*2] + mova m15, [r7+32*3] + sub r7, 32*24 + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + call m(inv_txfm_add_dct_dct_32x16_10bpc).write_16x16 + add r5, 32 + mov dstq, r5 + cmp r7, r4 + jl .pass2_loop + RET +ALIGN function_align +.main: + lea r5, [idct64_mul_16bpc] + mova m0, [cq+64* 1] + mova m1, [cq+64*31] + mova m2, [cq+64*17] + mova m3, [cq+64*15] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+64* 7] + mova m1, [cq+64*25] + mova m2, [cq+64*23] + mova m3, [cq+64* 9] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+64* 5] + mova m1, [cq+64*27] + mova m2, [cq+64*21] + mova m3, [cq+64*11] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+64* 3] + mova m1, [cq+64*29] + mova m2, [cq+64*19] + mova m3, [cq+64*13] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part2 + mova m0, [cq+64* 2] + mova m1, [cq+64*14] + mova m2, [cq+64*18] + mova m3, [cq+64*30] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_fast + mova m0, [cq+64* 6] + mova m1, [cq+64*10] + mova m2, [cq+64*22] + mova m3, [cq+64*26] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_fast + mova m0, [cq+64* 4] + mova m1, [cq+64*12] + mova m2, [cq+64*20] + mova m3, [cq+64*28] + call m(idct_8x16_internal_10bpc).main_oddhalf_fast + mova m0, [cq+64* 0] + mova m1, [cq+64* 8] + mova m2, [cq+64*16] + mova m3, [cq+64*24] + pxor m15, m15 + mov r7d, 64*30 +.main_zero_loop: + mova [cq+r7-64*2], m15 + mova [cq+r7-64*1], m15 + mova [cq+r7+64*0], m15 + mova [cq+r7+64*1], m15 + sub r7d, 64*4 + jg .main_zero_loop +.main_end: + psrld m15, m11, 10 ; pd_2 +.main_end2: + add cq, 32 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + add r6, 32*8 + call m(idct_8x16_internal_10bpc).main_evenhalf + mova [r6+32*2], m1 + mova [r6+32*1], m2 + mova [r6+32*0], m3 + mova [r6-32*1], m4 + mova [r6-32*2], m5 + mova [r6-32*3], m6 + mova [r6-32*4], m7 + jmp .main_end_loop_start +.main_end_loop: + mova m0, [r6+32* 3] ; idct8 0 + n +.main_end_loop_start: + mova m1, [r5+32* 4] ; idct16 15 - n + mova m2, [r5-32*12] ; idct32 16 + n + mova m3, [r6-32*13] ; idct32 31 - n + mova m4, [r6-32*29] ; idct64 63 - n + mova m5, [r5-32*28] ; idct64 48 + n + mova m6, [r6-32*45] ; idct64 47 - n + mova m7, [r5-32*44] ; idct64 32 + n + paddd m8, m0, m1 ; idct16 out0 + n + psubd m0, m1 ; idct16 out15 - n + REPX {pmaxsd x, m12}, m8, m0 + REPX {pminsd x, m13}, m8, m0 + paddd m1, m8, m3 ; idct32 out0 + n + psubd m8, m3 ; idct32 out31 - n + paddd m3, m0, m2 ; idct32 out15 - n + psubd m0, m2 ; idct32 out16 + n + REPX {pmaxsd x, m12}, m1, m8, m3, m0 + REPX {pminsd x, m13}, m1, m3, m8, m0 + REPX {paddd x, m15}, m1, m3, m0, m8 + paddd m2, m1, m4 ; idct64 out0 + n (unshifted) + psubd m1, m4 ; idct64 out63 - n (unshifted) + paddd m4, m3, m5 ; idct64 out15 - n (unshifted) + psubd m3, m5 ; idct64 out48 + n (unshifted) + paddd m5, m0, m6 ; idct64 out16 + n (unshifted) + psubd m0, m6 ; idct64 out47 - n (unshifted) + paddd m6, m8, m7 ; idct64 out31 - n (unshifted) + psubd m8, m7 ; idct64 out32 + n (unshifted) + mova [r5-32*44], m2 + mova [r6+32* 3], m1 + mova [r6-32*45], m4 + mova [r5+32* 4], m3 + mova [r5-32*28], m5 + mova [r6-32*13], m0 + mova [r6-32*29], m6 + mova [r5-32*12], m8 + add r5, 32 + sub r6, 32 + cmp r5, r6 + jl .main_end_loop + ret +.shift_transpose: +%macro IDCT64_SHIFT_TRANSPOSE 1 ; shift + sub r6, 32*48 + mov r5, r6 +%%loop: + mova m0, [r6-32* 4] + mova m4, [r6+32* 4] + mova m1, [r6-32* 3] + mova m5, [r6+32* 5] + mova m2, [r6-32* 2] + mova m6, [r6+32* 6] + mova m3, [r6-32* 1] + mova m7, [r6+32* 7] + REPX {psrad x, %1}, m0, m4, m1, m5, m2, m6, m3, m7 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + mova m4, [r6+32* 0] + mova m6, [r6+32* 8] + mova m5, [r6+32* 1] + mova m7, [r6+32* 9] + REPX {psrad x, %1}, m4, m6, m5, m7 + packssdw m4, m6 + packssdw m5, m7 + mova m6, [r6+32* 2] + mova m8, [r6+32*10] + mova m7, [r6+32* 3] + mova m9, [r6+32*11] + REPX {psrad x, %1}, m6, m8, m7, m9 + packssdw m6, m8 + packssdw m7, m9 + call m(idct_16x8_internal_10bpc).transpose3 + mova [r5-32*4], m0 + mova [r5-32*3], m1 + mova [r5-32*2], m2 + mova [r5-32*1], m3 + mova [r5+32*0], m4 + mova [r5+32*1], m5 + mova [r5+32*2], m6 + mova [r5+32*3], m7 + add r6, 32*16 + add r5, 32*8 + cmp r5, r4 + jl %%loop + mov r6, r4 +%endmacro + IDCT64_SHIFT_TRANSPOSE 2 + ret + +cglobal inv_txfm_add_dct_dct_64x32_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 8, 16, 32*163, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*7] + call .main + cmp eobd, 36 + jl .fast + call .main + cmp eobd, 136 + jl .fast + call .main + cmp eobd, 300 + jl .fast + call .main + jmp .pass2 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 32 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 + jmp m(inv_txfm_add_dct_dct_64x16_10bpc).dconly2 +.fast: + pxor m0, m0 + lea r4, [rsp+32*135] +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + cmp r6, r4 + jl .fast_loop +.pass2: + lea r7, [r6-32*32] + lea r5, [r6+32*8] + lea r6, [pw_5+128] + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq +.pass2_loop: + mova m0, [r7-32*99] + mova m1, [r7-32*97] + mova m2, [r7-32*95] + mova m3, [r7-32*93] + mova m4, [r7-32*67] + mova m5, [r7-32*65] + mova m6, [r7-32*63] + mova m7, [r7-32*61] + mova m8, [r7-32*35] + mova m9, [r7-32*33] + mova m10, [r7-32*31] + mova m11, [r7-32*29] + mova m12, [r7-32* 3] + mova m13, [r7-32* 1] + mova m14, [r7+32* 1] + mova m15, [r7+32* 3] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + mova m0, [r7-32*100] + mova m1, [r7-32*98] + mova m2, [r7-32*96] + mova m3, [r7-32*94] + mova m4, [r7-32*68] + mova m5, [r7-32*66] + mova m6, [r7-32*64] + mova m7, [r7-32*62] + mova m8, [r7-32*36] + mova m9, [r7-32*34] + mova m10, [r7-32*32] + mova m11, [r7-32*30] + mova m12, [r7-32* 4] + mova m13, [r7-32* 2] + mova m14, [r7+32* 0] + mova m15, [r7+32* 2] + add r7, 32*8 + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_16x32_10bpc).pass2_end + sub dstq, r3 + lea r2, [r2+r3+32] + add dstq, 32 + cmp r7, r4 + jl .pass2_loop + RET +ALIGN function_align +.main: + lea r5, [idct64_mul_16bpc] + pmulld m0, m14, [cq+128* 1] + pmulld m1, m14, [cq+128*31] + pmulld m2, m14, [cq+128*17] + pmulld m3, m14, [cq+128*15] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1_rect2 + pmulld m0, m14, [cq+128* 7] + pmulld m1, m14, [cq+128*25] + pmulld m2, m14, [cq+128*23] + pmulld m3, m14, [cq+128* 9] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1_rect2 + pmulld m0, m14, [cq+128* 5] + pmulld m1, m14, [cq+128*27] + pmulld m2, m14, [cq+128*21] + pmulld m3, m14, [cq+128*11] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1_rect2 + pmulld m0, m14, [cq+128* 3] + pmulld m1, m14, [cq+128*29] + pmulld m2, m14, [cq+128*19] + pmulld m3, m14, [cq+128*13] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1_rect2 + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part2 + pmulld m0, m14, [cq+128* 2] + pmulld m1, m14, [cq+128*14] + pmulld m2, m14, [cq+128*18] + pmulld m3, m14, [cq+128*30] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_fast_rect2 + pmulld m0, m14, [cq+128* 6] + pmulld m1, m14, [cq+128*10] + pmulld m2, m14, [cq+128*22] + pmulld m3, m14, [cq+128*26] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_fast_rect2 + pmulld m0, m14, [cq+128* 4] + pmulld m1, m14, [cq+128*12] + pmulld m2, m14, [cq+128*20] + pmulld m3, m14, [cq+128*28] + call m(idct_8x16_internal_10bpc).main_oddhalf_fast_rect2 + pmulld m0, m14, [cq+128* 0] + pmulld m1, m14, [cq+128* 8] + pmulld m2, m14, [cq+128*16] + pmulld m3, m14, [cq+128*24] + pxor m15, m15 + mov r7d, 128*29 +.main_zero_loop: + mova [cq+r7-128*1], m15 + mova [cq+r7+128*0], m15 + mova [cq+r7+128*1], m15 + mova [cq+r7+128*2], m15 + sub r7d, 128*4 + jg .main_zero_loop + psrld m15, m11, 11 ; pd_1 + REPX {paddd x, m11}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 + call m(inv_txfm_add_dct_dct_64x16_10bpc).main_end2 + IDCT64_SHIFT_TRANSPOSE 1 + ret + +cglobal inv_txfm_add_dct_dct_64x64_10bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 11, 16, 32*195, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + vpbroadcastd m14, [pd_2896] + lea r6, [rsp+32*7] + call .main + cmp eobd, 36 + jl .fast + call .main + cmp eobd, 136 + jl .fast + call .main + cmp eobd, 300 + jl .fast + call .main + jmp .pass2 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 64 + jmp m(inv_txfm_add_dct_dct_64x16_10bpc).dconly +.fast: + pxor m0, m0 + lea r4, [rsp+32*135] +.fast_loop: + REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 + add r6, 32*8 + cmp r6, r4 + jl .fast_loop +.pass2: + lea r10, [r6-32*32] + lea r6, [pw_5+128] + lea r8, [strideq*4] + lea r9, [strideq*5] + lea r3, [r9+strideq*1] ; stride*6 + lea r7, [r9+strideq*2] ; stride*7 +.pass2_loop: + mova m0, [r10-32*100] ; in0 + mova m1, [r10-32*96] ; in4 + mova m2, [r10-32*68] ; in8 + mova m3, [r10-32*64] ; in12 + mova m4, [r10-32*36] ; in16 + mova m5, [r10-32*32] ; in20 + mova m6, [r10-32* 4] ; in24 + mova m7, [r10+32* 0] ; in28 + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [r4-32*4], m0 + mova [r4-32*3], m1 + mova [r4-32*2], m2 + mova [r4-32*1], m3 + mova [r4+32*0], m4 + mova [r4+32*1], m5 + mova [r4+32*2], m6 + mova [r4+32*3], m7 + add r4, 32*8 + mova [r4-32*4], m8 + mova [r4-32*3], m9 + mova [r4-32*2], m10 + mova [r4-32*1], m11 + mova [r4+32*0], m12 + mova [r4+32*1], m13 + mova [r4+32*2], m14 + mova [r4+32*3], m15 + mova m0, [r10-32*98] ; in2 + mova m1, [r10-32*94] ; in6 + mova m2, [r10-32*66] ; in10 + mova m3, [r10-32*62] ; in14 + mova m4, [r10-32*34] ; in18 + mova m5, [r10-32*30] ; in22 + mova m6, [r10-32* 2] ; in26 + mova m7, [r10+32* 2] ; in30 + lea r5, [r4+32*16] + add r4, 32*8 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova m0, [r10-32*99] ; in1 + mova m1, [r10+32* 3] ; in31 + mova m2, [r10-32*35] ; in17 + mova m3, [r10-32*61] ; in15 + mova m4, [r10-32*67] ; in9 + mova m5, [r10-32*29] ; in23 + mova m6, [r10-32* 3] ; in25 + mova m7, [r10-32*93] ; in7 + lea r6, [idct64_mul - 8] + add r4, 32*16 + add r5, 32*32 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + mova m0, [r10-32*95] ; in5 + mova m1, [r10-32* 1] ; in27 + mova m2, [r10-32*31] ; in21 + mova m3, [r10-32*65] ; in11 + mova m4, [r10-32*63] ; in13 + mova m5, [r10-32*33] ; in19 + mova m6, [r10+32* 1] ; in29 + mova m7, [r10-32*97] ; in3 + add r6, 8 + add r4, 32*8 + sub r5, 32*8 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part2_pass2 + add r10, 32*8 + sub dstq, r8 + sub r4, 32*44 + add dstq, 32 + cmp r10, r4 + jl .pass2_loop + RET +ALIGN function_align +.main: + lea r5, [idct64_mul_16bpc] + mova m0, [cq+128* 1] + mova m1, [cq+128*31] + mova m2, [cq+128*17] + mova m3, [cq+128*15] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+128* 7] + mova m1, [cq+128*25] + mova m2, [cq+128*23] + mova m3, [cq+128* 9] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+128* 5] + mova m1, [cq+128*27] + mova m2, [cq+128*21] + mova m3, [cq+128*11] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + mova m0, [cq+128* 3] + mova m1, [cq+128*29] + mova m2, [cq+128*19] + mova m3, [cq+128*13] + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_10bpc).main_part2 + mova m0, [cq+128* 2] + mova m1, [cq+128*14] + mova m2, [cq+128*18] + mova m3, [cq+128*30] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_fast + mova m0, [cq+128* 6] + mova m1, [cq+128*10] + mova m2, [cq+128*22] + mova m3, [cq+128*26] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_fast + mova m0, [cq+128* 4] + mova m1, [cq+128*12] + mova m2, [cq+128*20] + mova m3, [cq+128*28] + call m(idct_8x16_internal_10bpc).main_oddhalf_fast + mova m0, [cq+128* 0] + mova m1, [cq+128* 8] + mova m2, [cq+128*16] + mova m3, [cq+128*24] + pxor m15, m15 + mov r7d, 128*29 +.main_zero_loop: + mova [cq+r7-128*1], m15 + mova [cq+r7+128*0], m15 + mova [cq+r7+128*1], m15 + mova [cq+r7+128*2], m15 + sub r7d, 128*4 + jg .main_zero_loop + call m(inv_txfm_add_dct_dct_64x16_10bpc).main_end + jmp m(inv_txfm_add_dct_dct_64x16_10bpc).shift_transpose + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/itx16_avx512.asm b/third_party/dav1d/src/x86/itx16_avx512.asm new file mode 100644 index 0000000000..d973655462 --- /dev/null +++ b/third_party/dav1d/src/x86/itx16_avx512.asm @@ -0,0 +1,4133 @@ +; Copyright © 2022-2023, VideoLAN and dav1d authors +; Copyright © 2022-2023, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +idct8x8p: db 0, 1, 4, 5, 2, 3, 6, 7, 16, 17, 20, 21, 18, 19, 22, 23 + db 8, 9, 12, 13, 10, 11, 14, 15, 24, 25, 28, 29, 26, 27, 30, 31 + db 32, 33, 36, 37, 34, 35, 38, 39, 48, 49, 52, 53, 50, 51, 54, 55 + db 40, 41, 44, 45, 42, 43, 46, 47, 56, 57, 60, 61, 58, 59, 62, 63 +idtx8x8p: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 + db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 + db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 + db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 +idct8x16p: db 54, 55, 2, 3, 22, 23, 34, 35, 38, 39, 18, 19, 6, 7, 50, 51 + db 62, 63, 10, 11, 30, 31, 42, 43, 46, 47, 26, 27, 14, 15, 58, 59 + db 52, 53, 4, 5, 20, 21, 36, 37, 32, 33, 0, 1, 48, 49, 16, 17 + db 60, 61, 12, 13, 28, 29, 44, 45, 40, 41, 8, 9, 56, 57, 24, 25 +iadst8x16p: db 0, 1, 54, 55, 48, 49, 6, 7, 16, 17, 38, 39, 32, 33, 22, 23 + db 8, 9, 62, 63, 56, 57, 14, 15, 24, 25, 46, 47, 40, 41, 30, 31 + db 4, 5, 50, 51, 52, 53, 2, 3, 20, 21, 34, 35, 36, 37, 18, 19 + db 12, 13, 58, 59, 60, 61, 10, 11, 28, 29, 42, 43, 44, 45, 26, 27 +permA: db 0, 1, 0, 8, 4, 5, 1, 9, 8, 9, 4, 12, 12, 13, 5, 13 + db 16, 17, 16, 24, 20, 21, 17, 25, 24, 25, 20, 28, 28, 29, 21, 29 + db 2, 3, 2, 10, 6, 7, 3, 11, 10, 11, 6, 14, 14, 15, 7, 15 + db 18, 19, 18, 26, 22, 23, 19, 27, 26, 27, 22, 30, 30, 31, 23, 31 +permB: db 4, 2, 1, 8, 0, 0, 1, 0, 12, 3, 3, 10, 8, 1, 3, 2 + db 5, 10, 5, 12, 1, 8, 5, 4, 13, 11, 7, 14, 9, 9, 7, 6 + db 6, 6, 13, 4, 2, 4, 4, 5, 14, 7, 15, 6, 10, 5, 6, 7 + db 7, 14, 9, 0, 3, 12, 0, 1, 15, 15, 11, 2, 11, 13, 2, 3 +permC: db 0, 9, 0, 0, 0, 1, 4, 4, 2, 11, 2, 2, 2, 3, 6, 6 + db 1, 8, 1, 8, 4, 5, 5, 12, 3, 10, 3, 10, 6, 7, 7, 14 + db 9, 1, 8, 1, 1, 0, 12, 5, 11, 3, 10, 3, 3, 2, 14, 7 + db 8, 0, 9, 9, 5, 4, 13, 13, 10, 2, 11, 11, 7, 6, 15, 15 +idct8x32p: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53 + db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61 + db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55 + db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63 +idct32x8p: db 2, 18, 0, 16, 3, 19, 1, 17, 10, 26, 8, 24, 11, 27, 9, 25 + db 34, 50, 32, 48, 35, 51, 33, 49, 42, 58, 40, 56, 43, 59, 41, 57 + db 6, 22, 4, 20, 7, 23, 5, 21, 14, 30, 12, 28, 15, 31, 13, 29 + db 38, 54, 36, 52, 39, 55, 37, 53, 46, 62, 44, 60, 47, 63, 45, 61 +idtx32x8p: db 0, 8, 16, 24, 4, 12, 20, 28, 2, 10, 18, 26, 6, 14, 22, 30 + db 32, 40, 48, 56, 36, 44, 52, 60, 34, 42, 50, 58, 38, 46, 54, 62 + db 1, 9, 17, 25, 5, 13, 21, 29, 3, 11, 19, 27, 7, 15, 23, 31 + db 33, 41, 49, 57, 37, 45, 53, 61, 35, 43, 51, 59, 39, 47, 55, 63 + +pw_2048_m2048: times 16 dw 2048 +pw_m2048_2048: times 16 dw -2048 +pw_2048: times 16 dw 2048 + +; flags: 0 = ++, 1 = +-, 2 = -+, 3 = ++- +%macro COEF_PAIR 2-3 0 ; a, b, flags +%if %3 == 1 +pd_%1_m%2: dd %1, %1, -%2, -%2 +%define pd_%1 (pd_%1_m%2 + 4*0) +%define pd_m%2 (pd_%1_m%2 + 4*2) +%elif %3 == 2 +pd_m%1_%2: dd -%1, -%1, %2, %2 +%define pd_m%1 (pd_m%1_%2 + 4*0) +%define pd_%2 (pd_m%1_%2 + 4*2) +%else +pd_%1_%2: dd %1, %1, %2, %2 +%define pd_%1 (pd_%1_%2 + 4*0) +%define pd_%2 (pd_%1_%2 + 4*2) +%if %3 == 3 +%define pd_%2_m%2 pd_%2 +dd -%2, -%2 +%endif +%endif +%endmacro + +COEF_PAIR 201, 995 +COEF_PAIR 401, 1189, 1 +COEF_PAIR 401, 1931 +COEF_PAIR 401, 3920 +COEF_PAIR 799, 2276, 1 +COEF_PAIR 799, 3406 +COEF_PAIR 799, 4017 +COEF_PAIR 1380, 601 +COEF_PAIR 1751, 2440 +COEF_PAIR 2598, 1189 +COEF_PAIR 2598, 1931, 2 +COEF_PAIR 2598, 3612 +COEF_PAIR 2751, 2106 +COEF_PAIR 2896, 1567, 3 +COEF_PAIR 2896, 3784, 3 +COEF_PAIR 3035, 3513 +COEF_PAIR 3166, 1931 +COEF_PAIR 3166, 3612 +COEF_PAIR 3166, 3920 +COEF_PAIR 3703, 3290 +COEF_PAIR 3857, 4052 +COEF_PAIR 4017, 2276 +COEF_PAIR 4017, 3406 +COEF_PAIR 4076, 1189 +COEF_PAIR 4076, 3612 +COEF_PAIR 4076, 3920 +COEF_PAIR 4091, 3973 + +pb_32: times 4 db 32 +pw_5: times 2 dw 5 +pw_4096: times 2 dw 4096 +pw_8192: times 2 dw 8192 +pw_1697x16: times 2 dw 1697*16 +pw_2896x8: times 2 dw 2896*8 +pixel_10bpc_max: times 2 dw 0x03ff +dconly_10bpc: times 2 dw 0x7c00 +clip_18b_min: dd -0x20000 +clip_18b_max: dd 0x1ffff +pd_1: dd 1 +pd_2: dd 2 +pd_1448: dd 1448 +pd_2048: dd 2048 +pd_3071: dd 3071 ; 1024 + 2048 - 1 +pd_3072: dd 3072 ; 1024 + 2048 +pd_5119: dd 5119 ; 1024 + 4096 - 1 +pd_5120: dd 5120 ; 1024 + 4096 +pd_5793: dd 5793 + +cextern dup16_perm +cextern int8_permA +cextern idct_8x8_internal_8bpc_avx512icl.main +cextern iadst_8x8_internal_8bpc_avx512icl.main_pass2 +cextern idct_8x16_internal_8bpc_avx512icl.main +cextern idct_8x16_internal_8bpc_avx512icl.main2 +cextern idct_8x16_internal_8bpc_avx512icl.main_fast +cextern idct_8x16_internal_8bpc_avx512icl.main_fast2 +cextern iadst_8x16_internal_8bpc_avx512icl.main2 +cextern idct_16x8_internal_8bpc_avx512icl.main +cextern iadst_16x8_internal_8bpc_avx512icl.main_pass2 +cextern idct_16x16_internal_8bpc_avx512icl.main +cextern idct_16x16_internal_8bpc_avx512icl.main2 +cextern idct_16x16_internal_8bpc_avx512icl.main_fast +cextern idct_16x16_internal_8bpc_avx512icl.main_fast2 +cextern iadst_16x16_internal_8bpc_avx512icl.main_pass2b +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast2 +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_end +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast2 +cextern inv_txfm_add_dct_dct_32x8_8bpc_avx512icl.main +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast2 +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast2 +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx512icl.main_oddhalf_fast + +SECTION .text + +%define o_base (pw_2048+4*128) +%define o_base_8bpc (int8_permA+64*18) +%define o(x) (r5 - o_base + (x)) +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +INIT_ZMM avx512icl + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +; flags: 1 = inv_dst1, 2 = inv_dst2 +; skip round/shift if rnd is not a number +%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], flags +%if %8 < 32 + pmulld m%4, m%1, m%8 + pmulld m%3, m%2, m%8 +%else +%if %8 < 4096 + vpbroadcastd m%3, [o(pd_%8)] +%else + vbroadcasti32x4 m%3, [o(pd_%8)] +%endif + pmulld m%4, m%1, m%3 + pmulld m%3, m%2 +%endif +%if %7 < 32 + pmulld m%1, m%7 + pmulld m%2, m%7 +%else +%if %7 < 4096 + vpbroadcastd m%5, [o(pd_%7)] +%else + vbroadcasti32x4 m%5, [o(pd_%7)] +%endif + pmulld m%1, m%5 + pmulld m%2, m%5 +%endif +%if %9 & 2 + psubd m%4, m%6, m%4 + psubd m%2, m%4, m%2 +%else +%ifnum %6 + paddd m%4, m%6 +%endif + paddd m%2, m%4 +%endif +%ifnum %6 + paddd m%1, m%6 +%endif +%if %9 & 1 + psubd m%1, m%3, m%1 +%else + psubd m%1, m%3 +%endif +%ifnum %6 + psrad m%2, 12 + psrad m%1, 12 +%endif +%endmacro + +%macro INV_TXFM_FN 4 ; type1, type2, eob_offset, size +cglobal inv_txfm_add_%1_%2_%4_10bpc, 4, 7, 0, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%4_internal_10bpc) + lea r5, [o_base] + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [m(i%2_%4_internal_10bpc).pass2] +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else +%if %3 + add eobd, %3 +%endif + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endmacro + +%macro INV_TXFM_8X8_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 8x8 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 8 +.dconly: + add r6d, 384 + sar r6d, 9 +.dconly2: + vpbroadcastd ym2, [o(dconly_10bpc)] + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + vpbroadcastw ym1, r6d + paddsw ym1, ym2 +.dconly_loop: + mova xm0, [dstq+strideq*0] + vinserti32x4 ym0, [dstq+strideq*1], 1 + paddsw ym0, ym1 + psubusw ym0, ym2 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%endif +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst +INV_TXFM_8X8_FN dct, identity + +cglobal idct_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call .load + vpermi2q m1, m0, m2 ; 1 5 + vpermi2q m3, m6, m4 ; 7 3 + vpermt2q m0, m5, m4 ; 0 2 + vpermt2q m2, m5, m6 ; 4 6 + call .main + call .main_end + mova m4, [o(idct8x8p)] + packssdw m0, m2 ; 0 1 4 5 + packssdw m1, m3 ; 3 2 7 6 + vpermb m0, m4, m0 + vprolq m1, 32 + vpermb m2, m4, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + jmp tx2q +.pass2: + lea r5, [o_base_8bpc] + vextracti32x8 ym2, m0, 1 + vextracti32x8 ym3, m1, 1 + call m(idct_8x8_internal_8bpc).main + mova m10, [permC] + vpbroadcastd m12, [pw_2048] +.end: + vpermt2q m0, m10, m1 + vpermt2q m2, m10, m3 +.end2: + vpbroadcastd m11, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m10, m10 + pmulhrsw m8, m12, m0 + call .write_8x4_start + pmulhrsw m8, m12, m2 +.write_8x4: + lea dstq, [dstq+strideq*4] + add cq, 64*2 +.write_8x4_start: + mova xm9, [dstq+strideq*0] + vinserti32x4 ym9, [dstq+strideq*1], 1 + vinserti32x4 m9, [dstq+strideq*2], 2 + vinserti32x4 m9, [dstq+r6 ], 3 + mova [cq+64*0], m10 + mova [cq+64*1], m10 + paddw m9, m8 + pmaxsw m9, m10 + pminsw m9, m11 + mova [dstq+strideq*0], xm9 + vextracti32x4 [dstq+strideq*1], ym9, 1 + vextracti32x4 [dstq+strideq*2], m9, 2 + vextracti32x4 [dstq+r6 ], m9, 3 + ret +ALIGN function_align +.load: + mova m0, [cq+64*0] ; 0 1 + mova m4, [cq+64*1] ; 2 3 + mova m1, [o(permB)] + mova m2, [cq+64*2] ; 4 5 + mova m6, [cq+64*3] ; 6 7 + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + psrlq m5, m1, 32 + vpbroadcastd m12, [o(pd_2896)] + mova m3, m1 + vpbroadcastd m11, [o(pd_1)] + ret +ALIGN function_align +.main_fast: ; bottom half is zero + vbroadcasti32x4 m3, [o(pd_4017_3406)] + vbroadcasti32x4 m8, [o(pd_799_m2276)] + vbroadcasti32x4 m2, [o(pd_2896_3784)] + vbroadcasti32x4 m9, [o(pd_2896_1567)] + pmulld m3, m1 ; t4a t5a + pmulld m1, m8 ; t7a t6a + pmulld m2, m0 ; t0 t3 + pmulld m0, m9 ; t1 t2 + jmp .main2 +.main: + ITX_MULSUB_2D 1, 3, 8, 9, 10, _, 799_3406, 4017_2276 + ITX_MULSUB_2D 0, 2, 8, 9, 10, _, 2896_1567, 2896_3784 +.main2: + REPX {paddd x, m13}, m1, m3, m0, m2 + REPX {psrad x, 12 }, m1, m3, m0, m2 + punpcklqdq m8, m1, m3 ; t4a t7a + punpckhqdq m1, m3 ; t5a t6a + psubd m3, m8, m1 ; t5a t6a + paddd m8, m1 ; t4 t7 + pmaxsd m3, m14 + punpckhqdq m1, m2, m0 ; t3 t2 + pminsd m3, m15 + punpcklqdq m2, m0 ; t0 t1 + pmulld m3, m12 + paddd m0, m2, m1 ; dct4 out0 out1 + psubd m2, m1 ; dct4 out3 out2 + REPX {pmaxsd x, m14}, m8, m0, m2 + REPX {pminsd x, m15}, m8, m0, m2 +.main3: + pshufd m1, m3, q1032 + paddd m3, m13 + psubd m9, m3, m1 + paddd m3, m1 + psrad m9, 12 + psrad m3, 12 + punpckhqdq m1, m8, m3 ; t7 t6 + shufpd m8, m9, 0xaa ; t4 t5 + ret +.main_end: + paddd m0, m11 + paddd m2, m11 + psubd m3, m0, m1 ; out7 out6 + paddd m0, m1 ; out0 out1 + paddd m1, m2, m8 ; out3 out2 + psubd m2, m8 ; out4 out5 + REPX {vpsravd x, m11}, m0, m2, m3, m1 + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity +INV_TXFM_8X8_FN adst, adst + +cglobal iadst_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call m(idct_8x8_internal_10bpc).load + vpermi2q m1, m6, m2 ; 7 5 + vpermi2q m3, m4, m0 ; 3 1 + vpermt2q m0, m5, m4 ; 0 2 + vpermt2q m2, m5, m6 ; 4 6 + call .main + punpckldq m1, m2, m4 ; out4 out6 + punpckhdq m2, m0 ; -out5 -out7 + punpckldq m0, m3 ; out0 out2 + punpckhdq m4, m3 ; -out1 -out3 + paddd m1, m11 + psubd m3, m11, m2 + paddd m0, m11 + psubd m4, m11, m4 +.pass1_end: + REPX {psrad x, 1}, m1, m0, m3, m4 + packssdw m0, m1 ; 0 2 4 6 + packssdw m4, m3 ; 1 3 5 7 + psrlq m1, [o(permB)], 8 + punpckhwd m3, m0, m4 + punpcklwd m0, m4 + psrlq m2, m1, 32 + vpermi2q m1, m0, m3 + vpermt2q m0, m2, m3 + jmp tx2q +.pass2: + call .main_pass2 + movu m10, [permC+2] + vbroadcasti32x8 m12, [pw_2048_m2048+16] + jmp m(idct_8x8_internal_10bpc).end +.main_pass2: + vextracti32x8 ym2, m0, 1 + vextracti32x8 ym3, m1, 1 + lea r5, [o_base_8bpc] + pshufd ym4, ym0, q1032 + pshufd ym5, ym1, q1032 + jmp m(iadst_8x8_internal_8bpc).main_pass2 +ALIGN function_align +.main: + ITX_MULSUB_2D 1, 0, 4, 5, 6, 13, 401_1931, 4076_3612 + ITX_MULSUB_2D 3, 2, 4, 5, 6, 13, 3166_3920, 2598_1189 + psubd m4, m0, m2 ; t4 t6 + paddd m0, m2 ; t0 t2 + psubd m2, m1, m3 ; t5 t7 + paddd m1, m3 ; t1 t3 + REPX {pmaxsd x, m14}, m4, m2, m0, m1 + REPX {pminsd x, m15}, m4, m2, m0, m1 + pxor m5, m5 + psubd m5, m4 + shufpd m4, m2, 0xaa ; t4 t7 + shufpd m2, m5, 0xaa ; t5 -t6 + ITX_MULSUB_2D 4, 2, 3, 5, 6, 13, 1567, 3784 + punpckhqdq m3, m0, m1 + punpcklqdq m0, m1 + psubd m1, m0, m3 ; t2 t3 + paddd m0, m3 ; out0 -out7 + punpckhqdq m3, m4, m2 ; t7a t6a + punpcklqdq m4, m2 ; t5a t4a + psubd m2, m4, m3 ; t7 t6 + paddd m4, m3 ; out6 -out1 + REPX {pmaxsd x, m14}, m1, m2 + REPX {pminsd x, m15}, m1, m2 + shufpd m3, m1, m2, 0xaa + shufpd m1, m2, 0x55 + pmulld m3, m12 + pmulld m1, m12 + paddd m3, m13 + psubd m2, m3, m1 + paddd m3, m1 + psrad m2, 12 ; out4 -out5 + pshufd m3, m3, q1032 + psrad m3, 12 ; out2 -out3 + ret + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, identity +INV_TXFM_8X8_FN flipadst, flipadst + +cglobal iflipadst_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call m(idct_8x8_internal_10bpc).load + vpermi2q m1, m6, m2 ; 7 5 + vpermi2q m3, m4, m0 ; 3 1 + vpermt2q m0, m5, m4 ; 0 2 + vpermt2q m2, m5, m6 ; 4 6 + call m(iadst_8x8_internal_10bpc).main + punpckhdq m1, m3, m4 ; -out3 -out1 + punpckldq m3, m0 ; out2 out0 + punpckhdq m0, m2 ; -out7 -out5 + punpckldq m4, m2 ; out6 out4 + psubd m1, m11, m1 + paddd m3, m11 + psubd m0, m11, m0 + paddd m4, m11 + jmp m(iadst_8x8_internal_10bpc).pass1_end +.pass2: + call m(iadst_8x8_internal_10bpc).main_pass2 + movu m10, [permC+1] + vbroadcasti32x8 m12, [pw_m2048_2048+16] + lea r6, [strideq*3] + vpermt2q m0, m10, m1 ; 7 6 5 4 + vpbroadcastd m11, [pixel_10bpc_max] + vpermt2q m2, m10, m3 ; 3 2 1 0 + pxor m10, m10 + pmulhrsw m8, m12, m2 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m8, m12, m0 + jmp m(idct_8x8_internal_10bpc).write_8x4 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + mova m1, [cq+64*0] + packssdw m1, [cq+64*2] ; 0 4 1 5 + mova m2, [cq+64*1] ; 2 6 3 7 + packssdw m2, [cq+64*3] + mova m0, [o(idtx8x8p)] + vpermb m1, m0, m1 + vpermb m2, m0, m2 + punpckldq m0, m1, m2 ; 0 1 4 5 + punpckhdq m1, m2 ; 2 3 6 7 + jmp tx2q +.pass2: + movu m3, [o(permC+2)] + vpbroadcastd m12, [o(pw_4096)] + psrlq m2, m3, 32 + vpermi2q m2, m0, m1 + vpermt2q m0, m3, m1 + jmp m(idct_8x8_internal_10bpc).end2 + +%macro INV_TXFM_8X16_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 8x16 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly +%endif +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, identity, 35 +INV_TXFM_8X16_FN dct, flipadst +INV_TXFM_8X16_FN dct, adst + +cglobal idct_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + cmp eobd, 43 + jl .fast + call .load + call .main + call .main_end +.pass1_end: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + jmp tx2q +.pass2: + mova m8, [o(idct8x16p)] + REPX {vpermb x, m8, x}, m0, m1, m2, m3 + punpckhdq m5, m0, m1 + punpckldq m0, m1 + punpckhdq m4, m2, m3 + punpckldq m2, m3 + punpcklqdq m8, m0, m2 ; 15 1 + punpckhqdq m0, m2 ; 7 9 + punpckhqdq m1, m5, m4 ; 3 13 + punpcklqdq m5, m4 ; 11 5 + lea r5, [o_base_8bpc] + vextracti32x8 ym7, m8, 1 ; 14 2 + vextracti32x8 ym3, m0, 1 ; 6 10 + vextracti32x8 ym6, m1, 1 ; 12 4 + vextracti32x8 ym9, m5, 1 ; 8 0 + call m(idct_8x16_internal_8bpc).main2 + mova m8, [permC] + vpbroadcastd m12, [pw_2048] + vpermt2q m0, m8, m1 + lea r6, [strideq*3] + vpermt2q m2, m8, m3 + vpbroadcastd m11, [pixel_10bpc_max] + vpermt2q m4, m8, m5 + pxor m10, m10 + vpermt2q m6, m8, m7 + pmulhrsw m8, m12, m0 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m8, m12, m2 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m8, m12, m4 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m8, m12, m6 + jmp m(idct_8x8_internal_10bpc).write_8x4 +.fast: + mova ym0, [cq+64*0] + mova ym4, [cq+64*2] + mova ym1, [cq+64*1] + mova ym5, [cq+64*5] + mova ym2, [cq+64*4] + mova ym6, [cq+64*6] + mova ym3, [cq+64*7] + mova ym7, [cq+64*3] + call .round_input_fast + call m(idct_8x8_internal_10bpc).main + call m(idct_8x8_internal_10bpc).main_end + movu m6, [o(permC+3)] + packssdw m3, m1, m3 + packssdw m1, m0, m2 + vprolq m3, 32 + vpermd m1, m6, m1 + vpermd m3, m6, m3 + mova ym0, ym1 ; 0 4 + vextracti32x8 ym1, m1, 1 ; 1 5 + mova ym2, ym3 ; 2 6 + vextracti32x8 ym3, m3, 1 ; 3 7 + jmp tx2q +ALIGN function_align +.round_input_fast: + movshdup m8, [o(permB)] + vpbroadcastd m12, [o(pd_2896)] + vpermt2q m0, m8, m4 + vpermt2q m1, m8, m5 + vpermt2q m2, m8, m6 + vpermt2q m3, m8, m7 + vpbroadcastd m13, [o(pd_2048)] + REPX {pmulld x, m12}, m0, m1, m2, m3 + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + REPX {paddd x, m13}, m0, m1, m2, m3 + vpbroadcastd m11, [o(pd_1)] + REPX {psrad x, 12 }, m0, m1, m2, m3 + ret +ALIGN function_align +.load: + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] +.load2: + vpbroadcastd m12, [o(pd_2896)] + pmulld m0, m12, [cq+64*0] + pmulld m1, m12, [cq+64*1] + pmulld m2, m12, [cq+64*2] + pmulld m3, m12, [cq+64*3] + vpbroadcastd m13, [o(pd_2048)] + pmulld m4, m12, [cq+64*4] + pmulld m5, m12, [cq+64*5] + pmulld m6, m12, [cq+64*6] + pmulld m7, m12, [cq+64*7] +.round: + REPX {paddd x, m13}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 + REPX {paddd x, m13}, m4, m5, m6, m7 + REPX {psrad x, 12 }, m4, m5, m6, m7 + ret +ALIGN function_align +.main_fast_rect2: + REPX {paddd x, m13}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_fast: + pmulld m0, m12 + pmulld m5, m3, [o(pd_2276)] {1to16} ; t5a + pmulld m3, [o(pd_3406)] {1to16} ; t6a + pmulld m7, m1, [o(pd_4017)] {1to16} ; t7a + pmulld m1, [o(pd_799)] {1to16} ; t4a + pmulld m6, m2, [o(pd_3784)] {1to16} ; t3 + pmulld m2, [o(pd_1567)] {1to16} ; t2 + paddd m0, m13 + psubd m5, m13, m5 + psrad m0, 12 ; t0 + mova m9, m0 ; t1 + jmp .main2 +.main_rect2: + call .round +.main: + pmulld m0, m12 + ITX_MULSUB_2D 5, 3, 8, 9, 10, _, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 8, 9, 10, _, 799, 4017 ; t4a t7a + ITX_MULSUB_2D 2, 6, 8, 9, 10, _, 1567, 3784 ; t2 t3 + pmulld m4, m12 + paddd m0, m13 + paddd m5, m13 + psubd m9, m0, m4 ; t1 + paddd m0, m4 ; t0 + psrad m9, 12 + psrad m0, 12 +.main2: + REPX {paddd x, m13}, m3, m1, m7 + REPX {psrad x, 12 }, m5, m1, m3, m7 + paddd m8, m1, m5 ; t4 + psubd m1, m5 ; t5a + psubd m5, m7, m3 ; t6a + paddd m7, m3 ; t7 + pmaxsd m5, m14 + pmaxsd m1, m14 + paddd m2, m13 + paddd m6, m13 + pminsd m5, m15 + pminsd m1, m15 + pmulld m5, m12 + pmulld m1, m12 + pmaxsd m8, m14 + pmaxsd m7, m14 + pminsd m8, m15 + paddd m5, m13 + psubd m4, m5, m1 + paddd m5, m1 + REPX {psrad x, 12 }, m2, m6, m5, m4 + paddd m1, m9, m2 ; dct4 out1 + psubd m2, m9, m2 ; dct4 out2 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + pminsd m6, m15, m7 + REPX {pmaxsd x, m14}, m0, m1, m2, m3 + REPX {pminsd x, m15}, m0, m1, m2, m3 + ret +.main_end: + vpbroadcastd m11, [o(pd_1)] +.main_end2: + REPX {paddd x, m11}, m0, m1, m2, m3 + psubd m7, m0, m6 ; out7 + paddd m0, m6 ; out0 + psubd m6, m1, m5 ; out6 + paddd m1, m5 ; out1 + psubd m5, m2, m4 ; out5 + paddd m2, m4 ; out2 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 + REPX {vpsravd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + ret + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, identity, 35 +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, adst + +cglobal iadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + cmp eobd, 43 + jl .fast + call m(idct_8x16_internal_10bpc).load + call .main + psrad m0, 1 + psrad m1, 1 + psrad m6, m10, 1 + psrad m7, m11, 1 + psrad m2, 12 + psrad m3, 12 + psrad m4, m8, 12 + psrad m5, m9, 12 + jmp m(idct_8x16_internal_10bpc).pass1_end +.fast: + call .fast_main + punpcklqdq m1, m2, m4 ; out4 out6 + punpckhqdq m2, m0 ; -out5 -out7 + punpcklqdq m0, m3 ; out0 out2 + punpckhqdq m4, m3 ; -out1 -out3 + paddd m1, m11 + psubd m3, m11, m2 + paddd m0, m11 + psubd m4, m11, m4 +.fast_end: + movu m5, [o(permC+3)] + REPX {psrad x, 1}, m1, m0, m3, m4 + packssdw m2, m0, m1 ; 0 2 4 6 + packssdw m3, m4, m3 ; 1 3 5 7 + vpermd m2, m5, m2 + vpermd m3, m5, m3 + mova ym0, ym2 + vextracti32x8 ym2, m2, 1 + mova ym1, ym3 + vextracti32x8 ym3, m3, 1 + jmp tx2q +.pass2: + call .pass2_main + movu m4, [permB+2] + vbroadcasti32x8 m12, [pw_2048_m2048+16] + psrlq m7, m4, 8 + vpermi2q m4, m0, m3 ; 0 1 2 3 + psrlq m5, m7, 24 + vpermi2q m7, m0, m3 ; 12 13 14 15 + psrlq m6, m5, 8 + vpermq m5, m5, m1 ; 4 5 6 7 + vpermq m6, m6, m2 ; 8 9 10 11 +.pass2_end: + vpbroadcastd m11, [pixel_10bpc_max] + pxor m10, m10 + lea r6, [strideq*3] + pmulhrsw m8, m12, m4 + call m(idct_8x8_internal_10bpc).write_8x4_start + pmulhrsw m8, m12, m5 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m8, m12, m6 + call m(idct_8x8_internal_10bpc).write_8x4 + pmulhrsw m8, m12, m7 + jmp m(idct_8x8_internal_10bpc).write_8x4 +ALIGN function_align +.main: + ITX_MULSUB_2D 7, 0, 8, 9, 10, 13, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D 1, 6, 8, 9, 10, 13, 3920, 1189 ; t7a, t6a + ITX_MULSUB_2D 5, 2, 8, 9, 10, 13, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D 3, 4, 8, 9, 10, 13, 3166, 2598 ; t5a, t4a + psubd m8, m2, m6 ; t6 + paddd m2, m6 ; t2 + psubd m6, m0, m4 ; t4 + paddd m0, m4 ; t0 + psubd m4, m5, m1 ; t7 + paddd m5, m1 ; t3 + psubd m1, m7, m3 ; t5 + paddd m7, m3 ; t1 + REPX {pmaxsd x, m14}, m6, m1, m8, m4, m2, m0, m5, m7 + REPX {pminsd x, m15}, m6, m1, m8, m4, m2, m0, m5, m7 + vpbroadcastd m10, [o(pd_1567)] + vpbroadcastd m11, [o(pd_3784)] + ITX_MULSUB_2D 6, 1, 3, 9, _, 13, 10, 11 ; t5a, t4a + ITX_MULSUB_2D 4, 8, 3, 9, _, 13, 11, 10 ; t6a, t7a + vpbroadcastd m12, [o(pd_1448)] + psubd m9, m6, m8 ; t7 + paddd m6, m8 ; out6 + psubd m3, m7, m5 ; t3 + paddd m7, m5 ; -out7 + psubd m5, m0, m2 ; t2 + paddd m0, m2 ; out0 + psubd m2, m1, m4 ; t6 + paddd m1, m4 ; -out1 + REPX {pmaxsd x, m14}, m5, m3, m2, m9 + REPX {pminsd x, m15}, m5, m3, m2, m9 + REPX {pmulld x, m12}, m5, m3, m2, m9 + vpbroadcastd m4, [o(pd_1)] + psubd m8, m5, m3 ; (t2 - t3) * 1448 + paddd m3, m5 ; (t2 + t3) * 1448 + psubd m5, m2, m9 ; (t6 - t7) * 1448 + paddd m2, m9 ; (t6 + t7) * 1448 + vpbroadcastd m9, [o(pd_3072)] + paddd m0, m4 + psubd m1, m4, m1 + paddd m10, m6, m4 + psubd m11, m4, m7 + paddd m2, m9 + paddd m8, m9 + vpbroadcastd m9, [o(pd_3071)] + psubd m3, m9, m3 + psubd m9, m5 + ret +ALIGN function_align +.fast_main: + mova ym0, [cq+64*0] + mova ym4, [cq+64*2] + mova ym1, [cq+64*7] + mova ym5, [cq+64*5] + mova ym2, [cq+64*4] + mova ym6, [cq+64*6] + mova ym3, [cq+64*3] + mova ym7, [cq+64*1] + call m(idct_8x16_internal_10bpc).round_input_fast + jmp m(iadst_8x8_internal_10bpc).main +ALIGN function_align +.pass2_main: + mova m8, [o(iadst8x16p)] + REPX {vpermb x, m8, x}, m0, m1, m2, m3 + vpbroadcastd m10, [o(pw_2896x8)] + punpckhdq m5, m0, m1 + punpckldq m0, m1 + punpckhdq m1, m2, m3 + punpckldq m2, m3 + lea r5, [o_base_8bpc] + punpckhqdq m4, m0, m2 ; 12 3 14 1 + punpcklqdq m0, m2 ; 0 15 2 13 + punpckhqdq m6, m5, m1 ; 8 7 10 5 + punpcklqdq m5, m1 ; 4 11 6 9 + call m(iadst_8x16_internal_8bpc).main2 + paddsw m1, m2, m4 + psubsw m2, m4 + pmulhrsw m1, m10 ; -out7 out4 out6 -out5 + pmulhrsw m2, m10 ; out8 -out11 -out9 out10 + ret + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, identity, 35 +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst + +cglobal iflipadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + cmp eobd, 43 + jl .fast + call m(idct_8x16_internal_10bpc).load + call m(iadst_8x16_internal_10bpc).main + psrad m7, m0, 1 + psrad m0, m11, 1 + psrad m6, m1, 1 + psrad m1, m10, 1 + psrad m5, m2, 12 + psrad m2, m9, 12 + psrad m4, m3, 12 + psrad m3, m8, 12 + jmp m(idct_8x16_internal_10bpc).pass1_end +.fast: + call m(iadst_8x16_internal_10bpc).fast_main + punpckhqdq m1, m3, m4 ; -out3 -out1 + punpcklqdq m3, m0 ; out2 out0 + punpckhqdq m0, m2 ; -out7 -out5 + punpcklqdq m4, m2 ; out6 out4 + psubd m1, m11, m1 + paddd m3, m11 + psubd m0, m11, m0 + paddd m4, m11 + jmp m(iadst_8x16_internal_10bpc).fast_end +.pass2: + call m(iadst_8x16_internal_10bpc).pass2_main + movu m7, [permB+2] + vbroadcasti32x8 m12, [pw_m2048_2048+16] + psrlq m4, m7, 8 + vpermi2q m7, m3, m0 ; 3 2 1 0 + psrlq m5, m4, 24 + vpermi2q m4, m3, m0 ; 15 14 13 12 + psrlq m6, m5, 8 + vpermq m5, m5, m2 ; 11 10 9 8 + vpermq m6, m6, m1 ; 7 6 5 4 + jmp m(iadst_8x16_internal_10bpc).pass2_end + +INV_TXFM_8X16_FN identity, dct +INV_TXFM_8X16_FN identity, adst +INV_TXFM_8X16_FN identity, flipadst +INV_TXFM_8X16_FN identity, identity + +cglobal iidentity_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call m(idct_8x16_internal_10bpc).load2 + jmp m(idct_8x16_internal_10bpc).pass1_end +.pass2: + vpbroadcastd m8, [o(pw_1697x16)] + pmulhrsw m4, m8, m0 + pmulhrsw m5, m8, m1 + pmulhrsw m6, m8, m2 + pmulhrsw m7, m8, m3 + REPX {paddsw x, x}, m0, m1, m2, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + vpbroadcastd m7, [o(pw_2048)] + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + vpbroadcastd m6, [o(pixel_10bpc_max)] + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m1 + punpckhdq m4, m1 + pxor m5, m5 + punpckhqdq m1, m0, m2 ; 1 5 9 13 + punpcklqdq m0, m2 ; 0 4 8 12 + punpcklqdq m2, m3, m4 ; 2 6 10 14 + punpckhqdq m3, m4 ; 3 7 11 15 + lea r6, [strideq*3] + pmulhrsw m0, m7 + call .write_8x4_start + pmulhrsw m0, m7, m1 + call .write_8x4 + pmulhrsw m0, m7, m2 + call .write_8x4 + pmulhrsw m0, m7, m3 +.write_8x4: + add dstq, strideq + add cq, 64*2 +.write_8x4_start: + mova xm4, [dstq+strideq*0] + vinserti32x4 ym4, [dstq+strideq*4], 1 + vinserti32x4 m4, [dstq+strideq*8], 2 + vinserti32x4 m4, [dstq+r6*4 ], 3 + mova [cq+64*0], m5 + mova [cq+64*1], m5 + paddw m4, m0 + pmaxsw m4, m5 + pminsw m4, m6 + mova [dstq+strideq*0], xm4 + vextracti32x4 [dstq+strideq*4], ym4, 1 + vextracti32x4 [dstq+strideq*8], m4, 2 + vextracti32x4 [dstq+r6*4 ], m4, 3 + ret + +%macro INV_TXFM_16X8_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 16x8 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 8 +.dconly: + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 +.dconly2: + vpbroadcastd m2, [o(dconly_10bpc)] + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + vpbroadcastw m1, r6d + paddsw m1, m2 +.dconly_loop: + mova ym0, [dstq+strideq*0] + vinserti32x8 m0, [dstq+strideq*1], 1 + paddsw m0, m1 + psubusw m0, m2 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%endif +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, identity, -21 +INV_TXFM_16X8_FN dct, flipadst +INV_TXFM_16X8_FN dct, adst + +cglobal idct_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m12, [o(pd_2896)] + pmulld m4, m12, [cq+64*0] ; 0 1 + pmulld m9, m12, [cq+64*1] ; 2 3 + pmulld m8, m12, [cq+64*2] ; 4 5 + pmulld m7, m12, [cq+64*3] ; 6 7 + vpbroadcastd m13, [o(pd_2048)] + pxor m2, m2 + mova m15, [o(permB)] + REPX {mova [cq+64*x], m2}, 0, 1, 2, 3 + psrlq m0, m15, 32 + REPX {paddd x, m13}, m4, m9, m8, m7 + vpbroadcastd m14, [o(clip_18b_min)] + REPX {psrad x, 12 }, m4, m8, m9, m7 + mova m1, m0 + vpermi2q m0, m4, m8 ; 0 4 + cmp eobd, 43 + jl .fast + pmulld m5, m12, [cq+64*4] ; 8 9 + pmulld m10, m12, [cq+64*5] ; 10 11 + pmulld m11, m12, [cq+64*6] ; 12 13 + pmulld m6, m12, [cq+64*7] ; 14 15 + REPX {mova [cq+64*x], m2}, 4, 5, 6, 7 + REPX {paddd x, m13}, m5, m10, m11, m6 + REPX {psrad x, 12 }, m10, m5, m11, m6 + mova m2, m1 + vpermi2q m1, m9, m10 ; 2 10 + mova m3, m2 + vpermi2q m2, m5, m11 ; 8 12 + vpermi2q m3, m6, m7 ; 14 6 + vpermt2q m4, m15, m11 ; 1 13 + vpermt2q m6, m15, m9 ; 15 3 + vpermt2q m5, m15, m8 ; 9 5 + vpermt2q m7, m15, m10 ; 7 11 + vpbroadcastd m15, [o(clip_18b_max)] + call m(idct_8x8_internal_10bpc).main + call .main + jmp .pass1_end +.fast: + vpermi2q m1, m9, m7 ; 2 6 + vpermt2q m4, m15, m9 ; 1 3 + vpermt2q m7, m15, m8 ; 7 5 + vpbroadcastd m15, [o(clip_18b_max)] + call m(idct_8x8_internal_10bpc).main_fast + call .main_fast +.pass1_end: + call m(idct_8x16_internal_10bpc).main_end + mova m8, [o(permA)] + psrlq m9, m8, 8 +.pass1_end2: + mova m10, m9 + mova m11, m8 + call .transpose_16x8 + jmp tx2q +.pass2: + lea r5, [o_base_8bpc] + call m(idct_16x8_internal_8bpc).main + movshdup m4, [permC] + vpbroadcastd m11, [pw_2048] + psrlq m5, m4, 8 +.end: + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 + vpermq m8, m4, m0 + vpermq m9, m5, m1 + lea r6, [strideq*3] + call .write_16x4 + vpermq m8, m4, m2 + vpermq m9, m5, m3 +.write_16x4: + pmulhrsw m8, m11 + pmulhrsw m9, m11 +.write_16x4_noround: + mova ym10, [dstq+strideq*0] + vinserti32x8 m10, [dstq+strideq*1], 1 + paddw m8, m10 + mova ym10, [dstq+strideq*2] + vinserti32x8 m10, [dstq+r6 ], 1 + paddw m9, m10 + pmaxsw m8, m12 + pmaxsw m9, m12 + pminsw m8, m13 + pminsw m9, m13 + mova [dstq+strideq*0], ym8 + vextracti32x8 [dstq+strideq*1], m8, 1 + mova [dstq+strideq*2], ym9 + vextracti32x8 [dstq+r6 ], m9, 1 + lea dstq, [dstq+strideq*4] + ret +ALIGN function_align +.main_fast: ; bottom half is zero + vbroadcasti32x4 m6, [o(pd_4076_3920)] + vbroadcasti32x4 m3, [o(pd_401_m1189)] + vbroadcasti32x4 m5, [o(pd_m2598_1931)] + vbroadcasti32x4 m9, [o(pd_3166_3612)] + pmulld m6, m4 ; t15a t12a + pmulld m4, m3 ; t8a t11a + pmulld m5, m7 ; t9a t10a + pmulld m7, m9 ; t14a t13a + jmp .main2 +.main: + ITX_MULSUB_2D 4, 6, 3, 9, 10, _, 401_3920, 4076_1189 + ITX_MULSUB_2D 5, 7, 3, 9, 10, _, 3166_1931, 2598_3612 +.main2: + REPX {paddd x, m13}, m4, m6, m5, m7 + REPX {psrad x, 12 }, m4, m5, m6, m7 + paddd m9, m4, m5 ; t8 t11 + psubd m4, m5 ; t9 t10 + psubd m5, m6, m7 ; t14 t13 + paddd m6, m7 ; t15 t12 + REPX {pmaxsd x, m14}, m5, m4, m9, m6 + REPX {pminsd x, m15}, m5, m4, m9, m6 +.main3: + psubd m3, m0, m1 ; dct8 out7 out6 + paddd m0, m1 ; dct8 out0 out1 + vbroadcasti32x4 m7, [o(pd_3784_m3784)] + pmulld m7, m5 + vpmulld m5, [o(pd_1567)] {1to16} + paddd m1, m2, m8 ; dct8 out3 out2 + psubd m2, m8 ; dct8 out4 out5 + vbroadcasti32x4 m8, [o(pd_1567_m1567)] + pmulld m8, m4 + vpmulld m4, [o(pd_3784)] {1to16} + REPX {pmaxsd x, m14}, m0, m1 + REPX {pminsd x, m15}, m0, m1 + paddd m7, m13 + paddd m5, m13 + paddd m7, m8 + psubd m5, m4 + psrad m7, 12 ; t14a t10a + psrad m5, 12 ; t9a t13a + punpckhqdq m4, m9, m7 + punpcklqdq m8, m9, m5 + punpckhqdq m5, m6, m5 + punpcklqdq m6, m7 + psubd m7, m8, m4 ; t11a t10 + paddd m8, m4 ; t8a t9 + psubd m4, m6, m5 ; t12a t13 + paddd m6, m5 ; t15a t14 + REPX {pmaxsd x, m14}, m4, m7 + REPX {pminsd x, m15}, m4, m7 + pmulld m4, m12 + pmulld m7, m12 + REPX {pmaxsd x, m14}, m2, m3, m6, m8 + REPX {pminsd x, m15}, m2, m3, m6, m8 + paddd m4, m13 + paddd m5, m4, m7 + psubd m4, m7 + psrad m4, 12 ; t11 t10a + psrad m5, 12 ; t12 t13a + ret +ALIGN function_align +.transpose_16x8: + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + vpermi2d m8, m0, m2 + vpermt2d m0, m9, m2 + vpermi2d m10, m1, m3 + vpermi2d m11, m1, m3 + punpckhwd m3, m8, m0 + punpcklwd m1, m8, m0 + punpckhwd m4, m10, m11 + punpcklwd m2, m10, m11 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + ret + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, identity, -21 +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, adst + +cglobal iadst_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + call .main_pass1 + vpbroadcastd m9, [o(pd_1)] + paddd m0, m9 + psubd m1, m9, m1 + paddd m2, m9 + psubd m3, m9, m3 + paddd m4, m9, m5 + psubd m5, m9, m6 + paddd m6, m9, m7 + psubd m7, m9, m8 +.pass1_end: + mova m9, [o(permA)] + psrlq m8, m9, 8 + REPX {psrad x, 1}, m0, m4, m1, m5, m2, m6, m3, m7 + jmp m(idct_16x8_internal_10bpc).pass1_end2 +.pass2: + call .main_pass2 + vpermq m8, m11, m0 + vpermq m9, m11, m1 + call m(idct_16x8_internal_10bpc).write_16x4_noround + vpermq m8, m11, m2 + vpermq m9, m11, m3 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround +ALIGN function_align +.main_pass1: + vpbroadcastd m12, [o(pd_2896)] + pmulld m2, m12, [cq+64*0] + pmulld m7, m12, [cq+64*1] + pmulld m1, m12, [cq+64*2] + pmulld m5, m12, [cq+64*3] + vpbroadcastd m13, [o(pd_2048)] + pxor m4, m4 + mova m10, [o(permB)] + REPX {mova [cq+64*x], m4}, 0, 1, 2, 3 + REPX {paddd x, m13}, m2, m7, m1, m5 + psrlq m6, m10, 32 + REPX {psrad x, 12 }, m2, m7, m1, m5 + mova m0, m6 + vpermi2q m0, m2, m7 ; 0 2 + vpermt2q m7, m10, m2 ; 3 1 + mova m2, m6 + vpermi2q m2, m1, m5 ; 4 6 + vpermt2q m5, m10, m1 ; 7 5 + cmp eobd, 43 + jl .main_fast + pmulld m8, m12, [cq+64*4] + pmulld m3, m12, [cq+64*5] + pmulld m9, m12, [cq+64*6] + pmulld m1, m12, [cq+64*7] + REPX {mova [cq+64*x], m4}, 4, 5, 6, 7 + REPX {paddd x, m13}, m8, m3, m9, m1 + REPX {psrad x, 12 }, m8, m3, m9, m1 + mova m4, m6 + vpermi2q m4, m8, m3 ; 8 10 + vpermt2q m3, m10, m8 ; 11 9 + vpermi2q m6, m9, m1 ; 12 14 + vpermt2q m1, m10, m9 ; 15 13 +.main: + ITX_MULSUB_2D 1, 0, 8, 9, 10, _, 201_995, 4091_3973, 1 + ITX_MULSUB_2D 3, 2, 8, 9, 10, _, 1751_2440, 3703_3290, 1 + ITX_MULSUB_2D 5, 4, 8, 9, 10, _, 3035_3513, 2751_2106 + ITX_MULSUB_2D 7, 6, 8, 9, 10, _, 3857_4052, 1380_601 + jmp .main2 +.main_fast: + vbroadcasti32x4 m1, [o(pd_4091_3973)] + vbroadcasti32x4 m8, [o(pd_201_995)] + vbroadcasti32x4 m3, [o(pd_3703_3290)] + vbroadcasti32x4 m9, [o(pd_1751_2440)] + vbroadcasti32x4 m4, [o(pd_2751_2106)] + vbroadcasti32x4 m10, [o(pd_3035_3513)] + vbroadcasti32x4 m6, [o(pd_1380_601)] + vbroadcasti32x4 m11, [o(pd_3857_4052)] + pmulld m1, m0 + pmulld m0, m8 + pmulld m3, m2 + pmulld m2, m9 + pmulld m4, m5 + pmulld m5, m10 + pmulld m6, m7 + pmulld m7, m11 +.main2: + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + REPX {psubd x, m13, x}, m1, m3 + REPX {paddd x, m13 }, m0, m2, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m4, m1, m5, m2, m6, m3, m7 + psubd m8, m0, m4 ; t8a t10a + paddd m0, m4 ; t0a t2a + psubd m4, m1, m5 ; t9a t11a + paddd m1, m5 ; t1a t3a + psubd m5, m2, m6 ; t12a t14a + paddd m2, m6 ; t4a t6a + psubd m6, m3, m7 ; t13a t15a + paddd m3, m7 ; t5a t7a + REPX {pmaxsd x, m14}, m8, m4, m5, m6 + REPX {pminsd x, m15}, m8, m4, m5, m6 + vbroadcasti32x4 m11, [o(pd_4017_2276)] + vbroadcasti32x4 m10, [o(pd_799_3406)] + ITX_MULSUB_2D 8, 4, 7, 9, _, 13, 10, 11 + ITX_MULSUB_2D 6, 5, 7, 9, _, 13, 11, 10 + REPX {pmaxsd x, m14}, m0, m2, m1, m3 + REPX {pminsd x, m15}, m0, m2, m1, m3 + psubd m7, m0, m2 ; t4 t6 + paddd m0, m2 ; t0 t2 + psubd m2, m1, m3 ; t5 t7 + paddd m1, m3 ; t1 t3 + psubd m3, m4, m6 ; t12a t14a + paddd m4, m6 ; t8a t10a + psubd m6, m8, m5 ; t13a t15a + paddd m8, m5 ; t9a t11a + REPX {pmaxsd x, m14}, m7, m3, m2, m6 + REPX {pminsd x, m15}, m7, m3, m2, m6 + punpcklqdq m5, m3, m7 ; t12a t4 + punpckhqdq m3, m7 ; t14a t6 + punpckhqdq m7, m6, m2 ; t15a t7 + punpcklqdq m6, m2 ; t13a t5 + vpbroadcastd m11, [o(pd_1567)] + vpbroadcastd m10, [o(pd_3784)] + ITX_MULSUB_2D 7, 3, 2, 9, 10, 13, 10, 11 + ITX_MULSUB_2D 5, 6, 2, 9, 10, 13, 11, 10 + REPX {pmaxsd x, m14}, m0, m4, m1, m8 + REPX {pminsd x, m15}, m0, m4, m1, m8 + punpckhqdq m2, m4, m0 ; t10a t2 + punpcklqdq m4, m0 ; t8a t0 + punpckhqdq m0, m8, m1 ; t11a t3 + punpcklqdq m8, m1 ; t9a t1 + paddd m1, m6, m7 ; out2 -out3 + psubd m6, m7 ; t14a t6 + paddd m7, m5, m3 ; -out13 out12 + psubd m5, m3 ; t15a t7 + psubd m3, m8, m0 ; t11 t3a + paddd m8, m0 ; out14 -out15 + paddd m0, m4, m2 ; -out1 out0 + psubd m4, m2 ; t10 t2a + REPX {pmaxsd x, m14}, m6, m5, m3, m4 + mov r6d, 0x3333 + REPX {pminsd x, m15}, m6, m5, m3, m4 + kmovw k1, r6d + REPX {pmulld x, m12}, m6, m5, m3, m4 + pxor m9, m9 + REPX {vpsubd x{k1}, m9, x}, m0, m1, m7, m8 + paddd m6, m13 + paddd m4, m13 + paddd m2, m6, m5 ; -out5 out4 + psubd m6, m5 ; out10 -out11 + psubd m5, m4, m3 ; -out9 out8 + paddd m3, m4 ; out6 -out7 + REPX {psrad x, 12}, m2, m3, m5, m6 + REPX {vpsubd x{k1}, m9, x}, m2, m3, m5, m6 + ret +ALIGN function_align +.main_pass2: + lea r5, [o_base_8bpc] + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call m(iadst_16x8_internal_8bpc).main_pass2 + movshdup m11, [permC] + pmulhrsw m0, m6 + pmulhrsw m1, m6 + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 + lea r6, [strideq*3] + ret + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, identity, -21 +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst + +cglobal iflipadst_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call m(iadst_16x8_internal_10bpc).main_pass1 + vpbroadcastd m9, [o(pd_1)] + psubd m4, m9, m3 + paddd m3, m9, m5 + paddd m5, m9, m2 + psubd m2, m9, m6 + psubd m6, m9, m1 + paddd m1, m9, m7 + paddd m7, m9, m0 + psubd m0, m9, m8 + jmp m(iadst_16x8_internal_10bpc).pass1_end +.pass2: + call m(iadst_16x8_internal_10bpc).main_pass2 + psrlq m11, 8 + vpermq m8, m11, m3 + vpermq m9, m11, m2 + call m(idct_16x8_internal_10bpc).write_16x4_noround + vpermq m8, m11, m1 + vpermq m9, m11, m0 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround + +INV_TXFM_16X8_FN identity, dct +INV_TXFM_16X8_FN identity, adst +INV_TXFM_16X8_FN identity, flipadst +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 + call m(idct_8x16_internal_10bpc).load2 + vpbroadcastd m8, [o(pd_5793)] + vpbroadcastd m13, [o(pd_3072)] + pxor m10, m10 + REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {mova [cq+64*x], m10}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x16_internal_10bpc).round + psrlq m8, [o(permA)], 16 + psrlq m9, m8, 8 + mova m10, m8 + mova m11, m9 + call m(idct_16x8_internal_10bpc).transpose_16x8 + jmp tx2q +.pass2: + movshdup m4, [o(permC)] + vpbroadcastd m11, [o(pw_4096)] + mova m5, m4 + jmp m(idct_16x8_internal_10bpc).end + +%macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 16x16 +%ifidn %1_%2, dct_dct + imul r6d, [cq], 181 + mov [cq], eobd ; 0 + or r3d, 16 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly2 +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, identity, 28 +INV_TXFM_16X16_FN dct, flipadst +INV_TXFM_16X16_FN dct, adst + +cglobal idct_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + cmp eobd, 36 + jl .fast + mova m0, [cq+64* 0] + mova m1, [cq+64* 2] + mova m2, [cq+64* 4] + mova m3, [cq+64* 6] + mova m4, [cq+64* 8] + mova m5, [cq+64*10] + mova m6, [cq+64*12] + mova m7, [cq+64*14] +%if WIN64 + movaps [cq+16*0], xmm6 + movaps [cq+16*1], xmm7 +%endif + call m(idct_8x16_internal_10bpc).main + mova m16, [cq+64* 1] + mova m17, [cq+64* 3] + mova m18, [cq+64* 5] + mova m19, [cq+64* 7] + mova m20, [cq+64* 9] + mova m21, [cq+64*11] + mova m22, [cq+64*13] + mova m23, [cq+64*15] + call .main + call .main_end +.pass1_end: +%if WIN64 + movaps xmm6, [cq+16*0] + movaps xmm7, [cq+16*1] +%endif + vzeroupper +.pass1_end2: + call .main_end3 +.pass1_end3: + mov r6d, 64*12 + pxor m8, m8 +.zero_loop: + mova [cq+r6+64*3], m8 + mova [cq+r6+64*2], m8 + mova [cq+r6+64*1], m8 + mova [cq+r6+64*0], m8 + sub r6d, 64*4 + jge .zero_loop + jmp tx2q +.pass2: + lea r5, [o_base_8bpc] + call m(idct_16x16_internal_8bpc).main + movshdup m12, [permC] + vpbroadcastd m11, [pw_2048] + psrlq m13, m12, 8 + vpermq m8, m12, m0 + vpermq m0, m13, m7 + vpermq m7, m13, m1 + vpermq m1, m12, m6 + vpermq m6, m12, m2 + vpermq m2, m13, m5 + vpermq m5, m13, m3 + vpermq m3, m12, m4 +.pass2_end: + lea r6, [strideq*3] + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 + pmulhrsw m8, m11, m8 + pmulhrsw m9, m11, m7 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m6 + pmulhrsw m9, m11, m5 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m3 + pmulhrsw m9, m11, m2 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m1 + pmulhrsw m9, m11, m0 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround +.fast: + mova ym0, [cq+64*0] + mova ym2, [cq+64*4] + movshdup m8, [o(permB)] + mova ym1, [cq+64*2] + mova ym3, [cq+64*6] + mova ym4, [cq+64*1] + mova ym5, [cq+64*3] + mova ym6, [cq+64*5] + mova ym7, [cq+64*7] + vpermt2q m0, m8, m2 ; 0 4 + vpermt2q m1, m8, m3 ; 2 6 + vpermt2q m4, m8, m5 ; 1 3 + vpermt2q m7, m8, m6 ; 7 5 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + vpbroadcastd m11, [o(pd_2)] + call m(idct_8x16_internal_10bpc).main_end2 + mova m8, [o(permA)] + psrlq m9, m8, 8 + jmp m(iadst_16x16_internal_10bpc).pass1_fast_end2 +ALIGN function_align +.main_fast_rect2: + REPX {paddd x, m13}, m16, m17, m18, m19 + REPX {psrad x, 12 }, m16, m17, m18, m19 +.main_fast: + pmulld m23, m16, [o(pd_4076)] {1to16} ; t15a + pmulld m16, [o(pd_401)] {1to16} ; t8a + pmulld m20, m19, [o(pd_2598)] {1to16} ; t9a + pmulld m19, [o(pd_3166)] {1to16} ; t14a + pmulld m22, m17, [o(pd_1189)] {1to16} ; t11a + pmulld m17, [o(pd_3920)] {1to16} ; t12a + pmulld m21, m18, [o(pd_3612)] {1to16} ; t13a + pmulld m18, [o(pd_1931)] {1to16} ; t10a + psubd m20, m13, m20 + psubd m22, m13, m22 + call .round2 + jmp .main2 +.main_rect2: + call .round +.main: + ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3166, 2598 ; t9a, t14a + ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3920, 1189 ; t11a, t12a + ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1931, 3612 ; t10a, t13a + call .round +.main2: + paddd m9, m20, m16 ; t8 + psubd m20, m16, m20 ; t9 + psubd m16, m22, m18 ; t10 + paddd m18, m22 ; t11 + paddd m22, m23, m19 ; t15 + psubd m23, m19 ; t14 + psubd m19, m17, m21 ; t13 + paddd m17, m21 ; t12 + vpbroadcastd m11, [o(pd_3784)] + REPX {pmaxsd x, m14}, m20, m23, m16, m19 + vpbroadcastd m10, [o(pd_1567)] + REPX {pminsd x, m15}, m20, m23, m16, m19 + ITX_MULSUB_2D 23, 20, 21, 7, _, 13, 10, 11 + ITX_MULSUB_2D 19, 16, 21, 7, _, 13, 10, 11, 2 + REPX {pmaxsd x, m14}, m9, m18, m22, m17 + REPX {pminsd x, m15}, m9, m18, m22, m17 + paddd m21, m20, m19 ; t14 + psubd m20, m19 ; t13 + psubd m19, m9, m18 ; t11a + paddd m9, m18 ; t8a + psubd m18, m23, m16 ; t10 + paddd m16, m23 ; t9 + psubd m23, m22, m17 ; t12a + paddd m22, m17 ; t15a + REPX {pmaxsd x, m14}, m20, m23, m18, m19 + REPX {pminsd x, m15}, m20, m23, m18, m19 + REPX {pmulld x, m12}, m20, m23, m18, m19 + psubd m7, m0, m6 ; dct8 out7 + paddd m0, m6 ; dct8 out0 + psubd m6, m1, m5 ; dct8 out6 + paddd m1, m5 ; dct8 out1 + REPX {pmaxsd x, m14}, m7, m0, m6, m1 + psubd m5, m2, m4 ; dct8 out5 + paddd m2, m4 ; dct8 out2 + REPX {pminsd x, m15}, m7, m0, m6, m1 + psubd m4, m3, m8 ; dct8 out4 + paddd m3, m8 ; dct8 out3 + REPX {pmaxsd x, m14}, m5, m2, m4, m3 + paddd m20, m13 + paddd m23, m13 + REPX {pminsd x, m15}, m5, m2, m4, m3 + psubd m17, m20, m18 ; t10a + paddd m20, m18 ; t13a + REPX {pmaxsd x, m14}, m22, m21, m16, m9 + psubd m18, m23, m19 ; t11 + paddd m19, m23 ; t12 + REPX {pminsd x, m15}, m22, m21, m16, m9 + REPX {psrad x, 12 }, m20, m19, m18, m17 + ret +.main_end: + vpbroadcastd m11, [o(pd_2)] +.main_end2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m23, m0, m22 ; out15 + paddd m0, m22 ; out0 + psubd m22, m1, m21 ; out14 + paddd m1, m21 ; out1 + psubd m21, m2, m20 ; out13 + paddd m2, m20 ; out2 + psubd m20, m3, m19 ; out12 + paddd m3, m19 ; out3 + psubd m19, m4, m18 ; out11 + paddd m4, m18 ; out4 + psubd m18, m5, m17 ; out10 + paddd m5, m17 ; out5 + psubd m17, m6, m16 ; out9 + paddd m6, m16 ; out6 + psubd m16, m7, m9 ; out8 + paddd m7, m9 ; out7 + REPX {vpsravd x, m11}, m0, m16, m1, m17, m2, m18, m3, m19, \ + m4, m20, m5, m21, m6, m22, m7, m23 + packssdw m0, m16 + packssdw m1, m17 + packssdw m2, m18 + packssdw m3, m19 + packssdw m4, m20 + packssdw m5, m21 + packssdw m6, m22 + packssdw m7, m23 + ret +.main_end3: + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m4, m5 + punpcklwd m4, m5 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 + punpckhdq m7, m0, m2 + punpckldq m0, m2 + punpckhdq m2, m8, m1 + punpckldq m8, m1 + punpckhdq m1, m4, m5 + punpckldq m4, m5 + punpckhdq m5, m3, m6 + punpckldq m3, m6 + vshufi32x4 m6, m0, m4, q3232 + vinserti32x8 m0, ym4, 1 + vinserti32x8 m4, m8, ym3, 1 + vshufi32x4 m8, m3, q3232 + vinserti32x8 m3, m7, ym1, 1 + vshufi32x4 m7, m1, q3232 + vshufi32x4 m1, m2, m5, q3232 + vinserti32x8 m2, ym5, 1 + vshufi32x4 m5, m7, m1, q2020 ; 10 11 + vshufi32x4 m7, m1, q3131 ; 14 15 + vshufi32x4 m1, m3, m2, q2020 ; 2 3 + vshufi32x4 m3, m2, q3131 ; 6 7 + vshufi32x4 m2, m0, m4, q3131 ; 4 5 + vshufi32x4 m0, m4, q2020 ; 0 1 + vshufi32x4 m4, m6, m8, q2020 ; 8 9 + vshufi32x4 m6, m8, q3131 ; 12 13 + ret +ALIGN function_align +.round: + paddd m20, m13 + paddd m22, m13 +.round2: + paddd m16, m13 + paddd m18, m13 +.round3: + REPX {psrad x, 12 }, m16, m18, m20, m22 + REPX {paddd x, m13}, m17, m19, m21, m23 + REPX {psrad x, 12 }, m17, m19, m21, m23 + ret + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, flipadst +INV_TXFM_16X16_FN adst, adst + +cglobal iadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + cmp eobd, 36 + jl .fast + call .main_pass1 + packssdw m0, m16 + packssdw m1, m17 + packssdw m2, m18 + packssdw m3, m19 + packssdw m4, m5, m20 + packssdw m5, m6, m21 + packssdw m6, m7, m22 + packssdw m7, m8, m23 + jmp m(idct_16x16_internal_10bpc).pass1_end +.fast: + call .main_pass1_fast + vpbroadcastd m9, [o(pd_2)] + paddd m0, m9 + psubd m1, m9, m1 + paddd m2, m9 + psubd m3, m9, m3 + paddd m4, m9, m5 + psubd m5, m9, m6 + paddd m6, m9, m7 + psubd m7, m9, m8 +.pass1_fast_end: + mova m9, [o(permA)] + psrlq m8, m9, 8 + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 +.pass1_fast_end2: + mova m10, m9 + mova m11, m8 + call m(idct_16x8_internal_10bpc).transpose_16x8 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + REPX {mova [cq+64*x], ym4}, 0, 1, 2, 3, 4, 5, 6, 7 + jmp tx2q +.pass2: + lea r5, [o_base_8bpc] + call m(iadst_16x16_internal_8bpc).main_pass2b + movshdup m12, [permC] + mova m11, [pw_2048_m2048] + psrlq m13, m12, 8 + vpermq m8, m13, m0 + vpermq m0, m12, m7 + vpermq m7, m13, m1 + vpermq m1, m12, m6 + vpermq m6, m13, m2 + vpermq m2, m12, m5 + vpermq m5, m13, m3 + vpermq m3, m12, m4 + jmp m(idct_16x16_internal_10bpc).pass2_end +ALIGN function_align +.main_pass1: + mova m0, [cq+64* 0] +%if WIN64 + movaps [cq+16*0], xmm6 + movaps [cq+16*1], xmm7 +%endif + mova m23, [cq+64*15] + vpbroadcastd m13, [o(pd_2048)] + ITX_MULSUB_2D 23, 0, 8, 9, 10, 13, 201, 4091 ; t1 t0 + mova m7, [cq+64* 7] + mova m16, [cq+64* 8] + ITX_MULSUB_2D 7, 16, 8, 9, 10, 13, 3035, 2751 ; t9 t8 + mova m2, [cq+64* 2] + mova m21, [cq+64*13] + ITX_MULSUB_2D 21, 2, 8, 9, 10, 13, 995, 3973 ; t3 t2 + mova m5, [cq+64* 5] + mova m18, [cq+64*10] + ITX_MULSUB_2D 5, 18, 8, 9, 10, 13, 3513, 2106 ; t11 t10 + mova m4, [cq+64* 4] + mova m19, [cq+64*11] + ITX_MULSUB_2D 19, 4, 8, 9, 10, 13, 1751, 3703 ; t5 t4 + mova m3, [cq+64* 3] + mova m20, [cq+64*12] + ITX_MULSUB_2D 3, 20, 8, 9, 10, 13, 3857, 1380 ; t13 t12 + mova m6, [cq+64* 6] + mova m17, [cq+64* 9] + ITX_MULSUB_2D 17, 6, 8, 9, 10, 13, 2440, 3290 ; t7 t6 + mova m1, [cq+64* 1] + mova m22, [cq+64*14] + ITX_MULSUB_2D 1, 22, 8, 9, 10, 13, 4052, 601 ; t15 t14 + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + psubd m9, m23, m7 ; t9a + paddd m23, m7 ; t1a + psubd m7, m2, m18 ; t10a + paddd m18, m2 ; t2a + REPX {pmaxsd x, m14}, m9, m23, m7, m18 + psubd m2, m17, m1 ; t15a + paddd m17, m1 ; t7a + REPX {pminsd x, m15}, m9, m23, m7, m18 + psubd m1, m21, m5 ; t11a + paddd m21, m5 ; t3a + REPX {pmaxsd x, m14}, m2, m17, m1, m21 + psubd m5, m4, m20 ; t12a + paddd m4, m20 ; t4a + REPX {pminsd x, m15}, m2, m17, m1, m21 + psubd m20, m19, m3 ; t13a + paddd m19, m3 ; t5a + REPX {pmaxsd x, m14}, m5, m4, m20, m19 + psubd m8, m6, m22 ; t14a + paddd m6, m22 ; t6a + REPX {pminsd x, m15}, m5, m4, m20, m19 + psubd m22, m0, m16 ; t8a + paddd m16, m0 ; t0a + REPX {pmaxsd x, m14}, m8, m6, m22, m16 + vpbroadcastd m11, [o(pd_4017)] + vpbroadcastd m10, [o(pd_799)] + REPX {pminsd x, m15}, m8, m6, m22, m16 + ITX_MULSUB_2D 22, 9, 0, 3, _, 13, 10, 11 ; t9 t8 + ITX_MULSUB_2D 20, 5, 0, 3, _, 13, 11, 10 ; t12 t13 + vpbroadcastd m11, [o(pd_2276)] + vpbroadcastd m10, [o(pd_3406)] + ITX_MULSUB_2D 7, 1, 0, 3, _, 13, 10, 11 ; t11 t10 + ITX_MULSUB_2D 2, 8, 0, 3, _, 13, 11, 10 ; t14 t15 + paddd m0, m16, m4 ; t0 + psubd m16, m4 ; t4 + psubd m3, m23, m19 ; t5 + paddd m23, m19 ; t1 + REPX {pmaxsd x, m14}, m0, m16, m3, m23 + psubd m19, m18, m6 ; t6 + paddd m18, m6 ; t2 + REPX {pminsd x, m15}, m0, m16, m3, m23 + psubd m6, m21, m17 ; t7 + paddd m21, m17 ; t3 + REPX {pmaxsd x, m14}, m19, m18, m6, m21 + paddd m17, m9, m20 ; t8a + psubd m9, m20 ; t12a + REPX {pminsd x, m15}, m19, m18, m6, m21 + psubd m20, m22, m5 ; t13a + paddd m22, m5 ; t9a + REPX {pmaxsd x, m14}, m17, m9, m20, m22 + psubd m5, m1, m2 ; t14a + paddd m1, m2 ; t10a + REPX {pminsd x, m15}, m17, m9, m20, m22 + psubd m2, m7, m8 ; t15a + paddd m7, m8 ; t11a + REPX {pmaxsd x, m14}, m5, m1, m2, m7 + vpbroadcastd m11, [o(pd_3784)] + vpbroadcastd m10, [o(pd_1567)] + REPX {pminsd x, m15}, m5, m1, m2, m7 + ITX_MULSUB_2D 16, 3, 4, 8, _, 13, 10, 11 ; t5a t4a + ITX_MULSUB_2D 6, 19, 4, 8, _, 13, 11, 10 ; t6a t7a + ITX_MULSUB_2D 9, 20, 4, 8, _, 13, 10, 11 ; t13 t12 + ITX_MULSUB_2D 2, 5, 4, 8, _, 13, 11, 10 ; t14 t15 + psubd m8, m0, m18 ; t2a + paddd m0, m18 ; out0 + psubd m18, m23, m21 ; t3a + paddd m23, m21 ; -out15 + paddd m21, m9, m5 ; -out13 + psubd m9, m5 ; t15a + psubd m5, m3, m6 ; t6 + paddd m3, m6 ; -out3 + REPX {pmaxsd x, m14}, m8, m18, m9, m5 + psubd m6, m20, m2 ; t14a + paddd m2, m20 ; out2 + paddd m20, m16, m19 ; out12 + psubd m16, m19 ; t7 + REPX {pminsd x, m15}, m8, m18, m9, m5 + psubd m19, m22, m7 ; t11 + paddd m22, m7 ; out14 + psubd m7, m17, m1 ; t10 + paddd m1, m17 ; -out1 + REPX {pmaxsd x, m14}, m6, m16, m19, m7 + vpbroadcastd m12, [o(pd_1448)] + vpbroadcastd m4, [o(pd_2)] + vpbroadcastd m10, [o(pd_5120)] + vpbroadcastd m11, [o(pd_5119)] + REPX {pminsd x, m15}, m6, m16, m19, m7 + psubd m17, m7, m19 ; -out9 + paddd m7, m19 ; out6 + psubd m19, m5, m16 ; -out11 + paddd m5, m16 ; out4 + REPX {pmulld x, m12}, m17, m7, m19, m5 + psubd m16, m8, m18 ; out8 + paddd m8, m18 ; -out7 + psubd m18, m6, m9 ; out10 + paddd m6, m9 ; -out5 + REPX {pmulld x, m12}, m16, m8, m18, m6 + REPX {paddd x, m4 }, m0, m2, m20, m22 + REPX {psubd x, m4, x}, m1, m3, m21, m23 + REPX {paddd x, m10 }, m7, m5, m16, m18 + REPX {psubd x, m11, x}, m17, m19, m8, m6 + REPX {psrad x, 2 }, m20, m22, m0, m2, m21, m23, m1, m3 + REPX {psrad x, 13}, m17, m19, m5, m7, m16, m18, m6, m8 + ret +ALIGN function_align +.main_pass1_fast: + mova ym0, [cq+64*0] + mova ym1, [cq+64*2] + movshdup m8, [o(permB)] + mova ym6, [cq+64*1] + mova ym7, [cq+64*3] + mova ym2, [cq+64*4] + mova ym3, [cq+64*6] + mova ym4, [cq+64*5] + mova ym5, [cq+64*7] + vpermt2q m0, m8, m1 ; 0 2 + vpermt2q m7, m8, m6 ; 3 1 + vpermt2q m2, m8, m3 ; 4 6 + vpermt2q m5, m8, m4 ; 7 5 + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m12, [o(pd_2896)] + jmp m(iadst_16x8_internal_10bpc).main_fast + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + cmp eobd, 36 + jl .fast + call m(iadst_16x16_internal_10bpc).main_pass1 + packssdw m4, m19, m3 + packssdw m3, m20, m5 + packssdw m5, m18, m2 + packssdw m2, m21, m6 + packssdw m6, m17, m1 + packssdw m1, m22, m7 + packssdw m7, m16, m0 + packssdw m0, m23, m8 + jmp m(idct_16x16_internal_10bpc).pass1_end +.fast: + call m(iadst_16x16_internal_10bpc).main_pass1_fast + vpbroadcastd m9, [o(pd_2)] + psubd m4, m9, m3 + paddd m3, m9, m5 + paddd m5, m9, m2 + psubd m2, m9, m6 + psubd m6, m9, m1 + paddd m1, m9, m7 + paddd m7, m9, m0 + psubd m0, m9, m8 + jmp m(iadst_16x16_internal_10bpc).pass1_fast_end +.pass2: + lea r5, [o_base_8bpc] + call m(iadst_16x16_internal_8bpc).main_pass2b + movshdup m12, [permC] + movu m11, [pw_m2048_2048] + psrlq m13, m12, 8 + vpermq m8, m13, m7 + vpermq m7, m13, m6 + vpermq m6, m13, m5 + vpermq m5, m13, m4 + vpermq m3, m12, m3 + vpermq m2, m12, m2 + vpermq m1, m12, m1 + vpermq m0, m12, m0 + jmp m(idct_16x16_internal_10bpc).pass2_end + +INV_TXFM_16X16_FN identity, dct, -92 +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 +%undef cmp + vpbroadcastd m10, [o(pd_5793)] + vpbroadcastd m11, [o(pd_5120)] + mov r6, cq + cmp eobd, 36 + jl .fast + call .pass1_main + packssdw m0, m6, m8 + packssdw m1, m7, m9 + call .pass1_main + packssdw m2, m6, m8 + packssdw m3, m7, m9 + call .pass1_main + packssdw m4, m6, m8 + packssdw m5, m7, m9 + call .pass1_main + packssdw m6, m8 + packssdw m7, m9 + jmp m(idct_16x16_internal_10bpc).pass1_end2 +.fast: + call .pass1_main_fast + packssdw m0, m6, m7 + call .pass1_main_fast + packssdw m1, m6, m7 + call .pass1_main_fast + packssdw m2, m6, m7 + call .pass1_main_fast + packssdw m3, m6, m7 + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckldq m3, m4, m1 + punpckhdq m4, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + pxor m7, m7 + vshufi32x4 m2, m0, m3, q3131 + vshufi32x4 m0, m3, q2020 + vshufi32x4 m3, m1, m4, q3131 + vshufi32x4 m1, m4, q2020 + REPX {mova x, m7}, m4, m5, m6 + jmp m(idct_16x16_internal_10bpc).pass1_end3 +.pass2: + movshdup m14, [o(permC)] + vpbroadcastd m15, [o(pw_1697x16)] + lea r6, [strideq*3] + vpbroadcastd m11, [o(pw_2048)] + pxor m12, m12 + vpbroadcastd m13, [pixel_10bpc_max] + vpermq m8, m14, m0 + vpermq m9, m14, m1 + call .pass2_main + vpermq m8, m14, m2 + vpermq m9, m14, m3 + call .pass2_main + vpermq m8, m14, m4 + vpermq m9, m14, m5 + call .pass2_main + vpermq m8, m14, m6 + vpermq m9, m14, m7 +.pass2_main: + pmulhrsw m0, m15, m8 + pmulhrsw m1, m15, m9 + paddsw m8, m8 + paddsw m9, m9 + paddsw m8, m0 + paddsw m9, m1 + jmp m(idct_16x8_internal_10bpc).write_16x4 +ALIGN function_align +.pass1_main: + pmulld m6, m10, [r6+64*0] + pmulld m7, m10, [r6+64*1] + pmulld m8, m10, [r6+64*8] + pmulld m9, m10, [r6+64*9] + add r6, 64*2 + REPX {paddd x, m11}, m6, m7, m8, m9 + REPX {psrad x, 13 }, m6, m8, m7, m9 + ret +ALIGN function_align +.pass1_main_fast: + mova ym6, [r6+64* 0] + vinserti32x8 m6, [r6+64* 4], 1 + mova ym7, [r6+64* 8] + vinserti32x8 m7, [r6+64*12], 1 + add r6, 64 + REPX {pmulld x, m10}, m6, m7 + REPX {paddd x, m11}, m6, m7 + REPX {psrad x, 13 }, m6, m7 + ret + +cglobal inv_txfm_add_dct_dct_8x32_10bpc, 4, 7, 22, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + vpbroadcastd m11, [o(pd_2)] + mova m20, [o(idct8x32p)] + pxor m21, m21 + cmp eobd, 43 + jl .fast + call .pass1_main + punpcklwd m16, m0, m1 + punpcklwd m17, m2, m3 + punpckhwd m18, m0, m1 + punpckhwd m19, m2, m3 + cmp eobd, 107 + jge .full + punpckldq m0, m16, m17 ; 0 2 + punpckhdq m1, m16, m17 ; 4 6 + punpckldq m2, m18, m19 ; 8 10 + punpckhdq m3, m18, m19 ; 12 14 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + vextracti32x8 ym16, m2, 1 + vextracti32x8 ym17, m3, 1 + call m(idct_8x16_internal_8bpc).main_fast + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast + jmp .end +.full: + add cq, 64 + call .pass1_main + punpcklwd m5, m0, m1 + punpcklwd m6, m2, m3 + punpckhwd m7, m0, m1 + punpckhwd m8, m2, m3 + punpckldq m0, m16, m17 ; 0 2 + punpckhdq m1, m16, m17 ; 4 6 + punpckldq m2, m18, m19 ; 8 10 + punpckhdq m3, m18, m19 ; 12 14 + punpckldq m4, m5, m6 ; 16 18 + punpckhdq m5, m6 ; 20 22 + punpckldq m6, m7, m8 ; 24 26 + punpckhdq m7, m8 ; 28 30 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + vextracti32x8 ym16, m2, 1 + vextracti32x8 ym17, m3, 1 + vextracti32x8 ym18, m4, 1 + vextracti32x8 ym19, m5, 1 + vextracti32x8 ym20, m6, 1 + vextracti32x8 ym21, m7, 1 + call m(idct_8x16_internal_8bpc).main + REPX {pshufd x, x, q1032}, ym18, ym19, ym20, ym21 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main + jmp .end +.fast: + movshdup m8, [o(permB)] + mova ym1, [cq+128*1] + mova ym5, [cq+128*5] + mova ym7, [cq+128*3] + mova ym3, [cq+128*7] + mova ym0, [cq+128*0] + mova ym4, [cq+128*2] + mova ym2, [cq+128*4] + mova ym6, [cq+128*6] + vpermt2q m1, m8, m5 ; 1 5 + vpermt2q m3, m8, m7 ; 7 3 + vpermt2q m0, m8, m4 ; 0 2 + vpermt2q m2, m8, m6 ; 4 6 + mova [cq+128*0], ym21 + REPX {vmovdqa32 [cq+128*x], ym21}, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x8_internal_10bpc).main_end + packssdw m0, m2 + packssdw m1, m3 + vpermb m0, m20, m0 + vprold m20, 16 + vpermb m2, m20, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + call m(idct_8x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast2 +.end: + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_end ; performs vzeroupper + lea r3, [strideq*2] + vpbroadcastd m12, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m11, m11 + lea r3, [dstq+r3*8] + pmulhrsw m0, m10 + pmulhrsw m1, m10 + call .write_8x4x2 + pmulhrsw m0, m10, m2 + pmulhrsw m1, m10, m3 + call .write_8x4x2 + pmulhrsw m0, m10, m4 + pmulhrsw m1, m10, m5 + call .write_8x4x2 + pmulhrsw m0, m10, m6 + pmulhrsw m1, m10, m7 +.write_8x4x2: + mova xm8, [dstq+strideq*0] + vinserti32x4 ym8, [dstq+strideq*1], 1 + vinserti32x4 m8, [dstq+strideq*2], 2 + vinserti32x4 m8, [dstq+r6 ], 3 + mova xm9, [r3 +r6 ] + vinserti32x4 ym9, [r3 +strideq*2], 1 + vinserti32x4 m9, [r3 +strideq*1], 2 + vinserti32x4 m9, [r3 +strideq*0], 3 + paddw m8, m0 + paddw m9, m1 + pmaxsw m8, m11 + pmaxsw m9, m11 + pminsw m8, m12 + pminsw m9, m12 + mova [dstq+strideq*0], xm8 + vextracti32x4 [dstq+strideq*1], ym8, 1 + vextracti32x4 [dstq+strideq*2], m8, 2 + vextracti32x4 [dstq+r6 ], m8, 3 + lea dstq, [dstq+strideq*4] + vextracti32x4 [r3 +strideq*0], m9, 3 + vextracti32x4 [r3 +strideq*1], m9, 2 + vextracti32x4 [r3 +strideq*2], ym9, 1 + mova [r3 +r6 ], xm9 + lea r3, [r3+strideq*4] + ret +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 +ALIGN function_align +.pass1_main: + mova m0, [cq+128*0] + mova m1, [cq+128*1] + mova m2, [cq+128*2] + mova m3, [cq+128*3] + mova m4, [cq+128*4] + mova m5, [cq+128*5] + mova m6, [cq+128*6] + mova m7, [cq+128*7] + REPX {mova [cq+128*x], m21}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x16_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_end2 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + REPX {vpermb x, m20, x}, m0, m1, m2, m3 + ret + +cglobal inv_txfm_add_identity_identity_8x32_10bpc, 4, 8, 12, dst, stride, c, eob + vpbroadcastd m9, [pw_5] + lea r4, [strideq*3] + pxor m10, m10 + lea r5, [strideq*5] + vpbroadcastd m11, [pixel_10bpc_max] + sub eobd, 107 + lea r6, [strideq+r4*2] +.loop: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + lea r7, [dstq+strideq*8] + REPX {mova [cq+128*x], m10}, 0, 1, 2, 3 + REPX {paddsw x, m9}, m0, m1, m2, m3 + REPX {mova [cq+128*x], m10}, 4, 5, 6, 7 + REPX {psraw x, 3 }, m0, m1, m2, m3 + add cq, 64 + mova xm4, [dstq+strideq*0] + mova xm5, [dstq+strideq*1] + mova xm6, [dstq+strideq*2] + mova xm7, [dstq+r4 *1] + punpckhwd m8, m0, m1 + vinserti32x4 ym4, [dstq+strideq*4], 1 + punpcklwd m0, m1 + vinserti32x4 ym5, [dstq+r5 *1], 1 + punpckhwd m1, m2, m3 + vinserti32x4 ym6, [dstq+r4 *2], 1 + punpcklwd m2, m3 + vinserti32x4 ym7, [dstq+r6 *1], 1 + punpckhwd m3, m0, m8 + vinserti32x4 m4, [r7 +strideq*0], 2 + punpcklwd m0, m8 + vinserti32x4 m5, [r7 +strideq*1], 2 + punpckhwd m8, m2, m1 + vinserti32x4 m6, [r7 +strideq*2], 2 + punpcklwd m2, m1 + vinserti32x4 m7, [r7 +r4 *1], 2 + punpckhqdq m1, m0, m2 + vinserti32x4 m4, [r7 +strideq*4], 3 + punpcklqdq m0, m2 + vinserti32x4 m5, [r7 +r5 *1], 3 + punpcklqdq m2, m3, m8 + vinserti32x4 m6, [r7 +r4 *2], 3 + punpckhqdq m3, m8 + vinserti32x4 m7, [r7 +r6 *1], 3 + paddw m0, m4 + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + REPX {pmaxsw x, m10}, m0, m1, m2, m3 + REPX {pminsw x, m11}, m0, m1, m2, m3 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + mova [dstq+strideq*2], xm2 + mova [dstq+r4 *1], xm3 + vextracti32x4 [dstq+strideq*4], ym0, 1 + vextracti32x4 [dstq+r5 *1], ym1, 1 + vextracti32x4 [dstq+r4 *2], ym2, 1 + vextracti32x4 [dstq+r6 *1], ym3, 1 + lea dstq, [r7+strideq*8] + vextracti32x4 [r7 +strideq*0], m0, 2 + vextracti32x4 [r7 +strideq*1], m1, 2 + vextracti32x4 [r7 +strideq*2], m2, 2 + vextracti32x4 [r7 +r4 *1], m3, 2 + vextracti32x4 [r7 +strideq*4], m0, 3 + vextracti32x4 [r7 +r5 *1], m1, 3 + vextracti32x4 [r7 +r4 *2], m2, 3 + vextracti32x4 [r7 +r6 *1], m3, 3 + add eobd, 0x80000000 + jnc .loop + RET + +cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + mova m11, [o(permB)] + mova m0, [cq+64* 0] ; 0 1 + mova m4, [cq+64* 1] ; 2 3 + mova m1, [cq+64* 2] ; 4 5 + mova m8, [cq+64* 3] ; 6 7 + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + psrlq m10, m11, 32 +%if WIN64 + movaps [cq+16*0], xmm6 + movaps [cq+16*1], xmm7 +%endif + mova m16, m11 + vpermi2q m16, m0, m1 ; 1 5 + mova m17, m11 + vpermi2q m17, m8, m4 ; 7 3 + cmp eobd, 43 + jl .fast + mova m18, [cq+64* 4] ; 8 9 + mova m20, [cq+64* 5] ; 10 11 + mova m6, [cq+64* 6] ; 12 13 + mova m7, [cq+64* 7] ; 14 15 + vpermt2q m0, m10, m18 ; 0 8 + vpermt2q m18, m11, m6 ; 9 13 + mova m19, m11 + vpermi2q m19, m7, m20 ; 15 11 + cmp eobd, 107 + jge .full + vpermt2q m1, m10, m6 ; 4 12 + vpermt2q m4, m10, m8 ; 2 6 + vpermt2q m7, m10, m20 ; 14 10 + mov r6d, 64*1 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + call .main_fast + call m(idct_16x16_internal_10bpc).main_end + jmp .end +.full: + mova m2, [cq+64* 8] ; 16 17 + mova m5, [cq+64* 9] ; 18 19 + mova m9, [cq+64*10] ; 20 21 + mova m21, [cq+64*11] ; 22 23 + vpermt2q m1, m10, m9 ; 4 20 + vpermt2q m7, m10, m21 ; 14 22 + vpermt2q m21, m11, m5 ; 23 19 + vpermt2q m5, m10, m20 ; 18 10 + mova m20, m11 + vpermi2q m20, m2, m9 ; 17 21 + mova m22, [cq+64*12] ; 24 25 + mova m9, [cq+64*13] ; 26 27 + mova m3, [cq+64*14] ; 28 29 + mova m23, [cq+64*15] ; 30 31 + vpermt2q m2, m10, m22 ; 16 24 + vpermt2q m22, m11, m3 ; 25 29 + vpermt2q m3, m10, m6 ; 28 12 + vpermt2q m4, m10, m9 ; 2 26 + mova m6, m10 + vpermi2q m6, m23, m8 ; 30 6 + vpermt2q m23, m11, m9 ; 31 27 + mov r6d, 64*3 + call m(idct_8x8_internal_10bpc).main + call m(idct_16x8_internal_10bpc).main + call .main + call m(idct_16x16_internal_10bpc).main_end + jmp .end +.fast: + vpermq m0, m10, m0 ; 0 0 + vpermq m1, m10, m1 ; 4 4 + vpermt2q m4, m10, m8 ; 2 6 + xor r6d, r6d + call .main_fast2 + call m(idct_16x16_internal_10bpc).main_end +.end: +%if WIN64 + movaps xmm6, [cq+16*0] + movaps xmm7, [cq+16*1] +%endif + vzeroupper + call .transpose_8x32 + pxor m14, m14 +.zero_loop: + mova [cq+r6*4+64*3], m14 + mova [cq+r6*4+64*2], m14 + mova [cq+r6*4+64*1], m14 + mova [cq+r6*4+64*0], m14 + sub r6d, 64 + jge .zero_loop + lea r5, [o_base_8bpc] + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m8 + punpcklqdq m6, m8 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + pxor m12, m12 +.write_32x8_start: + vpbroadcastd m11, [pw_2048] + vpbroadcastd m13, [pixel_10bpc_max] + lea r3, [strideq*3] +.write_32x8: + pmulhrsw m0, m11 + pmulhrsw m1, m11 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + call .write_32x4 + pmulhrsw m0, m11, m4 + pmulhrsw m1, m11, m5 + pmulhrsw m2, m11, m6 + pmulhrsw m3, m11, m7 +.write_32x4: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3 ] + REPX {pmaxsw x, m12}, m0, m1, m2, m3 + REPX {pminsw x, m13}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + lea dstq, [dstq+strideq*4] + ret +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 8 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 +ALIGN function_align +.main_fast2: ; bottom three-quarters are zero + vbroadcasti32x4 m8, [o(pd_799_4017)] + pmulld m8, m1 ; t4 t7 + vpmulld m0, [o(pd_2896)] {1to16} ; t0 t1 + REPX {paddd x, m13}, m8, m0 + REPX {psrad x, 12 }, m8, m0 + pmulld m3, m8, m12 + mova m2, m0 ; t3 t2 + call m(idct_8x8_internal_10bpc).main3 + vbroadcasti32x4 m6, [o(pd_4076_3920)] + vbroadcasti32x4 m3, [o(pd_401_m1189)] + pmulld m6, m4 ; t15 t12 + pmulld m4, m3 ; t9 t10 + REPX {paddd x, m13}, m6, m4 + REPX {psrad x, 12 }, m6, m4 + mova m5, m6 ; t14 t13 + mova m9, m4 ; t8 t11 + call m(idct_16x8_internal_10bpc).main3 + vbroadcasti32x4 m23, [o(pd_4091_3973)] + vbroadcasti32x4 m7, [o(pd_201_995)] + vbroadcasti32x4 m22, [o(pd_1380_601)] + vbroadcasti32x4 m9, [o(pd_3857_4052)] + pmulld m23, m16 ; t16 t20 + pmulld m16, m7 ; t31 t27 + pmulld m22, m17 ; -t19 -t25 + pmulld m17, m9 ; t28 t24 + REPX {paddd x, m13}, m23, m16, m17 + psubd m22, m13, m22 + REPX {psrad x, 12 }, m23, m16, m22, m17 + mova m20, m23 ; t30 t26 + mova m9, m16 ; t17 t21 + mova m19, m22 ; t18 t22 + mova m18, m17 ; t29 t25 + jmp .main3 +.main_fast: ; bottom half is zero + vbroadcasti32x4 m23, [o(pd_4091_3973)] + vbroadcasti32x4 m7, [o(pd_201_995)] + vbroadcasti32x4 m20, [o(pd_2751_2106)] + vbroadcasti32x4 m9, [o(pd_3035_3513)] + vbroadcasti32x4 m21, [o(pd_3703_3290)] + vbroadcasti32x4 m10, [o(pd_1751_2440)] + vbroadcasti32x4 m22, [o(pd_1380_601)] + vbroadcasti32x4 m11, [o(pd_3857_4052)] + pmulld m23, m16 ; t16a t20a + pmulld m16, m7 ; t31a t27a + pmulld m20, m19 ; -t17a -t21a + pmulld m19, m9 ; t30a t26a + pmulld m21, m18 ; t18a t22a + pmulld m18, m10 ; t29a t25a + pmulld m22, m17 ; -t19a -t25a + pmulld m17, m11 ; t28a t24a + psubd m20, m13, m20 + psubd m22, m13, m22 + jmp .main2 +.main: + ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 201_995, 4091_3973 + ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3035_3513, 2751_2106 + ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1751_2440, 3703_3290 + ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3857_4052, 1380_601 + paddd m20, m13 + paddd m22, m13 +.main2: + REPX {paddd x, m13}, m16, m23, m19 + REPX {psrad x, 12 }, m16, m20, m23, m19 + psubd m9, m16, m20 ; t17 t21 + paddd m16, m20 ; t16 t20 + psubd m20, m23, m19 ; t30 t26 + paddd m23, m19 ; t31 t27 + REPX {pmaxsd x, m14}, m9, m16, m20, m23 + REPX {paddd x, m13}, m21, m18, m17 + REPX {psrad x, 12 }, m18, m22, m21, m17 + psubd m19, m22, m18 ; t18 t22 + paddd m22, m18 ; t19 t23 + psubd m18, m17, m21 ; t29 t25 + paddd m17, m21 ; t28 t24 + REPX {pmaxsd x, m14}, m19, m22, m18, m17 + REPX {pminsd x, m15}, m20, m9, m18, m19, m16, m23, m22, m17 +.main3: + vbroadcasti32x4 m11, [o(pd_4017_2276)] + vbroadcasti32x4 m10, [o(pd_799_3406)] + psubd m7, m0, m6 ; dct16 out15 out14 + paddd m0, m6 ; dct16 out0 out1 + psubd m6, m1, m5 ; dct16 out12 out13 + paddd m1, m5 ; dct16 out3 out2 + psubd m5, m2, m4 ; dct16 out11 out10 + paddd m2, m4 ; dct16 out4 out5 + psubd m4, m3, m8 ; dct16 out8 out9 + paddd m3, m8 ; dct16 out7 out6 + ITX_MULSUB_2D 20, 9, 8, 21, _, 13, 10, 11 + ITX_MULSUB_2D 18, 19, 8, 21, _, 13, 10, 11, 2 + REPX {pmaxsd x, m14}, m7, m0, m6, m1, m5, m2, m4, m3 + punpckhqdq m21, m16, m20 ; t20 t21a + punpcklqdq m16, m20 ; t16 t17a + punpcklqdq m20, m22, m19 ; t19 t18a + punpckhqdq m22, m19 ; t23 t22a + REPX {pminsd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 + punpcklqdq m19, m23, m9 ; t31 t30a + punpckhqdq m23, m9 ; t27 t26a + punpckhqdq m9, m17, m18 ; t24 t25a + punpcklqdq m17, m18 ; t28 t29a + vpbroadcastd m11, [o(pd_3784)] + vpbroadcastd m10, [o(pd_1567)] + psubd m18, m16, m20 ; t19a t18 + paddd m20, m16 ; t16a t17 + psubd m16, m19, m17 ; t28a t29 + paddd m19, m17 ; t31a t30 + psubd m17, m22, m21 ; t20a t21 + paddd m22, m21 ; t23a t22 + psubd m21, m9, m23 ; t27a t26 + paddd m23, m9 ; t24a t25 + REPX {pmaxsd x, m14}, m18, m16, m17, m21 + REPX {pminsd x, m15}, m16, m18, m21, m17 + ITX_MULSUB_2D 16, 18, 8, 9, _, 13, 10, 11 + ITX_MULSUB_2D 21, 17, 8, 9, _, 13, 10, 11, 2 + REPX {pmaxsd x, m14}, m20, m22, m19, m23 + REPX {pminsd x, m15}, m20, m22, m19, m23 + paddd m9, m20, m22 ; t16 t17a + psubd m20, m22 ; t23 t22a + paddd m22, m19, m23 ; t31 t30a + psubd m19, m23 ; t24 t25a + psubd m23, m16, m17 ; t20a t21 + paddd m16, m17 ; t19a t18 + psubd m17, m18, m21 ; t27a t26 + paddd m21, m18 ; t28a t29 + REPX {pmaxsd x, m14}, m20, m19, m23, m17 + REPX {pminsd x, m15}, m19, m20, m17, m23 + REPX {pmulld x, m12}, m19, m20, m17, m23 + REPX {pmaxsd x, m14}, m22, m21, m16, m9 + paddd m19, m13 + paddd m17, m13 + REPX {pminsd x, m15}, m22, m21, m16, m9 + psubd m18, m19, m20 ; t23a t22 + paddd m19, m20 ; t24a t25 + paddd m20, m17, m23 ; t27 t26a + psubd m17, m23 ; t20 t21a + REPX {psrad x, 12 }, m20, m19, m18, m17 + ret +.transpose_8x32: + mova m10, [o(idct32x8p)] + psrlw m8, m10, 8 + mova m9, m8 + vpermi2w m8, m1, m5 + vpermt2w m1, m10, m5 + vprold m5, m9, 16 + vpermi2w m9, m3, m7 + vpermt2w m3, m10, m7 + vprold m10, 16 + mova m7, m5 + vpermi2w m5, m0, m4 + vpermt2w m0, m10, m4 + vpermi2w m7, m2, m6 + vpermt2w m2, m10, m6 + punpckhdq m6, m5, m8 + punpckldq m5, m8 + punpckhdq m8, m7, m9 + punpckldq m7, m9 + punpckhdq m4, m2, m3 + punpckldq m2, m3 + punpckhdq m3, m0, m1 + punpckldq m0, m1 + ret + +cglobal inv_txfm_add_identity_identity_32x8_10bpc, 4, 7, 10, dst, stride, c, eob + vpbroadcastd m5, [pw_4096] + lea r4, [strideq*3] + mova m6, [idtx32x8p] + lea r5, [strideq*5] + vpbroadcastd m9, [pixel_10bpc_max] + lea r6, [strideq+r4*2] + pxor m8, m8 + sub eobd, 107 + psrlw m7, m6, 8 +.loop: + mova m0, [cq+64*0] + packssdw m0, [cq+64*1] ; 02 13 + mova m1, [cq+64*2] + packssdw m1, [cq+64*3] ; 46 57 + mova m2, [cq+64*4] + packssdw m2, [cq+64*5] ; 8a 9b + mova m3, [cq+64*6] + packssdw m3, [cq+64*7] ; ce df + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + REPX {mova [cq+64*x], m8}, 0, 1, 2, 3 + mova m4, m6 + vpermi2w m4, m1, m3 + vpermt2w m1, m7, m3 + REPX {mova [cq+64*x], m8}, 4, 5, 6, 7 + mova m3, m7 + vpermi2w m3, m0, m2 + vpermt2w m0, m6, m2 + add cq, 64*8 + punpcklqdq m2, m3, m1 ; 4 5 + punpckhqdq m3, m1 ; 6 7 + punpckhqdq m1, m0, m4 ; 2 3 + punpcklqdq m0, m4 ; 0 1 + mova ym4, [dstq+strideq*0] + vinserti32x8 m4, [dstq+strideq*1], 1 + paddw m0, m4 + mova ym4, [dstq+strideq*2] + vinserti32x8 m4, [dstq+r4 *1], 1 + paddw m1, m4 + mova ym4, [dstq+strideq*4] + vinserti32x8 m4, [dstq+r5 *1], 1 + paddw m2, m4 + mova ym4, [dstq+r4 *2] + vinserti32x8 m4, [dstq+r6 *1], 1 + paddw m3, m4 + REPX {pmaxsw x, m8}, m0, m1, m2, m3 + REPX {pminsw x, m9}, m0, m1, m2, m3 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+r4 *1], m1, 1 + mova [dstq+strideq*4], ym2 + vextracti32x8 [dstq+r5 *1], m2, 1 + mova [dstq+r4 *2], ym3 + vextracti32x8 [dstq+r6 *1], m3, 1 + add dstq, 32 + add eobd, 0x80000000 + jnc .loop + RET + +cglobal inv_txfm_add_dct_dct_16x32_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] +%if WIN64 + movaps [rsp+ 8], xmm6 + movaps [rsp+24], xmm7 +%endif + cmp eobd, 36 + jl .fast + call .pass1 + cmp eobd, 151 + jge .full + lea r5, [o_base_8bpc] + pxor m9, m9 + punpcklwd m8, m1, m1 ; 2 + punpckhwd m14, m1, m1 ; 3 + punpcklwd m1, m3, m3 ; 6 + punpckhwd m15, m3, m3 ; 7 + punpcklwd m3, m6, m6 ; 12 + punpckhwd m19, m6, m6 ; 13 + punpcklwd m6, m9, m4 ; __ 8 + punpckhwd m20, m4, m4 ; 9 + punpckhwd m16, m5, m5 ; 11 + punpcklwd m5, m5 ; 10 + punpcklwd m9, m0 ; __ 0 + punpckhwd m21, m0, m0 ; 1 + punpcklwd m0, m7, m7 ; 14 + punpckhwd m17, m7, m7 ; 15 + punpcklwd m7, m2, m2 ; 4 + punpckhwd m18, m2, m2 ; 5 + call m(idct_16x16_internal_8bpc).main_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mov r6d, 64*3 + pxor m8, m8 +.zero_loop: + REPX {mova [cq+r6*8+128*x], m8}, 3, 2, 1, 0 + sub r6d, 64 + jge .zero_loop + jmp .pass2_end +.full: + mova [cq+128*0], m0 + mova [cq+128*1], m1 + mova [cq+128*2], m2 + mova [cq+128*3], m3 + mova [cq+128*4], m4 + mova [cq+128*5], m5 + mova [cq+128*6], m6 + mova [cq+128*7], m7 + add cq, 64 + call .pass1 + mova m9, [cq-64* 1] ; 0 1 + mova m14, [cq+64* 1] ; 2 3 + mova m18, [cq+64* 3] ; 4 5 + mova m15, [cq+64* 5] ; 6 7 + mova m20, [cq+64* 7] ; 8 9 + mova m16, [cq+64* 9] ; 10 11 + mova m22, [cq+64*11] ; 12 13 + mova m19, [cq+64*13] ; 14 15 + lea r5, [o_base_8bpc] + punpcklwd m8, m7, m14 ; 30 2 + punpckhwd m21, m7, m9 ; 31 1 + punpcklwd m7, m6, m18 ; 28 4 + punpckhwd m14, m6 ; 3 29 + punpcklwd m9, m0, m9 ; 16 0 + punpckhwd m17, m19, m0 ; 15 17 + punpcklwd m0, m19, m1 ; 14 18 + punpckhwd m19, m1, m22 ; 19 13 + punpcklwd m1, m15, m5 ; 6 26 + punpckhwd m18, m5, m18 ; 27 5 + punpcklwd m6, m4, m20 ; 24 8 + punpckhwd m15, m4 ; 7 25 + punpcklwd m5, m3, m16 ; 22 10 + punpckhwd m20, m3, m20 ; 23 9 + punpcklwd m3, m22, m2 ; 12 20 + punpckhwd m16, m2 ; 11 21 + call m(idct_16x16_internal_8bpc).main2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + mov r6d, 32*7 + pxor m8, m8 +.full_zero_loop: + REPX {mova [cq+r6*8+64*x], m8}, 2, 1, 0, -1 + sub r6d, 32 + jge .full_zero_loop + jmp .pass2_end +.fast: + mova ym0, [cq+128*0] + mova ym2, [cq+128*4] + movshdup m8, [o(permB)] + mova ym1, [cq+128*2] + mova ym3, [cq+128*6] + mova ym4, [cq+128*1] + mova ym5, [cq+128*3] + mova ym6, [cq+128*5] + mova ym7, [cq+128*7] + vpermt2q m0, m8, m2 ; 0 4 + vpermt2q m1, m8, m3 ; 2 6 + vpermt2q m4, m8, m5 ; 1 3 + vpermt2q m7, m8, m6 ; 7 5 + REPX {pmulld x, m12}, m0, m1, m4, m7 + pxor ym16, ym16 + mova [cq+128*0], ym16 + REPX {vmovdqa32 [cq+128*x], ym16}, 1, 2, 3, 4, 5, 6, 7 + REPX {paddd x, m13}, m0, m1, m4, m7 + REPX {psrad x, 12 }, m0, m1, m4, m7 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + vpbroadcastd m11, [o(pd_1)] + call m(idct_8x16_internal_10bpc).main_end2 + mova m8, [o(idct8x32p)] + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + mova m6, [dup16_perm] + vpermb m0, m8, m0 + vpermb m2, m8, m2 + vprold m8, 16 + vpermb m1, m8, m1 + vpermb m3, m8, m3 + punpckldq m4, m0, m2 + punpckhdq m0, m2 + punpckldq m2, m1, m3 + punpckhdq m1, m3 + punpckldq m21, m4, m2 + punpckhdq m14, m4, m2 + punpckldq m18, m0, m1 + punpckhdq m15, m0, m1 + vpermb m8, m6, m14 ; 2 + vpermb m1, m6, m15 ; 6 + vpermb m7, m6, m18 ; 4 + pmovzxwd m9, ym21 ; 0 + vpord m6, [o(pb_32)] {1to16} + lea r5, [o_base_8bpc] + vpermb m21, m6, m21 ; 1 + vpermb m15, m6, m15 ; 7 + vpermb m18, m6, m18 ; 5 + vpermb m14, m6, m14 ; 3 + pslld m9, 16 + call m(idct_16x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 +.pass2_end: + movshdup m22, [permC] + vpbroadcastd m11, [pw_2048] + vpbroadcastd m13, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m12, m12 + psrlq m23, m22, 8 + vpermq m8, m22, m0 + vpermq m9, m23, m1 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m2 + vpermq m9, m23, m3 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m4 + vpermq m9, m23, m5 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m6 + vpermq m9, m23, m7 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m14 + vpermq m9, m23, m15 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m16 + vpermq m9, m23, m17 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m18 + vpermq m9, m23, m19 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m20 + vpermq m9, m23, m21 +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + vzeroupper + jmp m(idct_16x8_internal_10bpc).write_16x4 +.pass1: + pmulld m0, m12, [cq+128* 0] + pmulld m1, m12, [cq+128* 2] + pmulld m2, m12, [cq+128* 4] + pmulld m3, m12, [cq+128* 6] + pmulld m4, m12, [cq+128* 8] + pmulld m5, m12, [cq+128*10] + pmulld m6, m12, [cq+128*12] + pmulld m7, m12, [cq+128*14] + call m(idct_8x16_internal_10bpc).main_rect2 + pmulld m16, m12, [cq+128* 1] + pmulld m17, m12, [cq+128* 3] + pmulld m18, m12, [cq+128* 5] + pmulld m19, m12, [cq+128* 7] + pmulld m20, m12, [cq+128* 9] + pmulld m21, m12, [cq+128*11] + pmulld m22, m12, [cq+128*13] + pmulld m23, m12, [cq+128*15] + call m(idct_16x16_internal_10bpc).main_rect2 + vpbroadcastd m11, [o(pd_1)] + call m(idct_16x16_internal_10bpc).main_end2 + jmp m(idct_16x16_internal_10bpc).main_end3 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly + +cglobal inv_txfm_add_identity_identity_16x32_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m10, [pw_2896x8] + vpbroadcastd m11, [pw_1697x16] + vpbroadcastd m13, [pw_8192] + vpbroadcastd m15, [pixel_10bpc_max] + lea r6, [strideq*9] + pxor m14, m14 + paddw m12, m13, m13 ; pw_16384 + cmp eobd, 151 + jl .main + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] +.main: + call .main_internal + add cq, 128*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal +.main2: + pmulhrsw m2, m13 + pmulhrsw m4, m13 + pmulhrsw m6, m13 + pmulhrsw m8, m13 + punpcklqdq m0, m1, m2 ; 0 8 + punpckhqdq m1, m2 ; 1 9 + call .write_16x2x2 + punpcklqdq m0, m3, m4 ; 2 10 + punpckhqdq m1, m3, m4 ; 3 11 + call .write_16x2x2 + punpcklqdq m0, m5, m6 ; 4 12 + punpckhqdq m1, m5, m6 ; 5 13 + call .write_16x2x2 + punpcklqdq m0, m7, m8 ; 6 14 + punpckhqdq m1, m7, m8 ; 7 15 +.write_16x2x2: + mova ym2, [dstq+strideq*0] + vinserti32x8 m2, [dstq+strideq*8], 1 + mova ym9, [dstq+strideq*1] + vinserti32x8 m9, [dstq+r6 ], 1 + paddw m0, m2 + paddw m1, m9 + pmaxsw m0, m14 + pmaxsw m1, m14 + pminsw m0, m15 + pminsw m1, m15 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*8], m0, 1 + mova [dstq+strideq*1], ym1 + vextracti32x8 [dstq+r6 ], m1, 1 + lea dstq, [dstq+strideq*2] + ret +.main_internal: + mova m8, [cq+128* 0] + packssdw m8, [cq+128* 8] + mova m6, [cq+128* 1] + packssdw m6, [cq+128* 9] + mova m0, [cq+128* 2] + packssdw m0, [cq+128*10] + mova m2, [cq+128* 3] + packssdw m2, [cq+128*11] + REPX {pmulhrsw x, m10}, m8, m6, m0, m2 + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + pmulhrsw m4, m11, m8 + pmulhrsw m9, m11, m6 + REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 + pmulhrsw m4, m12 + pmulhrsw m9, m12 + paddsw m8, m4 + paddsw m6, m9 + pmulhrsw m4, m11, m0 + pmulhrsw m9, m11, m2 + REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 + pmulhrsw m4, m12 + pmulhrsw m9, m12 + paddsw m0, m4 + paddsw m2, m9 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + +cglobal inv_txfm_add_dct_dct_32x16_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] +%if WIN64 + movaps [rsp+ 8], xmm6 + movaps [rsp+24], xmm7 +%endif + mov r6d, 8*12 + cmp eobd, 36 + jl .fast + pmulld m0, m12, [cq+64* 0] + pmulld m1, m12, [cq+64* 4] + pmulld m2, m12, [cq+64* 8] + pmulld m3, m12, [cq+64*12] + pmulld m16, m12, [cq+64* 2] + pmulld m17, m12, [cq+64* 6] + pmulld m18, m12, [cq+64*10] + pmulld m19, m12, [cq+64*14] + cmp eobd, 151 + jge .full + call m(idct_8x16_internal_10bpc).main_fast_rect2 + call m(idct_16x16_internal_10bpc).main_fast_rect2 + call .idct16_sumsub + call .pass1_load_spill + call .main_fast_rect2 + jmp .pass1_end +.full: + pmulld m4, m12, [cq+64*16] + pmulld m5, m12, [cq+64*20] + pmulld m6, m12, [cq+64*24] + pmulld m7, m12, [cq+64*28] + pmulld m20, m12, [cq+64*18] + pmulld m21, m12, [cq+64*22] + pmulld m22, m12, [cq+64*26] + pmulld m23, m12, [cq+64*30] + add r6d, 8*16 + call m(idct_8x16_internal_10bpc).main_rect2 + call m(idct_16x16_internal_10bpc).main_rect2 + call .idct16_sumsub + call .pass1_load_spill + pmulld m16, m12, [cq+64*17] + pmulld m17, m12, [cq+64*19] + pmulld m18, m12, [cq+64*21] + pmulld m19, m12, [cq+64*23] + pmulld m20, m12, [cq+64*25] + pmulld m21, m12, [cq+64*27] + pmulld m22, m12, [cq+64*29] + pmulld m23, m12, [cq+64*31] + call .main_rect2 +.pass1_end: + vpbroadcastd m11, [o(pd_1)] + lea r4, [cq+64] + call .idct32_pass1_end + lea r5, [o_base_8bpc] + punpckhqdq m19, m5, m16 ; 11 + punpcklqdq m5, m16 ; 10 + punpckhqdq m16, m2, m1 ; 5 + punpcklqdq m2, m1 ; 4 + punpcklqdq m1, m15, m4 ; 2 + punpckhqdq m15, m4 ; 3 + punpcklqdq m4, m14, m18 ; 8 + punpckhqdq m18, m14, m18 ; 9 + punpckhqdq m14, m0, m20 ; 1 + punpcklqdq m0, m20 ; 0 + punpckhqdq m20, m6, m17 ; 13 + punpcklqdq m6, m17 ; 12 + punpckhqdq m17, m3, m21 ; 7 + punpcklqdq m3, m21 ; 6 + punpckhqdq m21, m7, m8 ; 15 + punpcklqdq m7, m8 ; 14 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + jmp .end +.fast: + pmulld ym0, ym12, [cq+64*0] + pmulld ym1, ym12, [cq+64*4] + movshdup m7, [o(permB)] + mova ym4, [cq+64*2] + mova ym5, [cq+64*6] + mova ym16, [cq+64*1] + mova ym2, [cq+64*5] + mova ym3, [cq+64*3] + mova ym17, [cq+64*7] + vpermt2q m4, m7, m5 ; 2 6 + vpermt2q m16, m7, m2 ; 1 5 + vpermt2q m17, m7, m3 ; 7 3 + paddd ym0, ym13 + paddd ym1, ym13 + psrad ym0, 12 + psrad ym1, 12 + vpermq m0, m7, m0 ; 0 0 + vpermq m1, m7, m1 ; 4 4 + REPX {pmulld x, m12}, m4, m16, m17 + REPX {paddd x, m13}, m4, m16, m17 + REPX {psrad x, 12 }, m4, m16, m17 + call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 + vpbroadcastd m11, [o(pd_1)] + call m(idct_16x16_internal_10bpc).main_end2 + call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 + lea r5, [o_base_8bpc] + punpckhqdq m14, m0, m2 ; 1 + punpcklqdq m0, m2 ; 0 + punpcklqdq m1, m3, m4 ; 2 + punpckhqdq m15, m3, m4 ; 3 + punpcklqdq m2, m5, m7 ; 4 + punpckhqdq m16, m5, m7 ; 5 + punpcklqdq m3, m6, m8 ; 6 + punpckhqdq m17, m6, m8 ; 7 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast +.end: +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + pxor m12, m12 +.zero_loop: + mova [cq+r6*8+64*3], m12 + mova [cq+r6*8+64*2], m12 + mova [cq+r6*8+64*1], m12 + mova [cq+r6*8+64*0], m12 + sub r6d, 8*4 + jge .zero_loop + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start + pmulhrsw m0, m11, m14 + pmulhrsw m1, m11, m15 + pmulhrsw m2, m11, m16 + pmulhrsw m3, m11, m17 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m18 + pmulhrsw m1, m11, m19 + pmulhrsw m2, m11, m20 + pmulhrsw m3, m11, m21 + vzeroupper + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 16 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 +.dconly2: + vpbroadcastd m3, [o(dconly_10bpc)] + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + vpbroadcastw m2, r6d + paddsw m2, m3 +.dconly_loop: + paddsw m0, m2, [dstq+strideq*0] + paddsw m1, m2, [dstq+strideq*1] + psubusw m0, m3 + psubusw m1, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +ALIGN function_align +.idct16_sumsub: + psubd m23, m0, m22 ; t15 + paddd m0, m22 ; t0 + psubd m22, m1, m21 ; t14 + paddd m1, m21 ; t1 + REPX {pmaxsd x, m14}, m23, m0, m22, m1 + psubd m21, m2, m20 ; t13 + paddd m2, m20 ; t2 + REPX {pminsd x, m15}, m23, m0, m22, m1 + psubd m20, m3, m19 ; t12 + paddd m3, m19 ; t3 + REPX {pmaxsd x, m14}, m21, m2, m20, m3 + psubd m19, m4, m18 ; t11 + paddd m4, m18 ; t4 + REPX {pminsd x, m15}, m21, m2, m20, m3 + psubd m18, m5, m17 ; t10 + paddd m5, m17 ; t5 + REPX {pmaxsd x, m14}, m19, m4, m18, m5 + psubd m17, m6, m16 ; t9 + paddd m6, m16 ; t6 + REPX {pminsd x, m15}, m19, m4, m18, m5 + psubd m16, m7, m9 ; t8 + paddd m7, m9 ; t7 + REPX {pmaxsd x, m14}, m17, m6, m16, m7 + REPX {pminsd x, m15}, m17, m6, m16, m7 + ret +.idct32_pass1_end: + psrlq m12, [o(permC)], 24 ; 0 2 8 10 1 3 9 11 + psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 +%macro IDCT32_PASS1_END 2 ; low, high + paddd m8, m11, [r4+128*%1] + paddd m9, m11, [cq+128*%1] + psubd m10, m8, m%1 ; out 16+n + paddd m8, m%1 ; out 15-n + paddd m%1, m9, m%2 ; out 0+n + psubd m9, m%2 ; out 31-n + REPX {vpsravd x, m11}, m10, m%1, m8, m9 + packssdw m%1, m10 ; 0+n 16+n + packssdw m%2, m8, m9 ; 15-n 31-n +%endmacro + IDCT32_PASS1_END 0, 23 ; 0 16, 15 31 + IDCT32_PASS1_END 7, 16 ; 7 23, 8 24 + mova m14, m13 + vpermi2q m14, m0, m16 + vpermt2q m0, m12, m16 + IDCT32_PASS1_END 1, 22 ; 1 17, 14 30 + IDCT32_PASS1_END 6, 17 ; 6 22, 9 25 + mova m15, m13 + vpermi2q m15, m1, m17 + vpermt2q m1, m12, m17 + IDCT32_PASS1_END 2, 21 ; 2 18, 13 29 + IDCT32_PASS1_END 5, 18 ; 5 21, 10 26 + mova m16, m13 + vpermi2q m16, m2, m18 + vpermt2q m2, m12, m18 + IDCT32_PASS1_END 3, 20 ; 3 19, 12 28 + IDCT32_PASS1_END 4, 19 ; 4 20, 11 27 + mova m17, m13 + vpermi2q m17, m3, m19 + vpermt2q m3, m12, m19 + mova m18, m13 + vpermi2q m18, m4, m20 + vpermt2q m4, m12, m20 + mova m19, m13 + vpermi2q m19, m5, m21 + vpermt2q m5, m12, m21 + mova m20, m13 + vpermi2q m20, m6, m22 + vpermt2q m6, m12, m22 + mova m21, m13 + vpermi2q m21, m7, m23 + vpermt2q m7, m12, m23 + punpckhwd m8, m2, m3 ; c04 d04 c05 d05 c06 d06 c07 d07 + punpcklwd m2, m3 ; c00 d00 c01 d01 c02 d02 c03 d03 + punpckhwd m3, m0, m1 ; a04 b04 a05 b05 a06 b06 a07 b07 + punpcklwd m0, m1 ; a00 b00 a01 b01 a02 b02 a03 b03 + punpckhwd m1, m4, m5 ; e04 f04 e05 f05 e06 f06 e07 f07 + punpcklwd m4, m5 ; e00 f00 e01 f01 e02 f02 e03 f03 + punpckhwd m5, m6, m7 ; g04 h04 g05 h05 g06 h06 g07 h07 + punpcklwd m6, m7 ; g00 h00 g01 h01 g02 h02 g03 h03 + punpckhwd m7, m14, m15 ; a12 b12 a13 b13 a14 b14 a15 b15 + punpcklwd m14, m15 ; a08 b08 a09 b09 a10 b10 a11 b11 + punpckhwd m15, m16, m17 ; c12 d12 c13 d13 c14 d14 c15 d15 + punpcklwd m16, m17 ; c08 d08 c09 d09 c10 d10 c11 d11 + punpckhwd m17, m18, m19 ; e12 f12 e13 f13 e14 f14 e15 f15 + punpcklwd m18, m19 ; e08 f08 e09 f09 e10 f10 e11 f11 + punpckhwd m19, m20, m21 ; g12 h12 g13 h13 g14 h14 g15 h15 + punpcklwd m20, m21 ; g08 h08 g09 h09 g10 h10 g11 h11 + punpckhdq m21, m1, m5 ; e06 f06 g06 h06 e07 f07 g07 h07 + punpckldq m1, m5 ; e04 f04 g04 h04 e05 f05 g05 h05 + punpckhdq m5, m14, m16 ; a10 b10 c10 d10 a11 b11 c11 d11 + punpckldq m14, m16 ; a08 b08 c08 d08 a09 b09 c09 d09 + punpckhdq m16, m18, m20 ; e10 f10 g10 h10 e11 f11 g11 h11 + punpckldq m18, m20 ; e08 f08 g08 h08 e09 f09 g09 h09 + punpckldq m20, m4, m6 ; e00 f00 g00 h00 e01 f01 g01 h01 + punpckhdq m4, m6 ; e02 f02 g02 h02 e03 f03 g03 h03 + punpckldq m6, m7, m15 ; a12 b12 c12 d12 a13 b13 c13 d13 + punpckhdq m7, m15 ; a14 b14 c14 d14 a15 b15 c15 d15 + punpckhdq m15, m0, m2 ; a02 b02 c02 d02 a03 b03 c03 d03 + punpckldq m0, m2 ; a00 b00 c00 d00 a01 b01 c01 d01 + punpckldq m2, m3, m8 ; a04 b04 c04 d04 a05 b05 c05 d05 + punpckhdq m3, m8 ; a06 b06 c06 d06 a07 b07 c07 d07 + punpckhdq m8, m17, m19 ; e14 f14 g14 h14 e15 f15 g15 h15 + punpckldq m17, m19 ; e12 f12 g12 h12 e13 f13 g13 h13 + ret +.pass1_load_spill: + mova [cq+64* 0], m0 + mova [cq+64* 2], m1 + mova [cq+64* 4], m2 + mova [cq+64* 6], m3 + mova [cq+64* 8], m4 + mova [cq+64*10], m5 + mova [cq+64*12], m6 + mova [cq+64*14], m7 + pmulld m0, m12, [cq+64* 1] + pmulld m1, m12, [cq+64* 3] + pmulld m2, m12, [cq+64* 5] + pmulld m3, m12, [cq+64* 7] + pmulld m4, m12, [cq+64* 9] + pmulld m5, m12, [cq+64*11] + pmulld m6, m12, [cq+64*13] + pmulld m7, m12, [cq+64*15] + mova [cq+64* 1], m23 + mova [cq+64* 3], m22 + mova [cq+64* 5], m21 + mova [cq+64* 7], m20 + mova [cq+64* 9], m19 + mova [cq+64*11], m18 + mova [cq+64*13], m17 + mova [cq+64*15], m16 + ret +.main_fast_rect2: + call m(idct_8x16_internal_10bpc).round +.main_fast: ; bottom half is zero + pmulld m23, m0, [o(pd_4091)] {1to16} ; t31a + pmulld m0, [o(pd_201)] {1to16} ; t16a + pmulld m16, m7, [o(pd_2751)] {1to16} ; t17a + pmulld m7, [o(pd_3035)] {1to16} ; t30a + pmulld m19, m4, [o(pd_3703)] {1to16} ; t29a + pmulld m4, [o(pd_1751)] {1to16} ; t18a + pmulld m20, m3, [o(pd_1380)] {1to16} ; t19a + pmulld m3, [o(pd_3857)] {1to16} ; t28a + pmulld m21, m2, [o(pd_3973)] {1to16} ; t27a + pmulld m2, [o(pd_995)] {1to16} ; t20a + pmulld m18, m5, [o(pd_2106)] {1to16} ; t21a + pmulld m5, [o(pd_3513)] {1to16} ; t26a + pmulld m17, m6, [o(pd_3290)] {1to16} ; t25a + pmulld m6, [o(pd_2440)] {1to16} ; t22a + pmulld m22, m1, [o(pd_601)] {1to16} ; t23a + pmulld m1, [o(pd_4052)] {1to16} ; t24a + REPX {psubd x, m13, x}, m16, m20, m18, m22 + call m(idct_16x16_internal_10bpc).round3 + jmp .main2 +.main_rect2: + call m(idct_8x16_internal_10bpc).round + call m(idct_16x16_internal_10bpc).round +.main: + ITX_MULSUB_2D 0, 23, 8, 9, 10, _, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 16, 7, 8, 9, 10, _, 3035, 2751 ; t17a, t30a + ITX_MULSUB_2D 4, 19, 8, 9, 10, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 20, 3, 8, 9, 10, _, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2D 2, 21, 8, 9, 10, _, 995, 3973 ; t20a, t27a + ITX_MULSUB_2D 18, 5, 8, 9, 10, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 6, 17, 8, 9, 10, _, 2440, 3290 ; t22a, t25a + ITX_MULSUB_2D 22, 1, 8, 9, 10, _, 4052, 601 ; t23a, t24a + call m(idct_16x16_internal_10bpc).round +.main2: + call m(idct_8x16_internal_10bpc).round + psubd m8, m0, m16 ; t17 + paddd m0, m16 ; t16 + psubd m16, m23, m7 ; t30 + paddd m23, m7 ; t31 + REPX {pmaxsd x, m14}, m8, m0, m16, m23 + paddd m7, m20, m4 ; t19 + psubd m20, m4 ; t18 + REPX {pminsd x, m15}, m8, m0, m16, m23 + paddd m4, m3, m19 ; t28 + psubd m3, m19 ; t29 + REPX {pmaxsd x, m14}, m7, m20, m4, m3 + psubd m19, m2, m18 ; t21 + paddd m2, m18 ; t20 + REPX {pminsd x, m15}, m7, m20, m4, m3 + psubd m18, m21, m5 ; t26 + paddd m21, m5 ; t27 + REPX {pmaxsd x, m14}, m19, m2, m18, m21 + psubd m5, m22, m6 ; t22 + paddd m6, m22 ; t23 + REPX {pminsd x, m15}, m19, m2, m18, m21 + psubd m22, m1, m17 ; t25 + paddd m17, m1 ; t24 + REPX {pmaxsd x, m14}, m5, m6, m22, m17 + vpbroadcastd m11, [o(pd_4017)] + vpbroadcastd m10, [o(pd_799)] + REPX {pminsd x, m15}, m5, m6, m22, m17 + ITX_MULSUB_2D 16, 8, 9, 1, _, 13, 10, 11 ; t17a, t30a + ITX_MULSUB_2D 3, 20, 9, 1, _, 13, 10, 11, 2 ; t29a, t18a + vpbroadcastd m11, [o(pd_2276)] + vpbroadcastd m10, [o(pd_3406)] + ITX_MULSUB_2D 18, 19, 9, 1, _, 13, 10, 11 ; t21a, t26a + ITX_MULSUB_2D 22, 5, 9, 1, _, 13, 10, 11, 2 ; t25a, t22a + paddd m1, m6, m2 ; t23a + psubd m6, m2 ; t20a + psubd m2, m17, m21 ; t27a + paddd m17, m21 ; t24a + REPX {pmaxsd x, m14}, m1, m6, m2, m17 + psubd m21, m23, m4 ; t28a + paddd m23, m4 ; t31a + REPX {pminsd x, m15}, m1, m6, m2, m17 + psubd m4, m16, m20 ; t18 + paddd m16, m20 ; t17 + REPX {pmaxsd x, m14}, m21, m23, m4, m16 + psubd m20, m0, m7 ; t19a + paddd m0, m7 ; t16a + REPX {pminsd x, m15}, m21, m23, m4, m16 + psubd m7, m8, m3 ; t29 + paddd m3, m8 ; t30 + REPX {pmaxsd x, m14}, m20, m0, m7, m3 + paddd m8, m5, m18 ; t22 + psubd m5, m18 ; t21 + REPX {pminsd x, m15}, m20, m0, m7, m3 + psubd m18, m22, m19 ; t26 + paddd m22, m19 ; t25 + REPX {pmaxsd x, m14}, m8, m5, m18, m22 + vpbroadcastd m11, [o(pd_3784)] + vpbroadcastd m10, [o(pd_1567)] + REPX {pminsd x, m15}, m8, m5, m18, m22 + ITX_MULSUB_2D 21, 20, 9, 19, _, 13, 10, 11 ; t19, t28 + ITX_MULSUB_2D 2, 6, 9, 19, _, 13, 10, 11, 2 ; t27, t20 + ITX_MULSUB_2D 7, 4, 9, 19, _, 13, 10, 11 ; t18a, t29a + ITX_MULSUB_2D 18, 5, 9, 19, _, 13, 10, 11, 2 ; t26a, t21a + psubd m19, m0, m1 ; t23 + paddd m0, m1 ; t16 + paddd m1, m8, m16 ; t17a + psubd m8, m16, m8 ; t22a + REPX {pmaxsd x, m14}, m19, m0, m1, m8 + psubd m16, m23, m17 ; t24 + paddd m23, m17 ; t31 + REPX {pminsd x, m15}, m19, m0, m1, m8 + psubd m17, m3, m22 ; t25a + paddd m22, m3 ; t30a + REPX {pmaxsd x, m14}, m16, m23, m17, m22 + paddd m3, m6, m21 ; t19a + psubd m6, m21, m6 ; t20a + REPX {pminsd x, m15}, m16, m23, m17, m22 + paddd m21, m18, m4 ; t29 + psubd m18, m4, m18 ; t26 + REPX {pmaxsd x, m14}, m3, m6, m21, m18 + psubd m4, m20, m2 ; t27a + paddd m20, m2 ; t28a + REPX {pminsd x, m15}, m3, m6, m21, m18 + paddd m2, m7, m5 ; t18 + psubd m7, m5 ; t21 + REPX {pmaxsd x, m14}, m4, m20, m2, m7 + REPX {pminsd x, m15}, m4, m20, m2, m7 + REPX {pmulld x, m12}, m18, m16, m4, m17, m7, m19, m6, m8 + REPX {paddd x, m13}, m18, m16, m4, m17 + psubd m5, m18, m7 ; t21a + paddd m18, m7 ; t26a + psubd m7, m16, m19 ; t23a + paddd m16, m19 ; t24a + REPX {psrad x, 12 }, m5, m18, m7, m16 + paddd m19, m4, m6 ; t27 + psubd m4, m6 ; t20 + psubd m6, m17, m8 ; t22 + paddd m17, m8 ; t25 + REPX {psrad x, 12 }, m19, m4, m6, m17 + ret + +cglobal inv_txfm_add_identity_identity_32x16_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m10, [pw_2896x8] + vpbroadcastd m11, [pw_1697x16] + vpbroadcastd m13, [pw_2048] + vpbroadcastd m15, [pixel_10bpc_max] + lea r6, [strideq*9] + pxor m14, m14 + cmp eobd, 151 + jl .main + mov r4, dstq + call .main + add cq, 64*12 + lea dstq, [r4+32] +.main: + call .main_internal + add cq, 64*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 +.main_internal: + mova m8, [cq+64* 0] + packssdw m8, [cq+64* 8] + mova m6, [cq+64* 1] + packssdw m6, [cq+64* 9] + mova m0, [cq+64* 2] + packssdw m0, [cq+64*10] + mova m2, [cq+64* 3] + packssdw m2, [cq+64*11] + REPX {pmulhrsw x, m10}, m8, m6, m0, m2 + REPX {paddsw x, x }, m8, m6, m0, m2 + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + pmulhrsw m4, m11, m8 + pmulhrsw m9, m11, m6 + paddsw m8, m8 + paddsw m6, m6 + REPX {mova [cq+64*x], m14}, 0, 1, 2, 3 + paddsw m8, m4 + paddsw m6, m9 + pmulhrsw m4, m11, m0 + pmulhrsw m9, m11, m2 + paddsw m0, m0 + paddsw m2, m2 + REPX {mova [cq+64*x], m14}, 8, 9, 10, 11 + paddsw m0, m4 + paddsw m2, m9 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + +cglobal inv_txfm_add_dct_dct_32x32_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + WIN64_SPILL_XMM 30 + cmp eobd, 136 + jl .fast + add cq, 64 + cmp eobd, 543 + jge .full + call .pass1_fast ; bottomright 16x16 zero + mov r6d, 16*12 + jmp .lefthalf +.full: + call .pass1 + mov r6d, 16*28 +.lefthalf: + mova [cq+128* 0], m0 + mova [cq+128* 1], m1 + mova [cq+128* 2], m2 + mova [cq+128* 3], m3 + mova [cq+128* 4], m14 + mova [cq+128* 5], m15 + mova [cq+128* 6], m16 + mova [cq+128* 7], m17 + mova [cq+128* 8], m22 + mova [cq+128* 9], m23 + mova [cq+128*10], m24 + mova [cq+128*11], m25 + mova [cq+128*12], m26 + mova [cq+128*13], m27 + mova [cq+128*14], m28 + mova [cq+128*15], m29 + sub cq, 64 + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + call .pass1 + lea r5, [o_base_8bpc] + mova m4, [cq+64+128* 0] + mova m5, [cq+64+128* 1] + mova m6, [cq+64+128* 2] + mova m7, [cq+64+128* 3] + mova m18, [cq+64+128* 4] + mova m19, [cq+64+128* 5] + mova m20, [cq+64+128* 6] + mova m21, [cq+64+128* 7] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + mova m14, [cq+64+128* 8] + mova m15, [cq+64+128* 9] + mova m16, [cq+64+128*10] + mova m17, [cq+64+128*11] + mova m18, [cq+64+128*12] + mova m19, [cq+64+128*13] + mova m20, [cq+64+128*14] + mova m21, [cq+64+128*15] + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf + pxor m12, m12 +.right_zero_loop: + mova [cq+r6*8+64+128*3], m12 + mova [cq+r6*8+64+128*2], m12 + mova [cq+r6*8+64+128*1], m12 + mova [cq+r6*8+64+128*0], m12 + sub r6d, 16*4 + jge .right_zero_loop + mov r6d, 16*28 + jmp .end2 +.fast: ; topleft 16x16 nonzero + cmp eobd, 36 + jl .fast2 + call .pass1_fast + lea r5, [o_base_8bpc] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + jmp .end +.fast2: ; topleft 8x8 nonzero + movshdup m7, [o(permB)] + mova ym0, [cq+128*0] + mova ym1, [cq+128*4] + mova ym4, [cq+128*2] + mova ym5, [cq+128*6] + mova ym16, [cq+128*1] + mova ym2, [cq+128*5] + mova ym3, [cq+128*3] + mova ym17, [cq+128*7] + mov r6d, 16*4 + vpermq m0, m7, m0 ; 0 0 + vpermq m1, m7, m1 ; 4 4 + vpermt2q m4, m7, m5 ; 2 6 + vpermt2q m16, m7, m2 ; 1 5 + vpermt2q m17, m7, m3 ; 7 3 + call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 + call m(idct_16x16_internal_10bpc).main_end + call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 + lea r5, [o_base_8bpc] + punpckhqdq m22, m0, m2 ; 1 + punpcklqdq m0, m2 ; 0 + punpcklqdq m1, m5, m7 ; 4 + punpckhqdq m24, m5, m7 ; 5 + punpcklqdq m14, m3, m4 ; 2 + punpckhqdq m23, m3, m4 ; 3 + punpcklqdq m15, m6, m8 ; 6 + punpckhqdq m25, m6, m8 ; 7 + mova m10, m13 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 +.end: + pxor m12, m12 +.end2: + psubsw m9, m0, m29 ; out31 + paddsw m0, m29 ; out0 + psubsw m29, m1, m28 ; out30 + paddsw m1, m28 ; out1 + psubsw m28, m2, m27 ; out29 + paddsw m2, m27 ; out2 + psubsw m27, m3, m26 ; out28 + paddsw m3, m26 ; out3 + psubsw m26, m4, m25 ; out27 + paddsw m4, m25 ; out4 + psubsw m25, m5, m24 ; out26 + paddsw m5, m24 ; out5 + psubsw m24, m6, m23 ; out25 + paddsw m6, m23 ; out6 + psubsw m23, m7, m22 ; out24 + paddsw m7, m22 ; out7 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start + mova m0, [cq+128*0] + mova m1, [cq+128*1] + mova m2, [cq+128*2] + mova m3, [cq+128*3] + mova m4, [cq+128*4] + mova m5, [cq+128*5] + mova m6, [cq+128*6] + mova m7, [cq+128*7] + psubsw m22, m0, m21 ; out23 + paddsw m0, m21 ; out8 + psubsw m21, m1, m20 ; out22 + paddsw m1, m20 ; out9 + psubsw m20, m2, m19 ; out21 + paddsw m2, m19 ; out10 + psubsw m19, m3, m18 ; out20 + paddsw m3, m18 ; out11 + psubsw m18, m4, m17 ; out19 + paddsw m4, m17 ; out12 + psubsw m17, m5, m16 ; out18 + paddsw m5, m16 ; out13 + psubsw m16, m6, m15 ; out17 + paddsw m6, m15 ; out14 + psubsw m15, m7, m14 ; out16 + paddsw m7, m14 ; out15 +.zero_loop: + mova [cq+r6*8+128*3], m12 + mova [cq+r6*8+128*2], m12 + mova [cq+r6*8+128*1], m12 + mova [cq+r6*8+128*0], m12 + sub r6d, 16*4 + jge .zero_loop + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8 + pmulhrsw m0, m11, m15 + pmulhrsw m1, m11, m16 + pmulhrsw m2, m11, m17 + pmulhrsw m3, m11, m18 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m19 + pmulhrsw m1, m11, m20 + pmulhrsw m2, m11, m21 + pmulhrsw m3, m11, m22 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m23 + pmulhrsw m1, m11, m24 + pmulhrsw m2, m11, m25 + pmulhrsw m3, m11, m26 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m27 + pmulhrsw m1, m11, m28 + pmulhrsw m2, m11, m29 + pmulhrsw m3, m11, m9 + WIN64_RESTORE_XMM + vzeroupper + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 +.pass1_fast: + mova m0, [cq+128* 0] + mova m1, [cq+128* 4] + mova m2, [cq+128* 8] + mova m3, [cq+128*12] + mov r6d, 16*12 + call m(idct_8x16_internal_10bpc).main_fast + mova m16, [cq+128* 2] + mova m17, [cq+128* 6] + mova m18, [cq+128*10] + mova m19, [cq+128*14] + call m(idct_16x16_internal_10bpc).main_fast + call .pass1_load_spill + call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast + jmp .pass1_end +.pass1: + mova m0, [cq+128* 0] + mova m1, [cq+128* 4] + mova m2, [cq+128* 8] + mova m3, [cq+128*12] + mova m4, [cq+128*16] + mova m5, [cq+128*20] + mova m6, [cq+128*24] + mova m7, [cq+128*28] + call m(idct_8x16_internal_10bpc).main + mova m16, [cq+128* 2] + mova m17, [cq+128* 6] + mova m18, [cq+128*10] + mova m19, [cq+128*14] + mova m20, [cq+128*18] + mova m21, [cq+128*22] + mova m22, [cq+128*26] + mova m23, [cq+128*30] + call m(idct_16x16_internal_10bpc).main + call .pass1_load_spill + mova m16, [cq+128*17] + mova m17, [cq+128*19] + mova m18, [cq+128*21] + mova m19, [cq+128*23] + mova m20, [cq+128*25] + mova m21, [cq+128*27] + mova m22, [cq+128*29] + mova m23, [cq+128*31] + call m(inv_txfm_add_dct_dct_32x16_10bpc).main +.pass1_end: + vpbroadcastd m11, [o(pd_2)] + lea r4, [cq+128*8] + call m(inv_txfm_add_dct_dct_32x16_10bpc).idct32_pass1_end + punpckhqdq m22, m0, m20 ; 1 + punpcklqdq m0, m20 ; 0 + punpckhqdq m24, m2, m1 ; 5 + punpcklqdq m1, m2, m1 ; 4 + punpcklqdq m2, m14, m18 ; 8 + punpckhqdq m26, m14, m18 ; 9 + punpcklqdq m14, m15, m4 ; 2 + punpckhqdq m23, m15, m4 ; 3 + punpckhqdq m25, m3, m21 ; 7 + punpcklqdq m15, m3, m21 ; 6 + punpckhqdq m28, m6, m17 ; 13 + punpcklqdq m3, m6, m17 ; 12 + punpckhqdq m27, m5, m16 ; 11 + punpcklqdq m16, m5, m16 ; 10 + punpckhqdq m29, m7, m8 ; 15 + punpcklqdq m17, m7, m8 ; 14 + ret +.pass1_load_spill: + call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub + mova [cq+128* 0], m0 + mova m0, [cq+128* 1] + mova [cq+128* 1], m1 + mova [cq+128* 2], m2 + mova m1, [cq+128* 3] + mova m2, [cq+128* 5] + mova [cq+128* 3], m3 + mova [cq+128* 4], m4 + mova m3, [cq+128* 7] + mova m4, [cq+128* 9] + mova [cq+128* 5], m5 + mova [cq+128* 6], m6 + mova [cq+128* 7], m7 + mova m5, [cq+128*11] + mova m6, [cq+128*13] + mova m7, [cq+128*15] + mova [cq+128* 8], m23 + mova [cq+128* 9], m22 + mova [cq+128*10], m21 + mova [cq+128*11], m20 + mova [cq+128*12], m19 + mova [cq+128*13], m18 + mova [cq+128*14], m17 + mova [cq+128*15], m16 + ret + +cglobal inv_txfm_add_identity_identity_32x32_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m13, [pw_8192] + vpbroadcastd m15, [pixel_10bpc_max] + pxor m14, m14 + lea r6, [strideq*9] + cmp eobd, 136 + jl .main + mov r4, dstq + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] + call .main + add cq, 128*12-64 + lea dstq, [r4+32] + cmp eobd, 543 + jl .main + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] +.main: + call .main_internal + add cq, 128*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 +.main_internal: + mova m8, [cq+128* 0] + packssdw m8, [cq+128* 8] + mova m6, [cq+128* 1] + packssdw m6, [cq+128* 9] + mova m0, [cq+128* 2] + packssdw m0, [cq+128*10] + mova m2, [cq+128* 3] + packssdw m2, [cq+128*11] + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + +cglobal inv_txfm_add_dct_dct_16x64_10bpc, 4, 7, 0, dst, stride, c, eob + lea r5, [o_base] + test eobd, eobd + jz .dconly + + PROLOGUE 4, 7, 32, -8*mmsize, dst, stride, c, eob +%undef cmp + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + cmp eobd, 36 + jl .fast + call .pass1 + cmp eobd, 151 + jge .full + lea r5, [o_base_8bpc] + + punpckhwd m22, m0, m0 + punpckhwd m23, m1, m1 + punpckhwd m24, m2, m2 + punpckhwd m25, m3, m3 + punpckhwd m26, m4, m4 + punpckhwd m27, m5, m5 + punpckhwd m28, m6, m6 + punpckhwd m29, m7, m7 + punpcklwd m21, m1, m1 + punpcklwd m14, m3, m3 + punpcklwd m18, m5, m5 + punpcklwd m15, m7, m7 + pxor m9, m9 + punpcklwd m9, m9, m0 + punpcklwd m8, m2, m2 + punpcklwd m7, m4, m4 + punpcklwd m1, m6, m6 + call m(idct_16x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + mova [rsp+mmsize*0], m14 + mova [rsp+mmsize*1], m15 + mova [rsp+mmsize*2], m16 + mova [rsp+mmsize*3], m17 + mova [rsp+mmsize*4], m18 + mova [rsp+mmsize*5], m19 + mova [rsp+mmsize*6], m20 + mova [rsp+mmsize*7], m21 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast + + pxor m12, m12 + mov r3d, 64*3 +.zero_loop: + REPX {mova [cq+r3*8+128*x], m12}, 0, 1, 2, 3 + sub r3d, 64 + jge .zero_loop + + jmp .pass2_end +.full: + mova [cq+128*0], m0 + mova [cq+128*1], m1 + mova [cq+128*2], m2 + mova [cq+128*3], m3 + mova [cq+128*4], m4 + mova [cq+128*5], m5 + mova [cq+128*6], m6 + mova [cq+128*7], m7 + add cq, 64 + call .pass1 + sub cq, 64 + mova m22, [cq+128*0] ; 0 1 + mova m23, [cq+128*1] ; 2 3 + mova m24, [cq+128*2] ; 4 5 + mova m25, [cq+128*3] ; 6 7 + mova m26, [cq+128*4] ; 8 9 + mova m27, [cq+128*5] ; 10 11 + mova m28, [cq+128*6] ; 12 13 + mova m29, [cq+128*7] ; 14 15 + mova [cq+64* 8], m0 + mova [cq+64* 9], m1 + mova [cq+64*10], m2 + mova [cq+64*11], m3 + mova [cq+64*12], m4 + mova [cq+64*13], m5 + mova [cq+64*14], m6 + mova [cq+64*15], m7 + lea r5, [o_base_8bpc] + + punpcklwd m20, m1, m1 + punpcklwd m16, m3, m3 + punpcklwd m19, m5, m5 + punpcklwd m17, m7, m7 + punpcklwd m8, m24, m24 ; 4 + punpcklwd m5, m2, m2 ; 20 + punpcklwd m1, m28, m28 ; 12 + punpcklwd m7, m26, m26 ; 8 + punpcklwd m3, m4, m4 ; 24 + punpcklwd m4, m6, m6 ; 28 + pxor m9, m9 + punpcklwd m6, m9, m0 ; __ 16 + mova m0, m4 + punpcklwd m9, m9, m22 ; __ 0 + call m(idct_16x16_internal_8bpc).main_fast + punpcklwd m21, m23, m23 ; 2 + punpcklwd m15, m29, m29 ; 14 + punpcklwd m18, m27, m27 ; 10 + punpcklwd m14, m25, m25 ; 6 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova [rsp+mmsize*0], m14 + mova [rsp+mmsize*1], m15 + mova [rsp+mmsize*2], m16 + mova [rsp+mmsize*3], m17 + mova [rsp+mmsize*4], m18 + mova [rsp+mmsize*5], m19 + mova [rsp+mmsize*6], m20 + mova [rsp+mmsize*7], m21 + mova m21, [cq+64*15] + mova m14, [cq+64* 8] + mova m17, [cq+64*11] + mova m18, [cq+64*12] + mova m19, [cq+64*13] + mova m16, [cq+64*10] + mova m15, [cq+64* 9] + mova m20, [cq+64*14] + REPX {punpckhwd x, x}, m22, m21, m14, m29, m26, m17, m18, m25, \ + m24, m19, m16, m27, m28, m15, m20, m23 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf + + pxor m12, m12 + mov r3d, 32*7 +.full_zero_loop: + REPX {mova [cq+r3*8+64*x], m12}, 0, 1, 2, 3 + sub r3d, 32 + jge .full_zero_loop + + jmp .pass2_end +.fast: + mova ym0, [cq+128*0] + mova ym2, [cq+128*4] + movshdup m8, [o(permB)] + mova ym1, [cq+128*2] + mova ym3, [cq+128*6] + mova ym4, [cq+128*1] + mova ym5, [cq+128*3] + mova ym6, [cq+128*5] + mova ym7, [cq+128*7] + vpermt2q m0, m8, m2 ; 0 4 + vpermt2q m1, m8, m3 ; 2 6 + vpermt2q m4, m8, m5 ; 1 3 + vpermt2q m7, m8, m6 ; 7 5 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + vpbroadcastd m11, [o(pd_2)] + call m(idct_8x16_internal_10bpc).main_end2 + mova m8, [o(idct8x32p)] + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + mova m6, [dup16_perm] + vpermb m0, m8, m0 + vpermb m2, m8, m2 + vprold m8, 16 + vpermb m1, m8, m1 + vpermb m3, m8, m3 + punpckldq m4, m0, m2 + punpckhdq m0, m2 + punpckldq m2, m1, m3 + punpckhdq m1, m3 + punpckldq m21, m4, m2 + punpckhdq m14, m4, m2 + punpckldq m18, m0, m1 + punpckhdq m15, m0, m1 + vpord m7, m6, [o(pb_32)] {1to16} + vpermb m22, m7, m21 ; 1 + pmovzxwd m9, ym21 ; 0 + vpermb m8, m6, m18 ; 4 + vpermb m24, m7, m18 ; 5 + vpermb m21, m6, m14 ; 2 + vpermb m23, m7, m14 ; 3 + vpermb m14, m6, m15 ; 6 + vpermb m25, m7, m15 ; 7 + lea r5, [o_base_8bpc] + pslld m9, 16 + + pxor m7, m7 + REPX {mova x, m7}, m1, m18, m15, m26, m27, m28, m29 + + call m(idct_16x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + mova [rsp+mmsize*0], m14 + mova [rsp+mmsize*1], m15 + mova [rsp+mmsize*2], m16 + mova [rsp+mmsize*3], m17 + mova [rsp+mmsize*4], m18 + mova [rsp+mmsize*5], m19 + mova [rsp+mmsize*6], m20 + mova [rsp+mmsize*7], m21 + + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast + + pxor m12, m12 + REPX {mova [cq+128*x], ym12}, 0, 1, 2, 3, 4, 5, 6, 7 +.pass2_end: + movshdup m30, [permC] + vpbroadcastd m11, [pw_2048] + vpbroadcastd m13, [pixel_10bpc_max] + lea r6, [strideq*3] + psrlq m31, m30, 8 + vpermq m8, m30, m0 + vpermq m9, m31, m1 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m2 + vpermq m9, m31, m3 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m4 + vpermq m9, m31, m5 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m6 + vpermq m9, m31, m7 + call m(idct_16x8_internal_10bpc).write_16x4 + + mova m1, [rsp+mmsize*0] + mova m2, [rsp+mmsize*1] + mova m3, [rsp+mmsize*2] + mova m4, [rsp+mmsize*3] + mova m5, [rsp+mmsize*4] + mova m6, [rsp+mmsize*5] + mova m7, [rsp+mmsize*6] + mova m8, [rsp+mmsize*7] + + paddsw m0, m1, m21 + psubsw m21, m1, m21 + paddsw m1, m2, m20 + psubsw m20, m2, m20 + paddsw m2, m3, m19 + psubsw m19, m3, m19 + paddsw m3, m4, m18 + psubsw m18, m4, m18 + paddsw m4, m5, m17 + psubsw m17, m5, m17 + paddsw m5, m6, m16 + psubsw m16, m6, m16 + paddsw m6, m7, m15 + psubsw m15, m7, m15 + paddsw m7, m8, m14 + psubsw m14, m8, m14 + + vpermq m8, m30, m0 + vpermq m9, m31, m1 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m2 + vpermq m9, m31, m3 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m4 + vpermq m9, m31, m5 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m6 + vpermq m9, m31, m7 + call m(idct_16x8_internal_10bpc).write_16x4 + + vpermq m8, m30, m14 + vpermq m9, m31, m15 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m16 + vpermq m9, m31, m17 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m18 + vpermq m9, m31, m19 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m20 + vpermq m9, m31, m21 + call m(idct_16x8_internal_10bpc).write_16x4 + + vpermq m8, m30, m22 + vpermq m9, m31, m23 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m24 + vpermq m9, m31, m25 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m26 + vpermq m9, m31, m27 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m30, m28 + vpermq m9, m31, m29 + call m(idct_16x8_internal_10bpc).write_16x4 + RET +.pass1: + mova m0, [cq+128* 0] + mova m1, [cq+128* 2] + mova m2, [cq+128* 4] + mova m3, [cq+128* 6] + mova m4, [cq+128* 8] + mova m5, [cq+128*10] + mova m6, [cq+128*12] + mova m7, [cq+128*14] + call m(idct_8x16_internal_10bpc).main + mova m16, [cq+128* 1] + mova m17, [cq+128* 3] + mova m18, [cq+128* 5] + mova m19, [cq+128* 7] + mova m20, [cq+128* 9] + mova m21, [cq+128*11] + mova m22, [cq+128*13] + mova m23, [cq+128*15] + call m(idct_16x16_internal_10bpc).main + call m(idct_16x16_internal_10bpc).main_end + jmp m(idct_16x16_internal_10bpc).main_end3 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 64 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly2 + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/itx16_sse.asm b/third_party/dav1d/src/x86/itx16_sse.asm new file mode 100644 index 0000000000..3833e17c99 --- /dev/null +++ b/third_party/dav1d/src/x86/itx16_sse.asm @@ -0,0 +1,8135 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; Copyright © 2017-2021, The rav1e contributors +; Copyright © 2020, Nathan Egge +; Copyright © 2021, Matthias Dressel +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA +%macro COEF 1-2 +pd_%1: times 4 dd %1 +%if %0 == 2 +pd_m%1: times 4 dd -%1 +%endif +%endmacro + +COEF 201 +COEF 401 +COEF 601, 1 +COEF 799 +COEF 995 +COEF 1189, 1 +COEF 1380, 1 +COEF 1567 +COEF 1751 +COEF 1931 +COEF 2106, 1 +COEF 2276, 1 +COEF 2440 +COEF 2598, 1 +COEF 2751, 1 +COEF 2896 +COEF 3035 +COEF 3166 +COEF 3290 +COEF 3406 +COEF 3513 +COEF 3612 +COEF 3703 +COEF 3784 +COEF 3857 +COEF 3920 +COEF 3973 +COEF 4017 +COEF 4052 +COEF 4076 +COEF 4091 + +deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 + +%if ARCH_X86_32 +pd_1: times 4 dd 1 +%endif +pd_2: times 4 dd 2 +pw_5: times 8 dw 5 +pd_1321: times 4 dd 1321 +pd_2482: times 4 dd 2482 +pd_m3344: times 4 dd -3344 +pd_2048: times 4 dd 2048 +pw_4x2048_4xm2048: times 4 dw 2048 + times 4 dw -2048 +pw_4xm2048_4x2048: times 4 dw -2048 + times 4 dw 2048 +pw_2048: times 8 dw 2048 +pw_m2048: times 8 dw -2048 +pd_3803: times 4 dd 3803 +pw_4096: times 8 dw 4096 +pd_5793: times 4 dd 5793 +pd_6144: times 4 dd 6144 +pw_8192: times 8 dw 8192 +pd_10240: times 4 dd 10240 +pd_11586: times 4 dd 11586 +pw_1697x8: times 8 dw 1697*8 +pw_2896x8: times 8 dw 2896*8 +pw_1697x16: times 8 dw 1697*16 +pw_16384: times 8 dw 16384 +pixel_10bpc_max: times 8 dw 0x03ff + +pw_1567_3784: times 4 dw 1567, 3784 +pw_m3784_1567: times 4 dw -3784, 1567 +pw_2896_2896: times 4 dw 2896, 2896 +pw_m2896_2896: times 4 dw -2896, 2896 + +clip_18b_min: times 4 dd -0x20000 +clip_18b_max: times 4 dd 0x1ffff + +idct64_mul_16bpc: +dd 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474, 401, 4076, 799, 4017 +dd -700, 4036, 2359, 3349, -2191, 3461, 897, 3996, -2598, -3166, -4017, -799 +dd 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092, 1931, 3612, 3406, 2276 +dd -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889, -1189, -3920, -2276, -3406 + +cextern inv_txfm_add_dct_dct_4x4_8bpc_ssse3 +cextern iadst_4x4_internal_8bpc_ssse3.main +cextern idct_4x8_internal_8bpc_ssse3.main +cextern iadst_4x8_internal_8bpc_ssse3.main +cextern idct_16x4_internal_8bpc_ssse3.main +cextern iadst_16x4_internal_8bpc_ssse3.main +cextern iadst_16x4_internal_8bpc_ssse3.main_pass2_end +cextern idct_8x4_internal_8bpc_ssse3.main +cextern iadst_8x4_internal_8bpc_ssse3.main +cextern idct_8x8_internal_8bpc_ssse3.main +cextern idct_8x8_internal_8bpc_ssse3.pass1_end3 +cextern iadst_8x8_internal_8bpc_ssse3.main +cextern iadst_8x8_internal_8bpc_ssse3.main_pass2_end +cextern idct_16x8_internal_8bpc_ssse3.main +cextern iadst_16x8_internal_8bpc_ssse3.main +cextern iadst_16x8_internal_8bpc_ssse3.main_pass2_end +cextern idct_8x32_internal_8bpc_ssse3.main +cextern idct_8x32_internal_8bpc_ssse3.main_fast +cextern idct_8x32_internal_8bpc_ssse3.main_veryfast +cextern idct_16x64_internal_8bpc_ssse3.main +cextern idct_16x64_internal_8bpc_ssse3.main_fast + +tbl_4x16_2d: db 0, 13, 29, 45 +tbl_4x16_h: db 0, 16, 32, 48 +tbl_4x16_v: db 0, 4, 8, 12 + +tbl_8x16_2d: db 0, 14, 30, 46 +tbl_8x16_v: db 0, 4, 8, 12 +tbl_8x16_h: db 0, 32, 64, 96 + +tbl_16x16_2d: db 0, 10, 36, 78 +tbl_16x16_v: db 0, 4, 8, 12 +tbl_16x16_h: db 0, 64, 128, 192 + +tbl_8x32_2d: dw 0, 14, 43, 75, 107, 139, 171, 203 + +tbl_16x32_2d: dw 0, 14, 44, 90, 151, 215, 279, 343 + +tbl_32x16_2d: ; first 4 entries of 32x32 are identical to this one +tbl_32x32_2d: dw 0, 10, 36, 78, 136, 210, 300, 406 + +tbl_Nx32_odd_offset: db 2*16, 2*23 + db 2*20, 2*19 + db 2*18, 2*21 + db 2*22, 2*17 + db 2*30, 2*25 + db 2*26, 2*29 + db 2*28, 2*27 + db 2*24, 2*31 + +tbl_Nx64_offset: db 2* 0, 2*32, 2*16, 2*46 + db 2* 8, 2*40, 2*23, 2*38 + db 2* 1, 2*36, 2*20, 2*42 + db 2* 9, 2*44, 2*19, 2*34 + db 2* 2, 2*60, 2*18, 2*50 + db 2*10, 2*52, 2*21, 2*58 + db 2* 3, 2*56, 2*22, 2*54 + db 2*11, 2*48, 2*17, 2*62 + +SECTION .text + +%define m_suffix(x, sfx) mangle(private_prefix %+ _ %+ x %+ sfx) +%define m(x) m_suffix(x, SUFFIX) + +; This refers to the first function in itx_sse i.e. the start of the text section +; which is needed as a base pointer for constants. +%define itx8_start m_suffix(inv_txfm_add_dct_dct_4x4_8bpc, _ssse3) + +%if ARCH_X86_64 +%define o(x) x +%else +%define o(x) r6-$$+x ; PIC +%endif + +%macro IWHT4_1D 0 + ; m0 = in0, m1 = in1, m2 = in2, m3 = in3 + paddd m0, m1 ; in0 += in1 + psubd m4, m2, m3 ; tmp0 = in2 - in3 + psubd m5, m0, m4 ; tmp1 = (in0 - tmp0) >> 1 + psrad m5, 1 + psubd m2, m5, m1 ; in2 = tmp1 - in1 + psubd m5, m3 ; in1 = tmp1 - in3 + psubd m0, m5 ; in0 -= in1 + paddd m4, m2 ; in3 = tmp0 + in2 + ; m0 = out0, m1 = in1, m2 = out2, m3 = in3 + ; m4 = out3, m5 = out1 +%endmacro + +INIT_XMM sse2 +cglobal inv_txfm_add_wht_wht_4x4_16bpc, 3, 3, 6, dst, stride, c, eob, bdmax + mova m0, [cq+16*0] + mova m1, [cq+16*1] + mova m2, [cq+16*2] + mova m3, [cq+16*3] + REPX {psrad x, 2}, m0, m1, m2, m3 + IWHT4_1D + punpckldq m1, m0, m5 + punpckhdq m3, m0, m5 + punpckldq m5, m2, m4 + punpckhdq m2, m4 + punpcklqdq m0, m1, m5 + punpckhqdq m1, m5 + punpcklqdq m4, m3, m2 + punpckhqdq m3, m2 + mova m2, m4 + IWHT4_1D + packssdw m0, m4 ; low: out3, high: out0 + packssdw m2, m5 ; low: out2, high: out1 + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + lea r2, [dstq+strideq*2] + movq m1, [dstq+strideq*0] + movhps m1, [r2 +strideq*1] + movq m3, [r2 +strideq*0] + movhps m3, [dstq+strideq*1] + movd m5, bdmaxm + pshuflw m5, m5, q0000 ; broadcast + punpcklqdq m5, m5 ; broadcast + paddsw m0, m1 + paddsw m2, m3 + pmaxsw m0, m4 + pmaxsw m2, m4 + pminsw m0, m5 + pminsw m2, m5 + movhps [r2 +strideq*1], m0 ; write out0 + movhps [dstq+strideq*1], m2 ; write out1 + movq [r2 +strideq*0], m2 ; write out2 + movq [dstq+strideq*0], m0 ; write out3 + RET + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +; flags: 2 = inv_dst1, 4 = inv_dst2 +; skip round/shift if rnd is not a number +%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], flags +; %1 dst/src[1] +; %2 dst/src[2] +; %3 tmp[1] +; %4 tmp[2] +; %5 tmp[3] +; %6 rnd +; %7 coef[1] +; %8 coef[2] +; %9 flags +%ifnidn %7,%8 ; optimize when coef1 == coef2 +%if %8 < 32 + pmulld m%4, m%1, m%8 + pmulld m%3, m%2, m%8 +%else + mova m%3, [o(pd_%8)] + pmulld m%4, m%1, m%3 + pmulld m%3, m%2 +%endif +%endif +%if %7 < 32 + pmulld m%1, m%7 + pmulld m%2, m%7 +%else + mova m%5, [o(pd_%7)] + pmulld m%1, m%5 + pmulld m%2, m%5 +%endif +%if %9 & 4 ; invert dst2 + paddd m%4, m%2 + psubd m%2, m%6, m%4 +%else +%ifnum %6 +%ifnidn %7,%8 + paddd m%4, m%6 +%else + paddd m%1, m%6 +%endif +%endif +%ifnidn %7,%8 + paddd m%2, m%4 +%else + mova m%3, m%2 + paddd m%2, m%1 +%endif +%endif +%if %9 & 2 ; invert dst1 + psubd m%3, m%1 + paddd m%1, m%3, m%6 +%else +%ifnum %6 +%ifnidn %7,%8 + paddd m%1, m%6 +%endif +%endif + psubd m%1, m%3 +%endif +%ifnum %6 + psrad m%2, 12 + psrad m%1, 12 +%endif +%endmacro + +%macro INV_TXFM_FN 4-5+ 8 ; type1, type2, eob_offset, size, mmsize/stack +cglobal inv_txfm_add_%1_%2_%4_16bpc, 4, 7, %5, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%4_internal_16bpc) +%if ARCH_X86_32 + LEA r6, $$ +%endif +%if has_epilogue +%ifidn %1_%2, dct_dct + test eobd, eobd + jz %%end +%endif + lea tx2q, [o(m(i%2_%4_internal_16bpc).pass2)] +%ifnum %3 +%if %3 + add eobd, %3 +%endif +%else + lea r5, [o(%3)] +%endif + call %%p1 + RET +%%end: +%else + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [o(m(i%2_%4_internal_16bpc).pass2)] +%ifnum %3 +%if %3 + add eobd, %3 +%endif +%else + lea r5, [o(%3)] +%endif +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 0, 4x4 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 4 +.dconly: + add r5d, 128 + sar r5d, 8 +.dconly2: + imul r5d, 2896 + mova m2, [o(pixel_10bpc_max)] + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + pxor m3, m3 + punpcklqdq m0, m0 +.dconly_loop: + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + paddw m1, m0 + pminsw m1, m2 + pmaxsw m1, m3 + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%endif +%endmacro + +%macro IDCT4_1D 8 ; src[1-4], tmp[1-3], rnd + ; butterfly rotation + ITX_MULSUB_2D %1, %3, %5, %6, %7, %8, 2896, 2896 ; %1 out1 %3 out0 + ITX_MULSUB_2D %2, %4, %5, %6, %7, %8, 1567, 3784 ; %2 out2 %4 out3 + ; Hadamard rotation + psubd m%5, m%1, m%2 + paddd m%2, m%1 + paddd m%1, m%3, m%4 + psubd m%3, m%4 + ; %1 (src1) = out0 + ; %2 (src2) = out1 + ; %3 (src3) = out3 + ; $5 (tmp1) = out2 +%endmacro + +INIT_XMM sse4 + +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, identity +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst + +cglobal idct_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + mova m2, [cq+16*2] + mova m3, [cq+16*3] + mova m5, [o(pd_2048)] + call .pass1_main + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + ; transpose + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass1_main: + IDCT4_1D 0, 1, 2, 3, 4, 6, 7, 5 + ret +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + ; m5 = pd_2048 + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + pmaddwd m4, m2, [o(pw_m3784_1567)] + pmaddwd m2, [o(pw_1567_3784)] + pmaddwd m0, m1, [o(pw_m2896_2896)] + pmaddwd m1, [o(pw_2896_2896)] + REPX {paddd x, m5}, m4, m2, m0, m1 + packssdw m5, m5 ; pw_2048 + REPX {psrad x, 12}, m4, m2, m0, m1 + packssdw m2, m4 ; t3 t2 + packssdw m1, m0 ; t0 t1 + paddsw m0, m1, m2 ; out0 out1 + psubsw m1, m2 ; out3 out2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + movq m3, [r5 +strideq*1] + movhps m3, [r5 +strideq*0] + mova m5, [o(pixel_10bpc_max)] + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movhps [r5 +strideq*0], m1 + movq [r5 +strideq*1], m1 + RET + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +cglobal iadst_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call .main + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + ; transpose + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x4_internal_8bpc, _ssse3).main +.end: + mova m4, [o(pw_2048)] + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + movq m3, [r5 +strideq*0] + movhps m3, [r5 +strideq*1] + mova m5, [o(pixel_10bpc_max)] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [r5 +strideq*0], m1 + movhps [r5 +strideq*1], m1 + RET +ALIGN function_align +.main: + mova m1, [cq+16*2] + mova m3, [cq+16*3] + mova m5, [cq+16*0] + lea r3, [cq+16*1] +.main2: + mova m0, [o(pd_1321)] ; SINPI_1_9 + mova m2, [o(pd_2482)] ; SINPI_2_9 + mova m6, [o(pd_3803)] ; SINPI_4_9 + pmulld m4, m0, m1 ; s[4] = SINPI_1_9 * T[2] + pmulld m7, m3, m6 ; s[6] = SINPI_4_9 * T[3] + pmulld m6, m1 ; s[3] = SINPI_4_9 * T[2] + pmulld m0, m5 ; s[0] = SINPI_1_9 * T[0] + psubd m1, m3 ; T[2] - T[3] + pmulld m3, m2 ; s[5] = SINPI_2_9 * T[3] + pmulld m2, m5 ; s[1] = SINPI_2_9 * T[0] + paddd m0, m6 ; s[0] += s[3] + paddd m0, m3 ; s[0] += s[5] + mova m3, [o(pd_m3344)] ; -SINPI_3_9 + psubd m2, m4 ; s[1] -= s[4] + psubd m2, m7 ; s[1] -= s[6] + psubd m1, m5 ; -b7 = (T[2] -T[3]) - T[0] + pmulld m1, m3 ; s[2] = -SINPI_3_9 * -b7 + pmulld m3, [r3] ; -s[3] = -SINPI_3_9 * T[1] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m1 ; {s[0], s[2]} + 2048 + paddd m4, m0, m2 ; x[3] = s[0] + s[1] + psubd m2, m3 ; x[1] = s[1] + s[3] + psubd m0, m3 ; x[0] = s[0] + s[3] + paddd m4, m3 ; x[3] -= s[3] + paddd m2, m5 ; x[1] + 2048 + REPX {psrad x, 12}, m0, m2, m1, m4 + ret + + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call m(iadst_4x4_internal_16bpc).main + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + ; transpose + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x4_internal_8bpc, _ssse3).main + mova m4, [o(pw_2048)] + movq m3, [dstq+strideq*1] + movhps m3, [dstq+strideq*0] + lea r5, [dstq+strideq*2] + movq m2, [r5 +strideq*1] + movhps m2, [r5 +strideq*0] + mova m5, [o(pixel_10bpc_max)] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movhps [dstq+strideq*0], m1 + movq [dstq+strideq*1], m1 + movhps [r5 +strideq*0], m0 + movq [r5 +strideq*1], m0 + RET + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m3, [o(pd_5793)] + pmulld m0, m3, [cq+16*0] + pmulld m1, m3, [cq+16*1] + pmulld m2, m3, [cq+16*2] + pmulld m3, [cq+16*3] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + ; transpose + punpckhwd m3, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + ; m5 = pd_2048 + mova m4, [o(pw_1697x8)] + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + pmulhrsw m3, m4, m0 + pmulhrsw m4, m1 + paddsw m0, m3 + paddsw m1, m4 + movq m3, [r5 +strideq*0] + movhps m3, [r5 +strideq*1] + mova m4, [o(pixel_10bpc_max)] + packssdw m5, m5 ; pw_2048 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + pxor m5, m5 + mova [cq+16*0], m5 + mova [cq+16*1], m5 + mova [cq+16*2], m5 + mova [cq+16*3], m5 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m5 + pmaxsw m1, m5 + pminsw m0, m4 + pminsw m1, m4 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [r5 +strideq*0], m1 + movhps [r5 +strideq*1], m1 + RET + +%macro INV_TXFM_4X8_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 4x8 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 8 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + jmp m(inv_txfm_add_dct_dct_4x4_16bpc).dconly +%endif +%endmacro + +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, identity, 9 +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst + +cglobal idct_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp + mova m5, [o(pd_2048)] +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 13 + setge r5b +%else + mov r5d, 1 + cmp eobd, 13 + sbb r5d, 0 +%endif + shl r5d, 4 +.loop_pass1: + mova m3, [o(pd_2896)] + pmulld m0, m3, [cq+32*0+r5] + pmulld m1, m3, [cq+32*1+r5] + pmulld m2, m3, [cq+32*2+r5] + pmulld m3, [cq+32*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + call m(idct_4x4_internal_16bpc).pass1_main + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*0+16], m0 + mova [cq+32*1+16], m4 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + mova m2, [cq+32*0+16] + mova m6, [cq+32*1+16] + punpckhwd m4, m2, m6 + punpcklwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_4x8_internal_8bpc, _ssse3).main + ; m0-3 is now out0/1,3/2,4/5,7/6 + mova m4, [o(pw_2048)] + shufps m1, m1, q1032 + shufps m3, m3, q1032 +.end: + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + pxor m4, m4 + REPX {mova [cq+16*x], m4}, 0, 1, 2, 3, 4, 5, 6, 7 + mova m7, [o(pixel_10bpc_max)] + lea r2, [strideq*3] + movq m5, [dstq+strideq*0] + movq m6, [dstq+strideq*2] + movhps m5, [dstq+strideq*1] + movhps m6, [dstq+r2] + lea r4, [dstq+strideq*4] + paddw m0, m5 + paddw m1, m6 + movq m5, [r4+strideq*0] + movq m6, [r4+strideq*2] + movhps m5, [r4+strideq*1] + movhps m6, [r4+r2] + paddw m2, m5 + paddw m3, m6 + REPX {pminsw x, m7}, m0, m1, m2, m3 + REPX {pmaxsw x, m4}, m0, m1, m2, m3 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r2 ], m1 + movq [r4 +strideq*0], m2 + movhps [r4 +strideq*1], m2 + movq [r4 +strideq*2], m3 + movhps [r4 +r2 ], m3 + RET + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity, 9 + +cglobal iadst_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call .pass1_main + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + mova m2, [cq+32*2+16] + mova m6, [cq+32*3+16] + punpckhwd m4, m2, m6 + punpcklwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass1_main: +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 13 + setge r5b +%else + mov r5d, 1 + cmp eobd, 13 + sbb r5d, 0 +%endif + shl r5d, 4 + lea r3, [cq+32*1+16] +.loop_pass1: + mova m0, [o(pd_2048)] + mova m3, [o(pd_2896)] + pmulld m5, m3, [cq+32*0+r5] + pmulld m2, m3, [cq+32*1+r5] + pmulld m1, m3, [cq+32*2+r5] + pmulld m3, [cq+32*3+r5] + REPX {paddd x, m0}, m5, m2, m1, m3 + REPX {psrad x, 12}, m5, m2, m1, m3 + mova [r3], m2 + call m(iadst_4x4_internal_16bpc).main2 + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*2+16], m0 + mova [cq+32*3+16], m1 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + ret +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x8_internal_8bpc, _ssse3).main + mova m4, [o(pw_4x2048_4xm2048)] + jmp m(idct_4x8_internal_16bpc).end + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity, 9 + +cglobal iflipadst_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call m(iadst_4x8_internal_16bpc).pass1_main + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + mova m6, [cq+32*2+16] + mova m2, [cq+32*3+16] + punpcklwd m4, m2, m6 + punpckhwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x8_internal_8bpc, _ssse3).main + mova m4, m0 + mova m5, m1 + pshufd m0, m3, q1032 + pshufd m1, m2, q1032 + pshufd m2, m5, q1032 + pshufd m3, m4, q1032 + mova m4, [o(pw_4xm2048_4x2048)] + jmp m(idct_4x8_internal_16bpc).end + +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity, 3 + +cglobal iidentity_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp + mova m5, [o(pd_2048)] + mova m4, [o(pd_2896)] + mova m6, [o(pd_5793)] + ; clear m7 in case we skip the bottom square + pxor m7, m7 +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 16 + setge r5b +%else + mov r5d, 1 + cmp eobd, 16 + sbb r5d, 0 +%endif + shl r5d, 4 +.loop_pass1: + pmulld m0, m4, [cq+32*0+r5] + pmulld m1, m4, [cq+32*1+r5] + pmulld m2, m4, [cq+32*2+r5] + pmulld m3, m4, [cq+32*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + REPX {pmulld x, m6}, m0, m1, m2, m3 + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*0+16], m0 + mova m7, m2 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + punpckhwd m4, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m4 + punpcklwd m0, m4 + mova m2, [cq+32*0+16] + punpckhwd m4, m2, m7 + punpcklwd m2, m7 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: + mova m4, [o(pw_4096)] + jmp m(idct_4x8_internal_16bpc).end + +%macro INV_TXFM_4X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix + INV_TXFM_FN %1, %2, tbl_4x16_%3, 4x16 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 16 + add r5d, 384 + sar r5d, 9 + jmp m(inv_txfm_add_dct_dct_4x4_16bpc).dconly2 +%endif +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, identity, v +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst + +cglobal idct_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif + mova m5, [o(pd_2048)] +.loop_pass1: + mova m0, [cq+64*0+r5] + mova m1, [cq+64*1+r5] + mova m2, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(idct_4x4_internal_16bpc).pass1_main + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m1, m4, m2 + REPX {psrad x, 1}, m0, m1, m4, m2 + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + test r5d, r5d + jz .end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.end_pass1: + mova m2, [cq+64*0+16] + mova m3, [cq+64*1+16] + mova m4, [cq+64*0+32] + mova m5, [cq+64*1+32] + mova m6, [cq+64*0+48] + mova m7, [cq+64*1+48] + ; m0-7 = packed & transposed output + jmp tx2q +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_16x4_internal_8bpc, _ssse3).main + ; m0-6 is out0-13 [with odd registers having inversed output] + ; [coeffq+16*7] has out15/14 + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [cq+16*7] + REPX {shufps x, x, q1032}, m1, m3, m5, m7 + mova [cq+16*0], m4 + mova [cq+16*1], m5 + mova [cq+16*2], m6 + mova [cq+16*3], m7 +.end: + pxor m4, m4 + REPX {mova [cq+16*x], m4}, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + mova m7, [o(pixel_10bpc_max)] + mov r5d, 2 + lea r3, [strideq*3] +.loop: + movq m5, [dstq+strideq*0] + movq m6, [dstq+strideq*2] + movhps m5, [dstq+strideq*1] + movhps m6, [dstq+r3] + lea r4, [dstq+strideq*4] + paddw m0, m5 + paddw m1, m6 + movq m5, [r4+strideq*0] + movq m6, [r4+strideq*2] + movhps m5, [r4+strideq*1] + movhps m6, [r4+r3] + paddw m2, m5 + paddw m3, m6 + REPX {pminsw x, m7}, m0, m1, m2, m3 + REPX {pmaxsw x, m4}, m0, m1, m2, m3 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r3 ], m1 + movq [r4 +strideq*0], m2 + movhps [r4 +strideq*1], m2 + movq [r4 +strideq*2], m3 + movhps [r4 +r3 ], m3 + dec r5d + jz .end2 + lea dstq, [dstq+strideq*8] + mova m0, [cq+0*16] + mova m1, [cq+1*16] + mova m2, [cq+2*16] + mova m3, [cq+3*16] + REPX {mova [cq+x*16], m4}, 0, 1, 2, 3 + jmp .loop +.end2: + RET + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity, v + +cglobal iadst_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r6+r5] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif +.loop_pass1: + mova m5, [cq+64*0+r5] + lea r3, [cq+64*1+r5] + mova m1, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(iadst_4x4_internal_16bpc).main2 + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m2, m1, m4 + REPX {psrad x, 1}, m0, m2, m1, m4 + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main_pass2_end + ; m7/5/2/4 = out4/-11,-5/10,6/-9,-7/8 + ; m0/3 & cq6/7 = out0/-15,-3/12,-1/14,2/-13 + mova m1, [o(pw_4x2048_4xm2048)] + REPX {pmulhrsw x, m1}, m7, m2, m0 + pshufd m6, m1, q1032 ; 4x-2048,4x2048 + pmulhrsw m1, [cq+16*7] + REPX {pmulhrsw x, m6}, m5, m4, m3 + pmulhrsw m6, [cq+16*6] + ; m7/5/2/4 = out4/11,5/10,6/9,7/8 + ; m0/3/6/1 = out0/15,3/12,1/14,2/13 + ; output should be as 0-3 for out0-7, and cq+0-3*16 for out8-15 + movhps [cq+0*8], m4 + movhps [cq+1*8], m2 + movhps [cq+2*8], m5 + movhps [cq+3*8], m7 + movhps [cq+4*8], m3 + movhps [cq+5*8], m1 + movhps [cq+6*8], m6 + movhps [cq+7*8], m0 + punpcklqdq m0, m6 + punpcklqdq m1, m3 + punpcklqdq m3, m2, m4 + punpcklqdq m2, m7, m5 + jmp m(idct_4x16_internal_16bpc).end + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity, v + +cglobal iflipadst_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif +.loop_pass1: + mova m5, [cq+64*0+r5] + lea r3, [cq+64*1+r5] + mova m1, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(iadst_4x4_internal_16bpc).main2 + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m2, m1, m4 + REPX {psrad x, 1}, m0, m2, m1, m4 + packssdw m0, m2 ; out3 out2 + packssdw m1, m4 ; out1 out0 + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main_pass2_end + ; m7/5/2/4 = out11/-4,-10/5,9/-6,-8/7 + ; m0/3 & cq6/7 = out15/-0,-12/3,-14/1,13/-2 + mova m1, [o(pw_4x2048_4xm2048)] + REPX {pmulhrsw x, m1}, m7, m2, m0 + pshufd m6, m1, q1032 ; 4x-2048,4x2048 + pmulhrsw m1, [cq+16*7] + REPX {pmulhrsw x, m6}, m5, m4, m3 + pmulhrsw m6, [cq+16*6] + ; m7/5/2/4 = out11/4,10/5,9/6,8/7 + ; m0/3/6/1 = out15/0,12/3,14/1,13/2 + ; output should be as 0-3 for out0-7, and cq+0-3*16 for out8-15 + movq [cq+0*8], m4 + movq [cq+1*8], m2 + movq [cq+2*8], m5 + movq [cq+3*8], m7 + movq [cq+4*8], m3 + movq [cq+5*8], m1 + movq [cq+6*8], m6 + movq [cq+7*8], m0 + punpckhqdq m0, m6 + punpckhqdq m1, m3 + punpckhqdq m3, m2, m4 + punpckhqdq m2, m7, m5 + jmp m(idct_4x16_internal_16bpc).end + +INV_TXFM_4X16_FN identity, dct, h +INV_TXFM_4X16_FN identity, adst, h +INV_TXFM_4X16_FN identity, flipadst, h +INV_TXFM_4X16_FN identity, identity + +cglobal iidentity_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif + mova m5, [o(pd_6144)] + mova m4, [o(pd_5793)] +.loop_pass1: + pmulld m0, m4, [cq+64*0+r5] + pmulld m1, m4, [cq+64*1+r5] + pmulld m2, m4, [cq+64*2+r5] + pmulld m3, m4, [cq+64*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 13}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + punpckhwd m3, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: + mova [cq+16*4], m0 + mova [cq+16*5], m1 + mova [cq+16*6], m2 + mova [cq+16*7], m7 + mova m0, [o(pw_1697x16)] + mova m7, [o(pw_2048)] + pmulhrsw m1, m0, m4 + pmulhrsw m2, m0, m5 + REPX {paddsw x, x}, m4, m5 + paddsw m4, m1 + paddsw m5, m2 + REPX {pmulhrsw x, m7}, m4, m5 + mova [cq+16*0], m4 + mova [cq+16*1], m5 + mova m4, [cq+16*7] + pmulhrsw m1, m0, m6 + pmulhrsw m2, m0, m4 + REPX {paddsw x, x}, m6, m4 + paddsw m6, m1 + paddsw m4, m2 + REPX {pmulhrsw x, m7}, m6, m4 + mova [cq+16*2], m6 + mova [cq+16*3], m4 + mova m4, [cq+16*4] + mova m1, [cq+16*5] + mova m2, [cq+16*6] + pmulhrsw m5, m0, m2 + pmulhrsw m6, m0, m3 + REPX {paddsw x, x}, m2, m3 + paddsw m2, m5 + paddsw m3, m6 + pmulhrsw m6, m0, m1 + pmulhrsw m0, m4 + REPX {paddsw x, x}, m1, m4 + paddsw m1, m6 + paddsw m0, m4 + REPX {pmulhrsw x, m7}, m2, m3, m1, m0 + jmp m(idct_4x16_internal_16bpc).end + +%macro INV_TXFM_8X4_FN 2 ; type1, type2 +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, 0, 8x4, 15 +%else + INV_TXFM_FN %1, %2, 0, 8x4, 8, 0-4*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + add r5d, 128 + sar r5d, 8 + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 + lea r2, [strideq*3] + mova m1, [dstq+strideq*0] + mova m2, [dstq+strideq*1] + mova m3, [dstq+strideq*2] + mova m4, [dstq+r2] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+strideq*0], m1 + mova [dstq+strideq*1], m2 + mova [dstq+strideq*2], m3 + mova [dstq+r2 ], m4 + RET +%endif +%endmacro + +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, identity +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst + +cglobal idct_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] +.pass1_entry: +%if ARCH_X86_32 + lea r3, [rsp+gprsize] +%else + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+0*16] + mova m1, [cq+1*16] + mova m2, [cq+2*16] + mova m3, [cq+3*16] + mova m4, [cq+4*16] + mova m5, [cq+5*16] + mova m6, [cq+6*16] + mova m7, [cq+7*16] + call .rect2_mul + call r5 + call .transpose4x8packed + ; m0-3 = packed & transposed output + jmp tx2q +.transpose4x8packed: + ; transpose + punpcklwd m1, m2, m6 + punpckhwd m2, m6 + punpckhwd m6, m0, m4 + punpcklwd m0, m4 + + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m4, m6, m2 + punpcklwd m6, m2 + + punpcklwd m2, m3, m4 + punpckhwd m3, m4 + punpckhwd m1, m0, m6 + punpcklwd m0, m6 + ret +.main: + call .main_pass1 + call .round + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.rect2_mul: +%if ARCH_X86_64 + REPX {pmulld x, m14}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 +%else + mova [r3], m7 + mova m7, [o(pd_2896)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_2048)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] +%endif + REPX {psrad x, 12}, m0, m1, m2, m3, m4, m5, m6, m7 + ret +%if ARCH_X86_64 +.main_pass1_fast: + pmulld m5, m3, [o(pd_m2276)] + pmulld m3, [o(pd_3406)] + pmulld m7, m1, [o(pd_4017)] + pmulld m1, [o(pd_799)] + pmulld m6, m2, [o(pd_3784)] + pmulld m2, [o(pd_1567)] + pmulld m0, m14 + pxor m4, m4 + jmp .main_pass1_fast2 +.main_pass1: + ITX_MULSUB_2D 5, 3, 8, 9, 10, _, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 8, 9, 10, _, 799, 4017 ; t4a t7a + ITX_MULSUB_2D 2, 6, 8, 9, 10, _, 1567, 3784 ; t2 t3 + REPX {pmulld x, m14}, m0, m4 +.main_pass1_fast2: + REPX {paddd x, m11}, m1, m2, m3, m5, m6, m7 + REPX {psrad x, 12 }, m1, m2, m3, m5, m6, m7 + paddd m8, m1, m5 ; t4 + psubd m1, m5 ; t5a + paddd m9, m7, m3 ; t7 + psubd m7, m3 ; t6a + REPX {pmaxsd x, m12}, m1, m8, m7, m9 + REPX {pminsd x, m13}, m1, m8, m7, m9 + REPX {pmulld x, m14}, m7, m1 + paddd m0, m11 + paddd m7, m11 + psubd m5, m0, m4 + paddd m0, m4 + psubd m4, m7, m1 + paddd m7, m1 + REPX {psrad x, 12 }, m5, m0, m4, m7 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + paddd m6, m5, m2 ; dct4 out1 + psubd m5, m2 ; dct4 out2 + REPX {pmaxsd x, m12}, m0, m6, m5, m3 + REPX {pminsd x, m13}, m0, m6, m5, m3 + ret +.round: + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + psubd m7, m0, m9 ; out7 + paddd m0, m9 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 +%else +.main_pass1_fast: + pmulld m5, m3, [o(pd_m2276)] + pmulld m3, [o(pd_3406)] + pmulld m7, m1, [o(pd_4017)] + pmulld m1, [o(pd_799)] + pmulld m6, m2, [o(pd_3784)] + pmulld m2, [o(pd_1567)] + mova m4, [o(pd_2048)] + mova [r3+0*16], m2 + REPX {paddd x, m4}, m5, m3, m7, m1 + REPX {psrad x, 12}, m5, m3, m7, m1 + paddd m2, m1, m5 ; t4 + psubd m1, m5 ; t5a + pmulld m5, m0, [o(pd_2896)] + mova m0, m4 + paddd m4, m7, m3 ; t7 + psubd m7, m3 ; t6a + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3 }, m1, m2, m7, m4 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3 }, m1, m2, m7, m4 + mova [r3+3*16], m2 + mova [r3+1*16], m4 + pxor m4, m4 + mova m2, [r3+0*16] + mova m3, [o(pd_2896)] + jmp .main_pass1_fast2 +.main_pass1: + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m4 + mova [r3+3*16], m6 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 5, 3, 2, 4, 6, 0, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 2, 4, 6, 0, 799, 4017 ; t4a t7a + paddd m2, m1, m5 ; t4 + psubd m1, m5 ; t5a + paddd m4, m7, m3 ; t7 + psubd m7, m3 ; t6a + mova m6, [o(clip_18b_min)] + REPX {pmaxsd x, m6 }, m1, m2, m7, m4 + mova m6, [o(clip_18b_max)] + REPX {pminsd x, m6 }, m1, m2, m7, m4 + mova m6, [r3+3*16] + mova [r3+3*16], m2 + mova m2, [r3+1*16] + mova [r3+1*16], m4 + + ITX_MULSUB_2D 2, 6, 4, 3, 5, _, 1567, 3784 ; t2 t3 + mova m3, [o(pd_2896)] + mova m5, [r3+0*16] + mova m4, [r3+2*16] + REPX {pmulld x, m3 }, m5, m4 +.main_pass1_fast2: + REPX {paddd x, m0 }, m2, m6 + REPX {psrad x, 12 }, m2, m6 + REPX {pmulld x, m3 }, m7, m1 + paddd m7, m0 + paddd m0, m5 + + psubd m5, m0, m4 + paddd m0, m4 + psubd m4, m7, m1 + paddd m7, m1 + REPX {psrad x, 12 }, m5, m0, m4, m7 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + paddd m6, m5, m2 ; dct4 out1 + psubd m5, m2 ; dct4 out2 + + mova m1, [o(clip_18b_min)] + REPX {pmaxsd x, m1 }, m0, m6, m5, m3 + mova m1, [o(clip_18b_max)] + REPX {pminsd x, m1 }, m0, m6, m5, m3 + ret +.round: + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + mova [r3+0*16], m6 + mova m6, [r3+1*16] + psubd m7, m0, m6 ; out7 + paddd m0, m6 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + mova m6, [r3+3*16] + psubd m4, m3, m6 ; out4 + paddd m3, m6 ; out3 + mova m6, [r3+0*16] +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x4_internal_8bpc, _ssse3).main +.end: + lea r3, [strideq*3] + call .round2_and_write_8x4 + REPX {mova [cq+16*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + RET +.round2_and_write_8x4: + pxor m6, m6 + mova m5, [o(pixel_10bpc_max)] + mova m4, [o(pw_2048)] +.round1_and_write_8x4: + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 +.write_8x4: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3] + REPX {pminsw x, m5}, m0, m1, m2, m3 + REPX {pmaxsw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + ret + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + call .main_pass1 + call .round + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.main_pass1: +%if ARCH_X86_64 + ITX_MULSUB_2D 7, 0, 8, 9, 10, 11, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D 1, 6, 8, 9, 10, 11, 3920, 1189 ; t7a, t6a + ITX_MULSUB_2D 5, 2, 8, 9, 10, 11, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D 3, 4, 8, 9, 10, 11, 3166, 2598 ; t5a, t4a + psubd m8, m2, m6 ; t6 + paddd m2, m6 ; t2 + psubd m6, m0, m4 ; t4 + paddd m0, m4 ; t0 + psubd m4, m5, m1 ; t7 + paddd m5, m1 ; t3 + psubd m1, m7, m3 ; t5 + paddd m7, m3 ; t1 + REPX {pmaxsd x, m12}, m6, m1, m8, m4, m2, m0, m5, m7 + REPX {pminsd x, m13}, m6, m1, m8, m4, m2, m0, m5, m7 + ITX_MULSUB_2D 6, 1, 3, 9, 10, 11, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2D 4, 8, 3, 9, 10, 11, 3784, 10 ; t6a, t7a + psubd m9, m6, m8 ; t7 + paddd m6, m8 ; out6 + mova m8, [o(pd_2896)] + psubd m3, m7, m5 ; t3 + paddd m7, m5 ; -out7 + psubd m5, m0, m2 ; t2 + paddd m0, m2 ; out0 + psubd m2, m1, m4 ; t6 + paddd m1, m4 ; -out1 + REPX {pmaxsd x, m12}, m5, m3, m2, m9 + REPX {pminsd x, m13}, m5, m3, m2, m9 + REPX {pmulld x, m14}, m5, m3, m2, m9 + psubd m4, m5, m3 ; (t2 - t3) * 2896 + paddd m3, m5 ; (t2 + t3) * 2896 + psubd m5, m2, m9 ; (t6 - t7) * 2896 + paddd m2, m9 ; (t6 + t7) * 2896 + ret +.round: + + ; m0=out0,m1=-out1,m6=out6,m7=-out7 + + pcmpeqd m8, m8 + REPX {pxor x, m8 }, m1, m7, m3, m5 + REPX {psubd x, m8 }, m1, m7 + REPX {paddd x, m11}, m2, m3, m4, m5 + REPX {psrad x, 12 }, m2, m3, m4, m5 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m5, [o(pd_2048)] + + ITX_MULSUB_2D 7, 0, 2, 3, 4, 5, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D 1, 6, 2, 3, 4, 5, 3920, 1189 ; t7a, t6a + mova m2, [r3+0*16] + mova m3, [r3+1*16] + mova m4, [r3+2*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m6 + mova m1, [r3+3*16] + mova [r3+3*16], m7 + ITX_MULSUB_2D 1, 2, 0, 6, 7, 5, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D 3, 4, 0, 6, 7, 5, 3166, 2598 ; t5a, t4a + mova m0, [r3+0*16] + mova m6, [r3+2*16] + psubd m7, m2, m6 ; t6 + paddd m2, m6 ; t2 + psubd m6, m0, m4 ; t4 + paddd m0, m4 ; t0 + mova [r3+0*16], m7 + mova m5, [r3+1*16] + mova m7, [r3+3*16] + psubd m4, m1, m5 ; t7 + paddd m5, m1 ; t3 + psubd m1, m7, m3 ; t5 + paddd m7, m3 ; t1 + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3 }, m6, m1, m4, m2, m0, m5, m7 + mova [r3+1*16], m7 + mova m7, [o(clip_18b_max)] + pmaxsd m3, [r3+0*16] + REPX {pminsd x, m7 }, m6, m1, m3, m4, m2, m0, m5 + pminsd m7, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 6, 1, 2, 5, 7, 0, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2D 4, 3, 2, 5, 7, 0, 3784, 7 ; t6a, t7a + mova m5, [r3+2*16] + mova m7, [r3+3*16] + psubd m2, m6, m3 ; t7 + paddd m6, m3 ; out6 + mova [r3+3*16], m6 + mova m0, [r3+0*16] + mova m6, [r3+1*16] + psubd m3, m7, m5 ; t3 + paddd m7, m5 ; -out7 + psubd m5, m0, m6 ; t2 + paddd m0, m6 ; out0 + psubd m6, m1, m4 ; t6 + paddd m1, m4 ; -out1 + mova m4, [o(clip_18b_min)] + REPX {pmaxsd x, m4 }, m5, m3, m6, m2 + mova m4, [o(clip_18b_max)] + REPX {pminsd x, m4 }, m5, m3, m6, m2 + mova m4, [o(pd_2896)] + REPX {pmulld x, m4 }, m5, m3, m6, m2 + psubd m4, m5, m3 ; (t2 - t3) * 2896 + paddd m3, m5 ; (t2 + t3) * 2896 + psubd m5, m6, m2 ; (t6 - t7) * 2896 + paddd m2, m6 ; (t6 + t7) * 2896 + ret +.round: + mova [r3+2*16], m0 + + pcmpeqd m0, m0 + mova m6, [o(pd_2048)] + REPX {pxor x, m0 }, m1, m7, m3, m5 + REPX {psubd x, m0 }, m1, m7 + REPX {paddd x, m6 }, m2, m3, m4, m5 + REPX {psrad x, 12 }, m2, m3, m4, m5 + + mova m6, [r3+3*16] + mova m0, [r3+2*16] +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x4_internal_8bpc, _ssse3).main + jmp m(idct_8x4_internal_16bpc).end + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call m(iadst_8x4_internal_16bpc).round + packssdw m7, m6 + packssdw m5, m4 + packssdw m3, m2 + packssdw m1, m0 + mova m0, m7 + mova m2, m5 + mova m4, m3 + mova m6, m1 + ret +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x4_internal_8bpc, _ssse3).main + lea r3, [strideq*3] + add dstq, r3 + neg strideq + jmp m(idct_8x4_internal_16bpc).end + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + REPX {paddd x, x}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.pass2: + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(idct_8x4_internal_16bpc).end + +%macro INV_TXFM_8X8_FN 2-3 0 ; type1, type2, eob_offset +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, %3, 8x8, 15, 0-3*16 +%else + INV_TXFM_FN %1, %2, %3, 8x8, 8, 0-5*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 2 +.end: + add r5d, 384 + sar r5d, 9 +.end2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 + lea r2, [strideq*3] +.loop: + mova m1, [dstq+strideq*0] + mova m2, [dstq+strideq*1] + mova m3, [dstq+strideq*2] + mova m4, [dstq+r2] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+strideq*0], m1 + mova [dstq+strideq*1], m2 + mova [dstq+strideq*2], m3 + mova [dstq+r2 ], m4 + lea dstq, [dstq+strideq*4] + dec r3d + jg .loop + RET +%endif +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, identity, 6 +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst + +cglobal idct_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + DECLARE_REG_TMP 1 + mov [rsp+4*16+1*gprsize], r1 +%else + DECLARE_REG_TMP 6 +%endif + lea t0, [o(.pass1_main)] + +.pass1_full: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + shl r5d, 4 +%if ARCH_X86_32 + lea r3, [rsp+gprsize] +%endif +.loop_pass1: + mova m0, [cq+0*32+r5] + mova m1, [cq+1*32+r5] + mova m2, [cq+2*32+r5] + mova m3, [cq+3*32+r5] + mova m4, [cq+4*32+r5] + mova m5, [cq+5*32+r5] + mova m6, [cq+6*32+r5] + mova m7, [cq+7*32+r5] + call t0 + + test r5d, r5d + jz .end_pass1 + + mova [cq+0*32+16], m0 + mova [cq+1*32+16], m1 + mova [cq+2*32+16], m2 + mova [cq+3*32+16], m3 + + sub r5d, 16 + jmp .loop_pass1 +.end_pass1: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + mov r1, [rsp+4*16+1*gprsize] +%endif + jmp tx2q +.pass1_main: + call m(idct_8x4_internal_16bpc).main_pass1 + pcmpeqd m1, m1 + REPX {psubd x, m1}, m0, m6, m5, m3 + call m(idct_8x4_internal_16bpc).round + REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 +.pack_and_transpose: + packssdw m2, m3 + packssdw m6, m7 + packssdw m0, m1 + packssdw m4, m5 + jmp m(idct_8x4_internal_16bpc).transpose4x8packed + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 +%endif + call .round3_and_write_8x8 +.zero: +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+16*x], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +%undef mzero + RET + + ; round (rounded right-shift by 5) before writing + ; data in m0-7 + ; on x86-64, pw_2048 is in m8 + ; .round1 is for m0-7 + ; .round2 is for m0-6 & [rsp+gprsize*2] + ; .round3 is same, but without using m8 on x86-64 (.round2/3 are identical on x86-32) + ; .round4 is x86-32-only, it is similar to .round2 but with constant already in m7 +%if ARCH_X86_32 +.round1_and_write_8x8: + mova [rsp+gprsize*2], m7 +.round2_and_write_8x8: +%endif +.round3_and_write_8x8: + mova m7, [o(pw_2048)] +%if ARCH_X86_32 +.round4_and_write_8x8: +%endif + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [rsp+gprsize*2] +%if ARCH_X86_64 + jmp .write_8x8 +.round2_and_write_8x8: + mova m7, [rsp+gprsize*2] +.round1_and_write_8x8: + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 +%endif + + ; m0-7 have to-be-written data [pre-rounded] + ; on x86-64, m9-10 contain a zero/pixel_max + ; on x86-32, these are runtime-generated, and [rsp+gprsize*2] is scratch + ; r0,1,3 contain dstq/strideq/stride3q + ; r5 is a scratch register +.write_8x8: + lea r5, [dstq+strideq*4] + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3] + paddw m4, [r5 +strideq*0] + paddw m5, [r5 +strideq*1] + paddw m6, [r5 +strideq*2] + paddw m7, [r5 +r3] +%if ARCH_X86_64 + REPX {pmaxsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 +%else + mova [rsp+gprsize*2], m7 + pxor m7, m7 + REPX {pmaxsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmaxsw m7, [rsp+gprsize*2] + mova [rsp+gprsize*2], m7 + mova m7, [o(pixel_10bpc_max)] + REPX {pminsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pminsw m7, [rsp+gprsize*2] +%endif + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + mova [r5 +strideq*0], m4 + mova [r5 +strideq*1], m5 + mova [r5 +strideq*2], m6 + mova [r5 +r3 ], m7 + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity, 6 + +cglobal iadst_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+4*16+1*gprsize], r1 +%endif + lea t0, [o(.pass1_main)] + jmp m(idct_8x8_internal_16bpc).pass1_full +.pass1_main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call .round + jmp m(idct_8x8_internal_16bpc).pack_and_transpose +.round: +%if ARCH_X86_64 + pcmpeqd m8, m8 ; -1 + REPX {psubd x, m8 }, m0, m6 + REPX {pxor x, m8 }, m1, m7, m3, m5 + REPX {psrad x, 1 }, m0, m1, m6, m7 + REPX {psubd x, m8 }, m1, m7 + mova m8, [o(pd_6144)] + REPX {paddd x, m8 }, m2, m3, m4, m5 + REPX {psrad x, 13 }, m2, m3, m4, m5 +%else + mova [r3+2*16], m0 + + pcmpeqd m0, m0 ; -1 + mova m6, [o(pd_6144)] + REPX {pxor x, m0 }, m1, m7, m3, m5 + REPX {psrad x, 1 }, m1, m7 + REPX {psubd x, m0 }, m1, m7 + REPX {paddd x, m6 }, m2, m3, m4, m5 + REPX {psrad x, 13 }, m2, m3, m4, m5 + + mova m0, [r3+2*16] + psrld m6, 12 ; +1 + paddd m0, m6 + paddd m6, [r3+3*16] + REPX {psrad x, 1 }, m0, m6 +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main_pass2_end + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 +%endif + call .round3_and_write_8x8 + jmp m(idct_8x8_internal_16bpc).zero + + ; round (rounded right-shift by 5) before writing; odd registers are negated + ; data in m0-7 + ; on x86-64, pw_2048 is in m8 and pw_m2048 is in m11 + ; .round1 is for m0-7 + ; .round2 is for m0-6 & [rsp+gprsize*2] + ; .round3 is same, but without using m8 on x86-64 (.round2/3 are identical on x86-32) +%if ARCH_X86_64 +.round2_and_write_8x8: + mova m7, [rsp+gprsize*2] +.round1_and_write_8x8: + REPX {pmulhrsw x, m8 }, m0, m2, m4, m6 + REPX {pmulhrsw x, m11}, m1, m3, m5, m7 + jmp m(idct_8x8_internal_16bpc).write_8x8 +%else +.round1_and_write_8x8: + mova [rsp+gprsize*2], m7 +.round2_and_write_8x8: +%endif +.round3_and_write_8x8: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova m7, [o(pw_m2048)] + REPX {pmulhrsw x, m7}, m1, m3, m5 + pmulhrsw m7, [rsp+gprsize*2] + jmp m(idct_8x8_internal_16bpc).write_8x8 + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity, 6 + +cglobal iflipadst_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+4*16+1*gprsize], r1 +%endif + lea t0, [o(.pass1_main)] + jmp m(idct_8x8_internal_16bpc).pass1_full +.pass1_main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call m(iadst_8x8_internal_16bpc).round + ; invert registers + packssdw m7, m6 + packssdw m5, m4 + packssdw m3, m2 + packssdw m1, m0 + mova m0, m7 + mova m2, m5 + mova m4, m3 + mova m6, m1 + jmp m(idct_8x4_internal_16bpc).transpose4x8packed + +.pass2: + lea dstq, [dstq+strideq*8] + sub dstq, strideq + neg strideq + jmp m(iadst_8x8_internal_16bpc).pass2 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m0, [cq+0*32] + mova m1, [cq+1*32] + mova m2, [cq+2*32] + mova m3, [cq+3*32] + mova m4, [cq+4*32] + mova m5, [cq+5*32] + mova m6, [cq+6*32] + mova m7, [cq+7*32] + packssdw m0, [cq+0*32+16] + packssdw m1, [cq+1*32+16] + packssdw m2, [cq+2*32+16] + packssdw m3, [cq+3*32+16] + packssdw m4, [cq+4*32+16] + packssdw m5, [cq+5*32+16] + packssdw m6, [cq+6*32+16] + packssdw m7, [cq+7*32+16] + mova [rsp+gprsize+16*1], m6 + jmp m_suffix(idct_8x8_internal_8bpc, _ssse3).pass1_end3 + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mova m8, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%else + mova [rsp+gprsize], m7 + mova m7, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round4_and_write_8x8 +%endif + jmp m(idct_8x8_internal_16bpc).zero + +%macro INV_TXFM_8X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, tbl_8x16_%3, 8x16, 15, 0-16*16 +%else + INV_TXFM_FN %1, %2, tbl_8x16_%3, 8x16, 8, 0-17*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + mov r3d, 4 +%if stack_size_padded > 0 + ; adjust to caller's stack allocation + add rsp, (12+ARCH_X86_64)*16 +%endif + jmp m(inv_txfm_add_dct_dct_8x8_16bpc).end +%endif +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, identity, v +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst + +%if ARCH_X86_64 +DECLARE_REG_TMP 7 +%endif + +cglobal idct_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(idct_8x8_internal_16bpc).pass1_main)] +.pass1_full: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif +%undef cmp + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, [rsp+16*16+2*gprsize] + ; setup stack pointer + lea r3, [rsp+gprsize] +%endif +.loop_pass1: + mova m0, [cq+0*64+r5] + mova m1, [cq+1*64+r5] + mova m2, [cq+2*64+r5] + mova m3, [cq+3*64+r5] + mova m4, [cq+4*64+r5] + mova m5, [cq+5*64+r5] + mova m6, [cq+6*64+r5] + mova m7, [cq+7*64+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call t0 + + mova [cq+0*64+r5], m0 + mova [cq+1*64+r5], m1 + mova [cq+2*64+r5], m2 + mova [cq+3*64+r5], m3 + sub r5d, 16 + jge .loop_pass1 +%if WIN64 + POP r7 +%elif ARCH_X86_32 + mov r1, [rsp+16*16+1*gprsize] +%endif + jmp tx2q + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + + ; input is in cqN*16, where N=0/4/8/12/1/5/9/13/2/6/10/14/3/7/11/15 + ; some are still pre-loaded from the final loop iteration in pass=1 + + mova m1, m2 + mova m2, [cq+ 1*16] + mova m3, [cq+ 9*16] + mova m4, [cq+ 2*16] + mova m5, [cq+10*16] + mova m6, [cq+ 3*16] + mova m7, [cq+11*16] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + ; m7 is already stored in [rsp+gprsize+0*16] + mova m0, [cq+ 4*16] + mova m1, [cq+12*16] + mova m2, [cq+ 5*16] + mova m3, [cq+13*16] + mova m4, [cq+ 6*16] + mova m5, [cq+14*16] + mova m6, [cq+ 7*16] + mova m7, [cq+15*16] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mov r6, dstq +%else + mov [rsp+16*16+gprsize*1], dstq +%endif + lea r3, [strideq*3] + lea dstq, [dstq+strideq*8] + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +%undef mzero + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] +%if ARCH_X86_64 + mov dstq, r6 +%else + mov dstq, [rsp+16*16+gprsize*1] +%endif + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + RET + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity, v + +cglobal iadst_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(iadst_8x8_internal_16bpc).pass1_main)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m4, [cq+ 9*16] + mova m5, [cq+13*16] + mova [rsp+gprsize+7*16], m0 + mova [rsp+gprsize+8*16], m1 + mova [rsp+gprsize+5*16], m4 + mova [rsp+gprsize+6*16], m5 + mova m0, m2 + mova m1, m3 + mova m2, [cq+ 1*16] + mova m3, [cq+ 5*16] + mova m4, [cq+ 2*16] + mova m5, [cq+ 6*16] + mova m6, [cq+11*16] + mova m7, [cq+15*16] + mova [rsp+gprsize+ 3*16], m4 + mova [rsp+gprsize+ 4*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m4, [cq+10*16] + mova m5, [cq+14*16] + mova m6, [cq+ 3*16] + mova m7, [cq+ 7*16] + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main_pass2_end + +%if ARCH_X86_64 + mova m11, [o(pw_m2048)] + mova m8, [o(pw_2048)] + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mov r6, dstq +%else + mov [rsp+16*16+gprsize*1], dstq +%endif + lea r3, [strideq*3] + lea dstq, [dstq+strideq*8] + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +%undef mzero + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] +%if ARCH_X86_64 + mov dstq, r6 +%else + mov dstq, [rsp+16*16+gprsize*1] +%endif + call m(iadst_8x8_internal_16bpc).round1_and_write_8x8 + RET + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity, v + +cglobal iflipadst_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(iflipadst_8x8_internal_16bpc).pass1_main)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: + lea r3, [strideq*3] + lea r3, [r3*5] + add dstq, r3 + neg strideq + jmp m(iadst_8x16_internal_16bpc).pass2 + +INV_TXFM_8X16_FN identity, dct, h +INV_TXFM_8X16_FN identity, adst, h +INV_TXFM_8X16_FN identity, flipadst, h +INV_TXFM_8X16_FN identity, identity + +cglobal iidentity_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(idct_8x8_internal_16bpc).pack_and_transpose)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: +%if ARCH_X86_64 + mova m4, [o(pw_2048)] + mova m5, [o(pixel_10bpc_max)] + pxor m6, m6 + mova m7, [o(pw_1697x16)] +%endif + mov r5d, 4 + lea r3, [strideq*3] +.pass2_loop: + call .main +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).round1_and_write_8x4 +%else + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 +%endif + REPX {mova [cq+x*16], m6}, 0, 4, 8, 12, 16, 20, 24, 28 + dec r5d + jle .end + add cq, 16 + lea dstq, [dstq+strideq*4] + mova m0, [cq+ 0*16] + mova m1, [cq+ 4*16] + mova m2, [cq+ 8*16] + mova m3, [cq+12*16] + jmp .pass2_loop +.end: + RET +.main: + ; y = pmulhrsw(x, pw_1697x16); x = paddsw(x, x); x = paddsw(x, y) +%if ARCH_X86_32 + mova m7, [o(pw_1697x16)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 +%else + pmulhrsw m8, m7, m0 + pmulhrsw m9, m7, m1 + pmulhrsw m10, m7, m2 + pmulhrsw m11, m7, m3 +%endif + REPX {paddsw x, x}, m0, m1, m2, m3 +%if ARCH_X86_64 + paddsw m0, m8 + paddsw m1, m9 + paddsw m2, m10 + paddsw m3, m11 +%else + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 +%endif + ret + +%macro INV_TXFM_16X4_FN 2 ; type1, type2 +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, 0, 16x4, 16, 0-8*16 +%else + INV_TXFM_FN %1, %2, 0, 16x4, 8, 0-12*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 4 +.dconly: + add r5d, 384 + sar r5d, 9 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m3, [o(pixel_10bpc_max)] + pxor m4, m4 +.loop: + mova m1, [dstq+ 0] + mova m2, [dstq+16] + REPX {paddw x, m0}, m1, m2 + REPX {pminsw x, m3}, m1, m2 + REPX {pmaxsw x, m4}, m1, m2 + mova [dstq+ 0], m1 + mova [dstq+16], m2 + add dstq, strideq + dec r3d + jg .loop + RET +%endif +%endmacro + +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, identity +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst + +cglobal idct_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + ; setup stack pointer + lea r3, [rsp+gprsize] + + mova m0, [cq+ 1*16] + mova m1, [cq+ 3*16] + mova m2, [cq+ 5*16] + mova m3, [cq+ 7*16] + mova m4, [cq+ 9*16] + mova m5, [cq+11*16] + mova m6, [cq+13*16] + mova m7, [cq+15*16] + call .main_oddhalf + mova m0, [cq+ 0*16] + mova m1, [cq+ 2*16] + mova m2, [cq+ 4*16] + mova m3, [cq+ 6*16] + mova m4, [cq+ 8*16] + mova m5, [cq+10*16] + mova m6, [cq+12*16] + mova m7, [cq+14*16] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + ; t0-7 is in m0-7 + + call .round + +%if ARCH_X86_64 +.pack_transpose: + ; transpose in two parts + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +.transpose: + call m(idct_8x4_internal_16bpc).transpose4x8packed + call .transpose4x8packed_hi +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m4, [r3+10*16] + mova m6, [r3+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + jmp tx2q +%if ARCH_X86_64 +.transpose4x8packed_hi: + punpcklwd m9, m10, m14 + punpckhwd m10, m14 + punpckhwd m14, m8, m12 + punpcklwd m8, m12 + + punpckhwd m11, m8, m9 + punpcklwd m8, m9 + punpckhwd m12, m14, m10 + punpcklwd m14, m10 + + punpcklwd m10, m11, m12 + punpckhwd m11, m12 + punpckhwd m9, m8, m14 + punpcklwd m8, m14 + ret +%endif +.main_oddhalf_fast: ; lower half zero + pmulld m7, m0, [o(pd_4076)] + pmulld m0, [o(pd_401)] + pmulld m6, m1, [o(pd_m1189)] + pmulld m1, [o(pd_3920)] +%if ARCH_X86_32 + mova m4, [o(pd_2048)] + REPX {paddd x, m4}, m1, m6 + REPX {psrad x, 12}, m1, m6 + mova [r3+1*16], m1 +%endif + pmulld m5, m2, [o(pd_3612)] + pmulld m2, [o(pd_1931)] +%if ARCH_X86_32 + pmulld m1, m3, [o(pd_m2598)] +%else + pmulld m4, m3, [o(pd_m2598)] +%endif + pmulld m3, [o(pd_3166)] + jmp .main_oddhalf_fast2 +.main_oddhalf: +%if ARCH_X86_64 + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3920, 1189 ; t11a, t12a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3166, 2598 ; t9a, t14a +.main_oddhalf_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t9 + paddd m0, m4 ; t8 + psubd m4, m6, m2 ; t10 + paddd m2, m6 ; t11 + psubd m6, m1, m5 ; t13 + paddd m5, m1 ; t12 + psubd m1, m7, m3 ; t14 + paddd m7, m3 ; t15 + REPX {pmaxsd x, m12}, m8, m1, m4, m6, m0, m2, m5, m7 + REPX {pminsd x, m13}, m8, m1, m4, m6, m0, m2, m5, m7 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 1, 8, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 4, 3, 9, _, 11, 10, 15, 4 + psubd m3, m1, m4 ; t10 + paddd m1, m4 ; t9 + psubd m4, m0, m2 ; t11a + paddd m0, m2 ; t8a + psubd m2, m8, m6 ; t13 + paddd m6, m8 ; t14 + psubd m8, m7, m5 ; t12a + paddd m7, m5 ; t15a + REPX {pmaxsd x, m12}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pminsd x, m13}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pmulld x, m14}, m2, m8, m3, m4 + paddd m2, m11 + paddd m8, m11 + paddd m5, m2, m3 ; t13a + psubd m2, m3 ; t10a + psubd m3, m8, m4 ; t11 + paddd m4, m8 ; t12 + REPX {psrad x, 12}, m5, m2, m3, m4 + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova [r3+4*16], m4 + mova [r3+5*16], m5 + mova [r3+6*16], m6 + mova [r3+7*16], m7 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m4, [o(pd_2048)] + + ITX_MULSUB_2D 0, 7, 2, 3, 5, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 6, 1, 2, 3, 5, 4, 3920, 1189 ; t11a, t12a + + mova m2, [r3+0*16] + mova m3, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova m1, [r3+2*16] + mova m5, [r3+3*16] + mova [r3+2*16], m6 + mova [r3+3*16], m7 + + ITX_MULSUB_2D 2, 5, 0, 6, 7, _, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 1, 3, 0, 6, 7, _, 3166, 2598 ; t9a, t14a + + mova m0, [r3+0*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] +.main_oddhalf_fast2: + REPX {paddd x, m4}, m0, m7, m2, m5, m1, m3 + REPX {psrad x, 12}, m0, m7, m2, m5, m1, m3 + psubd m4, m0, m1 ; t9 + paddd m0, m1 ; t8 + mova m1, [r3+1*16] + mova [r3+0*16], m4 + psubd m4, m6, m2 ; t10 + paddd m2, m6 ; t11 + psubd m6, m1, m5 ; t13 + paddd m5, m1 ; t12 + psubd m1, m7, m3 ; t14 + paddd m7, m3 ; t15 + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m1, m4, m6, m0, m2, m5, m7 + pmaxsd m3, [r3+0*16] + mova [r3+0*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m1, m4, m6, m0, m2, m5, m7 + pminsd m3, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 1, 3, 0, 2, 5, 7, 1567, 3784 + ITX_MULSUB_2D 6, 4, 0, 2, _, 7, 5, 3784, 4 + mova m0, [r3+0*16] + mova m2, [r3+1*16] + psubd m5, m1, m4 ; t10 + mova [r3+1*16], m5 + paddd m1, m4 ; t9 + psubd m4, m0, m2 ; t11a + paddd m0, m2 ; t8a + mova m5, [r3+2*16] + mova m7, [r3+3*16] + psubd m2, m3, m6 ; t13 + paddd m6, m3 ; t14 + paddd m3, m7, m5 ; t15a + psubd m7, m5 ; t12a + mova [r3+0*16], m3 + mova m3, [r3+1*16] + mova m5, [o(clip_18b_min)] + REPX {pmaxsd x, m5}, m2, m7, m3, m4, m0, m1, m6 + pmaxsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(clip_18b_max)] + REPX {pminsd x, m5}, m2, m7, m3, m4, m0, m1, m6 + pminsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(pd_2896)] + REPX {pmulld x, m5}, m2, m7, m3, m4 + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m2, m7 + paddd m5, m2, m3 ; t13a + psubd m2, m3 ; t10a + psubd m3, m7, m4 ; t11 + paddd m4, m7 ; t12 + REPX {psrad x, 12}, m5, m2, m3, m4 + mova m7, [r3+0*16] + mova [r3+11*16], m0 + mova [r3+10*16], m1 + mova [r3+9*16], m2 + mova [r3+8*16], m3 + mova [r3+7*16], m4 + mova [r3+6*16], m5 + mova [r3+5*16], m6 + mova [r3+4*16], m7 +%endif + ret +.round: +%if ARCH_X86_64 + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + pcmpeqd m8, m8 + REPX {psubd x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + mova m8, [r3+1*16] + mova m9, [r3+2*16] + mova m10, [r3+3*16] + mova m11, [r3+4*16] + mova m12, [r3+5*16] + mova m13, [r3+6*16] + mova m14, [r3+7*16] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r3+0*16] ; out8 + paddd m7, [r3+0*16] ; out7 + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ; and out0-15 is now in m0-15 +%else + mova [r3+ 0*16], m0 + mova m0, [o(clip_18b_min)] + REPX {pmaxsd x, m0}, m1, m2, m3, m4, m5, m6, m7 + pmaxsd m0, [r3+ 0*16] + mova [r3+ 0*16], m7 + mova m7, [o(clip_18b_max)] + REPX {pminsd x, m7}, m0, m1, m2, m3, m4, m5, m6 + pminsd m7, [r3+ 0*16] + mova [r3+ 0*16], m0 + pcmpeqd m0, m0 + REPX {psubd x, m0}, m1, m2, m3, m4, m5, m6, m7 + mova [r3+ 1*16], m1 + mova [r3+ 2*16], m2 + mova m1, [r3+ 0*16] + psubd m1, m0 + mova [r3+ 0*16], m1 + mova m1, [r3+11*16] + mova m2, [r3+10*16] + psubd m0, m7, m1 + paddd m7, m1 + psubd m1, m6, m2 + paddd m6, m2 + REPX {psrad x, 1}, m0, m1, m6, m7 + packssdw m0, m1 ; out8-9 + packssdw m6, m7 ; out6-7 + mova [r3+11*16], m6 + mova m1, [r3+9*16] + mova m7, [r3+8*16] + psubd m2, m5, m1 + paddd m5, m1 + psubd m1, m4, m7 + paddd m4, m7 + REPX {psrad x, 1}, m2, m1, m4, m5 + packssdw m2, m1 ; out10-11 + packssdw m4, m5 ; out4-5 + mova m1, [r3+2*16] + mova [r3+10*16], m4 + mova m6, [r3+7*16] + mova m7, [r3+6*16] + psubd m4, m3, m6 + paddd m3, m6 + psubd m6, m1, m7 + paddd m1, m7 + REPX {psrad x, 1}, m4, m6, m1, m3 + packssdw m4, m6 ; out12-13 + packssdw m1, m3 ; out2-3 + mova m3, [r3+1*16] + mova [r3+9*16], m1 + mova m1, [r3+0*16] + mova m5, [r3+5*16] + mova m7, [r3+4*16] + psubd m6, m3, m5 + paddd m3, m5 + psubd m5, m1, m7 + paddd m1, m7 + REPX {psrad x, 1}, m6, m5, m1, m3 + packssdw m6, m5 ; out14-15 + packssdw m1, m3 ; out0-1 + mova [r3+8*16], m1 +%endif + ret + +.pass2: + lea r4, [o(m_suffix(idct_8x4_internal_8bpc, _ssse3).main)] +.pass2_loop: + lea r3, [strideq*3] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call r4 + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 + REPX {mova [cq+x*16], m6}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +%if ARCH_X86_64 + mova m0, m8 + mova m1, m9 + mova m2, m10 + mova m3, m11 +%else + mova m0, [rsp+gprsize+0*16] + mova m1, [rsp+gprsize+1*16] + mova m2, [rsp+gprsize+2*16] + mova m3, [rsp+gprsize+3*16] +%endif + add dstq, 16 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call r4 + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 + RET + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + ; setup stack pointer + lea r3, [rsp+gprsize] + call .main +%if ARCH_X86_64 + jmp m(idct_16x4_internal_16bpc).pack_transpose +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + jmp tx2q +%endif + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*16] + mova m1, [cq+13*16] + mova m2, [cq+ 6*16] + mova m3, [cq+ 9*16] + mova m4, [cq+10*16] + mova m5, [cq+ 5*16] + mova m6, [cq+14*16] + mova m7, [cq+ 1*16] + call .main_part1 + mova m0, [cq+ 0*16] + mova m1, [cq+15*16] + mova m2, [cq+ 4*16] + mova m3, [cq+11*16] + mova m4, [cq+ 8*16] + mova m5, [cq+ 7*16] + mova m6, [cq+12*16] + mova m7, [cq+ 3*16] + call .main_part2 +.round: +%if ARCH_X86_64 + mova m15, [o(pd_6144)] + psrld m14, 11 ; pd_1 + pcmpeqd m8, m8 ; -1 + psubd m13, m15, m14 ; pd_6143 + REPX {paddd x, m14}, m0, m2 + REPX {paddd x, m15}, m4, m6 + REPX {pxor x, m8 }, m1, m3, m5, m7 + REPX {psrad x, 1 }, m1, m3 + REPX {paddd x, m15}, m5, m7 + REPX {psubd x, m8 }, m1, m3 + paddd m8, m15, m9 + psubd m9, m13, m10 + paddd m10, m15, m11 + psubd m11, m13, m12 + paddd m12, m14, [r3+3*16] + psubd m13, m14, [r3+2*16] + psubd m15, m14, [r3+0*16] + paddd m14, [r3+1*16] + REPX {psrad x, 1 }, m0, m2, m12, m13, m14, m15 + REPX {psrad x, 13}, m4, m5, m6, m7, m8, m9, m10, m11 +%else + mova [r3+8*16], m1 + mova [r3+9*16], m3 + mova m3, [o(pd_6144)] + pcmpeqd m1, m1 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m5, m6, m7 + REPX {psrad x, 13}, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {pxor x, m1}, m5, m7 + REPX {psubd x, m1}, m4, m6 + REPX {psrad x, 1 }, m4, m5, m6, m7 + REPX {psubd x, m1}, m5, m7 + packssdw m4, m5 + packssdw m6, m7 + mova m5, [r3+8*16] + mova m7, [r3+9*16] + mova [r3+8*16], m4 + mova [r3+9*16], m6 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m0, m5, m2, m7 + REPX {psrad x, 13}, m0, m5, m2, m7 + packssdw m0, m5 + packssdw m2, m7 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + REPX {psubd x, m1}, m4, m6 + REPX {pxor x, m1}, m5, m7 + REPX {psrad x, 1 }, m4, m5, m6, m7 + REPX {psubd x, m1}, m5, m7 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret + +.main_part2: +%if ARCH_X86_64 + ITX_MULSUB_2D 1, 0, 8, 9, 10, 11, 201, 4091 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 11, 1751, 3703 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 11, 3035, 2751 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 11, 3857, 1380 + psubd m8, m0, m4 ; t8a + paddd m0, m4 ; t0a + psubd m4, m1, m5 ; t9a + paddd m1, m5 ; t1a + psubd m5, m2, m6 ; t12a + paddd m2, m6 ; t4a + psubd m6, m3, m7 ; t13a + paddd m7, m3 ; t5a + REPX {pmaxsd x, m12}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + mova m15, [o(pd_4017)] + mova m10, [o(pd_799)] + ITX_MULSUB_2D 8, 4, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 5, 3, 9, _, 11, 15, 10 + psubd m3, m0, m2 ; t4 + paddd m0, m2 ; t0 + psubd m2, m1, m7 ; t5 + paddd m1, m7 ; t1 + psubd m7, m4, m6 ; t12a + paddd m4, m6 ; t8a + psubd m6, m8, m5 ; t13a + paddd m5, m8 ; t9a + REPX {pmaxsd x, m12}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 3, 2, 8, 9, _, 11, 10, 15 + ITX_MULSUB_2D 7, 6, 8, 9, _, 11, 10, 15 + mova m10, [r3+0*16] ; t2 + mova m8, [r3+1*16] ; t3 + psubd m9, m0, m10 ; t2a + paddd m0, m10 ; out0 + psubd m10, m1, m8 ; t3a + paddd m1, m8 ; -out15 + mova [r3+0*16], m1 + mova m15, [r3+3*16] ; t7a + mova m1, [r3+2*16] ; t6a + psubd m8, m3, m15 ; t7 + paddd m15, m3 ; out12 + paddd m3, m2, m1 ; -out3 + psubd m2, m1 ; t6 + mova [r3+3*16], m15 + mova [r3+1*16], m2 + mova m1, [r3+7*16] ; t15 + mova m2, [r3+6*16] ; t14 + paddd m15, m7, m1 ; -out13 + psubd m7, m1 ; t15a + psubd m11, m6, m2 ; t14a + paddd m2, m6 ; out2 + mova [r3+2*16], m15 + mova m1, [r3+4*16] ; t10a + mova m15, [r3+5*16] ; t11a + psubd m6, m4, m1 ; t10 + paddd m1, m4 ; -out1 + psubd m4, m5, m15 ; t11 + paddd m5, m15 ; out14 + REPX {pmaxsd x, m12}, m11, m7, m9, m10, m6, m4, m8 + pmaxsd m12, [r3+1*16] ; t6 + mova [r3+1*16], m5 + REPX {pminsd x, m13}, m11, m7, m9, m10, m6, m4, m12, m8 + REPX {pmulld x, m14}, m11, m7, m9, m10, m6, m4, m12, m8 + paddd m5, m11, m7 ; -out5 (unshifted) + psubd m11, m7 ; out10 (unshifted) + paddd m7, m9, m10 ; -out7 (unshifted) + psubd m9, m10 ; out8 (unshifted) + psubd m10, m6, m4 ; -out9 (unshifted) + paddd m6, m4 ; out6 (unshifted) + paddd m4, m12, m8 ; out4 (unshifted) + psubd m12, m8 ; -out11 (unshifted) +%else + mova [r3+8*16], m0 + mova [r3+9*16], m1 + mova [r3+10*16], m2 + mova [r3+11*16], m3 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 5, 4, 0, 1, 2, 3, 3035, 2751 + ITX_MULSUB_2D 7, 6, 0, 1, 2, 3, 3857, 1380 + mova m0, [r3+8*16] + mova m1, [r3+9*16] + mova [r3+8*16], m4 + mova m4, [r3+10*16] + mova [r3+9*16], m5 + mova [r3+10*16], m6 + mova m5, [r3+11*16] + mova [r3+11*16], m7 + ITX_MULSUB_2D 1, 0, 2, 6, 7, 3, 201, 4091 + ITX_MULSUB_2D 5, 4, 2, 6, 7, 3, 1751, 3703 + mova m2, [r3+8*16] + mova m6, [r3+9*16] + psubd m3, m0, m2 ; t8a + paddd m0, m2 ; t0a + mova [r3+8*16], m3 + psubd m2, m1, m6 ; t9a + paddd m1, m6 ; t1a + mova m3, [r3+10*16] + psubd m6, m4, m3 ; t12a + paddd m4, m3 ; t4a + mova m3, [r3+11*16] + psubd m7, m5, m3 ; t13a + paddd m5, m3 ; t5a + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m2, m6, m7, m0, m1, m4, m5 + pmaxsd m3, [r3+8*16] + mova [r3+8*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m2, m6, m7, m0, m1, m4, m5 + pminsd m3, [r3+8*16] + mova [r3+8*16], m3 + psubd m3, m0, m4 ; t4 + paddd m0, m4 ; t0 + psubd m4, m1, m5 ; t5 + paddd m1, m5 ; t1 + mova m5, [o(pd_2048)] + mova [r3+9*16], m1 + mova [r3+10*16], m4 + mova [r3+11*16], m3 + mova m3, [r3+8*16] + mova [r3+8*16], m0 + ITX_MULSUB_2D 3, 2, 0, 1, 4, 5, 799, 4017 + ITX_MULSUB_2D 7, 6, 0, 1, 4, 5, 4017, 4 + psubd m5, m2, m7 ; t12a + paddd m2, m7 ; t8a + psubd m7, m3, m6 ; t13a + paddd m6, m3 ; t9a + mova m0, [r3+8*16] + mova m1, [r3+9*16] + mova m4, [r3+10*16] + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m4, m5, m7, m0, m1, m2, m6 + pmaxsd m3, [r3+11*16] + mova [r3+8*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m4, m5, m7, m0, m1, m2, m6 + pminsd m3, [r3+8*16] + mova [r3+8*16], m0 + mova [r3+9*16], m1 + mova [r3+10*16], m2 + mova [r3+11*16], m6 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 3, 4, 1, 2, 6, 0, 1567, 3784 + ITX_MULSUB_2D 5, 7, 1, 2, 6, 0, 6, 3784 + mova m0, [r3+7*16] ; t7a + mova m2, [r3+6*16] ; t6a + psubd m1, m3, m0 ; t7 + paddd m0, m3 ; out12 + paddd m3, m4, m2 ; -out3 + psubd m4, m2 ; t6 + mova [r3+7*16], m3 + mova m3, [r3+3*16] ; t15 + mova m2, [r3+2*16] ; t14 + paddd m6, m5, m3 ; -out13 + psubd m5, m3 ; t15a + psubd m3, m7, m2 ; t14a + paddd m2, m7 ; out2 + mova [r3+6*16], m2 + mova m7, [r3+0*16] ; t10a + mova m2, [r3+1*16] ; t11a + mova [r3+0*16], m0 + mova [r3+1*16], m6 + mova m6, [r3+11*16] + psubd m0, m6, m2 ; t11 + paddd m6, m2 ; out14 + mova [r3+2*16], m6 + mova m2, [r3+10*16] + psubd m6, m2, m7 ; t10 + paddd m2, m7 ; -out1 + mova m7, [r3+5*16] ; t3 + mova [r3+5*16], m2 + mova [r3+10*16], m1 + mova m1, [r3+9*16] + psubd m2, m1, m7 ; t3a + paddd m1, m7 ; -out15 + mova [r3+3*16], m1 + mova m1, [r3+4*16] ; t2 + mova m7, [r3+8*16] + psubd m7, m1 ; t2a + paddd m1, [r3+8*16] ; out0 + mova [r3+4*16], m1 + mova m1, [o(clip_18b_min)] + REPX {pmaxsd x, m1}, m0, m2, m3, m4, m5, m6, m7 + pmaxsd m1, [r3+10*16] + mova [r3+10*16], m1 + mova m1, [o(clip_18b_max)] + REPX {pminsd x, m1}, m0, m2, m3, m4, m5, m6, m7 + pminsd m1, [r3+10*16] + mova [r3+10*16], m1 + mova m1, [o(pd_2896)] + REPX {pmulld x, m1}, m0, m2, m3, m4, m5, m6, m7 + pmulld m1, [r3+10*16] + mova [r3+11*16], m3 + psubd m3, m4, m1 ; -out11 (unshifted) + paddd m4, m1 ; out4 (unshifted) + psubd m1, m6, m0 ; -out9 (unshifted) + paddd m6, m0 ; out6 (unshifted) + psubd m0, m7, m2 ; out8 (unshifted) + paddd m7, m2 ; -out7 (unshifted) + mova m2, [r3+11*16] + mova [r3+11*16], m5 + paddd m5, m2 ; -out5 (unshifted) + psubd m2, [r3+11*16] ; out10 (unshifted) + ; m0-3 contain out8-11 (unshifted), m4-7 contain out4-7 (unshifted) + ; r[-4,3] contain out0-3 and out12-15 +%endif + ret +.main_part1: +%if ARCH_X86_64 + ITX_MULSUB_2D 1, 0, 8, 9, 10, 11, 995, 3973 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 11, 2440, 3290 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 11, 3513, 2106 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 11, 4052, 601 + psubd m8, m0, m4 ; t10a + paddd m0, m4 ; t2a + psubd m4, m1, m5 ; t11a + paddd m1, m5 ; t3a + psubd m5, m2, m6 ; t14a + paddd m2, m6 ; t6a + psubd m6, m3, m7 ; t15a + paddd m7, m3 ; t7a + REPX {pmaxsd x, m12}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + mova m15, [o(pd_2276)] + mova m10, [o(pd_3406)] + ITX_MULSUB_2D 8, 4, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 5, 3, 9, _, 11, 15, 10 + psubd m3, m0, m2 ; t6 + paddd m0, m2 ; t2 + psubd m2, m1, m7 ; t7 + paddd m1, m7 ; t3 + psubd m7, m4, m6 ; t14a + paddd m4, m6 ; t10a + psubd m6, m8, m5 ; t15a + paddd m5, m8 ; t11a + REPX {pmaxsd x, m12}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + mova m15, [o(pd_1567)] + mova m10, [o(pd_3784)] + ITX_MULSUB_2D 2, 3, 8, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 7, 8, 9, _, 11, 10, 15 + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+4*16], m4 + mova [r3+5*16], m5 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova [r3+6*16], m6 + mova [r3+7*16], m7 +%else + mova [r3+4*16], m0 + mova [r3+5*16], m1 + mova [r3+6*16], m2 + mova [r3+7*16], m3 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 5, 4, 0, 1, 2, 3, 3513, 2106 + ITX_MULSUB_2D 7, 6, 0, 1, 2, 3, 4052, 601 + mova [r3+0*16], m4 + mova [r3+1*16], m5 + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m0, [r3+4*16] + mova m1, [r3+5*16] + mova m2, [r3+6*16] + mova m7, [r3+7*16] + ITX_MULSUB_2D 1, 0, 4, 5, 6, 3, 995, 3973 + ITX_MULSUB_2D 7, 2, 4, 5, 6, 3, 2440, 3290 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + psubd m6, m0, m4 ; t10a + paddd m0, m4 ; t2a + mova [r3+4*16], m6 + mova m6, [r3+2*16] + mova m3, [r3+3*16] + psubd m4, m1, m5 ; t11a + paddd m1, m5 ; t3a + psubd m5, m2, m6 ; t14a + paddd m2, m6 ; t6a + psubd m6, m7, m3 ; t15a + paddd m7, m3 ; t7a + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m4, m5, m6, m0, m1, m2, m7 + pmaxsd m3, [r3+4*16] + mova [r3+4*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m4, m5, m6, m0, m1, m2, m7 + pminsd m3, [r3+4*16] + mova [r3+4*16], m3 + psubd m3, m0, m2 ; t6 + paddd m0, m2 ; t2 + psubd m2, m1, m7 ; t7 + paddd m1, m7 ; t3 + mova [r3+5*16], m1 + mova [r3+6*16], m3 + mova [r3+7*16], m2 + mova m1, [r3+4*16] + mova [r3+4*16], m0 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 1, 4, 0, 7, 2, 3, 3406, 2276 + ITX_MULSUB_2D 6, 5, 0, 7, 2, 3, 2276, 2 + psubd m7, m4, m6 ; t14a + paddd m4, m6 ; t10a + psubd m6, m1, m5 ; t15a + paddd m5, m1 ; t11a + mova m1, [r3+5*16] + mova m3, [r3+6*16] + mova m2, [r3+7*16] + mova m0, [o(clip_18b_min)] + REPX {pmaxsd x, m0}, m3, m2, m7, m6, m1, m4, m5 + pmaxsd m0, [r3+4*16] + mova [r3+4*16], m0 + mova m0, [o(clip_18b_max)] + REPX {pminsd x, m0}, m3, m2, m7, m6, m1, m4, m5 + pminsd m0, [r3+4*16] + mova [r3+4*16], m0 + mova [r3+5*16], m1 + mova [r3+0*16], m4 + mova [r3+1*16], m5 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 2, 3, 1, 4, 5, 0, 3784, 1567 + ITX_MULSUB_2D 6, 7, 1, 4, 5, 0, 5, 1567 + mova [r3+6*16], m2 + mova [r3+7*16], m3 + mova [r3+2*16], m6 + mova [r3+3*16], m7 +%endif + ret + +.pass2: + lea r4, [o(m_suffix(iadst_8x4_internal_8bpc, _ssse3).main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r3, [rsp+gprsize] + call m(iadst_16x4_internal_16bpc).main +%if ARCH_X86_64 + packssdw m1, m0 + packssdw m3, m2 + packssdw m5, m4 + packssdw m7, m6 + packssdw m9, m8 + packssdw m11, m10 + packssdw m13, m12 + packssdw m15, m14 + mova m0, m15 + mova m2, m13 + mova m4, m11 + mova m6, m9 + mova m8, m7 + mova m10, m5 + mova m12, m3 + mova m14, m1 + jmp m(idct_16x4_internal_16bpc).transpose +%else + mova [rsp+gprsize+4*16], m0 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m4 + mova [rsp+gprsize+7*16], m6 + pshufd m6, [rsp+gprsize+ 8*16], q1032 + pshufd m4, [rsp+gprsize+ 9*16], q1032 + pshufd m2, [rsp+gprsize+10*16], q1032 + pshufd m0, [rsp+gprsize+11*16], q1032 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + pshufd m6, [rsp+gprsize+ 4*16], q1032 + pshufd m4, [rsp+gprsize+ 5*16], q1032 + pshufd m2, [rsp+gprsize+ 6*16], q1032 + pshufd m0, [rsp+gprsize+ 7*16], q1032 + call m(idct_8x4_internal_16bpc).transpose4x8packed + jmp tx2q +%endif + +.pass2: + lea r3, [strideq*3] + lea dstq, [dstq+r3] + neg strideq + lea r4, [o(m_suffix(iadst_8x4_internal_8bpc, _ssse3).main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + mova m15, [o(pd_11586)] + pmulld m0, m15, [cq+ 0*16] + pmulld m1, m15, [cq+ 1*16] + pmulld m2, m15, [cq+ 2*16] + pmulld m3, m15, [cq+ 3*16] + pmulld m4, m15, [cq+ 4*16] + pmulld m5, m15, [cq+ 5*16] + pmulld m6, m15, [cq+ 6*16] + pmulld m7, m15, [cq+ 7*16] + pmulld m8, m15, [cq+ 8*16] + pmulld m9, m15, [cq+ 9*16] + pmulld m10, m15, [cq+10*16] + pmulld m11, m15, [cq+11*16] + pmulld m12, m15, [cq+12*16] + pmulld m13, m15, [cq+13*16] + pmulld m14, m15, [cq+14*16] + pmulld m15, [cq+15*16] + mova [cq+ 0*16], m15 + mova m15, [o(pd_6144)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [cq+ 0*16] + REPX {psrad x, 13 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp m(idct_16x4_internal_16bpc).pack_transpose +%else + add cq, 8*16 + mov r5d, 2 +.loop_pass1: + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+0*16] + pmulld m1, m7, [cq+1*16] + pmulld m2, m7, [cq+2*16] + pmulld m3, m7, [cq+3*16] + pmulld m4, m7, [cq+4*16] + pmulld m5, m7, [cq+5*16] + pmulld m6, m7, [cq+6*16] + pmulld m7, [cq+7*16] + mova [cq+7*16], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [cq+7*16] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + dec r5d + jz .end_pass1 + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + sub cq, 8*16 + jmp .loop_pass1 +.end_pass1: + jmp tx2q +%endif + +.pass2: +%if ARCH_X86_64 + mova m12, [o(pw_1697x8)] +%endif + lea r4, [o(.main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop +.main: +%if ARCH_X86_64 + pmulhrsw m4, m0, m12 + pmulhrsw m5, m1, m12 + pmulhrsw m6, m2, m12 + pmulhrsw m7, m3, m12 +%else + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m0, m7 + pmulhrsw m5, m1, m7 + pmulhrsw m6, m2, m7 + pmulhrsw m7, m3 +%endif + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + ret + +%macro INV_TXFM_16X8_FN 2-3 0 ; type1, type2, eob_offset +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, %3, 16x8, 16, 0-8*16 +%else + INV_TXFM_FN %1, %2, %3, 16x8, 8, 0-13*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 8 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 +%if ARCH_X86_32 + add rsp, 1*16 +%endif + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly +%endif +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, identity, 6 +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst + +cglobal idct_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + DECLARE_REG_TMP 6, 4, 6 +%else + mov [rsp+gprsize+12*16], r1 + DECLARE_REG_TMP 1, 4, 3 +%endif + lea t0, [o(.main)] +.loop_main: +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + shl r5d, 4 + + lea r3, [rsp+gprsize] +.loop_pass1: + call t0 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+4*32+r5], m8 + mova [cq+5*32+r5], m9 + mova [cq+6*32+r5], m10 + mova [cq+7*32+r5], m11 +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+4*32+r5], m0 + mova [cq+5*32+r5], m1 + mova [cq+6*32+r5], m2 + mova [cq+7*32+r5], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + pxor m7, m7 + REPX {mova [cq+x*32+r5], m7}, 8, 9, 10, 11, 12, 13, 14, 15 + test r5d, r5d + jz .end + mova [cq+0*32+r5], m0 + mova [cq+1*32+r5], m1 + mova [cq+2*32+r5], m2 + mova [cq+3*32+r5], m3 + xor r5d, r5d + jmp .loop_pass1 +.end: + + jmp tx2q +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*32+r5] + mova m1, [cq+ 3*32+r5] + mova m2, [cq+ 5*32+r5] + mova m3, [cq+ 7*32+r5] + mova m4, [cq+ 9*32+r5] + mova m5, [cq+11*32+r5] + mova m6, [cq+13*32+r5] + mova m7, [cq+15*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*32+r5] + mova m1, [cq+ 2*32+r5] + mova m2, [cq+ 4*32+r5] + mova m3, [cq+ 6*32+r5] + mova m4, [cq+ 8*32+r5] + mova m5, [cq+10*32+r5] + mova m6, [cq+12*32+r5] + mova m7, [cq+14*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +.pass2_main: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity, 6 + +cglobal iadst_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*32+r5] + mova m1, [cq+13*32+r5] + mova m2, [cq+ 6*32+r5] + mova m3, [cq+ 9*32+r5] + mova m4, [cq+10*32+r5] + mova m5, [cq+ 5*32+r5] + mova m6, [cq+14*32+r5] + mova m7, [cq+ 1*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(iadst_16x4_internal_16bpc).main_part1 + mova m0, [cq+ 0*32+r5] + mova m1, [cq+15*32+r5] + mova m2, [cq+ 4*32+r5] + mova m3, [cq+11*32+r5] + mova m4, [cq+ 8*32+r5] + mova m5, [cq+ 7*32+r5] + mova m6, [cq+12*32+r5] + mova m7, [cq+ 3*32+r5] +%if ARCH_X86_32 + add r3, 8*16 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 8*16 +%endif + call m(iadst_16x4_internal_16bpc).main_part2 + call m(iadst_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mova m11, [o(pw_m2048)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main_pass2_end + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity, 6 + +cglobal iflipadst_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main +.main: + call m(iadst_16x8_internal_16bpc).main +%if ARCH_X86_64 + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, m14, q1032 + pshufd m2, m12, q1032 + pshufd m4, m10, q1032 + pshufd m6, m8, q1032 + mova m14, m1 + mova m12, m3 + mova m10, m5 + mova m8, m7 +%else + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, [r3+11*16], q1032 + pshufd m2, [r3+10*16], q1032 + pshufd m4, [r3+9*16], q1032 + pshufd m6, [r3+8*16], q1032 + mova [r3+8*16], m7 + mova [r3+9*16], m5 + mova [r3+10*16], m3 + mova [r3+11*16], m1 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + lea dstq, [dstq+strideq*8] + neg strideq + add dstq, strideq +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], strideq +%endif + jmp m(iadst_16x8_internal_16bpc).pass2 + +INV_TXFM_16X8_FN identity, dct, -54 +INV_TXFM_16X8_FN identity, adst, -54 +INV_TXFM_16X8_FN identity, flipadst, -54 +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main +.main: +%if ARCH_X86_64 + mova m15, [o(pd_2896)] + pmulld m0, m15, [cq+ 0*32+r5] + pmulld m1, m15, [cq+ 1*32+r5] + pmulld m2, m15, [cq+ 2*32+r5] + pmulld m3, m15, [cq+ 3*32+r5] + pmulld m4, m15, [cq+ 4*32+r5] + pmulld m5, m15, [cq+ 5*32+r5] + pmulld m6, m15, [cq+ 6*32+r5] + pmulld m7, m15, [cq+ 7*32+r5] + pmulld m8, m15, [cq+ 8*32+r5] + pmulld m9, m15, [cq+ 9*32+r5] + pmulld m10, m15, [cq+10*32+r5] + pmulld m11, m15, [cq+11*32+r5] + pmulld m12, m15, [cq+12*32+r5] + pmulld m13, m15, [cq+13*32+r5] + pmulld m14, m15, [cq+14*32+r5] + pmulld m15, [cq+15*32+r5] + mova [r3], m15 + mova m15, [o(pd_2048)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + mova [r3], m15 + mova m15, [o(pd_11586)] + REPX {pmulld x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + pmulld m15, [r3] + mova [r3], m15 + mova m15, [o(pd_6144)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 13 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%else + mova m0, [cq+ 0*32+r5] + mova m1, [cq+ 1*32+r5] + mova m2, [cq+ 2*32+r5] + mova m3, [cq+ 3*32+r5] + mova m4, [cq+ 4*32+r5] + mova m5, [cq+ 5*32+r5] + mova m6, [cq+ 6*32+r5] + mova m7, [cq+ 7*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + mova [r3], m7 + mova m7, [o(pd_11586)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m2 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m0, [cq+ 8*32+r5] + mova m1, [cq+ 9*32+r5] + mova m2, [cq+10*32+r5] + mova m3, [cq+11*32+r5] + mova m4, [cq+12*32+r5] + mova m5, [cq+13*32+r5] + mova m6, [cq+14*32+r5] + mova m7, [cq+15*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + mova [r3], m7 + mova m7, [o(pd_11586)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +%if ARCH_X86_64 + mova m8, [o(pw_4096)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_64 + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%else + mova [rsp+gprsize], m7 + mova m7, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round4_and_write_8x8 +%endif +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +%macro INV_TXFM_16X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, tbl_16x16_%3, 16x16, 16, 0-(16+WIN64)*16 +%else + INV_TXFM_FN %1, %2, tbl_16x16_%3, 16x16, 8, 0-17*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 16 + add r5d, 640 + sar r5d, 10 + add rsp, (5+ARCH_X86_64*3+WIN64)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly2 +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, identity, v +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst + +cglobal idct_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + DECLARE_REG_TMP 6, 7 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif +%elif ARCH_X86_32 + DECLARE_REG_TMP 1, 6 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] +.pass1_full: +%undef cmp + mov t1d, 4 +.zero_loop: + dec t1d + cmp eobb, byte [r5+t1] + jb .zero_loop + mov r5d, t1d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, [rsp+16*16+2*gprsize] +%endif + ; setup stack pointer + lea r3, [rsp+gprsize] +.loop_pass1: + call t0 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+4*64+r5], m8 + mova [cq+5*64+r5], m9 + mova [cq+6*64+r5], m10 + mova [cq+7*64+r5], m11 +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+4*64+r5], m0 + mova [cq+5*64+r5], m1 + mova [cq+6*64+r5], m2 + mova [cq+7*64+r5], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+0*64+r5], m0 + mova [cq+1*64+r5], m1 + mova [cq+2*64+r5], m2 + mova [cq+3*64+r5], m3 + pxor m0, m0 + REPX {mova [cq+x*64+r5], m0}, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 16 + jge .loop_pass1 + +%if ARCH_X86_32 + ; restore pic-ptr + mov r1, [rsp+16*16+1*gprsize] +%endif + jmp tx2q +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + + mova m0, [cq+ 1*64+r5] + mova m1, [cq+ 3*64+r5] + mova m2, [cq+ 5*64+r5] + mova m3, [cq+ 7*64+r5] + mova m4, [cq+ 9*64+r5] + mova m5, [cq+11*64+r5] + mova m6, [cq+13*64+r5] + mova m7, [cq+15*64+r5] + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*64+r5] + mova m1, [cq+ 2*64+r5] + mova m2, [cq+ 4*64+r5] + mova m3, [cq+ 6*64+r5] + mova m4, [cq+ 8*64+r5] + mova m5, [cq+10*64+r5] + mova m6, [cq+12*64+r5] + mova m7, [cq+14*64+r5] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call .round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret +.round: +%if ARCH_X86_64 + REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + psrld m8, m11, 10 ; 2 + REPX {paddd x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + mova m8, [r3+1*16] + mova m9, [r3+2*16] + mova m10, [r3+3*16] + mova m11, [r3+4*16] + mova m12, [r3+5*16] + mova m13, [r3+6*16] + mova m14, [r3+7*16] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r3+0*16] ; out8 + paddd m7, [r3+0*16] ; out7 + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ; and out0-15 is now in m0-15 +%else + mova [r3+ 0*16], m0 + mova m0, [o(clip_18b_min)] + REPX {pmaxsd x, m0}, m1, m2, m3, m4, m5, m6, m7 + pmaxsd m0, [r3+ 0*16] + mova [r3+ 0*16], m7 + mova m7, [o(clip_18b_max)] + REPX {pminsd x, m7}, m0, m1, m2, m3, m4, m5, m6 + pminsd m7, [r3+ 0*16] + mova [r3+ 0*16], m0 + mova m0, [o(pd_2)] + REPX {paddd x, m0}, m1, m2, m3, m4, m5, m6, m7 + paddd m0, [r3+ 0*16] + mova [r3+ 0*16], m0 + mova [r3+ 1*16], m1 + mova [r3+ 2*16], m2 + mova m1, [r3+11*16] + mova m2, [r3+10*16] + psubd m0, m7, m1 + paddd m7, m1 + psubd m1, m6, m2 + paddd m6, m2 + REPX {psrad x, 2}, m0, m1, m6, m7 + packssdw m0, m1 ; out8-9 + packssdw m6, m7 ; out6-7 + mova [r3+11*16], m6 + mova m1, [r3+9*16] + mova m7, [r3+8*16] + psubd m2, m5, m1 + paddd m5, m1 + psubd m1, m4, m7 + paddd m4, m7 + REPX {psrad x, 2}, m2, m1, m4, m5 + packssdw m2, m1 ; out10-11 + packssdw m4, m5 ; out4-5 + mova m1, [r3+2*16] + mova [r3+10*16], m4 + mova m6, [r3+7*16] + mova m7, [r3+6*16] + psubd m4, m3, m6 + paddd m3, m6 + psubd m6, m1, m7 + paddd m1, m7 + REPX {psrad x, 2}, m4, m6, m1, m3 + packssdw m4, m6 ; out12-13 + packssdw m1, m3 ; out2-3 + mova m3, [r3+1*16] + mova [r3+9*16], m1 + mova m1, [r3+0*16] + mova m5, [r3+5*16] + mova m7, [r3+4*16] + psubd m6, m3, m5 + paddd m3, m5 + psubd m5, m1, m7 + paddd m1, m7 + REPX {psrad x, 2}, m6, m5, m1, m3 + packssdw m6, m5 ; out14-15 + packssdw m1, m3 ; out0-1 + mova [r3+8*16], m1 +%endif + ret + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 2 +.loop_pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [cq+0*64+ 0] + mova m1, [cq+2*64+ 0] + mova m2, [cq+0*64+16] + mova m3, [cq+2*64+16] + mova m4, [cq+0*64+32] + mova m5, [cq+2*64+32] + mova m6, [cq+0*64+48] + mova m7, [cq+2*64+48] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + ; m7 is already stored in [rsp+gprsize+0*16] + mova m0, [cq+1*64+ 0] + mova m1, [cq+3*64+ 0] + mova m2, [cq+1*64+16] + mova m3, [cq+3*64+16] + mova m4, [cq+1*64+32] + mova m5, [cq+3*64+32] + mova m6, [cq+1*64+48] + mova m7, [cq+3*64+48] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + lea dstq, [r7+strideq*8] +%else + mov dstq, [rsp+2*gprsize+16*16] + lea dstq, [dstq+strideq*8] +%endif + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 + mov dstq, r7 +%else + mov dstq, [rsp+2*gprsize+16*16] +%endif + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%if ARCH_X86_64 + add r7, 16 +%define mzero m9 +%else + add dword [rsp+2*gprsize+16*16], 16 +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add cq, 64*4 + REPX {mova [cq+x*16], mzero}, -8, -7, -6, -5, -4, -3, -2, -1 +%undef mzero + dec r4d + jg .loop_pass2 +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*64+r5] + mova m1, [cq+13*64+r5] + mova m2, [cq+ 6*64+r5] + mova m3, [cq+ 9*64+r5] + mova m4, [cq+10*64+r5] + mova m5, [cq+ 5*64+r5] + mova m6, [cq+14*64+r5] + mova m7, [cq+ 1*64+r5] + call m(iadst_16x4_internal_16bpc).main_part1 + mova m0, [cq+ 0*64+r5] + mova m1, [cq+15*64+r5] + mova m2, [cq+ 4*64+r5] + mova m3, [cq+11*64+r5] + mova m4, [cq+ 8*64+r5] + mova m5, [cq+ 7*64+r5] + mova m6, [cq+12*64+r5] + mova m7, [cq+ 3*64+r5] + call m(iadst_16x4_internal_16bpc).main_part2 + call .round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret +.round: +%if ARCH_X86_64 + pcmpeqd m8, m8 ; -1 + mova m15, [o(pd_10240)] + psrld m14, 10 ; +2 + psubd m13, m14, m8 ; +3 + REPX {pxor x, m8 }, m1, m3, m5, m7 + REPX {paddd x, m14}, m0, m2 + REPX {paddd x, m13}, m1, m3 + REPX {paddd x, m15}, m4, m5, m6, m7 + paddd m13, m15, m8 ; +10239 + paddd m8, m15, m9 + psubd m9, m13, m10 + paddd m10, m15, m11 + psubd m11, m13, m12 + paddd m12, m14, [r3+3*16] + psubd m13, m14, [r3+2*16] + psubd m15, m14, [r3+0*16] + paddd m14, [r3+1*16] + REPX {psrad x, 2 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11 +%else + mova [r3+8*16], m1 + mova [r3+9*16], m3 + mova m3, [o(pd_10240)] + pcmpeqd m1, m1 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m5, m6, m7 + REPX {psrad x, 14}, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + mova m3, [o(pd_2)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m6 + psubd m3, m1 + REPX {paddd x, m3}, m5, m7 + REPX {psrad x, 2 }, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova m5, [r3+8*16] + mova m7, [r3+9*16] + mova [r3+8*16], m4 + mova [r3+9*16], m6 + mova m3, [o(pd_10240)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m0, m5, m2, m7 + REPX {psrad x, 14}, m0, m5, m2, m7 + packssdw m0, m5 + packssdw m2, m7 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + mova m3, [o(pd_2)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m6 + psubd m3, m1 + REPX {paddd x, m3}, m5, m7 + REPX {psrad x, 2 }, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + mova m11, [o(pw_m2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 2 +.loop_pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [cq+0*64+32] + mova m1, [cq+1*64+32] + mova m2, [cq+2*64+16] + mova m3, [cq+3*64+16] + mova m4, [cq+0*64+ 0] + mova m5, [cq+1*64+ 0] + mova m6, [cq+2*64+48] + mova m7, [cq+3*64+48] + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m0, [cq+2*64+ 0] + mova m1, [cq+3*64+ 0] + mova m2, [cq+0*64+16] + mova m3, [cq+1*64+16] + mova m4, [cq+2*64+32] + mova m5, [cq+3*64+32] + mova m6, [cq+0*64+48] + mova m7, [cq+1*64+48] + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main_pass2_end + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + lea dstq, [r7+strideq*8] +%else + mov dstq, [rsp+2*gprsize+16*16] + lea dstq, [dstq+strideq*8] +%endif + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 + mov dstq, r7 +%else + mov dstq, [rsp+2*gprsize+16*16] +%endif + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] + call m(iadst_8x8_internal_16bpc).round1_and_write_8x8 +%if ARCH_X86_64 + add r7, 16 +%define mzero m9 +%else + add dword [rsp+2*gprsize+16*16], 16 +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add cq, 64*4 + REPX {mova [cq+x*16], mzero}, -8, -7, -6, -5, -4, -3, -2, -1 +%undef mzero + dec r4d + jg .loop_pass2 +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: + call m(iadst_16x16_internal_16bpc).main +%if ARCH_X86_64 + mova m1, m0 + mova m3, m2 + mova m5, m4 + mova m7, m6 + pshufd m0, m14, q1032 + pshufd m2, m12, q1032 + pshufd m4, m10, q1032 + pshufd m6, m8, q1032 + pshufd m8, m7, q1032 + pshufd m10, m5, q1032 + pshufd m12, m3, q1032 + pshufd m14, m1, q1032 +%else + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, [r3+11*16], q1032 + pshufd m2, [r3+10*16], q1032 + pshufd m4, [r3+9*16], q1032 + pshufd m6, [r3+8*16], q1032 + mova [r3+11*16], m1 + mova [r3+10*16], m3 + mova [r3+ 9*16], m5 + mova [r3+ 8*16], m7 +%endif + ret + +.pass2: + lea r3, [strideq*3] + lea r3, [r3*5] + add dstq, r3 + neg strideq + jmp m(iadst_16x16_internal_16bpc).pass2 + +INV_TXFM_16X16_FN identity, dct, h +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: +%if ARCH_X86_64 + mova m15, [o(pd_11586)] + pmulld m0, m15, [cq+ 0*64+r5] + pmulld m1, m15, [cq+ 1*64+r5] + pmulld m2, m15, [cq+ 2*64+r5] + pmulld m3, m15, [cq+ 3*64+r5] + pmulld m4, m15, [cq+ 4*64+r5] + pmulld m5, m15, [cq+ 5*64+r5] + pmulld m6, m15, [cq+ 6*64+r5] + pmulld m7, m15, [cq+ 7*64+r5] + pmulld m8, m15, [cq+ 8*64+r5] + pmulld m9, m15, [cq+ 9*64+r5] + pmulld m10, m15, [cq+10*64+r5] + pmulld m11, m15, [cq+11*64+r5] + pmulld m12, m15, [cq+12*64+r5] + pmulld m13, m15, [cq+13*64+r5] + pmulld m14, m15, [cq+14*64+r5] + pmulld m15, [cq+15*64+r5] + mova [r3], m15 + mova m15, [o(pd_10240)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 14 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%else + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+ 0*64+r5] + pmulld m1, m7, [cq+ 1*64+r5] + pmulld m2, m7, [cq+ 2*64+r5] + pmulld m3, m7, [cq+ 3*64+r5] + pmulld m4, m7, [cq+ 4*64+r5] + pmulld m5, m7, [cq+ 5*64+r5] + pmulld m6, m7, [cq+ 6*64+r5] + pmulld m7, [cq+ 7*64+r5] + mova [r3], m7 + mova m7, [o(pd_10240)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+8*16], m0 + mova [r3+9*16], m2 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+ 8*64+r5] + pmulld m1, m7, [cq+ 9*64+r5] + pmulld m2, m7, [cq+10*64+r5] + pmulld m3, m7, [cq+11*64+r5] + pmulld m4, m7, [cq+12*64+r5] + pmulld m5, m7, [cq+13*64+r5] + pmulld m6, m7, [cq+14*64+r5] + pmulld m7, [cq+15*64+r5] + mova [r3], m7 + mova m7, [o(pd_10240)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret + +.pass2: +%if ARCH_X86_64 + mova m4, [o(pw_2048)] + mova m5, [o(pixel_10bpc_max)] + pxor m6, m6 + mova m7, [o(pw_1697x16)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + mov r5d, 4 + lea r3, [strideq*3] +.pass2_loop: + mova m0, [cq+0*64+0] + mova m1, [cq+1*64+0] + mova m2, [cq+2*64+0] + mova m3, [cq+3*64+0] + call m(iidentity_8x16_internal_16bpc).main +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).round1_and_write_8x4 +%else + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 +%endif + REPX {mova [cq+x*16], m6}, 0, 4, 8, 12 + add cq, 16 + lea dstq, [dstq+strideq*4] + dec r5w + jg .pass2_loop + add cq, 64*3 + btc r5d, 16 + jc .end +%if ARCH_X86_64 + lea dstq, [r7+16] +%else + mov dstq, [rsp+2*gprsize+16*16] + add dstq, 16 +%endif + add r5d, 4 + jmp .pass2_loop +.end: +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +cglobal inv_txfm_add_identity_identity_8x32_16bpc, 4, 7, 8, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_5)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + mov r5d, eobd + add eobb, 21 + cmovc eobd, r5d ; 43, 107, 171 -> 64, 128, 192 + lea r4, [strideq*3] +.loop: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {paddsw x, m5}, m0, m1, m2, m3 + REPX {psraw x, 3 }, m0, m1, m2, m3 + call .main_zero + add cq, 16 + lea dstq, [dstq+strideq*4] + btc eobd, 16 + jnc .loop + sub eobd, 64 + jge .loop + RET +ALIGN function_align +.main_zero: + REPX {mova [cq+128*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 +.main: + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m4 + punpcklwd m0, m4 + punpckhwd m4, m2, m1 + punpcklwd m2, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r4 ] + REPX {pmaxsw x, m6}, m0, m1, m2, m3 + REPX {pminsw x, m7}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r4 ], m3 + ret + +cglobal inv_txfm_add_identity_identity_32x8_16bpc, 4, 7, 8, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_4096)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + mov r4d, eobd + add eobb, 21 + cmovc eobd, r4d + lea r4, [strideq*3] + mov r5, dstq +.loop: + mova m0, [cq+32*0] + packssdw m0, [cq+32*1] + mova m1, [cq+32*2] + packssdw m1, [cq+32*3] + mova m2, [cq+32*4] + packssdw m2, [cq+32*5] + mova m3, [cq+32*6] + packssdw m3, [cq+32*7] + REPX {mova [cq+32*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .loop + add cq, 32*8-32 + add r5, 16 + mov dstq, r5 + sub eobd, 64 + jge .loop + RET + +cglobal inv_txfm_add_identity_identity_16x32_16bpc, 4, 7, 12, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%else + mova m8, [o(pw_2896x8)] + mova m9, [o(pw_1697x16)] + mova m11, [o(pw_8192)] +%endif + mova m7, [o(pixel_10bpc_max)] + lea r4, [strideq*3] + pxor m6, m6 +%if ARCH_X86_64 + paddw m10, m11, m11 ; pw_16384 +%endif + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 107 ; eob < 143 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 271 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 399 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] +%if ARCH_X86_64 + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 + pmulhrsw m4, m9, m0 + pmulhrsw m5, m9, m1 + REPX {pmulhrsw x, m10}, m4, m5 +%else + mova m6, [o(pw_2896x8)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + mova m5, [o(pw_1697x16)] + pmulhrsw m4, m5, m0 + pmulhrsw m5, m1 + mova m6, [o(pw_16384)] + REPX {pmulhrsw x, m6 }, m4, m5 +%endif + paddsw m0, m4 + paddsw m1, m5 +%if ARCH_X86_64 + pmulhrsw m4, m9, m2 + pmulhrsw m5, m9, m3 + REPX {pmulhrsw x, m10}, m4, m5 +%else + mova m5, [o(pw_1697x16)] + pmulhrsw m4, m5, m2 + pmulhrsw m5, m3 + REPX {pmulhrsw x, m6 }, m4, m5 +%endif + paddsw m2, m4 + paddsw m3, m5 +%if ARCH_X86_64 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 +%else + psrlw m6, 1 ; pw_8192 + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + pxor m6, m6 +%endif + call m(inv_txfm_add_identity_identity_8x32_16bpc).main_zero + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_identity_identity_32x16_16bpc, 4, 7, 11, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%else + mova m8, [o(pw_2896x8)] + mova m9, [o(pw_1697x16)] + mova m10, [o(pw_2048)] +%endif + mova m7, [o(pixel_10bpc_max)] + lea r4, [strideq*3] + pxor m6, m6 + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*1] + call .main + sub eobd, 107 ; eob < 143 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*2] + call .main + sub eobd, 128 ; eob < 271 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*3] + call .main + sub eobd, 128 ; eob < 399 + jl .ret + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+64*0] + packssdw m0, [cq+64*1] + mova m1, [cq+64*2] + packssdw m1, [cq+64*3] + mova m2, [cq+64*4] + packssdw m2, [cq+64*5] + mova m3, [cq+64*6] + packssdw m3, [cq+64*7] +%if ARCH_X86_64 + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 +%else + mova m6, [o(pw_2896x8)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 +%endif + REPX {paddsw x, x }, m0, m1, m2, m3 +%if ARCH_X86_64 + pmulhrsw m4, m9, m0 + pmulhrsw m5, m9, m1 +%else + mova m6, [o(pw_1697x16)] + pmulhrsw m4, m6, m0 + pmulhrsw m5, m6, m1 +%endif + REPX {paddsw x, x }, m0, m1 + paddsw m0, m4 + paddsw m1, m5 +%if ARCH_X86_64 + pmulhrsw m4, m9, m2 + pmulhrsw m5, m9, m3 +%else + pmulhrsw m4, m6, m2 + pmulhrsw m6, m3 +%endif + REPX {paddsw x, x }, m2, m3 + paddsw m2, m4 +%if ARCH_X86_64 + paddsw m3, m5 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3 +%else + paddsw m3, m6 + mova m6, [o(pw_2048)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + pxor m6, m6 +%endif + REPX {mova [cq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_identity_identity_32x32_16bpc, 4, 7, 8, dst, stride, c, eob +%undef cmp +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_8192)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + lea r4, [strideq*3] + mov r5, dstq + call .main ; 0 + cmp eobd, 36 + jl .ret + add cq, 128*8-32 ; 0 1 + lea dstq, [r5+16] ; 1 + call .main + call .main2 + cmp eobd, 136 + jl .ret + add cq, 128*16-64 ; 0 1 2 + lea dstq, [r5+16*2] ; 1 2 + call .main ; 2 + call .main2 + call .main2 + cmp eobd, 300 + jl .ret + add cq, 128*24-96 ; 0 1 2 3 + add r5, 16*3 ; 1 2 3 + mov dstq, r5 ; 2 3 + call .main ; 3 + call .main2 + call .main2 + call .main2 + cmp eobd, 535 + jl .ret + add cq, 128*24-96 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + mov r5, dstq ; 2 3 4 + call .main ; 3 4 + call .main2 + call .main2 + cmp eobd, 755 + jl .ret + add cq, 128*16-64 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + mov r5, dstq ; 2 3 4 5 + call .main ; 3 4 5 + call .main2 + cmp eobd, 911 + jl .ret + add cq, 128*8-32 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + call .main ; 2 3 4 5 +.ret: ; 3 4 5 6 + RET +ALIGN function_align +.main2: + sub cq, 128*8 + sub dstq, 16 +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main_zero + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_dct_dct_8x32_16bpc, 4, 7, 15, 0-36*16, \ + dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%define base $$ + DECLARE_REG_TMP 0, 4 +%else + lea r6, [tbl_Nx32_odd_offset] +%define base tbl_Nx32_odd_offset + DECLARE_REG_TMP 4, 7 +%if WIN64 + mov [rsp+gprsize*1+35*16], r7 +%endif +%endif +%define o2(x) r6-base+x + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + mov [rsp+gprsize*1+35*16], r0 +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_8x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+ 3*16+r5*8], m0 + mova [rsp+11*16+r5*8], m0 + mova [rsp+ 3*16+t0*8], m0 + mova [rsp+ 3*16+t1*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_8x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+35*16], eobd + mov r3, rsp +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+0*128+r5*8] + mova m1, [cq+1*128+r5*8] + mova m2, [cq+2*128+r5*8] + mova m3, [cq+3*128+r5*8] + mova m4, [cq+4*128+r5*8] + mova m5, [cq+5*128+r5*8] + mova m6, [cq+6*128+r5*8] + mova m7, [cq+7*128+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + mova m1, [o(pd_2)] + REPX {paddd x, m1}, m0, m6, m5, m3 + call m(idct_8x4_internal_16bpc).round + REPX {psrad x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [r3+ 3*16+r5*8], m0 + mova [r3+11*16+r5*8], m2 + mova [r3+ 3*16+t1*8], m1 + mova [r3+ 3*16+t0*8], m3 + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + sub r5d, 2 + jge .loop_pass1 + + ; pass 2 code starts here + ; m0 is already loaded from last iteration of first pass +%if ARCH_X86_32 + mov r0, [rsp+gprsize*1+35*16] +%endif + mov eobd, [rsp+gprsize*0+35*16] + cmp eobd, 43 + jl .load_veryfast + cmp eobd, 107 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: + call .pass2 +%if WIN64 + mov r7, [rsp+gprsize*1+35*16] +%endif + RET + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m1, [rsp+gprsize+16* 4] + mova m2, [rsp+gprsize+16* 5] + mova m3, [rsp+gprsize+16* 6] + mova m4, [rsp+gprsize+16* 7] + mova m5, [rsp+gprsize+16* 8] + mova m6, [rsp+gprsize+16* 9] + mova m7, [rsp+gprsize+16*10] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 4*16], m1 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 6*16], m3 + mova [rsp+gprsize+ 7*16], m4 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova m0, [rsp+gprsize+11*16] + mova m1, [rsp+gprsize+12*16] + mova m2, [rsp+gprsize+13*16] + mova m3, [rsp+gprsize+14*16] + mova m4, [rsp+gprsize+15*16] + mova m5, [rsp+gprsize+16*16] + mova m6, [rsp+gprsize+17*16] + mova m7, [rsp+gprsize+18*16] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + mova m7, [rsp+gprsize+ 0*16] + mova [rsp+gprsize+11*16], m0 + mova [rsp+gprsize+12*16], m1 + mova [rsp+gprsize+13*16], m2 + mova [rsp+gprsize+14*16], m3 + mova [rsp+gprsize+15*16], m4 + mova [rsp+gprsize+16*16], m5 + mova [rsp+gprsize+17*16], m6 + mova [rsp+gprsize+18*16], m7 + call r4 +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+11*16] + mova m1, [rsp+gprsize+12*16] + mova m2, [rsp+gprsize+13*16] + mova m3, [rsp+gprsize+14*16] + mova m4, [rsp+gprsize+15*16] + mova m5, [rsp+gprsize+16*16] + mova m6, [rsp+gprsize+17*16] + mova m7, [rsp+gprsize+18*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+19*16] + mova m1, [rsp+gprsize+20*16] + mova m2, [rsp+gprsize+21*16] + mova m3, [rsp+gprsize+22*16] + mova m4, [rsp+gprsize+23*16] + mova m5, [rsp+gprsize+24*16] + mova m6, [rsp+gprsize+25*16] + mova m7, [rsp+gprsize+26*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+27*16] + mova m1, [rsp+gprsize+28*16] + mova m2, [rsp+gprsize+29*16] + mova m3, [rsp+gprsize+30*16] + mova m4, [rsp+gprsize+31*16] + mova m5, [rsp+gprsize+32*16] + mova m6, [rsp+gprsize+33*16] + mova m7, [rsp+gprsize+34*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + ret +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 8 + add r5d, 640 + sar r5d, 10 + add rsp, (31+2*ARCH_X86_64)*16 + jmp m(inv_txfm_add_dct_dct_8x8_16bpc).end2 + +cglobal inv_txfm_add_dct_dct_16x32_16bpc, 4, 7, 16, 0-77*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + mov [rsp+gprsize*1+76*16], r0 +%elif WIN64 + mov [rsp+gprsize*1+76*16], r7 +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m0 + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m0 + mova [rsp+44*16+r5*8], m0 + mova [rsp+52*16+r5*8], m0 + mova [rsp+44*16+t0*8], m0 + mova [rsp+44*16+t1*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+76*16], eobd + mov r3, rsp +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*128+r5*8] + mova m1, [cq+ 3*128+r5*8] + mova m2, [cq+ 5*128+r5*8] + mova m3, [cq+ 7*128+r5*8] + mova m4, [cq+ 9*128+r5*8] + mova m5, [cq+11*128+r5*8] + mova m6, [cq+13*128+r5*8] + mova m7, [cq+15*128+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*128+r5*8] + mova m1, [cq+ 2*128+r5*8] + mova m2, [cq+ 4*128+r5*8] + mova m3, [cq+ 6*128+r5*8] + mova m4, [cq+ 8*128+r5*8] + mova m5, [cq+10*128+r5*8] + mova m6, [cq+12*128+r5*8] + mova m7, [cq+14*128+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 +%if ARCH_X86_64 + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m2 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t0*8], m3 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+44*16+r5*8], m8 + mova [rsp+52*16+r5*8], m10 + mova [rsp+44*16+t1*8], m9 + mova [rsp+44*16+t0*8], m11 +%else + mova [rsp+44*16+r5*8], m0 + mova [rsp+52*16+r5*8], m2 + mova [rsp+44*16+t1*8], m1 + mova [rsp+44*16+t0*8], m3 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m4, [r3+10*16] + mova m6, [r3+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m2 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t0*8], m3 +%endif + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + add rsp, 9*16 +%if ARCH_X86_64 + mov r6, dstq +%else + mov dstq, [rsp+gprsize*1+67*16] +%endif + mov eobd, [rsp+gprsize*0+67*16] + cmp eobd, 44 + jl .load_veryfast + cmp eobd, 151 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+32] + mov r7, -4 +%else + lea r2, [rsp+67*16] + mov dword [r2+0*gprsize], 2 +%endif + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [rsp+16* 3] +.loop_pass2_entry: +%if ARCH_X86_32 + mov dstq, [r2+1*gprsize] +%endif + call m(inv_txfm_add_dct_dct_8x32_16bpc).pass2 + add rsp, 32*16 +%if ARCH_X86_64 + add r7, 2 + lea dstq, [r2+r7*8] + jl .loop_pass2 +%if WIN64 + mov r7, [rsp+gprsize*1+3*16] +%endif +%else + add dword [r2+1*gprsize], 16 + dec dword [r2+0*gprsize] + jg .loop_pass2 +%endif +%assign stack_size (stack_size-73*16) +%if STACK_ALIGNMENT >= 16 +%assign stack_size_padded (stack_size_padded-73*16) +%assign stack_offset (stack_offset-73*16) +%else +%xdefine rstkm [rsp + stack_size] +%endif + RET +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 32 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + add rsp, (65+4*ARCH_X86_64)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly + +cglobal inv_txfm_add_dct_dct_32x8_16bpc, 4, 7, 16, 0-(24+8*ARCH_X86_32)*16, \ + dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + add r5d, r5d + + ; actual first pass after skipping all-zero data +.loop_pass1: + mova m0, [cq+32* 1+r5*8] + mova m1, [cq+32* 7+r5*8] + mova m2, [cq+32* 9+r5*8] + mova m3, [cq+32*15+r5*8] + mova m4, [cq+32*17+r5*8] + mova m5, [cq+32*23+r5*8] + mova m6, [cq+32*25+r5*8] + mova m7, [cq+32*31+r5*8] +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mov r3, rsp + call .main_oddhalf_part1 + mova m0, [cq+32* 3+r5*8] + mova m1, [cq+32* 5+r5*8] + mova m2, [cq+32*11+r5*8] + mova m3, [cq+32*13+r5*8] + mova m4, [cq+32*19+r5*8] + mova m5, [cq+32*21+r5*8] + mova m6, [cq+32*27+r5*8] + mova m7, [cq+32*29+r5*8] + call .main_oddhalf_part2 + mova m0, [cq+32* 2+r5*8] + mova m1, [cq+32* 6+r5*8] + mova m2, [cq+32*10+r5*8] + mova m3, [cq+32*14+r5*8] + mova m4, [cq+32*18+r5*8] + mova m5, [cq+32*22+r5*8] + mova m6, [cq+32*26+r5*8] + mova m7, [cq+32*30+r5*8] + add r3, 16*(16+4*ARCH_X86_32) + call m(idct_16x4_internal_16bpc).main_oddhalf + mova m0, [cq+32* 0+r5*8] + mova m1, [cq+32* 4+r5*8] + mova m2, [cq+32* 8+r5*8] + mova m3, [cq+32*12+r5*8] + mova m4, [cq+32*16+r5*8] + mova m5, [cq+32*20+r5*8] + mova m6, [cq+32*24+r5*8] + mova m7, [cq+32*28+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call .round_dct32 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32* 8+r5*8], m8 + mova [cq+32* 9+r5*8], m9 + mova [cq+32*10+r5*8], m10 + mova [cq+32*11+r5*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32* 4+r5*8], m8 + mova [cq+32* 5+r5*8], m9 + mova [cq+32* 6+r5*8], m10 + mova [cq+32* 7+r5*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32*12+r5*8], m8 + mova [cq+32*13+r5*8], m9 + mova [cq+32*14+r5*8], m10 + mova [cq+32*15+r5*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32* 4+r5*8], m0 + mova [cq+32* 5+r5*8], m1 + mova [cq+32* 6+r5*8], m2 + mova [cq+32* 7+r5*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32* 8+r5*8], m0 + mova [cq+32* 9+r5*8], m1 + mova [cq+32*10+r5*8], m2 + mova [cq+32*11+r5*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32*12+r5*8], m0 + mova [cq+32*13+r5*8], m1 + mova [cq+32*14+r5*8], m2 + mova [cq+32*15+r5*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + pxor m7, m7 + ; clear lower half of [cq] + REPX {mova [cq+x*32+r5*8], m7}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + test r5d, r5d + jz .end_pass1 + mova [cq+32* 0+r5*8], m0 + mova [cq+32* 1+r5*8], m1 + mova [cq+32* 2+r5*8], m2 + mova [cq+32* 3+r5*8], m3 + sub r5d, 2 + jmp .loop_pass1 +.end_pass1: + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + mov r4d, 4 + call m(idct_16x8_internal_16bpc).pass2_main + RET + +.main_oddhalf_part1_fast: ; lower half zero + pmulld m7, m0, [o(pd_4091)] + pmulld m0, [o(pd_201)] + pmulld m4, m3, [o(pd_m2751)] +%if ARCH_X86_32 + pmulld m3, [o(pd_3035)] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m7 + REPX {psrad x, 12}, m0, m7 + mova [r3+3*16], m7 + mova m7, m3 + mova m3, m5 +%else + pmulld m3, [o(pd_3035)] +%endif + pmulld m6, m1, [o(pd_m1380)] + pmulld m1, [o(pd_3857)] + pmulld m5, m2, [o(pd_3703)] + pmulld m2, [o(pd_1751)] + jmp .main_oddhalf_part1_fast2 +.main_oddhalf_part1: ; in1, in7, in9, in15, in17, in23, in25, in31 +%if ARCH_X86_64 + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3035, 2751 ; t17a, t30a +.main_oddhalf_part1_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t17 + paddd m0, m4 ; t16 + psubd m4, m6, m2 ; t18 + paddd m6, m2 ; t19 + psubd m2, m1, m5 ; t29 + paddd m1, m5 ; t28 + psubd m5, m7, m3 ; t30 + paddd m7, m3 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + mova m15, [o(pd_4017)] + mova m10, [o(pd_799)] + ITX_MULSUB_2D 5, 8, 3, 9, _, 11, 10, 15 ; t17a, t30a + ITX_MULSUB_2D 2, 4, 3, 9, _, 11, 10, 15, 4 ; t29a, t18a + psubd m3, m0, m6 ; t19a + paddd m0, m6 ; t16a + psubd m6, m7, m1 ; t28a + paddd m7, m1 ; t31a + psubd m1, m5, m4 ; t18 + paddd m5, m4 ; t17 + psubd m4, m8, m2 ; t29 + paddd m8, m2 ; t30 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15 ; t18a, t29a + ITX_MULSUB_2D 6, 3, 2, 9, _, 11, 10, 15 ; t19, t28 + mova [r3+16*0], m0 + mova [r3+16*1], m5 + mova [r3+16*2], m4 + mova [r3+16*3], m6 + mova [r3+16*4], m3 + mova [r3+16*5], m1 + mova [r3+16*6], m8 + mova [r3+16*7], m7 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 0, 7, 2, 4, 5, 3, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 6, 1, 2, 4, 5, _, 3857, 1380 ; t19a, t28a + mova m4, [r3+2*16] + mova m5, [r3+3*16] + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m2, [r3+0*16] + mova m7, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + ITX_MULSUB_2D 2, 5, 0, 1, 6, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 4, 7, 0, 1, 6, _, 3035, 2751 ; t17a, t30a + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m6, [r3+2*16] +.main_oddhalf_part1_fast2: + REPX {paddd x, m3}, m1, m2, m4, m5, m6, m7 + REPX {psrad x, 12}, m1, m2, m4, m5, m6, m7 + psubd m3, m0, m4 ; t17 + mova [r3+0*16], m3 + mova m3, [r3+3*16] + paddd m0, m4 ; t16 + psubd m4, m6, m2 ; t18 + paddd m6, m2 ; t19 + psubd m2, m1, m5 ; t29 + paddd m1, m5 ; t28 + psubd m5, m3, m7 ; t30 + paddd m7, m3 ; t31 + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pmaxsd m3, [r3+0*16] + mova [r3+0*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pminsd m3, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 5, 3, 1, 6, 7, 0, 799, 4017 ; t17a, t30a + ITX_MULSUB_2D 2, 4, 1, 6, _, 0, 7, 4017, 4 ; t29a, t18a + psubd m1, m5, m4 ; t18 + paddd m5, m4 ; t17 + psubd m4, m3, m2 ; t29 + paddd m3, m2 ; t30 + mova m0, [r3+0*16] + mova m2, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + mova [r3+0*16], m3 + psubd m3, m0, m6 ; t19a + paddd m0, m6 ; t16a + psubd m6, m7, m2 ; t28a + paddd m7, m2 ; t31a + mova m2, [o(clip_18b_min)] + REPX {pmaxsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pmaxsd m2, [r3+0*16] + mova [r3+0*16], m2 + mova m2, [o(clip_18b_max)] + REPX {pminsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pminsd m2, [r3+0*16] + mova [r3+16*0], m0 + mova [r3+16*1], m5 + mova [r3+16*6], m2 + mova [r3+16*7], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 4, 1, 0, 5, 2, 7, 1567, 3784 ; t18a, t29a + ITX_MULSUB_2D 6, 3, 0, 5, 2, 7, 2, 3784 ; t19, t28 + mova [r3+16*2], m4 + mova [r3+16*3], m6 + mova [r3+16*4], m3 + mova [r3+16*5], m1 +%endif + ret +.main_oddhalf_part2_fast: ; lower half zero + pmulld m7, m0, [o(pd_m601)] + pmulld m0, [o(pd_4052)] + pmulld m4, m3, [o(pd_3290)] +%if ARCH_X86_32 + pmulld m3, [o(pd_2440)] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m7 + REPX {psrad x, 12}, m0, m7 + mova [r3+11*16], m7 + mova m7, m3 + mova m3, m5 +%else + pmulld m3, [o(pd_2440)] +%endif + pmulld m6, m1, [o(pd_3973)] + pmulld m1, [o(pd_995)] + pmulld m5, m2, [o(pd_m2106)] + pmulld m2, [o(pd_3513)] + jmp .main_oddhalf_part2_fast2 +.main_oddhalf_part2: ; in3, in5, in11, in13, in19, in21, in27, in29 +%if ARCH_X86_64 + ITX_MULSUB_2D 7, 0, 8, 9, 10, _, 4052, 601 ; t23a, t24a + ITX_MULSUB_2D 1, 6, 8, 9, 10, _, 995, 3973 ; t20a, t27a + ITX_MULSUB_2D 5, 2, 8, 9, 10, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 3, 4, 8, 9, 10, _, 2440, 3290 ; t22a, t25a +.main_oddhalf_part2_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t25 + paddd m0, m4 ; t24 + psubd m4, m6, m2 ; t26 + paddd m6, m2 ; t27 + psubd m2, m1, m5 ; t21 + paddd m1, m5 ; t20 + psubd m5, m7, m3 ; t22 + paddd m7, m3 ; t23 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + mova m15, [o(pd_2276)] + mova m10, [o(pd_3406)] + ITX_MULSUB_2D 4, 2, 3, 9, _, 11, 10, 15 ; t21a, t26a + ITX_MULSUB_2D 8, 5, 3, 9, _, 11, 10, 15, 4 ; t25a, t22a + psubd m3, m0, m6 ; t27a + paddd m0, m6 ; t24a + psubd m6, m7, m1 ; t20a + paddd m7, m1 ; t23a + psubd m1, m5, m4 ; t21 + paddd m5, m4 ; t22 + psubd m4, m8, m2 ; t26 + paddd m8, m2 ; t25 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15, 4 ; t26a, t21a + ITX_MULSUB_2D 3, 6, 2, 9, _, 11, 10, 15, 4 ; t27, t20 + mova m9, [r3+16*0] ; t16a + mova m10, [r3+16*1] ; t17 + psubd m2, m9, m7 ; t23 + paddd m9, m7 ; t16 + psubd m7, m10, m5 ; t22a + paddd m10, m5 ; t17a + REPX {pmaxsd x, m12}, m9, m10, m2, m7 + REPX {pminsd x, m13}, m9, m10, m2, m7 + mova [r3+16*0], m9 + mova [r3+16*1], m10 + mova m9, [r3+16*2] ; t18a + mova m10, [r3+16*3] ; t19 + psubd m5, m9, m1 ; t21 + paddd m9, m1 ; t18 + psubd m1, m10, m6 ; t20a + paddd m10, m6 ; t19a + REPX {pmaxsd x, m12}, m9, m10, m5, m1 + REPX {pminsd x, m13}, m9, m10, m5, m1 + mova [r3+16*2], m9 + mova [r3+16*3], m10 + mova m9, [r3+16*4] ; t28 + mova m10, [r3+16*5] ; t29a + psubd m6, m9, m3 ; t27a + paddd m9, m3 ; t28a + psubd m3, m10, m4 ; t26 + paddd m10, m4 ; t29 + REPX {pmaxsd x, m12}, m9, m10, m6, m3 + REPX {pminsd x, m13}, m9, m10, m6, m3 + REPX {pmulld x, m14}, m6, m3, m1, m5 + paddd m6, m11 + paddd m3, m11 + psubd m4, m6, m1 ; t20 + paddd m6, m1 ; t27 + psubd m1, m3, m5 ; t21a + paddd m3, m5 ; t26a + REPX {psrad x, 12 }, m4, m1, m3, m6 + mova [r3+16*4], m4 + mova [r3+16*5], m1 + mova m4, [r3+16*6] ; t30 + mova m1, [r3+16*7] ; t31a + psubd m5, m4, m8 ; t25a + paddd m4, m8 ; t30a + psubd m8, m1, m0 ; t24 + paddd m1, m0 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m1 + REPX {pminsd x, m13}, m8, m5, m4, m1 + REPX {pmulld x, m14}, m5, m8, m7, m2 + paddd m5, m11 + paddd m8, m11 + psubd m0, m5, m7 ; t22 + paddd m5, m7 ; t25 + psubd m7, m8, m2 ; t23a + paddd m2, m8 ; t24a + REPX {psrad x, 12 }, m0, m7, m2, m5 + mova [r3+16*6], m0 + mova [r3+16*7], m7 + mova [r3+16*8], m2 + mova [r3+16*9], m5 + mova [r3+16*10], m3 + mova [r3+16*11], m6 + mova [r3+16*12], m9 + mova [r3+16*13], m10 + mova [r3+16*14], m4 + mova [r3+16*15], m1 +%else + mova [r3+ 8*16], m2 + mova [r3+ 9*16], m3 + mova [r3+10*16], m4 + mova [r3+11*16], m5 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 7, 0, 2, 4, 5, 3, 4052, 601 ; t23a, t24a + ITX_MULSUB_2D 1, 6, 2, 4, 5, _, 995, 3973 ; t20a, t27a + mova m2, [r3+ 8*16] + mova m4, [r3+10*16] + mova m5, [r3+11*16] + mova [r3+ 8*16], m0 + mova [r3+10*16], m6 + mova [r3+11*16], m7 + mova m7, [r3+ 9*16] + mova [r3+ 9*16], m1 + ITX_MULSUB_2D 5, 2, 0, 6, 1, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 7, 4, 0, 6, 1, _, 2440, 3290 ; t22a, t25a + mova m0, [r3+ 8*16] + mova m1, [r3+ 9*16] + mova m6, [r3+10*16] +.main_oddhalf_part2_fast2: + REPX {paddd x, m3}, m1, m2, m7, m4, m5, m6 + REPX {psrad x, 12}, m1, m2, m7, m4, m5, m6 + psubd m3, m0, m4 ; t25 + mova [r3+ 8*16], m3 + mova m3, [r3+11*16] + paddd m0, m4 ; t24 + psubd m4, m6, m2 ; t26 + paddd m6, m2 ; t27 + psubd m2, m1, m5 ; t21 + paddd m1, m5 ; t20 + psubd m5, m3, m7 ; t22 + paddd m7, m3 ; t23 + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pmaxsd m3, [r3+ 8*16] + mova [r3+ 8*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pminsd m3, [r3+ 8*16] + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m1 + mova [r3+10*16], m6 + mova [r3+11*16], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 4, 2, 0, 1, 6, 7, 3406, 2276 ; t21a, t26a + ITX_MULSUB_2D 3, 5, 0, 1, _, 7, 6, 2276, 4 ; t25a, t22a + psubd m1, m5, m4 ; t21 + paddd m5, m4 ; t22 + psubd m4, m3, m2 ; t26 + paddd m3, m2 ; t25 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m6, [r3+10*16] + mova m7, [r3+11*16] + mova [r3+ 8*16], m3 + psubd m3, m0, m6 ; t27a + paddd m0, m6 ; t24a + psubd m6, m7, m2 ; t20a + paddd m7, m2 ; t23a + mova m2, [o(clip_18b_min)] + REPX {pmaxsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pmaxsd m2, [r3+ 8*16] + mova [r3+ 8*16], m2 + mova m2, [o(clip_18b_max)] + REPX {pminsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pminsd m2, [r3+ 8*16] + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m2 + mova [r3+14*16], m5 + mova [r3+15*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 4, 1, 2, 5, 7, 0, 1567, 3784, 4 ; t26a, t21a + ITX_MULSUB_2D 3, 6, 2, 5, _, 0, 7, 3784, 4 ; t27, t20 + mova [r3+10*16], m3 + mova m0, [o(clip_18b_min)] + mova m2, [o(clip_18b_max)] + mova m5, [r3+16*2] ; t18a + mova m7, [r3+16*3] ; t19 + psubd m3, m5, m1 ; t21 + paddd m5, m1 ; t18 + psubd m1, m7, m6 ; t20a + paddd m7, m6 ; t19a + REPX {pmaxsd x, m0}, m5, m7, m3, m1 + REPX {pminsd x, m2}, m5, m7, m3, m1 + mova [r3+16*2], m5 + mova [r3+16*3], m7 + mova [r3+11*16], m3 + mova m3, [r3+10*16] + mova m5, [r3+16*4] ; t28 + mova m7, [r3+16*5] ; t29a + psubd m6, m5, m3 ; t27a + paddd m5, m3 ; t28a + psubd m3, m7, m4 ; t26 + paddd m7, m4 ; t29 + REPX {pmaxsd x, m0}, m5, m7, m6, m3 + REPX {pminsd x, m2}, m5, m7, m6, m3 + mova [r3+16*12], m5 + mova [r3+16*13], m7 + mova m5, [o(pd_2048)] + mova m7, [o(pd_2896)] + mova m4, [r3+11*16] + REPX {pmulld x, m7}, m6, m3, m1, m4 + paddd m6, m5 + paddd m3, m5 + psubd m5, m6, m1 ; t20 + paddd m6, m1 ; t27 + psubd m1, m3, m4 ; t21a + paddd m3, m4 ; t26a + REPX {psrad x, 12}, m5, m1, m3, m6 + mova [r3+16*4], m5 + mova [r3+16*5], m1 + mova [r3+16*10], m3 + mova [r3+16*11], m6 + + mova m5, [r3+14*16] + mova m6, [r3+15*16] + mova m3, [r3+16*0] ; t16a + mova m4, [r3+16*1] ; t17 + psubd m1, m3, m6 ; t23 + paddd m3, m6 ; t16 + psubd m6, m4, m5 ; t22a + paddd m4, m5 ; t17a + REPX {pmaxsd x, m0}, m3, m4, m1, m6 + REPX {pminsd x, m2}, m3, m4, m1, m6 + mova [r3+16*0], m3 + mova [r3+16*1], m4 + mova m5, [r3+ 8*16] + mova m3, [r3+ 9*16] + mova [r3+ 8*16], m1 + mova [r3+ 9*16], m6 + mova m4, [r3+16*6] ; t30 + mova m1, [r3+16*7] ; t31a + psubd m6, m1, m5 ; t24 + paddd m1, m5 ; t31 + psubd m5, m4, m3 ; t25a + paddd m4, m3 ; t30a + REPX {pmaxsd x, m0}, m6, m5, m4, m1 + REPX {pminsd x, m2}, m6, m5, m4, m1 + mova [r3+16*14], m4 + mova [r3+16*15], m1 + mova m4, [o(pd_2048)] + mova m1, [r3+ 9*16] + mova m2, [r3+ 8*16] + REPX {pmulld x, m7}, m5, m6, m1, m2 + paddd m5, m4 + paddd m6, m4 + psubd m0, m5, m1 ; t22 + paddd m5, m1 ; t25 + psubd m1, m6, m2 ; t23a + paddd m2, m6 ; t24a + REPX {psrad x, 12}, m0, m1, m2, m5 + mova [r3+16*6], m0 + mova [r3+16*7], m1 + mova [r3+16*8], m2 + mova [r3+16*9], m5 +%endif + ret + + ; final sumsub for idct16 as well as idct32, plus final downshift +%macro IDCT32_END 6 ; in/out1, out2-4, tmp, shift, idx + mova m%4, [r3+16*(23-%1)] + pmaxsd m%1, m12 + pminsd m%1, m13 + psubd m%3, m%1, m%4 ; idct16 out15 - n + paddd m%1, m%4 ; idct16 out0 + n + pmaxsd m%1, m12 + pmaxsd m%3, m12 + pminsd m%1, m13 + pminsd m%3, m13 + paddd m%1, m11 + paddd m%3, m11 + mova m%5, [r3+16*( 0+%1)] + mova m%2, [r3+16*(15-%1)] + psubd m%4, m%1, m%2 ; out31 - n + paddd m%1, m%2 ; out0 + n + paddd m%2, m%3, m%5 ; out15 - n + psubd m%3, m%5 ; out16 + n + REPX {psrad x, %6}, m%1, m%3, m%2, m%4 +%endmacro + +.round_dct32: +%if ARCH_X86_64 + psrld m11, 10 ; pd_2 + IDCT32_END 0, 15, 8, 9, 10, 2 ; 0 15 16 31 + mova [r3+ 0*16], m6 + mova [r3+23*16], m7 + IDCT32_END 1, 14, 6, 7, 10, 2 ; 1 14 17 30 + packssdw m0, m1 ; 0 1 + packssdw m14, m15 ; 14 15 + packssdw m8, m6 ; 16 17 + packssdw m7, m9 ; 30 31 + mova [r3+16*15], m14 + mova [r3+16*14], m7 + IDCT32_END 2, 15, 10, 7, 6, 2 ; 2 13 18 29 + IDCT32_END 3, 14, 1, 9, 6, 2 ; 3 12 19 28 + packssdw m2, m3 ; 2 3 + packssdw m14, m15 ; 12 13 + packssdw m10, m1 ; 18 19 + packssdw m9, m7 ; 28 29 + mova [r3+16*13], m14 + mova [r3+16*12], m9 + IDCT32_END 4, 15, 1, 7, 6, 2 ; 4 11 20 27 + IDCT32_END 5, 14, 3, 9, 6, 2 ; 5 10 21 26 + packssdw m4, m5 ; 4 5 + packssdw m14, m15 ; 10 11 + packssdw m1, m3 ; 20 21 + packssdw m9, m7 ; 26 27 + mova [r3+16*11], m14 + mova [r3+16*10], m9 + mova m6, [r3+ 0*16] + mova m7, [r3+23*16] + IDCT32_END 6, 15, 14, 5, 3, 2 ; 6 9 22 25 + IDCT32_END 7, 11, 3, 9, 13, 2 ; 7 8 23 24 + packssdw m6, m7 ; 6 7 + packssdw m11, m15 ; 8 9 + packssdw m14, m3 ; 22 23 + packssdw m9, m5 ; 24 25 + mova [r3+16*9], m11 + mova [r3+16*8], m9 + mova m12, m1 + ret +%else + mova [r3+16*16], m0 + mova [r3+17*16], m1 + mova [r3+18*16], m2 + mova [r3+19*16], m3 + mova [r3+20*16], m4 + mova [r3+21*16], m5 + mova [r3+22*16], m6 + mova [r3+23*16], m7 + mova m1, [o(pd_2)] + mova m2, [o(clip_18b_min)] + mova m3, [o(clip_18b_max)] + + mov r4, 15*16 +.loop_dct32_end: + mova m0, [r3+16*16] + mova m6, [r3+16*24] + pmaxsd m0, m2 + pminsd m0, m3 + psubd m5, m0, m6 ; idct16 out15 - n + paddd m0, m6 ; idct16 out0 + n + pmaxsd m0, m2 + pmaxsd m5, m2 + pminsd m0, m3 + pminsd m5, m3 + paddd m0, m1 + paddd m5, m1 + mova m7, [r3] + mova m4, [r3+r4] + psubd m6, m0, m4 ; out31 - n + paddd m0, m4 ; out0 + n + paddd m4, m5, m7 ; out15 - n + psubd m5, m7 ; out16 + n + REPX {psrad x, 2}, m0, m5, m4, m6 + mova [r3], m0 + mova [r3+r4], m4 + mova [r3+16*16], m5 + mova [r3+24*16], m6 + add r3, 16 + sub r4, 32 + jg .loop_dct32_end + ret +%endif + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 8 +.dconly1: + add r5d, 640 + sar r5d, 10 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 +.dconly_loop: + mova m1, [dstq+16*0] + mova m2, [dstq+16*1] + mova m3, [dstq+16*2] + mova m4, [dstq+16*3] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m2 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, strideq + dec r3d + jg .dconly_loop + RET + +cglobal inv_txfm_add_dct_dct_32x16_16bpc, 4, 7, 16, 0-(24+8*ARCH_X86_32)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp + mov r5d, 8 +.zero_loop: + sub r5d, 2 + cmp eobw, word [o2(tbl_32x16_2d)+r5] + jl .zero_loop + + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+64* 1+r5*8] + mova m1, [cq+64* 7+r5*8] + mova m2, [cq+64* 9+r5*8] + mova m3, [cq+64*15+r5*8] + mova m4, [cq+64*17+r5*8] + mova m5, [cq+64*23+r5*8] + mova m6, [cq+64*25+r5*8] + mova m7, [cq+64*31+r5*8] + mov r3, rsp + call m(idct_8x4_internal_16bpc).rect2_mul + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + + mova m0, [cq+64* 3+r5*8] + mova m1, [cq+64* 5+r5*8] + mova m2, [cq+64*11+r5*8] + mova m3, [cq+64*13+r5*8] + mova m4, [cq+64*19+r5*8] + mova m5, [cq+64*21+r5*8] + mova m6, [cq+64*27+r5*8] + mova m7, [cq+64*29+r5*8] +%if ARCH_X86_32 + add r3, 16*8 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 16*8 +%endif + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + add r3, 16*(16+4*ARCH_X86_32) + + mova m0, [cq+64* 2+r5*8] + mova m1, [cq+64* 6+r5*8] + mova m2, [cq+64*10+r5*8] + mova m3, [cq+64*14+r5*8] + mova m4, [cq+64*18+r5*8] + mova m5, [cq+64*22+r5*8] + mova m6, [cq+64*26+r5*8] + mova m7, [cq+64*30+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+64* 0+r5*8] + mova m1, [cq+64* 4+r5*8] + mova m2, [cq+64* 8+r5*8] + mova m3, [cq+64*12+r5*8] + mova m4, [cq+64*16+r5*8] + mova m5, [cq+64*20+r5*8] + mova m6, [cq+64*24+r5*8] + mova m7, [cq+64*28+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call .round_dct32 + +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64* 8+r5*8], m8 + mova [cq+64* 9+r5*8], m9 + mova [cq+64*10+r5*8], m10 + mova [cq+64*11+r5*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64* 4+r5*8], m8 + mova [cq+64* 5+r5*8], m9 + mova [cq+64* 6+r5*8], m10 + mova [cq+64* 7+r5*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64*12+r5*8], m8 + mova [cq+64*13+r5*8], m9 + mova [cq+64*14+r5*8], m10 + mova [cq+64*15+r5*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64* 4+r5*8], m0 + mova [cq+64* 5+r5*8], m1 + mova [cq+64* 6+r5*8], m2 + mova [cq+64* 7+r5*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64* 8+r5*8], m0 + mova [cq+64* 9+r5*8], m1 + mova [cq+64*10+r5*8], m2 + mova [cq+64*11+r5*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64*12+r5*8], m0 + mova [cq+64*13+r5*8], m1 + mova [cq+64*14+r5*8], m2 + mova [cq+64*15+r5*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [cq+64* 0+r5*8], m0 + mova [cq+64* 1+r5*8], m1 + mova [cq+64* 2+r5*8], m2 + mova [cq+64* 3+r5*8], m3 + pxor m0, m0 + REPX {mova [cq+x*64+r5*8], m0}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + call .pass2 + RET + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 4 + jmp m(idct_16x16_internal_16bpc).loop_pass2 + +.round_dct32: +%if ARCH_X86_64 + psrld m11, 11 ; pd_1 + IDCT32_END 0, 15, 8, 9, 10, 1 ; 0 15 16 31 + mova [r3+ 0*16], m6 + mova [r3+23*16], m7 + IDCT32_END 1, 14, 6, 7, 10, 1 ; 1 14 17 30 + packssdw m0, m1 ; 0 1 + packssdw m14, m15 ; 14 15 + packssdw m8, m6 ; 16 17 + packssdw m7, m9 ; 30 31 + mova [r3+16*15], m14 + mova [r3+16*14], m7 + IDCT32_END 2, 15, 10, 7, 6, 1 ; 2 13 18 29 + IDCT32_END 3, 14, 1, 9, 6, 1 ; 3 12 19 28 + packssdw m2, m3 ; 2 3 + packssdw m14, m15 ; 12 13 + packssdw m10, m1 ; 18 19 + packssdw m9, m7 ; 28 29 + mova [r3+16*13], m14 + mova [r3+16*12], m9 + IDCT32_END 4, 15, 1, 7, 6, 1 ; 4 11 20 27 + IDCT32_END 5, 14, 3, 9, 6, 1 ; 5 10 21 26 + packssdw m4, m5 ; 4 5 + packssdw m14, m15 ; 10 11 + packssdw m1, m3 ; 20 21 + packssdw m9, m7 ; 26 27 + mova [r3+16*11], m14 + mova [r3+16*10], m9 + mova m6, [r3+ 0*16] + mova m7, [r3+23*16] + IDCT32_END 6, 15, 14, 5, 3, 1 ; 6 9 22 25 + IDCT32_END 7, 11, 3, 9, 13, 1 ; 7 8 23 24 + packssdw m6, m7 ; 6 7 + packssdw m11, m15 ; 8 9 + packssdw m14, m3 ; 22 23 + packssdw m9, m5 ; 24 25 + mova [r3+16*9], m11 + mova [r3+16*8], m9 + mova m12, m1 + ret +%else + mova [r3+16*16], m0 + mova [r3+17*16], m1 + mova [r3+18*16], m2 + mova [r3+19*16], m3 + mova [r3+20*16], m4 + mova [r3+21*16], m5 + mova [r3+22*16], m6 + mova [r3+23*16], m7 + pcmpeqd m1, m1 ; -1 + mova m2, [o(clip_18b_min)] + mova m3, [o(clip_18b_max)] + + mov r4, 15*16 +.loop_dct32_end: + mova m0, [r3+16*16] + mova m6, [r3+16*24] + psubd m5, m0, m6 ; idct16 out15 - n + paddd m0, m6 ; idct16 out0 + n + pmaxsd m0, m2 + pmaxsd m5, m2 + pminsd m0, m3 + pminsd m5, m3 + psubd m0, m1 + psubd m5, m1 + mova m7, [r3] + mova m4, [r3+r4] + psubd m6, m0, m4 ; out31 - n + paddd m0, m4 ; out0 + n + paddd m4, m5, m7 ; out15 - n + psubd m5, m7 ; out16 + n + REPX {psrad x, 1}, m0, m5, m4, m6 + mova [r3], m0 + mova [r3+r4], m4 + mova [r3+16*16], m5 + mova [r3+24*16], m6 + add r3, 16 + sub r4, 32 + jg .loop_dct32_end + ret +%endif + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 16 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + add r5d, 384 + sar r5d, 9 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_32x32_16bpc, 4, 7, 16, 0-(5*32+1)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%if ARCH_X86_32 + mov [rsp+5*32*16+1*gprsize], dstq +%elif WIN64 + mov [rsp+5*32*16+1*gprsize], r7 +%endif +%undef cmp + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+32*16+r5*8+0*32*16], m0 + mova [rsp+40*16+r5*8+0*32*16], m0 + mova [rsp+32*16+t0*8+0*32*16], m0 + mova [rsp+32*16+t1*8+0*32*16], m0 + mova [rsp+32*16+r5*8+1*32*16], m0 + mova [rsp+40*16+r5*8+1*32*16], m0 + mova [rsp+32*16+t0*8+1*32*16], m0 + mova [rsp+32*16+t1*8+1*32*16], m0 + mova [rsp+32*16+r5*8+2*32*16], m0 + mova [rsp+40*16+r5*8+2*32*16], m0 + mova [rsp+32*16+t0*8+2*32*16], m0 + mova [rsp+32*16+t1*8+2*32*16], m0 + mova [rsp+32*16+r5*8+3*32*16], m0 + mova [rsp+40*16+r5*8+3*32*16], m0 + mova [rsp+32*16+t0*8+3*32*16], m0 + mova [rsp+32*16+t1*8+3*32*16], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+5*32*16], eobd +.loop_pass1: + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128* 7+r5*8] + mova m2, [cq+128* 9+r5*8] + mova m3, [cq+128*15+r5*8] + mova m4, [cq+128*17+r5*8] + mova m5, [cq+128*23+r5*8] + mova m6, [cq+128*25+r5*8] + mova m7, [cq+128*31+r5*8] +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mov r3, rsp + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128* 5+r5*8] + mova m2, [cq+128*11+r5*8] + mova m3, [cq+128*13+r5*8] + mova m4, [cq+128*19+r5*8] + mova m5, [cq+128*21+r5*8] + mova m6, [cq+128*27+r5*8] + mova m7, [cq+128*29+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128* 6+r5*8] + mova m2, [cq+128*10+r5*8] + mova m3, [cq+128*14+r5*8] + mova m4, [cq+128*18+r5*8] + mova m5, [cq+128*22+r5*8] + mova m6, [cq+128*26+r5*8] + mova m7, [cq+128*30+r5*8] + add r3, 16*(16+4*ARCH_X86_32) + call m(idct_16x4_internal_16bpc).main_oddhalf + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 4+r5*8] + mova m2, [cq+128* 8+r5*8] + mova m3, [cq+128*12+r5*8] + mova m4, [cq+128*16+r5*8] + mova m5, [cq+128*20+r5*8] + mova m6, [cq+128*24+r5*8] + mova m7, [cq+128*28+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call m(inv_txfm_add_dct_dct_32x8_16bpc).round_dct32 + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+2*32*16], m8 + mova [rsp+40*16+r5*8+2*32*16], m10 + mova [rsp+32*16+t1*8+2*32*16], m9 + mova [rsp+32*16+t0*8+2*32*16], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+1*32*16], m8 + mova [rsp+40*16+r5*8+1*32*16], m10 + mova [rsp+32*16+t1*8+1*32*16], m9 + mova [rsp+32*16+t0*8+1*32*16], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+3*32*16], m8 + mova [rsp+40*16+r5*8+3*32*16], m10 + mova [rsp+32*16+t1*8+3*32*16], m9 + mova [rsp+32*16+t0*8+3*32*16], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+1*32*16], m0 + mova [rsp+40*16+r5*8+1*32*16], m2 + mova [rsp+32*16+t1*8+1*32*16], m1 + mova [rsp+32*16+t0*8+1*32*16], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+2*32*16], m0 + mova [rsp+40*16+r5*8+2*32*16], m2 + mova [rsp+32*16+t1*8+2*32*16], m1 + mova [rsp+32*16+t0*8+2*32*16], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+3*32*16], m0 + mova [rsp+40*16+r5*8+3*32*16], m2 + mova [rsp+32*16+t1*8+3*32*16], m1 + mova [rsp+32*16+t0*8+3*32*16], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + pxor m7, m7 + ; clear lower half of [cq] + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + mova [rsp+32*16+r5*8+0*32*16], m0 + mova [rsp+40*16+r5*8+0*32*16], m2 + mova [rsp+32*16+t1*8+0*32*16], m1 + mova [rsp+32*16+t0*8+0*32*16], m3 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+5*32*16] + add rsp, 29*16 + cmp eobd, 36 + jl .load_veryfast + cmp eobd, 136 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+64] + mov r7, -8 +%else + lea r2, [rsp+(4*32+3)*16] + mov dword [r2+0*gprsize], 4 +%endif + jmp m(inv_txfm_add_dct_dct_16x32_16bpc).loop_pass2_entry + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 32 + add rsp, (5*32+1-(24+8*ARCH_X86_32))*16 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly1 + +cglobal inv_txfm_add_dct_dct_16x64_16bpc, 4, 7, 16, \ + 0-(12+2*64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0 + mov [rsp+gprsize*1+(64*2+12)*16], r0 + mov [rsp+gprsize*2+(64*2+12)*16], r1 + mov [rsp+gprsize*3+(64*2+12)*16], r2 +%else + DECLARE_REG_TMP 8, 9, 4, 7 + mov [rsp+gprsize*1+(64*2+12)*16], r9 +%if WIN64 + mov [rsp+gprsize*2+(64*2+12)*16], r7 + mov [rsp+gprsize*3+(64*2+12)*16], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m0 + mova [rsp+12*16+t2*8], m0 + mova [rsp+12*16+t3*8], m0 + mova [rsp+76*16+t0*8], m0 + mova [rsp+76*16+t1*8], m0 + mova [rsp+76*16+t2*8], m0 + mova [rsp+76*16+t3*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+(64*2+12)*16], eobd + mov r3, rsp +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 6, 0 + mov r2, [rsp+gprsize*3+(64*2+12)*16] + mov [rsp+gprsize*3+(64*2+12)*16], r6 +%endif +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*128+r5*8] + mova m1, [cq+ 3*128+r5*8] + mova m2, [cq+ 5*128+r5*8] + mova m3, [cq+ 7*128+r5*8] + mova m4, [cq+ 9*128+r5*8] + mova m5, [cq+11*128+r5*8] + mova m6, [cq+13*128+r5*8] + mova m7, [cq+15*128+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*128+r5*8] + mova m1, [cq+ 2*128+r5*8] + mova m2, [cq+ 4*128+r5*8] + mova m3, [cq+ 6*128+r5*8] + mova m4, [cq+ 8*128+r5*8] + mova m5, [cq+10*128+r5*8] + mova m6, [cq+12*128+r5*8] + mova m7, [cq+14*128+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x16_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+76*16+t0*8], m8 + mova [rsp+76*16+t1*8], m9 + mova [rsp+76*16+t2*8], m10 + mova [rsp+76*16+t3*8], m11 +%else + mova [rsp+76*16+t0*8], m0 + mova [rsp+76*16+t1*8], m1 + mova [rsp+76*16+t2*8], m2 + mova [rsp+76*16+t3*8], m3 + mova m0, [rsp+ 8*16] + mova m2, [rsp+ 9*16] + mova m4, [rsp+10*16] + mova m6, [rsp+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t2*8], m2 + mova [rsp+12*16+t3*8], m3 +%if ARCH_X86_32 + mov r6, [rsp+gprsize*3+(64*2+12)*16] +%endif + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + mov eobd, [rsp+gprsize*0+(64*2+12)*16] + cmp eobd, 151 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + add rsp, 9*16 + +%if ARCH_X86_64 + lea r2, [dstq+32] + mov r7, -4 +%else + lea r2, [rsp+(64*2+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 2 +%endif +.loop_pass2: +%if ARCH_X86_32 + mov dstq, [r2+1*gprsize] +%endif + call .pass2 + add rsp, 64*16 +%if ARCH_X86_64 + add r7, 2 + lea dstq, [r2+r7*8] + jl .loop_pass2 +%else + add dword [r2+1*gprsize], 16 + dec dword [r2+0*gprsize] + jg .loop_pass2 +%endif +%assign stack_size (stack_size-(64*2+9)*16) +%if STACK_ALIGNMENT >= 16 +%assign stack_size_padded (stack_size_padded-(64*2+9)*16) +%assign stack_offset (stack_offset-(64*2+9)*16) +%else +%xdefine rstkm [rsp + stack_size] +%endif +%if ARCH_X86_64 + mov r9, [rsp+gprsize*1+3*16] +%if WIN64 + mov r7, [rsp+gprsize*2+3*16] + mov r8, [rsp+gprsize*3+3*16] +%endif +%endif + RET + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [rsp+gprsize+16* 3] + mova m1, [rsp+gprsize+16* 4] + mova m2, [rsp+gprsize+16* 5] + mova m3, [rsp+gprsize+16* 6] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 4*16], m1 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 6*16], m3 + mova [rsp+gprsize+ 7*16], m4 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m0, [rsp+gprsize+16*11] + mova m1, [rsp+gprsize+16*12] + mova m2, [rsp+gprsize+16*13] + mova m3, [rsp+gprsize+16*14] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + mova m7, [rsp+gprsize+ 0*16] + mova [rsp+gprsize+11*16], m0 + mova [rsp+gprsize+12*16], m1 + mova [rsp+gprsize+13*16], m2 + mova [rsp+gprsize+14*16], m3 + mova [rsp+gprsize+15*16], m4 + mova [rsp+gprsize+16*16], m5 + mova [rsp+gprsize+17*16], m6 + mova [rsp+gprsize+18*16], m7 +%if ARCH_X86_64 + call r8 +%else + call [r2+4*gprsize] +%endif + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+10*16], m7 +%if ARCH_X86_64 + call r9 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%else + call [r2+5*gprsize] +%endif + lea r3, [strideq*3] + lea r4, [rsp+gprsize+ 3*16] +%if ARCH_X86_64 + mov r6d, 8 +%else + mov dword [r2+2*gprsize], 8 +%endif +.loop_write: + mova m0, [r4+0*16] + mova m1, [r4+1*16] + mova m2, [r4+2*16] + mova m3, [r4+3*16] + mova m4, [r4+4*16] + mova m5, [r4+5*16] + mova m6, [r4+6*16] + mova m7, [r4+7*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + add r4, 8*16 +%if ARCH_X86_64 + dec r6d +%else + dec dword [r2+2*gprsize] +%endif + jg .loop_write + ret + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 64 + add r5d, 640 + sar r5d, 10 + add rsp, (12+2*64)*16+(4+4*ARCH_X86_32)*gprsize-(8+4*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_32x64_16bpc, 4, 7, 16, \ + 0-(32+4*64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0 + mov [rsp+gprsize*1+(64*4+32)*16], r0 + mov [rsp+gprsize*2+(64*4+32)*16], r1 + mov [rsp+gprsize*3+(64*4+32)*16], r2 +%else + DECLARE_REG_TMP 8, 9, 4, 7 + mov [rsp+gprsize*1+(64*4+32)*16], r9 +%if WIN64 + mov [rsp+gprsize*2+(64*4+32)*16], r7 + mov [rsp+gprsize*3+(64*4+32)*16], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + mova [rsp+ 32*16+t0*8], m0 + mova [rsp+ 32*16+t1*8], m0 + mova [rsp+ 32*16+t2*8], m0 + mova [rsp+ 32*16+t3*8], m0 + mova [rsp+ 96*16+t0*8], m0 + mova [rsp+ 96*16+t1*8], m0 + mova [rsp+ 96*16+t2*8], m0 + mova [rsp+ 96*16+t3*8], m0 + mova [rsp+160*16+t0*8], m0 + mova [rsp+160*16+t1*8], m0 + mova [rsp+160*16+t2*8], m0 + mova [rsp+160*16+t3*8], m0 + mova [rsp+224*16+t0*8], m0 + mova [rsp+224*16+t1*8], m0 + mova [rsp+224*16+t2*8], m0 + mova [rsp+224*16+t3*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+(64*4+32)*16], eobd + mov r3, rsp +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 6, 0 + mov r2, [rsp+gprsize*3+(64*4+32)*16] + mov [rsp+gprsize*3+(64*4+32)*16], r6 +%endif +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128* 7+r5*8] + mova m2, [cq+128* 9+r5*8] + mova m3, [cq+128*15+r5*8] + mova m4, [cq+128*17+r5*8] + mova m5, [cq+128*23+r5*8] + mova m6, [cq+128*25+r5*8] + mova m7, [cq+128*31+r5*8] + mov r3, rsp + call m(idct_8x4_internal_16bpc).rect2_mul + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128* 5+r5*8] + mova m2, [cq+128*11+r5*8] + mova m3, [cq+128*13+r5*8] + mova m4, [cq+128*19+r5*8] + mova m5, [cq+128*21+r5*8] + mova m6, [cq+128*27+r5*8] + mova m7, [cq+128*29+r5*8] +%if ARCH_X86_32 + add r3, 16*8 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 16*8 +%endif + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + add r3, 16*(16+4*ARCH_X86_32) + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128* 6+r5*8] + mova m2, [cq+128*10+r5*8] + mova m3, [cq+128*14+r5*8] + mova m4, [cq+128*18+r5*8] + mova m5, [cq+128*22+r5*8] + mova m6, [cq+128*26+r5*8] + mova m7, [cq+128*30+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 4+r5*8] + mova m2, [cq+128* 8+r5*8] + mova m3, [cq+128*12+r5*8] + mova m4, [cq+128*16+r5*8] + mova m5, [cq+128*20+r5*8] + mova m6, [cq+128*24+r5*8] + mova m7, [cq+128*28+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call m(inv_txfm_add_dct_dct_32x16_16bpc).round_dct32 + + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+160*16+t0*8], m8 + mova [rsp+160*16+t1*8], m9 + mova [rsp+160*16+t2*8], m10 + mova [rsp+160*16+t3*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+ 96*16+t0*8], m8 + mova [rsp+ 96*16+t1*8], m9 + mova [rsp+ 96*16+t2*8], m10 + mova [rsp+ 96*16+t3*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+224*16+t0*8], m8 + mova [rsp+224*16+t1*8], m9 + mova [rsp+224*16+t2*8], m10 + mova [rsp+224*16+t3*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+ 96*16+t0*8], m0 + mova [rsp+ 96*16+t1*8], m1 + mova [rsp+ 96*16+t2*8], m2 + mova [rsp+ 96*16+t3*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+160*16+t0*8], m0 + mova [rsp+160*16+t1*8], m1 + mova [rsp+160*16+t2*8], m2 + mova [rsp+160*16+t3*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+224*16+t0*8], m0 + mova [rsp+224*16+t1*8], m1 + mova [rsp+224*16+t2*8], m2 + mova [rsp+224*16+t3*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [rsp+ 32*16+t0*8], m0 + mova [rsp+ 32*16+t1*8], m1 + mova [rsp+ 32*16+t2*8], m2 + mova [rsp+ 32*16+t3*8], m3 + pxor m0, m0 + REPX {mova [cq+x*128+r5*8], m0}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 +%if ARCH_X86_32 + mov r6, [rsp+gprsize*3+(64*4+32)*16] +%endif + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + mov eobd, [rsp+gprsize*0+(64*4+32)*16] + cmp eobd, 136 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + add rsp, 29*16 + +%if ARCH_X86_64 + lea r2, [dstq+64] + mov r7, -8 +%else + lea r2, [rsp+(64*4+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 4 +%endif + jmp m(inv_txfm_add_dct_dct_16x64_16bpc).loop_pass2 + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 64 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + add r5d, 384 + sar r5d, 9 + add rsp, (32+4*64)*16+(4+4*ARCH_X86_32)*gprsize-(24+8*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_64x16_16bpc, 4, 7, 16, 0-(64+8*ARCH_X86_32)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp + mov r5d, 8 +.zero_loop: + sub r5d, 2 + cmp eobw, word [o2(tbl_32x16_2d)+r5] + jl .zero_loop + + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+64* 1+r5*8] + mova m1, [cq+64*31+r5*8] + mova m2, [cq+64*17+r5*8] + mova m3, [cq+64*15+r5*8] + call .main_part1 + mova m0, [cq+64* 7+r5*8] + mova m1, [cq+64*25+r5*8] + mova m2, [cq+64*23+r5*8] + mova m3, [cq+64* 9+r5*8] + call .main_part1 + mova m0, [cq+64* 5+r5*8] + mova m1, [cq+64*27+r5*8] + mova m2, [cq+64*21+r5*8] + mova m3, [cq+64*11+r5*8] + call .main_part1 + mova m0, [cq+64* 3+r5*8] + mova m1, [cq+64*29+r5*8] + mova m2, [cq+64*19+r5*8] + mova m3, [cq+64*13+r5*8] + call .main_part1 + call .main_part2 + + mova m0, [cq+64* 2+r5*8] + mova m1, [cq+64*14+r5*8] + mova m2, [cq+64*18+r5*8] + mova m3, [cq+64*30+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+64* 6+r5*8] + mova m1, [cq+64*10+r5*8] + mova m2, [cq+64*22+r5*8] + mova m3, [cq+64*26+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+64* 4+r5*8] + mova m1, [cq+64*12+r5*8] + mova m2, [cq+64*20+r5*8] + mova m3, [cq+64*28+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+64* 0+r5*8] + mova m1, [cq+64* 8+r5*8] + mova m2, [cq+64*16+r5*8] + mova m3, [cq+64*24+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 10 ; pd_2 +%else + mova m7, [o(pd_2)] +%endif + call .main_end_loop_start + + lea r3, [rsp+56*16] + lea r4, [cq+r5*8+64*28] + call .shift_transpose + sub r5d, 2 + jge .loop_pass1 + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + call .pass2 + RET + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 8 + jmp m(idct_16x16_internal_16bpc).loop_pass2 + +.main_part1: ; idct64 steps 1-5 + ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a +%if ARCH_X86_64 + movd m7, [r4+4*0] + movd m8, [r4+4*1] + movd m6, [r4+4*2] + movd m9, [r4+4*3] + movd m5, [r4+4*4] + movd m10, [r4+4*5] + movd m4, [r4+4*6] + movd m15, [r4+4*7] + REPX {pshufd x, x, q0000}, m7, m8, m6, m9, m5, m10, m4, m15 + pmulld m7, m0 ; t63a + pmulld m0, m8 ; t32a + pmulld m6, m1 ; t62a + pmulld m1, m9 ; t33a + pmulld m5, m2 ; t61a + pmulld m2, m10 ; t34a + pmulld m4, m3 ; t60a + pmulld m3, m15 ; t35a + movd m10, [r4+4*8] + movd m15, [r4+4*9] + REPX {pshufd x, x, q0000}, m10, m15 + REPX {paddd x, m11}, m7, m0, m6, m1, m5, m2, m4, m3 + REPX {psrad x, 12 }, m0, m1, m7, m6, m2, m3, m5, m4 + psubd m8, m0, m1 ; t33 + paddd m0, m1 ; t32 + psubd m1, m7, m6 ; t62 + paddd m7, m6 ; t63 + psubd m6, m3, m2 ; t34 + paddd m3, m2 ; t35 + psubd m2, m4, m5 ; t61 + paddd m4, m5 ; t60 + REPX {pmaxsd x, m12}, m8, m1, m6, m2 + REPX {pminsd x, m13}, m8, m1, m6, m2 + ITX_MULSUB_2D 1, 8, 5, 9, _, 11, 10, 15 ; t33a, t62a + ITX_MULSUB_2D 2, 6, 5, 9, _, 11, 10, 15, 4 ; t61a, t34a + REPX {pmaxsd x, m12}, m0, m3, m7, m4 + REPX {pminsd x, m13}, m0, m3, m7, m4 + movd m10, [r4+4*10] + movd m15, [r4+4*11] + REPX {pshufd x, x, q0000}, m10, m15 + psubd m5, m0, m3 ; t35a + paddd m0, m3 ; t32a + psubd m3, m7, m4 ; t60a + paddd m7, m4 ; t63a + psubd m4, m1, m6 ; t34 + paddd m1, m6 ; t33 + psubd m6, m8, m2 ; t61 + paddd m8, m2 ; t62 + REPX {pmaxsd x, m12}, m5, m3, m4, m6 + REPX {pminsd x, m13}, m5, m3, m4, m6 + ITX_MULSUB_2D 3, 5, 2, 9, _, 11, 10, 15 ; t35, t60 + ITX_MULSUB_2D 6, 4, 2, 9, _, 11, 10, 15 ; t34a, t61a + REPX {pmaxsd x, m12}, m0, m7, m1, m8 + REPX {pminsd x, m13}, m0, m7, m1, m8 + add r4, 4*12 + mova [r3+16*0], m0 + mova [r3+16*7], m7 + mova [r3+16*1], m1 + mova [r3+16*6], m8 + mova [r3+16*2], m6 + mova [r3+16*5], m4 + mova [r3+16*3], m3 + mova [r3+16*4], m5 +%else + movd m7, [r4+4*0] + movd m6, [r4+4*2] + movd m5, [r4+4*4] + movd m4, [r4+4*6] + REPX {pshufd x, x, q0000}, m7, m6, m5, m4 + pmulld m7, m0 ; t63a + pmulld m6, m1 ; t62a + pmulld m5, m2 ; t61a + pmulld m4, m3 ; t60a + mova [r3+0*16], m6 + mova [r3+1*16], m7 + movd m6, [r4+4*1] + movd m7, [r4+4*3] + REPX {pshufd x, x, q0000}, m7, m6 + pmulld m0, m6 ; t32a + pmulld m1, m7 ; t33a + movd m6, [r4+4*5] + movd m7, [r4+4*7] + REPX {pshufd x, x, q0000}, m7, m6 + pmulld m2, m6 ; t34a + pmulld m3, m7 ; t35a + mova m6, [r3+0*16] + mova m7, [o(pd_2048)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3+1*16] + REPX {psrad x, 12}, m0, m1, m7, m6, m2, m3, m5, m4 + mova [r3+0*16], m5 + psubd m5, m0, m1 ; t33 + paddd m0, m1 ; t32 + mova [r3+1*16], m0 + mova m0, [r3+0*16] + psubd m1, m7, m6 ; t62 + paddd m7, m6 ; t63 + psubd m6, m3, m2 ; t34 + paddd m3, m2 ; t35 + psubd m2, m4, m0 ; t61 + paddd m4, m0 ; t60 + mova m0, [o(clip_18b_min)] + REPX {pmaxsd x, m0}, m5, m1, m7, m6, m3, m2, m4 + pmaxsd m0, [r3+1*16] + mova [r3+0*16], m0 + mova m0, [o(clip_18b_max)] + REPX {pminsd x, m0}, m5, m1, m7, m6, m3, m2, m4 + pminsd m0, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + movd m3, [r4+4*8] + movd m4, [r4+4*9] + REPX {pshufd x, x, q0000}, m3, m4 + mova [r3+4*16], m2 + ITX_MULSUB_2D 1, 5, 2, 7, _, 0, 3, 4 ; t33a, t62a + mova m2, [r3+4*16] + mova [r3+4*16], m5 + ITX_MULSUB_2D 2, 6, 5, 7, _, 0, 3, 4, 4 ; t61a, t34a + mova m0, [r3+0*16] + mova m3, [r3+1*16] + mova m4, [r3+2*16] + mova m7, [r3+3*16] + psubd m5, m0, m3 ; t35a + paddd m0, m3 ; t32a + mova [r3+0*16], m5 + mova m5, [r3+4*16] + psubd m3, m7, m4 ; t60a + paddd m7, m4 ; t63a + psubd m4, m1, m6 ; t34 + paddd m1, m6 ; t33 + psubd m6, m5, m2 ; t61 + paddd m2, m5 ; t62 + mova m5, [o(clip_18b_min)] + REPX {pmaxsd x, m5}, m0, m3, m7, m4, m1, m6, m2 + pmaxsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(clip_18b_max)] + REPX {pminsd x, m5}, m0, m3, m7, m4, m1, m6, m2 + pminsd m5, [r3+0*16] + mova [r3+16*0], m0 + mova [r3+16*7], m7 + mova [r3+16*1], m1 + mova [r3+16*6], m2 + mova [r3+16*2], m4 + mova m7, [o(pd_2048)] + movd m0, [r4+4*10] + movd m1, [r4+4*11] + REPX {pshufd x, x, q0000}, m0, m1 + ITX_MULSUB_2D 3, 5, 2, 4, _, 7, 0, 1 ; t35, t60 + mova [r3+16*3], m3 + mova [r3+16*4], m5 + mova m4, [r3+2*16] + ITX_MULSUB_2D 6, 4, 2, 3, _, 7, 0, 1 ; t34a, t61a + add r4, 4*12 + mova [r3+16*2], m6 + mova [r3+16*5], m4 +%endif + add r3, 16*8 + ret + +.main_part2: ; idct64 steps 6-9 + lea r4, [r3+16*7] +%if ARCH_X86_64 + mova m10, [o(pd_1567)] + mova m15, [o(pd_3784)] +.main_part2_loop: + mova m0, [r3-16*32] ; t32a + mova m1, [r4-16*24] ; t39a + mova m2, [r4-16*32] ; t63a + mova m3, [r3-16*24] ; t56a + mova m4, [r3-16*16] ; t40a + mova m5, [r4-16* 8] ; t47a + mova m6, [r4-16*16] ; t55a + mova m7, [r3-16* 8] ; t48a + psubd m8, m0, m1 ; t39 + paddd m0, m1 ; t32 + psubd m1, m2, m3 ; t56 + paddd m2, m3 ; t63 + psubd m3, m5, m4 ; t40 + paddd m5, m4 ; t47 + psubd m4, m7, m6 ; t55 + paddd m7, m6 ; t48 + REPX {pmaxsd x, m12}, m8, m1, m3, m4 + REPX {pminsd x, m13}, m8, m1, m3, m4 + ITX_MULSUB_2D 1, 8, 6, 9, _, 11, 10, 15 ; t39a, t56a + ITX_MULSUB_2D 4, 3, 6, 9, _, 11, 10, 15, 4 ; t55a, t40a + REPX {pmaxsd x, m12}, m0, m2, m5, m7 + REPX {pminsd x, m13}, m0, m5, m2, m7 + psubd m6, m2, m7 ; t48a + paddd m2, m7 ; t63a + psubd m7, m0, m5 ; t47a + paddd m0, m5 ; t32a + psubd m5, m8, m4 ; t55 + paddd m8, m4 ; t56 + psubd m4, m1, m3 ; t40 + paddd m1, m3 ; t39 + REPX {pmaxsd x, m12}, m6, m7, m5, m4 + REPX {pminsd x, m13}, m6, m7, m5, m4 + REPX {pmulld x, m14}, m6, m7, m5, m4 + REPX {pmaxsd x, m12}, m2, m0, m8, m1 + REPX {pminsd x, m13}, m2, m0, m8, m1 + paddd m6, m11 + paddd m5, m11 + psubd m3, m6, m7 ; t47 + paddd m6, m7 ; t48 + psubd m7, m5, m4 ; t40a + paddd m5, m4 ; t55a + REPX {psrad x, 12}, m3, m6, m7, m5 + mova [r4-16* 8], m2 + mova [r3-16*32], m0 + mova [r3-16* 8], m8 + mova [r4-16*32], m1 + mova [r4-16*24], m3 + mova [r3-16*16], m6 + mova [r3-16*24], m7 + mova [r4-16*16], m5 +%else +.main_part2_loop: + mova m0, [r3-16*32] ; t32a + mova m1, [r4-16*24] ; t39a + mova m2, [r4-16*32] ; t63a + mova m3, [r3-16*24] ; t56a + mova m4, [r3-16*16] ; t40a + mova m5, [r4-16* 8] ; t47a + mova m6, [r4-16*16] ; t55a + psubd m7, m0, m1 ; t39 + paddd m0, m1 ; t32 + mova [r3+0*16], m7 + mova m7, [r3-16* 8] ; t48a + psubd m1, m2, m3 ; t56 + paddd m2, m3 ; t63 + psubd m3, m5, m4 ; t40 + paddd m5, m4 ; t47 + psubd m4, m7, m6 ; t55 + paddd m7, m6 ; t48 + mova m6, [o(clip_18b_min)] + REPX {pmaxsd x, m6}, m0, m1, m2, m3, m5, m4, m7 + pmaxsd m6, [r3+0*16] + mova [r3+0*16], m6 + mova m6, [o(clip_18b_max)] + REPX {pminsd x, m6}, m0, m1, m2, m3, m5, m4, m7 + pminsd m6, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 1, 6, 2, 5, 7, 0, 1567, 3784 ; t39a, t56a + ITX_MULSUB_2D 4, 3, 2, 5, _, 0, 7, 3784, 4 ; t55a, t40a + mova m2, [r3+1*16] + mova m7, [r3+3*16] + psubd m5, m2, m7 ; t48a + paddd m2, m7 ; t63a + mova [r3+1*16], m5 + mova m0, [r3+0*16] + mova m5, [r3+2*16] + psubd m7, m0, m5 ; t47a + paddd m0, m5 ; t32a + psubd m5, m6, m4 ; t55 + paddd m6, m4 ; t56 + psubd m4, m1, m3 ; t40 + paddd m1, m3 ; t39 + mova m3, [o(clip_18b_min)] + REPX {pmaxsd x, m3}, m2, m7, m0, m5, m6, m4, m1 + pmaxsd m3, [r3+1*16] + mova [r3+0*16], m3 + mova m3, [o(clip_18b_max)] + REPX {pminsd x, m3}, m2, m7, m0, m5, m6, m4, m1 + pminsd m3, [r3+0*16] + mova [r4-16* 8], m2 + mova [r3-16*32], m0 + mova [r3-16* 8], m6 + mova [r4-16*32], m1 + mova m0, [o(pd_2896)] + mova m1, [o(pd_2048)] + REPX {pmulld x, m0}, m3, m7, m5, m4 + REPX {paddd x, m1}, m3, m5 + psubd m6, m3, m7 ; t47 + paddd m3, m7 ; t48 + psubd m7, m5, m4 ; t40a + paddd m5, m4 ; t55a + REPX {psrad x, 12}, m6, m3, m7, m5 + mova [r4-16*24], m6 + mova [r3-16*16], m3 + mova [r3-16*24], m7 + mova [r4-16*16], m5 +%endif + add r3, 16 + sub r4, 16 + cmp r3, r4 + jl .main_part2_loop + sub r3, 4*16 + ret + +.main_end_loop: + mova m0, [r3+16*28] ; idct8 0 + n +.main_end_loop_start: + mova m2, [r3+16*12] ; idct32 16 + n + mova m3, [r4+16*12] ; idct32 31 - n +%if ARCH_X86_64 + mova m1, [r4+16*28] ; idct16 15 - n + mova m4, [r4-16* 4] ; idct64 63 - n + mova m5, [r3-16* 4] ; idct64 48 + n + mova m6, [r4-16*20] ; idct64 47 - n + mova m7, [r3-16*20] ; idct64 32 + n + pmaxsd m0, m12 + pminsd m0, m13 + paddd m8, m0, m1 ; idct16 out0 + n + psubd m0, m1 ; idct16 out15 - n + REPX {pmaxsd x, m12}, m8, m0 + REPX {pminsd x, m13}, m8, m0 + paddd m1, m8, m3 ; idct32 out0 + n + psubd m8, m3 ; idct32 out31 - n + paddd m3, m0, m2 ; idct32 out15 - n + psubd m0, m2 ; idct32 out16 + n + REPX {pmaxsd x, m12}, m1, m8, m3, m0 + REPX {pminsd x, m13}, m1, m3, m8, m0 + REPX {paddd x, m15}, m1, m3, m0, m8 + paddd m2, m1, m4 ; idct64 out0 + n (unshifted) + psubd m1, m4 ; idct64 out63 - n (unshifted) + paddd m4, m3, m5 ; idct64 out15 - n (unshifted) + psubd m3, m5 ; idct64 out48 + n (unshifted) + paddd m5, m0, m6 ; idct64 out16 + n (unshifted) + psubd m0, m6 ; idct64 out47 - n (unshifted) + paddd m6, m8, m7 ; idct64 out31 - n (unshifted) + psubd m8, m7 ; idct64 out32 + n (unshifted) + mova [r3-16*20], m2 + mova [r4+16*28], m1 + mova [r4-16*20], m4 + mova [r3+16*28], m3 + mova [r3-16* 4], m5 + mova [r4+16*12], m0 + mova [r4-16* 4], m6 + mova [r3+16*12], m8 +%else + mova m5, [o(clip_18b_min)] + mova m6, [o(clip_18b_max)] + mova m1, [r3+16*44] ; idct16 15 - n + pmaxsd m0, m5 + pminsd m0, m6 + paddd m4, m0, m1 ; idct16 out0 + n + psubd m0, m1 ; idct16 out15 - n + REPX {pmaxsd x, m5}, m4, m0 + REPX {pminsd x, m6}, m4, m0 + paddd m1, m4, m3 ; idct32 out0 + n + psubd m4, m3 ; idct32 out31 - n + paddd m3, m0, m2 ; idct32 out15 - n + psubd m0, m2 ; idct32 out16 + n + REPX {pmaxsd x, m5}, m1, m4, m3, m0 + REPX {pminsd x, m6}, m1, m3, m4, m0 + REPX {paddd x, m7}, m1, m3, m0, m4 + mova m5, [r4-16* 4] ; idct64 63 - n + mova m6, [r3-16* 4] ; idct64 48 + n + paddd m2, m1, m5 ; idct64 out0 + n (unshifted) + psubd m1, m5 ; idct64 out63 - n (unshifted) + paddd m5, m3, m6 ; idct64 out15 - n (unshifted) + psubd m3, m6 ; idct64 out48 + n (unshifted) + mova [r4+16*28], m1 + mova [r3+16*28], m3 + mova m6, [r4-16*20] ; idct64 47 - n + mova m1, [r3-16*20] ; idct64 32 + n + mova [r3-16*20], m2 + mova [r4-16*20], m5 + paddd m5, m0, m6 ; idct64 out16 + n (unshifted) + psubd m0, m6 ; idct64 out47 - n (unshifted) + paddd m6, m4, m1 ; idct64 out31 - n (unshifted) + psubd m4, m1 ; idct64 out32 + n (unshifted) + mova [r3-16* 4], m5 + mova [r4+16*12], m0 + mova [r4-16* 4], m6 + mova [r3+16*12], m4 +%endif + sub r4, 16 + add r3, 16 + cmp r3, r4 + jl .main_end_loop + ret + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [r4+0*64], m0 + mova [r4+1*64], m1 + mova [r4+2*64], m2 + mova [r4+3*64], m3 + sub r4, 4*64 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 16 +.dconly1: + add r5d, 640 + sar r5d, 10 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 +.dconly_loop: + paddw m1, m0, [dstq+16*0] + paddw m2, m0, [dstq+16*1] + paddw m3, m0, [dstq+16*2] + paddw m4, m0, [dstq+16*3] + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m2 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, 64 + btc r3d, 16 + jnc .dconly_loop + lea dstq, [dstq+strideq-128] + dec r3d + jg .dconly_loop + RET + +cglobal inv_txfm_add_dct_dct_64x32_16bpc, 4, 7, 16, \ + 0-(1+64+8*ARCH_X86_32+8*32+1*WIN64)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 0, 4, 1 + mov [rsp+(8*32+64+8)*16+1*gprsize], dstq + mov [rsp+(8*32+64+8)*16+2*gprsize], strideq +%else + DECLARE_REG_TMP 4, 7, 8 +%if WIN64 + mov [rsp+(8*32+64+1)*16+1*gprsize], r7 + mov [rsp+64*16+0*gprsize], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + lea t2, [rsp+7*32*16] +.zero_loop_inner: + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+r5*8], m0 + mova [t2+(72+8*ARCH_X86_32+1*WIN64)*16+r5*8], m0 + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+t0*8], m0 + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+t1*8], m0 + sub t2, 32*16 + cmp t2, rsp + jge .zero_loop_inner + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + mov [rsp+(8*32+64+8*ARCH_X86_32+1*WIN64)*16+0*gprsize], eobd + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128*31+r5*8] + mova m2, [cq+128*17+r5*8] + mova m3, [cq+128*15+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 7+r5*8] + mova m1, [cq+128*25+r5*8] + mova m2, [cq+128*23+r5*8] + mova m3, [cq+128* 9+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 5+r5*8] + mova m1, [cq+128*27+r5*8] + mova m2, [cq+128*21+r5*8] + mova m3, [cq+128*11+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128*29+r5*8] + mova m2, [cq+128*19+r5*8] + mova m3, [cq+128*13+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part2 + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128*14+r5*8] + mova m2, [cq+128*18+r5*8] + mova m3, [cq+128*30+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+128* 6+r5*8] + mova m1, [cq+128*10+r5*8] + mova m2, [cq+128*22+r5*8] + mova m3, [cq+128*26+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+128* 4+r5*8] + mova m1, [cq+128*12+r5*8] + mova m2, [cq+128*20+r5*8] + mova m3, [cq+128*28+r5*8] + call .rect2_mul_fast + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 8+r5*8] + mova m2, [cq+128*16+r5*8] + mova m3, [cq+128*24+r5*8] + call .rect2_mul_fast + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 11 ; pd_1 +%else + mova m7, [o(pd_1)] +%endif + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_end_loop_start + + lea r3, [rsp+56*16] + lea t2, [rsp+7*32*16+(64+8*ARCH_X86_32+1*WIN64)*16] + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + call .shift_transpose + ; zero cq + pxor m7, m7 + lea r4, [cq+30*128+r5*8] +.zero_cq_loop: + REPX {mova [r4+x*128], m7}, -2, -1, 0, 1 + sub r4, 4*128 + cmp r4, cq + jg .zero_cq_loop + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+(8*32+64+8*ARCH_X86_32+1*WIN64)*16] +%if ARCH_X86_32 + mov strideq, [rsp+gprsize*2+(8*32+64+8)*16] +%elif WIN64 + mov r8, [rsp+gprsize*0+64*16] +%endif + add rsp, (64+8*ARCH_X86_32+1*WIN64-3)*16 + cmp eobd, 36 + jl .load_veryfast + cmp eobd, 136 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+128] + mov r7, -16 +%else + lea r2, [rsp+(8*32+3)*16] + mov dword [r2+0*gprsize], 8 +%endif + jmp m(inv_txfm_add_dct_dct_16x32_16bpc).loop_pass2_entry + +.rect2_mul_fast: +%if ARCH_X86_64 + REPX {pmulld x, m14}, m0, m1, m2, m3 + REPX {paddd x, m11}, m0, m1, m2, m3 +%else + mova m4, [o(pd_2896)] + mova m5, [o(pd_2048)] + REPX {pmulld x, m4 }, m0, m1, m2, m3 + REPX {paddd x, m5 }, m0, m1, m2, m3 +%endif + REPX {psrad x, 12 }, m0, m1, m2, m3 + ret + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [t2+0*16+r5*8], m0 + mova [t2+8*16+r5*8], m2 + mova [t2+0*16+t0*8], m3 + mova [t2+0*16+t1*8], m1 + sub t2, 16*32 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 32 + add r5d, 128 + sar r5d, 8 + imul r5d, 181 + add r5d, 384 + sar r5d, 9 + add rsp, (1+8*32+1*WIN64)*16 + jmp m(inv_txfm_add_dct_dct_64x16_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_64x64_16bpc, 4, 7, 16, \ + 0-(64+8*ARCH_X86_32+8*64+1*ARCH_X86_64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0, 6 + mov [rsp+gprsize*1+(64*9+8)*16], r0 + mov [rsp+gprsize*2+(64*9+8)*16], r1 + mov [rsp+gprsize*3+(64*9+8)*16], r2 + mov [rsp+gprsize*4+(64*9+8)*16], r6 +%else + DECLARE_REG_TMP 8, 9, 4, 7, 0 + mov [rsp+gprsize*1+(64*9+1)*16], r9 + mov [rsp+gprsize*0+64*16], r0 +%if WIN64 + mov [rsp+gprsize*2+(64*9+1)*16], r7 + mov [rsp+gprsize*3+(64*9+1)*16], r8 +%endif +%endif +%undef cmp + + ; remove entirely-zero iterations + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + lea t4, [rsp+7*64*16] +.zero_loop_inner: + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t0*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t1*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t2*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t3*8], m0 + sub t4, 64*16 + cmp t4, rsp + jge .zero_loop_inner +%if ARCH_X86_32 + mov r6, [rsp+gprsize*4+(64*9+8)*16] +%endif + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + mov [rsp+gprsize*0+(9*64+8*ARCH_X86_32+1*ARCH_X86_64)*16], eobd +%if ARCH_X86_32 + mov cq, [rsp+gprsize*3+(64*9+8)*16] +%endif + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_18b_min)] + mova m13, [o(clip_18b_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128*31+r5*8] + mova m2, [cq+128*17+r5*8] + mova m3, [cq+128*15+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 7+r5*8] + mova m1, [cq+128*25+r5*8] + mova m2, [cq+128*23+r5*8] + mova m3, [cq+128* 9+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 5+r5*8] + mova m1, [cq+128*27+r5*8] + mova m2, [cq+128*21+r5*8] + mova m3, [cq+128*11+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128*29+r5*8] + mova m2, [cq+128*19+r5*8] + mova m3, [cq+128*13+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part2 + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128*14+r5*8] + mova m2, [cq+128*18+r5*8] + mova m3, [cq+128*30+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+128* 6+r5*8] + mova m1, [cq+128*10+r5*8] + mova m2, [cq+128*22+r5*8] + mova m3, [cq+128*26+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+128* 4+r5*8] + mova m1, [cq+128*12+r5*8] + mova m2, [cq+128*20+r5*8] + mova m3, [cq+128*28+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 8+r5*8] + mova m2, [cq+128*16+r5*8] + mova m3, [cq+128*24+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 10 ; pd_2 +%else + mova m7, [o(pd_2)] +%endif + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_end_loop_start + + lea r3, [rsp+56*16] + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + lea t4, [rsp+7*64*16+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16] + call .shift_transpose + ; zero cq + pxor m7, m7 +%if ARCH_X86_32 + mov cq, [rsp+gprsize*3+(64*9+8)*16] +%endif + lea r4, [cq+30*128+r5*8] +.zero_cq_loop: + REPX {mova [r4+x*128], m7}, -2, -1, 0, 1 + sub r4, 4*128 + cmp r4, cq + jg .zero_cq_loop +%if ARCH_X86_32 + mov r6, [rsp+gprsize*4+(64*9+8)*16] +%endif + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+(9*64+8*ARCH_X86_32+1*ARCH_X86_64)*16] +%if ARCH_X86_32 + mov strideq, [rsp+gprsize*2+(9*64+8)*16] +%else + mov r0, [rsp+gprsize*0+64*16] +%endif + add rsp, (64+8*ARCH_X86_32+1*ARCH_X86_64-3)*16 + cmp eobd, 151 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + +%if ARCH_X86_64 + lea r2, [dstq+128] + mov r7, -16 +%else + lea r2, [rsp+(64*8+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 8 +%endif + jmp m(inv_txfm_add_dct_dct_16x64_16bpc).loop_pass2 + + ; copy of pass=1 tmp-regs +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0, 6 +%else + DECLARE_REG_TMP 8, 9, 4, 7, 0 +%endif + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [t4+t0*8], m0 + mova [t4+t1*8], m1 + mova [t4+t2*8], m2 + mova [t4+t3*8], m3 + sub t4, 16*64 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 181 + mov [cq], eobd ; 0 + mov r3d, 64 + add rsp, (64+8*ARCH_X86_32+8*64+1*ARCH_X86_64)*16 + \ + (4+4*ARCH_X86_32)*gprsize - (64+8*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_64x16_16bpc).dconly1 diff --git a/third_party/dav1d/src/x86/itx_avx2.asm b/third_party/dav1d/src/x86/itx_avx2.asm new file mode 100644 index 0000000000..a67f053a61 --- /dev/null +++ b/third_party/dav1d/src/x86/itx_avx2.asm @@ -0,0 +1,5542 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 16 + +; Note: The order of (at least some of) those constants matter! + +const deint_shuf, db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 + +%macro COEF_PAIR 2 +pw_%1_%2: dw %1, %2 +pw_m%2_%1: dw -%2, %1 +%endmacro + +; ADST-only +pw_3803_1321: dw 3803, 1321 +pw_m1321_2482: dw -1321, 2482 +pw_2482_3344: dw 2482, 3344 +pw_m3344_3344: dw -3344, 3344 +pw_m3803_3344: dw -3803, 3344 +pw_m3803_m6688: dw -3803, -6688 +pw_2896_m2896: dw 2896, -2896 + +const pw_5, times 2 dw 5 +const pw_2048, times 2 dw 2048 +const pw_4096, times 2 dw 4096 +const pw_8192, times 2 dw 8192 +const pw_16384, times 2 dw 16384 +const pw_1697x16, times 2 dw 1697*16 +const pw_1697x8, times 2 dw 1697*8 +const pw_2896x8, times 2 dw 2896*8 +const pd_2048, dd 2048 + +const pw_2896_2896, dw 2896, 2896 +const pw_m2896_2896, dw -2896, 2896 +const pw_1567_3784, dw 1567, 3784 +const pw_m3784_1567, dw -3784, 1567 +COEF_PAIR 3784, 1567 +COEF_PAIR 201, 4091 +COEF_PAIR 995, 3973 +COEF_PAIR 1751, 3703 +COEF_PAIR 2440, 3290 +COEF_PAIR 3035, 2751 +COEF_PAIR 3513, 2106 +COEF_PAIR 3857, 1380 +COEF_PAIR 4052, 601 +COEF_PAIR 401, 4076 +COEF_PAIR 1931, 3612 +COEF_PAIR 3166, 2598 +COEF_PAIR 3920, 1189 +COEF_PAIR 799, 4017 +COEF_PAIR 3406, 2276 +pw_m799_m4017: dw -799, -4017 +const pw_m1567_m3784, dw -1567, -3784 +pw_m3406_m2276: dw -3406, -2276 +pw_m401_m4076: dw -401, -4076 +pw_m3166_m2598: dw -3166, -2598 +pw_m1931_m3612: dw -1931, -3612 +pw_m3920_m1189: dw -3920, -1189 +COEF_PAIR 2276, 3406 +COEF_PAIR 4017, 799 + +%macro COEF_X8 1-* +%rep %0 + dw %1*8, %1*8 + %rotate 1 +%endrep +%endmacro + +pw_3703x8: COEF_X8 3703 +pw_1751x8: COEF_X8 1751 +pw_m1380x8: COEF_X8 -1380 +pw_3857x8: COEF_X8 3857 +pw_3973x8: COEF_X8 3973 +pw_995x8: COEF_X8 995 +pw_m2106x8: COEF_X8 -2106 +pw_3513x8: COEF_X8 3513 +pw_3290x8: COEF_X8 3290 +pw_2440x8: COEF_X8 2440 +pw_m601x8: COEF_X8 -601 +pw_4052x8: COEF_X8 4052 + +const idct64_mul +COEF_X8 4095, 101, 4065, 501, 2967, -2824, 3229, -2520 +COEF_X8 3745, 1660, 3564, 2019, 3822, -1474, 3948, -1092 +COEF_X8 3996, 897, 3889, 1285, 3461, -2191, 3659, -1842 +COEF_X8 3349, 2359, 3102, 2675, 4036, -700, 4085, -301 + +pw_201_4091x8: dw 201*8, 4091*8 +pw_m601_4052x8: dw -601*8, 4052*8 +pw_995_3973x8: dw 995*8, 3973*8 +pw_m1380_3857x8: dw -1380*8, 3857*8 +pw_1751_3703x8: dw 1751*8, 3703*8 +pw_m2106_3513x8: dw -2106*8, 3513*8 +pw_2440_3290x8: dw 2440*8, 3290*8 +pw_m2751_3035x8: dw -2751*8, 3035*8 + +%define o_idct64_offset idct64_mul - (o_base) - 8 + +SECTION .text + +; Code size reduction trickery: Instead of using rip-relative loads with +; mandatory 4-byte offsets everywhere, we can set up a base pointer with a +; single rip-relative lea and then address things relative from that with +; 1-byte offsets as long as data is within +-128 bytes of the base pointer. +%define o_base deint_shuf + 128 +%define o(x) (r6 - (o_base) + (x)) +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +; flags: 1 = swap, 2 = interleave, 4: coef_regs +%macro ITX_MUL2X_PACK 6-7 0 ; dst/src, tmp[1-2], rnd, coef[1-2], flags +%if %7 & 4 + pmaddwd m%2, m%5, m%1 + pmaddwd m%1, m%6 +%else +%if %7 & 1 + vpbroadcastd m%2, [o(pw_%5_%6)] + vpbroadcastd m%3, [o(pw_m%6_%5)] +%else + vpbroadcastd m%2, [o(pw_m%6_%5)] + vpbroadcastd m%3, [o(pw_%5_%6)] +%endif + pmaddwd m%2, m%1 + pmaddwd m%1, m%3 +%endif + paddd m%2, m%4 + paddd m%1, m%4 +%if %7 & 2 + pslld m%2, 4 + psrld m%1, 12 + pblendw m%1, m%2, 0xaa +%else + psrad m%2, 12 + psrad m%1, 12 + packssdw m%1, m%2 +%endif +%endmacro + +; flags: 1 = swap, 2 = interleave, 4 = coef_regs +%macro ITX_MUL4X_PACK 9-10 0 ; dst/src, tmp[1-3], rnd, coef[1-4], flags +%if %10 & 1 + vpbroadcastd m%3, [o(pw_%8_%9)] + vpbroadcastd m%4, [o(pw_m%9_%8)] + vpbroadcastd xm%2, [o(pw_%6_%7)] + vpblendd m%2, m%3, 0xf0 + vpbroadcastd xm%3, [o(pw_m%7_%6)] +%else + vpbroadcastd m%3, [o(pw_m%9_%8)] + vpbroadcastd m%4, [o(pw_%8_%9)] + vpbroadcastd xm%2, [o(pw_m%7_%6)] + vpblendd m%2, m%3, 0xf0 + vpbroadcastd xm%3, [o(pw_%6_%7)] +%endif + vpblendd m%3, m%4, 0xf0 + ITX_MUL2X_PACK %1, %4, _, %5, %2, %3, (4|%10) +%endmacro + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +%macro ITX_MULSUB_2W 7-8 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2 + punpckhwd m%3, m%2, m%1 + punpcklwd m%2, m%1 +%if %7 < 32 + pmaddwd m%1, m%7, m%2 + pmaddwd m%4, m%7, m%3 +%else + vpbroadcastd m%1, [o(pw_m%7_%6)] + pmaddwd m%4, m%3, m%1 + pmaddwd m%1, m%2 +%endif + paddd m%4, m%5 + paddd m%1, m%5 + psrad m%4, 12 + psrad m%1, 12 + packssdw m%1, m%4 +%if %7 < 32 + pmaddwd m%3, m%6 + pmaddwd m%2, m%6 +%else + vpbroadcastd m%4, [o(pw_%6_%7)] + pmaddwd m%3, m%4 + pmaddwd m%2, m%4 +%endif + paddd m%3, m%5 + paddd m%2, m%5 + psrad m%3, 12 + psrad m%2, 12 +%if %0 == 8 + packssdw m%8, m%2, m%3 +%else + packssdw m%2, m%3 +%endif +%endmacro + +%macro IDCT4_1D 7 ; src[1-4], tmp[1-2], pd_2048 + ITX_MULSUB_2W %2, %4, %5, %6, %7, 1567, 3784, %5 ; t2, t3 + ITX_MULSUB_2W %1, %3, %4, %6, %7, 2896, 2896, %4 ; t1, t0 + psubsw m%3, m%1, m%2 + paddsw m%2, m%1 + paddsw m%1, m%4, m%5 + psubsw m%4, m%5 +%endmacro + +%macro IDCT8_1D 11 ; src[1-8], tmp[1-2], pd_2048 + ITX_MULSUB_2W %6, %4, %9, %10, %11, 3406, 2276 ; t5a, t6a + ITX_MULSUB_2W %2, %8, %9, %10, %11, 799, 4017 ; t4a, t7a + ITX_MULSUB_2W %3, %7, %9, %10, %11, 1567, 3784 ; t2, t3 + paddsw m%9, m%2, m%6 ; t4 + psubsw m%2, m%6 ; t5a + paddsw m%10, m%8, m%4 ; t7 + psubsw m%8, m%4 ; t6a + ITX_MULSUB_2W %1, %5, %4, %6, %11, 2896, 2896 ; t1, t0 + ITX_MULSUB_2W %8, %2, %4, %6, %11, 2896, 2896 ; t5, t6 + psubsw m%6, m%1, m%3 ; dct4 out2 + paddsw m%3, m%1 ; dct4 out1 + paddsw m%1, m%5, m%7 ; dct4 out0 + psubsw m%5, m%7 ; dct4 out3 + psubsw m%7, m%3, m%2 ; out6 + paddsw m%2, m%3 ; out1 + paddsw m%3, m%6, m%8 ; out2 + psubsw m%6, m%8 ; out5 + psubsw m%8, m%1, m%10 ; out7 + paddsw m%1, m%10 ; out0 + paddsw m%4, m%5, m%9 ; out3 + psubsw m%5, m%9 ; out4 +%endmacro + +; in1 = %1, in3 = %2, in5 = %3, in7 = %4 +; in9 = %5, in11 = %6, in13 = %7, in15 = %8 +%macro IDCT16_1D_ODDHALF 11 ; src[1-8], tmp[1-2], pd_2048 + ITX_MULSUB_2W %1, %8, %9, %10, %11, 401, 4076 ; t8a, t15a + ITX_MULSUB_2W %5, %4, %9, %10, %11, 3166, 2598 ; t9a, t14a + ITX_MULSUB_2W %3, %6, %9, %10, %11, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2W %7, %2, %9, %10, %11, 3920, 1189 ; t11a, t12a + psubsw m%9, m%2, m%6 ; t13 + paddsw m%6, m%2 ; t12 + psubsw m%2, m%8, m%4 ; t14 + paddsw m%8, m%4 ; t15 + psubsw m%4, m%7, m%3 ; t10 + paddsw m%3, m%7 ; t11 + psubsw m%7, m%1, m%5 ; t9 + paddsw m%1, m%5 ; t8 + ITX_MULSUB_2W %2, %7, %5, %10, %11, 1567, 3784 ; t9a, t14a + ITX_MULSUB_2W %9, %4, %5, %10, %11, m3784, 1567 ; t10a, t13a + psubsw m%5, m%1, m%3 ; t11a + paddsw m%1, m%3 ; t8a + psubsw m%3, m%7, m%4 ; t13 + paddsw m%7, m%4 ; t14 + psubsw m%4, m%8, m%6 ; t12a + paddsw m%8, m%6 ; t15a + psubsw m%6, m%2, m%9 ; t10 + paddsw m%2, m%9 ; t9 + ITX_MULSUB_2W %3, %6, %9, %10, %11, 2896, 2896 ; t10a, t13a + ITX_MULSUB_2W %4, %5, %9, %10, %11, 2896, 2896 ; t11, t12 +%endmacro + +%macro WRAP_XMM 1+ + INIT_XMM cpuname + %1 + INIT_YMM cpuname +%endmacro + +%macro ITX4_END 4-5 2048 ; row[1-4], rnd +%if %5 + vpbroadcastd m2, [o(pw_%5)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 +%endif + lea r2, [dstq+strideq*2] +%assign %%i 1 +%rep 4 + %if %1 & 2 + CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) + %else + CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) + %endif + %assign %%i %%i + 1 + %rotate 1 +%endrep + movd m2, [%%row_adr1] + pinsrd m2, [%%row_adr2], 1 + movd m3, [%%row_adr3] + pinsrd m3, [%%row_adr4], 1 + pmovzxbw m2, m2 + pmovzxbw m3, m3 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + movd [%%row_adr1], m0 + pextrd [%%row_adr2], m0, 1 + pextrd [%%row_adr3], m0, 2 + pextrd [%%row_adr4], m0, 3 + ret +%endmacro + +%macro IWHT4_1D_PACKED 0 + punpckhqdq m3, m0, m1 ; in1 in3 + punpcklqdq m0, m1 ; in0 in2 + psubw m2, m0, m3 + paddw m0, m3 + punpckhqdq m2, m2 ; t2 t2 + punpcklqdq m0, m0 ; t0 t0 + psubw m1, m0, m2 + psraw m1, 1 + psubw m1, m3 ; t1 t3 + psubw m0, m1 ; ____ out0 + paddw m2, m1 ; out3 ____ +%endmacro + +INIT_XMM avx2 +cglobal inv_txfm_add_wht_wht_4x4_8bpc, 3, 3, 4, dst, stride, c + mova m0, [cq+16*0] + mova m1, [cq+16*1] + pxor m2, m2 + mova [cq+16*0], m2 + mova [cq+16*1], m2 + psraw m0, 2 + psraw m1, 2 + IWHT4_1D_PACKED + punpckhwd m0, m1 + punpcklwd m3, m1, m2 + punpckhdq m1, m0, m3 + punpckldq m0, m3 + IWHT4_1D_PACKED + vpblendd m0, m2, 0x03 + ITX4_END 3, 0, 2, 1, 0 + +%macro INV_TXFM_FN 3 ; type1, type2, size +cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 5, 0, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%3_internal_8bpc) + lea r6, [o_base] + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [m(i%2_%3_internal_8bpc).pass2] +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x4 +%ifidn %1_%2, dct_dct + vpbroadcastw m0, [cq] + vpbroadcastd m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mov [cq], eobd ; 0 + pmulhrsw m0, m1 + mova m1, m0 + jmp m(iadst_4x4_internal_8bpc).end2 +%endif +%endmacro + +%macro IDCT4_1D_PACKED 0 + vpbroadcastd m4, [o(pd_2048)] + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784 + ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896 + paddsw m0, m1, m2 ; out0 out1 + psubsw m1, m2 ; out3 out2 +%endmacro + +%macro IADST4_1D_PACKED 0 + punpcklwd m2, m1, m0 + punpckhwd m3, m1, m0 + vpbroadcastd m5, [o(pw_m3344_3344)] + vpbroadcastd m0, [o(pw_3803_1321)] + vpbroadcastd m4, [o(pw_m1321_2482)] + pmaddwd m1, m5, m2 ; 3344*in3 - 3344*in2 + psrld m5, 16 + pmaddwd m0, m2 + pmaddwd m2, m4 + pmaddwd m5, m3 ; 3344*in0 + paddd m1, m5 ; 3344*in0 - 3344*in2 + 3344*in3 + vpbroadcastd m4, [o(pw_2482_3344)] + vpbroadcastd m5, [o(pw_m3803_3344)] + pmaddwd m4, m3 + pmaddwd m5, m3 + paddd m4, m0 ; 1321*in0 + 3344*in1 + 3803*in2 + 2482*in3 + vpbroadcastd m0, [o(pw_m3803_m6688)] + pmaddwd m3, m0 + vpbroadcastd m0, [o(pd_2048)] + paddd m2, m0 + paddd m1, m0 + paddd m0, m4 + paddd m5, m2 ; 2482*in0 + 3344*in1 - 1321*in2 - 3803*in3 + paddd m2, m4 + paddd m2, m3 + REPX {psrad x, 12}, m1, m2, m0, m5 + packssdw m0, m5 ; out0 out1 + packssdw m1, m2 ; out2 out3 +%endmacro + +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst +INV_TXFM_4X4_FN dct, identity + +cglobal idct_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + IDCT4_1D_PACKED + mova m2, [o(deint_shuf)] + shufps m3, m0, m1, q1331 + shufps m0, m1, q0220 + pshufb m0, m2 + pshufb m1, m3, m2 + jmp tx2q +.pass2: + IDCT4_1D_PACKED + pxor m2, m2 + mova [cq+16*0], m2 + mova [cq+16*1], m2 + ITX4_END 0, 1, 3, 2 + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +cglobal iadst_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + call .main + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call .main +.end: + pxor m2, m2 + mova [cq+16*0], m2 + mova [cq+16*1], m2 +.end2: + ITX4_END 0, 1, 2, 3 +ALIGN function_align +cglobal_label .main + IADST4_1D_PACKED + ret + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + call m(iadst_4x4_internal_8bpc).main + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + jmp tx2q +.pass2: + call m(iadst_4x4_internal_8bpc).main +.end: + pxor m2, m2 + mova [cq+16*0], m2 + mova [cq+16*1], m2 +.end2: + ITX4_END 3, 2, 1, 0 + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_4x4_internal_8bpc).end + +%macro WRITE_4X8 2 ; coefs[1-2] + movd xm4, [dstq+strideq*0] + pinsrd xm4, [dstq+strideq*1], 1 + movd xm5, [dstq+strideq*2] + pinsrd xm5, [dstq+r3 ], 1 + pinsrd xm4, [r2 +strideq*0], 2 + pinsrd xm4, [r2 +strideq*1], 3 + pinsrd xm5, [r2 +strideq*2], 2 + pinsrd xm5, [r2 +r3 ], 3 + pmovzxbw m4, xm4 + pmovzxbw m5, xm5 + paddw m4, m%1 + paddw m5, m%2 + packuswb m4, m5 + vextracti128 xm5, m4, 1 + movd [dstq+strideq*0], xm4 + pextrd [dstq+strideq*1], xm4, 1 + pextrd [dstq+strideq*2], xm4, 2 + pextrd [dstq+r3 ], xm4, 3 + movd [r2 +strideq*0], xm5 + pextrd [r2 +strideq*1], xm5, 1 + pextrd [r2 +strideq*2], xm5, 2 + pextrd [r2 +r3 ], xm5, 3 +%endmacro + +%macro INV_TXFM_4X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x8 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_2048)] + mov [cq], eobd + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 + mova m1, m0 + jmp m(iadst_4x8_internal_8bpc).end3 +%endif +%endmacro + +%macro IDCT8_1D_PACKED 0 + vpbroadcastd m6, [o(pd_2048)] + punpckhwd m5, m3, m0 ; in7 in1 + punpckhwd m4, m1, m2 ; in3 in5 + punpcklwd m3, m1 ; in6 in2 + punpcklwd m2, m0 ; in4 in0 + ITX_MUL2X_PACK 5, 0, 1, 6, 799, 4017, 3 ; t4a t7a + ITX_MUL2X_PACK 4, 0, 1, 6, 3406, 2276, 3 ; t5a t6a + ITX_MUL2X_PACK 3, 0, 1, 6, 1567, 3784 ; t3 t2 + psubsw m0, m5, m4 ; t5a t6a (interleaved) + paddsw m4, m5 ; t4 t7 (interleaved) + ITX_MUL2X_PACK 2, 1, 5, 6, 2896, 2896 ; t0 t1 + vpbroadcastd m1, [o(pw_m2896_2896)] + ITX_MUL2X_PACK 0, 1, _, 6, 1, 5, 4 ; t6 t5 +%if mmsize > 16 + vbroadcasti128 m1, [o(deint_shuf)] + pshufb m4, m1 +%else + pshufb m4, [o(deint_shuf)] +%endif + psubsw m1, m2, m3 ; tmp3 tmp2 + paddsw m3, m2 ; tmp0 tmp1 + shufps m2, m4, m0, q1032 ; t7 t6 + vpblendd m4, m0, 0xcc ; t4 t5 + paddsw m0, m3, m2 ; out0 out1 + psubsw m3, m2 ; out7 out6 + psubsw m2, m1, m4 ; out4 out5 + paddsw m1, m4 ; out3 out2 +%endmacro + +%macro IADST8_1D_PACKED 1 ; pass + vpbroadcastd m6, [o(pd_2048)] + punpckhwd m0, m4, m3 ; 0 7 + punpckhwd m1, m5, m2 ; 2 5 + punpcklwd m2, m5 ; 4 3 + punpcklwd m3, m4 ; 6 1 +%if %1 == 1 + ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076, 3 ; t1a t0a + ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612, 2 ; t2a t3a + ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598, 3 ; t5a t4a + ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189, 2 ; t6a t7a + psubsw m4, m0, m2 ; t5 t4 + paddsw m0, m2 ; t1 t0 + psubsw m5, m1, m3 ; t6 t7 + paddsw m1, m3 ; t2 t3 + ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784, 3 ; t5a t4a + ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 2 ; t7a t6a +%if mmsize > 16 + vbroadcasti128 m2, [o(deint_shuf)] +%else + mova m2, [o(deint_shuf)] +%endif + pshuflw m1, m1, q2301 + pshufhw m1, m1, q2301 + psubsw m3, m0, m1 ; t3 t2 + paddsw m0, m1 ; -out7 out0 + psubsw m1, m4, m5 ; t7 t6 + paddsw m4, m5 ; out6 -out1 + pshufb m0, m2 + pshufb m4, m2 + vpbroadcastd m5, [o(pw_m2896_2896)] + pmaddwd m2, m5, m3 + pmaddwd m5, m1 + paddd m2, m6 + paddd m5, m6 + psrad m2, 12 + psrad m5, 12 + packssdw m2, m5 ; out4 -out5 + vpbroadcastd m5, [o(pw_2896_2896)] + pmaddwd m3, m5 + pmaddwd m1, m5 + paddd m3, m6 + paddd m1, m6 + psrad m3, 12 + psrad m1, 12 + packssdw m1, m3 ; out2 -out3 + punpcklqdq m3, m4, m0 ; out6 -out7 + punpckhqdq m0, m4 ; out0 -out1 +%else + ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076 ; t0a t1a + ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612 ; t2a t3a + ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598 ; t4a t5a + ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189 ; t6a t7a + psubsw m4, m0, m2 ; t4 t5 + paddsw m0, m2 ; t0 t1 + psubsw m5, m1, m3 ; t6 t7 + paddsw m1, m3 ; t2 t3 + shufps m2, m5, m4, q1032 + punpckhwd m4, m2 + punpcklwd m5, m2 + ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784, 1 ; t5a t4a + ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567 ; t7a t6a + psubsw m2, m0, m1 ; t2 t3 + paddsw m0, m1 ; out0 -out7 + psubsw m1, m4, m5 ; t7 t6 + paddsw m4, m5 ; out6 -out1 + vpbroadcastd m5, [o(pw_2896x8)] + vpblendd m3, m0, m4, 0x33 ; out6 -out7 + vpblendd m0, m4, 0xcc ; out0 -out1 + shufps m4, m2, m1, q1032 ; t3 t7 + vpblendd m1, m2, 0x33 ; t2 t6 + psubsw m2, m1, m4 ; t2-t3 t6-t7 + paddsw m1, m4 ; t2+t3 t6+t7 + pmulhrsw m2, m5 ; out4 -out5 + pshufd m1, m1, q1032 + pmulhrsw m1, m5 ; out2 -out3 +%endif +%endmacro + +INIT_YMM avx2 +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst +INV_TXFM_4X8_FN dct, identity + +cglobal idct_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + IDCT4_1D_PACKED + vbroadcasti128 m2, [o(deint_shuf)] + shufps m3, m0, m1, q1331 + shufps m0, m1, q0220 + pshufb m0, m2 + pshufb m1, m3, m2 + jmp tx2q +.pass2: + vextracti128 xm2, m0, 1 + vextracti128 xm3, m1, 1 + call .main + vpbroadcastd m4, [o(pw_2048)] + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 + pshufd m1, m1, q1032 + jmp m(iadst_4x8_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + WRAP_XMM IDCT8_1D_PACKED + ret + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity + +cglobal iadst_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + call m(iadst_8x4_internal_8bpc).main + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + vextracti128 xm2, m0, 1 + vextracti128 xm3, m1, 1 + pshufd xm4, xm0, q1032 + pshufd xm5, xm1, q1032 + call .main_pass2 + vpbroadcastd m4, [o(pw_2048)] + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 + pxor m5, m5 + psubw m5, m4 +.end: + vpblendd m4, m5, 0xcc +.end2: + pmulhrsw m0, m4 + pmulhrsw m1, m4 + WIN64_RESTORE_XMM + pxor m2, m2 + mova [cq+32*0], m2 + mova [cq+32*1], m2 +.end3: + lea r2, [dstq+strideq*4] + lea r3, [strideq*3] + WRITE_4X8 0, 1 + RET +ALIGN function_align +.main_pass1: + WRAP_XMM IADST8_1D_PACKED 1 + ret +ALIGN function_align +cglobal_label .main_pass2 + WRAP_XMM IADST8_1D_PACKED 2 + ret + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity + +cglobal iflipadst_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + call m(iadst_8x4_internal_8bpc).main + punpcklwd m3, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m3 + punpckhwd m1, m3 + jmp tx2q +.pass2: + vextracti128 xm2, m0, 1 + vextracti128 xm3, m1, 1 + pshufd xm4, xm0, q1032 + pshufd xm5, xm1, q1032 + call m(iadst_4x8_internal_8bpc).main_pass2 + vpbroadcastd m5, [o(pw_2048)] + vinserti128 m3, xm1, 1 + vinserti128 m2, xm0, 1 + pxor m4, m4 + psubw m4, m5 + pshufd m0, m3, q1032 + pshufd m1, m2, q1032 + jmp m(iadst_4x8_internal_8bpc).end + +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity + +cglobal iidentity_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m2, [cq+32*0], q3120 + vpermq m0, [cq+32*1], q3120 + vpbroadcastd m3, [o(pw_2896x8)] + vpbroadcastd m4, [o(pw_1697x8)] + punpcklwd m1, m2, m0 + punpckhwd m2, m0 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + pmulhrsw m2, m4, m0 + pmulhrsw m4, m1 + paddsw m0, m2 + paddsw m1, m4 + jmp tx2q +.pass2: + vpbroadcastd m4, [o(pw_4096)] + jmp m(iadst_4x8_internal_8bpc).end2 + +%macro INV_TXFM_4X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x16 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + movd xm3, [o(pw_2048)] + mov [cq], eobd + pmulhrsw xm0, xm2 + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm3 + vpbroadcastw m0, xm0 + mova m1, m0 + mova m2, m0 + mova m3, m0 + jmp m(iadst_4x16_internal_8bpc).end3 +%endif +%endmacro + +%macro IDCT16_1D_PACKED 0 + vpbroadcastd m10, [o(pd_2048)] +.main2: + punpckhwd m8, m7, m0 ; dct16 in15 in1 + punpcklwd m9, m4, m0 ; dct4 in2 in0 + punpckhwd m0, m3, m4 ; dct16 in7 in9 + punpcklwd m7, m1 ; dct8 in7 in1 + punpckhwd m1, m6 ; dct16 in3 in13 + punpcklwd m3, m5 ; dct8 in3 in5 + punpckhwd m5, m2 ; dct16 in11 in5 + punpcklwd m6, m2 ; dct4 in3 in1 + ITX_MUL2X_PACK 8, 2, 4, 10, 401, 4076, 3 ; t8a t15a + ITX_MUL2X_PACK 0, 2, 4, 10, 3166, 2598, 3 ; t9a t14a + ITX_MUL2X_PACK 1, 2, 4, 10, 3920, 1189, 3 ; t11a t12a + ITX_MUL2X_PACK 5, 2, 4, 10, 1931, 3612, 3 ; t10a t13a + ITX_MUL2X_PACK 7, 2, 4, 10, 799, 4017, 3 ; t4a t7a + ITX_MUL2X_PACK 3, 2, 4, 10, 3406, 2276, 3 ; t5a t6a + ITX_MUL2X_PACK 6, 2, 4, 10, 1567, 3784 ; t3 t2 + psubsw m2, m8, m0 ; t9 t14 + paddsw m8, m0 ; t8 t15 + psubsw m0, m1, m5 ; t10 t13 + paddsw m1, m5 ; t11 t12 + vpbroadcastd m5, [o(pw_m3784_1567)] ; reuse pw_1567_3784 + ITX_MUL2X_PACK 2, 4, _, 10, 4, 5, 6 ; t9a t14a + vpbroadcastd m4, [o(pw_m1567_m3784)] ; reuse pw_m3784_1567 + ITX_MUL2X_PACK 0, 5, _, 10, 5, 4, 6 ; t10a t13a + psubsw m4, m8, m1 ; t11a t12a + paddsw m8, m1 ; t8a t15a + psubsw m1, m7, m3 ; t5a t6a + paddsw m7, m3 ; t4 t7 + paddsw m3, m2, m0 ; t9 t14 + psubsw m2, m0 ; t10 t13 +%if mmsize > 16 + vbroadcasti128 m0, [o(deint_shuf)] +%else + mova m0, [o(deint_shuf)] +%endif + pshufb m8, m0 + pshufb m7, m0 + pshufb m3, m0 + ITX_MUL2X_PACK 9, 0, 5, 10, 2896, 2896 ; t0 t1 + vpbroadcastd m0, [o(pw_m2896_2896)] + ITX_MUL2X_PACK 4, 5, _, 10, 5, 0, 4 ; t11 t12 + vpbroadcastd m5, [o(pw_2896_2896)] + ITX_MUL2X_PACK 1, 0, _, 10, 0, 5, 4 ; t6 t5 + vpbroadcastd m0, [o(pw_m2896_2896)] + ITX_MUL2X_PACK 2, 0, _, 10, 0, 5, 4 ; t13a t10a + punpckhqdq m0, m8, m3 ; t15a t14 + punpcklqdq m8, m3 ; t8a t9 + shufps m5, m4, m2, q1032 ; t12 t13a + vpblendd m4, m2, 0xcc ; t11 t10a + shufps m2, m7, m1, q1032 ; t7 t6 + vpblendd m7, m1, 0xcc ; t4 t5 + psubsw m1, m9, m6 ; dct4 out3 out2 + paddsw m9, m6 ; dct4 out0 out1 + psubsw m3, m9, m2 ; dct8 out7 out6 + paddsw m9, m2 ; dct8 out0 out1 + psubsw m2, m1, m7 ; dct8 out4 out5 + paddsw m1, m7 ; dct8 out3 out2 + psubsw m7, m9, m0 ; out15 out14 + paddsw m0, m9 ; out0 out1 + psubsw m6, m1, m5 ; out12 out13 + paddsw m1, m5 ; out3 out2 + psubsw m5, m2, m4 ; out11 out10 + paddsw m2, m4 ; out4 out5 + psubsw m4, m3, m8 ; out8 out9 + paddsw m3, m8 ; out7 out6 +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst +INV_TXFM_4X16_FN dct, identity + +cglobal idct_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + call m(idct_16x4_internal_8bpc).main + vpbroadcastd m5, [o(pw_16384)] + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + REPX {pmulhrsw x, m5}, m0, m4, m2, m3 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + vextracti128 xm4, m0, 1 + vextracti128 xm5, m1, 1 + vextracti128 xm6, m2, 1 + vextracti128 xm7, m3, 1 + call .main + vinserti128 m0, xm4, 1 + vinserti128 m1, xm5, 1 + vpbroadcastd m5, [o(pw_2048)] + vinserti128 m2, xm6, 1 + vinserti128 m3, xm7, 1 + pshufd m1, m1, q1032 + pshufd m3, m3, q1032 + jmp m(iadst_4x16_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + WRAP_XMM IDCT16_1D_PACKED + ret + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity + +cglobal iadst_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m5, [o(pw_16384)] + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + REPX {pmulhrsw x, m5}, m4, m2, m3, m0 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + call .main + vpbroadcastd m5, [o(pw_2896x8)] + paddsw m1, m2, m4 + psubsw m2, m4 + pmulhrsw m1, m5 ; -out7 out4 out6 -out5 + pmulhrsw m2, m5 ; out8 -out11 -out9 out10 + vpbroadcastd m5, [o(pw_2048)] + pshufd m1, m1, q1032 + vpblendd m4, m1, m0, 0x33 + vpblendd m0, m2, 0x33 + vpblendd m2, m3, 0x33 + vpblendd m3, m1, 0x33 + vpermq m0, m0, q2031 + vpermq m1, m2, q1302 + vpermq m2, m3, q3120 + vpermq m3, m4, q0213 + psubw m6, m7, m5 +.end: + vpblendd m5, m6, 0xcc +.end2: + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + WIN64_RESTORE_XMM + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + mova [cq+32*2], m4 + mova [cq+32*3], m4 +.end3: + lea r2, [dstq+strideq*8] + lea r3, [strideq*3] + WRITE_4X8 0, 1 + lea dstq, [dstq+strideq*4] + lea r2, [r2 +strideq*4] + WRITE_4X8 2, 3 + RET +ALIGN function_align +.main: + vpblendd m4, m1, m0, 0xcc + vpblendd m1, m0, 0x33 + vpblendd m5, m2, m3, 0xcc + vpblendd m2, m3, 0x33 + vperm2i128 m3, m5, m2, 0x31 + vinserti128 m0, m1, xm4, 1 ; in0 in3 in2 in1 + vperm2i128 m4, m1, m4, 0x31 + vinserti128 m1, m5, xm2, 1 ; in4 in7 in6 in5 + pshufd m3, m3, q1032 ; in15 in12 in13 in14 + pshufd m2, m4, q1032 ; in11 in8 in9 in10 +cglobal_label .main2 + vpbroadcastd m8, [o(pd_2048)] + pxor m7, m7 + punpckhwd m4, m3, m0 ; in12 in3 in14 in1 + punpcklwd m0, m3 ; in0 in15 in2 in13 + punpckhwd m3, m2, m1 ; in8 in7 in10 in5 + punpcklwd m1, m2 ; in4 in11 in6 in9 + ITX_MUL4X_PACK 0, 2, 5, 6, 8, 201, 4091, 995, 3973, 3 + ITX_MUL4X_PACK 1, 2, 5, 6, 8, 1751, 3703, 2440, 3290, 3 + ITX_MUL4X_PACK 3, 2, 5, 6, 8, 3035, 2751, 3513, 2106, 3 + ITX_MUL4X_PACK 4, 2, 5, 6, 8, 3857, 1380, 4052, 601, 3 + psubsw m2, m0, m3 ; t9a t8a t11a t10a + paddsw m0, m3 ; t1a t0a t3a t2a + psubsw m3, m1, m4 ; t13a t12a t15a t14a + paddsw m1, m4 ; t5a t4a t7a t6a + ITX_MUL4X_PACK 2, 4, 5, 6, 8, 799, 4017, 3406, 2276, 3 + psubw m6, m7, m5 + ITX_MUL2X_PACK 3, 5, _, 8, 6, 4, 6 + vpbroadcastd m6, [o(pw_m3784_1567)] + vpbroadcastd m5, [o(pw_1567_3784)] + psubsw m4, m0, m1 ; t5 t4 t7 t6 + paddsw m0, m1 ; t1 t0 t3 t2 + psubsw m1, m2, m3 ; t13a t12a t15a t14a + paddsw m2, m3 ; t9a t8a t11a t10a + psubw m3, m7, m6 ; pw_3784_m1567 + vpblendd m6, m3, 0xf0 + ITX_MUL2X_PACK 4, 3, _, 8, 6, 5, 4 ; t4a t5a t7a t6a + ITX_MUL2X_PACK 1, 3, _, 8, 6, 5, 4 ; t12 t13 t15 t14 + vbroadcasti128 m5, [o(deint_shuf)] + pshufb m0, m5 + pshufb m2, m5 + vperm2i128 m3, m0, m2, 0x31 ; t3 t2 t11a t10a + vinserti128 m0, xm2, 1 ; t1 t0 t9a t8a + vperm2i128 m2, m4, m1, 0x31 ; t7a t6a t15 t14 + vinserti128 m4, xm1, 1 ; t4a t5a t12 t13 + pshufd m2, m2, q1032 ; t6a t7a t14 t15 + psubsw m1, m0, m3 ; t3a t2a t11 t10 + paddsw m0, m3 ; -out15 out0 out14 -out1 + paddsw m3, m4, m2 ; -out3 out12 out2 -out13 + psubsw m4, m2 ; t6 t7 t14a t15a + shufps m2, m1, m4, q1032 ; t2a t6 t10 t14a + vpblendd m4, m1, 0x33 ; t3a t7 t11 t15a + ret +ALIGN function_align +.main_pass1_end: + vpbroadcastd m5, [o(pw_m2896_2896)] + vpbroadcastd m6, [o(pw_2896_2896)] + punpcklwd m1, m4, m2 + punpckhwd m4, m2 + pmaddwd m2, m5, m4 + pmaddwd m4, m6 + pmaddwd m5, m1 + pmaddwd m1, m6 + REPX {paddd x, m8}, m5, m1, m2, m4 + REPX {psrad x, 12}, m5, m2, m1, m4 + packssdw m2, m5 ; -out11 out8 out10 -out9 + packssdw m1, m4 ; -out7 out4 out6 -out5 + ret + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity + +cglobal iflipadst_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m5, [o(pw_16384)] + punpcklwd m4, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m3, m2 + punpckhwd m3, m2 + REPX {pmulhrsw x, m5}, m4, m1, m0, m3 + punpckldq m2, m3, m1 + punpckhdq m3, m1 + punpckhdq m1, m0, m4 + punpckldq m0, m4 + jmp tx2q +.pass2: + call m(iadst_4x16_internal_8bpc).main + vpbroadcastd m5, [o(pw_2896x8)] + paddsw m1, m2, m4 + psubsw m2, m4 + pmulhrsw m1, m5 ; -out7 out4 out6 -out5 + pmulhrsw m2, m5 ; out8 -out11 -out9 out10 + vpbroadcastd m6, [o(pw_2048)] + pshufd m1, m1, q1032 + vpblendd m4, m0, m2, 0x33 + vpblendd m0, m1, 0xcc + vpblendd m1, m3, 0xcc + vpblendd m2, m3, 0x33 + vpermq m0, m0, q3120 + vpermq m1, m1, q0213 + vpermq m2, m2, q2031 + vpermq m3, m4, q1302 + psubw m5, m7, m6 + jmp m(iadst_4x16_internal_8bpc).end + +INV_TXFM_4X16_FN identity, dct +INV_TXFM_4X16_FN identity, adst +INV_TXFM_4X16_FN identity, flipadst +INV_TXFM_4X16_FN identity, identity + +cglobal iidentity_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova m3, [cq+32*0] + mova m2, [cq+32*1] + mova m4, [cq+32*2] + mova m5, [cq+32*3] + vpbroadcastd m8, [o(pw_1697x8)] + pcmpeqw m0, m0 ; -1 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m5 + punpckhwd m4, m5 + pmulhrsw m5, m8, m1 + pmulhrsw m6, m8, m2 + pmulhrsw m7, m8, m3 + pmulhrsw m8, m4 + pcmpeqw m9, m0, m1 ; we want to do a signed avg, but pavgw is + pxor m1, m9 ; unsigned. as long as both signs are equal + pcmpeqw m9, m0, m2 ; it still works, but if the input is -1 the + pxor m2, m9 ; pmulhrsw result will become 0 which causes + pcmpeqw m9, m0, m3 ; pavgw to output -32768 instead of 0 unless + pxor m3, m9 ; we explicitly deal with that case here. + pcmpeqw m0, m4 + pxor m4, m0 + pavgw m1, m5 + pavgw m2, m6 + pavgw m3, m7 + pavgw m4, m8 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + vpbroadcastd m8, [o(pw_1697x16)] + vpbroadcastd m5, [o(pw_2048)] + pmulhrsw m4, m8, m0 + pmulhrsw m6, m8, m1 + pmulhrsw m7, m8, m2 + pmulhrsw m8, m3 + REPX {paddsw x, x}, m0, m1, m2, m3 + paddsw m0, m4 + paddsw m1, m6 + paddsw m2, m7 + paddsw m3, m8 + jmp m(iadst_4x16_internal_8bpc).end2 + +%macro WRITE_8X4 4-7 strideq*1, strideq*2, r3 ; coefs[1-2], tmp[1-2], off[1-3] + movq xm%3, [dstq ] + movhps xm%3, [dstq+%5] + movq xm%4, [dstq+%6] + movhps xm%4, [dstq+%7] + pmovzxbw m%3, xm%3 + pmovzxbw m%4, xm%4 +%ifnum %1 + paddw m%3, m%1 +%else + paddw m%3, %1 +%endif +%ifnum %2 + paddw m%4, m%2 +%else + paddw m%4, %2 +%endif + packuswb m%3, m%4 + vextracti128 xm%4, m%3, 1 + movq [dstq ], xm%3 + movhps [dstq+%6], xm%3 + movq [dstq+%5], xm%4 + movhps [dstq+%7], xm%4 +%endmacro + +%macro INV_TXFM_8X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x4 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + mov [cq], eobd + pmulhrsw xm0, xm1 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2 +%endif +%endmacro + +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst +INV_TXFM_8X4_FN dct, identity + +cglobal idct_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpbroadcastd xm3, [o(pw_2896x8)] + pmulhrsw xm0, xm3, [cq+16*0] + pmulhrsw xm1, xm3, [cq+16*1] + pmulhrsw xm2, xm3, [cq+16*2] + pmulhrsw xm3, [cq+16*3] + call m(idct_4x8_internal_8bpc).main + vbroadcasti128 m4, [o(deint_shuf)] + vinserti128 m3, m1, xm3, 1 + vinserti128 m1, m0, xm2, 1 + shufps m0, m1, m3, q0220 + shufps m1, m3, q1331 + pshufb m0, m4 + pshufb m1, m4 + jmp tx2q +.pass2: + IDCT4_1D_PACKED + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + jmp m(iadst_8x4_internal_8bpc).end2 + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpbroadcastd xm0, [o(pw_2896x8)] + pshufd xm4, [cq+16*0], q1032 + pmulhrsw xm3, xm0, [cq+16*3] + pshufd xm5, [cq+16*1], q1032 + pmulhrsw xm2, xm0, [cq+16*2] + pmulhrsw xm4, xm0 + pmulhrsw xm5, xm0 + call m(iadst_4x8_internal_8bpc).main_pass1 + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pxor m3, m3 + psubsw m3, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call .main +.end: + vpermq m0, m0, q3120 + vpermq m1, m1, q3120 +.end2: + vpbroadcastd m2, [o(pw_2048)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + WIN64_RESTORE_XMM +.end3: + pxor m2, m2 + mova [cq+32*0], m2 + mova [cq+32*1], m2 + lea r3, [strideq*3] + WRITE_8X4 0, 1, 4, 5 + RET +ALIGN function_align +cglobal_label .main + IADST4_1D_PACKED + ret + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpbroadcastd xm0, [o(pw_2896x8)] + pshufd xm4, [cq+16*0], q1032 + pmulhrsw xm3, xm0, [cq+16*3] + pshufd xm5, [cq+16*1], q1032 + pmulhrsw xm2, xm0, [cq+16*2] + pmulhrsw xm4, xm0 + pmulhrsw xm5, xm0 + call m(iadst_4x8_internal_8bpc).main_pass1 + vinserti128 m3, xm1, 1 + vinserti128 m2, xm0, 1 + punpckhwd m1, m3, m2 + punpcklwd m3, m2 + pxor m0, m0 + psubsw m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call m(iadst_8x4_internal_8bpc).main + mova m2, m1 + vpermq m1, m0, q2031 + vpermq m0, m2, q2031 + jmp m(iadst_8x4_internal_8bpc).end2 + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + mova xm2, [cq+16*0] + mova xm0, [cq+16*1] + vinserti128 m2, [cq+16*2], 1 + vinserti128 m0, [cq+16*3], 1 + vpbroadcastd m3, [o(pw_2896x8)] + punpcklwd m1, m2, m0 + punpckhwd m2, m0 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + paddsw m0, m0 + paddsw m1, m1 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_8x4_internal_8bpc).end + +%macro INV_TXFM_8X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x8 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + or r3d, 8 +.dconly: + pmulhrsw xm0, xm2 +.dconly2: + movd xm2, [pw_2048] + pmulhrsw xm0, xm1 + lea r2, [strideq*3] + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 +.dconly_loop: + WRITE_8X4 0, 0, 1, 2, strideq*1, strideq*2, r2 + lea dstq, [dstq+strideq*4] + sub r3d, 4 + jg .dconly_loop + RET +%endif +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst +INV_TXFM_8X8_FN dct, identity + +cglobal idct_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 ; 0 1 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m2, [cq+32*2], q3120 ; 4 5 + vpermq m1, [cq+32*1], q3120 ; 2 3 + call .main + shufps m4, m0, m1, q0220 + shufps m5, m0, m1, q1331 + shufps m1, m2, m3, q0220 + shufps m3, m2, m3, q1331 + vbroadcasti128 m0, [o(deint_shuf)] + vpbroadcastd m2, [o(pw_16384)] + REPX {pshufb x, m0}, m4, m5, m1, m3 + REPX {pmulhrsw x, m2}, m4, m5, m1, m3 + vinserti128 m0, m4, xm1, 1 + vperm2i128 m2, m4, m1, 0x31 + vinserti128 m1, m5, xm3, 1 + vperm2i128 m3, m5, m3, 0x31 + jmp tx2q +.pass2: + call .main + vpbroadcastd m4, [o(pw_2048)] + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + jmp m(iadst_8x8_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + IDCT8_1D_PACKED + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity + +cglobal iadst_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m4, [cq+32*0], q1302 ; 1 0 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m5, [cq+32*1], q1302 ; 3 2 + vpermq m2, [cq+32*2], q3120 ; 4 5 + call .main_pass1 + vpbroadcastd m5, [o(pw_16384)] + punpcklwd m4, m0, m1 + punpckhwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + pxor m3, m3 + psubw m3, m5 ; negate odd elements during rounding + pmulhrsw m4, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m5 + pmulhrsw m2, m3 + punpcklwd m3, m4, m0 + punpckhwd m4, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + vperm2i128 m2, m3, m0, 0x31 + vinserti128 m0, m3, xm0, 1 + vperm2i128 m3, m4, m1, 0x31 + vinserti128 m1, m4, xm1, 1 + jmp tx2q +.pass2: + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call .main_pass2 + vpbroadcastd m5, [o(pw_2048)] + vpbroadcastd xm4, [o(pw_4096)] + psubw m4, m5 ; lower half = 2048, upper half = -2048 +.end: + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 +.end2: + pmulhrsw m0, m4 + pmulhrsw m1, m4 +.end3: + pmulhrsw m2, m4 + pmulhrsw m3, m4 + WIN64_RESTORE_XMM +.end4: + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + mova [cq+32*2], m4 + mova [cq+32*3], m4 + lea r3, [strideq*3] + WRITE_8X4 0, 1, 4, 5 + lea dstq, [dstq+strideq*4] + WRITE_8X4 2, 3, 4, 5 + RET +ALIGN function_align +.main_pass1: + IADST8_1D_PACKED 1 + ret +ALIGN function_align +cglobal_label .main_pass2 + IADST8_1D_PACKED 2 + ret + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity + +cglobal iflipadst_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + vpermq m4, [cq+32*0], q1302 ; 1 0 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m5, [cq+32*1], q1302 ; 3 2 + vpermq m2, [cq+32*2], q3120 ; 4 5 + call m(iadst_8x8_internal_8bpc).main_pass1 + vpbroadcastd m5, [o(pw_16384)] + punpckhwd m4, m3, m2 + punpcklwd m3, m2 + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + pxor m0, m0 + psubw m0, m5 + pmulhrsw m4, m0 + pmulhrsw m3, m5 + pmulhrsw m2, m0 + pmulhrsw m1, m5 + punpckhwd m0, m4, m3 + punpcklwd m4, m3 + punpckhwd m3, m2, m1 + punpcklwd m2, m1 + vinserti128 m1, m0, xm3, 1 + vperm2i128 m3, m0, m3, 0x31 + vinserti128 m0, m4, xm2, 1 + vperm2i128 m2, m4, m2, 0x31 + jmp tx2q +.pass2: + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call m(iadst_8x8_internal_8bpc).main_pass2 + vpbroadcastd m4, [o(pw_2048)] + vpbroadcastd xm5, [o(pw_4096)] + psubw m4, m5 ; lower half = -2048, upper half = 2048 + vpermq m5, m3, q2031 + vpermq m3, m0, q2031 + vpermq m0, m2, q2031 + vpermq m2, m1, q2031 + pmulhrsw m1, m0, m4 + pmulhrsw m0, m5, m4 + jmp m(iadst_8x8_internal_8bpc).end3 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 + mova xm3, [cq+16*0] + mova xm2, [cq+16*1] + vinserti128 m3, [cq+16*4], 1 + vinserti128 m2, [cq+16*5], 1 + mova xm4, [cq+16*2] + mova xm0, [cq+16*3] + vinserti128 m4, [cq+16*6], 1 + vinserti128 m0, [cq+16*7], 1 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m0 + punpckhwd m4, m0 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + vpbroadcastd m4, [o(pw_4096)] + jmp m(iadst_8x8_internal_8bpc).end + +%macro INV_TXFM_8X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x16 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 16 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly +%endif +%endmacro + +%macro ITX_8X16_LOAD_COEFS 0 + vpbroadcastd m4, [o(pw_2896x8)] + pmulhrsw m0, m4, [cq+32*0] + add cq, 32*4 + pmulhrsw m7, m4, [cq+32*3] + pmulhrsw m1, m4, [cq-32*3] + pmulhrsw m6, m4, [cq+32*2] + pmulhrsw m2, m4, [cq-32*2] + pmulhrsw m5, m4, [cq+32*1] + pmulhrsw m3, m4, [cq-32*1] + pmulhrsw m4, [cq+32*0] +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst +INV_TXFM_8X16_FN dct, identity + +cglobal idct_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_8X16_LOAD_COEFS + call m(idct_16x8_internal_8bpc).main + vpbroadcastd m10, [o(pw_16384)] +.pass1_end: + vperm2i128 m9, m3, m7, 0x31 + vinserti128 m3, xm7, 1 + vperm2i128 m8, m2, m6, 0x31 + vinserti128 m2, xm6, 1 + vperm2i128 m6, m1, m5, 0x31 + vinserti128 m1, xm5, 1 + vperm2i128 m5, m0, m4, 0x31 + vinserti128 m0, xm4, 1 + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m1 + punpcklwd m0, m1 +.pass1_end2: + punpckhwd m7, m5, m6 + punpcklwd m5, m6 + punpcklwd m6, m8, m9 + punpckhwd m8, m9 + REPX {pmulhrsw x, m10}, m2, m0, m4, m3, m5, m6, m7, m8 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + punpckldq m4, m5, m6 + punpckhdq m5, m6 + punpckldq m6, m7, m8 + punpckhdq m7, m8 + jmp tx2q +.pass2: + call .main + REPX {vpermq x, x, q3120}, m0, m2, m4, m6 + REPX {vpermq x, x, q2031}, m1, m3, m5, m7 +.end: + vpbroadcastd m8, [o(pw_2048)] +.end2: + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 +.end3: + pxor m8, m8 + REPX {mova [cq+32*x], m8}, -4, -3, -2, -1, 0, 1, 2, 3 + lea r3, [strideq*3] + WRITE_8X4 0, 1, 8, 9 + lea dstq, [dstq+strideq*4] + WRITE_8X4 2, 3, 0, 1 + lea dstq, [dstq+strideq*4] + WRITE_8X4 4, 5, 0, 1 + lea dstq, [dstq+strideq*4] + WRITE_8X4 6, 7, 0, 1 + RET +ALIGN function_align +cglobal_label .main + IDCT16_1D_PACKED + ret + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity + +cglobal iadst_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_8X16_LOAD_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + vpbroadcastd m10, [o(pw_16384)] + pslld m9, m10, 17 + psubw m10, m9 ; 16384, -16384 + jmp m(idct_8x16_internal_8bpc).pass1_end +ALIGN function_align +.pass2: + call .main + call .main_pass2_end + vpbroadcastd m9, [o(pw_2048)] + vpbroadcastd xm8, [o(pw_4096)] + psubw m8, m9 + REPX {vpermq x, x, q2031}, m0, m1, m2, m3 + REPX {vpermq x, x, q3120}, m4, m5, m6, m7 + jmp m(idct_8x16_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + REPX {pshufd x, x, q1032}, m7, m1, m5, m3 +.main2: + vpbroadcastd m10, [o(pd_2048)] + punpckhwd m8, m7, m0 ; in14 in1 + punpcklwd m0, m7 ; in0 in15 + punpcklwd m7, m6, m1 ; in12 in3 + punpckhwd m1, m6 ; in2 in13 + punpckhwd m6, m5, m2 ; in10 in5 + punpcklwd m2, m5 ; in4 in11 + punpcklwd m5, m4, m3 ; in8 in7 + punpckhwd m3, m4 ; in6 in9 + ITX_MUL2X_PACK 0, 4, 9, 10, 201, 4091, 3 ; t0 t1 + ITX_MUL2X_PACK 1, 4, 9, 10, 995, 3973, 3 ; t2 t3 + ITX_MUL2X_PACK 2, 4, 9, 10, 1751, 3703, 3 ; t4 t5 + ITX_MUL2X_PACK 3, 4, 9, 10, 2440, 3290, 3 ; t6 t7 + ITX_MUL2X_PACK 5, 4, 9, 10, 3035, 2751, 3 ; t8 t9 + ITX_MUL2X_PACK 6, 4, 9, 10, 3513, 2106, 3 ; t10 t11 + ITX_MUL2X_PACK 7, 4, 9, 10, 3857, 1380, 3 ; t12 t13 + ITX_MUL2X_PACK 8, 4, 9, 10, 4052, 601, 3 ; t14 t15 + psubsw m4, m0, m5 ; t9a t8a + paddsw m0, m5 ; t1a t0a + psubsw m5, m1, m6 ; t11a t10a + paddsw m1, m6 ; t3a t2a + psubsw m6, m2, m7 ; t13a t12a + paddsw m2, m7 ; t5a t4a + psubsw m7, m3, m8 ; t15a t14a + paddsw m3, m8 ; t7a t6a + vpbroadcastd m11, [o(pw_m4017_799)] + vpbroadcastd m12, [o(pw_799_4017)] + pxor m9, m9 + ITX_MUL2X_PACK 4, 8, _, 10, 11, 12, 6 ; t8 t9 + psubw m8, m9, m11 ; pw_4017_m799 + ITX_MUL2X_PACK 6, 12, _, 10, 12, 8, 6 ; t12 t13 + vpbroadcastd m11, [o(pw_m2276_3406)] + vpbroadcastd m12, [o(pw_3406_2276)] + ITX_MUL2X_PACK 5, 8, _, 10, 11, 12, 6 ; t10 t11 + psubw m8, m9, m11 ; pw_2276_m3406 + ITX_MUL2X_PACK 7, 12, _, 10, 12, 8, 6 ; t14 t15 + psubsw m8, m1, m3 ; t7 t6 + paddsw m1, m3 ; t3 t2 + psubsw m3, m0, m2 ; t5 t4 + paddsw m0, m2 ; t1 t0 + psubsw m2, m5, m7 ; t14a t15a + paddsw m7, m5 ; t10a t11a + psubsw m5, m4, m6 ; t12a t13a + paddsw m4, m6 ; t8a t9a + vpbroadcastd m11, [o(pw_m3784_1567)] + vpbroadcastd m12, [o(pw_1567_3784)] + ITX_MUL2X_PACK 3, 6, _, 10, 12, 11, 6 ; t5a t4a + psubw m6, m9, m11 ; pw_3784_m1567 + ITX_MUL2X_PACK 8, 6, _, 10, 6, 12, 6 ; t7a t6a + vpbroadcastd m11, [o(pw_m1567_3784)] + vpbroadcastd m12, [o(pw_3784_1567)] + ITX_MUL2X_PACK 2, 6, _, 10, 11, 12, 6 ; t15 t14 + psubw m6, m9, m11 ; pw_1567_m3784 + ITX_MUL2X_PACK 5, 12, _, 10, 12, 6, 6 ; t13 t12 + vbroadcasti128 m12, [o(deint_shuf)] + paddsw m6, m4, m7 ; -out1 out14 + psubsw m4, m7 ; t10 t11 + psubsw m11, m3, m8 ; t7 t6 + paddsw m8, m3 ; out12 -out3 + psubsw m3, m0, m1 ; t3a t2a + paddsw m0, m1 ; -out15 out0 + paddsw m1, m2, m5 ; -out13 out2 + psubsw m5, m2 ; t15a t14a + pshufb m0, m12 + pshufb m6, m12 + pshufb m8, m12 + pshufb m1, m12 + shufps m7, m6, m0, q1032 ; out14 -out15 + vpblendd m0, m6, 0x33 ; -out1 out0 + punpcklqdq m6, m8, m1 ; out12 -out13 + punpckhqdq m1, m8, m1 ; -out3 out2 + ret +ALIGN function_align +.main_pass1_end: + vpbroadcastd m8, [o(pw_m2896_2896)] + vpbroadcastd m12, [o(pw_2896_2896)] + pmaddwd m9, m8, m11 ; -out11 + pmaddwd m2, m12, m5 ; -out5 + pmaddwd m5, m8 ; out10 + pmaddwd m11, m12 ; out4 + REPX {paddd x, m10}, m9, m5, m2, m11 + REPX {psrad x, 12 }, m9, m5, m2, m11 + packssdw m5, m9 ; out10 -out11 + packssdw m2, m11 ; -out5 out4 + pmaddwd m11, m8, m3 ; out8 + vpbroadcastd m8, [o(pw_2896_m2896)] + pmaddwd m3, m12 ; -out7 + pmaddwd m8, m4 ; -out9 + pmaddwd m4, m12 ; out6 + REPX {paddd x, m10}, m11, m3, m8, m4 + REPX {psrad x, 12 }, m11, m3, m8, m4 + packssdw m3, m4 ; -out7 out6 + packssdw m4, m11, m8 ; out8 -out9 + vpbroadcastd m10, [o(pw_16384)] + pxor m9, m9 + ret +ALIGN function_align +cglobal_label .main_pass2_end + vpbroadcastd m8, [o(pw_2896x8)] + pshufb m2, m11, m12 + pshufb m5, m12 + pshufb m3, m12 + pshufb m4, m12 + punpcklqdq m11, m5, m2 ; t15a t7 + punpckhqdq m5, m2 ; t14a t6 + shufps m2, m3, m4, q1032 ; t2a t10 + vpblendd m3, m4, 0xcc ; t3a t11 + psubsw m4, m2, m3 ; out8 -out9 + paddsw m3, m2 ; -out7 out6 + paddsw m2, m5, m11 ; -out5 out4 + psubsw m5, m11 ; out10 -out11 + REPX {pmulhrsw x, m8}, m2, m3, m4, m5 + ret + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity + +cglobal iflipadst_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_8X16_LOAD_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + vpbroadcastd m9, [o(pw_16384)] + pslld m10, m9, 17 + psubw m10, m9 ; -16384, 16384 + vperm2i128 m9, m4, m0, 0x31 + vinserti128 m0, m4, xm0, 1 + vperm2i128 m8, m5, m1, 0x31 + vinserti128 m4, m5, xm1, 1 + vperm2i128 m5, m7, m3, 0x31 + vinserti128 m3, m7, xm3, 1 + vinserti128 m1, m6, xm2, 1 + vperm2i128 m6, m6, m2, 0x31 + punpcklwd m2, m4, m0 + punpckhwd m4, m0 + punpcklwd m0, m3, m1 + punpckhwd m3, m1 + jmp m(idct_8x16_internal_8bpc).pass1_end2 +.pass2: + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end + vpbroadcastd m8, [o(pw_2048)] + vpbroadcastd xm9, [o(pw_4096)] + psubw m8, m9 + vpermq m9, m0, q3120 + vpermq m0, m7, q2031 + vpermq m7, m1, q3120 + vpermq m1, m6, q2031 + vpermq m6, m2, q3120 + vpermq m2, m5, q2031 + vpermq m5, m3, q3120 + vpermq m3, m4, q2031 + pmulhrsw m0, m8 + pmulhrsw m1, m8 + pmulhrsw m2, m8 + pmulhrsw m3, m8 + pmulhrsw m4, m5, m8 + pmulhrsw m5, m6, m8 + pmulhrsw m6, m7, m8 + pmulhrsw m7, m9, m8 + jmp m(idct_8x16_internal_8bpc).end3 + +INV_TXFM_8X16_FN identity, dct +INV_TXFM_8X16_FN identity, adst +INV_TXFM_8X16_FN identity, flipadst +INV_TXFM_8X16_FN identity, identity + +%macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] + pmulhrsw m%2, m%3, m%1 +%if %0 == 4 ; if downshifting by 1 + pmulhrsw m%2, m%4 +%else + paddsw m%1, m%1 +%endif + paddsw m%1, m%2 +%endmacro + +cglobal iidentity_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + mova xm3, [cq+16*0] + mova xm2, [cq+16*2] + add cq, 16*8 + vinserti128 m3, [cq+16*0], 1 + vinserti128 m2, [cq+16*2], 1 + vpbroadcastd m9, [o(pw_2896x8)] + mova xm4, [cq-16*4] + mova xm5, [cq-16*2] + vinserti128 m4, [cq+16*4], 1 + vinserti128 m5, [cq+16*6], 1 + mova xm7, [cq-16*7] + mova xm6, [cq-16*5] + vinserti128 m7, [cq+16*1], 1 + vinserti128 m6, [cq+16*3], 1 + mova xm8, [cq-16*3] + mova xm0, [cq-16*1] + vinserti128 m8, [cq+16*5], 1 + vinserti128 m0, [cq+16*7], 1 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m5 + punpckhwd m4, m5 + punpcklwd m5, m7, m6 + punpckhwd m7, m6 + punpcklwd m6, m8, m0 + punpckhwd m8, m0 + REPX {pmulhrsw x, m9}, m1, m2, m3, m4, m5, m6, m7, m8 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + punpckldq m4, m5, m6 + punpckhdq m5, m6 + punpckldq m6, m7, m8 + punpckhdq m7, m8 + jmp tx2q +.pass2: + vpbroadcastd m8, [o(pw_1697x16)] + REPX {vpermq x, x, q3120}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {IDTX16 x, 9, 8}, 0, 1, 2, 3, 4, 5, 6, 7 + jmp m(idct_8x16_internal_8bpc).end + +%macro WRITE_16X2 6 ; coefs[1-2], tmp[1-2], offset[1-2] + pmovzxbw m%3, [dstq+%5] +%ifnum %1 + paddw m%3, m%1 +%else + paddw m%3, %1 +%endif + pmovzxbw m%4, [dstq+%6] +%ifnum %2 + paddw m%4, m%2 +%else + paddw m%4, %2 +%endif + packuswb m%3, m%4 + vpermq m%3, m%3, q3120 + mova [dstq+%5], xm%3 + vextracti128 [dstq+%6], m%3, 1 +%endmacro + +%macro INV_TXFM_16X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x4 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + or r3d, 4 +.dconly: + pmulhrsw xm0, xm2 + movd xm2, [pw_2048] ; intentionally rip-relative + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 + pxor m3, m3 +.dconly_loop: + mova xm1, [dstq+strideq*0] + vinserti128 m1, [dstq+strideq*1], 1 + punpckhbw m2, m1, m3 + punpcklbw m1, m3 + paddw m2, m0 + paddw m1, m0 + packuswb m1, m2 + mova [dstq+strideq*0], xm1 + vextracti128 [dstq+strideq*1], m1, 1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +%endif +%endmacro + +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst +INV_TXFM_16X4_FN dct, identity + +cglobal idct_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova xm0, [cq+16*0] + mova xm1, [cq+16*1] + mova xm2, [cq+16*2] + mova xm3, [cq+16*3] + mova xm4, [cq+16*4] + mova xm5, [cq+16*5] + mova xm6, [cq+16*6] + mova xm7, [cq+16*7] + call m(idct_4x16_internal_8bpc).main + vinserti128 m6, m2, xm6, 1 + vinserti128 m2, m0, xm4, 1 + vinserti128 m0, m1, xm5, 1 + vinserti128 m1, m3, xm7, 1 + punpcklwd m3, m2, m6 + punpckhwd m2, m6 + vpbroadcastd m6, [o(pw_16384)] + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + mova m1, m6 + jmp m(iadst_16x4_internal_8bpc).pass1_end +.pass2: + call .main + jmp m(iadst_16x4_internal_8bpc).end +ALIGN function_align +cglobal_label .main + vpbroadcastd m6, [o(pd_2048)] + IDCT4_1D 0, 1, 2, 3, 4, 5, 6 + ret + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q1230 + vpermq m3, [cq+32*3], q2103 + vpermq m1, [cq+32*1], q1230 + vpermq m2, [cq+32*2], q2103 + call m(iadst_4x16_internal_8bpc).main2 + call m(iadst_4x16_internal_8bpc).main_pass1_end + punpcklwd m4, m3, m1 + punpcklwd m5, m2, m0 + punpckhwd m0, m1 + punpckhwd m2, m3 + vpbroadcastd m1, [o(pw_16384)] + vinserti128 m3, m0, xm2, 1 + vperm2i128 m2, m0, m2, 0x31 + vinserti128 m0, m4, xm5, 1 + vperm2i128 m4, m4, m5, 0x31 + psubw m6, m7, m1 +.pass1_end: + pmulhrsw m3, m1 + pmulhrsw m2, m6 + pmulhrsw m4, m1 + pmulhrsw m0, m6 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m0 + punpckhwd m4, m0 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + call .main +.end: + vpbroadcastd m4, [o(pw_2048)] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + WIN64_RESTORE_XMM +.end2: + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + mova [cq+32*2], m4 + mova [cq+32*3], m4 +.end3: + WRITE_16X2 0, 1, 4, 5, strideq*0, strideq*1 + lea dstq, [dstq+strideq*2] + WRITE_16X2 2, 3, 4, 5, strideq*0, strideq*1 + RET +ALIGN function_align +cglobal_label .main + vpbroadcastd m6, [o(pw_m3344_3344)] + vpbroadcastd m7, [o(pw_3803_1321)] + vpbroadcastd m8, [o(pw_m1321_2482)] + vpbroadcastd m9, [o(pw_2482_3344)] + punpcklwd m4, m2, m0 ; in2 in0 l + punpckhwd m2, m0 ; in2 in0 h + psrld m5, m6, 16 + pmaddwd m10, m6, m4 ; t2:02 l + pmaddwd m6, m2 ; t2:02 h + pmaddwd m0, m7, m4 ; t0:02 l + pmaddwd m7, m2 ; t0:02 h + pmaddwd m4, m8 ; t1:02 l + pmaddwd m8, m2 ; t1:02 h + punpckhwd m2, m3, m1 ; in3 in1 h + punpcklwd m3, m1 ; in3 in1 l + pmaddwd m1, m5, m2 ; t2:3 h + pmaddwd m5, m3 ; t2:3 l + paddd m6, m1 + vpbroadcastd m1, [o(pd_2048)] + paddd m10, m5 + pmaddwd m5, m9, m3 + pmaddwd m9, m2 + paddd m0, m1 + paddd m7, m1 + paddd m0, m5 ; t0 + t3 + 2048 l + paddd m7, m9 ; t0 + t3 + 2048 h + vpbroadcastd m9, [o(pw_m3803_3344)] + pmaddwd m5, m9, m2 + pmaddwd m9, m3 + paddd m10, m1 ; t2 + 2048 l + paddd m6, m1 ; t2 + 2048 h + paddd m5, m1 ; t1:13 + 2048 h + paddd m1, m9 ; t1:13 + 2048 l + vpbroadcastd m9, [o(pw_m3803_m6688)] + pmaddwd m2, m9 + pmaddwd m3, m9 + paddd m5, m8 ; t1 + t3 + 2048 h + paddd m1, m4 ; t1 + t3 + 2048 l + paddd m8, m7 + paddd m4, m0 + paddd m2, m8 ; t0 + t1 - t3 + 2048 h + paddd m3, m4 ; t0 + t1 - t3 + 2048 l + REPX {psrad x, 12}, m10, m6, m0, m7, m5, m1, m2, m3 + packssdw m0, m7 + packssdw m1, m5 + packssdw m3, m2 + packssdw m2, m10, m6 + ret + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q1230 + vpermq m3, [cq+32*3], q2103 + vpermq m1, [cq+32*1], q1230 + vpermq m2, [cq+32*2], q2103 + call m(iadst_4x16_internal_8bpc).main2 + call m(iadst_4x16_internal_8bpc).main_pass1_end + punpckhwd m4, m3, m2 + punpckhwd m5, m1, m0 + punpcklwd m0, m2 + punpcklwd m1, m3 + vpbroadcastd m6, [o(pw_16384)] + vinserti128 m3, m0, xm1, 1 + vperm2i128 m2, m0, m1, 0x31 + vinserti128 m0, m4, xm5, 1 + vperm2i128 m4, m4, m5, 0x31 + psubw m1, m7, m6 + jmp m(iadst_16x4_internal_8bpc).pass1_end +ALIGN function_align +.pass2: + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m4, [o(pw_2048)] + REPX {pmulhrsw x, m4}, m3, m2, m1, m0 + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + mova [cq+32*2], m4 + mova [cq+32*3], m4 + WRITE_16X2 3, 2, 4, 5, strideq*0, strideq*1 + lea dstq, [dstq+strideq*2] + WRITE_16X2 1, 0, 4, 5, strideq*0, strideq*1 + RET + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 + mova xm2, [cq+16*0] + mova xm4, [cq+16*1] + vinserti128 m2, [cq+16*4], 1 + vinserti128 m4, [cq+16*5], 1 + mova xm0, [cq+16*2] + mova xm1, [cq+16*3] + vinserti128 m0, [cq+16*6], 1 + vinserti128 m1, [cq+16*7], 1 + vpbroadcastd m7, [o(pw_1697x16)] + vpbroadcastd m8, [o(pw_16384)] + punpcklwd m3, m2, m4 + punpckhwd m2, m4 + punpcklwd m4, m0, m1 + punpckhwd m0, m1 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m0 + punpckhwd m4, m0 + pmulhrsw m0, m7, m1 + pmulhrsw m5, m7, m2 + pmulhrsw m6, m7, m3 + pmulhrsw m7, m4 + REPX {pmulhrsw x, m8}, m0, m5, m6, m7 + paddsw m1, m0 + paddsw m2, m5 + paddsw m3, m6 + paddsw m4, m7 + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + jmp tx2q +.pass2: + vpbroadcastd m7, [o(pw_1697x8)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(iadst_16x4_internal_8bpc).end + +%macro INV_TXFM_16X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x8 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 8 + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +%endif +%endmacro + +%macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd + vpbroadcastd m8, [o(pw_2896x8)] + vpermq m0, [cq+32*0], q3120 + add cq, 32*4 + vpermq m7, [cq+32*3], q%1 + vpermq m1, [cq-32*3], q%1 + vpermq m6, [cq+32*2], q3120 + vpermq m2, [cq-32*2], q3120 + vpermq m5, [cq+32*1], q%1 + vpermq m3, [cq-32*1], q%1 + vpermq m4, [cq+32*0], q3120 + REPX {pmulhrsw x, m8}, m0, m7, m1, m6, m2, m5, m3, m4 +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst +INV_TXFM_16X8_FN dct, identity + +cglobal idct_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_16X8_LOAD_COEFS 3120 + call m(idct_8x16_internal_8bpc).main + vpbroadcastd m10, [o(pw_16384)] + punpckhwd m8, m0, m2 + punpcklwd m0, m2 + punpckhwd m2, m1, m3 + punpcklwd m1, m3 + punpcklwd m9, m4, m6 + punpckhwd m4, m6 + punpcklwd m6, m5, m7 + punpckhwd m5, m7 + REPX {pmulhrsw x, m10}, m8, m1, m4, m6 +.pass1_end: + REPX {pmulhrsw x, m10}, m0, m2, m9, m5 + punpckhwd m3, m0, m8 + punpcklwd m0, m8 + punpckhwd m8, m2, m1 + punpcklwd m2, m1 + punpcklwd m7, m9, m4 + punpckhwd m9, m4 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m8 + punpckhdq m3, m8 + punpckldq m6, m7, m4 + punpckhdq m7, m4 + punpckldq m8, m9, m5 + punpckhdq m9, m5 + vperm2i128 m4, m0, m6, 0x31 + vinserti128 m0, xm6, 1 + vperm2i128 m5, m1, m7, 0x31 + vinserti128 m1, xm7, 1 + vperm2i128 m6, m2, m8, 0x31 + vinserti128 m2, xm8, 1 + vperm2i128 m7, m3, m9, 0x31 + vinserti128 m3, xm9, 1 + jmp tx2q +.pass2: + call .main + vpbroadcastd m8, [o(pw_2048)] +.end: + REPX {pmulhrsw x, m8}, m0, m2, m4, m6 +.end2: + REPX {pmulhrsw x, m8}, m1, m3, m5, m7 + lea r3, [strideq*3] + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r3 +.end3: + pxor m0, m0 + REPX {mova [cq+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 +.end4: + lea dstq, [dstq+strideq*4] + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r3 + RET +ALIGN function_align +cglobal_label .main + vpbroadcastd m10, [o(pd_2048)] +.main2: + IDCT8_1D 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + ret + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity + +cglobal iadst_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_16X8_LOAD_COEFS 1302 + call m(iadst_8x16_internal_8bpc).main2 + call m(iadst_8x16_internal_8bpc).main_pass1_end + psubw m11, m9, m10 + punpcklwd m8, m0, m2 + punpckhwd m0, m2 + punpckhwd m2, m1, m3 + punpcklwd m1, m3 + punpcklwd m9, m4, m6 + punpckhwd m4, m6 + punpckhwd m6, m5, m7 + punpcklwd m5, m7 + REPX {pmulhrsw x, m11}, m8, m1, m4, m6 + jmp m(idct_16x8_internal_8bpc).pass1_end +ALIGN function_align +.pass2: + call .main + call .main_pass2_end + pxor m8, m8 + psubw m8, m9 + REPX {pmulhrsw x, m9}, m0, m2, m4, m6 + jmp m(idct_16x8_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + vpbroadcastd m10, [o(pd_2048)] + ITX_MULSUB_2W 7, 0, 8, 9, 10, 401, 4076 ; t1a, t0a + ITX_MULSUB_2W 3, 4, 8, 9, 10, 3166, 2598 ; t5a, t4a + ITX_MULSUB_2W 1, 6, 8, 9, 10, 3920, 1189 ; t7a, t6a + ITX_MULSUB_2W 5, 2, 8, 9, 10, 1931, 3612 ; t3a, t2a + psubsw m8, m2, m6 ; t6 + paddsw m2, m6 ; t2 + psubsw m6, m0, m4 ; t4 + paddsw m0, m4 ; t0 + psubsw m4, m5, m1 ; t7 + paddsw m5, m1 ; t3 + psubsw m1, m7, m3 ; t5 + paddsw m7, m3 ; t1 + ITX_MULSUB_2W 6, 1, 3, 9, 10, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2W 4, 8, 3, 9, 10, 3784, 1567 ; t6a, t7a + psubsw m9, m6, m8 ; t7 + paddsw m6, m8 ; out6 + psubsw m3, m7, m5 ; t3 + paddsw m7, m5 ; -out7 + psubsw m5, m0, m2 ; t2 + paddsw m0, m2 ; out0 + psubsw m2, m1, m4 ; t6 + paddsw m1, m4 ; -out1 + ret +ALIGN function_align +.main_pass1_end: + vpbroadcastd m11, [o(pw_m2896_2896)] + vpbroadcastd m12, [o(pw_2896_2896)] + punpckhwd m4, m3, m5 + punpcklwd m3, m5 + pmaddwd m5, m11, m4 + pmaddwd m4, m12 + pmaddwd m8, m11, m3 + pmaddwd m3, m12 + REPX {paddd x, m10}, m5, m4, m8, m3 + REPX {psrad x, 12 }, m5, m8, m4, m3 + packssdw m3, m4 ; -out3 + packssdw m4, m8, m5 ; out4 + punpcklwd m5, m9, m2 + punpckhwd m9, m2 + pmaddwd m2, m12, m5 + pmaddwd m5, m11 + pmaddwd m12, m9 + pmaddwd m11, m9 + REPX {paddd x, m10}, m2, m5, m12, m11 + REPX {psrad x, 12 }, m2, m12, m5, m11 + packssdw m2, m12 ; out2 + packssdw m5, m11 ; -out5 + ret +ALIGN function_align +cglobal_label .main_pass2_end + vpbroadcastd m8, [o(pw_2896x8)] + psubsw m4, m5, m3 + paddsw m3, m5 + psubsw m5, m2, m9 + paddsw m2, m9 + pmulhrsw m2, m8 ; out2 + pmulhrsw m3, m8 ; -out3 + pmulhrsw m4, m8 ; out4 + pmulhrsw m5, m8 ; -out5 + vpbroadcastd m9, [o(pw_2048)] + ret + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity + +cglobal iflipadst_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + ITX_16X8_LOAD_COEFS 1302 + call m(iadst_8x16_internal_8bpc).main2 + call m(iadst_8x16_internal_8bpc).main_pass1_end + psubw m9, m10 + punpcklwd m8, m6, m4 + punpckhwd m6, m4 + punpcklwd m4, m7, m5 + punpckhwd m7, m5 + punpckhwd m5, m3, m1 + punpcklwd m3, m1 + punpckhwd m1, m2, m0 + punpcklwd m2, m0 + REPX {pmulhrsw x, m10}, m8, m4, m5, m1 + REPX {pmulhrsw x, m9 }, m6, m7, m3, m2 + punpcklwd m0, m7, m4 + punpckhwd m7, m4 + punpckhwd m4, m6, m8 + punpcklwd m6, m8 + punpckhwd m8, m3, m5 + punpcklwd m3, m5 + punpcklwd m5, m2, m1 + punpckhwd m2, m1 + punpckhdq m1, m0, m6 + punpckldq m0, m6 + punpckldq m6, m7, m4 + punpckhdq m7, m4 + punpckhdq m4, m3, m5 + punpckldq m3, m5 + punpckldq m5, m8, m2 + punpckhdq m8, m2 + vinserti128 m2, m6, xm5, 1 + vperm2i128 m6, m5, 0x31 + vperm2i128 m5, m1, m4, 0x31 + vinserti128 m1, xm4, 1 + vperm2i128 m4, m0, m3, 0x31 + vinserti128 m0, xm3, 1 + vinserti128 m3, m7, xm8, 1 + vperm2i128 m7, m8, 0x31 + jmp tx2q +.pass2: + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + pxor m8, m8 + psubw m8, m9 + pmulhrsw m10, m7, m8 + pmulhrsw m7, m0, m9 + pmulhrsw m0, m6, m9 + pmulhrsw m6, m1, m8 + pmulhrsw m1, m5, m8 + pmulhrsw m5, m2, m9 + pmulhrsw m2, m4, m9 + pmulhrsw m4, m3, m8 + lea r3, [strideq*3] + WRITE_16X2 10, 0, 8, 9, strideq*0, strideq*1 + WRITE_16X2 1, 2, 0, 1, strideq*2, r3 + jmp m(idct_16x8_internal_8bpc).end3 + +INV_TXFM_16X8_FN identity, dct +INV_TXFM_16X8_FN identity, adst +INV_TXFM_16X8_FN identity, flipadst +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 + mova xm7, [cq+16*0] + mova xm2, [cq+16*1] + add cq, 16*8 + vpbroadcastd m3, [o(pw_2896x8)] + vinserti128 m7, [cq+16*0], 1 + vinserti128 m2, [cq+16*1], 1 + mova xm6, [cq-16*6] + mova xm4, [cq-16*5] + vinserti128 m6, [cq+16*2], 1 + vinserti128 m4, [cq+16*3], 1 + mova xm8, [cq-16*4] + mova xm5, [cq-16*3] + vinserti128 m8, [cq+16*4], 1 + vinserti128 m5, [cq+16*5], 1 + mova xm0, [cq-16*2] + mova xm1, [cq-16*1] + vinserti128 m0, [cq+16*6], 1 + vinserti128 m1, [cq+16*7], 1 + vpbroadcastd m10, [o(pw_1697x16)] + vpbroadcastd m11, [o(pw_16384)] + REPX {pmulhrsw x, m3}, m7, m2, m6, m4, m8, m5, m0, m1 + punpcklwd m3, m7, m2 + punpckhwd m7, m2 + punpcklwd m2, m6, m4 + punpckhwd m6, m4 + punpcklwd m4, m8, m5 + punpckhwd m8, m5 + punpcklwd m5, m0, m1 + punpckhwd m0, m1 + punpckldq m1, m3, m2 + punpckhdq m3, m2 + punpckldq m2, m4, m5 + punpckhdq m4, m5 + punpckldq m5, m7, m6 + punpckhdq m7, m6 + punpckldq m6, m8, m0 + punpckhdq m8, m0 + REPX {IDTX16 x, 0, 10, 11}, 1, 3, 2, 4, 5, 7, 6, 8 + punpcklqdq m0, m1, m2 + punpckhqdq m1, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m6 + punpckhqdq m5, m6 + punpcklqdq m6, m7, m8 + punpckhqdq m7, m8 + jmp tx2q +.pass2: + vpbroadcastd m8, [o(pw_4096)] + jmp m(idct_16x8_internal_8bpc).end + +%define o_base pw_5 + 128 + +%macro INV_TXFM_16X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x16 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 16 + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +%endif +%endmacro + +%macro ITX_16X16_LOAD_COEFS 0 + mova m0, [cq+32*0] + mova m1, [cq+32*1] + mova m2, [cq+32*2] + mova m3, [cq+32*3] + add cq, 32*8 + mova m4, [cq-32*4] + mova m5, [cq-32*3] + mova m6, [cq-32*2] + mova m7, [cq-32*1] + mova m8, [cq+32*0] + mova m9, [cq+32*1] + mova m10, [cq+32*2] + mova m11, [cq+32*3] + mova m12, [cq+32*4] + mova m13, [cq+32*5] + mova m14, [cq+32*6] + mova m15, [cq+32*7] + mova [rsp], m15 +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst +INV_TXFM_16X16_FN dct, identity + +cglobal idct_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 + ITX_16X16_LOAD_COEFS + call .main +.pass1_end: + vpbroadcastd m1, [o(pw_8192)] + REPX {pmulhrsw x, m1}, m0, m2, m4, m6, m8, m10, m12, m14 + vextracti128 [rsp+16*5], m8, 1 + mova [rsp+16*1], xm8 +.pass1_end2: + vextracti128 [rsp+16*4], m0, 1 + mova [rsp+16*0], xm0 + REPX {pmulhrsw x, m1}, m3, m5, m7, m9, m11, m13, m15 + pmulhrsw m1, [rsp+32*1] + vperm2i128 m8, m1, m9, 0x31 + vinserti128 m1, xm9, 1 + vperm2i128 m9, m2, m10, 0x31 + vinserti128 m2, xm10, 1 + vperm2i128 m10, m3, m11, 0x31 + vinserti128 m3, xm11, 1 + vperm2i128 m11, m4, m12, 0x31 + vinserti128 m4, xm12, 1 + vperm2i128 m12, m5, m13, 0x31 + vinserti128 m5, xm13, 1 + vperm2i128 m13, m6, m14, 0x31 + vinserti128 m6, xm14, 1 + vperm2i128 m14, m7, m15, 0x31 + vinserti128 m7, xm15, 1 + mova m15, [rsp+32*2] +.pass1_end3: + punpcklwd m0, m9, m10 + punpckhwd m9, m10 + punpcklwd m10, m15, m8 + punpckhwd m15, m8 + punpckhwd m8, m11, m12 + punpcklwd m11, m12 + punpckhwd m12, m13, m14 + punpcklwd m13, m14 + punpckhdq m14, m11, m13 + punpckldq m11, m13 + punpckldq m13, m15, m9 + punpckhdq m15, m9 + punpckldq m9, m10, m0 + punpckhdq m10, m0 + punpckhdq m0, m8, m12 + punpckldq m8, m12 + punpcklqdq m12, m13, m8 + punpckhqdq m13, m8 + punpcklqdq m8, m9, m11 + punpckhqdq m9, m11 + punpckhqdq m11, m10, m14 + punpcklqdq m10, m14 + punpcklqdq m14, m15, m0 + punpckhqdq m15, m0 + mova m0, [rsp] + mova [rsp], m15 + punpckhwd m15, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m7 + punpcklwd m6, m7 + punpckhwd m7, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckhdq m6, m5, m7 + punpckldq m5, m7 + punpckldq m7, m15, m1 + punpckhdq m15, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m15 + punpcklqdq m6, m15 + jmp tx2q +.pass2: + call .main +.end: + vpbroadcastd m1, [o(pw_2048)] + REPX {pmulhrsw x, m1}, m0, m2, m4, m6, m8, m10, m12, m14 + mova [rsp], m6 +.end2: + REPX {pmulhrsw x, m1}, m3, m5, m7, m9, m11, m13, m15 + pmulhrsw m1, [rsp+32*1] + lea r3, [strideq*3] + WRITE_16X2 0, 1, 6, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 [rsp], 7, 0, 1, strideq*2, r3 +.end3: + pxor m2, m2 + REPX {mova [cq+32*x], m2}, -8, -7, -6, -5, -4, -3, -2, -1 + lea dstq, [dstq+strideq*4] + WRITE_16X2 8, 9, 0, 1, strideq*0, strideq*1 + WRITE_16X2 10, 11, 0, 1, strideq*2, r3 + REPX {mova [cq+32*x], m2}, 0, 1, 2, 3, 4, 5, 6, 7 + lea dstq, [dstq+strideq*4] + WRITE_16X2 12, 13, 0, 1, strideq*0, strideq*1 + WRITE_16X2 14, 15, 0, 1, strideq*2, r3 + RET +ALIGN function_align +cglobal_label .main + vpbroadcastd m15, [o(pd_2048)] + mova [rsp+gprsize+32*1], m1 + mova [rsp+gprsize+32*2], m9 + IDCT8_1D 0, 2, 4, 6, 8, 10, 12, 14, 1, 9, 15 + mova m1, [rsp+gprsize+32*2] ; in9 + mova [rsp+gprsize+32*2], m14 ; tmp7 + mova m9, [rsp+gprsize+32*1] ; in1 + mova [rsp+gprsize+32*1], m10 ; tmp5 + mova m14, [rsp+gprsize+32*0] ; in15 + mova [rsp+gprsize+32*0], m6 ; tmp3 + IDCT16_1D_ODDHALF 9, 3, 5, 7, 1, 11, 13, 14, 6, 10, 15 + mova m6, [rsp+gprsize+32*1] ; tmp5 + psubsw m15, m0, m14 ; out15 + paddsw m0, m14 ; out0 + psubsw m14, m2, m13 ; out14 + paddsw m2, m13 ; out1 + mova [rsp+gprsize+32*1], m2 + psubsw m13, m4, m11 ; out13 + paddsw m2, m4, m11 ; out2 + psubsw m11, m8, m7 ; out11 + paddsw m4, m8, m7 ; out4 + mova m7, [rsp+gprsize+32*2] ; tmp7 + psubsw m10, m6, m5 ; out10 + paddsw m5, m6 ; out5 + psubsw m8, m7, m9 ; out8 + paddsw m7, m9 ; out7 + psubsw m9, m12, m3 ; out9 + paddsw m6, m12, m3 ; out6 + mova m3, [rsp+gprsize+32*0] ; tmp3 + psubsw m12, m3, m1 ; out12 + paddsw m3, m1 ; out3 + ret + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 + ITX_16X16_LOAD_COEFS + call .main + call .main_pass1_end + pmulhrsw m0, m1, [cq+32*0] + pmulhrsw m2, m1, [cq+32*1] + REPX {pmulhrsw x, m1}, m4, m6, m8, m10 + pmulhrsw m12, m1, [cq+32*2] + pmulhrsw m14, m1, [cq+32*3] + vextracti128 [rsp+16*5], m8, 1 + mova [rsp+16*1], xm8 + pxor m8, m8 + psubw m1, m8, m1 + jmp m(idct_16x16_internal_8bpc).pass1_end2 +ALIGN function_align +.pass2: + call .main + call .main_pass2_end + REPX {pmulhrsw x, m1}, m0, m2, m4, m6, m8, m10, m12, m14 + mova [rsp+32*0], m6 + pxor m6, m6 + psubw m1, m6, m1 + jmp m(idct_16x16_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + vpbroadcastd m15, [o(pd_2048)] + mova [rsp+gprsize+32*1], m0 + mova [rsp+gprsize+32*2], m4 + ITX_MULSUB_2W 13, 2, 0, 4, 15, 995, 3973 ; t3, t2 + ITX_MULSUB_2W 9, 6, 0, 4, 15, 2440, 3290 ; t7, t6 + ITX_MULSUB_2W 5, 10, 0, 4, 15, 3513, 2106 ; t11, t10 + ITX_MULSUB_2W 1, 14, 0, 4, 15, 4052, 601 ; t15, t14 + psubsw m0, m2, m10 ; t10a + paddsw m2, m10 ; t2a + psubsw m10, m13, m5 ; t11a + paddsw m13, m5 ; t3a + psubsw m5, m6, m14 ; t14a + paddsw m6, m14 ; t6a + psubsw m14, m9, m1 ; t15a + paddsw m9, m1 ; t7a + ITX_MULSUB_2W 0, 10, 1, 4, 15, 3406, 2276 ; t11, t10 + ITX_MULSUB_2W 14, 5, 1, 4, 15, 2276, 3406 ; t14, t15 + psubsw m1, m10, m14 ; t14a + paddsw m10, m14 ; t10a + psubsw m14, m0, m5 ; t15a + paddsw m0, m5 ; t11a + psubsw m5, m2, m6 ; t6 + paddsw m2, m6 ; t2 + psubsw m6, m13, m9 ; t7 + paddsw m13, m9 ; t3 + ITX_MULSUB_2W 6, 5, 4, 9, 15, 3784, 1567 ; t6a, t7a + ITX_MULSUB_2W 14, 1, 4, 9, 15, 3784, 1567 ; t14, t15 + mova m9, [rsp+gprsize+32*0] ; in15 + mova [rsp+gprsize+32*0], m10 ; t10a + mova m4, [rsp+gprsize+32*1] ; in0 + mova [rsp+gprsize+32*1], m6 ; t6a + mova m6, [rsp+gprsize+32*2] ; in4 + mova [rsp+gprsize+32*2], m2 ; t2 + ITX_MULSUB_2W 9, 4, 2, 10, 15, 201, 4091 ; t1, t0 + ITX_MULSUB_2W 11, 6, 2, 10, 15, 1751, 3703 ; t5, t4 + ITX_MULSUB_2W 7, 8, 2, 10, 15, 3035, 2751 ; t9, t8 + ITX_MULSUB_2W 3, 12, 2, 10, 15, 3857, 1380 ; t13, t12 + psubsw m10, m4, m8 ; t8a + paddsw m8, m4 ; t0a + psubsw m4, m9, m7 ; t9a + paddsw m9, m7 ; t1a + psubsw m7, m6, m12 ; t12a + paddsw m6, m12 ; t4a + psubsw m12, m11, m3 ; t13a + paddsw m11, m3 ; t5a + ITX_MULSUB_2W 10, 4, 2, 3, 15, 799, 4017 ; t9, t8 + ITX_MULSUB_2W 12, 7, 2, 3, 15, 4017, 799 ; t12, t13 + psubsw m3, m9, m11 ; t5 + paddsw m9, m11 ; t1 + psubsw m11, m4, m12 ; t12a + paddsw m4, m12 ; t8a + paddsw m12, m8, m6 ; t0 + psubsw m8, m6 ; t4 + paddsw m6, m10, m7 ; t9a + psubsw m10, m7 ; t13a + ITX_MULSUB_2W 8, 3, 2, 7, 15, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2W 11, 10, 2, 7, 15, 1567, 3784 ; t13, t12 + mova m7, [rsp+gprsize+32*0] ; t10a + mova m2, [rsp+gprsize+32*1] ; t6a + paddsw m15, m9, m13 ; -out15 + psubsw m9, m13 ; t3a + paddsw m13, m11, m1 ; -out13 + psubsw m11, m1 ; t15a + psubsw m1, m4, m7 ; t10 + paddsw m7, m4 ; -out1 + psubsw m4, m3, m2 ; t6 + paddsw m3, m2 ; -out3 + paddsw m2, m10, m14 ; out2 + psubsw m10, m14 ; t14a + paddsw m14, m6, m0 ; out14 + psubsw m6, m0 ; t11 + mova m0, [rsp+gprsize+32*2] ; t2 + mova [rsp+gprsize+32*1], m7 + psubsw m7, m12, m0 ; t2a + paddsw m0, m12 ; out0 + paddsw m12, m8, m5 ; out12 + psubsw m8, m5 ; t7 + ret +ALIGN function_align +.main_pass1_end: + mova [cq+32*0], m0 + mova [cq+32*1], m2 + mova [cq+32*2], m12 + mova [cq+32*3], m14 + vpbroadcastd m14, [pw_m2896_2896] + vpbroadcastd m12, [pw_2896_2896] + vpbroadcastd m2, [pd_2048] + punpcklwd m5, m11, m10 + punpckhwd m11, m10 + pmaddwd m10, m14, m5 + pmaddwd m0, m14, m11 + pmaddwd m5, m12 + pmaddwd m11, m12 + REPX {paddd x, m2}, m10, m0, m5, m11 + REPX {psrad x, 12}, m10, m0, m5, m11 + packssdw m10, m0 ; out10 + packssdw m5, m11 ; -out5 + punpcklwd m11, m8, m4 + punpckhwd m8, m4 + pmaddwd m4, m12, m11 + pmaddwd m0, m12, m8 + pmaddwd m11, m14 + pmaddwd m8, m14 + REPX {paddd x, m2}, m4, m0, m11, m8 + REPX {psrad x, 12}, m4, m0, m11, m8 + packssdw m4, m0 ; out4 + packssdw m11, m8 ; -out11 + punpcklwd m8, m9, m7 + punpckhwd m9, m7 + pmaddwd m7, m12, m8 + pmaddwd m0, m12, m9 + pmaddwd m8, m14 + pmaddwd m9, m14 + REPX {paddd x, m2}, m7, m0, m8, m9 + REPX {psrad x, 12}, m7, m0, m8, m9 + packssdw m7, m0 ; -out7 + packssdw m8, m9 ; out8 + punpckhwd m0, m6, m1 + punpcklwd m6, m1 + pmaddwd m1, m14, m0 + pmaddwd m9, m14, m6 + pmaddwd m0, m12 + pmaddwd m6, m12 + REPX {paddd x, m2}, m1, m9, m0, m6 + REPX {psrad x, 12}, m1, m9, m0, m6 + packssdw m9, m1 ; -out7 + packssdw m6, m0 ; out8 + vpbroadcastd m1, [o(pw_8192)] + ret +ALIGN function_align +cglobal_label .main_pass2_end + ; In pass 2 we're going to clip to pixels afterwards anyway, so clipping to + ; 16-bit here will produce the same result as using 32-bit intermediates. + paddsw m5, m10, m11 ; -out5 + psubsw m10, m11 ; out10 + psubsw m11, m4, m8 ; -out11 + paddsw m4, m8 ; out4 + psubsw m8, m7, m9 ; out8 + paddsw m7, m9 ; -out7 + psubsw m9, m1, m6 ; -out9 + paddsw m6, m1 ; out6 + vpbroadcastd m1, [o(pw_2896x8)] + REPX {pmulhrsw x, m1}, m4, m5, m6, m7, m8, m9, m10, m11 + vpbroadcastd m1, [o(pw_2048)] + ret + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 + ITX_16X16_LOAD_COEFS + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass1_end + pmulhrsw m6, m1 + pmulhrsw m2, m1, m8 + mova [rsp+32*2], m6 + pmulhrsw m6, m1, m4 + pmulhrsw m4, m1, m10 + pmulhrsw m8, m1, [cq+32*3] + pmulhrsw m10, m1, [cq+32*2] + pmulhrsw m12, m1, [cq+32*1] + pmulhrsw m14, m1, [cq+32*0] + pxor m0, m0 + psubw m0, m1 + REPX {pmulhrsw x, m0}, m3, m5, m7, m11, m15 + pmulhrsw m1, m0, m9 + pmulhrsw m9, m0, m13 + pmulhrsw m0, [rsp+32*1] + mova [rsp+16*0], xm15 + mova [rsp+16*1], xm7 + vperm2i128 m15, m15, m7, 0x31 + vinserti128 m7, m2, xm14, 1 + vperm2i128 m14, m2, m14, 0x31 + vinserti128 m2, m9, xm5, 1 + vperm2i128 m9, m9, m5, 0x31 + vinserti128 m5, m4, xm12, 1 + vperm2i128 m12, m4, m12, 0x31 + vinserti128 m4, m11, xm3, 1 + vperm2i128 m11, m11, m3, 0x31 + vinserti128 m3, m10, xm6, 1 + vperm2i128 m10, m10, m6, 0x31 + vinserti128 m6, m1, xm0, 1 + vperm2i128 m13, m1, m0, 0x31 + vinserti128 m1, m8, [rsp+32*2], 1 + vperm2i128 m8, m8, [rsp+32*2], 0x31 + jmp m(idct_16x16_internal_8bpc).pass1_end3 +.pass2: + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end + pmulhrsw m0, m1 + pmulhrsw m8, m1 + mova [rsp+32*0], m0 + mova [rsp+32*2], m8 + pxor m0, m0 + psubw m0, m1 + pmulhrsw m8, m0, m7 + pmulhrsw m7, m0, m9 + pmulhrsw m9, m1, m6 + pmulhrsw m6, m1, m10 + pmulhrsw m10, m0, m5 + pmulhrsw m5, m0, m11 + pmulhrsw m11, m1, m4 + pmulhrsw m4, m1, m12 + pmulhrsw m12, m0, m3 + pmulhrsw m3, m0, m13 + pmulhrsw m13, m1, m2 + pmulhrsw m1, m14 + pmulhrsw m14, m0, [rsp+32*1] + pmulhrsw m0, m15 + lea r3, [strideq*3] + WRITE_16X2 0, 1, 2, 0, strideq*0, strideq*1 + mova m15, [rsp+32*0] + WRITE_16X2 3, 4, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + WRITE_16X2 5, 6, 0, 1, strideq*0, strideq*1 + WRITE_16X2 7, [rsp+32*2], 0, 1, strideq*2, r3 + jmp m(idct_16x16_internal_8bpc).end3 + +%macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 + pmulhrsw m%2, m%3, m%1 + psraw m%2, 1 + pavgw m%1, m%2 ; signs are guaranteed to be equal +%endmacro + +INV_TXFM_16X16_FN identity, dct +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 + vpbroadcastd m7, [o(pw_1697x16)] + mova xm0, [cq+16* 0] + vinserti128 m0, [cq+16*16], 1 + mova xm15, [cq+16* 1] + vinserti128 m15, [cq+16*17], 1 + mova xm1, [cq+16* 2] + vinserti128 m1, [cq+16*18], 1 + mova xm8, [cq+16* 3] + vinserti128 m8, [cq+16*19], 1 + mova xm2, [cq+16* 4] + vinserti128 m2, [cq+16*20], 1 + mova xm9, [cq+16* 5] + vinserti128 m9, [cq+16*21], 1 + mova xm3, [cq+16* 6] + vinserti128 m3, [cq+16*22], 1 + mova xm10, [cq+16* 7] + add cq, 16*16 + vinserti128 m10, [cq+16* 7], 1 + mova xm4, [cq-16* 8] + vinserti128 m4, [cq+16* 8], 1 + mova xm11, [cq-16* 7] + vinserti128 m11, [cq+16* 9], 1 + mova xm5, [cq-16* 6] + vinserti128 m5, [cq+16*10], 1 + mova xm12, [cq-16* 5] + vinserti128 m12, [cq+16*11], 1 + mova xm13, [cq-16* 3] + vinserti128 m13, [cq+16*13], 1 + mova xm14, [cq-16* 1] + vinserti128 m14, [cq+16*15], 1 + REPX {IDTX16B x, 6, 7}, 0, 15, 1, 8, 2, 9, 3, \ + 10, 4, 11, 5, 12, 13, 14 + mova xm6, [cq-16* 4] + vinserti128 m6, [cq+16*12], 1 + mova [rsp], m0 + IDTX16B 6, 0, 7 + mova xm0, [cq-16* 2] + vinserti128 m0, [cq+16*14], 1 + pmulhrsw m7, m0 + psraw m7, 1 + pavgw m7, m0 + jmp m(idct_16x16_internal_8bpc).pass1_end3 +ALIGN function_align +.pass2: + vpbroadcastd m15, [o(pw_1697x16)] + mova [rsp+32*1], m0 + REPX {IDTX16 x, 0, 15}, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14 + mova m0, [rsp+32*1] + mova [rsp+32*1], m1 + IDTX16 0, 1, 15 + mova m1, [rsp+32*0] + pmulhrsw m15, m1 + paddsw m1, m1 + paddsw m15, m1 + jmp m(idct_16x16_internal_8bpc).end + +%define o_base deint_shuf + 128 + +%macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 +%if %3 + vpbroadcastd m15, [o(pw_2896x8)] + pmulhrsw m0, m15, [%1+%2*0] + pmulhrsw m1, m15, [%1+%2*1] + pmulhrsw m2, m15, [%1+%2*2] + pmulhrsw m3, m15, [%1+%2*3] + pmulhrsw m4, m15, [%1+%2*4] + pmulhrsw m5, m15, [%1+%2*5] + pmulhrsw m6, m15, [%1+%2*6] + pmulhrsw m7, m15, [%1+%2*7] +%else + mova m0, [%1+%2*0] + mova m1, [%1+%2*1] + mova m2, [%1+%2*2] + mova m3, [%1+%2*3] + mova m4, [%1+%2*4] + mova m5, [%1+%2*5] + mova m6, [%1+%2*6] + mova m7, [%1+%2*7] +%endif +%endmacro + +%macro LOAD_8ROWS_H 2-3 0 ; src, stride, is_rect2 +%if %3 +%if %3 == 1 + vpbroadcastd m15, [o(pw_2896x8)] +%endif + pmulhrsw m8, m15, [%1+%2*0] + pmulhrsw m9, m15, [%1+%2*1] + pmulhrsw m10, m15, [%1+%2*2] + pmulhrsw m11, m15, [%1+%2*3] + pmulhrsw m12, m15, [%1+%2*4] + pmulhrsw m13, m15, [%1+%2*5] + pmulhrsw m14, m15, [%1+%2*6] + pmulhrsw m15, [%1+%2*7] +%else + mova m8, [%1+%2*0] + mova m9, [%1+%2*1] + mova m10, [%1+%2*2] + mova m11, [%1+%2*3] + mova m12, [%1+%2*4] + mova m13, [%1+%2*5] + mova m14, [%1+%2*6] + mova m15, [%1+%2*7] +%endif +%endmacro + +%macro ITX_UNPACK_MULHRSW 7 ; dst1, dst2/src, tmp, coef[1-4] + vpbroadcastd m%3, [r5-pw_201_4091x8+pw_%4_%5x8] + punpcklwd m%1, m%2, m%2 + pmulhrsw m%1, m%3 + vpbroadcastd m%3, [r5-pw_201_4091x8+pw_%6_%7x8] + punpckhwd m%2, m%2 + pmulhrsw m%2, m%3 +%endmacro + +cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jz .dconly + PROLOGUE 0, 4, 16, 32*3, dst, stride, c, eob + %undef cmp + cmp eobd, 106 + jle .fast + LOAD_8ROWS cq+32*1, 32*2 + call m(idct_16x8_internal_8bpc).main + vperm2i128 m11, m0, m4, 0x31 + vinserti128 m0, xm4, 1 + vperm2i128 m4, m1, m5, 0x31 + vinserti128 m1, xm5, 1 + vperm2i128 m5, m2, m6, 0x31 + vinserti128 m2, xm6, 1 + vperm2i128 m6, m3, m7, 0x31 + vinserti128 m3, xm7, 1 + pxor m7, m7 + REPX {mova [cq+32*x], m7}, 1, 3, 5, 7, 9, 11, 13, 15 + punpckhwd m7, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpcklwd m3, m11, m4 + punpckhwd m11, m4 + punpckhwd m4, m5, m6 + punpcklwd m5, m6 + punpckhdq m6, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m5 + punpckhdq m3, m5 + punpckhdq m5, m11, m4 + punpckldq m11, m4 + punpckldq m4, m7, m1 + punpckhdq m7, m1 + punpckhqdq m12, m6, m0 + punpcklqdq m0, m6 ; out4 + punpckhqdq m13, m7, m4 + punpcklqdq m4, m7 ; out5 + punpckhqdq m14, m3, m2 + punpcklqdq m2, m3 ; out6 + punpckhqdq m15, m5, m11 + punpcklqdq m11, m5 ; out7 + mova [rsp+32*0], m0 + mova [rsp+32*1], m4 + mova [rsp+32*2], m2 +.fast: + LOAD_8ROWS cq+32*0, 32*2 + call m(idct_16x8_internal_8bpc).main + vperm2i128 m8, m0, m4, 0x31 + vinserti128 m0, xm4, 1 + vperm2i128 m4, m1, m5, 0x31 + vinserti128 m1, xm5, 1 + vperm2i128 m5, m2, m6, 0x31 + vinserti128 m2, xm6, 1 + vperm2i128 m6, m3, m7, 0x31 + vinserti128 m3, xm7, 1 + vpbroadcastd m9, [o(pw_8192)] + pxor m7, m7 + REPX {mova [cq+32*x], m7}, 0, 2, 4, 6, 8, 10, 12, 14 + punpckhwd m7, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m8, m4 + punpcklwd m8, m4 + punpckhwd m4, m5, m6 + punpcklwd m5, m6 + punpckhdq m6, m0, m2 + punpckldq m0, m2 + punpckldq m2, m8, m5 + punpckhdq m8, m5 + punpckhdq m5, m3, m4 + punpckldq m3, m4 + punpckhdq m4, m7, m1 + punpckldq m7, m1 + punpcklqdq m1, m7, m4 + punpckhqdq m7, m4 ; out9 + punpckhqdq m4, m2, m8 ; out10 + punpcklqdq m2, m8 + punpckhqdq m8, m3, m5 + punpcklqdq m3, m5 + punpckhqdq m5, m0, m6 ; out8 + punpcklqdq m0, m6 + REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m7 + cmp eobd, 106 + jg .full + mova [rsp+32*0], m5 + mova [rsp+32*1], m7 + mova [rsp+32*2], m4 + pmulhrsw m11, m9, m8 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call .main_fast + jmp .pass2 +.dconly: + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly +.full: + REPX {pmulhrsw x, m9}, m12, m13, m14, m15 + pmulhrsw m6, m9, [rsp+32*2] + mova [rsp+32*2], m4 + pmulhrsw m4, m9, [rsp+32*0] + mova [rsp+32*0], m5 + pmulhrsw m5, m9, [rsp+32*1] + mova [rsp+32*1], m7 + pmulhrsw m7, m9, m11 + pmulhrsw m11, m9, m8 + call .main +.pass2: + vpbroadcastd m12, [o(pw_2048)] + REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m13, m14, m15 + pmulhrsw m12, [rsp] + REPX {vpermq x, x, q3120}, m0, m2, m4, m6, m8, m10, m12, m14 + REPX {vpermq x, x, q2031}, m1, m3, m5, m7, m9, m11, m13, m15 + mova [rsp+32*0], m4 + mova [rsp+32*1], m6 + lea r3, [strideq*3] + WRITE_8X4 0, 1, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 2, 3, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 [rsp+32*0], 5, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 [rsp+32*1], 7, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 8, 9, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 10, 11, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 12, 13, 4, 6 + lea dstq, [dstq+strideq*4] + WRITE_8X4 14, 15, 4, 6 + RET +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + call m(idct_8x16_internal_8bpc).main + mova m8, [rsp+gprsize+0*32] + mova [rsp+gprsize+0*32], m0 + mova m9, [rsp+gprsize+1*32] + mova [rsp+gprsize+1*32], m1 + mova m0, [rsp+gprsize+2*32] + mova [rsp+gprsize+2*32], m6 + lea r5, [r6-(o_base)+pw_201_4091x8] + ITX_UNPACK_MULHRSW 1, 8, 6, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a + ITX_UNPACK_MULHRSW 15, 9, 6, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a + ITX_UNPACK_MULHRSW 14, 0, 6, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a + ITX_UNPACK_MULHRSW 13, 11, 6, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a + jmp .main2 +ALIGN function_align +cglobal_label .main + call m(idct_8x16_internal_8bpc).main + mova m8, [rsp+gprsize+0*32] + mova [rsp+gprsize+0*32], m0 + mova m9, [rsp+gprsize+1*32] + mova [rsp+gprsize+1*32], m1 + mova m0, [rsp+gprsize+2*32] + mova [rsp+gprsize+2*32], m6 + punpcklwd m1, m15, m8 ; in31 in1 + punpckhwd m8, m15 ; in3 in29 + punpcklwd m15, m14, m9 ; in27 in5 + punpckhwd m9, m14 ; in7 in25 + punpcklwd m14, m13, m0 ; in23 in9 + punpckhwd m0, m13 ; in11 in21 + punpcklwd m13, m12, m11 ; in19 in13 + punpckhwd m11, m12 ; in15 in17 + ITX_MUL2X_PACK 1, 6, 12, 10, 201, 4091, 3 ; t16a, t31a + ITX_MUL2X_PACK 8, 6, 12, 10, 4052, 601, 3 ; t23a, t24a + ITX_MUL2X_PACK 15, 6, 12, 10, 995, 3973, 3 ; t20a, t27a + ITX_MUL2X_PACK 9, 6, 12, 10, 3857, 1380, 3 ; t19a, t28a + ITX_MUL2X_PACK 14, 6, 12, 10, 1751, 3703, 3 ; t18a, t29a + ITX_MUL2X_PACK 0, 6, 12, 10, 3513, 2106, 3 ; t21a, t26a + ITX_MUL2X_PACK 13, 6, 12, 10, 2440, 3290, 3 ; t22a, t25a + ITX_MUL2X_PACK 11, 6, 12, 10, 3035, 2751, 3 ; t17a, t30a +.main2: + psubsw m6, m1, m11 ; t17 t30 + paddsw m1, m11 ; t16 t31 + psubsw m11, m9, m14 ; t18 t29 + paddsw m9, m14 ; t19 t28 + psubsw m14, m15, m0 ; t21 t26 + paddsw m15, m0 ; t20 t27 + psubsw m0, m8, m13 ; t22 t25 + paddsw m8, m13 ; t23 t24 + ITX_MUL2X_PACK 6, 12, 13, 10, 799, 4017, 3 ; t17a t30a + ITX_MUL2X_PACK 11, 12, 13, 10, m4017, 799, 3 ; t18a t29a + ITX_MUL2X_PACK 14, 12, 13, 10, 3406, 2276, 3 ; t21a t26a + ITX_MUL2X_PACK 0, 12, 13, 10, m2276, 3406, 3 ; t22a t25a + psubsw m13, m1, m9 ; t19a t28a + paddsw m1, m9 ; t16a t31a + psubsw m9, m8, m15 ; t20a t27a + paddsw m8, m15 ; t23a t24a + psubsw m15, m6, m11 ; t18 t29 + paddsw m6, m11 ; t17 t30 + psubsw m11, m0, m14 ; t21 t26 + paddsw m0, m14 ; t22 t25 + ITX_MUL2X_PACK 15, 12, 14, 10, 1567, 3784, 3 ; t18a t29a + ITX_MUL2X_PACK 13, 12, 14, 10, 1567, 3784, 3 ; t19 t28 + ITX_MUL2X_PACK 9, 12, 14, 10, m3784, 1567, 3 ; t20 t27 + ITX_MUL2X_PACK 11, 12, 14, 10, m3784, 1567, 3 ; t21a t26a + vbroadcasti128 m12, [o(deint_shuf)] + psubsw m14, m1, m8 ; t23 t24 + paddsw m1, m8 ; t16 t31 + psubsw m8, m6, m0 ; t22a t25a + paddsw m6, m0 ; t17a t30a + psubsw m0, m15, m11 ; t21 t26 + paddsw m15, m11 ; t18 t29 + psubsw m11, m13, m9 ; t20a t27a + paddsw m13, m9 ; t19a t28a + REPX {pshufb x, m12}, m1, m6, m15, m13 + ITX_MUL2X_PACK 14, 9, 12, 10, 2896, 2896 ; t24a t23a + vpbroadcastd m9, [o(pw_m2896_2896)] + ITX_MUL2X_PACK 8, 12, _, 10, 12, 9, 4 ; t22 t25 + vpbroadcastd m12, [o(pw_2896_2896)] + ITX_MUL2X_PACK 0, 12, _, 10, 12, 9, 4 ; t21a t26a + vpbroadcastd m12, [o(pw_2896_2896)] + ITX_MUL2X_PACK 11, 9, _, 10, 9, 12, 4 ; t27 t20 + shufps m9, m14, m8, q1032 ; t23a t22 + vpblendd m14, m8, 0xcc ; t24a t25 + shufps m8, m11, m0, q1032 ; t20 t21a + vpblendd m11, m0, 0xcc ; t27 t26a + punpcklqdq m0, m1, m6 ; t16 t17a + punpckhqdq m1, m6 ; t31 t30a + psubsw m10, m5, m8 ; out20 out21 + paddsw m5, m8 ; out11 out10 + psubsw m6, m3, m14 ; out24 out25 + paddsw m3, m14 ; out7 out6 + psubsw m8, m7, m0 ; out16 out17 + paddsw m7, m0 ; out15 out14 + mova m0, [rsp+gprsize+0*32] + punpcklqdq m12, m13, m15 ; t19a t18 + punpckhqdq m13, m15 ; t28a t29 + psubsw m15, m0, m1 ; out31 out30 + paddsw m0, m1 ; out0 out1 + mova m1, [rsp+gprsize+1*32] + mova [rsp+gprsize+0*32], m6 + mova m6, [rsp+gprsize+2*32] + psubsw m14, m1, m13 ; out28 out29 + paddsw m1, m13 ; out3 out2 + psubsw m13, m2, m11 ; out27 out26 + paddsw m2, m11 ; out4 out5 + psubsw m11, m4, m9 ; out23 out22 + paddsw m4, m9 ; out8 out9 + psubsw m9, m6, m12 ; out19 out18 + paddsw m6, m12 ; out12 out13 + ret + +%macro LOAD_PACKED_16X2 4 ; dst, tmp, row[1-2] + vbroadcasti128 m%1, [cq+16*%3] + vbroadcasti128 m%2, [cq+16*%4] + shufpd m%1, m%2, 0x0c +%endmacro + +cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 8 +.dconly: + pmulhrsw xm0, xm2 + movd xm2, [pw_2048] ; intentionally rip-relative + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 + pxor m3, m3 +.dconly_loop: + mova m1, [dstq] + punpckhbw m2, m1, m3 + punpcklbw m1, m3 + paddw m2, m0 + paddw m1, m0 + packuswb m1, m2 + mova [dstq], m1 + add dstq, strideq + dec r3d + jg .dconly_loop + RET +.normal: + PROLOGUE 0, 4, 16, 32*3, dst, stride, c, eob + %undef cmp + LOAD_PACKED_16X2 0, 7, 0, 2 ; in0 in2 + LOAD_PACKED_16X2 4, 7, 1, 3 ; in1 in3 + LOAD_PACKED_16X2 1, 7, 4, 6 ; in4 in6 + LOAD_PACKED_16X2 5, 7, 5, 7 ; in5 in7 + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 0, 1, 2, 3 + add cq, 16*16 + LOAD_PACKED_16X2 2, 7, -8, -6 ; in8 in10 + LOAD_PACKED_16X2 6, 7, -7, -5 ; in9 in11 + LOAD_PACKED_16X2 3, 7, -4, -2 ; in12 in14 + LOAD_PACKED_16X2 11, 7, -3, -1 ; in13 in15 + REPX {mova [cq+32*x], m8}, -4, -3, -2, -1 + mova [rsp+32*0], m4 + mova [rsp+32*1], m5 + mova [rsp+32*2], m6 + cmp eobd, 106 + jg .full + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast + jmp .pass2 +.full: + LOAD_PACKED_16X2 4, 7, 0, 2 ; in16 in18 + LOAD_PACKED_16X2 12, 7, 3, 1 ; in19 in17 + LOAD_PACKED_16X2 5, 7, 4, 6 ; in20 in22 + LOAD_PACKED_16X2 13, 7, 7, 5 ; in23 in21 + REPX {mova [cq+32*x], m8}, 0, 1, 2, 3 + add cq, 16*8 + LOAD_PACKED_16X2 6, 7, 0, 2 ; in24 in26 + LOAD_PACKED_16X2 14, 7, 3, 1 ; in27 in25 + LOAD_PACKED_16X2 7, 8, 4, 6 ; in28 in30 + LOAD_PACKED_16X2 15, 8, 7, 5 ; in31 in29 + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 0, 1, 2, 3 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main +.pass2: + vpbroadcastd m12, [o(pw_8192)] + REPX {pmulhrsw x, m12}, m8, m9, m10, m11, m13, m14, m15 + mova [rsp+32*1], m9 + mova [rsp+32*2], m10 + punpckhwd m9, m0, m2 + punpcklwd m0, m2 + punpckhwd m2, m1, m3 + punpcklwd m1, m3 + punpcklwd m10, m4, m6 + punpckhwd m4, m6 + punpcklwd m6, m5, m7 + punpckhwd m5, m7 + punpckhwd m3, m0, m9 + punpcklwd m0, m9 + punpckhwd m9, m2, m1 + punpcklwd m2, m1 + punpcklwd m7, m10, m4 + punpckhwd m10, m4 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m9 + punpckhdq m3, m9 + punpckldq m6, m7, m4 + punpckhdq m7, m4 + punpckldq m9, m10, m5 + punpckhdq m10, m5 + REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m6, m7, m9, m10 + pmulhrsw m12, [rsp+32*0] + mova [rsp+32*0], m8 + vperm2i128 m4, m0, m6, 0x31 + vinserti128 m0, xm6, 1 + vperm2i128 m5, m1, m7, 0x31 + vinserti128 m1, xm7, 1 + vperm2i128 m6, m2, m9, 0x31 + vinserti128 m2, xm9, 1 + vperm2i128 m7, m3, m10, 0x31 + vinserti128 m3, xm10, 1 + call m(idct_16x8_internal_8bpc).main + vpbroadcastd m8, [o(pw_2048)] + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + lea r2, [strideq*3] + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r2 + lea r3, [dstq+strideq*4] + %define dstq r3 + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r2 + mova m0, [rsp+32*0] + mova m1, [rsp+32*1] + mova m2, [rsp+32*2] + punpckhwd m7, m0, m2 + punpcklwd m0, m2 + punpckhwd m2, m1, m11 + punpcklwd m1, m11 + punpckhwd m4, m12, m14 + punpcklwd m12, m14 + punpckhwd m5, m13, m15 + punpcklwd m13, m15 + punpckhwd m3, m0, m7 + punpcklwd m0, m7 + punpckhwd m9, m2, m1 + punpcklwd m2, m1 + punpcklwd m7, m12, m4 + punpckhwd m12, m4 + punpcklwd m4, m5, m13 + punpckhwd m5, m13 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + punpckldq m2, m3, m9 + punpckhdq m3, m9 + punpckldq m6, m7, m4 + punpckhdq m7, m4 + punpckldq m9, m12, m5 + punpckhdq m12, m5 + vperm2i128 m4, m0, m6, 0x31 + vinserti128 m0, xm6, 1 + vperm2i128 m5, m1, m7, 0x31 + vinserti128 m1, xm7, 1 + vperm2i128 m6, m2, m9, 0x31 + vinserti128 m2, xm9, 1 + vperm2i128 m7, m3, m12, 0x31 + vinserti128 m3, xm12, 1 + call m(idct_16x8_internal_8bpc).main2 + vpbroadcastd m8, [o(pw_2048)] + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + add r0, 16 + add r3, 16 + %define dstq r0 + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r2 + %define dstq r3 + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r2 + RET + +cglobal inv_txfm_add_identity_identity_8x32_8bpc, 4, 5, 11, dst, stride, c, eob + vpbroadcastd m9, [pw_5] + lea r4, [strideq*3] + sub eobd, 107 ; loop_iterations = 1 + (eobd >= 107) +.loop: + mova xm0,[cq+16* 0] + mova xm1, [cq+16* 4] + vinserti128 m0, [cq+16* 1], 1 + vinserti128 m1, [cq+16* 5], 1 + pxor m8, m8 + mova [cq+32*0], m8 + mova [cq+32*2], m8 + add cq, 16*16 + mova xm2, [cq-16* 8] + mova xm3, [cq-16* 4] + vinserti128 m2, [cq-16* 7], 1 + vinserti128 m3, [cq-16* 3], 1 + mova xm4, [cq+16* 0] + mova xm5, [cq+16* 4] + vinserti128 m4, [cq+16* 1], 1 + vinserti128 m5, [cq+16* 5], 1 + mova xm6, [cq+16* 8] + mova xm7, [cq+16*12] + vinserti128 m6, [cq+16* 9], 1 + vinserti128 m7, [cq+16*13], 1 + REPX {mova [cq+32*x], m8}, -4, -2, 0, 2, 4, 6 + REPX {paddsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + call .transpose8x8 + REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 + WRITE_8X4 0, 4, 8, 10, strideq*8, strideq*4, r4*4 + add dstq, strideq + WRITE_8X4 1, 5, 0, 4, strideq*8, strideq*4, r4*4 + add dstq, strideq + WRITE_8X4 2, 6, 0, 4, strideq*8, strideq*4, r4*4 + add dstq, strideq + WRITE_8X4 3, 7, 0, 4, strideq*8, strideq*4, r4*4 + add dstq, strideq + sub cq, 16*16-32 + lea dstq, [dstq+r4*4] + add eobd, 0x80000000 + jnc .loop + RET +ALIGN function_align +.transpose8x8: + punpckhwd m8, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m7 + punpcklwd m6, m7 + punpckhwd m7, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckhdq m6, m5, m7 + punpckldq m5, m7 + punpckldq m7, m8, m1 + punpckhdq m8, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m8 + punpcklqdq m6, m8 + ret + +cglobal inv_txfm_add_identity_identity_32x8_8bpc, 4, 6, 10, dst, stride, c, eob + add cq, 16*8 + vpbroadcastd m9, [pw_4096] + lea r4, [strideq*3] + lea r5, [dstq+strideq*4] + sub eobd, 107 +.loop: + mova xm0, [cq-16*8] + mova xm1, [cq-16*7] + vinserti128 m0, [cq+16*0], 1 + vinserti128 m1, [cq+16*1], 1 + mova xm2, [cq-16*6] + mova xm3, [cq-16*5] + vinserti128 m2, [cq+16*2], 1 + vinserti128 m3, [cq+16*3], 1 + mova xm4, [cq-16*4] + mova xm5, [cq-16*3] + vinserti128 m4, [cq+16*4], 1 + vinserti128 m5, [cq+16*5], 1 + mova xm6, [cq-16*2] + mova xm7, [cq-16*1] + vinserti128 m6, [cq+16*6], 1 + vinserti128 m7, [cq+16*7], 1 + pxor m8, m8 + REPX {mova [cq+32*x], m8}, -4, -3, -2, -1, 0, 1, 2, 3 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r4 + %define dstq r5 + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r4 + add cq, 16*16 + add r0, 16 + add r5, 16 + add eobd, 0x80000000 + jnc .loop + RET + +%define o_base pw_5 + 128 + +%macro LOAD_16ROWS 2-4 0, 1 ; src, stride, is_rect2, zero_coefs +%if %3 + vpbroadcastd m15, [o(pw_2896x8)] + pmulhrsw m0, m15, [%1+%2* 0] + pmulhrsw m1, m15, [%1+%2* 1] + pmulhrsw m2, m15, [%1+%2* 2] + pmulhrsw m3, m15, [%1+%2* 3] + pmulhrsw m4, m15, [%1+%2* 4] + pmulhrsw m5, m15, [%1+%2* 5] + pmulhrsw m6, m15, [%1+%2* 6] + pmulhrsw m7, m15, [%1+%2* 7] + pmulhrsw m8, m15, [%1+%2* 8] + pmulhrsw m9, m15, [%1+%2* 9] + pmulhrsw m10, m15, [%1+%2*10] + pmulhrsw m11, m15, [%1+%2*11] + pmulhrsw m12, m15, [%1+%2*12] + pmulhrsw m13, m15, [%1+%2*13] + pmulhrsw m14, m15, [%1+%2*14] + pmulhrsw m15, [%1+%2*15] +%else + mova m0, [%1+%2* 0] + mova m1, [%1+%2* 1] + mova m2, [%1+%2* 2] + mova m3, [%1+%2* 3] + mova m4, [%1+%2* 4] + mova m5, [%1+%2* 5] + mova m6, [%1+%2* 6] + mova m7, [%1+%2* 7] + mova m8, [%1+%2* 8] + mova m9, [%1+%2* 9] + mova m10, [%1+%2*10] + mova m11, [%1+%2*11] + mova m12, [%1+%2*12] + mova m13, [%1+%2*13] + mova m14, [%1+%2*14] + mova m15, [%1+%2*15] +%endif + mova [rsp], m15 +%if %4 + pxor m15, m15 + REPX {mova [%1+%2*x], m15}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15 +%endif +%endmacro + +%macro IDCT32_PASS2_END 7 ; coefs[1-2], tmp[1-2], rnd, offset[1-2] + mova m%4, [%2] + paddsw m%3, m%1, m%4 + psubsw m%1, m%4 + pmovzxbw m%4, [dstq+%6] + pmulhrsw m%3, m%5 + pmulhrsw m%1, m%5 + paddw m%3, m%4 + pmovzxbw m%4, [r2+%7] + paddw m%1, m%4 + packuswb m%3, m%1 + vpermq m%3, m%3, q3120 + mova [dstq+%6], xm%3 + vextracti128 [r2+%7], m%3, 1 +%endmacro + +cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jz .dconly + PROLOGUE 0, 8, 16, 32*35, dst, stride, c, eob, tmp1, tmp2, \ + base, tmp3 + %undef cmp + LOAD_16ROWS cq, 64, 1 + call m(idct_16x16_internal_8bpc).main + lea tmp1q, [rsp+32*7] + lea tmp2q, [tmp1q+32*8] + lea tmp3q, [tmp1q+32*16] + mova m1, [rsp+32*1] + mova [rsp+32*0], m6 + mova [rsp+32*1], m7 + vpbroadcastd m7, [o(pw_16384)] + call .transpose_2x8x8_round + mova m15, [rsp+32*0] + mova [tmp3q-32*4+ 0], xm0 + vextracti128 [tmp3q+32*0+ 0], m0, 1 + mova [tmp3q-32*3+ 0], xm2 + vextracti128 [tmp3q+32*1+ 0], m2, 1 + mova [tmp3q-32*2+ 0], xm4 + vextracti128 [tmp3q+32*2+ 0], m4, 1 + mova [tmp3q-32*1+ 0], xm6 + vextracti128 [tmp3q+32*3+ 0], m6, 1 + mova [tmp3q-32*4+16], xm8 + vextracti128 [tmp3q+32*0+16], m8, 1 + mova [tmp3q-32*3+16], xm10 + vextracti128 [tmp3q+32*1+16], m10, 1 + mova [tmp3q-32*2+16], xm12 + vextracti128 [tmp3q+32*2+16], m12, 1 + mova [tmp3q-32*1+16], xm14 + vextracti128 [tmp3q+32*3+16], m14, 1 + cmp eobd, 150 + jg .full + vinserti128 m0, m1, xm9, 1 + vperm2i128 m4, m1, m9, 0x31 + vinserti128 m2, m5, xm13, 1 + vperm2i128 m6, m5, m13, 0x31 + vinserti128 m1, m3, xm11, 1 + vperm2i128 m5, m3, m11, 0x31 + vinserti128 m3, m7, xm15, 1 + vperm2i128 m7, m7, m15, 0x31 + call .main_oddhalf_fast + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 + jmp .idct16 +.dconly: + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +.full: + mova [tmp1q-32*4], m1 + mova [tmp1q-32*3], m3 + mova [tmp1q-32*2], m5 + mova [tmp1q-32*1], m7 + mova [tmp1q+32*0], m9 + mova [tmp1q+32*1], m11 + mova [tmp1q+32*2], m13 + mova [tmp1q+32*3], m15 + LOAD_16ROWS cq+32, 64, 1 + call m(idct_16x16_internal_8bpc).main + lea r2, [tmp3q+32*8] + mova m1, [rsp+32*1] + mova [rsp+32*0], m6 + mova [rsp+32*1], m7 + vpbroadcastd m7, [o(pw_16384)] + call .transpose_2x8x8_round + mova m15, [rsp+32*0] + mova [r2-32*4+ 0], xm0 + vextracti128 [r2+32*0+ 0], m0, 1 + mova [r2-32*3+ 0], xm2 + vextracti128 [r2+32*1+ 0], m2, 1 + mova [r2-32*2+ 0], xm4 + vextracti128 [r2+32*2+ 0], m4, 1 + mova [r2-32*1+ 0], xm6 + vextracti128 [r2+32*3+ 0], m6, 1 + mova [r2-32*4+16], xm8 + vextracti128 [r2+32*0+16], m8, 1 + mova [r2-32*3+16], xm10 + vextracti128 [r2+32*1+16], m10, 1 + mova [r2-32*2+16], xm12 + vextracti128 [r2+32*2+16], m12, 1 + mova [r2-32*1+16], xm14 + vextracti128 [r2+32*3+16], m14, 1 + vinserti128 m8, m1, xm9, 1 + vperm2i128 m12, m1, m9, 0x31 + mova xm0, [tmp1q-32*4] + mova xm1, [tmp1q-32*3] + vinserti128 m0, [tmp1q+32*0], 1 + vinserti128 m1, [tmp1q+32*1], 1 + vinserti128 m10, m5, xm13, 1 + vperm2i128 m14, m5, m13, 0x31 + mova xm4, [tmp1q-32*4+16] + mova xm5, [tmp1q-32*3+16] + vinserti128 m4, [tmp1q+32*0+16], 1 + vinserti128 m5, [tmp1q+32*1+16], 1 + vinserti128 m9, m3, xm11, 1 + vperm2i128 m13, m3, m11, 0x31 + mova xm2, [tmp1q-32*2] + mova xm3, [tmp1q-32*1] + vinserti128 m2, [tmp1q+32*2], 1 + vinserti128 m3, [tmp1q+32*3], 1 + vinserti128 m11, m7, xm15, 1 + vperm2i128 m15, m7, m15, 0x31 + mova xm6, [tmp1q-32*2+16] + mova xm7, [tmp1q-32*1+16] + vinserti128 m6, [tmp1q+32*2+16], 1 + vinserti128 m7, [tmp1q+32*3+16], 1 + call .main_oddhalf + LOAD_8ROWS_H r2-32*4, 32 +.idct16: + LOAD_8ROWS tmp3q-32*4, 32 + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq + call .pass2_end + RET +ALIGN function_align +cglobal_label .main_oddhalf_fast ; lower half is zero + mova [rsp+gprsize+32*1], m7 + pxor m7, m7 + mova [rsp+gprsize+32*0], m7 + mova [rsp+gprsize+32*2], m7 + vpbroadcastd m11, [o(pw_3703x8)] + vpbroadcastd m7, [o(pw_1751x8)] + vpbroadcastd m12, [o(pw_m1380x8)] + vpbroadcastd m8, [o(pw_3857x8)] + vpbroadcastd m13, [o(pw_3973x8)] + vpbroadcastd m15, [o(pw_995x8)] + pmulhrsw m11, m4 ; t29a + pmulhrsw m4, m7 ; t18a + pmulhrsw m12, m3 ; t19a + pmulhrsw m3, m8 ; t28a + pmulhrsw m13, m2 ; t27a + pmulhrsw m2, m15 ; t20a + vpbroadcastd m10, [o(pw_m2106x8)] + vpbroadcastd m7, [o(pw_3513x8)] + vpbroadcastd m9, [o(pw_3290x8)] + vpbroadcastd m8, [o(pw_2440x8)] + vpbroadcastd m14, [o(pw_m601x8)] + vpbroadcastd m15, [o(pw_4052x8)] + pmulhrsw m10, m5 ; t21a + pmulhrsw m5, m7 ; t26a + pmulhrsw m9, m6 ; t25a + pmulhrsw m6, m8 ; t22a + pmulhrsw m14, m1 ; t23a + pmulhrsw m1, m15 ; t24a + vpbroadcastd m15, [o(pd_2048)] + jmp .main2 +ALIGN function_align +cglobal_label .main_oddhalf + mova [rsp+gprsize+32*0], m15 + mova [rsp+gprsize+32*1], m7 + mova [rsp+gprsize+32*2], m8 + vpbroadcastd m15, [o(pd_2048)] + ITX_MULSUB_2W 4, 11, 7, 8, 15, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2W 12, 3, 7, 8, 15, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2W 2, 13, 7, 8, 15, 995, 3973 ; t20a, t27a + ITX_MULSUB_2W 10, 5, 7, 8, 15, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2W 6, 9, 7, 8, 15, 2440, 3290 ; t22a, t25a + ITX_MULSUB_2W 14, 1, 7, 8, 15, 4052, 601 ; t23a, t24a +.main2: + psubsw m7, m12, m4 ; t18 + paddsw m12, m4 ; t19 + psubsw m4, m2, m10 ; t21 + paddsw m2, m10 ; t20 + psubsw m10, m14, m6 ; t22 + paddsw m14, m6 ; t23 + psubsw m6, m1, m9 ; t25 + paddsw m1, m9 ; t24 + psubsw m9, m13, m5 ; t26 + paddsw m13, m5 ; t27 + psubsw m5, m3, m11 ; t29 + paddsw m3, m11 ; t28 + ITX_MULSUB_2W 5, 7, 8, 11, 15, m4017, 799 ; t18a, t29a + ITX_MULSUB_2W 9, 4, 8, 11, 15, 3406, 2276 ; t21a, t26a + ITX_MULSUB_2W 6, 10, 8, 11, 15, m2276, 3406 ; t22a, t25a + psubsw m8, m14, m2 ; t20a + paddsw m14, m2 ; t23a + psubsw m2, m1, m13 ; t27a + paddsw m1, m13 ; t24a + psubsw m13, m6, m9 ; t21 + paddsw m6, m9 ; t22 + psubsw m9, m10, m4 ; t26 + paddsw m10, m4 ; t25 + ITX_MULSUB_2W 2, 8, 4, 11, 15, m3784, 1567 ; t20, t27 + ITX_MULSUB_2W 9, 13, 4, 11, 15, m3784, 1567 ; t21a, t26a + mova m4, [rsp+gprsize+32*0] ; in31 + mova [rsp+gprsize+32*0], m6 ; t22 + mova m6, [rsp+gprsize+32*1] ; in15 + mova [rsp+gprsize+32*1], m14 ; t23a + mova m14, [rsp+gprsize+32*2] ; in17 + mova [rsp+gprsize+32*2], m1 ; t24a + ITX_MULSUB_2W 0, 4, 1, 11, 15, 201, 4091 ; t16a, t31a + ITX_MULSUB_2W 14, 6, 1, 11, 15, 3035, 2751 ; t17a, t30a + psubsw m1, m0, m14 ; t17 + paddsw m0, m14 ; t16 + psubsw m14, m4, m6 ; t30 + paddsw m4, m6 ; t31 + ITX_MULSUB_2W 14, 1, 6, 11, 15, 799, 4017 ; t17a, t30a + psubsw m6, m0, m12 ; t19a + paddsw m0, m12 ; t16a + psubsw m12, m4, m3 ; t28a + paddsw m4, m3 ; t31a + psubsw m3, m14, m5 ; t18 + paddsw m14, m5 ; t17 + psubsw m5, m1, m7 ; t29 + paddsw m1, m7 ; t30 + ITX_MULSUB_2W 5, 3, 7, 11, 15, 1567, 3784 ; t18a, t29a + ITX_MULSUB_2W 12, 6, 7, 11, 15, 1567, 3784 ; t19, t28 + psubsw m7, m1, m10 ; t25a + paddsw m1, m10 ; t30a + psubsw m10, m5, m9 ; t21 + paddsw m5, m9 ; t18 + psubsw m9, m12, m2 ; t20a + paddsw m12, m2 ; t19a + psubsw m2, m3, m13 ; t26 + paddsw m3, m13 ; t29 + psubsw m13, m6, m8 ; t27a + paddsw m6, m8 ; t28a + mova [tmp1q-32*2], m5 + mova [tmp1q-32*1], m12 + mova [tmp2q+32*0], m6 + mova [tmp2q+32*1], m3 + mova [tmp2q+32*2], m1 + mova m5, [rsp+gprsize+32*0] ; t22 + mova m6, [rsp+gprsize+32*1] ; t23 + mova m3, [rsp+gprsize+32*2] ; t24a + psubsw m1, m14, m5 ; t22a + paddsw m14, m5 ; t17a + psubsw m5, m0, m6 ; t23 + paddsw m0, m6 ; t16 + psubsw m6, m4, m3 ; t24 + paddsw m4, m3 ; t31 + vpbroadcastd m8, [o(pw_m2896_2896)] + vpbroadcastd m3, [o(pw_2896_2896)] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m14 + mova [tmp2q+32*3], m4 + ITX_MULSUB_2W 13, 9, 0, 4, 15, 3, 8 ; t20, t27 + ITX_MULSUB_2W 2, 10, 0, 4, 15, 3, 8 ; t21a, t26a + ITX_MULSUB_2W 7, 1, 0, 4, 15, 3, 8 ; t22, t25 + ITX_MULSUB_2W 6, 5, 0, 4, 15, 3, 8 ; t23a, t24a + mova [tmp1q+32*0], m13 + mova [tmp1q+32*1], m2 + mova [tmp1q+32*2], m7 + mova [tmp1q+32*3], m6 + mova [tmp2q-32*4], m5 + mova [tmp2q-32*3], m1 + mova [tmp2q-32*2], m10 + mova [tmp2q-32*1], m9 + ret +ALIGN function_align +.transpose_2x8x8_round: + punpckhwd m6, m12, m13 + punpcklwd m12, m13 + punpckhwd m13, m8, m9 + punpcklwd m8, m9 + punpckhwd m9, m14, m15 + punpcklwd m14, m15 + punpckhwd m15, m10, m11 + punpcklwd m10, m11 + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5 + punpckhdq m11, m8, m10 + punpckldq m8, m10 + punpckldq m10, m12, m14 + punpckhdq m12, m14 + punpckhdq m14, m13, m15 + punpckldq m13, m15 + punpckldq m15, m6, m9 + punpckhdq m6, m9 + punpckhqdq m9, m8, m10 + punpcklqdq m8, m10 + punpcklqdq m10, m11, m12 + punpckhqdq m11, m12 + punpcklqdq m12, m13, m15 + punpckhqdq m13, m15 + punpckhqdq m15, m14, m6 + punpcklqdq m14, m6 + pmulhrsw m6, m7, [rsp+gprsize+32*0] + REPX {pmulhrsw x, m7}, m8, m9, m10, m11, m12, m13, m14, m15 + pmulhrsw m7, [rsp+gprsize+32*1] + mova [rsp+gprsize+32*0], m15 + punpckhwd m15, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m7 + punpcklwd m6, m7 + punpckhwd m7, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckhdq m6, m5, m7 + punpckldq m5, m7 + punpckldq m7, m15, m1 + punpckhdq m15, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m15 + punpcklqdq m6, m15 + ret +ALIGN function_align +.pass2_end: + mova [rsp+gprsize+32*0], m7 + mova [rsp+gprsize+32*2], m15 + vpbroadcastd m15, [o(pw_2048)] + IDCT32_PASS2_END 0, tmp2q+32*3, 1, 7, 15, strideq*0, r3*4 + IDCT32_PASS2_END 4, tmp2q-32*1, 0, 7, 15, strideq*4, strideq*8 + IDCT32_PASS2_END 8, tmp1q+32*3, 0, 4, 15, strideq*8, strideq*4 + IDCT32_PASS2_END 12, tmp1q-32*1, 0, 4, 15, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + mova m1, [rsp+gprsize+32*1] + IDCT32_PASS2_END 1, tmp2q+32*2, 0, 4, 15, strideq*0, r3*4 + IDCT32_PASS2_END 5, tmp2q-32*2, 0, 4, 15, strideq*4, strideq*8 + IDCT32_PASS2_END 9, tmp1q+32*2, 0, 4, 15, strideq*8, strideq*4 + IDCT32_PASS2_END 13, tmp1q-32*2, 0, 4, 15, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + IDCT32_PASS2_END 2, tmp2q+32*1, 0, 4, 15, strideq*0, r3*4 + IDCT32_PASS2_END 6, tmp2q-32*3, 0, 4, 15, strideq*4, strideq*8 + IDCT32_PASS2_END 10, tmp1q+32*1, 0, 4, 15, strideq*8, strideq*4 + IDCT32_PASS2_END 14, tmp1q-32*3, 0, 4, 15, r3*4, strideq*0 + add dstq, strideq + sub r2, strideq + mova m7, [rsp+gprsize+32*0] + mova m1, [rsp+gprsize+32*2] + IDCT32_PASS2_END 3, tmp2q+32*0, 0, 4, 15, strideq*0, r3*4 + IDCT32_PASS2_END 7, tmp2q-32*4, 0, 4, 15, strideq*4, strideq*8 + IDCT32_PASS2_END 11, tmp1q+32*0, 0, 4, 15, strideq*8, strideq*4 + IDCT32_PASS2_END 1, tmp1q-32*4, 0, 4, 15, r3*4, strideq*0 + ret + +; Perform the final sumsub step and YMM lane shuffling +%macro IDCT32_PASS1_END 4 ; row[1-2], tmp[1-2] + mova m%3, [tmp2q+32*( 3-%1)] + psubsw m%4, m%1, m%3 + paddsw m%1, m%3 + mova m%3, [tmp1q+32*(11-%2)] + mova [tmp1q+32*(11-%2)+16], xm%4 + vextracti128 [tmp2q+32*( 3-%1)+16], m%4, 1 + paddsw m%4, m%2, m%3 + psubsw m%2, m%3 + mova [tmp1q+32*(11-%2)], xm%2 + vextracti128 [tmp2q+32*( 3-%1)], m%2, 1 + vperm2i128 m%2, m%1, m%4, 0x31 + vinserti128 m%1, xm%4, 1 +%endmacro + +cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 16 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly +.normal: + PROLOGUE 0, 6, 16, 32*19, dst, stride, c, eob, tmp1, tmp2 + vpbroadcastd m15, [o(pw_2896x8)] + pmulhrsw m0, m15, [cq+32* 1] + pmulhrsw m1, m15, [cq+32* 3] + pmulhrsw m2, m15, [cq+32* 5] + pmulhrsw m3, m15, [cq+32* 7] + pmulhrsw m4, m15, [cq+32* 9] + pmulhrsw m5, m15, [cq+32*11] + pmulhrsw m6, m15, [cq+32*13] + pmulhrsw m7, m15, [cq+32*15] + pmulhrsw m8, m15, [cq+32*17] + pmulhrsw m9, m15, [cq+32*19] + pmulhrsw m10, m15, [cq+32*21] + pmulhrsw m11, m15, [cq+32*23] + pmulhrsw m12, m15, [cq+32*25] + pmulhrsw m13, m15, [cq+32*27] + pmulhrsw m14, m15, [cq+32*29] + pmulhrsw m15, [cq+32*31] + lea tmp1q, [rsp+32*7] + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + LOAD_16ROWS cq+32*0, 32*2, 1, 0 + pxor m15, m15 + mov r3d, 8 +.zero_loop: + mova [cq+32*0], m15 + mova [cq+32*1], m15 + mova [cq+32*2], m15 + mova [cq+32*3], m15 + add cq, 32*4 + dec r3d + jg .zero_loop + call m(idct_16x16_internal_8bpc).main + call .pass1_end + lea r2, [strideq*3] + mov r3, dstq +.pass2: + vpbroadcastd m7, [o(pw_16384)] + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + call m(idct_16x16_internal_8bpc).main + mova [rsp+32*2], m15 + vpbroadcastd m15, [o(pw_2048)] + REPX {pmulhrsw x, m15}, m2, m3, m0 + WRITE_16X2 2, 3, 1, 2, strideq*2, r2 + pmulhrsw m1, m15, [rsp+32*1] + WRITE_16X2 0, 1, 2, 3, strideq*0, strideq*1 + lea dstq, [dstq+strideq*4] + REPX {pmulhrsw x, m15}, m4, m5, m6, m7 + WRITE_16X2 4, 5, 2, 3, strideq*0, strideq*1 + WRITE_16X2 6, 7, 2, 3, strideq*2, r2 + lea dstq, [dstq+strideq*4] + REPX {pmulhrsw x, m15}, m8, m9, m10, m11 + WRITE_16X2 8, 9, 2, 3, strideq*0, strideq*1 + WRITE_16X2 10, 11, 2, 3, strideq*2, r2 + lea dstq, [dstq+strideq*4] + REPX {pmulhrsw x, m15}, m11, m12, m13, m14 + pmulhrsw m15, [rsp+32*2] + WRITE_16X2 12, 13, 2, 3, strideq*0, strideq*1 + WRITE_16X2 14, 15, 2, 3, strideq*2, r2 + test r3, r3 + jnz .right_half + RET +.right_half: + LOAD_8ROWS tmp1q-32*4, 32 + LOAD_8ROWS_H tmp2q-32*4, 32 + lea dstq, [r3+16] + xor r3d, r3d + mova [rsp+32*0], m6 + mova [rsp+32*1], m7 + jmp .pass2 +ALIGN function_align +.pass1_end: + mova [rsp+gprsize+32*0], m9 + IDCT32_PASS1_END 0, 8, 1, 9 + IDCT32_PASS1_END 2, 10, 1, 9 + IDCT32_PASS1_END 3, 11, 1, 9 + IDCT32_PASS1_END 4, 12, 1, 9 + IDCT32_PASS1_END 5, 13, 1, 9 + IDCT32_PASS1_END 6, 14, 1, 9 + IDCT32_PASS1_END 7, 15, 1, 9 + mova m1, [rsp+gprsize+32*1] + mova m9, [rsp+gprsize+32*0] + mova [rsp+gprsize+32*0], m6 + mova [rsp+gprsize+32*1], m7 + IDCT32_PASS1_END 1, 9, 6, 7 + ret + +cglobal inv_txfm_add_identity_identity_16x32_8bpc, 4, 5, 13, dst, stride, c, eob +%undef cmp + lea r6, [o_base] + vpbroadcastd m9, [o(pw_2896x8)] + vpbroadcastd m10, [o(pw_1697x16)] + vpbroadcastd m12, [o(pw_8192)] + cmp eobd, 43 ; if (eob > 43) + setg r4b ; iteration_count++ + cmp eobd, 150 ; if (eob > 150) + setg al ; iteration_count++ + add eobd, -279 ; if (eob > 278) + adc r4b, al ; iteration_count++ + lea r3, [strideq*3] + mov r6, cq + paddw m11, m12, m12 ; pw_16384 +.loop: + mova xm0, [cq+64* 0] + mova xm1, [cq+64* 1] + vinserti128 m0, [cq+64* 8], 1 + vinserti128 m1, [cq+64* 9], 1 + mova xm2, [cq+64* 2] + mova xm3, [cq+64* 3] + vinserti128 m2, [cq+64*10], 1 + vinserti128 m3, [cq+64*11], 1 + mova xm4, [cq+64* 4] + mova xm5, [cq+64* 5] + vinserti128 m4, [cq+64*12], 1 + vinserti128 m5, [cq+64*13], 1 + mova xm6, [cq+64* 6] + mova xm7, [cq+64* 7] + vinserti128 m6, [cq+64*14], 1 + vinserti128 m7, [cq+64*15], 1 + REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {IDTX16 x, 8, 10, 11}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + add cq, 16 + dec r4b + jge .loop + sub cq, 32 + pxor m0, m0 + mov r0d, 8 + cmp cq, r6 + ja .zero_loop +.zero_loop_half: + mova [r6+64*0], m0 + mova [r6+64*1], m0 + add r6, 64*4 + mova [r6-64*2], m0 + mova [r6-64*1], m0 + sub r0d, 2 + jg .zero_loop_half + RET +.zero_loop: + mova [r6+32*0], m0 + mova [r6+32*1], m0 + mova [r6+32*2], m0 + mova [r6+32*3], m0 + add r6, 32*4 + dec r0d + jg .zero_loop + RET + +cglobal inv_txfm_add_identity_identity_32x16_8bpc, 4, 6, 12, dst, stride, c, eob +%undef cmp + lea r6, [o_base] + vpbroadcastd m9, [o(pw_2896x8)] + vpbroadcastd m10, [o(pw_1697x16)] + vpbroadcastd m11, [o(pw_2048)] + cmp eobd, 35 ; if (eob > 35) + setg r4b ; iteration_count++ + cmp eobd, 150 ; if (eob > 150) + setg r3b ; iteration_count += 2 + lea r4d, [r4+r3*2] + lea r3, [strideq*3] + mov r5, dstq + mov r6, cq +.loop: + mova xm0, [cq+32* 0] + mova xm1, [cq+32* 1] + vinserti128 m0, [cq+32* 8], 1 + vinserti128 m1, [cq+32* 9], 1 + mova xm2, [cq+32* 2] + mova xm3, [cq+32* 3] + vinserti128 m2, [cq+32*10], 1 + vinserti128 m3, [cq+32*11], 1 + mova xm4, [cq+32* 4] + mova xm5, [cq+32* 5] + vinserti128 m4, [cq+32*12], 1 + vinserti128 m5, [cq+32*13], 1 + mova xm6, [cq+32* 6] + mova xm7, [cq+32* 7] + vinserti128 m6, [cq+32*14], 1 + vinserti128 m7, [cq+32*15], 1 + REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddsw x, x }, m0, m1, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + REPX {IDTX16 x, 8, 10}, 0, 1, 2, 3, 4, 5, 6, 7 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r3 + lea dstq, [dstq+strideq*4] + add cq, 16 + dec r4b + jl .ret + test r4b, 1 + jz .loop + add cq, 32*15 + lea dstq, [r5+16] + jmp .loop +.ret: + sub cd, eax + pxor m0, m0 + add cd, 384 +.zero_loop: + mova [r6+32*0], m0 + mova [r6+32*1], m0 + mova [r6+32*2], m0 + mova [r6+32*3], m0 + add r6, 32*4 + sub cd, 128 + jge .zero_loop + RET + +cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly +.normal: + PROLOGUE 0, 9, 16, 32*67, dst, stride, c, eob, tmp1, tmp2, \ + base, tmp3, tmp4 + %undef cmp + lea tmp1q, [rsp+32*7] + lea tmp2q, [tmp1q+32*8] + sub eobd, 136 + mov tmp4d, eobd +.pass1_loop: + LOAD_8ROWS cq+64*1, 64*2 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 1, 3, 5, 7, 9, 11, 13, 15 + test tmp4d, tmp4d + jl .fast + LOAD_8ROWS_H cq+64*17, 64*2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + LOAD_8ROWS_H cq+64*16, 64*2 + pxor m0, m0 + REPX {mova [cq+64*x], m0}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + mova [rsp], m15 + jmp .idct16 +.fast: + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 +.idct16: + LOAD_8ROWS cq+64*0, 64*2 + pxor m15, m15 + REPX {mova [cq+64*x], m15}, 0, 2, 4, 6, 8, 10, 12, 14 + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).pass1_end + vpbroadcastd m7, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + lea tmp3q, [tmp1q+32*32] + mova m15, [rsp] + mova [tmp3q-32*4], m0 + mova [tmp3q-32*3], m2 + mova [tmp3q-32*2], m4 + mova [tmp3q-32*1], m6 + mova [tmp3q+32*0], m8 + mova [tmp3q+32*1], m10 + mova [tmp3q+32*2], m12 + mova [tmp3q+32*3], m14 + add tmp3q, 32*8 + mova [tmp3q-32*4], m1 + mova [tmp3q-32*3], m3 + mova [tmp3q-32*2], m5 + mova [tmp3q-32*1], m7 + mova [tmp3q+32*0], m9 + mova [tmp3q+32*1], m11 + mova [tmp3q+32*2], m13 + mova [tmp3q+32*3], m15 + vpbroadcastd m9, [o(pw_8192)] + pmulhrsw m0, m9, [tmp1q-32*4] + pmulhrsw m1, m9, [tmp1q-32*3] + pmulhrsw m2, m9, [tmp1q-32*2] + pmulhrsw m3, m9, [tmp1q-32*1] + pmulhrsw m4, m9, [tmp1q+32*0] + pmulhrsw m5, m9, [tmp1q+32*1] + pmulhrsw m6, m9, [tmp1q+32*2] + pmulhrsw m7, m9, [tmp1q+32*3] + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova [tmp1q-32*4], m0 + pmulhrsw m0, m9, [tmp2q-32*4] + mova [tmp2q-32*4], m1 + pmulhrsw m1, m9, [tmp2q-32*3] + mova [tmp1q-32*3], m2 + pmulhrsw m2, m9, [tmp2q-32*2] + mova [tmp2q-32*3], m3 + pmulhrsw m3, m9, [tmp2q-32*1] + mova [tmp1q-32*2], m4 + pmulhrsw m4, m9, [tmp2q+32*0] + mova [tmp2q-32*2], m5 + pmulhrsw m5, m9, [tmp2q+32*1] + mova [tmp1q-32*1], m6 + pmulhrsw m6, m9, [tmp2q+32*2] + mova [tmp2q-32*1], m7 + pmulhrsw m7, m9, [tmp2q+32*3] + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova [tmp1q+32*0], m0 + mova [tmp2q+32*0], m1 + mova [tmp1q+32*1], m2 + mova [tmp2q+32*1], m3 + mova [tmp1q+32*2], m4 + mova [tmp2q+32*2], m5 + mova [tmp1q+32*3], m6 + mova [tmp2q+32*3], m7 + add cq, 32 + add tmp1q, 32*16 + add tmp2q, 32*16 + add eobd, 0x80000000 + jnc .pass1_loop + add tmp1q, 32*24 + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq + test tmp4d, tmp4d + jge .pass2_loop + add tmp1q, 32*16 + add tmp2q, 32*16 + add tmp3q, 32*16 +.pass2_loop: + LOAD_8ROWS tmp2q-32*4, 32 + test tmp4d, tmp4d + jl .fast2 + LOAD_8ROWS_H tmp3q-32*4, 32 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + sub tmp3q, 32*8 + LOAD_8ROWS_H tmp3q-32*4, 32 + sub tmp3q, 32*16 + jmp .pass2_loop_end +.fast2: + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + sub tmp3q, 32*24 + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 +.pass2_loop_end: + LOAD_8ROWS tmp3q-32*4, 32 + mova [rsp], m15 + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).pass2_end + lea tmp3q, [tmp1q-32*32] + cmp tmp2q, tmp3q + jb .ret + sub tmp2q, 32*32 + sub dstq, r3 + lea r2, [r2+r3+16] + add dstq, 16 + jmp .pass2_loop +.ret: + RET + +cglobal inv_txfm_add_identity_identity_32x32_8bpc, 4, 6, 10, dst, stride, c, eob + %undef cmp + vpbroadcastd m9, [pw_8192] + sub eobd, 136 ; if (eob < 136) + shr eobd, 30 ; topleft 16x16 only + lea eobd, [eobq*2-8] + lea r4, [strideq*3] + mov r5, dstq + lea r6, [cq+32] +.loop: + mova xm0, [cq+64* 0] + mova xm1, [cq+64* 1] + vinserti128 m0, [cq+64* 8], 1 + vinserti128 m1, [cq+64* 9], 1 + mova xm2, [cq+64* 2] + mova xm3, [cq+64* 3] + vinserti128 m2, [cq+64*10], 1 + vinserti128 m3, [cq+64*11], 1 + mova xm4, [cq+64* 4] + mova xm5, [cq+64* 5] + vinserti128 m4, [cq+64*12], 1 + vinserti128 m5, [cq+64*13], 1 + mova xm6, [cq+64* 6] + mova xm7, [cq+64* 7] + vinserti128 m6, [cq+64*14], 1 + vinserti128 m7, [cq+64*15], 1 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 + WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 + WRITE_16X2 2, 3, 0, 1, strideq*2, r4 + lea dstq, [dstq+strideq*4] + WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 + WRITE_16X2 6, 7, 0, 1, strideq*2, r4 + lea dstq, [dstq+strideq*4] + add cq, 16 + inc eobd + jz .ret + test eobd, 3 + jnz .loop + add cq, 64*15 + lea dstq, [r5+16] + jmp .loop +.ret: + pxor m0, m0 + mov r0d, 16 + cmp cq, r6 + jne .zero_loop +.zero_loop_topleft: + mova [r6-32*1], m0 + mova [r6+32*1], m0 + mova [r6+32*3], m0 + mova [r6+32*5], m0 + add r6, 64*4 + sub r0d, 4 + jg .zero_loop_topleft + RET +.zero_loop: + mova [r6-32*1], m0 + mova [r6+32*0], m0 + mova [r6+32*1], m0 + mova [r6+32*2], m0 + add r6, 32*4 + dec r0d + jg .zero_loop + RET + +%macro IDCT64_PART2_END 6-10 ; out, src[1-2], tmp[1-3], (offset[1-4]) +%if %1 & 1 + mova m%5, [tmp2q-32*(51-%1)] ; idct16 out 0+n + mova m%4, [tmp1q-32*(14+%1)] ; idct32 out31-n +%else + mova m%5, [tmp1q-32*(45-%1)] + mova m%4, [tmp2q-32*(20+%1)] +%endif + psubsw m%6, m%5, m%4 ; idct32 out31-n + paddsw m%5, m%4 ; idct32 out 0+n + psubsw m%4, m%6, m%3 ; out32+n + paddsw m%6, m%3 ; out31-n + psubsw m%3, m%5, m%2 ; out63-n + paddsw m%5, m%2 ; out 0+n +%if %0 == 6 ; pass 1 +%if %1 & 1 + mova [tmp2q-32*(19-%1)], m%4 + mova [tmp1q-32*(14+%1)], m%6 + mova [tmp1q+32*(18-%1)], m%3 + mova [tmp2q-32*(51-%1)], m%5 +%else + mova [tmp1q-32*(13-%1)], m%4 + mova [tmp2q-32*(20+%1)], m%6 + mova [tmp2q+32*(12-%1)], m%3 + mova [tmp1q-32*(45-%1)], m%5 +%endif +%else ; pass 2 + REPX {pmulhrsw x, m14}, m%4, m%6, m%3, m%5 +%if %1 & 1 + %define %%d0 r2 + %define %%d1 dstq +%else + %define %%d0 dstq + %define %%d1 r2 +%endif + pmovzxbw m%2, [%%d0+%9 ] + paddw m%2, m%4 + pmovzxbw m%4, [%%d1+%8 ] + paddw m%4, m%6 + pmovzxbw m%6, [%%d1+%10] + paddw m%3, m%6 + pmovzxbw m%6, [%%d0+%7 ] + paddw m%5, m%6 + packuswb m%2, m%4 + packuswb m%3, m%5 + vpermq m%2, m%2, q3120 + vpermq m%3, m%3, q3120 + mova [%%d0+%9 ], xm%2 + vextracti128 [%%d1+%8 ], m%2, 1 + mova [%%d1+%10], xm%3 + vextracti128 [%%d0+%7 ], m%3, 1 +%endif +%endmacro + +cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 64 + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +.normal: + PROLOGUE 0, 10, 16, 32*67, dst, stride, c, eob, tmp1, tmp2 + %undef cmp + lea tmp1q, [rsp+32*23] + lea tmp2q, [tmp1q+32*24] + sub eobd, 151 + mov r7d, eobd +.pass1_loop: + LOAD_16ROWS cq, 64 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [rsp+32*0], m6 + mova [rsp+32*1], m7 + vpbroadcastd m7, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + mova m15, [rsp+32*0] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m2 + mova [tmp1q-32*2], m4 + mova [tmp1q-32*1], m6 + mova [tmp1q+32*0], m8 + mova [tmp1q+32*1], m10 + mova [tmp1q+32*2], m12 + mova [tmp1q+32*3], m14 + mova [tmp2q-32*4], m1 + mova [tmp2q-32*3], m3 + mova [tmp2q-32*2], m5 + mova [tmp2q-32*1], m7 + mova [tmp2q+32*0], m9 + mova [tmp2q+32*1], m11 + mova [tmp2q+32*2], m13 + mova [tmp2q+32*3], m15 + add cq, 32 + add tmp1q, 32*8 + add tmp2q, 32*8 + add eobd, 0x80000000 + jnc .pass1_loop + lea r2, [rsp+32*23] + mova xm0, [r2-32*4+ 0] + mova xm1, [r2-32*2+ 0] + vinserti128 m0, [r2+32*0+ 0], 1 + vinserti128 m1, [r2+32*2+ 0], 1 + mova xm2, [r2-32*4+16] + mova xm3, [r2-32*2+16] + vinserti128 m2, [r2+32*0+16], 1 + vinserti128 m3, [r2+32*2+16], 1 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14 + test r7d, r7d + jl .fast + lea r3, [r2+32*8] + mova xm4, [r3-32*4+ 0] + mova xm5, [r3-32*2+ 0] + vinserti128 m4, [r3+32*0+ 0], 1 + vinserti128 m5, [r3+32*2+ 0], 1 + mova xm6, [r3-32*4+16] + mova xm7, [r3-32*2+16] + vinserti128 m6, [r3+32*0+16], 1 + vinserti128 m7, [r3+32*2+16], 1 +.fast: + mova [rsp], m8 + lea tmp1q, [rsp+32*7] + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + mova xm0, [r2-32*3+ 0] + mova xm1, [r2-32*1+ 0] + vinserti128 m0, [r2+32*1+ 0], 1 + vinserti128 m1, [r2+32*3+ 0], 1 + mova xm2, [r2-32*3+16] + mova xm3, [r2-32*1+16] + vinserti128 m2, [r2+32*1+16], 1 + vinserti128 m3, [r2+32*3+16], 1 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + test r7d, r7d + jl .fast2 + mova xm4, [r3-32*3+ 0] + mova xm5, [r3-32*1+ 0] + vinserti128 m4, [r3+32*1+ 0], 1 + vinserti128 m5, [r3+32*3+ 0], 1 + mova xm6, [r3-32*3+16] + mova xm7, [r3-32*1+16] + vinserti128 m6, [r3+32*1+16], 1 + vinserti128 m7, [r3+32*3+16], 1 +.fast2: + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + add r2, 32*24 + vpbroadcastd m15, [o(pd_2048)] + add tmp1q, 32*16 + add tmp2q, 32*32 + mova xm0, [r2-32*4+ 0] + mova xm3, [r2-32*1+16] + vinserti128 m0, [r2+32*0+ 0], 1 + vinserti128 m3, [r2+32*3+16], 1 + mova xm4, [r2-32*4+16] + mova xm7, [r2-32*1+ 0] + vinserti128 m4, [r2+32*0+16], 1 + vinserti128 m7, [r2+32*3+ 0], 1 + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r7d, r7d + jl .fast3 + add r3, 32*24 + mova xm1, [r3-32*1+16] + mova xm2, [r3-32*4+ 0] + vinserti128 m1, [r3+32*3+16], 1 + vinserti128 m2, [r3+32*0+ 0], 1 + mova xm5, [r3-32*1+ 0] + mova xm6, [r3-32*4+16] + vinserti128 m5, [r3+32*3+ 0], 1 + vinserti128 m6, [r3+32*0+16], 1 +.fast3: + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + mova xm0, [r2-32*2+ 0] + mova xm3, [r2-32*3+16] + vinserti128 m0, [r2+32*2+ 0], 1 + vinserti128 m3, [r2+32*1+16], 1 + mova xm4, [r2-32*2+16] + mova xm7, [r2-32*3+ 0] + vinserti128 m4, [r2+32*2+16], 1 + vinserti128 m7, [r2+32*1+ 0], 1 + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r7d, r7d + jl .fast4 + mova xm1, [r3-32*3+16] + mova xm2, [r3-32*2+ 0] + vinserti128 m1, [r3+32*1+16], 1 + vinserti128 m2, [r3+32*2+ 0], 1 + mova xm5, [r3-32*3+ 0] + mova xm6, [r3-32*2+16] + vinserti128 m5, [r3+32*1+ 0], 1 + vinserti128 m6, [r3+32*2+16], 1 +.fast4: + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 + RET +ALIGN function_align +%define o_base idct64_mul - 8 +cglobal_label .main_part1 + ; idct64 steps 1-5: + ; in1/31/17/15/ 9/23/25/ 7 -> + ; t32a/33/34a/35/36/37a/38/39a/56a/57/58a/59/60/61a/62/63a + ; in5/27/21/11/13/19/29/ 3 -> + ; t40a/41/42a/43/44/45a/46/47a/48a/49/50a/51/52/53a/54/55a + vpbroadcastd m11, [o(idct64_mul+4* 0)] + vpbroadcastd m13, [o(idct64_mul+4* 1)] + vpbroadcastd m10, [o(idct64_mul+4* 4)] + vpbroadcastd m12, [o(idct64_mul+4* 5)] + pmulhrsw m11, m0 ; t63a + pmulhrsw m0, m13 ; t32a + pmulhrsw m10, m1 ; t62a + pmulhrsw m1, m12 ; t33a + vpbroadcastd m9, [o(idct64_mul+4* 8)] + vpbroadcastd m13, [o(idct64_mul+4* 9)] + vpbroadcastd m8, [o(idct64_mul+4*12)] + vpbroadcastd m12, [o(idct64_mul+4*13)] + pmulhrsw m9, m2 ; t61a + pmulhrsw m2, m13 ; t34a + pmulhrsw m8, m3 ; t60a + pmulhrsw m3, m12 ; t35a + psubsw m12, m0, m1 ; t33 + paddsw m0, m1 ; t32 + psubsw m1, m3, m2 ; t34 + paddsw m3, m2 ; t35 + psubsw m2, m8, m9 ; t61 + paddsw m8, m9 ; t60 + psubsw m9, m11, m10 ; t62 + paddsw m11, m10 ; t63 + ITX_MULSUB_2W 2, 1, 10, 13, 15, m4076, 401 ; t34a, t61a + vpbroadcastd m14, [o(pw_401_4076)] + ITX_MULSUB_2W 9, 12, 10, 13, 15, 14, 13 ; t33a, t62a + psubsw m10, m0, m3 ; t35a + paddsw m0, m3 ; t32a + psubsw m3, m11, m8 ; t60a + paddsw m11, m8 ; t63a + psubsw m8, m9, m2 ; t34 + paddsw m9, m2 ; t33 + psubsw m2, m12, m1 ; t61 + paddsw m12, m1 ; t62 + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m9 + mova [tmp2q+32*2], m12 + mova [tmp2q+32*3], m11 + vpbroadcastd m13, [o(pw_m4017_799)] + vpbroadcastd m14, [o(pw_799_4017)] + ITX_MULSUB_2W 2, 8, 0, 1, 15, 14, 13 ; t34a, t61a + ITX_MULSUB_2W 3, 10, 0, 1, 15, 14, 13 ; t35, t60 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp2q+32*0], m10 + mova [tmp2q+32*1], m8 + vpbroadcastd m3, [o(idct64_mul+4*16)] + vpbroadcastd m11, [o(idct64_mul+4*17)] + vpbroadcastd m2, [o(idct64_mul+4*20)] + vpbroadcastd m10, [o(idct64_mul+4*21)] + vpbroadcastd m1, [o(idct64_mul+4*24)] + vpbroadcastd m9, [o(idct64_mul+4*25)] + vpbroadcastd m0, [o(idct64_mul+4*28)] + vpbroadcastd m8, [o(idct64_mul+4*29)] + pmulhrsw m3, m4 ; t59a + pmulhrsw m4, m11 ; t36a + pmulhrsw m2, m5 ; t58a + pmulhrsw m5, m10 ; t37a + pmulhrsw m1, m6 ; t57a + pmulhrsw m6, m9 ; t38a + pmulhrsw m0, m7 ; t56a + pmulhrsw m7, m8 ; t39a + psubsw m8, m4, m5 ; t37 + paddsw m4, m5 ; t36 + psubsw m5, m7, m6 ; t38 + paddsw m7, m6 ; t39 + psubsw m6, m0, m1 ; t57 + paddsw m0, m1 ; t56 + psubsw m1, m3, m2 ; t58 + paddsw m3, m2 ; t59 + ITX_MULSUB_2W 6, 5, 2, 9, 15, m2598, 3166 ; t38a, t57a + vpbroadcastd m10, [o(pw_3166_2598)] + ITX_MULSUB_2W 1, 8, 2, 9, 15, 10, 9 ; t37a, t58a + psubsw m2, m7, m4 ; t36a + paddsw m7, m4 ; t39a + psubsw m4, m0, m3 ; t59a + paddsw m0, m3 ; t56a + psubsw m3, m6, m1 ; t37 + paddsw m6, m1 ; t38 + psubsw m1, m5, m8 ; t58 + paddsw m5, m8 ; t57 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + mova [tmp2q-32*4], m0 + mova [tmp2q-32*3], m5 + vpbroadcastd m6, [o(pw_m799_m4017)] + vpbroadcastd m7, [o(pw_m4017_799)] + ITX_MULSUB_2W 4, 2, 0, 5, 15, 7, 6 ; t36, t59 + ITX_MULSUB_2W 1, 3, 0, 5, 15, 7, 6 ; t37a, t58a + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m1 + mova [tmp2q-32*2], m3 + mova [tmp2q-32*1], m2 + ret +%define o_base pw_5 + 128 +.main_part2_pass1: ; idct64 steps 6-9 + idct16/32/64 sumsub + sub r6, o_idct64_offset + 8 + vpbroadcastd m11, [o(pw_1567_3784)] + vpbroadcastd m12, [o(pw_m3784_1567)] + vpbroadcastd m13, [o(pw_2896_2896)] + vpbroadcastd m14, [o(pw_m2896_2896)] +.main_part2_pass1_loop: + call .main_part2_internal + IDCT64_PART2_END 0, 7, 0, 6, 9, 10 + IDCT64_PART2_END 7, 8, 5, 0, 6, 7 + IDCT64_PART2_END 8, 2, 1, 0, 6, 7 + IDCT64_PART2_END 15, 3, 4, 0, 6, 7 + cmp tmp1q, tmp2q + jne .main_part2_pass1_loop + ret +cglobal_label .main_part2_internal + mova m0, [tmp1q-32*12] ; t32a + mova m6, [tmp2q-32*13] ; t39a + mova m1, [tmp1q-32* 4] ; t40a + mova m5, [tmp2q+32* 3] ; t55a + add tmp1q, 32 + sub tmp2q, 32 + mova m2, [tmp1q+32* 3] ; t48a + mova m4, [tmp2q-32* 4] ; t47a + mova m3, [tmp1q+32*11] ; t56a + mova m7, [tmp2q+32*12] ; t63a + psubsw m8, m0, m6 ; t39 + paddsw m0, m6 ; t32 + psubsw m6, m4, m1 ; t40 + paddsw m4, m1 ; t47 + psubsw m1, m2, m5 ; t55 + paddsw m2, m5 ; t48 + psubsw m5, m7, m3 ; t56 + paddsw m7, m3 ; t63 + ITX_MULSUB_2W 5, 8, 3, 9, 15, 11, 12 ; t39a, t56a + vpbroadcastd m9, [o(pw_m1567_m3784)] + ITX_MULSUB_2W 1, 6, 3, 9, 15, 12, 9 ; t40a, t55a + psubsw m3, m0, m4 ; t47a + paddsw m0, m4 ; t32a + psubsw m4, m7, m2 ; t48a + paddsw m7, m2 ; t63a + psubsw m2, m5, m1 ; t40 + paddsw m5, m1 ; t39 + psubsw m1, m8, m6 ; t55 + paddsw m8, m6 ; t56 + ITX_MULSUB_2W 4, 3, 6, 9, 15, 13, 14 ; t47, t48 + ITX_MULSUB_2W 1, 2, 6, 9, 15, 13, 14 ; t40a, t55a + ret +.main_part2_pass2: + sub r6, o_idct64_offset + 8 + vpbroadcastd m11, [o(pw_1567_3784)] + vpbroadcastd m12, [o(pw_m3784_1567)] + vpbroadcastd m13, [o(pw_2896_2896)] + lea r9, [strideq*5] ; stride*5 + lea r3, [r9+strideq*1] ; stride*6 + lea r7, [r9+strideq*2] ; stride*7 + lea r8, [r3+strideq*2] ; stride*8 + lea r2, [dstq+r7] +.main_part2_pass2_loop: + vpbroadcastd m14, [o(pw_m2896_2896)] + call .main_part2_internal + vpbroadcastd m14, [o(pw_2048)] + IDCT64_PART2_END 0, 7, 0, 6, 9, 10, strideq*0, r3*4, r8*4, r7*8 + IDCT64_PART2_END 7, 8, 5, 0, 6, 7, strideq*0, r3*4, r8*4, r7*8 + IDCT64_PART2_END 8, 2, 1, 0, 6, 7, strideq*8, r8*2, r9*8, r3*8 + IDCT64_PART2_END 15, 3, 4, 0, 6, 7, strideq*8, r8*2, r9*8, r3*8 + add dstq, strideq + sub r2, strideq + cmp tmp1q, tmp2q + jne .main_part2_pass2_loop + ret + +cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 16 +.dconly: + pmulhrsw xm0, xm2 + movd xm2, [o(pw_2048)] + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 + pxor m1, m1 +.dconly_loop: + mova m2, [dstq+32*0] + mova m3, [dstq+32*1] + punpckhbw m4, m2, m1 + punpcklbw m2, m1 + punpckhbw m5, m3, m1 + punpcklbw m3, m1 + paddw m4, m0 + paddw m2, m0 + paddw m5, m0 + paddw m3, m0 + packuswb m2, m4 + packuswb m3, m5 + mova [dstq+32*0], m2 + mova [dstq+32*1], m3 + add dstq, strideq + dec r3d + jg .dconly_loop + RET +.normal: + PROLOGUE 0, 7, 16, 32*67, dst, stride, c, eob, tmp1, tmp2 + LOAD_8ROWS cq+32*0, 32*4 + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + lea tmp1q, [rsp+32*7] + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + LOAD_8ROWS cq+32*2, 32*4 + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m15, [o(pd_2048)] + add tmp1q, 32*16 + add tmp2q, 32*32 + mova m0, [cq+32* 1] + mova m1, [cq+32*31] + mova m2, [cq+32*17] + mova m3, [cq+32*15] + mova m4, [cq+32* 9] + mova m5, [cq+32*23] + mova m6, [cq+32*25] + mova m7, [cq+32* 7] + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + mova m0, [cq+32* 5] + mova m1, [cq+32*27] + mova m2, [cq+32*21] + mova m3, [cq+32*11] + mova m4, [cq+32*13] + mova m5, [cq+32*19] + mova m6, [cq+32*29] + mova m7, [cq+32* 3] + pxor m8, m8 + REPX {mova [cq+32*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 + sub tmp1q, 32*36 + lea r2, [strideq*3] + mov tmp2d, 4 +.pass2_loop: + lea r3, [tmp1q-32*8] + mova xm0, [r3 -32*4] + mova xm1, [r3 -32*3] + vinserti128 m0, [tmp1q-32*4], 1 + vinserti128 m1, [tmp1q-32*3], 1 + mova xm2, [r3 -32*2] + mova xm3, [r3 -32*1] + vinserti128 m2, [tmp1q-32*2], 1 + vinserti128 m3, [tmp1q-32*1], 1 + mova xm4, [r3 +32*0] + mova xm5, [r3 +32*1] + vinserti128 m4, [tmp1q+32*0], 1 + vinserti128 m5, [tmp1q+32*1], 1 + mova xm6, [r3 +32*2] + mova xm7, [r3 +32*3] + vinserti128 m6, [tmp1q+32*2], 1 + vinserti128 m7, [tmp1q+32*3], 1 + mova xm8, [r3 -32*4+16] + mova xm9, [r3 -32*3+16] + vinserti128 m8, [tmp1q-32*4+16], 1 + vinserti128 m9, [tmp1q-32*3+16], 1 + mova xm10, [r3 -32*2+16] + mova xm11, [r3 -32*1+16] + vinserti128 m10, [tmp1q-32*2+16], 1 + vinserti128 m11, [tmp1q-32*1+16], 1 + mova xm12, [r3 +32*0+16] + mova xm13, [r3 +32*1+16] + vinserti128 m12, [tmp1q+32*0+16], 1 + vinserti128 m13, [tmp1q+32*1+16], 1 + mova xm14, [r3 +32*2+16] + mova xm15, [r3 +32*3+16] + vinserti128 m14, [tmp1q+32*2+16], 1 + vinserti128 m15, [tmp1q+32*3+16], 1 + mova [rsp+32*0], m6 + mova [rsp+32*1], m7 + vpbroadcastd m7, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + call m(idct_16x16_internal_8bpc).main + mova [rsp+32*0], m15 + vpbroadcastd m15, [o(pw_2048)] + REPX {pmulhrsw x, m15}, m0, m2, m3, m4, m5, m6, m7 + WRITE_16X2 2, 3, 1, 2, strideq*2, r2 + pmulhrsw m1, m15, [rsp+32*1] + WRITE_16X2 0, 1, 2, 3, strideq*0, strideq*1 + lea r3, [dstq+strideq*4] + %define dstq r3 + WRITE_16X2 4, 5, 2, 3, strideq*0, strideq*1 + WRITE_16X2 6, 7, 2, 3, strideq*2, r2 + REPX {pmulhrsw x, m15}, m8, m9, m10, m11, m12, m13, m14 + lea r3, [r3+strideq*4] + WRITE_16X2 8, 9, 2, 3, strideq*0, strideq*1 + WRITE_16X2 10, 11, 2, 3, strideq*2, r2 + pmulhrsw m15, [rsp+32*0] + lea r3, [r3+strideq*4] + WRITE_16X2 12, 13, 2, 3, strideq*0, strideq*1 + WRITE_16X2 14, 15, 2, 3, strideq*2, r2 + add tmp1q, 32*16 + add r0, 16 + dec tmp2d + jg .pass2_loop + RET + +cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 64 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly +.normal: + PROLOGUE 0, 11, 16, 32*99, dst, stride, c, eob, tmp1, tmp2 + lea tmp1q, [rsp+32*7] + lea r10d, [eobq-136] + sar r10d, 31 +.pass1_loop: + lea tmp2q, [tmp1q+32*16] + LOAD_8ROWS cq+64*1, 64*2, 1 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 1, 3, 5, 7, 9, 11, 13, 15 + test r10b, r10b + jnz .fast + LOAD_8ROWS_H cq+64*17, 64*2, 2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + LOAD_8ROWS_H cq+64*16, 64*2, 1 + mova [rsp], m15 + pxor m15, m15 + REPX {mova [cq+64*x], m15}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + jmp .idct16 +.fast: + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 +.idct16: + LOAD_8ROWS cq+64*0, 64*2, 1 + pxor m15, m15 + REPX {mova [cq+64*x], m15}, 0, 2, 4, 6, 8, 10, 12, 14 + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).pass1_end + vpbroadcastd m7, [o(pw_16384)] + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + lea r3, [tmp1q+32*48] + mova m15, [rsp] + mova [r3-32*4], m0 + mova [r3-32*3], m2 + mova [r3-32*2], m4 + mova [r3-32*1], m6 + mova [r3+32*0], m8 + mova [r3+32*1], m10 + mova [r3+32*2], m12 + mova [r3+32*3], m14 + add r3, 32*24 + mova [r3-32*4], m1 + mova [r3-32*3], m3 + mova [r3-32*2], m5 + mova [r3-32*1], m7 + mova [r3+32*0], m9 + mova [r3+32*1], m11 + mova [r3+32*2], m13 + mova [r3+32*3], m15 + vpbroadcastd m9, [o(pw_16384)] + pmulhrsw m0, m9, [tmp1q-32*4] + pmulhrsw m1, m9, [tmp1q-32*3] + pmulhrsw m2, m9, [tmp1q-32*2] + pmulhrsw m3, m9, [tmp1q-32*1] + pmulhrsw m4, m9, [tmp1q+32*0] + pmulhrsw m5, m9, [tmp1q+32*1] + pmulhrsw m6, m9, [tmp1q+32*2] + pmulhrsw m7, m9, [tmp1q+32*3] + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova [tmp1q-32*4], m0 + pmulhrsw m0, m9, [tmp2q-32*4] + mova [tmp2q-32*4], m1 + pmulhrsw m1, m9, [tmp2q-32*3] + mova [tmp1q-32*3], m2 + pmulhrsw m2, m9, [tmp2q-32*2] + mova [tmp2q-32*3], m3 + pmulhrsw m3, m9, [tmp2q-32*1] + mova [tmp1q-32*2], m4 + pmulhrsw m4, m9, [tmp2q+32*0] + mova [tmp2q-32*2], m5 + pmulhrsw m5, m9, [tmp2q+32*1] + mova [tmp1q-32*1], m6 + pmulhrsw m6, m9, [tmp2q+32*2] + mova [tmp2q-32*1], m7 + pmulhrsw m7, m9, [tmp2q+32*3] + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova [tmp1q+32*0], m0 + mova [tmp2q+32*0], m1 + mova [tmp1q+32*1], m2 + mova [tmp2q+32*1], m3 + mova [tmp1q+32*2], m4 + mova [tmp2q+32*2], m5 + mova [tmp1q+32*3], m6 + mova [tmp2q+32*3], m7 + add cq, 32 + add tmp1q, 32*8 + add r10d, 0x80000000 + jnc .pass1_loop + lea r2, [rsp+32*55] + lea r7, [r2+32*24] +.pass2_loop: + lea r3, [r2+32*8] + lea r8, [r7+32*8] + mova m0, [r2-32*4] + mova m1, [r2-32*2] + mova m2, [r2+32*0] + mova m3, [r2+32*2] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14 + test r10b, r10b + jnz .fast2 + mova m4, [r3-32*4] + mova m5, [r3-32*2] + mova m6, [r3+32*0] + mova m7, [r3+32*2] +.fast2: + mova [rsp], m8 + lea tmp1q, [rsp+32*39] + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + mova m0, [r2-32*3] + mova m1, [r2-32*1] + mova m2, [r2+32*1] + mova m3, [r2+32*3] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + test r10b, r10b + jnz .fast3 + mova m4, [r3-32*3] + mova m5, [r3-32*1] + mova m6, [r3+32*1] + mova m7, [r3+32*3] +.fast3: + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m15, [o(pd_2048)] + add tmp1q, 32*16 + add tmp2q, 32*32 + mova m0, [r7-32*4] + mova m3, [r7+32*3] + mova m4, [r7+32*0] + mova m7, [r7-32*1] + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r10b, r10b + jnz .fast4 + mova m1, [r8+32*3] + mova m2, [r8-32*4] + mova m5, [r8-32*1] + mova m6, [r8+32*0] +.fast4: + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + mova m0, [r7-32*2] + mova m3, [r7+32*1] + mova m4, [r7+32*2] + mova m7, [r7-32*3] + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r10b, r10b + jnz .fast5 + mova m1, [r8+32*1] + mova m2, [r8-32*2] + mova m5, [r8-32*3] + mova m6, [r8+32*2] +.fast5: + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 + add r10d, 0x80000000 + jc .ret + lea r2, [rsp+32*7] + lea r7, [r2+32*16] + sub dstq, r8 + lea dstq, [dstq+strideq*4+16] + jmp .pass2_loop +.ret: + RET + +cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_16384)] + mov [cq], eobd + pmulhrsw xm0, xm1 + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly +.normal: + PROLOGUE 0, 9, 16, 32*131, dst, stride, c, eob, tmp1, tmp2, \ + base, tmp3, tmp4 + lea tmp1q, [rsp+32*7] + lea tmp4d, [eobq-136] +.pass1_loop: + LOAD_8ROWS cq+64*0, 64*4, 1 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + LOAD_8ROWS cq+64*2, 64*4, 1 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m15, [o(pd_2048)] + add tmp1q, 32*16 + add tmp2q, 32*32 + vpbroadcastd m7, [o(pw_2896x8)] + pmulhrsw m0, m7, [cq+64* 1] + pmulhrsw m1, m7, [cq+64*31] + pmulhrsw m2, m7, [cq+64*17] + pmulhrsw m3, m7, [cq+64*15] + pmulhrsw m4, m7, [cq+64* 9] + pmulhrsw m5, m7, [cq+64*23] + pmulhrsw m6, m7, [cq+64*25] + pmulhrsw m7, [cq+64* 7] + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + vpbroadcastd m7, [o(pw_2896x8-(o_idct64_offset))] + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + pmulhrsw m0, m7, [cq+64* 5] + pmulhrsw m1, m7, [cq+64*27] + pmulhrsw m2, m7, [cq+64*21] + pmulhrsw m3, m7, [cq+64*11] + pmulhrsw m4, m7, [cq+64*13] + pmulhrsw m5, m7, [cq+64*19] + pmulhrsw m6, m7, [cq+64*29] + pmulhrsw m7, [cq+64* 3] + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 + sub tmp1q, 32*44 + vpbroadcastd m10, [o(pw_16384)] + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_round_interleave + add cq, 32 + add tmp4d, 0x80000000 + jnc .pass1_loop + lea tmp1q, [rsp+32*15] + imul r2, strideq, 19 + lea r3, [strideq*3] + add r2, dstq + mov tmp4b, 4 +.pass2_loop: + lea tmp2q, [tmp1q+32*64] + LOAD_8ROWS tmp1q-32*4, 32 + test tmp4d, 0x40000000 + jnz .fast + LOAD_8ROWS_H tmp2q-32*4, 32 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + lea tmp3q, [tmp2q-32*8] + LOAD_8ROWS_H tmp3q-32*4, 32 + mova [rsp], m15 + jmp .idct16 +.fast: + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + pxor m8, m8 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 +.idct16: + lea tmp3q, [tmp1q-32*8] + LOAD_8ROWS tmp3q-32*4, 32 + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).pass2_end + add tmp1q, 32*16 + sub dstq, r3 + lea r2, [r2+r3+16] + add dstq, 16 + dec tmp4b + jg .pass2_loop + RET +ALIGN function_align +.transpose_round_interleave: + mov tmp3d, 4 +.loop: + lea tmp2q, [tmp1q+32*8] + mova xm0, [tmp1q-32*4] + mova xm1, [tmp1q-32*3] + vinserti128 m0, [tmp2q-32*4], 1 + vinserti128 m1, [tmp2q-32*3], 1 + mova xm2, [tmp1q-32*2] + mova xm3, [tmp1q-32*1] + vinserti128 m2, [tmp2q-32*2], 1 + vinserti128 m3, [tmp2q-32*1], 1 + mova xm4, [tmp1q+32*0] + mova xm5, [tmp1q+32*1] + vinserti128 m4, [tmp2q+32*0], 1 + vinserti128 m5, [tmp2q+32*1], 1 + mova xm6, [tmp1q+32*2] + mova xm7, [tmp1q+32*3] + vinserti128 m6, [tmp2q+32*2], 1 + vinserti128 m7, [tmp2q+32*3], 1 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova xm8, [tmp1q-32*4+16] + mova xm9, [tmp1q-32*3+16] + vinserti128 m8, [tmp2q-32*4+16], 1 + vinserti128 m9, [tmp2q-32*3+16], 1 + mova [tmp1q-32*4], m0 + mova [tmp2q-32*4], m1 + mova [tmp1q-32*3], m2 + mova [tmp2q-32*3], m3 + mova xm2, [tmp1q-32*2+16] + mova xm3, [tmp1q-32*1+16] + vinserti128 m2, [tmp2q-32*2+16], 1 + vinserti128 m3, [tmp2q-32*1+16], 1 + mova [tmp1q-32*2], m4 + mova [tmp2q-32*2], m5 + mova [tmp1q-32*1], m6 + mova [tmp2q-32*1], m7 + mova xm4, [tmp1q+32*0+16] + mova xm5, [tmp1q+32*1+16] + vinserti128 m4, [tmp2q+32*0+16], 1 + vinserti128 m5, [tmp2q+32*1+16], 1 + mova xm6, [tmp1q+32*2+16] + mova xm7, [tmp1q+32*3+16] + vinserti128 m6, [tmp2q+32*2+16], 1 + vinserti128 m7, [tmp2q+32*3+16], 1 + pmulhrsw m0, m8, m10 + pmulhrsw m1, m9, m10 + REPX {pmulhrsw x, m10}, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 + mova [tmp1q+32*0], m0 + mova [tmp2q+32*0], m1 + mova [tmp1q+32*1], m2 + mova [tmp2q+32*1], m3 + mova [tmp1q+32*2], m4 + mova [tmp2q+32*2], m5 + mova [tmp1q+32*3], m6 + mova [tmp2q+32*3], m7 + add tmp1q, 32*16 + dec tmp3d + jg .loop + ret + +cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 4, 0, dst, stride, c, eob + lea r6, [o_base] + test eobd, eobd + jnz .normal + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_8192)] + mov [cq], eobd + or r3d, 64 + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly +.normal: + PROLOGUE 0, 11, 16, 32*199, dst, stride, c, eob, tmp1, tmp2 + lea tmp1q, [rsp+32*71] + lea r10d, [eobq-136] +.pass1_loop: + LOAD_8ROWS cq+64*0, 64*4 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 + REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 + mova [rsp], m8 + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + LOAD_8ROWS cq+64*2, 64*4 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m15, [o(pd_2048)] + add tmp1q, 32*16 + add tmp2q, 32*32 + mova m0, [cq+64* 1] + mova m1, [cq+64*31] + mova m2, [cq+64*17] + mova m3, [cq+64*15] + mova m4, [cq+64* 9] + mova m5, [cq+64*23] + mova m6, [cq+64*25] + mova m7, [cq+64* 7] + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + mova m0, [cq+64* 5] + mova m1, [cq+64*27] + mova m2, [cq+64*21] + mova m3, [cq+64*11] + mova m4, [cq+64*13] + mova m5, [cq+64*19] + mova m6, [cq+64*29] + mova m7, [cq+64* 3] + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 + sub tmp1q, 32*44 + vpbroadcastd m10, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_round_interleave + add cq, 32 + add r10d, 0x80000000 + jnc .pass1_loop + lea tmp1q, [rsp+32*7] + mov r10b, 4 +.pass2_loop: + lea r2, [tmp1q+32*64] + mova m0, [r2-32*4] + mova m1, [r2-32*2] + mova m2, [r2+32*0] + mova m3, [r2+32*2] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14 + mova [rsp], m4 + test r10d, 0x40000000 + jnz .fast + lea r3, [r2+32*64] + mova m4, [r3-32*4] + mova m5, [r3-32*2] + mova m6, [r3+32*0] + mova m7, [r3+32*2] +.fast: + call m(idct_16x16_internal_8bpc).main + mova m1, [rsp+32*1] + mova [tmp1q-32*4], m0 + mova [tmp1q-32*3], m1 + mova [tmp1q-32*2], m2 + mova [tmp1q-32*1], m3 + mova [tmp1q+32*0], m4 + mova [tmp1q+32*1], m5 + mova [tmp1q+32*2], m6 + mova [tmp1q+32*3], m7 + add tmp1q, 32*8 + mova [tmp1q-32*4], m8 + mova [tmp1q-32*3], m9 + mova [tmp1q-32*2], m10 + mova [tmp1q-32*1], m11 + mova [tmp1q+32*0], m12 + mova [tmp1q+32*1], m13 + mova [tmp1q+32*2], m14 + mova [tmp1q+32*3], m15 + mova m0, [r2-32*3] + mova m1, [r2-32*1] + mova m2, [r2+32*1] + mova m3, [r2+32*3] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + test r10d, 0x40000000 + jnz .fast2 + mova m4, [r3-32*3] + mova m5, [r3-32*1] + mova m6, [r3+32*1] + mova m7, [r3+32*3] +.fast2: + add tmp1q, 32*8 + lea tmp2q, [tmp1q+32*8] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m15, [o(pd_2048)] + add r2, 32*8 + add r3, 32*8 + add tmp1q, 32*16 + add tmp2q, 32*32 + mova m0, [r2-32*4] ; 1 + mova m3, [r2+32*3] ; 15 + mova m4, [r2+32*0] ; 9 + mova m7, [r2-32*1] ; 7 + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r10d, 0x40000000 + jnz .fast3 + mova m1, [r3+32*3] ; 31 + mova m2, [r3-32*4] ; 17 + mova m5, [r3-32*1] ; 23 + mova m6, [r3+32*0] ; 25 +.fast3: + add r6, o_idct64_offset + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + add r6, 8 + add tmp1q, 32*8 + sub tmp2q, 32*8 + mova m0, [r2-32*2] ; 5 + mova m3, [r2+32*1] ; 11 + mova m4, [r2+32*2] ; 13 + mova m7, [r2-32*3] ; 3 + pxor m1, m1 + REPX {mova x, m1}, m2, m5, m6 + test r10d, 0x40000000 + jnz .fast4 + mova m1, [r3+32*1] ; 27 + mova m2, [r3-32*2] ; 21 + mova m5, [r3-32*3] ; 19 + mova m6, [r3+32*2] ; 29 +.fast4: + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 + sub tmp1q, 32*28 + sub dstq, r8 + lea dstq, [dstq+strideq*4+16] + dec r10b + jg .pass2_loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/itx_avx512.asm b/third_party/dav1d/src/x86/itx_avx512.asm new file mode 100644 index 0000000000..31c60fdd45 --- /dev/null +++ b/third_party/dav1d/src/x86/itx_avx512.asm @@ -0,0 +1,7389 @@ +; Copyright © 2020-2023, VideoLAN and dav1d authors +; Copyright © 2020-2023, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 +const \ +dup16_perm, db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7 + db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15 + db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23 + db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31 +const \ +int8_permA, db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 + db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55 + db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59 + db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 +int8_permB: db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 + db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59 + db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55 + db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 +int16_perm: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 + db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 + db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 + db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 +idtx_16x4p: db 0, 1, 4, 5, 16, 17, 20, 21, 2, 3, 6, 7, 18, 19, 22, 23 + db 32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55 + db 8, 9, 12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31 + db 40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63 +idct_8x32p: db 60, 61, 4, 5, 32, 33, 0, 1, 28, 29, 36, 37, 56, 57, 8, 9 + db 12, 13, 52, 53, 24, 25, 40, 41, 44, 45, 20, 21, 48, 49, 16, 17 + db 62, 63, 2, 3, 6, 7, 58, 59, 54, 55, 10, 11, 14, 15, 50, 51 + db 46, 47, 18, 19, 22, 23, 42, 43, 38, 39, 26, 27, 30, 31, 34, 35 +idct_16x32p: db 6, 7, 58, 59, 38, 39, 26, 27, 32, 33, 0, 1, 30, 31, 34, 35 + db 46, 47, 18, 19, 22, 23, 42, 43, 24, 25, 40, 41, 44, 45, 20, 21 + db 62, 63, 2, 3, 48, 49, 16, 17, 56, 57, 8, 9, 14, 15, 50, 51 + db 54, 55, 10, 11, 60, 61, 4, 5, 12, 13, 52, 53, 28, 29, 36, 37 +end_16x32p: db 0, 32, 1, 48, 2, 36, 3, 52, 16, 40, 17, 56, 18, 44, 19, 60 + db 4, 33, 5, 49, 6, 37, 7, 53, 20, 41, 21, 57, 22, 45, 23, 61 + db 8, 35, 9, 51, 10, 39, 11, 55, 24, 43, 25, 59, 26, 47, 27, 63 + db 12, 34, 13, 50, 14, 38, 15, 54, 28, 42, 29, 58, 30, 46, 31, 62 + +; packed 4-bit qword shuffle indices +permA: dq 0x1c0d0d1ce0d94040, 0x5849495868fb6262 + dq 0x3e2f2f3ef1c85151, 0x7a6b6b7a79ea7373 + dq 0x94858594a451c8d9, 0xd0c1c1d02c73eafb + dq 0xb6a7a7b6b540d9c8, 0xf2e3e3f23d62fbea +permB: dq 0x40acbd0fcadb0f40, 0x518e9f3ce8f99604 + dq 0xc824352d56128751, 0xd906171e74301e15 + dq 0x6271604b03472d62, 0x735342782165b426 + dq 0xeaf9e8699f8ea573, 0xfbdbca5abdac3c37 +permC: dq 0x9d409d041551c2e0, 0xbf62bf263773a486 + dq 0xc88c8c15409dd3f1, 0xeaaeae3762bfb597 + dq 0x04d9158c8cc84a68, 0x26fb37aeaeea2c0e + dq 0x5115049dd9045b79, 0x733726bffb263d1f +permD: dq 0x0cda098800041504, 0x0edb09b2028c3726 + dq 0x0f11fa9c01150415, 0x0988f326039d2637 + dq 0x05640f1108269d8c, 0x05290edb0aaebfae + dq 0x0005000509378c9d, 0xffffffff0bbfaebf + +pd_0to15: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +gather8a: dd 0, 2, 1, 3, 8, 10, 9, 11 +gather8b: dd 0, 1, 4, 5, 8, 9, 12, 13 +gather8c: dd 0, 4, 2, 6, 12, 8, 14, 10 +gather8d: dd 0, 19, 1, 18, 2, 17, 3, 16 + +int_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +int_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 +int_shuf3: db 0, 1, 8, 9, 4, 5, 12, 13, 2, 3, 10, 11, 6, 7, 14, 15 +int_shuf4: db 8, 9, 0, 1, 12, 13, 4, 5, 10, 11, 2, 3, 14, 15, 6, 7 +deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +int_mshift: db 12, 20, 0, 0, 44, 52, 0, 0 + +pb_32: times 4 db 32 +pw_2048: times 2 dw 2048 +pw_4096: times 2 dw 4096 +pw_8192: times 2 dw 8192 +pw_16384: times 2 dw 16384 +pw_1697x16: times 2 dw 1697*16 +pw_1697x8: times 2 dw 1697*8 +pw_2896x8: times 2 dw 2896*8 +pd_2048: dd 2048 + +%define pw_5 (permD+52) +%define pd_m1 (permD+60) +%define pw_3803_1321 (permD+44) +%define pw_2482_3803 (permD+12) +%define pw_2440_3290 (permD+ 4) +%define pw_m3290_2440 (permD+28) +%define pw_3857_1380 (permD+36) +%define pw_m1380_3857 (permD+20) + +pw_8192_m8192: dw 8192, -8192 +pw_m8192_8192: dw -8192, 8192 +pw_16384_m16384: dw 16384, -16384 +pw_m16384_16384: dw -16384, 16384 + +pw_m1321_2482: dw -1321, 2482 +pw_m3344_3344: dw -3344, 3344 +pw_2482_3344: dw 2482, 3344 +pw_m3803_3344: dw -3803, 3344 +pd_3344: dd 3344 +pw_m1321_m3344: dw -1321, -3344 +pw_2896_m2896: dw 2896, -2896 + +pw_1567_m3784: dw 1567, -3784 +pw_3784_m1567: dw 3784, -1567 +pw_4017_m799: dw 4017, -799 +pw_2276_m3406: dw 2276, -3406 +pw_m799_m4017: dw -799, -4017 +pw_m3406_m2276: dw -3406, -2276 + +%macro COEF_PAIR 2-3 0 +pw_%1_%2: dw %1, %2 +pw_m%2_%1: dw -%2, %1 +%if %3 +pw_m%1_m%2: dw -%1, -%2 +%endif +%endmacro + +COEF_PAIR 2896, 2896 +COEF_PAIR 1567, 3784, 1 +COEF_PAIR 3784, 1567 +COEF_PAIR 201, 4091 +COEF_PAIR 995, 3973 +COEF_PAIR 1751, 3703 +COEF_PAIR 3035, 2751 +COEF_PAIR 3513, 2106 +COEF_PAIR 4052, 601 +COEF_PAIR 3166, 2598, 1 +COEF_PAIR 3920, 1189, 1 +COEF_PAIR 2276, 3406 +COEF_PAIR 4017, 799 + +%macro COEF_X8 1-* +%rep %0 + dw %1*8, %1*8 + %rotate 1 +%endrep +%endmacro + +pw_m2276x8: COEF_X8 -2276 +pw_3406x8: COEF_X8 3406 +pw_4017x8: COEF_X8 4017 +pw_799x8: COEF_X8 799 +pw_3784x8: COEF_X8 3784 +pw_1567x8: COEF_X8 1567 + +pw_4076x8: COEF_X8 4076 +pw_401x8: COEF_X8 401 +pw_m2598x8: COEF_X8 -2598 +pw_3166x8: COEF_X8 3166 +pw_3612x8: COEF_X8 3612 +pw_1931x8: COEF_X8 1931 +pw_m1189x8: COEF_X8 -1189 +pw_3920x8: COEF_X8 3920 + +pw_4091x8: COEF_X8 4091 +pw_201x8: COEF_X8 201 +pw_m2751x8: COEF_X8 -2751 +pw_3035x8: COEF_X8 3035 +pw_3703x8: COEF_X8 3703 +pw_1751x8: COEF_X8 1751 +pw_m1380x8: COEF_X8 -1380 +pw_3857x8: COEF_X8 3857 +pw_3973x8: COEF_X8 3973 +pw_995x8: COEF_X8 995 +pw_m2106x8: COEF_X8 -2106 +pw_3513x8: COEF_X8 3513 +pw_3290x8: COEF_X8 3290 +pw_2440x8: COEF_X8 2440 +pw_m601x8: COEF_X8 -601 +pw_4052x8: COEF_X8 4052 + +pw_401_4076x8: dw 401*8, 4076*8 +pw_m2598_3166x8: dw -2598*8, 3166*8 +pw_1931_3612x8: dw 1931*8, 3612*8 +pw_m1189_3920x8: dw -1189*8, 3920*8 +pw_799_4017x8: dw 799*8, 4017*8 +pw_m2276_3406x8: dw -2276*8, 3406*8 + +pw_201_4091x8: dw 201*8, 4091*8 +pw_m601_4052x8: dw -601*8, 4052*8 +pw_995_3973x8: dw 995*8, 3973*8 +pw_m1380_3857x8: dw -1380*8, 3857*8 +pw_1751_3703x8: dw 1751*8, 3703*8 +pw_m2106_3513x8: dw -2106*8, 3513*8 +pw_2440_3290x8: dw 2440*8, 3290*8 +pw_m2751_3035x8: dw -2751*8, 3035*8 + +pw_101_4095x8: dw 101*8, 4095*8 +pw_m2824_2967x8: dw -2824*8, 2967*8 +pw_1660_3745x8: dw 1660*8, 3745*8 +pw_m1474_3822x8: dw -1474*8, 3822*8 +pw_897_3996x8: dw 897*8, 3996*8 +pw_m2191_3461x8: dw -2191*8, 3461*8 +pw_2359_3349x8: dw 2359*8, 3349*8 +pw_m700_4036x8: dw -700*8, 4036*8 +pw_501_4065x8: dw 501*8, 4065*8 +pw_m2520_3229x8: dw -2520*8, 3229*8 +pw_2019_3564x8: dw 2019*8, 3564*8 +pw_m1092_3948x8: dw -1092*8, 3948*8 +pw_1285_3889x8: dw 1285*8, 3889*8 +pw_m1842_3659x8: dw -1842*8, 3659*8 +pw_2675_3102x8: dw 2675*8, 3102*8 +pw_m301_4085x8: dw -301*8, 4085*8 + +idct64_mul: COEF_X8 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474 +COEF_PAIR 401, 4076, 1 +COEF_PAIR 799, 4017 + COEF_X8 -700, 4036, 2359, 3349, -2191, 3461, 897, 3996 +dw -2598, -3166, 3166, -2598, 2598, 3166, -4017, -799, 799, -4017 + COEF_X8 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092 +COEF_PAIR 1931, 3612, 1 +COEF_PAIR 3406, 2276 + COEF_X8 -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889 +dw -1189, -3920, 3920, -1189, 1189, 3920, -2276, -3406, 3406, -2276 + +SECTION .text + +%define o_base int8_permA+64*18 +%define o(x) (r5 - (o_base) + (x)) +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +; flags: 1 = swap, 2 = interleave (l), 4 = interleave (t), 8 = no_pack, +; 16 = special_mul1, 32 = special_mul2 +%macro ITX_MUL2X_PACK 6-7 0 ; dst/src, tmp[1-2], rnd, coef[1-2], flags + mova m%2, m%4 +%if %7 & 16 + vpdpwssd m%2, m%1, [o(pw_%5)] {bcstd} + mova m%3, m%4 +%if %7 & 32 + vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd} +%else + vpdpwssd m%3, m%1, m%6 +%endif +%elif %7 & 32 + vpdpwssd m%2, m%1, m%5 + mova m%3, m%4 + vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd} +%elif %6 < 32 + vpdpwssd m%2, m%1, m%5 + mova m%3, m%4 + vpdpwssd m%3, m%1, m%6 +%elif %7 & 1 + vpdpwssd m%2, m%1, [o(pw_%5_%6)] {bcstd} + mova m%3, m%4 + vpdpwssd m%3, m%1, [o(pw_m%6_%5)] {bcstd} +%else + vpdpwssd m%2, m%1, [o(pw_m%6_%5)] {bcstd} + mova m%3, m%4 + vpdpwssd m%3, m%1, [o(pw_%5_%6)] {bcstd} +%endif +%if %7 & 2 + psrld m%2, 12 + pslld m%3, 4 + vpshrdd m%1, m%3, m%2, 16 +%elif %7 & 4 + ; compared to using shifts (as above) this has better throughput, + ; but worse latency and requires setting up the opmask/index + ; registers, so only use this method for the larger transforms + pslld m%1, m%2, 4 + vpmultishiftqb m%1{k7}, m13, m%3 +%else + psrad m%2, 12 + psrad m%3, 12 +%if %7 & 8 == 0 + packssdw m%1, m%3, m%2 +%endif +%endif +%endmacro + +; flags: same as ITX_MUL2X_PACK +%macro ITX_MUL4X_PACK 10-11 0 ; dst/src, tmp[1-2], coef_tmp[1-2], rnd, coef[1-4], flags +%if %11 & 1 + vpbroadcastd m%4, [o(pw_%9_%10)] + vpbroadcastd m%4{k1}, [o(pw_%7_%8)] + vpbroadcastd m%5, [o(pw_m%10_%9)] + vpbroadcastd m%5{k1}, [o(pw_m%8_%7)] +%else + vpbroadcastd m%4, [o(pw_m%10_%9)] + vpbroadcastd m%4{k1}, [o(pw_m%8_%7)] + vpbroadcastd m%5, [o(pw_%9_%10)] + vpbroadcastd m%5{k1}, [o(pw_%7_%8)] +%endif + ITX_MUL2X_PACK %1, %2, %3, %6, %4, %5, %11 +%endmacro + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +%macro ITX_MULSUB_2W 7-8 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2 + punpcklwd m%3, m%2, m%1 + punpckhwd m%2, m%1 +%if %7 < 32 + mova m%1, m%5 + vpdpwssd m%1, m%3, m%7 + mova m%4, m%5 + vpdpwssd m%4, m%2, m%7 +%else + mova m%1, m%5 + vpdpwssd m%1, m%3, [o(pw_m%7_%6)] {bcstd} + mova m%4, m%5 + vpdpwssd m%4, m%2, [o(pw_m%7_%6)] {bcstd} +%endif + psrad m%1, 12 + psrad m%4, 12 + packssdw m%1, m%4 + mova m%4, m%5 +%if %7 < 32 + vpdpwssd m%4, m%2, m%6 + mova m%2, m%5 + vpdpwssd m%2, m%3, m%6 +%else + vpdpwssd m%4, m%2, [o(pw_%6_%7)] {bcstd} + mova m%2, m%5 + vpdpwssd m%2, m%3, [o(pw_%6_%7)] {bcstd} +%endif + psrad m%4, 12 + psrad m%2, 12 +%if %0 == 8 + packssdw m%8, m%2, m%4 +%else + packssdw m%2, m%4 +%endif +%endmacro + +%macro WRAP_XMM 1+ + %xdefine %%reset RESET_MM_PERMUTATION + INIT_XMM cpuname + DEFINE_MMREGS xmm + AVX512_MM_PERMUTATION + %1 + %%reset +%endmacro + +%macro WRAP_YMM 1+ + INIT_YMM cpuname + %1 + INIT_ZMM cpuname +%endmacro + +%macro ITX4_END 4-5 2048 ; row[1-4], rnd +%if %5 + vpbroadcastd m2, [o(pw_%5)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 +%endif + lea r2, [dstq+strideq*2] +%assign %%i 1 +%rep 4 + %if %1 & 2 + CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) + %else + CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) + %endif + %assign %%i %%i + 1 + %rotate 1 +%endrep + movd m2, [%%row_adr1] + pinsrd m2, [%%row_adr2], 1 + movd m3, [%%row_adr3] + pinsrd m3, [%%row_adr4], 1 + pmovzxbw m2, m2 + pmovzxbw m3, m3 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + movd [%%row_adr1], m0 + pextrd [%%row_adr2], m0, 1 + pextrd [%%row_adr3], m0, 2 + pextrd [%%row_adr4], m0, 3 + ret +%endmacro + +%macro INV_TXFM_FN 3 ; type1, type2, size +cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, 0, dst, stride, c, eob, tx2, base + %define %%p1 m(i%1_%3_internal_8bpc) + lea baseq, [o_base] + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [m(i%2_%3_internal_8bpc).pass2] +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x4 +%ifidn %1_%2, dct_dct + vpbroadcastw m0, [cq] + vpbroadcastd m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mov [cq], eobd + pmulhrsw m0, m1 + mova m1, m0 + jmp m(iadst_4x4_internal_8bpc).end2 +%endif +%endmacro + +%macro IDCT4_1D_PACKED 0 + vpbroadcastd m4, [o(pd_2048)] + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784 + ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896 + paddsw m0, m1, m2 ; out0 out1 + psubsw m1, m2 ; out3 out2 +%endmacro + +%macro IADST4_1D_PACKED 0 + punpcklwd m4, m1, m0 ; in2 in0 + punpckhwd m5, m1, m0 ; in3 in1 +.main2: + vpbroadcastd m3, [o(pd_2048)] + mova m0, m3 + vpdpwssd m0, m4, [o(pw_3803_1321)] {bcstd} + mova m2, m3 + vpdpwssd m2, m4, [o(pw_m1321_2482)] {bcstd} + mova m1, m3 + vpdpwssd m1, m4, [o(pw_m3344_3344)] {bcstd} + vpdpwssd m3, m4, [o(pw_2482_3803)] {bcstd} + vpdpwssd m0, m5, [o(pw_2482_3344)] {bcstd} + vpdpwssd m2, m5, [o(pw_m3803_3344)] {bcstd} + vpdpwssd m1, m5, [o(pd_3344)] {bcstd} + vpdpwssd m3, m5, [o(pw_m1321_m3344)] {bcstd} + REPX {psrad x, 12}, m0, m2, m1, m3 + packssdw m0, m2 ; out0 out1 + packssdw m1, m3 ; out2 out3 +%endmacro + +INIT_XMM avx512icl +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst +INV_TXFM_4X4_FN dct, identity + +cglobal idct_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + IDCT4_1D_PACKED + mova m2, [o(deint_shuf)] + shufps m3, m0, m1, q1331 + shufps m0, m0, m1, q0220 + pshufb m0, m2 + pshufb m1, m3, m2 + jmp tx2q +.pass2: + IDCT4_1D_PACKED + pxor ymm16, ymm16 + mova [cq], ymm16 + ITX4_END 0, 1, 3, 2 + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +cglobal iadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + call .main + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call .main +.end: + pxor ymm16, ymm16 + mova [cq], ymm16 +.end2: + ITX4_END 0, 1, 2, 3 +ALIGN function_align +.main: + IADST4_1D_PACKED + ret + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + call m(iadst_4x4_internal_8bpc).main + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + jmp tx2q +.pass2: + call m(iadst_4x4_internal_8bpc).main +.end: + pxor ymm16, ymm16 + mova [cq], ymm16 +.end2: + ITX4_END 3, 2, 1, 0 + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_4x4_internal_8bpc).end + +%macro INV_TXFM_4X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x8 +%ifidn %1_%2, dct_dct + movd xmm1, [o(pw_2896x8)] + pmulhrsw xmm0, xmm1, [cq] + movd xmm2, [o(pw_2048)] + pmulhrsw xmm0, xmm1 + pmulhrsw xmm0, xmm1 + pmulhrsw xmm0, xmm2 + vpbroadcastw ym0, xmm0 + mova ym1, ym0 + jmp m(iadst_4x8_internal_8bpc).end3 +%endif +%endmacro + +%macro IDCT8_1D_PACKED 0 + punpckhwd m5, m3, m0 ; in7 in1 + punpckhwd m4, m1, m2 ; in3 in5 + punpcklwd m3, m1 ; in6 in2 + punpcklwd m2, m0 ; in4 in0 +.main2: + vpbroadcastd m6, [o(pd_2048)] + ITX_MUL2X_PACK 5, 0, 1, 6, 799, 4017, 3 ; t4a t7a + ITX_MUL2X_PACK 4, 0, 1, 6, 3406, 2276, 3 ; t5a t6a + ITX_MUL2X_PACK 3, 0, 1, 6, 1567, 3784 ; t3 t2 + psubsw m0, m5, m4 ; t5a t6a (interleaved) + paddsw m4, m5 ; t4 t7 (interleaved) + ITX_MUL2X_PACK 2, 1, 5, 6, 2896, 2896 ; t0 t1 + ITX_MUL2X_PACK 0, 1, 5, 6, 2896, 2896, 1 ; t6 t5 +%if mmsize > 16 + vbroadcasti32x4 m1, [o(deint_shuf)] + pshufb m4, m1 +%else + pshufb m4, [o(deint_shuf)] +%endif + psubsw m1, m2, m3 ; tmp3 tmp2 + paddsw m3, m2 ; tmp0 tmp1 + punpckhqdq m2, m4, m0 ; t7 t6 + punpcklqdq m4, m0 ; t4 t5 + paddsw m0, m3, m2 ; out0 out1 + psubsw m3, m2 ; out7 out6 + psubsw m2, m1, m4 ; out4 out5 + paddsw m1, m4 ; out3 out2 +%endmacro + +%macro IADST8_1D_PACKED 1 ; pass + vpbroadcastd m6, [o(pd_2048)] +%if %1 == 1 + ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076, 3 ; t1a t0a + ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612, 2 ; t2a t3a + ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598, 3 ; t5a t4a + ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189, 2 ; t6a t7a + psubsw m4, m0, m2 ; t5 t4 + paddsw m0, m2 ; t1 t0 + psubsw m5, m1, m3 ; t6 t7 + paddsw m1, m3 ; t2 t3 + ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784, 3 ; t5a t4a + ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 2 ; t7a t6a +%if mmsize > 16 + vbroadcasti32x4 m2, [o(deint_shuf)] +%else + mova m2, [o(deint_shuf)] +%endif + vprord m1, 16 + psubsw m3, m0, m1 ; t3 t2 + paddsw m0, m1 ; -out7 out0 + psubsw m1, m4, m5 ; t7 t6 + paddsw m4, m5 ; out6 -out1 + pshufb m0, m2 + pshufb m4, m2 + mova m2, m6 + vpdpwssd m2, m3, [o(pw_m2896_2896)] {bcstd} + mova m5, m6 + vpdpwssd m5, m1, [o(pw_m2896_2896)] {bcstd} + psrad m2, 12 + psrad m5, 12 + packssdw m2, m5 ; out4 -out5 + mova m5, m6 + vpdpwssd m5, m3, [o(pw_2896_2896)] {bcstd} + mova m3, m6 + vpdpwssd m3, m1, [o(pw_2896_2896)] {bcstd} + psrad m5, 12 + psrad m3, 12 + packssdw m1, m3, m5 ; out2 -out3 +%else + punpckhwd m0, m4, m3 ; 0 7 + punpckhwd m1, m5, m2 ; 2 5 + punpcklwd m2, m5 ; 4 3 + punpcklwd m3, m4 ; 6 1 + ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076 ; t0a t1a + ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612 ; t2a t3a + ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598 ; t4a t5a + ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189 ; t6a t7a + psubsw m4, m0, m2 ; t4 t5 + paddsw m0, m2 ; t0 t1 + psubsw m5, m1, m3 ; t6 t7 + paddsw m1, m3 ; t2 t3 + shufps m2, m5, m4, q1032 + punpckhwd m4, m2 + punpcklwd m5, m2 + ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784 ; t4a t5a + ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 1 ; t6a t7a + psubsw m2, m0, m1 ; t2 t3 + paddsw m0, m1 ; out0 -out7 + psubsw m1, m4, m5 ; t6 t7 + paddsw m4, m5 ; -out1 out6 + vpbroadcastd m5, [o(pw_2896x8)] + punpckhqdq m3, m2, m1 ; t3 t7 + punpcklqdq m2, m1 ; t2 t6 + paddsw m1, m2, m3 ; t2+t3 t6+t7 + psubsw m2, m3 ; t2-t3 t6-t7 + punpckhqdq m3, m4, m0 ; out6 -out7 + punpcklqdq m0, m4 ; out0 -out1 + pmulhrsw m2, m5 ; out4 -out5 + pshufd m1, m1, q1032 + pmulhrsw m1, m5 ; out2 -out3 +%endif +%endmacro + +INIT_YMM avx512icl +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, identity +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst + +cglobal idct_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + IDCT4_1D_PACKED + vbroadcasti32x4 m2, [o(deint_shuf)] + shufps m3, m0, m1, q1331 + shufps m0, m0, m1, q0220 + pshufb m0, m2 + pshufb m1, m3, m2 + jmp tx2q +.pass2: + vextracti32x4 xm2, m0, 1 + vextracti32x4 xm3, m1, 1 + call .main + vpbroadcastd m4, [o(pw_2048)] + vinserti32x4 m0, m0, xm2, 1 + vinserti32x4 m1, m1, xm3, 1 + pshufd m1, m1, q1032 + jmp m(iadst_4x8_internal_8bpc).end2 +ALIGN function_align +.main: + WRAP_XMM IDCT8_1D_PACKED + ret + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity + +cglobal iadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + call m(iadst_8x4_internal_8bpc).main + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + vextracti32x4 xm2, m0, 1 + vextracti32x4 xm3, m1, 1 + pshufd xm4, xm0, q1032 + pshufd xm5, xm1, q1032 + call .main_pass2 + vpbroadcastd m4, [o(pw_2048)] + vinserti32x4 m0, xm2, 1 + vinserti32x4 m1, xm3, 1 + pxor m5, m5 + psubw m5, m4 +.end: + punpcklqdq m4, m5 +.end2: + pmulhrsw m0, m4 + pmulhrsw m1, m4 +.end3: + vpbroadcastd m3, strided + pmulld m5, m3, [o(pd_0to15)] + kxnorb k1, k1, k1 + kmovb k2, k1 + vpgatherdd m3{k1}, [dstq+m5] + pxor m4, m4 + mova [cq], zmm20 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + vpscatterdd [dstq+m5]{k2}, m0 + RET +ALIGN function_align +.main_pass1: + punpckhwd xm0, xm4, xm3 ; 0 7 + punpckhwd xm1, xm5, xm2 ; 2 5 + punpcklwd xm2, xm5 ; 4 3 + punpcklwd xm3, xm4 ; 6 1 + WRAP_XMM IADST8_1D_PACKED 1 + punpcklqdq xm3, xm4, xm0 ; out6 -out7 + punpckhqdq xm0, xm4 ; out0 -out1 + ret +ALIGN function_align +.main_pass2: + WRAP_XMM IADST8_1D_PACKED 2 + ret + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity + +cglobal iflipadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 + vpermq m1, [cq+32*1], q3120 + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + call m(iadst_8x4_internal_8bpc).main + punpcklwd m3, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m3 + punpckhwd m1, m3 + jmp tx2q +.pass2: + vextracti32x4 xm2, m0, 1 + vextracti32x4 xm3, m1, 1 + pshufd xm4, xm0, q1032 + pshufd xm5, xm1, q1032 + call m(iadst_4x8_internal_8bpc).main_pass2 + vpbroadcastd m5, [o(pw_2048)] + vinserti32x4 m3, xm1, 1 + vinserti32x4 m2, xm0, 1 + pxor m4, m4 + psubw m4, m5 + pshufd m0, m3, q1032 + pshufd m1, m2, q1032 + jmp m(iadst_4x8_internal_8bpc).end + +INIT_ZMM avx512icl +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity + +cglobal iidentity_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd m0, [o(pw_2896x8)] + pmulhrsw m0, [cq] + mova m1, [o(int8_permB)] + vpbroadcastd m2, [o(pw_1697x8)] + vpermb m0, m1, m0 + pmulhrsw m2, m0 + paddsw m0, m2 + vextracti32x8 ym1, m0, 1 + jmp tx2q +.pass2: + vpbroadcastd ym4, [o(pw_4096)] + jmp m(iadst_4x8_internal_8bpc).end2 + +%macro INV_TXFM_4X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x16 +%ifidn %1_%2, dct_dct + movsx r6d, word [cq] + mov [cq], eobd + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 + imul r6d, 181 + add r6d, 128+2048 + sar r6d, 8+4 + vpbroadcastw m0, r6d + mova m1, m0 + jmp m(iadst_4x16_internal_8bpc).end3 +%endif +%endmacro + +%macro IDCT16_1D_PACKED 0 + punpckhwd m8, m7, m0 ; dct16 in15 in1 + punpcklwd m9, m4, m0 ; dct4 in2 in0 + punpckhwd m0, m3, m4 ; dct16 in7 in9 + punpcklwd m7, m1 ; dct8 in7 in1 + punpckhwd m1, m6 ; dct16 in3 in13 + punpcklwd m3, m5 ; dct8 in3 in5 + punpckhwd m5, m2 ; dct16 in11 in5 + punpcklwd m6, m2 ; dct4 in3 in1 +cglobal_label .main2 + vpbroadcastd m10, [o(pd_2048)] +.main3: + vpbroadcastq m13, [o(int_mshift)] + vpcmpub k7, m13, m10, 6 ; 0x33... + ITX_MUL2X_PACK 8, 2, 4, 10, 401, 4076, 5 ; t8a t15a + ITX_MUL2X_PACK 0, 2, 4, 10, 3166, 2598, 5 ; t9a t14a + ITX_MUL2X_PACK 5, 2, 4, 10, 1931, 3612, 5 ; t10a t13a + ITX_MUL2X_PACK 1, 2, 4, 10, 3920, 1189, 5 ; t11a t12a + ITX_MUL2X_PACK 7, 2, 4, 10, 799, 4017, 5 ; t4a t7a + ITX_MUL2X_PACK 3, 2, 4, 10, 3406, 2276, 5 ; t5a t6a +.main4: + psubsw m2, m8, m0 ; t9 t14 + paddsw m8, m0 ; t8 t15 + psubsw m4, m1, m5 ; t10 t13 + paddsw m1, m5 ; t11 t12 + ITX_MUL2X_PACK 6, 0, 5, 10, 1567, 3784 ; t3 t2 + psubsw m0, m8, m1 ; t11a t12a + paddsw m8, m1 ; t8a t15a + psubsw m1, m7, m3 ; t5a t6a + paddsw m7, m3 ; t4 t7 +.main5: + ITX_MUL2X_PACK 2, 3, 5, 10, 1567, 3784, 5 ; t9a t14a + ITX_MUL2X_PACK 4, 3, 5, 10, m3784, 1567, 5 ; t10a t13a +%if mmsize > 16 + vbroadcasti32x4 m5, [o(deint_shuf)] +%else + mova m5, [o(deint_shuf)] +%endif + vpbroadcastd m11, [o(pw_m2896_2896)] + vpbroadcastd m12, [o(pw_2896_2896)] + paddsw m3, m2, m4 ; t9 t14 + psubsw m2, m4 ; t10 t13 + pshufb m8, m5 + pshufb m7, m5 + pshufb m3, m5 + ITX_MUL2X_PACK 9, 4, 5, 10, 11, 12 ; t0 t1 + ITX_MUL2X_PACK 1, 4, 5, 10, 12, 11 ; t5 t6 + ITX_MUL2X_PACK 0, 4, 5, 10, 11, 12, 8 ; t11 t12 + ITX_MUL2X_PACK 2, 0, 11, 10, 11, 12, 8 ; t10a t13a + punpckhqdq m2, m7, m1 ; t7 t6 + punpcklqdq m7, m1 ; t4 t5 + psubsw m1, m9, m6 ; dct4 out3 out2 + paddsw m9, m6 ; dct4 out0 out1 + packssdw m5, m11 ; t12 t13a + packssdw m4, m0 ; t11 t10a + punpckhqdq m0, m8, m3 ; t15a t14 + punpcklqdq m8, m3 ; t8a t9 + psubsw m3, m9, m2 ; dct8 out7 out6 + paddsw m9, m2 ; dct8 out0 out1 + psubsw m2, m1, m7 ; dct8 out4 out5 + paddsw m1, m7 ; dct8 out3 out2 + psubsw m7, m9, m0 ; out15 out14 + paddsw m0, m9 ; out0 out1 + psubsw m6, m1, m5 ; out12 out13 + paddsw m1, m5 ; out3 out2 + psubsw m5, m2, m4 ; out11 out10 + paddsw m2, m4 ; out4 out5 + psubsw m4, m3, m8 ; out8 out9 + paddsw m3, m8 ; out7 out6 +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, identity +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst + +cglobal idct_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova ym1, [cq+32*2] + vinserti32x8 m1, [cq+32*0], 1 + mova m0, [o(int16_perm)] + mova ym2, [cq+32*3] + vinserti32x8 m2, [cq+32*1], 1 + vpbroadcastd m4, [o(pd_2048)] + vpermb m1, m0, m1 ; c0 a0 c1 a1 c2 a2 c3 a3 + vpermb m2, m0, m2 ; d0 b0 d1 b1 d2 b2 d3 b3 + ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896, 2 + ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784, 2 + vpbroadcastd m4, [o(pw_16384)] + psubsw m3, m1, m2 + paddsw m1, m2 ; out0 out1 + vprord m3, 16 ; out2 out3 + punpckldq m0, m1, m3 + punpckhdq m1, m3 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + jmp tx2q +.pass2: + vextracti32x4 xm2, ym0, 1 + vextracti32x4 xm3, ym1, 1 + vextracti32x4 xm4, m0, 2 + vextracti32x4 xm5, m1, 2 + vextracti32x4 xm6, m0, 3 + vextracti32x4 xm7, m1, 3 + call .main + vinserti32x4 ym0, xm2, 1 + vinserti32x4 ym1, xm3, 1 + vinserti32x4 ym4, xm6, 1 + vinserti32x4 ym5, xm7, 1 + vinserti32x8 m0, ym4, 1 + vinserti32x8 m1, ym5, 1 + vpbroadcastd m5, [o(pw_2048)] + pshufd m1, m1, q1032 + jmp m(iadst_4x16_internal_8bpc).end2 +ALIGN function_align +.main: + WRAP_XMM IDCT16_1D_PACKED + ret + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity + +cglobal iadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m1, [o(permB)] + vpermq m0, m1, [cq+64*0] + vpermq m1, m1, [cq+64*1] + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m3, [o(pw_16384)] + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + pmulhrsw m2, m3 + pmulhrsw m0, m3 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + jmp tx2q +.pass2: + call .main + vpbroadcastd m5, [o(pw_2048)] + psrlq m10, 4 + psubw m6, m8, m5 +.end: + vpbroadcastd m7, [o(pw_2896x8)] + paddsw ym1, ym2, ym4 + psubsw ym2, ym4 + vinserti32x8 m1, ym2, 1 + pmulhrsw m1, m7 ; -out7 out4 out6 -out5 out8 -out11 -out9 out10 + psrlq m0, m10, 4 + vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d + vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f + punpcklqdq m5, m6 +.end2: + pmulhrsw m0, m5 + pmulhrsw m1, m5 +.end3: + vpbroadcastd m3, strided + pmulld m5, m3, [o(pd_0to15)] + kxnorw k1, k1, k1 + kmovw k2, k1 + vpgatherdd m3{k1}, [dstq+m5] + pxor m4, m4 + mova [cq+64*0], m4 + mova [cq+64*1], m4 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + vpscatterdd [dstq+m5]{k2}, m0 + RET +ALIGN function_align +.main: + movu m3, [o(permB+1)] + psrlq m10, m3, 4 +.main2: + vpermi2q m3, m0, m1 ; in15 in12 in13 in14 in11 in8 in9 in10 + vpermt2q m0, m10, m1 ; in0 in3 in2 in1 in4 in7 in6 in5 + vpbroadcastd m9, [o(pd_2048)] + vpbroadcastq ym13, [o(int_mshift)] + kxnorb k1, k1, k1 + punpckhwd m4, m3, m0 ; in12 in3 in14 in1 + punpcklwd m0, m3 ; in0 in15 in2 in13 + kshiftrb k1, k1, 4 + vextracti32x8 ym3, m4, 1 ; in8 in7 in10 in5 + vextracti32x8 ym1, m0, 1 ; in4 in11 in6 in9 +INIT_YMM avx512icl + vpcmpub k7, m13, m9, 6 ; 0x33... + pxor m8, m8 + ITX_MUL4X_PACK 0, 2, 5, 6, 7, 9, 201, 4091, 995, 3973, 5 + ITX_MUL4X_PACK 1, 2, 5, 6, 7, 9, 1751, 3703, 2440, 3290, 5 + ITX_MUL4X_PACK 3, 2, 5, 6, 7, 9, 3035, 2751, 3513, 2106, 5 + ITX_MUL4X_PACK 4, 2, 5, 6, 7, 9, 3857, 1380, 4052, 601, 5 + psubsw m2, m0, m3 ; t9a t8a t11a t10a + paddsw m0, m3 ; t1a t0a t3a t2a + psubsw m3, m1, m4 ; t13a t12a t15a t14a + paddsw m4, m1 ; t5a t4a t7a t6a + ITX_MUL4X_PACK 2, 1, 5, 6, 7, 9, 799, 4017, 3406, 2276, 5 + psubw m7, m8, m7 + ITX_MUL2X_PACK 3, 1, 5, 9, 7, 6, 4 + vpbroadcastd m6, [o(pw_3784_m1567)] + vpbroadcastd m6{k1}, [o(pw_m3784_1567)] + psubsw m1, m0, m4 ; t5 t4 t7 t6 + paddsw m0, m4 ; t1 t0 t3 t2 + psubsw m4, m2, m3 ; t13a t12a t15a t14a + paddsw m2, m3 ; t9a t8a t11a t10a + ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t4a t5a t7a t6a + ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t12 t13 t15 t14 + vbroadcasti32x4 m5, [o(deint_shuf)] + pshufb m0, m5 + pshufb m2, m5 + vshufi32x4 m3, m0, m2, 0x03 ; t3 t2 t11a t10a + vinserti32x4 m0, xm2, 1 ; t1 t0 t9a t8a + vshufi32x4 m2, m1, m4, 0x03 ; t7a t6a t15 t14 + vinserti32x4 m1, xm4, 1 ; t4a t5a t12 t13 + pshufd m2, m2, q1032 ; t6a t7a t14 t15 + psubsw m4, m0, m3 ; t3a t2a t11 t10 + paddsw m0, m3 ; -out15 out0 out14 -out1 + paddsw m3, m1, m2 ; out12 -out3 -out13 out2 + psubsw m1, m2 ; t7 t6 t15a t14a + punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a + punpcklqdq m4, m1 ; t3a t7 t11 t15a +INIT_ZMM avx512icl + vinserti32x8 m3, ym0, 1 ; out12 -out3 -out13 out2 -out15 out0 out14 -out1 + ret + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity + +cglobal iflipadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m1, [o(permB)] + vpermq m0, m1, [cq+64*0] + vpermq m1, m1, [cq+64*1] + call m(iadst_16x4_internal_8bpc).main + vpbroadcastd m3, [o(pw_16384)] + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + pmulhrsw m2, m3 + pmulhrsw m1, m3 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + jmp tx2q +.pass2: + call m(iadst_4x16_internal_8bpc).main + vpbroadcastd m6, [o(pw_2048)] + psrlq m10, 12 + psubw m5, m8, m6 + jmp m(iadst_4x16_internal_8bpc).end + +INV_TXFM_4X16_FN identity, dct +INV_TXFM_4X16_FN identity, adst +INV_TXFM_4X16_FN identity, flipadst +INV_TXFM_4X16_FN identity, identity + +cglobal iidentity_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m2, [o(int16_perm)] + vpermb m1, m2, [cq+64*0] + vpermb m2, m2, [cq+64*1] + vpbroadcastd m4, [o(pw_1697x8)] + vpbroadcastd m0, [o(pd_m1)] + pmulhrsw m3, m4, m1 ; we want to do a signed avg, but pavgw is + vpcmpw k1, m1, m0, 4 ; unsigned. as long as both signs are equal + pmulhrsw m4, m2 ; it still works, but if the input is -1 the + vpcmpw k2, m2, m0, 4 ; pmulhrsw result will become 0 which causes + vpavgw m1{k1}{z}, m3 ; pavgw to output -32768 instead of 0 unless + vpavgw m2{k2}{z}, m4 ; we explicitly deal with that case here. + punpckldq m0, m1, m2 + punpckhdq m1, m2 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x16)] + vpbroadcastd m5, [o(pw_2048)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m0 + paddsw m1, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_4x16_internal_8bpc).end2 + +%macro WRITE_8X4 4-7 strideq*1, strideq*2, r6 ; coefs[1-2], tmp[1-2], off[1-3] + movq xm%3, [dstq ] + movhps xm%3, [dstq+%5] + movq xm%4, [dstq+%6] + movhps xm%4, [dstq+%7] + pmovzxbw m%3, xm%3 + pmovzxbw m%4, xm%4 +%ifnum %1 + paddw m%3, m%1 +%else + paddw m%3, %1 +%endif +%ifnum %2 + paddw m%4, m%2 +%else + paddw m%4, %2 +%endif + packuswb m%3, m%4 + vextracti32x4 xm%4, m%3, 1 + movq [dstq ], xm%3 + movhps [dstq+%6], xm%3 + movq [dstq+%5], xm%4 + movhps [dstq+%7], xm%4 +%endmacro + +%macro INV_TXFM_8X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x4 +%ifidn %1_%2, dct_dct + movd xm1, [o(pw_2896x8)] + pmulhrsw xm0, xm1, [cq] + movd xm2, [o(pw_2048)] + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm1 + pmulhrsw xm0, xm2 + vpbroadcastw m0, xm0 + mova m1, m0 + jmp m(iadst_8x4_internal_8bpc).end3 +%endif +%endmacro + +INIT_YMM avx512icl +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst +INV_TXFM_8X4_FN dct, identity + +cglobal idct_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd xm3, [o(pw_2896x8)] + pmulhrsw xm0, xm3, [cq+16*0] + pmulhrsw xm1, xm3, [cq+16*1] + pmulhrsw xm2, xm3, [cq+16*2] + pmulhrsw xm3, [cq+16*3] + call m(idct_4x8_internal_8bpc).main + vbroadcasti32x4 m4, [o(deint_shuf)] + vinserti32x4 m3, m1, xm3, 1 + vinserti32x4 m1, m0, xm2, 1 + shufps m0, m1, m3, q0220 + shufps m1, m3, q1331 + pshufb m0, m4 + pshufb m1, m4 + jmp tx2q +.pass2: + IDCT4_1D_PACKED + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + jmp m(iadst_8x4_internal_8bpc).end2 + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd xm0, [o(pw_2896x8)] + pshufd xm4, [cq+16*0], q1032 + pmulhrsw xm3, xm0, [cq+16*3] + pshufd xm5, [cq+16*1], q1032 + pmulhrsw xm2, xm0, [cq+16*2] + pmulhrsw xm4, xm0 + pmulhrsw xm5, xm0 + call m(iadst_4x8_internal_8bpc).main_pass1 + vinserti32x4 m0, xm2, 1 + vinserti32x4 m1, xm3, 1 + pxor m3, m3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + psubsw m3, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call .main +.end: + vpermq m0, m0, q3120 + vpermq m1, m1, q3120 +.end2: + vpbroadcastd m2, [o(pw_2048)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 +.end3: + pxor m2, m2 + mova [cq], zmm18 + lea r6, [strideq*3] + WRITE_8X4 0, 1, 4, 5 + RET +ALIGN function_align +.main: + IADST4_1D_PACKED + ret + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd xm0, [o(pw_2896x8)] + pshufd xm4, [cq+16*0], q1032 + pmulhrsw xm3, xm0, [cq+16*3] + pshufd xm5, [cq+16*1], q1032 + pmulhrsw xm2, xm0, [cq+16*2] + pmulhrsw xm4, xm0 + pmulhrsw xm5, xm0 + call m(iadst_4x8_internal_8bpc).main_pass1 + vinserti32x4 m3, m3, xm1, 1 + vinserti32x4 m2, m2, xm0, 1 + punpckhwd m1, m3, m2 + punpcklwd m3, m2 + pxor m0, m0 + psubsw m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + jmp tx2q +.pass2: + call m(iadst_8x4_internal_8bpc).main + mova m2, m1 + vpermq m1, m0, q2031 + vpermq m0, m2, q2031 + jmp m(iadst_8x4_internal_8bpc).end2 + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova xm2, [cq+16*0] + mova xm0, [cq+16*1] + vinserti32x4 m2, [cq+16*2], 1 + vinserti32x4 m0, [cq+16*3], 1 + vpbroadcastd m3, [o(pw_2896x8)] + punpcklwd m1, m2, m0 + punpckhwd m2, m0 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + paddsw m0, m0 + paddsw m1, m1 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_8x4_internal_8bpc).end + +%macro INV_TXFM_8X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x8 +%ifidn %1_%2, dct_dct +INIT_ZMM avx512icl + movsx r6d, word [cq] + mov [cq], eobd +.dconly: + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 +.dconly2: + vpbroadcastd ym2, strided + imul r6d, 181 + pmulld ym5, ym2, [o(pd_0to15)] + kxnorb k1, k1, k1 + add r6d, 128+2048 + sar r6d, 8+4 + pxor m3, m3 + vpbroadcastw m4, r6d +.dconly_loop: + kmovb k2, k1 + vpgatherdq m2{k1}, [dstq+ym5] + punpcklbw m0, m2, m3 + punpckhbw m1, m2, m3 + paddw m0, m4 + paddw m1, m4 + packuswb m0, m1 + kmovb k1, k2 + vpscatterdq [dstq+ym5]{k2}, m0 + lea dstq, [dstq+strideq*8] + sub r3d, 8 + jg .dconly_loop + RET +INIT_YMM avx512icl +%endif +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, identity +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst + +cglobal idct_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m0, [cq+32*0], q3120 ; 0 1 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m2, [cq+32*2], q3120 ; 4 5 + vpermq m1, [cq+32*1], q3120 ; 2 3 + call .main + shufps m4, m0, m1, q0220 + shufps m5, m0, m1, q1331 + shufps m1, m2, m3, q0220 + shufps m3, m2, m3, q1331 + vbroadcasti32x4 m0, [o(deint_shuf)] + vpbroadcastd m2, [o(pw_16384)] + REPX {pshufb x, m0}, m4, m5, m1, m3 + REPX {pmulhrsw x, m2}, m4, m5, m1, m3 + vinserti32x4 m0, m4, xm1, 1 + vshufi32x4 m2, m4, m1, 0x03 + vinserti32x4 m1, m5, xm3, 1 + vshufi32x4 m3, m5, m3, 0x03 + jmp tx2q +.pass2: + call .main + vpbroadcastd m4, [o(pw_2048)] + vpermq m0, m0, q3120 + vpermq m1, m1, q2031 + vpermq m2, m2, q3120 + vpermq m3, m3, q2031 + jmp m(iadst_8x8_internal_8bpc).end2 +ALIGN function_align +cglobal_label .main + IDCT8_1D_PACKED + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity + +cglobal iadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m4, [cq+32*0], q1302 ; 1 0 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m5, [cq+32*1], q1302 ; 3 2 + vpermq m2, [cq+32*2], q3120 ; 4 5 + call .main_pass1 + vpbroadcastd m5, [o(pw_16384_m16384)] + punpcklwd m4, m0, m1 + punpckhwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + punpcklwd m3, m4, m0 + punpckhwd m4, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + REPX {pmulhrsw x, m5}, m3, m4, m0, m1 + vshufi32x4 m2, m3, m0, 0x03 + vinserti32x4 m0, m3, xm0, 1 + vshufi32x4 m3, m4, m1, 0x03 + vinserti32x4 m1, m4, xm1, 1 + jmp tx2q +.pass2: + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call .main_pass2 + vpbroadcastd m5, [o(pw_2048)] + vpbroadcastd xm4, [o(pw_4096)] + psubw m4, m5 ; lower half = 2048, upper half = -2048 +.end: + REPX {vpermq x, x, q3120}, m0, m1, m2, m3 +.end2: + pmulhrsw m0, m4 + pmulhrsw m1, m4 +.end3: + pmulhrsw m2, m4 + pmulhrsw m3, m4 +.end4: + pxor m4, m4 + mova [cq+32*0], m4 + mova [cq+32*1], m4 + mova [cq+32*2], m4 + mova [cq+32*3], m4 + lea r6, [strideq*3] + WRITE_8X4 0, 1, 4, 5 + lea dstq, [dstq+strideq*4] + WRITE_8X4 2, 3, 4, 5 + RET +ALIGN function_align +.main_pass1: + punpckhwd m0, m4, m3 ; 0 7 + punpckhwd m1, m5, m2 ; 2 5 + punpcklwd m2, m5 ; 4 3 + punpcklwd m3, m4 ; 6 1 + IADST8_1D_PACKED 1 + punpcklqdq m3, m4, m0 ; out6 -out7 + punpckhqdq m0, m4 ; out0 -out1 + ret +ALIGN function_align +cglobal_label .main_pass2 + IADST8_1D_PACKED 2 + ret + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity + +cglobal iflipadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpermq m4, [cq+32*0], q1302 ; 1 0 + vpermq m3, [cq+32*3], q3120 ; 6 7 + vpermq m5, [cq+32*1], q1302 ; 3 2 + vpermq m2, [cq+32*2], q3120 ; 4 5 + call m(iadst_8x8_internal_8bpc).main_pass1 + vpbroadcastd m5, [o(pw_m16384_16384)] + punpckhwd m4, m3, m2 + punpcklwd m3, m2 + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + punpckhwd m0, m4, m3 + punpcklwd m4, m3 + punpckhwd m3, m2, m1 + punpcklwd m2, m1 + REPX {pmulhrsw x, m5}, m0, m4, m3, m2 + vinserti32x4 m1, m0, xm3, 1 + vshufi32x4 m3, m0, m3, 0x03 + vinserti32x4 m0, m4, xm2, 1 + vshufi32x4 m2, m4, m2, 0x03 + jmp tx2q +.pass2: + pshufd m4, m0, q1032 + pshufd m5, m1, q1032 + call m(iadst_8x8_internal_8bpc).main_pass2 + vpbroadcastd m4, [o(pw_2048)] + vpbroadcastd xm5, [o(pw_4096)] + psubw m4, m5 ; lower half = -2048, upper half = 2048 + vpermq m5, m3, q2031 + vpermq m3, m0, q2031 + vpermq m0, m2, q2031 + vpermq m2, m1, q2031 + pmulhrsw m1, m0, m4 + pmulhrsw m0, m5, m4 + jmp m(iadst_8x8_internal_8bpc).end3 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova xm3, [cq+16*0] + mova xm2, [cq+16*1] + vinserti32x4 m3, [cq+16*4], 1 + vinserti32x4 m2, [cq+16*5], 1 + mova xm4, [cq+16*2] + mova xm0, [cq+16*3] + vinserti32x4 m4, [cq+16*6], 1 + vinserti32x4 m0, [cq+16*7], 1 + punpcklwd m1, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m4, m0 + punpckhwd m4, m0 + punpckldq m0, m1, m2 + punpckhdq m1, m2 + punpckldq m2, m3, m4 + punpckhdq m3, m4 + jmp tx2q +.pass2: + vpbroadcastd m4, [o(pw_4096)] + jmp m(iadst_8x8_internal_8bpc).end + +%macro INV_TXFM_8X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x16 +%ifidn %1_%2, dct_dct + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 16 + imul r6d, 181 + add r6d, 128 + sar r6d, 8 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly +%endif +%endmacro + +%macro ITX_8X16_LOAD_COEFS 0 + vpbroadcastd m4, [o(pw_2896x8)] + pmulhrsw m0, m4, [cq+32*0] + add cq, 32*4 + pmulhrsw m7, m4, [cq+32*3] + pmulhrsw m1, m4, [cq-32*3] + pmulhrsw m6, m4, [cq+32*2] + pmulhrsw m2, m4, [cq-32*2] + pmulhrsw m5, m4, [cq+32*1] + pmulhrsw m3, m4, [cq-32*1] + pmulhrsw m4, [cq+32*0] +%endmacro + +INIT_ZMM avx512icl +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, identity +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst + +cglobal idct_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m3, [o(permB)] + vpermq m0, m3, [cq+64*0] + vpbroadcastd m4, [o(pw_2896x8)] + vpermq m1, m3, [cq+64*1] + vpermq m2, m3, [cq+64*2] + vpermq m3, m3, [cq+64*3] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + call m(idct_16x8_internal_8bpc).main + vpbroadcastd m5, [o(pw_16384)] + punpckhwd m4, m0, m2 ; b0 f0 b1 f1 b2 f2 b3 f3 + punpcklwd m0, m2 ; a0 e0 a1 e1 a2 e2 a3 e3 + punpckhwd m2, m1, m3 ; c0 g0 c1 g1 c2 g2 c3 g3 + punpcklwd m1, m3 ; d0 h0 d1 h1 d2 h2 d3 h3 + REPX {pmulhrsw x, m5}, m4, m0, m2, m1 + punpckhwd m3, m0, m4 ; a2 b2 e2 f2 a3 b3 e3 f3 + punpcklwd m0, m4 ; a0 b0 e0 f0 a1 b1 e1 f1 + punpckhwd m4, m2, m1 ; c2 d2 g2 h2 c3 d3 g3 h3 + punpcklwd m2, m1 ; c0 d0 g0 h0 c1 d1 g1 h1 + punpckhdq m1, m0, m2 ; 1 5 9 13 + punpckldq m0, m2 ; 0 4 8 12 + punpckldq m2, m3, m4 ; 2 6 10 14 + punpckhdq m3, m4 ; 3 7 11 15 + jmp tx2q +.pass2: + vprord m5, [o(int16_perm)], 16 + vshufi32x4 m2, m2, q1320 ; 2 10 14 6 + vshufi32x4 m4, m1, m3, q2310 ; 1 5 15 11 + vshufi32x4 m1, m3, q0132 ; 9 13 7 3 + vpermb m9, m5, m0 + vpermb m7, m5, m2 + vpermb m8, m5, m4 + vpermb m0, m5, m1 + vextracti32x8 ym6, m9, 1 + vextracti32x8 ym3, m7, 1 + vextracti32x8 ym5, m8, 1 + vextracti32x8 ym1, m0, 1 + call .main2 + mova ym8, [o(gather8a)] + lea r3, [dstq+strideq*4] + pmovzxdq m9, ym8 + pshufd ym8, ym8, q1230 + vpermt2q m0, m9, m4 + vpermt2q m1, m9, m5 + vpermt2q m2, m9, m6 + vpermt2q m3, m9, m7 +.end: + vpbroadcastd m7, [o(pw_2048)] +.end2: + pmulhrsw m0, m7 + pmulhrsw m1, m7 +.end3: + pmulhrsw m2, m7 + pmulhrsw m3, m7 +.end4: + vpbroadcastd ym6, strided + kxnorb k1, k1, k1 + pxor m4, m4 + pmulld ym8, ym6 + kmovb k2, k1 + vpgatherdq m6{k1}, [dstq+ym8] + kmovb k1, k2 + vpgatherdq m7{k2}, [r3+ym8] + mova [cq+64*0], m4 + mova [cq+64*1], m4 + kmovb k2, k1 + mova [cq+64*2], m4 + mova [cq+64*3], m4 + punpcklbw m5, m6, m4 + punpckhbw m6, m4 + paddw m0, m5 + paddw m1, m6 + packuswb m0, m1 + vpscatterdq [dstq+ym8]{k1}, m0 + punpcklbw m6, m7, m4 + punpckhbw m7, m4 + paddw m2, m6 + paddw m3, m7 + packuswb m2, m3 + vpscatterdq [r3+ym8]{k2}, m2 + RET +ALIGN function_align +cglobal_label .main_fast2 ; bottom three-quarters are zero + vpbroadcastd ym10, [o(pd_2048)] + vpbroadcastq ym13, [o(int_mshift)] + vpbroadcastd ym3, [o(pw_401_4076x8)] + vpbroadcastd ym5, [o(pw_799_4017x8)] + vpbroadcastd ym4, [o(pw_m1189_3920x8)] + pxor ym6, ym6 + punpckhwd ym2, ym0, ym0 + pmulhrsw ym2, ym3 ; t8a t15a + punpcklwd ym7, ym1, ym1 + pmulhrsw ym7, ym5 ; t4a t7a + punpckhwd ym1, ym1 + pmulhrsw ym4, ym1 ; t11a t12a + vpcmpub k7, ym13, ym10, 6 + punpcklwd ym9, ym6, ym0 + psubsw ym0, ym2, ym4 ; t11a t12a + paddsw ym8, ym2, ym4 ; t8a t15a + mova ym1, ym7 + jmp .main5 +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + vpbroadcastd ym10, [o(pd_2048)] + vpbroadcastq ym13, [o(int_mshift)] + pxor ym6, ym6 + punpckhwd ym8, ym0, ym0 + punpckhwd ym4, ym3, ym3 + punpckhwd ym5, ym2, ym2 + punpcklwd ym7, ym1, ym1 + punpckhwd ym1, ym1 + punpcklwd ym3, ym3 + punpcklwd ym9, ym6, ym0 + punpcklwd ym6, ym2 + vpbroadcastd ym2, [o(pw_401_4076x8)] + vpbroadcastd ym0, [o(pw_m2598_3166x8)] + vpbroadcastd ym11, [o(pw_1931_3612x8)] + vpbroadcastd ym12, [o(pw_m1189_3920x8)] + pmulhrsw ym8, ym2 ; t8a t15a + vpbroadcastd ym2, [o(pw_799_4017x8)] + pmulhrsw ym0, ym4 ; t9a t14a + vpbroadcastd ym4, [o(pw_m2276_3406x8)] + pmulhrsw ym5, ym11 ; t10a t13a + pmulhrsw ym1, ym12 ; t11a t12a + pmulhrsw ym7, ym2 ; t4a t7a + pmulhrsw ym3, ym4 ; t5a t6a + vpcmpub k7, ym13, ym10, 6 + jmp .main4 +ALIGN function_align +cglobal_label .main + WRAP_YMM IDCT16_1D_PACKED + ret + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity + +cglobal iadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call m(iadst_16x8_internal_8bpc).main_pass1 + vbroadcasti32x4 m6, [o(int_shuf1)] + vpbroadcastd m7, [o(pw_16384_m16384)] + punpckhwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpcklwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3 + pshufb m5, m1, m6 ; c0 d0 c1 d1 c2 d2 c3 d3 + pshufb m2, m6 ; e0 f0 e1 f1 e2 f2 e3 f3 +.pass1_end: + REPX {pmulhrsw x, m7}, m3, m5, m4, m2 + punpckldq m0, m3, m5 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m3, m5 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckhdq m5, m2, m4 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpckldq m2, m4 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m5 + punpckhqdq m3, m5 + jmp tx2q +.pass2: + call .main_pass2 + vpbroadcastd m6, [o(pw_2048)] + psrlq m10, 4 + psubw m7, m8, m6 +.pass2_end: + vpbroadcastd m5, [o(pw_2896x8)] + paddsw m1, m2, m4 + psubsw m2, m4 + pmulhrsw m1, m5 ; -out7 out4 out6 -out5 + pmulhrsw m5, m2 ; out8 -out11 -out9 out10 + mova ym8, [o(gather8c)] + lea r3, [dstq+strideq] + psrlq m2, m10, 4 + vpermi2q m2, m0, m3 ; 1 3 13 15 + vpermt2q m0, m10, m3 ; 0 2 12 14 + psrlq m3, m10, 8 + vpermi2q m3, m1, m5 ; 5 7 9 11 + psrlq m10, 12 + vpermt2q m1, m10, m5 ; 4 6 8 10 + pmulhrsw m0, m6 + pmulhrsw m1, m6 + jmp m(idct_8x16_internal_8bpc).end3 +ALIGN function_align +.main_pass1: + vpbroadcastd m2, [o(pw_2896x8)] + pmulhrsw m5, m2, [cq+64*0] + pmulhrsw m3, m2, [cq+64*3] + pmulhrsw m1, m2, [cq+64*1] + pmulhrsw m2, [cq+64*2] + movu m4, [o(permA+3)] + psrlq m10, m4, 4 + mova m6, m4 + vpermi2q m4, m5, m3 ; in0 in12 in2 in14 + vpermt2q m5, m10, m3 ; in15 in3 in13 in1 + vpermi2q m6, m1, m2 ; in4 in8 in6 in10 + vpermt2q m1, m10, m2 ; in11 in7 in9 in5 + jmp .main +ALIGN function_align +.main_pass2: + mova m4, [o(permC)] + psrlq m5, m4, 4 + vpermi2q m4, m0, m2 ; in0 in12 in2 in14 + psrlq m6, m5, 4 + vpermi2q m5, m1, m3 ; in15 in3 in13 in1 + psrlq m10, m6, 4 + vpermi2q m6, m0, m2 ; in4 in8 in6 in10 + vpermt2q m1, m10, m3 ; in11 in7 in9 in5 +.main: + punpcklwd m0, m4, m5 ; in0 in15 in2 in13 + punpckhwd m4, m5 ; in12 in3 in14 in1 + punpcklwd m5, m6, m1 ; in4 in11 in6 in9 + punpckhwd m6, m1 ; in8 in7 in10 in5 +cglobal_label .main2 + vpbroadcastd m9, [o(pd_2048)] + vpbroadcastq m13, [o(int_mshift)] + kxnorb k1, k1, k1 + vpcmpub k7, m13, m9, 6 ; 0x33... + pxor m8, m8 + ITX_MUL4X_PACK 0, 1, 2, 3, 7, 9, 201, 4091, 995, 3973, 5 + ITX_MUL4X_PACK 6, 1, 2, 3, 7, 9, 3035, 2751, 3513, 2106, 5 + ITX_MUL4X_PACK 4, 1, 2, 3, 7, 9, 3857, 1380, 4052, 601, 5 + ITX_MUL4X_PACK 5, 1, 2, 3, 7, 9, 1751, 3703, 2440, 3290, 5 + psubsw m2, m0, m6 ; t9a t8a t11a t10a + paddsw m0, m6 ; t1a t0a t3a t2a + psubsw m3, m5, m4 ; t13a t12a t15a t14a + paddsw m5, m4 ; t5a t4a t7a t6a + ITX_MUL4X_PACK 2, 4, 1, 6, 7, 9, 799, 4017, 3406, 2276, 5 + psubw m7, m8, m7 + ITX_MUL2X_PACK 3, 4, 1, 9, 7, 6, 4 + vpbroadcastd m6, [o(pw_3784_m1567)] + vpbroadcastd m6{k1}, [o(pw_m3784_1567)] + psubsw m1, m0, m5 ; t5 t4 t7 t6 + paddsw m0, m5 ; t1 t0 t3 t2 + psubsw m4, m2, m3 ; t13a t12a t15a t14a + paddsw m2, m3 ; t9a t8a t11a t10a + ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t5a t4a t6a t7a + ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t13 t12 t14 t15 + vbroadcasti32x4 m5, [o(deint_shuf)] + pshufb m0, m5 + pshufb m2, m5 + vshufi32x4 m3, m0, m2, q3232 ; t3 t2 t11a t10a + vinserti32x8 m0, ym2, 1 ; t1 t0 t9a t8a + vshufi32x4 m2, m1, m4, q3232 ; t6a t7a t14 t15 + vinserti32x8 m1, ym4, 1 ; t5a t4a t13 t12 + pshufd m2, m2, q1032 ; t7a t6a t15 t14 + psubsw m4, m0, m3 ; t3a t2a t11 t10 + paddsw m0, m3 ; -out15 out0 out14 -out1 + paddsw m3, m1, m2 ; out12 -out3 -out13 out2 + psubsw m1, m2 ; t7 t6 t15a t14a + punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a + punpcklqdq m4, m1 ; t3a t7 t11 t15a + ret + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity + +cglobal iflipadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call m(iadst_16x8_internal_8bpc).main_pass1 + vbroadcasti32x4 m6, [o(int_shuf2)] + vpbroadcastd m7, [o(pw_m16384_16384)] + punpcklwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpckhwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3 + pshufb m5, m2, m6 ; c0 d0 c1 d1 c2 d2 c3 d3 + pshufb m2, m1, m6 ; e0 f0 e1 f1 e2 f2 e3 f3 + jmp m(iadst_8x16_internal_8bpc).pass1_end +.pass2: + call m(iadst_8x16_internal_8bpc).main_pass2 + vpbroadcastd m7, [o(pw_2048)] + psrlq m10, 36 + psubw m6, m8, m7 + jmp m(iadst_8x16_internal_8bpc).pass2_end + +INV_TXFM_8X16_FN identity, dct +INV_TXFM_8X16_FN identity, adst +INV_TXFM_8X16_FN identity, flipadst +INV_TXFM_8X16_FN identity, identity + +cglobal iidentity_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [o(int16_perm)] + vpermb m3, m0, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3 + vpermb m2, m0, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3 + vpermb m4, m0, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3 + vpermb m0, m0, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3 + vpbroadcastd m5, [o(pw_2896x8)] + punpckldq m1, m3, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m3, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m2, m4, m0 ; e0 f0 g0 h0 a1 f1 g1 h1 + punpckhdq m4, m0 ; e2 f2 g2 h2 e3 f3 g3 h3 + REPX {pmulhrsw x, m5}, m1, m2, m3, m4 + punpcklqdq m0, m1, m2 ; a0 b0 c0 d0 e0 f0 g0 h0 + punpckhqdq m1, m2 ; a1 b1 c1 d1 e1 f1 g1 h1 + punpcklqdq m2, m3, m4 ; a2 b2 c2 d2 e2 f2 g2 h2 + punpckhqdq m3, m4 ; a3 b3 c3 d3 e3 f3 g3 h3 + jmp tx2q +.pass2: + vpbroadcastd m7, [o(pw_1697x16)] + mova ym8, [o(gather8b)] + lea r3, [dstq+strideq*2] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + REPX {paddsw x, x}, m0, m1, m2, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(idct_8x16_internal_8bpc).end + +%macro WRITE_16X2 6 ; coefs[1-2], tmp[1-2], offset[1-2] + pmovzxbw m%3, [dstq+%5] +%ifnum %1 + paddw m%3, m%1 +%else + paddw m%3, %1 +%endif + pmovzxbw m%4, [dstq+%6] +%ifnum %2 + paddw m%4, m%2 +%else + paddw m%4, %2 +%endif + packuswb m%3, m%4 + vpermq m%3, m%3, q3120 + mova [dstq+%5], xm%3 + vextracti32x4 [dstq+%6], m%3, 1 +%endmacro + +%macro INV_TXFM_16X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x4 +%ifidn %1_%2, dct_dct + movsx r6d, word [cq] + mov [cq], eobd + jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly2 +%endif +%endmacro + +INIT_ZMM avx512icl +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst +INV_TXFM_16X4_FN dct, identity + +cglobal idct_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova xm0, [cq+16*0] + mova xm1, [cq+16*1] + mova xm2, [cq+16*2] + mova xm3, [cq+16*3] + mova xm4, [cq+16*4] + mova xm5, [cq+16*5] + mova xm6, [cq+16*6] + mova xm7, [cq+16*7] + call m(idct_4x16_internal_8bpc).main + vpbroadcastd m8, [o(pw_16384)] + vinserti32x4 ym1, xm3, 1 ; 3 2 7 6 + vinserti32x4 ym5, xm7, 1 ; b a f e + vinserti32x4 ym0, xm2, 1 ; 0 1 4 5 + vinserti32x4 ym4, xm6, 1 ; 8 9 c d + vinserti32x8 m1, ym5, 1 ; 3 2 7 6 b a f e + vinserti32x8 m0, ym4, 1 ; 0 1 4 5 8 9 c d + pmulhrsw m1, m8 + pmulhrsw m0, m8 + pshufd m1, m1, q1032 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + jmp tx2q +.pass2: + IDCT4_1D_PACKED + mova m2, [o(permA)] + jmp m(iadst_16x4_internal_8bpc).end + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+64*0] + mova m1, [cq+64*1] + movshdup m3, [o(permB)] + psrlq m10, m3, 4 + call m(iadst_4x16_internal_8bpc).main2 + vpbroadcastd m6, [o(pw_16384_m16384)] + psrlq m0, m10, 4 + psrlq m10, 8 +.pass1_end: + punpcklwd ym5, ym4, ym2 + punpckhwd ym4, ym2 + vinserti32x8 m5, ym4, 1 + mova m1, m9 + vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16} + mova m4, m9 + vpdpwssd m4, m5, [o(pw_2896_2896)] {1to16} + psrad m1, 12 + psrad m4, 12 + packssdw m1, m4 ; out8 -out7 -out9 out6 -out11 out4 out10 -out5 + vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d + vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + pmulhrsw m0, m6 + pmulhrsw m1, m6 + jmp tx2q +.pass2: + call .main + movu m2, [o(permA+1)] +.end: + vpbroadcastd m3, [o(pw_2048)] + pmulhrsw m0, m3 + pmulhrsw m1, m3 +.end2: + psrlq m3, m2, 4 + vpermi2q m2, m0, m1 + vpermi2q m3, m0, m1 +.end3: + lea r3, [dstq+strideq*2] + mova xm1, [dstq+strideq*0] + vinserti32x4 ym1, [dstq+strideq*1], 1 + vinserti32x4 m1, [r3 +strideq*0], 2 + vinserti32x4 m1, [r3 +strideq*1], 3 + pxor m4, m4 + mova [cq+64*0], m4 + mova [cq+64*1], m4 + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [r3 +strideq*0], m0, 2 + vextracti32x4 [r3 +strideq*1], m0, 3 + RET +ALIGN function_align +.main: + IADST4_1D_PACKED + ret + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m0, [cq+64*0] + mova m1, [cq+64*1] + movshdup m3, [o(permB)] + psrlq m10, m3, 4 + call m(iadst_4x16_internal_8bpc).main2 + vpbroadcastd m6, [o(pw_m16384_16384)] + psrlq m0, m10, 12 + psrlq m10, 16 + jmp m(iadst_16x4_internal_8bpc).pass1_end +.pass2: + call m(iadst_16x4_internal_8bpc).main + movu m2, [o(permA+2)] + jmp m(iadst_16x4_internal_8bpc).end + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m1, [cq+64*0] + mova m2, [cq+64*1] + vpbroadcastd m3, [o(pw_1697x16)] + vpbroadcastd m4, [o(pw_16384)] + mova m5, [o(idtx_16x4p)] + shufps m0, m1, m2, q2020 + shufps m1, m2, q3131 + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddsw m0, m2 + paddsw m1, m3 + vpermb m0, m5, m0 + vpermb m1, m5, m1 + jmp tx2q +.pass2: + vpbroadcastd m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + movu m2, [o(permA+1)] + jmp m(iadst_16x4_internal_8bpc).end + +%macro INV_TXFM_16X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x8 +%ifidn %1_%2, dct_dct + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 8 +.dconly: + imul r6d, 181 + add r6d, 128 + sar r6d, 8 +.dconly2: + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 +.dconly3: + imul r6d, 181 + lea r2, [strideq*3] + add r6d, 128+2048 + sar r6d, 8+4 + pxor m2, m2 + vpbroadcastw m3, r6d +.dconly_loop: + mova xm1, [dstq+strideq*0] + vinserti32x4 ym1, [dstq+strideq*1], 1 + vinserti32x4 m1, [dstq+strideq*2], 2 + vinserti32x4 m1, [dstq+r2 ], 3 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + paddw m0, m3 + paddw m1, m3 + packuswb m0, m1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+r2 ], m0, 3 + lea dstq, [dstq+strideq*4] + sub r3d, 4 + jg .dconly_loop + RET +%endif +%endmacro + +%macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd + vpbroadcastd m8, [o(pw_2896x8)] + vpermq m0, [cq+32*0], q3120 + add cq, 32*4 + vpermq m7, [cq+32*3], q%1 + vpermq m1, [cq-32*3], q%1 + vpermq m6, [cq+32*2], q3120 + vpermq m2, [cq-32*2], q3120 + vpermq m5, [cq+32*1], q%1 + vpermq m3, [cq-32*1], q%1 + vpermq m4, [cq+32*0], q3120 + REPX {pmulhrsw x, m8}, m0, m7, m1, m6, m2, m5, m3, m4 +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, identity +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst + +cglobal idct_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd m1, [o(pw_2896x8)] + vpermq m0, [cq+64*0], q3120 + vpermq m2, [cq+64*1], q3120 + vpermq m4, [cq+64*2], q3120 + vpermq m6, [cq+64*3], q3120 + REPX {pmulhrsw x, m1}, m0, m2, m4, m6 + vextracti32x8 ym1, m0, 1 + vextracti32x8 ym3, m2, 1 + vextracti32x8 ym5, m4, 1 + vextracti32x8 ym7, m6, 1 + call m(idct_8x16_internal_8bpc).main + vbroadcasti32x4 m8, [o(int_shuf1)] + vbroadcasti32x4 m9, [o(int_shuf2)] + vinserti32x8 m0, ym2, 1 ; a0 a1 a2 a3 b0 b1 b2 b3 + vinserti32x8 m1, ym3, 1 ; d0 d1 d2 d3 c0 c1 c2 c3 + vinserti32x8 m4, ym6, 1 ; i0 i1 i2 i3 j0 j1 j2 j3 + vinserti32x8 m5, ym7, 1 ; l0 l1 l2 l3 k0 k1 k2 k3 + vpbroadcastd m2, [o(pw_16384)] + pshufb m0, m8 ; a0 b0 a1 b1 a2 b2 a3 b3 + pshufb m1, m9 ; c0 d0 c1 d1 c2 d2 c3 d3 + pshufb m6, m4, m8 ; i0 j0 i1 j1 i2 j2 i3 j3 + pshufb m7, m5, m9 ; m0 n0 m1 n1 m2 n2 m3 n3 + REPX {pmulhrsw x, m2}, m0, m1, m6, m7 + punpckldq m2, m0, m1 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m3, m0, m1 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m4, m6, m7 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhdq m5, m6, m7 ; i2 j2 k2 l2 i3 j3 k3 l3 + jmp tx2q +.pass2: + vshufi32x4 m0, m2, m4, q2020 ; 0 1 + vshufi32x4 m2, m4, q3131 ; 4 5 + vshufi32x4 m1, m3, m5, q2020 ; 2 3 + vshufi32x4 m3, m5, q3131 ; 6 7 + call .main + movshdup m4, [o(permC)] + psrlq m6, m4, 4 + vpermq m5, m4, q1032 + vpermi2q m4, m0, m2 ; a2 a3 b2 b3 e2 e3 f2 f3 + vpermt2q m0, m6, m2 ; a0 a1 b0 b1 e0 e1 f0 f1 + psrlq m6, m5, 4 + vpermi2q m5, m1, m3 ; c2 c3 d2 d3 g2 g3 h2 h3 + vpermt2q m1, m6, m3 ; c0 c1 d0 d1 g0 g1 h0 h1 + vpbroadcastd m6, [o(pw_2048)] +.end: + REPX {pmulhrsw x, m6}, m0, m4, m1, m5 +.end2: + lea r3, [dstq+strideq*4] + lea r4, [strideq*3] + mova xm3, [dstq+strideq*0] + mova xm6, [dstq+strideq*2] + vinserti32x4 ym3, [dstq+strideq*1], 1 + vinserti32x4 ym6, [dstq+r4 ], 1 + vinserti32x4 m3, [r3 +strideq*0], 2 + vinserti32x4 m6, [r3 +strideq*2], 2 + vinserti32x4 m3, [r3 +strideq*1], 3 + vinserti32x4 m6, [r3 +r4 ], 3 + pxor m7, m7 + mova [cq+64*0], m7 + mova [cq+64*1], m7 + mova [cq+64*2], m7 + mova [cq+64*3], m7 + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + paddw m0, m2 + paddw m4, m3 + packuswb m0, m4 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [r3 +strideq*0], m0, 2 + vextracti32x4 [r3 +strideq*1], m0, 3 + punpcklbw m3, m6, m7 + punpckhbw m6, m7 + paddw m1, m3 + paddw m5, m6 + packuswb m1, m5 + mova [dstq+strideq*2], xm1 + vextracti32x4 [dstq+r4 ], ym1, 1 + vextracti32x4 [r3 +strideq*2], m1, 2 + vextracti32x4 [r3 +r4 ], m1, 3 + RET +ALIGN function_align +cglobal_label .main + IDCT8_1D_PACKED + ret + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity + +cglobal iadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call m(iadst_8x16_internal_8bpc).main_pass1 + vpbroadcastd m7, [o(pw_16384_m16384)] + psrlq m10, 4 +.pass1_end: + punpcklwd m5, m4, m2 + punpckhwd m4, m2 + mova m1, m9 + vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16} + mova m6, m9 + vpdpwssd m6, m5, [o(pw_2896_2896)] {1to16} + mova m2, m9 + vpdpwssd m2, m4, [o(pw_m2896_2896)] {1to16} + vpdpwssd m9, m4, [o(pw_2896_2896)] {1to16} + psrad m1, 12 + psrad m6, 12 + packssdw m1, m6 ; out8 -out7 -out9 out6 + psrad m2, 12 + psrad m9, 12 + packssdw m2, m9 ; -out11 out4 out10 -out5 + psrlq m4, m10, 4 + vpermi2q m4, m0, m2 + vpermt2q m0, m10, m2 + psrlq m5, m10, 8 + vpermi2q m5, m1, m3 + psrlq m10, 12 + vpermt2q m1, m10, m3 + punpcklwd m3, m4, m5 ; a0 c0 a1 c1 a2 c2 a3 c3 + punpckhwd m4, m5 ; b0 d0 b1 d1 b2 d2 b3 d3 + punpcklwd m5, m1, m0 ; i0 k0 i1 k1 2i k2 i3 k3 + punpckhwd m1, m0 ; j0 l0 j1 l1 j2 l2 j3 l3 + punpcklwd m2, m3, m4 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhwd m3, m4 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpcklwd m4, m5, m1 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhwd m5, m1 ; i2 j2 k2 l2 i3 j3 k3 l3 + REPX {pmulhrsw x, m7}, m2, m3, m4, m5 + jmp tx2q +.pass2: + vshufi32x4 m0, m2, m4, q2020 + vshufi32x4 m2, m4, q3131 ; 4 5 + vshufi32x4 m1, m3, m5, q2020 + vshufi32x4 m3, m5, q3131 ; 6 7 + pshufd m4, m0, q1032 ; 1 0 + pshufd m5, m1, q1032 ; 3 2 + call .main_pass2 + movshdup m4, [o(permC)] + pmulhrsw m0, m6 + pmulhrsw m1, m6 + psrlq m6, m4, 4 + mova m5, m4 + vpermi2q m4, m0, m2 + vpermt2q m0, m6, m2 + vpermi2q m5, m1, m3 + vpermt2q m1, m6, m3 + jmp m(idct_16x8_internal_8bpc).end2 +ALIGN function_align +.main_pass1: + vpbroadcastd m4, [o(pw_2896x8)] + pmulhrsw m3, m4, [cq+64*0] + pmulhrsw m1, m4, [cq+64*3] + pmulhrsw m2, m4, [cq+64*1] + pmulhrsw m4, [cq+64*2] + mova m5, [o(int16_perm)] + kxnorb k1, k1, k1 + vpblendmd m0{k1}, m1, m3 ; 0 7 + vmovdqa32 m3{k1}, m1 ; 6 1 + vpblendmd m1{k1}, m4, m2 ; 2 5 + vmovdqa32 m2{k1}, m4 ; 4 3 + REPX {vpermb x, m5, x}, m0, m1, m2, m3 + IADST8_1D_PACKED 1 + ret +ALIGN function_align +cglobal_label .main_pass2 + IADST8_1D_PACKED 2 + pxor m5, m5 + psubd m5, m6 + packssdw m6, m5 + pmulhrsw m2, m6 + pmulhrsw m3, m6 + ret + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity + +cglobal iflipadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call m(iadst_8x16_internal_8bpc).main_pass1 + vpbroadcastd m7, [o(pw_m16384_16384)] + psrlq m10, 20 + jmp m(iadst_16x8_internal_8bpc).pass1_end +.pass2: + vshufi32x4 m0, m2, m4, q2020 + vshufi32x4 m2, m4, q3131 ; 4 5 + vshufi32x4 m1, m3, m5, q2020 + vshufi32x4 m3, m5, q3131 ; 6 7 + pshufd m4, m0, q1032 ; 1 0 + pshufd m5, m1, q1032 ; 3 2 + call m(iadst_16x8_internal_8bpc).main_pass2 + movshdup m4, [o(permC)] + pmulhrsw m5, m6, m0 + pmulhrsw m0, m6, m1 + psrlq m1, m4, 12 + psrlq m4, 8 + mova m7, m4 + vpermi2q m4, m0, m3 + vpermt2q m0, m1, m3 + vpermi2q m1, m5, m2 + vpermt2q m5, m7, m2 + jmp m(idct_16x8_internal_8bpc).end2 + +INV_TXFM_16X8_FN identity, dct +INV_TXFM_16X8_FN identity, adst +INV_TXFM_16X8_FN identity, flipadst +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + vpbroadcastd m0, [o(pw_2896x8)] + pmulhrsw m3, m0, [cq+64*0] + pmulhrsw m4, m0, [cq+64*1] + pmulhrsw m5, m0, [cq+64*2] + pmulhrsw m0, [cq+64*3] + vpbroadcastd m7, [o(pw_1697x16)] + vpbroadcastd m8, [o(pw_16384)] + shufps m2, m3, m4, q2020 ; a0 a1 a4 a5 e0 e1 e4 e5 + shufps m3, m4, q3131 ; a2 a3 a6 a7 e2 e3 e6 e7 + shufps m4, m5, m0, q2020 ; i0 i1 i4 i5 m0 m1 m4 m5 + shufps m5, m0, q3131 ; i2 i3 i6 i7 m2 m3 m6 m7 + mova m9, [o(int8_permA)] + pmulhrsw m0, m7, m2 + pmulhrsw m1, m7, m3 + pmulhrsw m6, m7, m4 + pmulhrsw m7, m5 + REPX {pmulhrsw x, m8}, m0, m1, m6, m7 + paddsw m2, m0 + paddsw m3, m1 + paddsw m4, m6 + paddsw m5, m7 + REPX {vpermb x, m9, x}, m2, m3, m4, m5 + jmp tx2q +.pass2: + mova m7, [o(permB)] + vpbroadcastd m6, [o(pw_4096)] + vpermq m0, m7, m2 + vpermq m4, m7, m4 + vpermq m1, m7, m3 + vpermq m5, m7, m5 + jmp m(idct_16x8_internal_8bpc).end + +%macro INV_TXFM_16X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x16 +%ifidn %1_%2, dct_dct + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 16 + imul r6d, 181 + add r6d, 128+512 + sar r6d, 8+2 + jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, identity +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst + +cglobal idct_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m7, [o(permB)] + vpermq m0, m7, [cq+64*0] + vpermq m1, m7, [cq+64*1] + vpermq m2, m7, [cq+64*2] + vpermq m3, m7, [cq+64*3] + vpermq m4, m7, [cq+64*4] + vpermq m5, m7, [cq+64*5] + vpermq m6, m7, [cq+64*6] + vpermq m7, m7, [cq+64*7] + call .main + vbroadcasti32x4 m12, [o(int_shuf1)] + vbroadcasti32x4 m11, [o(int_shuf2)] + vpbroadcastd m13, [o(pw_8192)] + pshufb m0, m12 + pshufb m8, m1, m11 + pshufb m2, m12 + pshufb m9, m3, m11 + pshufb m4, m12 + pshufb m10, m5, m11 + pshufb m6, m12 + pshufb m11, m7, m11 + REPX {pmulhrsw x, m13}, m0, m8, m2, m9, m4, m10, m6, m11 + punpckhdq m1, m0, m8 + punpckldq m0, m8 + punpckhdq m3, m2, m9 + punpckldq m2, m9 + punpckhdq m5, m4, m10 + punpckldq m4, m10 + punpckhdq m7, m6, m11 + punpckldq m6, m11 + jmp tx2q +.pass2: + vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc + vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4 + vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec + vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4 + vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me + vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6 + vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee + vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6 + vshufi32x4 m2, m0, m4, q3131 ; 4 5 + vshufi32x4 m0, m4, q2020 ; 0 1 + vshufi32x4 m4, m6, m8, q2020 ; 8 9 + vshufi32x4 m6, m8, q3131 ; 12 13 + vshufi32x4 m3, m1, m5, q3131 ; 6 7 + vshufi32x4 m1, m5, q2020 ; 2 3 + vshufi32x4 m5, m7, m9, q2020 ; 10 11 + vshufi32x4 m7, m9, q3131 ; 14 15 + call .main + mova m8, [o(permD)] + psrlq m12, m8, 4 + psrlq m9, m8, 8 + psrlq m13, m8, 12 + mova m10, m8 + vpermi2q m8, m0, m2 ; 0 1 4 5 + vpermt2q m0, m12, m2 + mova m11, m9 + vpermi2q m9, m1, m3 ; 2 3 6 7 + vpermt2q m1, m13, m3 + vpermi2q m10, m4, m6 ; 8 9 12 13 + vpermt2q m4, m12, m6 + vpermi2q m11, m5, m7 ; 10 11 14 15 + vpermt2q m5, m13, m7 +.end: + vpbroadcastd m12, [o(pw_2048)] +.end2: + REPX {pmulhrsw x, m12}, m0, m1, m4, m5 +.end3: + REPX {pmulhrsw x, m12}, m8, m9, m10, m11 + lea r3, [strideq*3] + lea r4, [dstq+strideq*4] + lea r5, [dstq+strideq*8] + lea r6, [r4 +strideq*8] + mova xm3, [dstq+strideq*0] + mova xm6, [dstq+strideq*2] + vinserti32x4 ym3, [dstq+strideq*1], 1 + vinserti32x4 ym6, [dstq+r3 ], 1 + vinserti32x4 m3, [r4+strideq*0], 2 + vinserti32x4 m6, [r4+strideq*2], 2 + vinserti32x4 m3, [r4+strideq*1], 3 + vinserti32x4 m6, [r4+r3 ], 3 + mova xm12, [r5+strideq*0] + mova xm13, [r5+strideq*2] + vinserti32x4 ym12, [r5+strideq*1], 1 + vinserti32x4 ym13, [r5+r3 ], 1 + vinserti32x4 m12, [r6+strideq*0], 2 + vinserti32x4 m13, [r6+strideq*2], 2 + vinserti32x4 m12, [r6+strideq*1], 3 + vinserti32x4 m13, [r6+r3 ], 3 + pxor m7, m7 + REPX {mova [cq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + punpcklbw m2, m3, m7 + punpckhbw m3, m7 + paddw m0, m2 + paddw m8, m3 + packuswb m0, m8 + punpcklbw m2, m6, m7 + punpckhbw m6, m7 + paddw m1, m2 + paddw m9, m6 + packuswb m1, m9 + punpcklbw m2, m12, m7 + punpckhbw m12, m7 + paddw m2, m4 + paddw m10, m12 + packuswb m2, m10 + punpcklbw m3, m13, m7 + punpckhbw m13, m7 + paddw m3, m5 + paddw m11, m13 + packuswb m3, m11 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + mova [dstq+strideq*2], xm1 + vextracti32x4 [dstq+r3 ], ym1, 1 + vextracti32x4 [r4+strideq*0], m0, 2 + vextracti32x4 [r4+strideq*1], m0, 3 + vextracti32x4 [r4+strideq*2], m1, 2 + vextracti32x4 [r4+r3 ], m1, 3 + mova [r5+strideq*0], xm2 + vextracti32x4 [r5+strideq*1], ym2, 1 + mova [r5+strideq*2], xm3 + vextracti32x4 [r5+r3 ], ym3, 1 + vextracti32x4 [r6+strideq*0], m2, 2 + vextracti32x4 [r6+strideq*1], m2, 3 + vextracti32x4 [r6+strideq*2], m3, 2 + vextracti32x4 [r6+r3 ], m3, 3 + RET +ALIGN function_align +cglobal_label .main_fast2 ; bottom three-quarters are zero + vpbroadcastd m10, [o(pd_2048)] + vpbroadcastq m13, [o(int_mshift)] + vpcmpub k7, m13, m10, 6 +.main_fast4: + vpbroadcastd m2, [o(pw_401_4076x8)] + vpbroadcastd m4, [o(pw_m1189_3920x8)] + vpbroadcastd m3, [o(pw_799_4017x8)] + pmulhrsw m2, m8 ; t8a t15a + pmulhrsw m4, m1 ; t11a t12a + pmulhrsw m7, m3 ; t4a t7a + pxor m6, m6 + psubsw m0, m2, m4 ; t11a t12a + paddsw m8, m2, m4 ; t8a t15a + mova m1, m7 + jmp .main5 +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + vpbroadcastd m10, [o(pd_2048)] +.main_fast3: + vpbroadcastq m13, [o(int_mshift)] + vpcmpub k7, m13, m10, 6 +.main_fast5: + vpbroadcastd m2, [o(pw_401_4076x8)] + vpbroadcastd m4, [o(pw_m2598_3166x8)] + vpbroadcastd m11, [o(pw_1931_3612x8)] + vpbroadcastd m12, [o(pw_m1189_3920x8)] + pmulhrsw m8, m2 ; t8a t15a + vpbroadcastd m2, [o(pw_799_4017x8)] + pmulhrsw m0, m4 ; t9a t14a + vpbroadcastd m4, [o(pw_m2276_3406x8)] + pmulhrsw m5, m11 ; t10a t13a + pmulhrsw m1, m12 ; t11a t12a + pmulhrsw m7, m2 ; t4a t7a + pmulhrsw m3, m4 ; t5a t6a + jmp .main4 +ALIGN function_align +cglobal_label .main + IDCT16_1D_PACKED + ret + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call .main_pass1 + vpbroadcastd m10, [o(pw_8192_m8192)] + punpcklwd m8, m0, m1 ; b0 d0 b1 d1 b2 d2 b3 d3 + punpckhwd m0, m1 ; a0 c0 a1 c1 a2 c2 a3 c3 + punpckhwd m1, m0, m8 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpcklwd m0, m8 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpcklwd m8, m2, m3 ; f0 h0 f1 h1 f2 h2 f3 h3 + punpckhwd m2, m3 ; e0 g0 e1 g1 e2 g2 e3 g3 + punpckhwd m3, m2, m8 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpcklwd m2, m8 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhwd m8, m4, m5 ; i0 k0 i1 k1 i2 k2 i3 k3 + punpcklwd m4, m5 ; j0 l0 j1 l1 j2 l2 j3 l3 + punpckhwd m5, m4, m8 ; i2 j2 k2 l2 i3 j3 k3 l3 + punpcklwd m4, m8 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhwd m8, m6, m7 ; m0 o0 m1 o1 m2 o2 m3 o3 + punpcklwd m6, m7 ; n0 p0 n1 p1 n2 p2 n3 p3 + punpckhwd m7, m6, m8 ; m2 n2 o2 p2 m3 n3 o3 p3 + punpcklwd m6, m8 ; m0 n0 o0 p0 m1 n1 o1 p1 +.pass1_end: + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + jmp tx2q +.pass2: + call .main_pass2 + mova m10, [o(permD)] + psrlq m8, m10, 8 + psrlq m12, m10, 12 + psrlq m13, m10, 4 + mova m9, m8 + vpermi2q m8, m0, m2 ; 0 1 4 5 + vpermt2q m0, m12, m2 + vpermi2q m9, m1, m3 ; 2 3 6 7 + vpermt2q m1, m12, m3 + vpbroadcastd m12, [o(pw_2048)] + mov r3d, 0xff00ff00 + mova m11, m10 + vpermi2q m10, m4, m6 ; 8 9 12 13 + vpermt2q m4, m13, m6 + kmovd k1, r3d + vpermi2q m11, m5, m7 ; 10 11 14 15 + vpermt2q m5, m13, m7 + pxor m7, m7 + vpsubw m12{k1}, m7, m12 + jmp m(idct_16x16_internal_8bpc).end2 +ALIGN function_align +.main_pass1: + mova m4, [o(permB)] + psrlq m3, m4, 4 + vpermq m0, m4, [cq+64*0] + vpermq m7, m3, [cq+64*7] + vpermq m6, m4, [cq+64*6] + vpermq m1, m3, [cq+64*1] + vpermq m2, m4, [cq+64*2] + vpermq m5, m3, [cq+64*5] + vpermq m4, m4, [cq+64*4] + vpermq m3, m3, [cq+64*3] + call .main + vpbroadcastd m13, [o(pw_2896_2896)] + vpbroadcastd m12, [o(pw_m2896_2896)] + mova m2, m10 + vpdpwssd m2, m5, m13 ; -out5 + mova m8, m10 + vpdpwssd m8, m11, m13 ; out4 + mova m9, m10 + vpdpwssd m9, m5, m12 ; out10 + mova m5, m10 + vpdpwssd m5, m11, m12 ; -out11 + mova m11, m10 + vpdpwssd m11, m3, m13 ; -out7 + mova m14, m10 + vpdpwssd m14, m4, m13 ; out6 + mova m13, m10 + vpdpwssd m13, m3, m12 ; out8 + vpdpwssd m10, m4, [o(pw_2896_m2896)] {1to16} ; -out9 + REPX {psrad x, 12}, m2, m8, m9, m5, m11, m14, m13, m10 + packssdw m2, m8 ; -out5 out4 + packssdw m5, m9, m5 ; out10 -out11 + packssdw m3, m11, m14 ; -out7 out6 + packssdw m4, m13, m10 ; out8 -out9 + ret +ALIGN function_align +.main_pass2: + vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc + vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4 + vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec + vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4 + vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me + vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6 + vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee + vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6 + vshufi32x4 m2, m0, m4, q3131 ; 4 5 + vshufi32x4 m0, m4, q2020 ; 0 1 + vshufi32x4 m4, m6, m8, q2020 ; 8 9 + vshufi32x4 m6, m8, q3131 ; 12 13 + vshufi32x4 m3, m1, m5, q3131 ; 6 7 + vshufi32x4 m1, m5, q2020 ; 2 3 + vshufi32x4 m5, m7, m9, q2020 ; 10 11 + vshufi32x4 m7, m9, q3131 ; 14 15 +cglobal_label .main_pass2b + REPX {pshufd x, x, q1032}, m1, m3, m5, m7 + call .main + vpbroadcastd m8, [o(pw_2896x8)] + pshufb m2, m11, m12 + pshufb m5, m12 + pshufb m3, m12 + pshufb m4, m12 + punpcklqdq m9, m5, m2 ; t15a t7 + punpckhqdq m5, m2 ; t14a t6 + shufps m2, m3, m4, q1032 ; t2a t10 + shufps m3, m4, q3210 ; t3a t11 + psubsw m4, m2, m3 ; out8 -out9 + paddsw m3, m2 ; -out7 out6 + paddsw m2, m5, m9 ; -out5 out4 + psubsw m5, m9 ; out10 -out11 + REPX {pmulhrsw x, m8}, m2, m3, m4, m5 + ret +ALIGN function_align +.main: + vpbroadcastd m10, [o(pd_2048)] + vpbroadcastq m13, [o(int_mshift)] + punpckhwd m8, m7, m0 ; in14 in1 + punpcklwd m0, m7 ; in0 in15 + punpcklwd m7, m6, m1 ; in12 in3 + punpckhwd m1, m6 ; in2 in13 + punpckhwd m6, m5, m2 ; in10 in5 + punpcklwd m2, m5 ; in4 in11 + punpcklwd m5, m4, m3 ; in8 in7 + punpckhwd m3, m4 ; in6 in9 + vpcmpub k7, m13, m10, 6 ; 0x33... + ITX_MUL2X_PACK 0, 4, 9, 10, 201, 4091, 5 ; t0 t1 + ITX_MUL2X_PACK 1, 4, 9, 10, 995, 3973, 5 ; t2 t3 + ITX_MUL2X_PACK 2, 4, 9, 10, 1751, 3703, 5 ; t4 t5 + ITX_MUL2X_PACK 3, 4, 9, 10, 2440, 3290, 5 ; t6 t7 + ITX_MUL2X_PACK 5, 4, 9, 10, 3035, 2751, 5 ; t8 t9 + ITX_MUL2X_PACK 6, 4, 9, 10, 3513, 2106, 5 ; t10 t11 + ITX_MUL2X_PACK 7, 4, 9, 10, 3857, 1380, 5 ; t12 t13 + ITX_MUL2X_PACK 8, 4, 9, 10, 4052, 601, 5 ; t14 t15 + psubsw m4, m0, m5 ; t9a t8a + paddsw m0, m5 ; t1a t0a + psubsw m5, m1, m6 ; t11a t10a + paddsw m1, m6 ; t3a t2a + psubsw m6, m2, m7 ; t13a t12a + paddsw m2, m7 ; t5a t4a + psubsw m7, m3, m8 ; t15a t14a + paddsw m3, m8 ; t7a t6a + ITX_MUL2X_PACK 4, 8, 9, 10, 799, 4017, 4 ; t8 t9 + ITX_MUL2X_PACK 6, 8, 9, 10, 799_4017, 4017_m799, 52 ; t12 t13 + ITX_MUL2X_PACK 5, 8, 9, 10, 3406, 2276, 4 ; t10 t11 + ITX_MUL2X_PACK 7, 8, 9, 10, 3406_2276, 2276_m3406, 52 ; t14 t15 + psubsw m8, m1, m3 ; t7 t6 + paddsw m1, m3 ; t3 t2 + psubsw m3, m0, m2 ; t5 t4 + paddsw m0, m2 ; t1 t0 + psubsw m2, m5, m7 ; t14a t15a + paddsw m7, m5 ; t10a t11a + psubsw m5, m4, m6 ; t12a t13a + paddsw m4, m6 ; t8a t9a + ITX_MUL2X_PACK 3, 6, 9, 10, 1567, 3784, 5 ; t5a t4a + ITX_MUL2X_PACK 8, 6, 9, 10, 3784_m1567, 1567_3784, 52 ; t7a t6a + ITX_MUL2X_PACK 2, 6, 9, 10, 3784, 1567, 4 ; t15 t14 + ITX_MUL2X_PACK 5, 6, 9, 10, 3784_1567, 1567_m3784, 52 ; t13 t12 + vbroadcasti32x4 m12, [o(deint_shuf)] + paddsw m6, m4, m7 ; -out1 out14 + psubsw m4, m7 ; t10 t11 + psubsw m11, m3, m8 ; t7 t6 + paddsw m8, m3 ; out12 -out3 + psubsw m3, m0, m1 ; t3a t2a + paddsw m0, m1 ; -out15 out0 + paddsw m1, m2, m5 ; -out13 out2 + psubsw m5, m2 ; t15a t14a + pshufb m0, m12 + pshufb m6, m12 + pshufb m8, m12 + pshufb m1, m12 + shufps m7, m6, m0, q1032 ; out14 -out15 + shufps m0, m6, m0, q3210 ; -out1 out0 + punpcklqdq m6, m8, m1 ; out12 -out13 + punpckhqdq m1, m8, m1 ; -out3 out2 + ret + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + call m(iadst_16x16_internal_8bpc).main_pass1 + vpbroadcastd m10, [o(pw_m8192_8192)] + punpcklwd m8, m1, m0 ; m0 o0 m1 o1 m2 o2 m3 o3 + punpckhwd m9, m1, m0 ; n0 p0 n1 p1 n2 p2 n3 p3 + punpckhwd m1, m7, m6 ; a0 c0 a1 c1 a2 c2 a3 c3 + punpcklwd m7, m6 ; b0 d0 b1 d1 b2 d2 b3 d3 + punpcklwd m0, m1, m7 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhwd m1, m7 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpcklwd m6, m8, m9 ; m0 n0 o0 p0 m1 n1 o1 p1 + punpckhwd m7, m8, m9 ; m2 n2 o2 p2 m3 n3 o3 p3 + punpcklwd m8, m3, m2 ; i0 k0 i1 k1 i2 k2 i3 k3 + punpckhwd m9, m3, m2 ; j0 l0 j1 l1 j2 l2 j3 l3 + punpckhwd m3, m5, m4 ; e0 g0 e1 g1 e2 g2 e3 g3 + punpcklwd m5, m4 ; f0 h0 f1 h1 f2 h2 f3 h3 + punpcklwd m2, m3, m5 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhwd m3, m5 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpcklwd m4, m8, m9 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhwd m5, m8, m9 ; i2 j2 k2 l2 i3 j3 k3 l3 + jmp m(iadst_16x16_internal_8bpc).pass1_end +.pass2: + call m(iadst_16x16_internal_8bpc).main_pass2 + mova m10, [o(permD)] + psrlq m8, m10, 8 + psrlq m12, m10, 12 + psrlq m13, m10, 4 + mova m9, m8 + vpermi2q m8, m7, m5 ; 0 1 4 5 + vpermt2q m7, m12, m5 + vpermi2q m9, m6, m4 ; 2 3 6 7 + vpermt2q m6, m12, m4 + vpbroadcastd m12, [o(pw_2048)] + mov r3d, 0x00ff00ff + mova m11, m10 + vpermi2q m10, m3, m1 ; 8 9 12 13 + vpermt2q m3, m13, m1 + kmovd k1, r3d + vpermi2q m11, m2, m0 ; 10 11 14 15 + vpermt2q m2, m13, m0 + pxor m0, m0 + vpsubw m12{k1}, m0, m12 + pmulhrsw m0, m7, m12 + pmulhrsw m1, m6, m12 + pmulhrsw m4, m3, m12 + pmulhrsw m5, m2, m12 + jmp m(idct_16x16_internal_8bpc).end3 + +INV_TXFM_16X16_FN identity, dct +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2 + mova m8, [o(int16_perm)] + vpermb m1, m8, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3 + vpermb m2, m8, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3 + vpbroadcastd m0, [o(pw_1697x16)] + vpermb m3, m8, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3 + vpermb m4, m8, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3 + vpermb m5, m8, [cq+64*4] ; i0 j0 i1 j1 i2 j2 i3 j3 + vpermb m6, m8, [cq+64*5] ; k0 l0 k1 l1 k2 l2 k3 l3 + vpermb m7, m8, [cq+64*6] ; m0 n0 m1 n1 m2 n2 m3 n3 + vpermb m8, m8, [cq+64*7] ; o0 p0 o1 p1 o2 p2 o3 p3 + pmulhrsw m9, m0, m1 + pmulhrsw m10, m0, m2 + pmulhrsw m11, m0, m3 + pmulhrsw m12, m0, m4 + pmulhrsw m13, m0, m5 + pmulhrsw m14, m0, m6 + pmulhrsw m15, m0, m7 + pmulhrsw m0, m8 + REPX {psraw x, 1}, m9, m10, m11, m12 + pavgw m1, m9 + pavgw m2, m10 + pavgw m3, m11 + pavgw m4, m12 + REPX {psraw x, 1}, m13, m14, m15, m0 + pavgw m5, m13 + pavgw m6, m14 + pavgw m7, m15 + pavgw m8, m0 + punpckldq m0, m1, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m1, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m2, m3, m4 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhdq m3, m4 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpckldq m4, m5, m6 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhdq m5, m6 ; i2 j2 k2 l2 i3 j3 k3 l3 + punpckldq m6, m7, m8 ; m0 n0 o0 p0 m1 n1 o1 p1 + punpckhdq m7, m8 ; m2 n2 o2 p2 m3 n3 o3 p3 + jmp tx2q +ALIGN function_align +.pass2: + vpbroadcastd m11, [o(pw_1697x16)] + pmulhrsw m12, m11, m0 + pmulhrsw m13, m11, m1 + pmulhrsw m14, m11, m2 + pmulhrsw m15, m11, m3 + pmulhrsw m8, m11, m4 + pmulhrsw m9, m11, m5 + pmulhrsw m10, m11, m6 + pmulhrsw m11, m7 + REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 + paddsw m0, m12 + paddsw m1, m13 + paddsw m2, m14 + paddsw m3, m15 + paddsw m8, m4 + movu m4, [o(permD+2)] + paddsw m9, m5 + paddsw m6, m10 + paddsw m7, m11 + psrlq m12, m4, 4 + mova m5, m4 + mova m10, m4 + mova m11, m4 + vpermi2q m4, m0, m2 ; 8 9 12 13 + vpermt2q m0, m12, m2 ; 0 1 4 5 + vpermi2q m5, m1, m3 ; 10 11 14 15 + vpermt2q m1, m12, m3 ; 2 3 6 7 + vpermi2q m10, m8, m6 + vpermt2q m8, m12, m6 + vpermi2q m11, m9, m7 + vpermt2q m9, m12, m7 + jmp m(idct_16x16_internal_8bpc).end + +%macro ITX_UNPACK_MULHRSW 8 ; dst[1-2], src, tmp, coef[1-4] + vpbroadcastd m%4, [o(pw_%5_%6x8)] + punpcklwd m%1, m%3, m%3 + pmulhrsw m%1, m%4 + vpbroadcastd m%4, [o(pw_%7_%8x8)] + punpckhwd m%2, m%3, m%3 + pmulhrsw m%2, m%4 +%endmacro + +cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + cmp eobd, 107 + jb .fast + mova m5, [cq+64*5] + mova m3, [cq+64*3] + mova m1, [cq+64*1] + mova m7, [cq+64*7] + mova m2, [cq+64*2] + mova m6, [cq+64*6] + mova m0, [cq+64*0] + mova m4, [cq+64*4] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + mova m8, [o(idct_8x32p)] + vpbroadcastd m9, [o(pw_8192)] + REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7 + punpckldq m8, m0, m1 ; ab + punpckhdq m0, m1 + punpckldq m1, m2, m3 ; cd + punpckhdq m2, m3 + punpckldq m3, m4, m5 ; ef + punpckhdq m4, m5 + punpckldq m5, m6, m7 ; gh + punpckhdq m6, m7 + REPX {pmulhrsw x, m9}, m8, m0, m1, m2, m3, m4, m5, m6 + punpcklqdq m18, m8, m1 ; 30 2 6 26 31 1 23 9 + punpckhqdq m14, m8, m1 ; 16 0 12 20 3 29 11 21 + punpcklqdq m21, m0, m2 ; 14 18 22 10 27 5 19 13 + punpckhqdq m15, m0, m2 ; 18 4 24 8 7 25 15 17 + punpcklqdq m20, m3, m5 + punpckhqdq m16, m3, m5 + punpcklqdq m19, m4, m6 + punpckhqdq m17, m4, m6 + vinserti32x4 ym8, ym18, xm20, 1 + vshufi32x4 ym1, ym18, ym20, 0x03 + vinserti32x4 ym9, ym14, xm16, 1 + vshufi32x4 ym3, ym14, ym16, 0x03 + vinserti32x4 ym0, ym21, xm19, 1 + vshufi32x4 ym5, ym21, ym19, 0x03 + vinserti32x4 ym7, ym15, xm17, 1 + vshufi32x4 ym6, ym15, ym17, 0x03 + call m(idct_8x16_internal_8bpc).main2 + psrlq m12, [o(permB)], 60 + vpermt2q m14, m12, m16 + vpermt2q m21, m12, m19 + vpermt2q m15, m12, m17 + vpermi2q m12, m18, m20 + vextracti32x8 ym16, m14, 1 + vextracti32x8 ym19, m21, 1 + vextracti32x8 ym17, m15, 1 + vextracti32x8 ym20, m12, 1 + call .main2 + jmp .end +.fast: ; right half is zero + mova m0, [o(int16_perm)] + mova ym2, [cq+64*4] + vinserti32x8 m2, [cq+64*0], 1 + mova ym3, [cq+64*6] + vinserti32x8 m3, [cq+64*2], 1 + mova ym4, [cq+64*3] + vinserti32x8 m4, [cq+64*5], 1 + mova ym5, [cq+64*7] + vinserti32x8 m5, [cq+64*1], 1 + REPX {vpermb x, m0, x}, m2, m3, m4, m5 + call m(idct_16x8_internal_8bpc).main2 + vbroadcasti32x4 m4, [o(int_shuf3)] + vbroadcasti32x4 m5, [o(int_shuf4)] + pshufb m2, m4 ; e0 f0 e2 f2 e1 f1 e3 f3 + pshufb m3, m5 ; g0 h0 g2 h2 g1 h1 g3 h3 + pshufb m0, m4 ; a0 b0 a2 b2 a1 b1 a3 b3 + pshufb m1, m5 ; c0 d0 c2 d2 c1 d1 c3 d3 + vpbroadcastd m4, [o(pw_8192)] + psrlq m5, [o(permB)], 60 + punpckldq m6, m2, m3 ; e0 f0 g0 h0 e2 f2 g2 h2 + punpckhdq m17, m2, m3 ; e1 f1 g1 h1 e3 f3 g3 h3 + punpckldq m2, m0, m1 ; a0 b0 c0 d0 a2 b2 c2 d2 + punpckhdq m16, m0, m1 ; a1 b1 c1 d1 a3 b3 c3 d3 + REPX {pmulhrsw x, m4}, m6, m17, m2, m16 + vinserti32x4 ym0, ym2, xm6, 1 ; 0 2 + vshufi32x4 ym1, ym2, ym6, 0x03 ; 4 6 + vinserti32x4 ym14, ym16, xm17, 1 ; 1 3 + vshufi32x4 ym15, ym16, ym17, 0x03 ; 5 7 + vpermt2q m2, m5, m6 ; 8 10 + vpermt2q m16, m5, m17 ; 9 11 + vextracti32x8 ym3, m2, 1 ; 12 14 + vextracti32x8 ym17, m16, 1 ; 13 15 + call m(idct_8x16_internal_8bpc).main_fast + call .main_fast +.end: + vpbroadcastd ym8, strided + pmulld ym8, [o(gather8d)] + call .main_end + lea r3, [dstq+strideq*4] + kxnorb k1, k1, k1 + lea r4, [dstq+strideq*8] + pxor m9, m9 + lea r1, [r3+strideq*8] + kmovb k2, k1 + vpgatherdq m12{k1}, [r0+ym8] + kmovb k1, k2 + vpgatherdq m13{k2}, [r3+ym8] + kmovb k2, k1 + vpgatherdq m14{k1}, [r4+ym8] + kmovb k1, k2 + vpgatherdq m15{k2}, [r1+ym8] + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {mova [cq+64*x], m9}, 0, 1, 2, 3, 4, 5, 6, 7 + punpcklbw m11, m12, m9 + punpckhbw m12, m9 + paddw m0, m11 + paddw m1, m12 + packuswb m0, m1 + kmovb k2, k1 + vpscatterdq [r0+ym8]{k1}, m0 + punpcklbw m12, m13, m9 + punpckhbw m13, m9 + paddw m2, m12 + paddw m3, m13 + packuswb m2, m3 + kmovb k1, k2 + vpscatterdq [r3+ym8]{k2}, m2 + punpcklbw m13, m14, m9 + punpckhbw m14, m9 + paddw m4, m13 + paddw m5, m14 + packuswb m4, m5 + kmovb k2, k1 + vpscatterdq [r4+ym8]{k1}, m4 + punpcklbw m14, m15, m9 + punpckhbw m15, m9 + paddw m6, m14 + paddw m7, m15 + packuswb m6, m7 + vpscatterdq [r1+ym8]{k2}, m6 + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 32 + imul r6d, 181 + add r6d, 128+512 + sar r6d, 8+2 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2 +INIT_YMM avx512icl +ALIGN function_align +cglobal_label .main_fast2 ; bottom three-quarters are zero + ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a + ITX_UNPACK_MULHRSW 21, 20, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a + mova m11, m12 + mova m17, m20 + mova m15, m21 + mova m16, m14 + jmp .main4 +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a + ITX_UNPACK_MULHRSW 21, 15, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a + ITX_UNPACK_MULHRSW 20, 16, 16, 8, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a + ITX_UNPACK_MULHRSW 19, 17, 17, 8, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a + jmp .main3 +ALIGN function_align +cglobal_label .main + punpcklwd m12, m21, m14 ; in31 in1 + punpckhwd m14, m21 ; in3 in29 + punpcklwd m21, m20, m15 ; in27 in5 + punpckhwd m15, m20 ; in7 in25 + punpcklwd m20, m19, m16 ; in23 in9 + punpckhwd m16, m19 ; in11 in21 + punpcklwd m19, m18, m17 ; in19 in13 + punpckhwd m17, m18 ; in15 in17 +.main2: + ITX_MUL2X_PACK 12, 8, 9, 10, 201, 4091, 5 ; t16a, t31a + ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a + ITX_MUL2X_PACK 21, 8, 9, 10, 995, 3973, 5 ; t20a, t27a + ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a + ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a + ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a + ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a + ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a +.main3: + psubsw m11, m12, m17 ; t17 t30 + paddsw m12, m17 ; t16 t31 + psubsw m17, m15, m20 ; t18 t29 + paddsw m20, m15 ; t19 t28 + psubsw m15, m21, m16 ; t21 t26 + paddsw m21, m16 ; t20 t27 + psubsw m16, m14, m19 ; t22 t25 + paddsw m14, m19 ; t23 t24 +.main4: + ITX_MUL2X_PACK 11, 18, 19, 10, 799, 4017, 5 ; t17a t30a + ITX_MUL2X_PACK 17, 18, 19, 10, m4017, 799, 5 ; t18a t29a + ITX_MUL2X_PACK 15, 18, 19, 10, 3406, 2276, 5 ; t21a t26a + ITX_MUL2X_PACK 16, 18, 19, 10, m2276, 3406, 5 ; t22a t25a + vpbroadcastd m8, [o(pw_m3784_1567)] + psubsw m19, m12, m20 ; t19a t28a + paddsw m20, m12 ; t16a t31a + psubsw m12, m14, m21 ; t20a t27a + paddsw m14, m21 ; t23a t24a + psubsw m21, m11, m17 ; t18 t29 + paddsw m11, m17 ; t17 t30 + psubsw m17, m16, m15 ; t21 t26 + paddsw m16, m15 ; t22 t25 + ITX_MUL2X_PACK 21, 18, 15, 10, 1567_3784, 8, 20 ; t18a t29a + ITX_MUL2X_PACK 19, 18, 15, 10, 1567_3784, 8, 20 ; t19 t28 + ITX_MUL2X_PACK 12, 18, 15, 10, 8, m1567_m3784, 36 ; t20 t27 + ITX_MUL2X_PACK 17, 18, 15, 10, 8, m1567_m3784, 36 ; t21a t26a + vbroadcasti32x4 m18, [o(deint_shuf)] + vpbroadcastd m8, [o(pw_m2896_2896)] + vpbroadcastd m9, [o(pw_2896_2896)] + psubsw m15, m20, m14 ; t23 t24 + paddsw m20, m14 ; t16 t31 + psubsw m14, m11, m16 ; t22a t25a + paddsw m11, m16 ; t17a t30a + psubsw m16, m21, m17 ; t21 t26 + paddsw m21, m17 ; t18 t29 + psubsw m17, m19, m12 ; t20a t27a + paddsw m19, m12 ; t19a t28a + REPX {pshufb x, m18}, m20, m11, m21, m19 + ITX_MUL2X_PACK 15, 18, 12, 10, 8, 9, 8 ; t23a t22a + ITX_MUL2X_PACK 14, 13, 15, 10, 8, 9, 8 ; t22 t25 + packssdw m18, m13 ; t23a t22 + packssdw m12, m15 ; t24a t25 + ITX_MUL2X_PACK 16, 13, 15, 10, 8, 9, 8 ; t21a t26a + ITX_MUL2X_PACK 17, 16, 14, 10, 8, 9, 8 ; t20 t27 + packssdw m16, m13 ; t20 t21a + packssdw m14, m15 ; t27 t26a + punpcklqdq m13, m19, m21 ; t19a t18 + punpckhqdq m19, m21 ; t28a t29 + punpcklqdq m21, m20, m11 ; t16 t17a + punpckhqdq m20, m11 ; t31 t30a +INIT_ZMM avx512icl + mova m15, [o(permA)] + ret +cglobal_label .main_end + vpbroadcastd m10, [o(pw_2048)] + vpermt2q m0, m15, m1 ; t0 t1 t2 t3 + vpermt2q m20, m15, m19 ; t31 t30a t29 t28a + vpermt2q m2, m15, m3 ; t4 t5 t6 t7 + vpermt2q m14, m15, m12 ; t27 t26a t25 t24a + vpermt2q m4, m15, m5 ; t8 t9 t10 t11 + vpermt2q m18, m15, m16 ; t23a t22 t21a t20 + vpermt2q m6, m15, m7 ; t12 t13 t14 t15 + vpermt2q m13, m15, m21 ; t19a t18 t17a t16 + psubsw m7, m0, m20 ; out31 out30 out29 out28 + paddsw m0, m20 ; out0 out1 out2 out3 + psubsw m5, m2, m14 ; out27 out26 out25 out24 + paddsw m2, m14 ; out4 out5 out6 out7 + psubsw m3, m4, m18 ; out23 out22 out21 out20 + paddsw m4, m18 ; out8 out9 out10 out11 + psubsw m1, m6, m13 ; out19 out18 out17 out16 + paddsw m6, m13 ; out12 out13 out14 out15 + vzeroupper + ret + +%macro LOAD_PACKED_16X2 3 ; dst, row[1-2] + vbroadcasti32x4 ym%1, [cq+16*%2] + vbroadcasti32x4 ym8, [cq+16*%3] + shufpd ym%1, ym8, 0x0c +%endmacro + +cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 4, 0, dst, stride, c, eob +%undef cmp + test eobd, eobd + jz .dconly + lea r5, [o_base] + LOAD_PACKED_16X2 0, 0, 2 ; in0 in2 + LOAD_PACKED_16X2 1, 4, 6 ; in4 in6 + LOAD_PACKED_16X2 2, 8, 10 ; in8 in10 + LOAD_PACKED_16X2 3, 12, 14 ; in12 in14 + LOAD_PACKED_16X2 14, 1, 3 ; in1 in3 + LOAD_PACKED_16X2 15, 5, 7 ; in5 in7 + LOAD_PACKED_16X2 16, 9, 11 ; in9 in11 + LOAD_PACKED_16X2 17, 13, 15 ; in13 in15 + pxor m4, m4 + REPX {mova [cq+64*x], m4}, 0, 1, 2, 3 + cmp eobd, 107 + jb .fast + LOAD_PACKED_16X2 4, 16, 18 ; in16 in18 + LOAD_PACKED_16X2 5, 20, 22 ; in20 in22 + LOAD_PACKED_16X2 6, 24, 26 ; in24 in26 + LOAD_PACKED_16X2 7, 28, 30 ; in28 in30 + call m(idct_8x16_internal_8bpc).main + LOAD_PACKED_16X2 18, 19, 17 ; in19 in17 + LOAD_PACKED_16X2 19, 23, 21 ; in23 in21 + LOAD_PACKED_16X2 20, 27, 25 ; in27 in25 + LOAD_PACKED_16X2 21, 31, 29 ; in31 in29 + pxor m8, m8 + REPX {mova [cq+64*x], m8}, 4, 5, 6, 7 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main + jmp .pass2 +.fast: ; bottom half is zero + mova ym5, ym4 + mova ym6, ym4 + mova ym7, ym4 + call m(idct_8x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast +.pass2: + vpbroadcastd m10, [o(pw_8192)] + vpermt2q m0, m15, m4 ; t0 t1 t9 t8 + vpermt2q m20, m15, m18 ; t31 t30a t23a t22 + vpermt2q m3, m15, m7 ; t7 t6 t14 t15 + vpermt2q m12, m15, m21 ; t25 t24a t17a t16 + vpermt2q m2, m15, m6 ; t4 t5 t13 t12 + vpermt2q m14, m15, m13 ; t23a t22 t21a t20 + vpermt2q m1, m15, m5 ; t3 t2 t10 t11 + vpermt2q m19, m15, m16 ; t27 t26a t19a t18 + psubsw m8, m0, m20 ; out31 out30 out22 out23 + paddsw m0, m20 ; out0 out1 out9 out8 + paddsw m6, m3, m12 ; out7 out6 out14 out15 + psubsw m3, m12 ; out24 out25 out17 out16 + psubsw m5, m2, m14 ; out27 out26 out18 out19 + paddsw m4, m2, m14 ; out4 out5 out13 out12 + psubsw m7, m1, m19 ; out28 out29 out21 out20 + paddsw m2, m1, m19 ; out3 out2 out10 out11 + vzeroupper + vshufi32x4 m1, m0, m3, q1221 ; out1 out9 out17 out25 + vshufi32x4 m0, m3, q0330 ; out0 out8 out16 out24 + vshufi32x4 m3, m2, m5, q0330 ; out3 out11 out19 out27 + vshufi32x4 m2, m5, q1221 ; out2 out10 out18 out26 + vshufi32x4 m5, m4, m7, q1221 ; out5 out13 out21 out29 + vshufi32x4 m4, m7, q0330 ; out4 out12 out20 out28 + vshufi32x4 m7, m6, m8, q0330 ; out7 out15 out23 out31 + vshufi32x4 m6, m8, q1221 ; out6 out14 out22 out30 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8 + call .main + vpbroadcastd m8, [o(pw_2048)] + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + lea r2, [strideq*3] + lea r3, [dstq+strideq*4] + movshdup m12, [o(permD)] + pmovzxbw m8, [dstq+strideq*0] + pmovzxbw m9, [dstq+strideq*1] + pmovzxbw m10, [dstq+strideq*2] + pmovzxbw m11, [dstq+r2 ] + paddw m0, m8 + paddw m1, m9 + paddw m2, m10 + paddw m3, m11 + pmovzxbw m8, [r3+strideq*0] + pmovzxbw m9, [r3+strideq*1] + pmovzxbw m10, [r3+strideq*2] + pmovzxbw m11, [r3+r2 ] + paddw m4, m8 + paddw m5, m9 + paddw m6, m10 + paddw m7, m11 + packuswb m0, m1 + packuswb m2, m3 + vpermq m0, m12, m0 + vpermq m2, m12, m2 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym2 + vextracti32x8 [dstq+r2 ], m2, 1 + packuswb m4, m5 + packuswb m6, m7 + vpermq m4, m12, m4 + vpermq m6, m12, m6 + mova [r3+strideq*0], ym4 + vextracti32x8 [r3+strideq*1], m4, 1 + mova [r3+strideq*2], ym6 + vextracti32x8 [r3+r2 ], m6, 1 + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 8 +.dconly2: + imul r6d, 181 + add r6d, 128+512 + sar r6d, 8+2 +.dconly3: + imul r6d, 181 + add r6d, 128+2048 + sar r6d, 8+4 + pxor m2, m2 + vpbroadcastw m3, r6d +.dconly_loop: + mova ym1, [dstq+strideq*0] + vinserti32x8 m1, [dstq+strideq*1], 1 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + paddw m0, m3 + paddw m1, m3 + packuswb m0, m1 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +ALIGN function_align +cglobal_label .main + vpbroadcastd m10, [o(pd_2048)] +.main2: + ITX_MULSUB_2W 5, 3, 8, 9, 10, 3406, 2276 ; t5a, t6a + ITX_MULSUB_2W 1, 7, 8, 9, 10, 799, 4017 ; t4a, t7a + ITX_MULSUB_2W 2, 6, 8, 9, 10, 1567, 3784 ; t2, t3 + vpbroadcastd m11, [o(pw_2896_2896)] + vpbroadcastd m12, [o(pw_m2896_2896)] + ITX_MULSUB_2W 0, 4, 8, 9, 10, 11, 12 ; t1, t0 +.main3: + paddsw m8, m1, m5 ; t4 + psubsw m1, m5 ; t5a + paddsw m9, m7, m3 ; t7 + psubsw m7, m3 ; t6a + ITX_MULSUB_2W 7, 1, 3, 5, 10, 11, 12 ; t5, t6 + psubsw m5, m0, m2 ; dct4 out2 + paddsw m2, m0 ; dct4 out1 + paddsw m0, m4, m6 ; dct4 out0 + psubsw m4, m6 ; dct4 out3 + psubsw m6, m2, m1 ; out6 + paddsw m1, m2 ; out1 + paddsw m2, m5, m7 ; out2 + psubsw m5, m7 ; out5 + psubsw m7, m0, m9 ; out7 + paddsw m0, m9 ; out0 + paddsw m3, m4, m8 ; out3 + psubsw m4, m8 ; out4 + ret + +cglobal inv_txfm_add_identity_identity_8x32_8bpc, 3, 5, 0, dst, stride, c + vpbroadcastd m7, [pw_5] + paddsw m0, m7, [cq+64*0] + paddsw m1, m7, [cq+64*1] + vpbroadcastd ym9, strided + paddsw m2, m7, [cq+64*2] + paddsw m3, m7, [cq+64*3] + paddsw m4, m7, [cq+64*4] + paddsw m5, m7, [cq+64*5] + paddsw m6, m7, [cq+64*6] + paddsw m7, [cq+64*7] + pmulld ym14, ym9, [pd_0to15] + lea r3, [dstq+strideq*1] + lea r4, [dstq+strideq*2] + kxnorb k1, k1, k1 + pxor m13, m13 + add r1, r4 ; dstq+strideq*3 + kmovb k2, k1 + vpgatherdq m9{k1}, [r0+ym14*4] + kmovb k1, k2 + vpgatherdq m10{k2}, [r3+ym14*4] + kmovb k2, k1 + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8 + REPX {psraw x, 3}, m0, m1, m2, m3, m4, m5, m6, m7 + vpgatherdq m11{k1}, [r4+ym14*4] + kmovb k1, k2 + vpgatherdq m12{k2}, [r1+ym14*4] + REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7 + punpcklbw m8, m9, m13 ; 0 8 16 24 + punpckhbw m9, m13 ; 4 12 20 28 + paddw m0, m8 + paddw m4, m9 + packuswb m0, m4 + kmovb k2, k1 + vpscatterdq [r0+ym14*4]{k1}, m0 + punpcklbw m8, m10, m13 ; 1 9 17 25 + punpckhbw m10, m13 ; 5 13 21 29 + paddw m1, m8 + paddw m5, m10 + packuswb m1, m5 + kmovb k1, k2 + vpscatterdq [r3+ym14*4]{k2}, m1 + punpcklbw m8, m11, m13 ; 2 10 18 26 + punpckhbw m11, m13 ; 6 14 22 30 + paddw m2, m8 + paddw m6, m11 + packuswb m2, m6 + kmovb k2, k1 + vpscatterdq [r4+ym14*4]{k1}, m2 + punpcklbw m8, m12, m13 ; 3 11 19 27 + punpckhbw m12, m13 ; 7 15 23 31 + paddw m3, m8 + paddw m7, m12 + packuswb m3, m7 + vpscatterdq [r1+ym14*4]{k2}, m3 + RET + +cglobal inv_txfm_add_identity_identity_32x8_8bpc, 3, 5, 0, dst, stride, c + vpbroadcastd m0, [pw_4096] + pmulhrsw m3, m0, [cq+64*0] + pmulhrsw m4, m0, [cq+64*4] + pmulhrsw m6, m0, [cq+64*1] + pmulhrsw m5, m0, [cq+64*5] + pmulhrsw m7, m0, [cq+64*2] + pmulhrsw m2, m0, [cq+64*6] + pmulhrsw m8, m0, [cq+64*3] + pmulhrsw m0, [cq+64*7] + mova m13, [int8_permA] + lea r3, [strideq*3] + lea r4, [dstq+strideq*4] + punpckldq m1, m3, m4 + punpckhdq m3, m4 + punpckldq m4, m6, m5 + punpckhdq m6, m5 + punpckldq m5, m7, m2 + punpckhdq m7, m2 + punpckldq m2, m8, m0 + punpckhdq m8, m0 + mova ym9, [dstq+strideq*0] + vinserti32x8 m9, [dstq+strideq*2], 1 + mova ym10, [dstq+strideq*1] + vinserti32x8 m10, [dstq+r3 ], 1 + mova ym11, [r4+strideq*0] + vinserti32x8 m11, [r4+strideq*2], 1 + mova ym12, [r4+strideq*1] + vinserti32x8 m12, [r4+r3 ], 1 + REPX {vpermb x, m13, x}, m1, m4, m5, m2, m3, m6, m7, m8 + pxor m13, m13 + REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7 + punpcklqdq m0, m1, m4 ; a0 a2 c0 c2 + punpckhqdq m1, m4 ; b0 b2 d0 d2 + punpcklqdq m4, m5, m2 ; a1 a3 c1 c3 + punpckhqdq m5, m2 ; b1 b3 d1 d3 + punpcklqdq m2, m3, m6 ; e0 e2 g0 g2 + punpckhqdq m3, m6 ; f0 f2 h0 h2 + punpcklqdq m6, m7, m8 ; e1 e3 g1 g3 + punpckhqdq m7, m8 ; f1 f3 h1 h3 + punpcklbw m8, m9, m13 + punpckhbw m9, m13 + paddw m0, m8 + paddw m4, m9 + packuswb m0, m4 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*2], m0, 1 + punpcklbw m8, m10, m13 + punpckhbw m10, m13 + paddw m1, m8 + paddw m5, m10 + packuswb m1, m5 + mova [dstq+strideq*1], ym1 + vextracti32x8 [dstq+r3 ], m1, 1 + punpcklbw m8, m11, m13 + punpckhbw m11, m13 + paddw m2, m8 + paddw m6, m11 + packuswb m2, m6 + mova [r4+strideq*0], ym2 + vextracti32x8 [r4+strideq*2], m2, 1 + punpcklbw m8, m12, m13 + punpckhbw m12, m13 + paddw m3, m8 + paddw m7, m12 + packuswb m3, m7 + mova [r4+strideq*1], ym3 + vextracti32x8 [r4+r3 ], m3, 1 + RET + +%macro IDCT_16x32_END 3 ; src[1-2], row + mova xm8, [dstq+strideq*0] + vinserti32x4 ym8, [dstq+strideq*1], 1 + mova xm9, [dstq+r3 ] + vinserti32x4 ym9, [dstq+strideq*2], 1 + pmulhrsw m%1, m10 + pmulhrsw m%2, m10 + vpermb m8, m11, m8 + vpermb m9, m11, m9 + mova [cq+64*(%3*2+0)], m13 + mova [cq+64*(%3*2+1)], m13 + paddw m8, m%1 + paddw m9, m%2 + packuswb m8, m9 + vpermd m8, m12, m8 + mova [dstq+strideq*0], xm8 + vextracti32x4 [dstq+strideq*1], ym8, 1 + vextracti32x4 [dstq+strideq*2], m8, 2 + vextracti32x4 [dstq+r3 ], m8, 3 +%if %1 != 20 + lea dstq, [dstq+strideq*4] +%endif +%endmacro + +cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 4, 22, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m15, [o(pw_2896x8)] + cmp eobd, 151 + jb .fast + pmulhrsw m5, m15, [cq+64*10] + pmulhrsw m3, m15, [cq+64* 6] + pmulhrsw m1, m15, [cq+64* 2] + pmulhrsw m7, m15, [cq+64*14] + pmulhrsw m2, m15, [cq+64* 4] + pmulhrsw m6, m15, [cq+64*12] + pmulhrsw m0, m15, [cq+64* 0] + pmulhrsw m4, m15, [cq+64* 8] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + pmulhrsw m14, m15, [cq+64* 1] + pmulhrsw m21, m15, [cq+64*15] + pmulhrsw m18, m15, [cq+64* 9] + pmulhrsw m17, m15, [cq+64* 7] + pmulhrsw m16, m15, [cq+64* 5] + pmulhrsw m19, m15, [cq+64*11] + pmulhrsw m20, m15, [cq+64*13] + pmulhrsw m15, [cq+64* 3] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova m8, [o(idct_16x32p)] + vpbroadcastd m9, [o(pw_16384)] + REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m14, m15, m16, m17, m18, m19, m20, m21 + punpckldq m8, m0, m1 + punpckhdq m0, m1 + punpckldq m1, m2, m3 + punpckhdq m2, m3 + REPX {pmulhrsw x, m9}, m8, m0, m1, m2 + punpckldq m3, m4, m5 + punpckhdq m4, m5 + punpckldq m5, m6, m7 + punpckhdq m6, m7 + REPX {pmulhrsw x, m9}, m3, m4, m5, m6 + punpckldq m7, m14, m15 + punpckhdq m14, m15 + punpckldq m15, m16, m17 + punpckhdq m16, m17 + REPX {pmulhrsw x, m9}, m7, m14, m15, m16 + punpckldq m17, m18, m19 + punpckhdq m18, m19 + punpckldq m19, m20, m21 + punpckhdq m20, m21 + REPX {pmulhrsw x, m9}, m17, m18, m19, m20 + punpcklqdq m21, m8, m1 + punpckhqdq m8, m1 + punpcklqdq m1, m0, m2 + punpckhqdq m0, m2 + punpcklqdq m2, m3, m5 + punpckhqdq m3, m5 + punpcklqdq m5, m4, m6 + punpckhqdq m4, m6 + punpcklqdq m6, m7, m15 + punpckhqdq m7, m15 + punpcklqdq m15, m14, m16 + punpckhqdq m14, m16 + punpcklqdq m16, m17, m19 + punpckhqdq m17, m19 + punpcklqdq m19, m18, m20 + punpckhqdq m18, m20 + vinserti32x8 m20, m21, ym2, 1 + vshufi32x4 m21, m2, q3232 + vinserti32x8 m2, m8, ym3, 1 + vshufi32x4 m8, m3, q3232 + vinserti32x8 m3, m1, ym5, 1 + vshufi32x4 m1, m5, q3232 + vinserti32x8 m5, m0, ym4, 1 + vshufi32x4 m0, m4, q3232 + vinserti32x8 m4, m6, ym16, 1 + vshufi32x4 m6, m16, q3232 + vinserti32x8 m16, m7, ym17, 1 + vshufi32x4 m7, m17, q3232 + vinserti32x8 m17, m15, ym19, 1 + vshufi32x4 m15, m19, q3232 + vinserti32x8 m19, m14, ym18, 1 + vshufi32x4 m14, m18, q3232 + vshufi32x4 m18, m21, m6, q3131 ; 27 5 + vshufi32x4 m21, m6, q2020 ; 31 1 + vshufi32x4 m6, m8, m7, q2020 ; 24 8 + vshufi32x4 m8, m7, q3131 ; 30 2 + vshufi32x4 m7, m1, m15, q2020 ; 28 4 + vshufi32x4 m1, m15, q3131 ; 6 26 + vshufi32x4 m15, m0, m14, q2020 ; 7 25 + vshufi32x4 m0, m14, q3131 ; 14 18 + vshufi32x4 m14, m20, m4, q2020 ; 3 29 + vshufi32x4 m20, m4, q3131 ; 23 9 + vshufi32x4 m9, m3, m17, q2020 ; 16 0 + vshufi32x4 m3, m17, q3131 ; 12 20 + vshufi32x4 m17, m5, m19, q2020 ; 15 17 + vshufi32x4 m5, m19, q3131 ; 22 10 + vshufi32x4 m19, m2, m16, q2020 ; 19 13 + vshufi32x4 m16, m2, m16, q3131 ; 11 21 + call m(idct_16x16_internal_8bpc).main3 + call .main_oddhalf + jmp .pass2 +.fast: ; right half is zero + mova ym8, [cq+64*15] + vinserti32x8 m8, [cq+64* 1], 1 + mova m2, [o(int16_perm)] + mova ym9, [cq+64* 8] + vinserti32x8 m9, [cq+64* 0], 1 + mova ym0, [cq+64* 7] + vinserti32x8 m0, [cq+64* 9], 1 + mova ym7, [cq+64*14] + vinserti32x8 m7, [cq+64* 2], 1 + mova ym1, [cq+64* 3] + vinserti32x8 m1, [cq+64*13], 1 + mova ym3, [cq+64* 6] + vinserti32x8 m3, [cq+64*10], 1 + mova ym5, [cq+64*11] + vinserti32x8 m5, [cq+64* 5], 1 + mova ym6, [cq+64*12] + vinserti32x8 m6, [cq+64* 4], 1 + REPX {pmulhrsw x, m15}, m8, m9, m0, m7, m1, m3, m5, m6 + REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6 + call m(idct_16x16_internal_8bpc).main2 + vbroadcasti32x4 m8, [o(int_shuf3)] + vbroadcasti32x4 m9, [o(int_shuf4)] + vpbroadcastd m11, [o(pw_16384)] + pshufb m0, m8 + pshufb m1, m9 + pshufb m2, m8 + pshufb m3, m9 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 + pshufb m4, m8 + pshufb m5, m9 + pshufb m6, m8 + pshufb m7, m9 + REPX {pmulhrsw x, m11}, m4, m5, m6, m7 + punpckhdq m17, m0, m1 + punpckldq m0, m1 + punpckhdq m16, m2, m3 + punpckldq m2, m3 + punpckhdq m18, m4, m5 + punpckldq m4, m5 + punpckhdq m5, m6, m7 + punpckldq m6, m7 + vinserti32x8 m1, m0, ym2, 1 + vshufi32x4 m3, m0, m2, q3232 + vinserti32x8 m2, m4, ym6, 1 + vshufi32x4 m4, m6, q3232 + vinserti32x8 m15, m17, ym16, 1 + vshufi32x4 m17, m16, q3232 + vinserti32x8 m16, m18, ym5, 1 + vshufi32x4 m18, m5, q3232 + vshufi32x4 m0, m1, m2, q2020 ; 0 2 + vshufi32x4 m1, m2, q3131 ; 4 6 + vshufi32x4 m2, m3, m4, q2020 ; 8 10 + vshufi32x4 m3, m4, q3131 ; 12 14 + vshufi32x4 m14, m15, m16, q2020 ; 1 3 + vshufi32x4 m15, m16, q3131 ; 5 7 + vshufi32x4 m16, m17, m18, q2020 ; 9 11 + vshufi32x4 m17, m18, q3131 ; 13 15 + pxor m6, m6 + punpckhwd m8, m0, m0 + punpcklwd m9, m6, m0 + punpckhwd m0, m3, m3 + punpckhwd m5, m2, m2 + punpcklwd m7, m1, m1 + punpckhwd m1, m1 + punpcklwd m3, m3 + punpcklwd m6, m2 + call m(idct_16x16_internal_8bpc).main_fast5 + punpcklwd m21, m14, m14 + punpckhwd m14, m14 + punpcklwd m18, m15, m15 + punpckhwd m15, m15 + punpcklwd m20, m16, m16 + punpckhwd m16, m16 + punpcklwd m19, m17, m17 + punpckhwd m17, m17 + call .main_oddhalf_fast +.pass2: + vpbroadcastd m10, [o(pw_2048)] + mova m11, [o(end_16x32p)] + lea r3, [strideq*3] + pxor m13, m13 + psrld m12, m11, 8 + IDCT_16x32_END 0, 1, 0 + IDCT_16x32_END 2, 3, 1 + IDCT_16x32_END 4, 5, 2 + IDCT_16x32_END 6, 7, 3 + IDCT_16x32_END 14, 15, 4 + IDCT_16x32_END 16, 17, 5 + IDCT_16x32_END 18, 19, 6 + IDCT_16x32_END 20, 21, 7 + RET +ALIGN function_align +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly +ALIGN function_align +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero + vpbroadcastd m8, [o(pw_201_4091x8)] + vpbroadcastd m20, [o(pw_m1380_3857x8)] + vpbroadcastd m9, [o(pw_995_3973x8)] + vpbroadcastd m16, [o(pw_m601_4052x8)] + pmulhrsw m21, m8 ; t16a, t31a + pmulhrsw m20, m15 ; t19a, t28a + pmulhrsw m18, m9 ; t20a, t27a + pmulhrsw m14, m16 ; t23a, t24a + mova m8, m21 + mova m17, m20 + mova m15, m18 + mova m16, m14 + jmp .main3 +ALIGN function_align +cglobal_label .main_oddhalf_fast ; bottom half is zero + vpbroadcastd m8, [o(pw_201_4091x8)] + vpbroadcastd m9, [o(pw_m2751_3035x8)] + vpbroadcastd m11, [o(pw_1751_3703x8)] + vpbroadcastd m12, [o(pw_m1380_3857x8)] + pmulhrsw m21, m8 ; t16a, t31a + vpbroadcastd m8, [o(pw_995_3973x8)] + pmulhrsw m17, m9 ; t17a, t30a + vpbroadcastd m9, [o(pw_m2106_3513x8)] + pmulhrsw m20, m11 ; t18a, t29a + vpbroadcastd m11, [o(pw_2440_3290x8)] + pmulhrsw m15, m12 ; t19a, t28a + vpbroadcastd m12, [o(pw_m601_4052x8)] + pmulhrsw m18, m8 ; t20a, t27a + pmulhrsw m16, m9 ; t21a, t26a + pmulhrsw m19, m11 ; t22a, t25a + pmulhrsw m14, m12 ; t23a, t24a + jmp .main2 +ALIGN function_align +cglobal_label .main_oddhalf + ITX_MUL2X_PACK 21, 8, 9, 10, 201, 4091, 5 ; t16a, t31a + ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a + ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a + ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a + ITX_MUL2X_PACK 18, 8, 9, 10, 995, 3973, 5 ; t20a, t27a + ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a + ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a + ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a +.main2: + psubsw m8, m21, m17 ; t17 t30 + paddsw m21, m17 ; t16 t31 + psubsw m17, m15, m20 ; t18 t29 + paddsw m20, m15 ; t19 t28 + psubsw m15, m18, m16 ; t21 t26 + paddsw m18, m16 ; t20 t27 + psubsw m16, m14, m19 ; t22 t25 + paddsw m14, m19 ; t23 t24 +.main3: + ITX_MUL2X_PACK 8, 9, 19, 10, 799, 4017, 5 ; t17a t30a + ITX_MUL2X_PACK 17, 9, 19, 10, m4017, 799, 5 ; t18a t29a + ITX_MUL2X_PACK 15, 9, 19, 10, 3406, 2276, 5 ; t21a t26a + ITX_MUL2X_PACK 16, 9, 19, 10, m2276, 3406, 5 ; t22a t25a + vpbroadcastd m11, [o(pw_m3784_1567)] + psubsw m19, m21, m20 ; t19a t28a + paddsw m21, m20 ; t16a t31a + psubsw m20, m14, m18 ; t20a t27a + paddsw m14, m18 ; t23a t24a + psubsw m18, m8, m17 ; t18 t29 + paddsw m8, m17 ; t17 t30 + psubsw m17, m16, m15 ; t21 t26 + paddsw m15, m16 ; t22 t25 + ITX_MUL2X_PACK 18, 9, 16, 10, 1567_3784, 11, 20 ; t18a t29a + ITX_MUL2X_PACK 19, 9, 16, 10, 1567_3784, 11, 20 ; t19 t28 + ITX_MUL2X_PACK 20, 9, 16, 10, 11, m1567_m3784, 36 ; t20 t27 + ITX_MUL2X_PACK 17, 9, 16, 10, 11, m1567_m3784, 36 ; t21a t26a + vbroadcasti32x4 m9, [o(deint_shuf)] + psubsw m16, m21, m14 ; t23 t24 + paddsw m14, m21 ; t16 t31 + psubsw m21, m8, m15 ; t22a t25a + paddsw m15, m8 ; t17a t30a + psubsw m8, m18, m17 ; t21 t26 + paddsw m18, m17 ; t18 t29 + paddsw m17, m19, m20 ; t19a t28a + psubsw m19, m20 ; t20a t27a + vpbroadcastd m11, [o(pw_m2896_2896)] + vpbroadcastd m12, [o(pw_2896_2896)] + REPX {pshufb x, m9}, m14, m15, m18, m17 + mova m9, m10 + vpdpwssd m9, m16, m11 + mova m20, m10 + vpdpwssd m20, m21, m11 + psrad m9, 12 + psrad m20, 12 + packssdw m9, m20 ; t23a t22 + mova m20, m10 + vpdpwssd m20, m16, m12 + mova m16, m10 + vpdpwssd m16, m21, m12 + psrad m20, 12 + psrad m16, 12 + packssdw m16, m20, m16 ; t24a t25 + ITX_MUL2X_PACK 8, 21, 20, 10, 11, 12, 8 ; t21a t26a + ITX_MUL2X_PACK 19, 8, 11, 10, 11, 12, 8 ; t20 t27 + packssdw m11, m20 ; t27 t26a + packssdw m8, m21 ; t20 t21a + punpcklqdq m20, m14, m15 ; t16 t17a + punpckhqdq m14, m15 ; t31 t30a + punpckhqdq m15, m17, m18 ; t28a t29 + punpcklqdq m17, m18 ; t19a t18 + psubsw m21, m0, m14 ; out31 out30 + paddsw m0, m14 ; out0 out1 + psubsw m14, m7, m20 ; out16 out17 + paddsw m7, m20 ; out15 out14 + psubsw m20, m1, m15 ; out28 out29 + paddsw m1, m15 ; out3 out2 + psubsw m15, m6, m17 ; out19 out18 + paddsw m6, m17 ; out12 out13 + psubsw m17, m4, m9 ; out23 out22 + paddsw m4, m9 ; out8 out9 + psubsw m18, m3, m16 ; out24 out25 + paddsw m3, m16 ; out7 out6 + psubsw m16, m5, m8 ; out20 out21 + paddsw m5, m8 ; out11 out10 + psubsw m19, m2, m11 ; out27 out26 + paddsw m2, m11 ; out4 out5 + ret + +cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 22, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + mova m21, [o(permB)] + vpermq m1, m21, [cq+64* 0] ; 0 1 + vpermq m14, m21, [cq+64* 1] ; 2 3 + vpermq m20, m21, [cq+64* 2] ; 4 5 + vpermq m15, m21, [cq+64* 3] ; 6 7 + vpbroadcastd m8, [o(pw_2896x8)] + vpermq m2, m21, [cq+64* 4] ; 8 9 + vpermq m16, m21, [cq+64* 5] ; 10 11 + vpermq m3, m21, [cq+64* 6] ; 12 13 + vpermq m17, m21, [cq+64* 7] ; 14 15 + REPX {pmulhrsw x, m8}, m1, m14, m20, m15, m2, m16, m3, m17 + pxor m12, m12 + REPX {mova [cq+64*x], m12}, 0, 1, 2, 3, 4, 5, 6, 7 + cmp eobd, 151 + jb .fast + vpermq m9, m21, [cq+64* 8] ; 16 17 + vpermq m19, m21, [cq+64* 9] ; 18 19 + vpermq m4, m21, [cq+64*10] ; 20 21 + vpermq m5, m21, [cq+64*11] ; 22 23 + vpermq m6, m21, [cq+64*12] ; 24 25 + vpermq m18, m21, [cq+64*13] ; 26 27 + vpermq m7, m21, [cq+64*14] ; 28 29 + vpermq m21, m21, [cq+64*15] ; 30 31 + REPX {pmulhrsw x, m8}, m9, m19, m4, m5, m6, m18, m7, m21 + REPX {mova [cq+64*x], m12}, 8, 9, 10, 11, 12, 13, 14, 15 + punpcklwd m8, m21, m14 ; 30 2 + punpckhwd m21, m1 ; 31 1 + punpcklwd m0, m17, m19 ; 14 18 + punpckhwd m17, m9 ; 15 17 + punpcklwd m9, m1 ; 16 0 + punpckhwd m14, m7 ; 3 29 + punpcklwd m1, m15, m18 ; 6 26 + punpckhwd m15, m6 ; 7 25 + punpcklwd m6, m2 ; 24 8 + punpckhwd m19, m3 ; 19 13 + punpcklwd m3, m4 ; 12 20 + punpckhwd m18, m20 ; 27 5 + punpcklwd m7, m20 ; 28 4 + punpckhwd m20, m5, m2 ; 23 9 + punpcklwd m5, m16 ; 22 10 + punpckhwd m16, m4 ; 11 21 + call m(idct_16x16_internal_8bpc).main2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + jmp .pass2 +.fast: ; bottom half zero + punpcklwd m8, m14, m14 ; 2 + punpcklwd m0, m17, m17 ; 14 + punpcklwd m5, m16, m16 ; 10 + punpcklwd m9, m12, m1 ; __ 0 + punpckhwd m21, m1, m1 ; 1 + punpcklwd m1, m15, m15 ; 6 + punpcklwd m7, m20, m20 ; 4 + punpckhwd m19, m3, m3 ; 13 + punpcklwd m3, m3 ; 12 + punpcklwd m6, m12, m2 ; __ 8 + punpckhwd m18, m20, m20 ; 5 + punpckhwd m20, m2, m2 ; 9 + call m(idct_16x16_internal_8bpc).main_fast + punpckhwd m15, m15 ; 7 + punpckhwd m14, m14 ; 3 + punpckhwd m16, m16 ; 11 + punpckhwd m17, m17 ; 15 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast +.pass2: + vpbroadcastd m9, [o(pw_16384)] + call .transpose_round + vshufi32x4 m16, m14, m2, q3131 ; 5 + vshufi32x4 m14, m2, q2020 ; 1 + vshufi32x4 m2, m0, m3, q3131 ; 4 + vshufi32x4 m0, m3, q2020 ; 0 + vshufi32x4 m3, m1, m18, q3131 ; 6 + vshufi32x4 m1, m18, q2020 ; 2 + vshufi32x4 m18, m20, m6, q2020 ; 9 + vshufi32x4 m20, m6, q3131 ; 13 + vshufi32x4 m6, m21, m4, q3131 ; 12 + vshufi32x4 m4, m21, m4, q2020 ; 8 + vshufi32x4 m21, m19, m7, q3131 ; 15 + vshufi32x4 m19, m7, q2020 ; 11 + vshufi32x4 m7, m5, m15, q3131 ; 14 + vshufi32x4 m5, m15, q2020 ; 10 + vshufi32x4 m15, m17, m9, q2020 ; 3 + vshufi32x4 m17, m9, q3131 ; 7 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main2 + call .main_oddhalf + vpbroadcastd m12, [o(pw_2048)] + movshdup m13, [o(permD)] + lea r2, [strideq*3] + pmovzxbw m8, [dstq+strideq*0] + pmovzxbw m9, [dstq+strideq*1] + pmovzxbw m10, [dstq+strideq*2] + pmovzxbw m11, [dstq+r2 ] + REPX {pmulhrsw x, m12}, m0, m1, m2, m3 + lea r3, [dstq+strideq*4] + paddw m0, m8 + paddw m1, m9 + paddw m2, m10 + paddw m3, m11 + pmovzxbw m8, [r3+strideq*0] + pmovzxbw m9, [r3+strideq*1] + pmovzxbw m10, [r3+strideq*2] + pmovzxbw m11, [r3+r2 ] + REPX {pmulhrsw x, m12}, m4, m5, m6, m7 + lea r4, [dstq+strideq*8] + packuswb m0, m1 + paddw m4, m8 + paddw m5, m9 + packuswb m2, m3 + paddw m6, m10 + paddw m7, m11 + pmovzxbw m8, [r4+strideq*0] + pmovzxbw m9, [r4+strideq*1] + pmovzxbw m10, [r4+strideq*2] + pmovzxbw m11, [r4+r2 ] + REPX {pmulhrsw x, m12}, m14, m15, m16, m17 + lea r5, [r3+strideq*8] + packuswb m4, m5 + paddw m14, m8 + paddw m15, m9 + packuswb m6, m7 + paddw m16, m10 + paddw m17, m11 + pmovzxbw m8, [r5+strideq*0] + pmovzxbw m9, [r5+strideq*1] + pmovzxbw m10, [r5+strideq*2] + pmovzxbw m11, [r5+r2 ] + REPX {pmulhrsw x, m12}, m18, m19, m20, m21 + packuswb m14, m15 + paddw m18, m8 + paddw m19, m9 + packuswb m16, m17 + paddw m20, m10 + paddw m21, m11 + packuswb m18, m19 + packuswb m20, m21 + REPX {vpermq x, m13, x}, m0, m2, m4, m6, m14, m16, m18, m20 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym2 + vextracti32x8 [dstq+r2 ], m2, 1 + mova [r3+strideq*0], ym4 + vextracti32x8 [r3+strideq*1], m4, 1 + mova [r3+strideq*2], ym6 + vextracti32x8 [r3+r2 ], m6, 1 + mova [r4+strideq*0], ym14 + vextracti32x8 [r4+strideq*1], m14, 1 + mova [r4+strideq*2], ym16 + vextracti32x8 [r4+r2 ], m16, 1 + mova [r5+strideq*0], ym18 + vextracti32x8 [r5+strideq*1], m18, 1 + mova [r5+strideq*2], ym20 + vextracti32x8 [r5+r2 ], m20, 1 + RET +ALIGN function_align +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 16 + imul r6d, 181 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3 +ALIGN function_align +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero + vpbroadcastd m9, [o(pw_2896x8)] + vpbroadcastd m2, [o(pw_4017x8)] + vpbroadcastd m3, [o(pw_799x8)] + vpbroadcastd m18, [o(pw_4076x8)] + vpbroadcastd m19, [o(pw_401x8)] + vpbroadcastd m20, [o(pw_m1189x8)] + vpbroadcastd m16, [o(pw_3920x8)] + pmulhrsw m9, m0 ; t0 + pmulhrsw m2, m1 ; t7a + pmulhrsw m1, m3 ; t4a + pmulhrsw m18, m14 ; t15a + pmulhrsw m14, m19 ; t8a + pmulhrsw m20, m15 ; t11a + pmulhrsw m15, m16 ; t12a + psubsw m7, m9, m2 ; idct8 out7 + paddsw m0, m9, m2 ; idct8 out0 + psubsw m4, m9, m1 ; idct8 out4 + paddsw m3, m9, m1 ; idct8 out3 + ITX_MULSUB_2W 2, 1, 5, 6, 10, 2896, 2896 ; t5, t6 + mova m21, m18 + mova m19, m14 + mova m16, m15 + mova m8, m20 + psubsw m6, m9, m1 ; idct8 out6 + paddsw m1, m9 ; idct8 out1 + psubsw m5, m9, m2 ; idct8 out5 + paddsw m2, m9 ; idct8 out2 + jmp .main3 +ALIGN function_align +cglobal_label .main_oddhalf_fast ; bottom half is zero + vpbroadcastd m5, [o(pw_m2276x8)] + vpbroadcastd m11, [o(pw_3406x8)] + vpbroadcastd m7, [o(pw_4017x8)] + vpbroadcastd m12, [o(pw_799x8)] + vpbroadcastd m6, [o(pw_3784x8)] + vpbroadcastd m10, [o(pw_1567x8)] + vpbroadcastd m4, [o(pw_2896x8)] + pmulhrsw m5, m3 ; t5a + pmulhrsw m3, m11 ; t6a + pmulhrsw m7, m1 ; t7a + pmulhrsw m1, m12 ; t4a + pmulhrsw m6, m2 ; t3 + pmulhrsw m2, m10 ; t2 + pmulhrsw m4, m0 ; t0 + vpbroadcastd m11, [o(pw_2896_2896)] + vpbroadcastd m12, [o(pw_m2896_2896)] + vpbroadcastd m10, [o(pd_2048)] + mova m0, m4 ; t1 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main3 + vpbroadcastd m21, [o(pw_4076x8)] + vpbroadcastd m8, [o(pw_401x8)] + vpbroadcastd m18, [o(pw_m2598x8)] + vpbroadcastd m9, [o(pw_3166x8)] + vpbroadcastd m19, [o(pw_3612x8)] + vpbroadcastd m11, [o(pw_1931x8)] + vpbroadcastd m20, [o(pw_m1189x8)] + vpbroadcastd m12, [o(pw_3920x8)] + pmulhrsw m21, m14 ; t15a + pmulhrsw m14, m8 ; t8a + pmulhrsw m18, m17 ; t9a + pmulhrsw m17, m9 ; t14a + pmulhrsw m19, m16 ; t13a + pmulhrsw m16, m11 ; t10a + pmulhrsw m20, m15 ; t11a + pmulhrsw m15, m12 ; t12a + jmp .main2 +ALIGN function_align +cglobal_label .main_oddhalf + ITX_MULSUB_2W 14, 21, 8, 9, 10, 401, 4076 ; t8a, t15a + ITX_MULSUB_2W 18, 17, 8, 9, 10, 3166, 2598 ; t9a, t14a + ITX_MULSUB_2W 16, 19, 8, 9, 10, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2W 20, 15, 8, 9, 10, 3920, 1189 ; t11a, t12a +.main2: + paddsw m8, m20, m16 ; t11 + psubsw m20, m16 ; t10 + paddsw m16, m15, m19 ; t12 + psubsw m15, m19 ; t13 + psubsw m19, m14, m18 ; t9 + paddsw m14, m18 ; t8 + psubsw m18, m21, m17 ; t14 + paddsw m21, m17 ; t15 +.main3: + vpbroadcastd m11, [o(pw_1567_3784)] + vpbroadcastd m12, [o(pw_m3784_1567)] + ITX_MULSUB_2W 18, 19, 9, 17, 10, 11, 12 ; t9a, t14a + vpbroadcastd m11, [o(pw_m1567_m3784)] + ITX_MULSUB_2W 15, 20, 9, 17, 10, 12, 11 ; t10a, t13a + vpbroadcastd m11, [o(pw_2896_2896)] + vpbroadcastd m12, [o(pw_m2896_2896)] + psubsw m17, m14, m8 ; t11a + paddsw m8, m14 ; t8a + paddsw m14, m18, m15 ; t9 + psubsw m18, m15 ; t10 + psubsw m15, m19, m20 ; t13 + paddsw m19, m20 ; t14 + paddsw m20, m21, m16 ; t15a + psubsw m16, m21, m16 ; t12a + ITX_MULSUB_2W 15, 18, 9, 21, 10, 11, 12 ; t10a, t13a + ITX_MULSUB_2W 16, 17, 9, 21, 10, 11, 12 ; t11, t12 + psubsw m21, m0, m20 ; out15 + paddsw m0, m20 ; out0 + psubsw m20, m1, m19 ; out14 + paddsw m1, m19 ; out1 + psubsw m19, m2, m18 ; out13 + paddsw m2, m18 ; out2 + psubsw m18, m3, m17 ; out12 + paddsw m3, m17 ; out3 + psubsw m17, m4, m16 ; out11 + paddsw m4, m16 ; out4 + psubsw m16, m5, m15 ; out10 + paddsw m5, m15 ; out5 + psubsw m15, m6, m14 ; out9 + paddsw m6, m14 ; out6 + psubsw m14, m7, m8 ; out8 + paddsw m7, m8 ; out7 + ret +.transpose_round: + punpcklwd m8, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m1, m3 + punpckhwd m1, m3 + punpcklwd m3, m4, m6 + punpckhwd m4, m6 + punpcklwd m6, m5, m7 + punpckhwd m5, m7 + punpcklwd m7, m14, m16 + punpckhwd m14, m16 + punpcklwd m16, m15, m17 + punpckhwd m15, m17 + punpcklwd m17, m19, m21 + punpckhwd m19, m21 + punpckhwd m21, m18, m20 + punpcklwd m18, m20 + punpcklwd m20, m8, m1 + punpckhwd m8, m1 + punpcklwd m1, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m6 + punpckhwd m4, m6 + REPX {pmulhrsw x, m9}, m20, m8, m1, m0 + punpcklwd m6, m7, m15 + punpckhwd m7, m15 + punpcklwd m15, m14, m16 + punpckhwd m14, m16 + REPX {pmulhrsw x, m9}, m2, m3, m5, m4 + punpckhwd m16, m18, m19 + punpcklwd m18, m19 + punpcklwd m19, m21, m17 + punpckhwd m21, m17 + REPX {pmulhrsw x, m9}, m6, m7, m15, m14 + punpcklwd m17, m8, m0 ; a2 a6 aa ae + punpckhwd m8, m0 ; a3 a7 ab af + punpcklwd m0, m20, m1 ; a0 a4 a8 ac + punpckhwd m20, m1 ; a1 a5 a9 ad + REPX {pmulhrsw x, m9}, m16, m18, m19, m21 + punpcklwd m1, m2, m5 ; b0 b4 b8 bc + punpckhwd m2, m5 ; b1 b5 b9 bd + punpcklwd m5, m3, m4 ; b2 b6 ba be + punpckhwd m3, m4 ; b3 b7 bb bf + punpcklwd m4, m6, m15 ; c0 c4 c8 cc + punpckhwd m6, m15 ; c1 c5 c9 cd + punpcklwd m15, m7, m14 ; c2 c6 ca ce + punpckhwd m7, m14 ; c3 c7 cb cf + punpcklwd m14, m18, m19 ; d0 d4 d8 dc + punpckhwd m18, m19 ; d1 d5 d9 dd + punpcklwd m9, m16, m21 ; d2 d6 da de + punpckhwd m16, m21 ; d3 d7 db df + vshufi32x4 m21, m0, m1, q3232 ; a8 ac b8 bc + vinserti32x8 m0, ym1, 1 ; a0 a4 b0 b4 + vinserti32x8 m1, m17, ym5, 1 ; a2 a6 b2 b6 + vshufi32x4 m5, m17, m5, q3232 ; aa ae ba be + vinserti32x8 m17, m8, ym3, 1 ; a3 a7 b3 b7 + vshufi32x4 m19, m8, m3, q3232 ; ab af bb bf + vinserti32x8 m3, m4, ym14, 1 ; c0 c4 d0 d4 + vshufi32x4 m4, m14, q3232 ; c8 cc d8 dc + vinserti32x8 m14, m20, ym2, 1 ; a1 a5 b1 b5 + vshufi32x4 m20, m2, q3232 ; a9 ad b9 bd + vinserti32x8 m2, m6, ym18, 1 ; c1 c5 d1 d5 + vshufi32x4 m6, m18, q3232 ; c9 cd d9 dd + vinserti32x8 m18, m15, ym9, 1 ; c2 c6 d2 d6 + vshufi32x4 m15, m9, q3232 ; ca ce da de + vinserti32x8 m9, m7, ym16, 1 ; c3 c7 d3 d7 + vshufi32x4 m7, m16, q3232 ; cb cf db df + ret + +%macro IDTX_16x32 4 ; src/dst[1-4] + pmulhrsw m%1, m15, [cq+64*%1] + pmulhrsw m%2, m15, [cq+64*%2] + pmulhrsw m%3, m15, [cq+64*%3] + pmulhrsw m%4, m15, [cq+64*%4] + pmulhrsw m18, m16, m%1 + pmulhrsw m19, m16, m%2 + pmulhrsw m20, m16, m%3 + pmulhrsw m21, m16, m%4 + REPX {pmulhrsw x, m17}, m18, m19, m20, m21 + paddsw m%1, m18 + paddsw m%2, m19 + paddsw m%3, m20 + paddsw m%4, m21 +%endmacro + +%macro IDTX_16x32_STORE 2 ; src[1-2] + mova xm17, [dstq+r3*0] + vinserti128 ym17, [dstq+r3*4], 1 + vinserti32x4 m17, [dstq+r3*8], 2 + vinserti32x4 m17, [dstq+r4*8], 3 + mova [cq+64*(%1*2+0)], m18 + mova [cq+64*(%1*2+1)], m18 + punpcklbw m16, m17, m18 + punpckhbw m17, m18 + paddw m16, m%1 + paddw m17, m%2 + packuswb m16, m17 + mova [dstq+r3*0], xm16 + vextracti128 [dstq+r3*4], ym16, 1 + vextracti32x4 [dstq+r3*8], m16, 2 + vextracti32x4 [dstq+r4*8], m16, 3 +%if %1 != 7 + add dstq, strideq +%endif +%endmacro + +cglobal inv_txfm_add_identity_identity_16x32_8bpc, 3, 5, 22, dst, stride, c + vpbroadcastd m15, [pw_2896x8] + vpbroadcastd m16, [pw_1697x16] + vpbroadcastd m17, [pw_16384] + IDTX_16x32 0, 1, 2, 3 + IDTX_16x32 4, 5, 6, 7 + IDTX_16x32 8, 9, 10, 11 + IDTX_16x32 12, 13, 14, 15 + vpbroadcastd m16, [pw_8192] + call .transpose_2x8x8_round + lea r3, [strideq*2] + lea r4, [strideq*3] + pxor m18, m18 + IDTX_16x32_STORE 0, 8 + IDTX_16x32_STORE 1, 9 + IDTX_16x32_STORE 2, 10 + IDTX_16x32_STORE 3, 11 + IDTX_16x32_STORE 4, 12 + IDTX_16x32_STORE 5, 13 + IDTX_16x32_STORE 6, 14 + IDTX_16x32_STORE 7, 15 + RET +ALIGN function_align +.transpose_2x8x8_round: + punpckhwd m17, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m7 + punpcklwd m6, m7 + punpckhwd m7, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckhdq m6, m5, m7 + punpckldq m5, m7 + punpckldq m7, m17, m1 + punpckhdq m17, m1 + REPX {pmulhrsw x, m16}, m0, m2, m3, m4, m5, m7, m6, m17 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m17 + punpcklqdq m6, m17 + punpckhwd m17, m12, m13 + punpcklwd m12, m13 + punpckhwd m13, m8, m9 + punpcklwd m8, m9 + punpckhwd m9, m14, m15 + punpcklwd m14, m15 + punpckhwd m15, m10, m11 + punpcklwd m10, m11 + punpckhdq m11, m8, m10 + punpckldq m8, m10 + punpckldq m10, m12, m14 + punpckhdq m12, m14 + punpckhdq m14, m13, m15 + punpckldq m13, m15 + punpckldq m15, m17, m9 + punpckhdq m17, m9 + REPX {pmulhrsw x, m16}, m8, m10, m11, m12, m13, m15, m14, m17 + punpckhqdq m9, m8, m10 + punpcklqdq m8, m10 + punpcklqdq m10, m11, m12 + punpckhqdq m11, m12 + punpcklqdq m12, m13, m15 + punpckhqdq m13, m15 + punpckhqdq m15, m14, m17 + punpcklqdq m14, m17 + ret + +%macro IDTX_32x16 4 ; dst[1-4] + pmulhrsw m%2, m12, [cq+32*(%1+ 0)] + pmulhrsw m18, m12, [cq+32*(%1+16)] + pmulhrsw m%4, m12, [cq+32*(%3+ 0)] + pmulhrsw m19, m12, [cq+32*(%3+16)] + REPX {paddsw x, x}, m%2, m18, m%4, m19 + mova m%1, m14 + vpermi2q m%1, m%2, m18 + vpermt2q m%2, m16, m18 +%if %3 != 14 + mova m%3, m14 +%endif + vpermi2q m%3, m%4, m19 + vpermt2q m%4, m16, m19 + pmulhrsw m18, m17, m%1 + pmulhrsw m19, m17, m%2 + pmulhrsw m20, m17, m%3 + pmulhrsw m21, m17, m%4 + REPX {paddsw x, x}, m%1, m%2, m%3, m%4 + paddsw m%1, m18 + paddsw m%2, m19 + paddsw m%3, m20 + paddsw m%4, m21 +%endmacro + +%macro IDTX_32x16_STORE 2-3 0 ; src[1-2], 32x32 + mova ym19, [dstq+strideq*0] + vinserti32x8 m19, [dstq+strideq*8], 1 +%if %3 == 0 + mova [cq+64*(%1*2+0)], m20 + mova [cq+64*(%1*2+1)], m20 +%endif + punpcklbw m18, m19, m20 + punpckhbw m19, m20 + paddw m18, m%1 + paddw m19, m%2 + packuswb m18, m19 + mova [dstq+strideq*0], ym18 + vextracti32x8 [dstq+strideq*8], m18, 1 +%if %3 || %1 != 7 + add dstq, strideq +%endif +%endmacro + +cglobal inv_txfm_add_identity_identity_32x16_8bpc, 3, 3, 22, dst, stride, c + vpbroadcastd m12, [pw_2896x8] + movu m14, [permB+7] + vpbroadcastd m17, [pw_1697x16] + psrlq m16, m14, 4 + IDTX_32x16 0, 1, 2, 3 + IDTX_32x16 4, 5, 6, 7 + IDTX_32x16 8, 9, 10, 11 + IDTX_32x16 12, 13, 14, 15 + vpbroadcastd m16, [pw_2048] + call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round + pxor m20, m20 + IDTX_32x16_STORE 0, 8 + IDTX_32x16_STORE 1, 9 + IDTX_32x16_STORE 2, 10 + IDTX_32x16_STORE 3, 11 + IDTX_32x16_STORE 4, 12 + IDTX_32x16_STORE 5, 13 + IDTX_32x16_STORE 6, 14 + IDTX_32x16_STORE 7, 15 + RET + +%macro IDCT_32x32_END 4 ; src, mem, stride[1-2] + pmovzxbw m10, [dstq+%3] + pmovzxbw m11, [r3 +%4] +%if %2 < 8 + paddsw m8, m%2, m%1 + psubsw m9, m%2, m%1 +%else + mova m9, [cq+64*(%2*2-16)] + paddsw m8, m9, m%1 + psubsw m9, m%1 +%endif + pmulhrsw m8, m12 + pmulhrsw m9, m12 +%if %2 >= 8 +%if %2 == 8 + pxor m0, m0 +%endif + mova [cq+64*(%2*2-16)], m0 + mova [cq+64*(%2*2-15)], m0 +%endif + paddw m8, m10 + paddw m9, m11 + packuswb m8, m9 + vpermq m8, m13, m8 + mova [dstq+%3], ym8 + vextracti32x8 [r3 +%4], m8, 1 +%if %2 == 3 || %2 == 7 || %2 == 11 + add dstq, r5 + sub r3, r5 +%endif +%endmacro + +cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + WIN64_SPILL_XMM 30 + cmp eobd, 136 + jb .fast + mova m5, [cq+64*20] + mova m3, [cq+64*12] + mova m1, [cq+64* 4] + mova m7, [cq+64*28] + mova m2, [cq+64* 8] + mova m6, [cq+64*24] + mova m0, [cq+64* 0] + mova m4, [cq+64*16] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + mova m14, [cq+64* 2] + mova m21, [cq+64*30] + mova m18, [cq+64*18] + mova m17, [cq+64*14] + mova m16, [cq+64*10] + mova m19, [cq+64*22] + mova m20, [cq+64*26] + mova m15, [cq+64* 6] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova [cq+64* 0], m14 + mova [cq+64* 2], m15 + mova [cq+64* 4], m16 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64*12], m20 + mova [cq+64*14], m21 + mova m22, [cq+64* 1] + mova m21, [cq+64*31] + mova m14, [cq+64*17] + mova m29, [cq+64*15] + mova m26, [cq+64* 9] + mova m17, [cq+64*23] + mova m18, [cq+64*25] + mova m25, [cq+64* 7] + mova m24, [cq+64* 5] + mova m19, [cq+64*27] + mova m16, [cq+64*21] + mova m27, [cq+64*11] + mova m28, [cq+64*13] + mova m15, [cq+64*19] + mova m20, [cq+64*29] + mova m23, [cq+64* 3] + call .main_oddhalf + vpbroadcastd m10, [o(pw_8192)] + psubsw m13, m0, m29 ; 31 + paddsw m0, m29 ; 0 + psubsw m29, m1, m28 ; 30 + paddsw m1, m28 ; 1 + psubsw m28, m2, m27 ; 29 + paddsw m2, m27 ; 2 + psubsw m27, m3, m26 ; 28 + paddsw m3, m26 ; 3 + psubsw m26, m4, m25 ; 27 + paddsw m4, m25 ; 4 + psubsw m25, m5, m24 ; 26 + paddsw m5, m24 ; 5 + psubsw m24, m6, m23 ; 25 + paddsw m6, m23 ; 6 + psubsw m23, m7, m22 ; 24 + paddsw m7, m22 ; 7 + pxor m9, m9 + punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7 + punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7 + punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3 + REPX {mova [cq+64*x], m9}, 16, 17, 18, 19 + punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7 + punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3 + punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7 + punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3 + REPX {mova [cq+64*x], m9}, 20, 21, 22, 23 + punpckhwd m3, m23, m24 + punpcklwd m23, m24 + punpckhwd m24, m25, m26 + punpcklwd m25, m26 + REPX {mova [cq+64*x], m9}, 24, 25, 26, 27 + punpckhwd m26, m27, m28 + punpcklwd m27, m28 + punpckhwd m28, m29, m13 + punpcklwd m29, m13 + REPX {mova [cq+64*x], m9}, 28, 29, 30, 31 + punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7 + punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5 + punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7 + punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5 + REPX {pmulhrsw x, m10}, m0, m4, m8, m22 + punpckhdq m13, m23, m25 + punpckldq m23, m25 + punpckhdq m25, m27, m29 + punpckldq m27, m29 + REPX {pmulhrsw x, m10}, m13, m23, m25, m27 + punpckhdq m9, m3, m24 + punpckldq m3, m24 + punpckhdq m24, m26, m28 + punpckldq m26, m28 + punpcklqdq m5, m23, m27 ; d00 d08 d16 d24 + punpckhqdq m23, m27 ; d01 d09 d17 d25 + punpckhqdq m27, m13, m25 ; d03 d11 d19 d27 + punpcklqdq m13, m25 ; d02 d10 d18 d26 + punpckhqdq m25, m3, m26 ; d05 d13 d21 d29 + punpcklqdq m3, m26 ; d04 d12 d20 d28 + punpckhqdq m26, m9, m24 ; d07 d15 d23 d31 + punpcklqdq m9, m24 ; d06 d14 d22 d30 + REPX {pmulhrsw x, m10}, m25, m3, m26 + mova [cq+64* 9], m23 + mova [cq+64*11], m27 + mova [cq+64*13], m25 + mova [cq+64*15], m26 + punpckhqdq m24, m8, m22 ; a05 a13 a21 a29 + punpcklqdq m8, m22 ; a04 a12 a20 a28 + punpckhqdq m22, m0, m4 ; a01 a09 a17 a25 + punpcklqdq m0, m4 ; a00 a08 a16 a24 + punpckhqdq m23, m7, m2 ; a03 a11 a19 a27 + punpcklqdq m7, m2 ; a02 a10 a18 a26 + punpckhqdq m25, m6, m1 ; a07 a15 a23 a31 + punpcklqdq m6, m1 ; a06 a14 a22 a30 + mova m2, [cq+64* 0] + mova m11, [cq+64* 2] + mova m12, [cq+64* 4] + mova m29, [cq+64* 6] + mova m27, [cq+64* 8] + mova m26, [cq+64*10] + mova m4, [cq+64*12] + mova m28, [cq+64*14] + psubsw m1, m2, m21 ; 23 + paddsw m2, m21 ; 8 + psubsw m21, m11, m20 ; 22 + paddsw m11, m20 ; 9 + psubsw m20, m12, m19 ; 21 + paddsw m12, m19 ; 10 + psubsw m19, m29, m18 ; 20 + paddsw m29, m18 ; 11 + psubsw m18, m27, m17 ; 19 + paddsw m27, m17 ; 12 + psubsw m17, m26, m16 ; 18 + paddsw m26, m16 ; 13 + paddsw m16, m4, m15 ; 14 + psubsw m4, m15 ; 17 + pmulhrsw m15, m6, m10 + psubsw m6, m28, m14 ; 16 + paddsw m28, m14 ; 15 + pmulhrsw m14, m7, m10 + punpcklwd m7, m6, m4 + punpckhwd m6, m4 + punpckhwd m4, m17, m18 + punpcklwd m17, m18 + punpckhwd m18, m19, m20 + punpcklwd m19, m20 + punpckhwd m20, m21, m1 + punpcklwd m21, m1 + punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7 + punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3 + punpckhwd m11, m12, m29 ; k4 l4 k5 l5 k6 l6 k7 l7 + punpcklwd m12, m29 ; k0 l0 k1 l1 k2 l2 k3 l3 + punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7 + punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3 + punpckhwd m26, m16, m28 ; o4 p4 o5 p5 o6 p6 o7 p7 + punpcklwd m16, m28 ; o0 p0 o1 p1 o2 p2 o3 p3 + pmulhrsw m23, m10 + pmulhrsw m25, m10 + punpckhdq m28, m2, m12 ; i2 j2 k2 l2 i3 j3 k3 l3 + punpckldq m2, m12 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhdq m12, m27, m16 ; m2 n2 o2 p2 m3 n3 o3 p3 + punpckldq m27, m16 ; m0 n0 o0 p0 m1 n1 o1 p1 + REPX {pmulhrsw x, m10}, m28, m2, m12, m27 + punpckhdq m16, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7 + punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5 + punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7 + punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5 + REPX {pmulhrsw x, m10}, m16, m1, m11, m29 + punpckhdq m26, m19, m21 + punpckldq m19, m21 + punpckhdq m21, m6, m4 + punpckldq m6, m4 + REPX {pmulhrsw x, m10}, m26, m19, m21, m6 + punpckhdq m4, m18, m20 + punpckldq m18, m20 + punpckhdq m20, m7, m17 + punpckldq m7, m17 + REPX {pmulhrsw x, m10}, m4, m18, m20, m7 + punpcklqdq m17, m28, m12 ; b02 b10 b18 b26 + punpckhqdq m28, m12 ; b03 b11 b19 b27 + punpckhqdq m12, m2, m27 ; b01 b09 b17 b25 + punpcklqdq m2, m27 ; b00 b08 b16 b24 + punpckhqdq m27, m1, m29 ; b05 b13 b21 b29 + punpcklqdq m1, m29 ; b04 b12 b20 b28 + punpckhqdq m29, m16, m11 ; b07 b15 b23 b31 + punpcklqdq m16, m11 ; b06 b14 b22 b30 + mova [cq+64* 1], m12 + mova [cq+64* 3], m28 + mova [cq+64* 5], m27 + mova [cq+64* 7], m29 + punpckhqdq m27, m20, m26 ; c03 c11 c19 c27 + punpcklqdq m20, m26 ; c02 c10 c18 c26 + punpckhqdq m26, m7, m19 ; c01 c09 c17 c25 + punpcklqdq m7, m19 ; c00 c08 c16 c24 + punpckhqdq m28, m6, m18 ; c05 c13 c21 c29 + punpcklqdq m6, m18 ; c04 c12 c20 c28 + punpckhqdq m29, m21, m4 ; c07 c15 c23 c31 + punpcklqdq m21, m4 ; c06 c14 c22 c30 + pmulhrsw m19, m9, m10 + vshufi32x4 m4, m0, m2, q3232 ; a16 a24 b16 b24 + vinserti32x8 m0, ym2, 1 ; a00 a08 b00 b08 + vshufi32x4 m2, m7, m5, q3232 ; c16 c24 d16 d24 + vinserti32x8 m7, ym5, 1 ; c00 c08 d00 d08 + vshufi32x4 m5, m8, m1, q3232 ; a20 a28 b20 b28 + vinserti32x8 m1, m8, ym1, 1 ; a04 a12 b04 b12 + vshufi32x4 m8, m6, m3, q3232 ; c20 c28 d20 d28 + vinserti32x8 m6, ym3, 1 ; c04 c12 d04 d12 + vshufi32x4 m3, m1, m6, q3131 ; 12 + vshufi32x4 m1, m6, q2020 ; 4 + vshufi32x4 m6, m4, m2, q3131 ; 24 + vshufi32x4 m4, m2, q2020 ; 16 + vshufi32x4 m2, m0, m7, q3131 ; 8 + vshufi32x4 m0, m7, q2020 ; 0 + vshufi32x4 m7, m5, m8, q3131 ; 28 + vshufi32x4 m5, m8, q2020 ; 20 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + vshufi32x4 m18, m14, m17, q3232 ; a18 a26 b18 b26 + vinserti32x8 m14, ym17, 1 ; a02 a10 b02 b10 + vshufi32x4 m17, m20, m13, q3232 ; c18 c26 d18 d26 + vinserti32x8 m20, ym13, 1 ; c02 c10 d02 d10 + vshufi32x4 m13, m21, m19, q3232 ; c22 c30 d22 d30 + vinserti32x8 m21, ym19, 1 ; c06 c14 d06 d14 + vshufi32x4 m19, m15, m16, q3232 ; a22 a30 b22 b30 + vinserti32x8 m15, ym16, 1 ; a06 a14 b06 b14 + vshufi32x4 m16, m14, m20, q3131 ; 10 + vshufi32x4 m14, m20, q2020 ; 2 + vshufi32x4 m20, m18, m17, q3131 ; 26 + vshufi32x4 m18, m17, q2020 ; 18 + vshufi32x4 m17, m15, m21, q3131 ; 14 + vshufi32x4 m15, m21, q2020 ; 6 + vshufi32x4 m21, m19, m13, q3131 ; 30 + vshufi32x4 m19, m13, q2020 ; 22 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova [cq+64* 0], m14 + mova [cq+64* 2], m15 + mova [cq+64* 4], m16 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64*12], m20 + mova [cq+64*14], m21 + mova m15, [cq+64* 1] + mova m16, [cq+64* 3] + mova m17, [cq+64* 5] + mova m19, [cq+64* 7] + mova m20, [cq+64* 9] + mova m21, [cq+64*11] + mova m13, [cq+64*13] + mova m18, [cq+64*15] + vshufi32x4 m14, m22, m15, q3232 ; a17 a25 b17 b25 + vinserti32x8 m22, ym15, 1 ; a01 a09 b01 b09 + vshufi32x4 m15, m23, m16, q3232 ; a19 a27 b19 b27 + vinserti32x8 m23, ym16, 1 ; a03 a11 b03 b11 + vshufi32x4 m16, m24, m17, q3232 ; a21 a29 b21 b29 + vinserti32x8 m24, ym17, 1 ; a05 a13 b05 b13 + vshufi32x4 m17, m25, m19, q3232 ; a23 a31 b23 b31 + vinserti32x8 m25, ym19, 1 ; a07 a15 b07 b15 + vinserti32x8 m8, m26, ym20, 1 ; c01 c09 d01 d09 + vshufi32x4 m26, m20, q3232 ; c17 c25 d17 d25 + vinserti32x8 m9, m27, ym21, 1 ; c03 c11 d03 d11 + vshufi32x4 m27, m21, q3232 ; c19 c27 d19 d27 + vinserti32x8 m11, m28, ym13, 1 ; c05 c13 d05 d13 + vshufi32x4 m28, m13, q3232 ; c21 c29 d21 d29 + vinserti32x8 m12, m29, ym18, 1 ; c07 c15 d07 d15 + vshufi32x4 m29, m18, q3232 ; c23 c31 d23 d31 + vshufi32x4 m18, m14, m26, q3131 ; 25 + vshufi32x4 m14, m26, q2020 ; 17 + vshufi32x4 m19, m15, m27, q3131 ; 27 + vshufi32x4 m15, m27, q2020 ; 19 + vshufi32x4 m20, m16, m28, q3131 ; 29 + vshufi32x4 m16, m28, q2020 ; 21 + vshufi32x4 m21, m17, m29, q3131 ; 31 + vshufi32x4 m17, m29, q2020 ; 23 + vshufi32x4 m26, m22, m8, q3131 ; 9 + vshufi32x4 m22, m8, q2020 ; 1 + vshufi32x4 m27, m23, m9, q3131 ; 11 + vshufi32x4 m23, m9, q2020 ; 3 + vshufi32x4 m28, m24, m11, q3131 ; 13 + vshufi32x4 m24, m11, q2020 ; 5 + vshufi32x4 m29, m25, m12, q3131 ; 15 + vshufi32x4 m25, m12, q2020 ; 7 + call .main_oddhalf + jmp .end +.fast: ; bottom/right halves are zero + mova m14, [o(dup16_perm)] + pmovzxwd m9, [cq+64* 0] + pmovzxwd m6, [cq+64* 8] + vpermb m8, m14, [cq+64* 2] + vpermb ym0, ym14, [cq+64*14] + vpermb ym5, ym14, [cq+64*10] + vpermb m1, m14, [cq+64* 6] + vpermb m7, m14, [cq+64* 4] + vpermb ym3, ym14, [cq+64*12] + pslld m9, 16 + pslld m6, 16 + call m(idct_16x16_internal_8bpc).main_fast + vpermb m21, m14, [cq+64* 1] + vpermb ym17, ym14, [cq+64*15] + vpermb ym20, ym14, [cq+64* 9] + vpermb m15, m14, [cq+64* 7] + vpermb m18, m14, [cq+64* 5] + vpermb ym16, ym14, [cq+64*11] + vpermb ym19, ym14, [cq+64*13] + vpermb m14, m14, [cq+64* 3] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m9, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round + vshufi32x4 m22, m14, m2, q2020 ; 1 + vshufi32x4 m24, m14, m2, q3131 ; 5 + vshufi32x4 m23, m17, m9, q2020 ; 3 + vshufi32x4 m25, m17, m9, q3131 ; 7 + vshufi32x4 m16, m5, m15, q2020 ; 10 + vshufi32x4 m17, m5, m15, q3131 ; 14 + vshufi32x4 m14, m1, m18, q2020 ; 2 + vshufi32x4 m15, m1, m18, q3131 ; 6 + vshufi32x4 m1, m0, m3, q3131 ; 4 + vshufi32x4 m0, m3, q2020 ; 0 + vshufi32x4 m3, m21, m4, q3131 ; 12 + vshufi32x4 m2, m21, m4, q2020 ; 8 + vshufi32x4 m26, m20, m6, q2020 ; 9 + vshufi32x4 m28, m20, m6, q3131 ; 13 + vshufi32x4 m27, m19, m7, q2020 ; 11 + vshufi32x4 m29, m19, m7, q3131 ; 15 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + mova [cq+64* 0], m14 + mova [cq+64* 2], m15 + mova [cq+64* 4], m16 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64*12], m20 + mova [cq+64*14], m21 + call .main_oddhalf_fast +.end: + lea r4, [strideq*3] + vpbroadcastd m12, [o(pw_2048)] + movshdup m13, [o(permD)] + lea r3, [dstq+r4*8] + lea r5, [strideq+r4] ; stride*4 + add r3, r5 ; dst+stride*28 + IDCT_32x32_END 29, 0, strideq*0, r4 + IDCT_32x32_END 28, 1, strideq*1, strideq*2 + IDCT_32x32_END 27, 2, strideq*2, strideq*1 + IDCT_32x32_END 26, 3, r4 , strideq*0 + IDCT_32x32_END 25, 4, strideq*0, r4 + IDCT_32x32_END 24, 5, strideq*1, strideq*2 + IDCT_32x32_END 23, 6, strideq*2, strideq*1 + IDCT_32x32_END 22, 7, r4 , strideq*0 + IDCT_32x32_END 21, 8, strideq*0, r4 + IDCT_32x32_END 20, 9, strideq*1, strideq*2 + IDCT_32x32_END 19, 10, strideq*2, strideq*1 + IDCT_32x32_END 18, 11, r4 , strideq*0 + IDCT_32x32_END 17, 12, strideq*0, r4 + IDCT_32x32_END 16, 13, strideq*1, strideq*2 + IDCT_32x32_END 15, 14, strideq*2, strideq*1 + IDCT_32x32_END 14, 15, r4 , strideq*0 + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly2 +ALIGN function_align +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero + vpbroadcastd m21, [o(pw_4091x8)] + vpbroadcastd m8, [o(pw_201x8)] + vpbroadcastd m18, [o(pw_m1380x8)] + vpbroadcastd m9, [o(pw_3857x8)] + vpbroadcastd m19, [o(pw_3973x8)] + vpbroadcastd m11, [o(pw_995x8)] + vpbroadcastd m28, [o(pw_m601x8)] + vpbroadcastd m12, [o(pw_4052x8)] + pmulhrsw m21, m22 ; t31a + pmulhrsw m22, m8 ; t16a + pmulhrsw m18, m25 ; t19a + pmulhrsw m25, m9 ; t28a + pmulhrsw m19, m24 ; t27a + pmulhrsw m24, m11 ; t20a + pmulhrsw m28, m23 ; t23a + pmulhrsw m23, m12 ; t24a + mova m15, m21 + mova m8, m22 + mova m14, m18 + mova m27, m25 + mova m29, m19 + mova m26, m24 + mova m16, m28 + mova m20, m23 + jmp .main3 +ALIGN function_align +cglobal_label .main_oddhalf_fast ; bottom half is zero + vpbroadcastd m21, [o(pw_4091x8)] + vpbroadcastd m8, [o(pw_201x8)] + vpbroadcastd m14, [o(pw_m2751x8)] + vpbroadcastd m9, [o(pw_3035x8)] + vpbroadcastd m17, [o(pw_3703x8)] + vpbroadcastd m11, [o(pw_1751x8)] + vpbroadcastd m18, [o(pw_m1380x8)] + vpbroadcastd m12, [o(pw_3857x8)] + pmulhrsw m21, m22 ; t31a + vpbroadcastd m19, [o(pw_3973x8)] + pmulhrsw m22, m8 ; t16a + vpbroadcastd m8, [o(pw_995x8)] + pmulhrsw m14, m29 ; t30a + vpbroadcastd m16, [o(pw_m2106x8)] + pmulhrsw m29, m9 ; t17a + vpbroadcastd m9, [o(pw_3513x8)] + pmulhrsw m17, m26 ; t29a + vpbroadcastd m15, [o(pw_3290x8)] + pmulhrsw m26, m11 ; t18a + vpbroadcastd m11, [o(pw_2440x8)] + pmulhrsw m18, m25 ; t19a + vpbroadcastd m20, [o(pw_m601x8)] + pmulhrsw m25, m12 ; t28a + vpbroadcastd m12, [o(pw_4052x8)] + pmulhrsw m19, m24 ; t27a + pmulhrsw m24, m8 ; t20a + pmulhrsw m16, m27 ; t21a + pmulhrsw m27, m9 ; t26a + pmulhrsw m15, m28 ; t25a + pmulhrsw m28, m11 ; t22a + pmulhrsw m20, m23 ; t23a + pmulhrsw m23, m12 ; t24a + jmp .main2 +ALIGN function_align +cglobal_label .main_oddhalf + ITX_MULSUB_2W 22, 21, 8, 9, 10, 201, 4091 ; t16a, t31a + ITX_MULSUB_2W 14, 29, 8, 9, 10, 3035, 2751 ; t17a, t30a + ITX_MULSUB_2W 26, 17, 8, 9, 10, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2W 18, 25, 8, 9, 10, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2W 24, 19, 8, 9, 10, 995, 3973 ; t20a, t27a + ITX_MULSUB_2W 16, 27, 8, 9, 10, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2W 28, 15, 8, 9, 10, 2440, 3290 ; t22a, t25a + ITX_MULSUB_2W 20, 23, 8, 9, 10, 4052, 601 ; t23a, t24a +.main2: + psubsw m8, m22, m14 ; t17 + paddsw m22, m14 ; t16 + paddsw m14, m18, m26 ; t19 + psubsw m18, m26 ; t18 + psubsw m26, m24, m16 ; t21 + paddsw m24, m16 ; t20 + psubsw m16, m20, m28 ; t22 + paddsw m28, m20 ; t23 + psubsw m20, m23, m15 ; t25 + paddsw m23, m15 ; t24 + psubsw m15, m21, m29 ; t30 + paddsw m21, m29 ; t31 + psubsw m29, m19, m27 ; t26 + paddsw m19, m27 ; t27 + paddsw m27, m25, m17 ; t28 + psubsw m25, m17 ; t29 +.main3: + ITX_MULSUB_2W 15, 8, 9, 17, 10, 799, 4017 ; t17a, t30a + ITX_MULSUB_2W 25, 18, 9, 17, 10, m4017, 799 ; t18a, t29a + ITX_MULSUB_2W 29, 26, 9, 17, 10, 3406, 2276 ; t21a, t26a + ITX_MULSUB_2W 20, 16, 9, 17, 10, m2276, 3406 ; t22a, t25a + vpbroadcastd m12, [o(pw_m3784_1567)] + vpbroadcastd m11, [o(pw_1567_3784)] + psubsw m17, m21, m27 ; t28a + paddsw m21, m27 ; t31a + psubsw m27, m15, m25 ; t18 + paddsw m15, m25 ; t17 + psubsw m25, m20, m29 ; t21 + paddsw m20, m29 ; t22 + psubsw m29, m8, m18 ; t29 + paddsw m8, m18 ; t30 + psubsw m18, m22, m14 ; t19a + paddsw m22, m14 ; t16a + psubsw m14, m28, m24 ; t20a + paddsw m24, m28 ; t23a + paddsw m28, m16, m26 ; t25 + psubsw m16, m26 ; t26 + psubsw m26, m23, m19 ; t27a + paddsw m23, m19 ; t24a + ITX_MULSUB_2W 29, 27, 9, 19, 10, 11, 12 ; t18a, t29a + ITX_MULSUB_2W 17, 18, 9, 19, 10, 11, 12 ; t19, t28 + vpbroadcastd m11, [o(pw_m1567_m3784)] + ITX_MULSUB_2W 16, 25, 9, 19, 10, 12, 11 ; t21a, t26a + ITX_MULSUB_2W 26, 14, 9, 19, 10, 12, 11 ; t20, t27 + vpbroadcastd m12, [o(pw_m2896_2896)] + vpbroadcastd m11, [o(pw_2896_2896)] + psubsw m19, m27, m25 ; t26 + paddsw m27, m25 ; t29 + psubsw m25, m17, m26 ; t20a + paddsw m17, m26 ; t19a + paddsw m26, m18, m14 ; t28a + psubsw m18, m14 ; t27a + paddsw m14, m22, m24 ; t16 + psubsw m22, m24 ; t23 + psubsw m24, m29, m16 ; t21 + paddsw m16, m29 ; t18 + paddsw m29, m21, m23 ; t31 + psubsw m21, m23 ; t24 + psubsw m23, m15, m20 ; t22a + paddsw m15, m20 ; t17a + psubsw m20, m8, m28 ; t25a + paddsw m28, m8 ; t30a + ITX_MULSUB_2W 18, 25, 8, 9, 10, 11, 12 ; t20, t27 + ITX_MULSUB_2W 19, 24, 8, 9, 10, 11, 12 ; t21a, t26a + ITX_MULSUB_2W 21, 22, 8, 9, 10, 11, 12 ; t23a, t24a + ITX_MULSUB_2W 20, 23, 8, 9, 10, 11, 12 ; t22, t25 + ret + +%macro IDTX_32x32 2 ; dst[1-2] + vmovdqa32 ym%1, [cq+64*(%1+ 0)] ; force EVEX encoding, which + vmovdqa32 ym17, [cq+64*(%1+16)] ; reduces code size due to + vmovdqa32 ym%2, [cq+64*(%2+ 0)] ; compressed displacements + vmovdqa32 ym18, [cq+64*(%2+16)] + vpermt2q m%1, m21, m17 + vpermt2q m%2, m21, m18 +%endmacro + +cglobal inv_txfm_add_identity_identity_32x32_8bpc, 3, 3, 22, dst, stride, c + movu m21, [permB+7] + vpbroadcastd m16, [pw_8192] + pxor m20, m20 +.loop: + IDTX_32x32 0, 1 + IDTX_32x32 2, 3 + IDTX_32x32 4, 5 + IDTX_32x32 6, 7 + IDTX_32x32 8, 9 + IDTX_32x32 10, 11 + IDTX_32x32 12, 13 + IDTX_32x32 14, 15 + call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round + IDTX_32x16_STORE 0, 8, 1 + IDTX_32x16_STORE 1, 9, 1 + IDTX_32x16_STORE 2, 10, 1 + IDTX_32x16_STORE 3, 11, 1 + IDTX_32x16_STORE 4, 12, 1 + IDTX_32x16_STORE 5, 13, 1 + IDTX_32x16_STORE 6, 14, 1 + IDTX_32x16_STORE 7, 15, 1 + lea dstq, [dstq+strideq*8] + btc cq, 5 + jnc .loop + mov r0d, 8 +.zero_loop: + mova [cq+64*0], m20 + mova [cq+64*1], m20 + mova [cq+64*2], m20 + mova [cq+64*3], m20 + add cq, 64*4 + dec r0d + jg .zero_loop + RET + +cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + WIN64_SPILL_XMM 30 + cmp eobd, 151 + jb .fast + mova m5, [cq+64*10] + mova m3, [cq+64* 6] + mova m1, [cq+64* 2] + mova m7, [cq+64*14] + mova m2, [cq+64* 4] + mova m6, [cq+64*12] + mova m0, [cq+64* 0] + mova m4, [cq+64* 8] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + mova m14, [cq+64* 1] + mova m21, [cq+64*15] + mova m18, [cq+64* 9] + mova m17, [cq+64* 7] + mova m16, [cq+64* 5] + mova m19, [cq+64*11] + mova m20, [cq+64*13] + mova m15, [cq+64* 3] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + vpbroadcastd m9, [o(pw_8192)] +%macro TRANSPOSE_8x4_ROUND 4 + punpckhwd m8, m%3, m%4 ; c4 d4 c5 d5 c6 d6 c7 d7 + punpcklwd m%3, m%4 ; c0 d0 c1 d1 c2 d2 c3 d3 + punpckhwd m%4, m%1, m%2 ; a4 b4 a5 b5 a6 b6 a7 b7 + punpcklwd m%1, m%2 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpckhdq m%2, m%1, m%3 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m%1, m%3 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckldq m%3, m%4, m8 ; a4 b4 c4 d4 a5 b5 c5 d5 + punpckhdq m%4, m8 ; a6 b6 c6 d6 a7 b7 c7 d7 + REPX {pmulhrsw x, m9}, m%2, m%1, m%3, m%4 +%endmacro + TRANSPOSE_8x4_ROUND 0, 1, 2, 3 + TRANSPOSE_8x4_ROUND 4, 5, 6, 7 + TRANSPOSE_8x4_ROUND 14, 15, 16, 17 + TRANSPOSE_8x4_ROUND 18, 19, 20, 21 + vinserti32x8 m26, m0, ym4, 1 ; a0 a4 b0 b4 + vshufi32x4 m0, m4, q3232 ; a8 a12 b8 b12 + vinserti32x8 m27, m1, ym5, 1 ; a1 a5 b1 b5 + vshufi32x4 m1, m5, q3232 ; a9 a13 b9 b13 + vinserti32x8 m28, m2, ym6, 1 ; a2 a6 b2 b6 + vshufi32x4 m2, m6, q3232 ; a10 a14 b10 b14 + vinserti32x8 m29, m3, ym7, 1 ; a3 a7 b3 b7 + vshufi32x4 m8, m3, m7, q3232 ; a11 a15 b11 b15 + vinserti32x8 m4, m14, ym18, 1 ; c0 c4 d0 d4 + vshufi32x4 m14, m18, q3232 ; c8 c12 d8 d12 + vinserti32x8 m5, m15, ym19, 1 ; c1 c5 d1 d5 + vshufi32x4 m15, m19, q3232 ; c9 c13 d9 d13 + vinserti32x8 m6, m16, ym20, 1 ; c2 c6 d2 d6 + vshufi32x4 m16, m20, q3232 ; c10 c14 d10 d14 + vinserti32x8 m7, m17, ym21, 1 ; c3 c7 d3 d7 + vshufi32x4 m17, m21, q3232 ; c11 c15 d11 d15 + vshufi32x4 m22, m26, m4, q2020 ; 0 1 + vshufi32x4 m26, m4, q3131 ; 8 9 + vshufi32x4 m23, m27, m5, q2020 ; 2 3 + vshufi32x4 m27, m5, q3131 ; 10 11 + vshufi32x4 m24, m28, m6, q2020 ; 4 5 + vshufi32x4 m28, m6, q3131 ; 12 13 + vshufi32x4 m25, m29, m7, q2020 ; 6 7 + vshufi32x4 m29, m7, q3131 ; 14 15 + vshufi32x4 m4, m0, m14, q2020 ; 16 17 + vshufi32x4 m3, m0, m14, q3131 ; 24 25 + vshufi32x4 m20, m1, m15, q2020 ; 18 19 + vshufi32x4 m19, m1, m15, q3131 ; 26 27 + vshufi32x4 m5, m2, m16, q2020 ; 20 21 + vshufi32x4 m0, m2, m16, q3131 ; 28 29 + vshufi32x4 m16, m8, m17, q2020 ; 22 23 + vshufi32x4 m17, m8, m17, q3131 ; 30 31 + pxor m6, m6 + mova [cq+64* 0], m4 + mova [cq+64* 2], m5 + mova [cq+64* 4], m3 + mova [cq+64* 6], m0 + punpcklwd m8, m24, m24 ; 4 + punpcklwd m0, m0 ; 28 + punpcklwd m5, m5 ; 20 + punpcklwd m1, m28, m28 ; 12 + punpcklwd m7, m26, m26 ; 8 + punpcklwd m3, m3 ; 24 + punpcklwd m9, m6, m22 ; __ 0 + punpcklwd m6, m4 ; __ 16 + call m(idct_16x16_internal_8bpc).main_fast3 + mova [cq+64* 1], m20 + mova [cq+64* 3], m16 + mova [cq+64* 5], m19 + mova [cq+64* 7], m17 + punpcklwd m21, m23, m23 ; 2 + punpcklwd m17, m17 ; 30 + punpcklwd m20, m20 ; 18 + punpcklwd m15, m29, m29 ; 14 + punpcklwd m18, m27, m27 ; 10 + punpcklwd m16, m16 ; 22 + punpcklwd m19, m19 ; 26 + punpcklwd m14, m25, m25 ; 6 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova [cq+64* 8], m14 + mova [cq+64* 9], m15 + mova [cq+64*10], m16 + mova [cq+64*11], m17 + mova [cq+64*12], m18 + mova [cq+64*13], m19 + mova [cq+64*14], m20 + mova [cq+64*15], m21 + mova m21, [cq+64* 7] + mova m14, [cq+64* 0] + mova m17, [cq+64* 3] + mova m18, [cq+64* 4] + mova m19, [cq+64* 5] + mova m16, [cq+64* 2] + mova m15, [cq+64* 1] + mova m20, [cq+64* 6] + REPX {punpckhwd x, x}, m22, m21, m14, m29, m26, m17, m18, m25, \ + m24, m19, m16, m27, m28, m15, m20, m23 + call .main_oddhalf + jmp .end +.fast: ; right half is zero + mova ym8, [cq+64*15] + vinserti32x8 m8, [cq+64* 1], 1 + mova m2, [o(int16_perm)] + mova ym9, [cq+64* 8] + vinserti32x8 m9, [cq+64* 0], 1 + mova ym0, [cq+64* 7] + vinserti32x8 m0, [cq+64* 9], 1 + mova ym7, [cq+64*14] + vinserti32x8 m7, [cq+64* 2], 1 + mova ym1, [cq+64* 3] + vinserti32x8 m1, [cq+64*13], 1 + mova ym3, [cq+64* 6] + vinserti32x8 m3, [cq+64*10], 1 + mova ym5, [cq+64*11] + vinserti32x8 m5, [cq+64* 5], 1 + mova ym6, [cq+64*12] + vinserti32x8 m6, [cq+64* 4], 1 + REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6 + call m(idct_16x16_internal_8bpc).main2 + vbroadcasti32x4 m8, [o(int_shuf3)] + vbroadcasti32x4 m9, [o(int_shuf4)] + vpbroadcastd m11, [o(pw_8192)] + pshufb m0, m8 + pshufb m1, m9 + pshufb m2, m8 + pshufb m3, m9 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 + pshufb m4, m8 + pshufb m5, m9 + pshufb m6, m8 + pshufb m7, m9 + REPX {pmulhrsw x, m11}, m4, m5, m6, m7 + punpckhdq m28, m0, m1 + punpckldq m0, m1 + punpckhdq m27, m2, m3 + punpckldq m2, m3 + punpckhdq m22, m4, m5 + punpckldq m4, m5 + punpckhdq m23, m6, m7 + punpckldq m6, m7 + vinserti32x8 m14, m0, ym2, 1 + vshufi32x4 m15, m0, m2, q3232 + vinserti32x8 m2, m4, ym6, 1 + vshufi32x4 m4, m6, q3232 + vshufi32x4 m21, m14, m2, q2020 ; 0 2 + vshufi32x4 m14, m2, q3131 ; 4 6 + vshufi32x4 m18, m15, m4, q2020 ; 8 10 + vshufi32x4 m15, m4, q3131 ; 12 14 + pxor m9, m9 + punpcklwd m8, m14, m14 ; 4 + punpcklwd m1, m15, m15 ; 12 + punpcklwd m7, m18, m18 ; 8 + punpcklwd m9, m21 ; __ 0 + call m(idct_16x16_internal_8bpc).main_fast4 + punpckhwd m21, m21 ; 2 + punpckhwd m15, m15 ; 14 + punpckhwd m18, m18 ; 10 + punpckhwd m14, m14 ; 6 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + vinserti32x8 m24, m28, ym27, 1 + vshufi32x4 m28, m27, q3232 + vinserti32x8 m27, m22, ym23, 1 + vshufi32x4 m22, m23, q3232 + vshufi32x4 m23, m24, m27, q2020 ; 1 3 + vshufi32x4 m24, m27, q3131 ; 5 7 + vshufi32x4 m27, m28, m22, q2020 ; 9 11 + vshufi32x4 m28, m22, q3131 ; 13 15 + punpcklwd m22, m23, m23 ; 1 + punpckhwd m29, m28, m28 ; 15 + punpcklwd m26, m27, m27 ; 9 + punpckhwd m25, m24, m24 ; 7 + mova [cq+64* 8], m14 + mova [cq+64* 9], m15 + mova [cq+64*10], m16 + mova [cq+64*11], m17 + punpcklwd m24, m24 ; 5 + punpckhwd m27, m27 ; 11 + punpcklwd m28, m28 ; 13 + punpckhwd m23, m23 ; 3 + mova [cq+64*12], m18 + mova [cq+64*13], m19 + mova [cq+64*14], m20 + mova [cq+64*15], m21 + call .main_oddhalf_fast +.end: + imul r6, strideq, 60 + mova m10, [o(end_16x32p)] + vpbroadcastd m11, [o(pw_2048)] + lea r3, [strideq*3] + pxor m12, m12 + add r6, dstq ; dst+stride*60 + psrldq m13, m10, 1 + lea r4, [strideq+r3] ; stride*4 +%macro IDCT_16x64_END 3 ; idct32, idct64, tmp +%if %1 & 1 + %define %%s0 r3 + %define %%s1 strideq*2 + %define %%s2 strideq*1 + %define %%s3 strideq*0 +%else + %define %%s0 strideq*0 + %define %%s1 strideq*1 + %define %%s2 strideq*2 + %define %%s3 r3 +%if %1 + add dstq, r4 + sub r6, r4 +%endif +%endif +%if %1 < 8 + pmulhrsw m8, m11, m%1 + pmulhrsw m9, m11, m%2 +%else + mova m9, [cq+64*%1] + paddsw m8, m9, m%2 ; out 0+n, 1+n + psubsw m9, m%2 ; out 63-n, 62-n + pmulhrsw m8, m11 + pmulhrsw m9, m11 +%endif + mova xm29, [dstq+%%s0] + vinserti128 ym29, [dstq+%%s1], 1 + mova xm%3, [r6 +%%s3] + vinserti128 ym%3, [r6 +%%s2], 1 + vpermb m29, m10, m29 + vpermb m%3, m10, m%3 + mova [cq+64*%1], m12 + paddw m29, m8 + paddw m%3, m9 + packuswb m29, m%3 + vpermd m29, m13, m29 + mova [dstq+%%s0], xm29 + vextracti128 [dstq+%%s1], ym29, 1 + vextracti32x4 [r6 +%%s2], m29, 2 + vextracti32x4 [r6 +%%s3], m29, 3 +%endmacro + IDCT_16x64_END 0, 29, 0 + IDCT_16x64_END 1, 28, 28 + IDCT_16x64_END 2, 27, 28 + IDCT_16x64_END 3, 26, 28 + IDCT_16x64_END 4, 25, 28 + IDCT_16x64_END 5, 24, 28 + IDCT_16x64_END 6, 23, 28 + IDCT_16x64_END 7, 22, 28 + IDCT_16x64_END 8, 21, 28 + IDCT_16x64_END 9, 20, 28 + IDCT_16x64_END 10, 19, 28 + IDCT_16x64_END 11, 18, 28 + IDCT_16x64_END 12, 17, 28 + IDCT_16x64_END 13, 16, 28 + IDCT_16x64_END 14, 15, 28 + IDCT_16x64_END 15, 14, 28 + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 64 + imul r6d, 181 + add r6d, 128+512 + sar r6d, 8+2 + jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 +ALIGN function_align +cglobal_label .main_oddhalf_fast ; bottom three-quarters are zero + vpbroadcastd m8, [o(pw_101_4095x8)] + vpbroadcastd m21, [o(pw_m1474_3822x8)] + vpbroadcastd m14, [o(pw_897_3996x8)] + vpbroadcastd m17, [o(pw_m700_4036x8)] + vpbroadcastd m18, [o(pw_501_4065x8)] + vpbroadcastd m19, [o(pw_m1092_3948x8)] + vpbroadcastd m16, [o(pw_1285_3889x8)] + vpbroadcastd m15, [o(pw_m301_4085x8)] + pmulhrsw m8, m22 ; t32a t63a + pmulhrsw m21, m29 ; t35a t60a + pmulhrsw m14, m26 ; t36a t59a + pmulhrsw m17, m25 ; t39a t56 + pmulhrsw m18, m24 ; t40a t55a + pmulhrsw m19, m27 ; t43a t52a + pmulhrsw m16, m28 ; t44a t51a + pmulhrsw m15, m23 ; t47a t48a + mova m22, m8 + mova m29, m21 + mova m26, m14 + mova m25, m17 + mova m24, m18 + mova m27, m19 + mova m28, m16 + mova m20, m15 + jmp .main_oddhalf2 +ALIGN function_align +cglobal_label .main_oddhalf + vpbroadcastd m8, [o(pw_101_4095x8)] + vpbroadcastd m9, [o(pw_m2824_2967x8)] + vpbroadcastd m11, [o(pw_1660_3745x8)] + vpbroadcastd m12, [o(pw_m1474_3822x8)] + pmulhrsw m22, m8 ; t32a t63a + vpbroadcastd m8, [o(pw_897_3996x8)] + pmulhrsw m21, m9 ; t33a t62a + vpbroadcastd m9, [o(pw_m2191_3461x8)] + pmulhrsw m14, m11 ; t34a t61a + vpbroadcastd m11, [o(pw_2359_3349x8)] + pmulhrsw m29, m12 ; t35a t60a + vpbroadcastd m12, [o(pw_m700_4036x8)] + pmulhrsw m26, m8 ; t36a t59a + vpbroadcastd m8, [o(pw_501_4065x8)] + pmulhrsw m17, m9 ; t37a t58a + vpbroadcastd m9, [o(pw_m2520_3229x8)] + pmulhrsw m18, m11 ; t38a t57a + vpbroadcastd m11, [o(pw_2019_3564x8)] + pmulhrsw m25, m12 ; t39a t56a + vpbroadcastd m12, [o(pw_m1092_3948x8)] + pmulhrsw m24, m8 ; t40a t55a + vpbroadcastd m8, [o(pw_1285_3889x8)] + pmulhrsw m19, m9 ; t41a t54a + vpbroadcastd m9, [o(pw_m1842_3659x8)] + pmulhrsw m16, m11 ; t42a t53a + vpbroadcastd m11, [o(pw_2675_3102x8)] + pmulhrsw m27, m12 ; t43a t52a + vpbroadcastd m12, [o(pw_m301_4085x8)] + pmulhrsw m28, m8 ; t44a t51a + pmulhrsw m15, m9 ; t45a t50a + pmulhrsw m20, m11 ; t46a t49a + pmulhrsw m23, m12 ; t47a t48a + psubsw m8, m22, m21 ; t33 t62 + paddsw m22, m21 ; t32 t63 + psubsw m21, m29, m14 ; t34 t61 + paddsw m29, m14 ; t35 t60 + psubsw m14, m26, m17 ; t37 t58 + paddsw m26, m17 ; t36 t59 + psubsw m17, m25, m18 ; t38 t57 + paddsw m25, m18 ; t39 t56 + psubsw m18, m24, m19 ; t41 t54 + paddsw m24, m19 ; t40 t55 + psubsw m19, m27, m16 ; t42 t53 + paddsw m27, m16 ; t43 t52 + psubsw m16, m28, m15 ; t45 t50 + paddsw m28, m15 ; t44 t51 + psubsw m15, m23, m20 ; t46 t49 + paddsw m20, m23 ; t47 t48 +.main_oddhalf2: + ITX_MUL2X_PACK 8, 9, 23, 10, 401, 4076, 5 ; t33a t62a + ITX_MUL2X_PACK 21, 9, 23, 10, m4076, 401, 5 ; t34a t61a + ITX_MUL2X_PACK 14, 9, 23, 10, 3166, 2598, 5 ; t37a t58a + ITX_MUL2X_PACK 17, 9, 23, 10, m2598, 3166, 5 ; t38a t57a + ITX_MUL2X_PACK 18, 9, 23, 10, 1931, 3612, 5 ; t41a t54a + ITX_MUL2X_PACK 19, 9, 23, 10, m3612, 1931, 5 ; t42a t53a + ITX_MUL2X_PACK 16, 9, 23, 10, 3920, 1189, 5 ; t45a t50a + ITX_MUL2X_PACK 15, 9, 23, 10, m1189, 3920, 5 ; t46a t49a + vpbroadcastd m11, [o(pw_m4017_799)] + psubsw m23, m25, m26 ; t36a t59a + paddsw m25, m26 ; t39a t56a + psubsw m26, m24, m27 ; t43a t52a + paddsw m27, m24 ; t40a t55a + psubsw m24, m20, m28 ; t44a t51a + paddsw m20, m28 ; t47a t48a + psubsw m28, m8, m21 ; t34 t61 + paddsw m8, m21 ; t33 t62 + psubsw m21, m17, m14 ; t37 t58 + paddsw m17, m14 ; t38 t57 + psubsw m14, m18, m19 ; t42 t53 + paddsw m18, m19 ; t41 t54 + psubsw m19, m15, m16 ; t45 t50 + paddsw m15, m16 ; t46 t49 + psubsw m16, m22, m29 ; t35a t60a + paddsw m22, m29 ; t32a t63a + ITX_MUL2X_PACK 16, 9, 29, 10, 799_4017, 11, 20 ; t35 t60 + ITX_MUL2X_PACK 28, 9, 29, 10, 799_4017, 11, 20 ; t34a t61a + ITX_MUL2X_PACK 23, 9, 29, 10, 11, m799_m4017, 36 ; t36 t59 + ITX_MUL2X_PACK 21, 9, 29, 10, 11, m799_m4017, 36 ; t37a t58a + vpbroadcastd m11, [o(pw_m2276_3406)] + ITX_MUL2X_PACK 26, 9, 29, 10, 3406_2276, 11, 20 ; t43 t52 + ITX_MUL2X_PACK 14, 9, 29, 10, 3406_2276, 11, 20 ; t42a t53a + ITX_MUL2X_PACK 24, 9, 29, 10, 11, m3406_m2276, 36 ; t44 t51 + ITX_MUL2X_PACK 19, 9, 29, 10, 11, m3406_m2276, 36 ; t45a t50a + vpbroadcastd m11, [o(pw_1567_3784)] + vpbroadcastd m12, [o(pw_m3784_1567)] + psubsw m29, m22, m25 ; t39 t56 + paddsw m22, m25 ; t32 t63 + psubsw m25, m20, m27 ; t40 t55 + paddsw m20, m27 ; t47 t48 + psubsw m27, m8, m17 ; t38a t57a + paddsw m8, m17 ; t33a t62a + psubsw m17, m15, m18 ; t41a t54a + paddsw m15, m18 ; t46a t49a + paddsw m18, m16, m23 ; t35a t60a + psubsw m16, m23 ; t36a t59a + psubsw m23, m24, m26 ; t43a t52a + paddsw m24, m26 ; t44a t51a + paddsw m26, m28, m21 ; t34 t61 + psubsw m28, m21 ; t37 t58 + psubsw m21, m19, m14 ; t42 t53 + paddsw m19, m14 ; t45 t50 + ITX_MUL2X_PACK 29, 9, 14, 10, 11, 12, 4 ; t39a t56a + ITX_MUL2X_PACK 27, 9, 14, 10, 11, 12, 4 ; t38 t57 + ITX_MUL2X_PACK 16, 9, 14, 10, 11, 12, 4 ; t36 t59 + ITX_MUL2X_PACK 28, 9, 14, 10, 11, 12, 4 ; t37a t58a + vpbroadcastd m11, [o(pw_m1567_m3784)] + ITX_MUL2X_PACK 25, 9, 14, 10, 12, 11, 4 ; t40a t55a + ITX_MUL2X_PACK 17, 9, 14, 10, 12, 11, 4 ; t41 t54 + ITX_MUL2X_PACK 23, 9, 14, 10, 12, 11, 4 ; t43 t52 + ITX_MUL2X_PACK 21, 9, 14, 10, 12, 11, 4 ; t42a t53a + vbroadcasti32x4 m13, [o(deint_shuf)] + vpbroadcastd m11, [o(pw_2896_2896)] + vpbroadcastd m12, [o(pw_m2896_2896)] + paddsw m14, m22, m20 ; t32a t63a + psubsw m22, m20 ; t47a t48a + psubsw m20, m8, m15 ; t46 t49 + paddsw m8, m15 ; t33 t62 + paddsw m15, m18, m24 ; t35 t60 + psubsw m18, m24 ; t44 t51 + psubsw m24, m26, m19 ; t45a t50a + paddsw m26, m19 ; t34a t61a + REPX {pshufb x, m13}, m14, m8, m15, m26 + psubsw m19, m29, m25 ; t40 t55 + paddsw m25, m29 ; t39 t56 + psubsw m29, m27, m17 ; t41a t54a + paddsw m27, m17 ; t38a t57a + psubsw m17, m16, m23 ; t43a t52a + paddsw m16, m23 ; t36a t59a + psubsw m9, m28, m21 ; t42 t53 + paddsw m28, m21 ; t37 t58 + REPX {pshufb x, m13}, m25, m27, m16, m28 + ITX_MUL2X_PACK 22, 13, 21, 10, 11, 12, 8 ; t47 t48 + ITX_MUL2X_PACK 20, 23, 22, 10, 11, 12, 8 ; t46a t49a + packssdw m21, m22 ; t47 t46a + packssdw m13, m23 ; t48 t49a + ITX_MUL2X_PACK 18, 22, 20, 10, 11, 12, 8 ; t44a t51a + ITX_MUL2X_PACK 24, 23, 18, 10, 11, 12, 8 ; t45 t50 + packssdw m20, m18 ; t44a t45 + packssdw m22, m23 ; t51a t50 + ITX_MUL2X_PACK 19, 24, 18, 10, 11, 12, 8 ; t40a t55a + ITX_MUL2X_PACK 29, 23, 19, 10, 11, 12, 8 ; t41 t54 + packssdw m18, m19 ; t40a t41 + packssdw m24, m23 ; t55a t54 + ITX_MUL2X_PACK 17, 23, 19, 10, 11, 12, 8 ; t43 t52 + ITX_MUL2X_PACK 9, 29, 17, 10, 11, 12, 8 ; t42a t53a + packssdw m19, m17 ; t43 t42a + packssdw m23, m29 ; t52 t53a + punpcklqdq m17, m25, m27 ; t39 t38a + punpckhqdq m25, m27 ; t56 t57a + punpckhqdq m27, m15, m26 ; t60 t61a + punpcklqdq m15, m26 ; t35 t34a + punpckhqdq m26, m16, m28 ; t59a t58 + punpcklqdq m16, m28 ; t36a t37 + punpckhqdq m28, m14, m8 ; t63a t62 + punpcklqdq m14, m8 ; t32a t33 + psubsw m29, m0, m28 ; out63 out62 + paddsw m0, m28 ; out0 out1 + psubsw m28, m1, m27 ; out60 out61 + paddsw m1, m27 ; out3 out2 + psubsw m27, m2, m26 ; out59 out58 + paddsw m2, m26 ; out4 out5 + psubsw m26, m3, m25 ; out56 out57 + paddsw m3, m25 ; out7 out6 + psubsw m25, m4, m24 ; out55 out54 + paddsw m4, m24 ; out8 out9 + psubsw m24, m5, m23 ; out52 out53 + paddsw m5, m23 ; out11 out10 + psubsw m23, m6, m22 ; out51 out50 + paddsw m6, m22 ; out12 out13 + psubsw m22, m7, m13 ; out48 out49 + paddsw m7, m13 ; out15 out14 + ret + +cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jnz .normal + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 16 +.dconly: + imul r6d, 181 + add r6d, 128+512 + sar r6d, 8+2 +.dconly2: + imul r6d, 181 + add r6d, 128+2048 + sar r6d, 8+4 + pxor m2, m2 + vpbroadcastw m3, r6d +.dconly_loop: + mova m1, [dstq] + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + paddw m0, m3 + paddw m1, m3 + packuswb m0, m1 + mova [dstq], m0 + add dstq, strideq + dec r3d + jg .dconly_loop + RET +.normal: + WIN64_SPILL_XMM 31 + mova m19, [o(dup16_perm)] + mova m24, [cq+64* 2] + mova m28, [cq+64* 6] + mova m26, [cq+64* 4] + mova m22, [cq+64* 0] + mova m23, [cq+64* 1] + mova m29, [cq+64* 7] + mova m27, [cq+64* 5] + mova m25, [cq+64* 3] + vpermb m8, m19, m24 ; 4 + vpermb m1, m19, m28 ; 12 + vpermb m7, m19, m26 ; 8 + vpermb m9, m19, m22 ; __ 0 + vpermb m21, m19, m23 ; 2 + vpermb m15, m19, m29 ; 14 + vpermb m18, m19, m27 ; 10 + vpermb m14, m19, m25 ; 6 + pslld m9, 16 + vpord m30, m19, [o(pb_32)] {1to16} + REPX {vpermb x, m30, x}, m22, m29, m26, m25, m24, m27, m28, m23 + cmp eobd, 151 + jb .fast + vpermb m0, m19, [cq+64*14] ; 28 + vpermb m5, m19, [cq+64*10] ; 20 + vpermb m3, m19, [cq+64*12] ; 24 + vpermb m6, m19, [cq+64* 8] ; __ 16 + pslld m6, 16 + call m(idct_16x16_internal_8bpc).main_fast + vpermb m17, m19, [cq+64*15] ; 30 + vpermb m20, m19, [cq+64* 9] ; 18 + vpermb m16, m19, [cq+64*11] ; 22 + vpermb m19, m19, [cq+64*13] ; 26 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + vpermb m21, m30, [cq+64*15] + vpermb m14, m30, [cq+64* 8] + vpermb m17, m30, [cq+64*11] + vpermb m18, m30, [cq+64*12] + vpermb m19, m30, [cq+64*13] + vpermb m16, m30, [cq+64*10] + vpermb m15, m30, [cq+64* 9] + vpermb m20, m30, [cq+64*14] + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf + jmp .end +.fast: ; bottom half is zero + call m(idct_16x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast +.end: + mova [cq+64* 8], m4 + mova [cq+64* 9], m5 + mova [cq+64*10], m6 + mova [cq+64*11], m7 + mova [cq+64*12], m26 + mova [cq+64*13], m27 + mova [cq+64*14], m28 + mova [cq+64*15], m29 + vpbroadcastd m13, [o(pw_8192)] + call .pass1_end + call .pass2 + mova [cq+64* 0], m0 + mova [cq+64* 1], m1 + mova [cq+64* 2], m2 + mova [cq+64* 3], m3 + mova [cq+64* 4], m4 + mova [cq+64* 5], m5 + mova [cq+64* 6], m6 + mova [cq+64* 7], m7 + pmulhrsw m0, m13, [cq+64* 8] + pmulhrsw m1, m13, [cq+64* 9] + pmulhrsw m2, m13, [cq+64*10] + pmulhrsw m3, m13, [cq+64*11] + vpbroadcastd m30, [o(pw_2048)] + pmulhrsw m4, m13, m22 + pmulhrsw m5, m13, m23 + pmulhrsw m6, m13, m24 + pmulhrsw m7, m13, m25 + pmulhrsw m22, m30, m14 + pmulhrsw m14, m13, m26 + pmulhrsw m23, m30, m15 + pmulhrsw m15, m13, m27 + pmulhrsw m24, m30, m16 + pmulhrsw m16, m13, m28 + pmulhrsw m25, m30, m17 + pmulhrsw m17, m13, m29 + pmulhrsw m26, m30, m18 + pmulhrsw m18, m13, [cq+64*12] + pmulhrsw m27, m30, m19 + pmulhrsw m19, m13, [cq+64*13] + pmulhrsw m28, m30, m20 + pmulhrsw m20, m13, [cq+64*14] + pmulhrsw m29, m30, m21 + pmulhrsw m21, m13, [cq+64*15] + call .transpose_round + call .pass2 + pxor m10, m10 + lea r3, [strideq*3] +%macro IDCT_64x16_END 4 + mova m9, [dstq+%4] +%if %1 < 8 + pmulhrsw m%3, m30, [cq+64*%1] +%endif + pmulhrsw m%2, m30 + mova [cq+64*%1], m10 + punpcklbw m8, m9, m10 + punpckhbw m9, m10 + paddw m8, m%3 + paddw m9, m%2 + packuswb m8, m9 + mova [dstq+%4], m8 +%if %1 == 3 || %1 == 7 || %1 == 11 + lea dstq, [dstq+strideq*4] +%endif +%endmacro + IDCT_64x16_END 0, 0, 11, strideq*0 + IDCT_64x16_END 1, 1, 11, strideq*1 + IDCT_64x16_END 2, 2, 11, strideq*2 + IDCT_64x16_END 3, 3, 11, r3 + IDCT_64x16_END 4, 4, 11, strideq*0 + IDCT_64x16_END 5, 5, 11, strideq*1 + IDCT_64x16_END 6, 6, 11, strideq*2 + IDCT_64x16_END 7, 7, 11, r3 + IDCT_64x16_END 8, 14, 22, strideq*0 + IDCT_64x16_END 9, 15, 23, strideq*1 + IDCT_64x16_END 10, 16, 24, strideq*2 + IDCT_64x16_END 11, 17, 25, r3 + IDCT_64x16_END 12, 18, 26, strideq*0 + IDCT_64x16_END 13, 19, 27, strideq*1 + IDCT_64x16_END 14, 20, 28, strideq*2 + IDCT_64x16_END 15, 21, 29, r3 + RET +ALIGN function_align +.pass1_end: + mova m4, [cq+64* 0] + mova m5, [cq+64* 1] + mova m6, [cq+64* 2] + mova m7, [cq+64* 3] + mova m8, [cq+64* 4] + mova m9, [cq+64* 5] + mova m11, [cq+64* 6] + mova m12, [cq+64* 7] + psubsw m29, m4, m21 ; out47 out46 + paddsw m4, m21 ; out16 out17 + psubsw m28, m5, m20 ; out44 out45 + paddsw m5, m20 ; out19 out18 + REPX {pmulhrsw x, m13}, m0, m1, m2, m3 + psubsw m27, m6, m19 ; out43 out42 + paddsw m6, m19 ; out20 out21 + psubsw m26, m7, m18 ; out40 out41 + paddsw m7, m18 ; out23 out22 + pmulhrsw m18, m13, m22 + pmulhrsw m19, m13, m23 + pmulhrsw m20, m13, m24 + pmulhrsw m21, m13, m25 + paddsw m25, m12, m14 ; out31 out30 + psubsw m14, m12, m14 ; out32 out33 + paddsw m24, m11, m15 ; out28 out29 + psubsw m15, m11, m15 ; out35 out34 + REPX {pmulhrsw x, m13}, m4, m5, m6, m7 + paddsw m23, m9, m16 ; out27 out26 + psubsw m16, m9, m16 ; out36 out37 + paddsw m22, m8, m17 ; out24 out25 + psubsw m17, m8, m17 ; out39 out38 + REPX {pmulhrsw x, m13}, m14, m15, m16, m17 +.transpose_round: +%macro TRANSPOSE_8x4_PACKED 4 + punpckhwd m8, m%1, m%3 ; b0 f0 b1 f1 b2 f2 b3 f3 + punpcklwd m%1, m%3 ; a0 e0 a1 e1 a2 e2 a3 e3 + punpcklwd m%3, m%2, m%4 ; d0 h0 d1 h1 d2 h2 d3 h3 + punpckhwd m%2, m%4 ; c0 g0 c1 g1 c2 g2 c3 g3 + punpckhwd m%4, m%1, m%2 ; a2 c2 e2 g2 a3 c3 e3 g3 + punpcklwd m%1, m%2 ; a0 c0 e0 g0 a1 c1 e1 g1 + punpckhwd m%2, m8, m%3 ; b2 d2 f2 h2 b3 d3 f3 h3 + punpcklwd m8, m%3 ; b0 d0 f0 h0 b1 d1 f1 h1 + punpcklwd m%3, m%4, m%2 ; 2 + punpckhwd m%4, m%2 ; 3 + punpckhwd m%2, m%1, m8 ; 1 + punpcklwd m%1, m8 ; 0 +%endmacro + TRANSPOSE_8x4_PACKED 0, 1, 2, 3 + TRANSPOSE_8x4_PACKED 18, 19, 20, 21 + TRANSPOSE_8x4_PACKED 4, 5, 6, 7 + TRANSPOSE_8x4_PACKED 14, 15, 16, 17 + vshufi32x4 m8, m0, m4, q3232 ; a02 a03 b02 b03 + vinserti32x8 m0, ym4, 1 ; a00 a01 b00 b01 + vshufi32x4 m4, m1, m5, q3232 ; a12 a13 b12 b13 + vinserti32x8 m9, m1, ym5, 1 ; a10 a11 b10 b11 + vshufi32x4 m5, m2, m6, q3232 ; a22 a23 b22 b23 + vinserti32x8 m1, m2, ym6, 1 ; a20 a21 b20 b21 + vshufi32x4 m6, m3, m7, q3232 ; a32 a33 b32 b33 + vinserti32x8 m11, m3, ym7, 1 ; a30 a31 b30 b31 + vshufi32x4 m2, m14, m18, q3232 ; c02 c03 d02 d03 + vinserti32x8 m3, m14, ym18, 1 ; c00 c01 d00 d01 + vshufi32x4 m18, m15, m19, q3232 ; c12 c13 d12 d13 + vinserti32x8 m15, ym19, 1 ; c10 c11 d10 d11 + vshufi32x4 m19, m16, m20, q3232 ; c22 c23 d22 d23 + vinserti32x8 m16, ym20, 1 ; c20 c21 d20 d21 + vshufi32x4 m20, m17, m21, q3232 ; c32 c33 d32 d33 + vinserti32x8 m17, ym21, 1 ; c30 c31 d30 d31 + ret +.pass2: + vshufi32x4 m7, m5, m19, q3131 ; 14 + vshufi32x4 m5, m19, q2020 ; 10 + vshufi32x4 m21, m6, m20, q3131 ; 15 + vshufi32x4 m19, m6, m20, q2020 ; 11 + vshufi32x4 m20, m4, m18, q3131 ; 13 + vshufi32x4 m18, m4, m18, q2020 ; 9 + vshufi32x4 m6, m8, m2, q3131 ; 12 + vshufi32x4 m4, m8, m2, q2020 ; 8 + vshufi32x4 m2, m0, m3, q3131 ; 4 + vshufi32x4 m0, m3, q2020 ; 0 + vshufi32x4 m3, m1, m16, q3131 ; 6 + vshufi32x4 m1, m16, q2020 ; 2 + vshufi32x4 m16, m9, m15, q3131 ; 5 + vshufi32x4 m14, m9, m15, q2020 ; 1 + vshufi32x4 m15, m11, m17, q2020 ; 3 + vshufi32x4 m17, m11, m17, q3131 ; 7 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main2 + jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + +cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 7, 0, dst, stride, c, eob + lea r5, [o_base] + test eobd, eobd + jz .dconly + PROLOGUE 0, 9, 30, 64*32, dst, stride, c, eob + vpbroadcastd m23, [o(pw_2896x8)] +%undef cmp + cmp eobd, 136 + jb .fast + pmulhrsw m5, m23, [cq+64*20] + pmulhrsw m3, m23, [cq+64*12] + pmulhrsw m1, m23, [cq+64* 4] + pmulhrsw m7, m23, [cq+64*28] + pmulhrsw m2, m23, [cq+64* 8] + pmulhrsw m6, m23, [cq+64*24] + pmulhrsw m0, m23, [cq+64* 0] + pmulhrsw m4, m23, [cq+64*16] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + pmulhrsw m14, m23, [cq+64* 2] + pmulhrsw m21, m23, [cq+64*30] + pmulhrsw m18, m23, [cq+64*18] + pmulhrsw m17, m23, [cq+64*14] + pmulhrsw m16, m23, [cq+64*10] + pmulhrsw m19, m23, [cq+64*22] + pmulhrsw m20, m23, [cq+64*26] + pmulhrsw m15, m23, [cq+64* 6] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova [cq+64* 0], m14 + mova [cq+64* 2], m15 + mova [cq+64* 4], m16 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64*12], m20 + mova [cq+64*14], m21 + pmulhrsw m22, m23, [cq+64* 1] + pmulhrsw m21, m23, [cq+64*31] + pmulhrsw m14, m23, [cq+64*17] + pmulhrsw m29, m23, [cq+64*15] + pmulhrsw m26, m23, [cq+64* 9] + pmulhrsw m17, m23, [cq+64*23] + pmulhrsw m18, m23, [cq+64*25] + pmulhrsw m25, m23, [cq+64* 7] + pmulhrsw m24, m23, [cq+64* 5] + pmulhrsw m19, m23, [cq+64*27] + pmulhrsw m16, m23, [cq+64*21] + pmulhrsw m27, m23, [cq+64*11] + pmulhrsw m28, m23, [cq+64*13] + pmulhrsw m15, m23, [cq+64*19] + pmulhrsw m20, m23, [cq+64*29] + pmulhrsw m23, [cq+64* 3] + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf + vpbroadcastd m12, [o(pw_16384)] + psubsw m13, m0, m29 ; 31 + paddsw m0, m29 ; 0 + psubsw m29, m1, m28 ; 30 + paddsw m1, m28 ; 1 + psubsw m28, m2, m27 ; 29 + paddsw m2, m27 ; 2 + psubsw m27, m3, m26 ; 28 + paddsw m3, m26 ; 3 + psubsw m26, m4, m25 ; 27 + paddsw m4, m25 ; 4 + psubsw m25, m5, m24 ; 26 + paddsw m5, m24 ; 5 + psubsw m24, m6, m23 ; 25 + paddsw m6, m23 ; 6 + psubsw m23, m7, m22 ; 24 + paddsw m7, m22 ; 7 + pxor m9, m9 + punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7 + punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7 + punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3 + REPX {mova [cq+64*x], m9}, 16, 17, 18, 19 + punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7 + punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3 + punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7 + punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3 + REPX {mova [cq+64*x], m9}, 20, 21, 22, 23 + punpckhwd m3, m23, m24 + punpcklwd m23, m24 + punpckhwd m24, m25, m26 + punpcklwd m25, m26 + REPX {mova [cq+64*x], m9}, 24, 25, 26, 27 + punpckhwd m26, m27, m28 + punpcklwd m27, m28 + punpckhwd m28, m29, m13 + punpcklwd m29, m13 + REPX {mova [cq+64*x], m9}, 28, 29, 30, 31 + punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1 + REPX {pmulhrsw x, m12}, m7, m0, m2, m4 + punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7 + punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5 + punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7 + punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5 + REPX {pmulhrsw x, m12}, m6, m8, m1, m22 + punpckhdq m13, m23, m25 + punpckldq m23, m25 + punpckhdq m25, m27, m29 + punpckldq m27, m29 + REPX {pmulhrsw x, m12}, m13, m23, m25, m27 + punpckhdq m9, m3, m24 + punpckldq m3, m24 + punpckhdq m24, m26, m28 + punpckldq m26, m28 + REPX {pmulhrsw x, m12}, m9, m3, m24, m26 + punpckhqdq m5, m23, m27 ; d01 d09 d17 d25 + punpcklqdq m23, m27 ; d00 d08 d16 d24 + punpcklqdq m27, m13, m25 ; d02 d10 d18 d26 + punpckhqdq m13, m25 ; d03 d11 d19 d27 + punpcklqdq m25, m3, m26 ; d04 d12 d20 d28 + punpckhqdq m3, m26 ; d05 d13 d21 d29 + punpcklqdq m26, m9, m24 ; d06 d14 d22 d30 + punpckhqdq m9, m24 ; d07 d15 d23 d31 + mova [cq+64* 3], m23 + mova [cq+64*13], m27 + mova [cq+64* 7], m25 + mova [cq+64*15], m26 + punpckhqdq m24, m8, m22 ; a05 a13 a21 a29 + punpcklqdq m8, m22 ; a04 a12 a20 a28 + punpckhqdq m22, m0, m4 ; a01 a09 a17 a25 + punpcklqdq m0, m4 ; a00 a08 a16 a24 + punpckhqdq m23, m7, m2 ; a03 a11 a19 a27 + punpcklqdq m7, m2 ; a02 a10 a18 a26 + punpckhqdq m25, m6, m1 ; a07 a15 a23 a31 + punpcklqdq m6, m1 ; a06 a14 a22 a30 + mova [cq+64* 1], m0 + mova [cq+64* 9], m7 + mova [cq+64* 5], m8 + mova [cq+64*11], m6 + mova m2, [cq+64* 0] + mova m11, [cq+64* 2] + mova m8, [cq+64* 4] + mova m29, [cq+64* 6] + mova m27, [cq+64* 8] + mova m26, [cq+64*10] + mova m4, [cq+64*12] + mova m28, [cq+64*14] + psubsw m1, m2, m21 ; 23 + paddsw m2, m21 ; 8 + psubsw m21, m11, m20 ; 22 + paddsw m11, m20 ; 9 + psubsw m20, m8, m19 ; 21 + paddsw m8, m19 ; 10 + psubsw m19, m29, m18 ; 20 + paddsw m29, m18 ; 11 + psubsw m18, m27, m17 ; 19 + paddsw m27, m17 ; 12 + psubsw m17, m26, m16 ; 18 + paddsw m26, m16 ; 13 + psubsw m16, m4, m15 ; 17 + paddsw m4, m15 ; 14 + psubsw m15, m28, m14 ; 16 + paddsw m28, m14 ; 15 + punpcklwd m14, m15, m16 + punpckhwd m15, m16 + punpckhwd m16, m17, m18 + punpcklwd m17, m18 + punpckhwd m18, m19, m20 + punpcklwd m19, m20 + punpckhwd m20, m21, m1 + punpcklwd m21, m1 + punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7 + punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3 + punpckhwd m11, m8, m29 ; k4 l4 k5 l5 k6 l6 k7 l7 + punpcklwd m8, m29 ; k0 l0 k1 l1 k2 l2 k3 l3 + punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7 + punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3 + punpckhwd m26, m4, m28 ; o4 p4 o5 p5 o6 p6 o7 p7 + punpcklwd m4, m28 ; o0 p0 o1 p1 o2 p2 o3 p3 + punpckhdq m28, m2, m8 ; i2 j2 k2 l2 i3 j3 k3 l3 + punpckldq m2, m8 ; i0 j0 k0 l0 i1 j1 k1 l1 + punpckhdq m8, m27, m4 ; m2 n2 o2 p2 m3 n3 o3 p3 + punpckldq m27, m4 ; m0 n0 o0 p0 m1 n1 o1 p1 + REPX {pmulhrsw x, m12}, m28, m2, m8, m27 + punpckhdq m4, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7 + punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5 + punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7 + punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5 + REPX {pmulhrsw x, m12}, m4, m1, m11, m29 + punpckhdq m26, m19, m21 + punpckldq m19, m21 + punpckhdq m21, m15, m16 + punpckldq m15, m16 + REPX {pmulhrsw x, m12}, m26, m19, m21, m15 + punpckhdq m16, m18, m20 + punpckldq m18, m20 + punpckhdq m20, m14, m17 + punpckldq m14, m17 + REPX {pmulhrsw x, m12}, m16, m18, m20, m14 + punpckhqdq m17, m28, m8 ; b03 b11 b19 b27 + punpcklqdq m28, m8 ; b02 b10 b18 b26 + punpckhqdq m8, m2, m27 ; b01 b09 b17 b25 + punpcklqdq m2, m27 ; b00 b08 b16 b24 + punpcklqdq m27, m1, m29 ; b04 b12 b20 b28 + punpckhqdq m1, m29 ; b05 b13 b21 b29 + punpcklqdq m29, m4, m11 ; b06 b14 b22 b30 + punpckhqdq m4, m11 ; b07 b15 b23 b31 + mova [cq+64* 0], m2 + mova [cq+64* 8], m28 + mova [cq+64* 4], m27 + mova [cq+64*10], m29 + punpckhqdq m27, m20, m26 ; c03 c11 c19 c27 + punpcklqdq m20, m26 ; c02 c10 c18 c26 + punpckhqdq m26, m14, m19 ; c01 c09 c17 c25 + punpcklqdq m14, m19 ; c00 c08 c16 c24 + punpckhqdq m28, m15, m18 ; c05 c13 c21 c29 + punpcklqdq m15, m18 ; c04 c12 c20 c28 + punpckhqdq m29, m21, m16 ; c07 c15 c23 c31 + punpcklqdq m21, m16 ; c06 c14 c22 c30 + mova [cq+64* 2], m14 + mova [cq+64*12], m20 + mova [cq+64* 6], m15 + mova [cq+64*14], m21 + vshufi32x4 m14, m22, m8, q3232 ; a17 a25 b17 b25 + vinserti32x8 m22, ym8, 1 ; a01 a09 b01 b09 + vshufi32x4 m15, m23, m17, q3232 ; a19 a27 b19 b27 + vinserti32x8 m23, ym17, 1 ; a03 a11 b03 b11 + vshufi32x4 m16, m24, m1, q3232 ; a21 a29 b21 b29 + vinserti32x8 m24, ym1, 1 ; a05 a13 b05 b13 + vshufi32x4 m17, m25, m4, q3232 ; a23 a31 b23 b31 + vinserti32x8 m25, ym4, 1 ; a07 a15 b07 b15 + vinserti32x8 m19, m26, ym5, 1 ; c01 c09 d01 d09 + vshufi32x4 m26, m5, q3232 ; c17 c25 d17 d25 + vinserti32x8 m20, m27, ym13, 1 ; c03 c11 d03 d11 + vshufi32x4 m27, m13, q3232 ; c19 c27 d19 d27 + vinserti32x8 m21, m28, ym3, 1 ; c05 c13 d05 d13 + vshufi32x4 m28, m3, q3232 ; c21 c29 d21 d29 + vinserti32x8 m18, m29, ym9, 1 ; c07 c15 d07 d15 + vshufi32x4 m29, m9, q3232 ; c23 c31 d23 d31 + mov r4, rsp + vshufi32x4 m0, m22, m19, q2020 ; 1 + vshufi32x4 m1, m17, m29, q3131 ; 31 + vshufi32x4 m2, m14, m26, q2020 ; 17 + vshufi32x4 m3, m25, m18, q3131 ; 15 + call .main_part1 + vshufi32x4 m0, m25, m18, q2020 ; 7 + vshufi32x4 m1, m14, m26, q3131 ; 25 + vshufi32x4 m2, m17, m29, q2020 ; 23 + vshufi32x4 m3, m22, m19, q3131 ; 9 + call .main_part1 + vshufi32x4 m0, m24, m21, q2020 ; 5 + vshufi32x4 m1, m15, m27, q3131 ; 27 + vshufi32x4 m2, m16, m28, q2020 ; 21 + vshufi32x4 m3, m23, m20, q3131 ; 11 + call .main_part1 + vshufi32x4 m0, m23, m20, q2020 ; 3 + vshufi32x4 m1, m16, m28, q3131 ; 29 + vshufi32x4 m2, m15, m27, q2020 ; 19 + vshufi32x4 m3, m24, m21, q3131 ; 13 + call .main_part1 + call .main_part2 + mova m0, [cq+64* 1] ; a0 + mova m15, [cq+64* 0] ; b0 + mova m3, [cq+64* 2] ; c0 + mova m16, [cq+64* 3] ; d0 + mova m14, [cq+64* 5] ; a4 + mova m8, [cq+64* 4] ; b4 + mova m17, [cq+64* 6] ; c4 + mova m1, [cq+64* 7] ; d4 + vshufi32x4 m2, m0, m15, q3232 ; a16 a24 b16 b24 + vinserti32x8 m0, ym15, 1 ; a00 a08 b00 b08 + vshufi32x4 m15, m3, m16, q3232 ; c16 c24 d16 d24 + vinserti32x8 m3, ym16, 1 ; c00 c08 d00 d08 + vshufi32x4 m16, m14, m8, q3232 ; a20 a28 b20 b28 + vinserti32x8 m14, ym8, 1 ; a04 a12 b04 b12 + vshufi32x4 m8, m17, m1, q3232 ; c20 c28 d20 d28 + vinserti32x8 m17, ym1, 1 ; c04 c12 d04 d12 + vshufi32x4 m1, m0, m3, q3131 ; 8 + vshufi32x4 m0, m3, q2020 ; 0 + vshufi32x4 m3, m2, m15, q3131 ; 24 + vshufi32x4 m2, m15, q2020 ; 16 + vshufi32x4 m15, m14, m17, q3131 ; 12 + vshufi32x4 m14, m17, q2020 ; 4 + vshufi32x4 m17, m16, m8, q3131 ; 28 + vshufi32x4 m16, m8, q2020 ; 20 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + mova m8, [cq+64* 8] + mova m9, [cq+64*12] + mova m11, [cq+64*10] + mova m12, [cq+64*14] + mova [cq+64* 0], m14 + mova [cq+64* 2], m15 + mova [cq+64* 4], m16 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64*12], m20 + mova [cq+64*14], m21 + mova m22, [cq+64* 9] + mova m27, [cq+64*13] + mova m23, [cq+64*11] + mova m24, [cq+64*15] + vshufi32x4 m26, m22, m8, q3232 ; a18 a26 b18 b26 + vinserti32x8 m22, ym8, 1 ; a02 a10 b02 b10 + vshufi32x4 m8, m9, m27, q3232 ; c18 c26 d18 d26 + vinserti32x8 m9, ym27, 1 ; c02 c10 d02 d10 + vshufi32x4 m27, m23, m11, q3232 ; a22 a30 b22 b30 + vinserti32x8 m23, ym11, 1 ; a06 a14 b06 b14 + vshufi32x4 m11, m12, m24, q3232 ; c22 c30 d22 d30 + vinserti32x8 m12, ym24, 1 ; c06 c14 d06 d14 + vshufi32x4 m28, m26, m8, q3131 ; 26 + vshufi32x4 m26, m8, q2020 ; 18 + vshufi32x4 m24, m22, m9, q3131 ; 10 + vshufi32x4 m22, m9, q2020 ; 2 + vshufi32x4 m29, m27, m11, q3131 ; 30 + vshufi32x4 m27, m11, q2020 ; 22 + vshufi32x4 m25, m23, m12, q3131 ; 14 + vshufi32x4 m23, m12, q2020 ; 6 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + jmp .end +.fast: ; bottom/right halves are zero + pmulhrsw ym9, ym23, [cq+64* 0] + pmulhrsw ym6, ym23, [cq+64* 8] + mova m14, [o(dup16_perm)] + pmulhrsw ym8, ym23, [cq+64* 2] + pmulhrsw xm0, xm23, [cq+64*14] + pmulhrsw xm5, xm23, [cq+64*10] + pmulhrsw ym1, ym23, [cq+64* 6] + pmulhrsw ym7, ym23, [cq+64* 4] + pmulhrsw xm3, xm23, [cq+64*12] + pmovzxwd m9, ym9 + pmovzxwd m6, ym6 + vpermb m8, m14, m8 + punpcklwd xm0, xm0 + vpermb ym5, ym14, ym5 + vpermb m1, m14, m1 + vpermb m7, m14, m7 + punpcklwd xm3, xm3 + pslld m9, 16 + pslld m6, 16 + call m(idct_16x16_internal_8bpc).main_fast + vpmulhrsw ym21, ym23, [cq+64* 1] + {evex}vpmulhrsw xm17, xm23, [cq+64*15] ; force EVEX encoding, which + {evex}vpmulhrsw xm20, xm23, [cq+64* 9] ; reduces code size due to + {evex}vpmulhrsw ym15, ym23, [cq+64* 7] ; compressed displacements + {evex}vpmulhrsw ym18, ym23, [cq+64* 5] + {evex}vpmulhrsw xm16, xm23, [cq+64*11] + {evex}vpmulhrsw xm19, xm23, [cq+64*13] + {evex}vpmulhrsw ym23, [cq+64* 3] + vpermb m21, m14, m21 + punpcklwd xm17, xm17 + vpermb ym20, ym14, ym20 + vpermb m15, m14, m15 + vpermb m18, m14, m18 + vpermb ym16, ym14, ym16 + punpcklwd xm19, xm19 + vpermb m14, m14, m23 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + vpbroadcastd m9, [o(pw_16384)] + call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round + vshufi32x4 m16, m0, m3, q2020 ; 0 + vshufi32x4 m26, m0, m3, q3131 ; 4 + vshufi32x4 m0, m14, m2, q2020 ; 1 + vshufi32x4 m14, m2, q3131 ; 5 + vshufi32x4 m3, m19, m7, q3131 ; 15 + vshufi32x4 m19, m7, q2020 ; 11 + vshufi32x4 m27, m17, m9, q2020 ; 3 + vshufi32x4 m17, m9, q3131 ; 7 + vshufi32x4 m28, m20, m6, q2020 ; 9 + vshufi32x4 m20, m6, q3131 ; 13 + vshufi32x4 m22, m1, m18, q2020 ; 2 + vshufi32x4 m23, m1, m18, q3131 ; 6 + vshufi32x4 m24, m5, m15, q2020 ; 10 + vshufi32x4 m25, m5, m15, q3131 ; 14 + vshufi32x4 m15, m21, m4, q3131 ; 12 + vshufi32x4 m21, m21, m4, q2020 ; 8 + mov r4, rsp + call .main_part1_fast + mova m0, m17 + mova m3, m28 + call .main_part1_fast + mova m0, m14 + mova m3, m19 + call .main_part1_fast + mova m0, m27 + mova m3, m20 + call .main_part1_fast + call .main_part2 + mova m0, m16 + mova m1, m21 + mova m14, m26 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 + mova [cq+64*14], m21 + mova [cq+64* 0], m14 + mova [cq+64* 6], m17 + mova [cq+64* 8], m18 + mova [cq+64*10], m19 + mova [cq+64* 4], m16 + mova [cq+64* 2], m15 + mova [cq+64*12], m20 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 +.end: + lea r4, [strideq*3] + vpbroadcastd m12, [o(pw_2048)] + movshdup m13, [o(permD)] + lea r5, [r4+strideq] ; stride*4 + lea r3, [dstq+r4*8] + lea r6, [strideq+r5*8] ; stride*33 + lea r8, [r4+r5*8] ; stride*35 + add r3, r5 ; dst+stride*28 + lea r7, [r6+strideq] ; stride*34 +%macro IDCT_32x64_END 6 ; src, mem, stride[1-4] +%if %2 < 8 + paddsw m10, m%2, m%1 + psubsw m11, m%2, m%1 +%else + mova m11, [cq+64*(%2*2-16)] + paddsw m10, m11, m%1 + psubsw m11, m%1 +%endif + mova m9, [rsp+64*(31-%2)] + mova m%1, [rsp+64*%2] + paddsw m8, m10, m9 + psubsw m10, m9 + paddsw m9, m11, m%1 + pmovzxbw m0, [dstq+%3] + psubsw m11, m%1 + pmovzxbw m%1, [r3 +%4] + REPX {pmulhrsw x, m12}, m8, m10, m9, m11 + paddw m8, m0 + pmovzxbw m0, [r3 +%5] + paddw m10, m%1 + pmovzxbw m%1, [dstq+%6] + paddw m9, m0 + paddw m11, m%1 +%if %2 >= 8 +%if %2 == 8 + pxor m1, m1 +%endif + mova [cq+64*(%2*2-16)], m1 + mova [cq+64*(%2*2-15)], m1 +%endif + packuswb m8, m10 + packuswb m9, m11 + vpermq m8, m13, m8 + vpermq m9, m13, m9 + mova [dstq+%3], ym8 + vextracti32x8 [r3 +%4], m8, 1 + mova [r3 +%5], ym9 + vextracti32x8 [dstq+%6], m9, 1 +%if %2 == 3 || %2 == 7 || %2 == 11 + add dstq, r5 + sub r3, r5 +%endif +%endmacro + IDCT_32x64_END 29, 0, strideq*0, r8, r4 , r5*8 + IDCT_32x64_END 28, 1, strideq*1, r7, strideq*2, r6 + IDCT_32x64_END 27, 2, strideq*2, r6, strideq*1, r7 + IDCT_32x64_END 26, 3, r4 , r5*8, strideq*0, r8 + IDCT_32x64_END 25, 4, strideq*0, r8, r4 , r5*8 + IDCT_32x64_END 24, 5, strideq*1, r7, strideq*2, r6 + IDCT_32x64_END 23, 6, strideq*2, r6, strideq*1, r7 + IDCT_32x64_END 22, 7, r4 , r5*8, strideq*0, r8 + IDCT_32x64_END 21, 8, strideq*0, r8, r4 , r5*8 + IDCT_32x64_END 20, 9, strideq*1, r7, strideq*2, r6 + IDCT_32x64_END 19, 10, strideq*2, r6, strideq*1, r7 + IDCT_32x64_END 18, 11, r4 , r5*8, strideq*0, r8 + IDCT_32x64_END 17, 12, strideq*0, r8, r4 , r5*8 + IDCT_32x64_END 16, 13, strideq*1, r7, strideq*2, r6 + IDCT_32x64_END 15, 14, strideq*2, r6, strideq*1, r7 + IDCT_32x64_END 14, 15, r4 , r5*8, strideq*0, r8 + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 64 + imul r6d, 181 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3 +ALIGN function_align ; bottom three-quarters are zero +.main_part1_fast: + vpbroadcastd m1, [o(idct64_mul+4*0)] + vpbroadcastd m8, [o(idct64_mul+4*1)] + vpbroadcastd m2, [o(idct64_mul+4*6)] + vpbroadcastd m9, [o(idct64_mul+4*7)] + pmulhrsw m1, m0 ; t63a + pmulhrsw m0, m8 ; t32a + pmulhrsw m2, m3 ; t60a + pmulhrsw m3, m9 ; t35a + mova m8, m0 + mova m7, m1 + mova m6, m3 + mova m5, m2 + jmp .main_part1b +.main_part1: + ; idct64 steps 1-5: + ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + vpbroadcastd m7, [o(idct64_mul+4*0)] + vpbroadcastd m8, [o(idct64_mul+4*1)] + vpbroadcastd m6, [o(idct64_mul+4*2)] + vpbroadcastd m9, [o(idct64_mul+4*3)] + pmulhrsw m7, m0 ; t63a + vpbroadcastd m5, [o(idct64_mul+4*4)] + pmulhrsw m0, m8 ; t32a + vpbroadcastd m8, [o(idct64_mul+4*5)] + pmulhrsw m6, m1 ; t62a + vpbroadcastd m4, [o(idct64_mul+4*6)] + pmulhrsw m1, m9 ; t33a + vpbroadcastd m9, [o(idct64_mul+4*7)] + pmulhrsw m5, m2 ; t61a + pmulhrsw m2, m8 ; t34a + pmulhrsw m4, m3 ; t60a + pmulhrsw m3, m9 ; t35a + psubsw m8, m0, m1 ; t33 + paddsw m0, m1 ; t32 + psubsw m1, m7, m6 ; t62 + paddsw m7, m6 ; t63 + psubsw m6, m3, m2 ; t34 + paddsw m3, m2 ; t35 + psubsw m2, m4, m5 ; t61 + paddsw m5, m4 ; t60 +.main_part1b: + vpbroadcastd m11, [o(idct64_mul+4*8)] + vpbroadcastd m12, [o(idct64_mul+4*9)] + ITX_MULSUB_2W 1, 8, 4, 9, 10, 11, 12 ; t33a, t62a + vpbroadcastd m11, [o(idct64_mul+4*10)] + ITX_MULSUB_2W 2, 6, 4, 9, 10, 12, 11 ; t34a, t61a + vpbroadcastd m11, [o(idct64_mul+4*11)] + vpbroadcastd m12, [o(idct64_mul+4*12)] + psubsw m4, m0, m3 ; t35a + paddsw m0, m3 ; t32a + psubsw m3, m7, m5 ; t60a + paddsw m7, m5 ; t63a + psubsw m5, m1, m2 ; t34 + paddsw m1, m2 ; t33 + psubsw m2, m8, m6 ; t61 + paddsw m6, m8 ; t62 + add r5, 4*13 + ITX_MULSUB_2W 3, 4, 8, 9, 10, 11, 12 ; t35, t60 + ITX_MULSUB_2W 2, 5, 8, 9, 10, 11, 12 ; t34a, t61a + mova [r4+64*0], m0 + mova [r4+64*7], m7 + mova [r4+64*1], m1 + mova [r4+64*6], m6 + mova [r4+64*3], m3 + mova [r4+64*4], m4 + mova [r4+64*2], m2 + mova [r4+64*5], m5 + add r4, 64*8 + ret +.main_part2: + vpbroadcastd m11, [o(pw_1567_3784 -16*13)] + vpbroadcastd m12, [o(pw_m3784_1567 -16*13)] + lea r6, [r4+64*7] + vpbroadcastd m17, [o(pw_m1567_m3784-16*13)] + vpbroadcastd m18, [o(pw_2896_2896 -16*13)] + vpbroadcastd m19, [o(pw_m2896_2896 -16*13)] + sub r5, 16*13 +.main_part2_loop: + mova m0, [r4-64*32] ; t32a + mova m1, [r6-64*24] ; t39a + mova m2, [r6-64*32] ; t63a + mova m3, [r4-64*24] ; t56a + mova m4, [r4-64*16] ; t40a + mova m5, [r6-64* 8] ; t47a + mova m6, [r6-64*16] ; t55a + mova m7, [r4-64* 8] ; t48a + psubsw m8, m0, m1 ; t39 + paddsw m0, m1 ; t32 + psubsw m1, m2, m3 ; t56 + paddsw m2, m3 ; t63 + psubsw m3, m5, m4 ; t40 + paddsw m5, m4 ; t47 + psubsw m4, m7, m6 ; t55 + paddsw m7, m6 ; t48 + ITX_MULSUB_2W 1, 8, 6, 9, 10, 11, 12 ; t39a, t56a + ITX_MULSUB_2W 4, 3, 6, 9, 10, 12, 17 ; t40a, t55a + psubsw m6, m2, m7 ; t48a + paddsw m2, m7 ; t63a + psubsw m7, m0, m5 ; t47a + paddsw m0, m5 ; t32a + psubsw m5, m8, m3 ; t55 + paddsw m8, m3 ; t56 + psubsw m3, m1, m4 ; t40 + paddsw m1, m4 ; t39 + ITX_MULSUB_2W 6, 7, 4, 9, 10, 18, 19 ; t47, t48 + ITX_MULSUB_2W 5, 3, 4, 9, 10, 18, 19 ; t40a, t55a + mova [r6-64* 8], m2 + mova [r4-64*32], m0 + mova [r4-64* 8], m8 + mova [r6-64*32], m1 + mova [r6-64*24], m6 + mova [r4-64*16], m7 + mova [r4-64*24], m5 + mova [r6-64*16], m3 + add r4, 64 + sub r6, 64 + cmp r4, r6 + jb .main_part2_loop + ret + +cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 7, 0, dst, stride, c, eob + lea r5, [o_base] + test eobd, eobd + jz .dconly + PROLOGUE 0, 7, 30, 64*32, dst, stride, c, eob + vpbroadcastd m23, [o(pw_2896x8)] +%undef cmp + cmp eobd, 136 + jb .fast + pmulhrsw m0, m23, [cq+64* 1] + pmulhrsw m1, m23, [cq+64*31] + pmulhrsw m2, m23, [cq+64*17] + pmulhrsw m3, m23, [cq+64*15] + vpbroadcastd m10, [o(pd_2048)] + mov r4, rsp + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + pmulhrsw m0, m23, [cq+64* 7] + pmulhrsw m1, m23, [cq+64*25] + pmulhrsw m2, m23, [cq+64*23] + pmulhrsw m3, m23, [cq+64* 9] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + pmulhrsw m0, m23, [cq+64* 5] + pmulhrsw m1, m23, [cq+64*27] + pmulhrsw m2, m23, [cq+64*21] + pmulhrsw m3, m23, [cq+64*11] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + pmulhrsw m0, m23, [cq+64* 3] + pmulhrsw m1, m23, [cq+64*29] + pmulhrsw m2, m23, [cq+64*19] + pmulhrsw m3, m23, [cq+64*13] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 + pmulhrsw m3, m23, [cq+64*24] + pmulhrsw m1, m23, [cq+64* 8] + pmulhrsw m2, m23, [cq+64*16] + pmulhrsw m0, m23, [cq+64* 0] + pmulhrsw m14, m23, [cq+64* 4] + pmulhrsw m17, m23, [cq+64*28] + pmulhrsw m16, m23, [cq+64*20] + pmulhrsw m15, m23, [cq+64*12] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + pmulhrsw m22, m23, [cq+64* 2] + pmulhrsw m29, m23, [cq+64*30] + pmulhrsw m26, m23, [cq+64*18] + pmulhrsw m25, m23, [cq+64*14] + pmulhrsw m24, m23, [cq+64*10] + pmulhrsw m27, m23, [cq+64*22] + pmulhrsw m28, m23, [cq+64*26] + pmulhrsw m23, [cq+64* 6] + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + vpbroadcastd m13, [o(pw_16384)] + call .pass1_end_part1 + mova [cq+64*16], m1 + mova [cq+64*17], m3 + mova [cq+64*18], m5 + mova [cq+64*19], m7 + mova [cq+64*24], m23 + mova [cq+64*25], m25 + mova [cq+64*26], m27 + mova [cq+64*27], m29 + pmulhrsw m23, m13, m0 ; a0 + pmulhrsw m25, m13, m2 ; a2 + pmulhrsw m27, m13, m4 ; a4 + pmulhrsw m29, m13, m6 ; a6 + REPX {pmulhrsw x, m13}, m22, m24, m26, m28 ; e0 e2 e4 e6 + call .pass1_end_part2 + mova [cq+64*20], m15 + mova [cq+64*21], m17 + mova [cq+64*22], m19 + mova [cq+64*23], m21 + mova [cq+64*28], m1 + mova [cq+64*29], m3 + mova [cq+64*30], m5 + mova [cq+64*31], m7 + REPX {pmulhrsw x, m13}, m14, m16, m18, m20 ; c0 c2 c4 c6 + REPX {pmulhrsw x, m13}, m0, m2, m4, m6 ; g0 g2 g4 g6 + vinserti32x8 m3, m23, ym14, 1 ; a00 a01 c00 c01 + vshufi32x4 m23, m14, q3232 ; a02 a03 c02 c03 + vinserti32x8 m15, m22, ym0, 1 ; e00 e01 g00 g01 + vshufi32x4 m22, m0, q3232 ; e02 e03 g02 g03 + vinserti32x8 m1, m27, ym18, 1 ; a40 a41 c40 c41 + vshufi32x4 m27, m18, q3232 ; a42 a43 c42 c43 + vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41 + vshufi32x4 m26, m4, q3232 ; e42 e43 g42 g43 + vinserti32x8 m14, m25, ym16, 1 ; a20 a21 c20 c21 + vshufi32x4 m25, m16, q3232 ; a22 a23 c22 c23 + vinserti32x8 m17, m24, ym2, 1 ; e20 e21 g20 g21 + vshufi32x4 m24, m2, q3232 ; e22 e23 g22 g23 + vinserti32x8 m19, m29, ym20, 1 ; a60 a61 c60 c61 + vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63 + vinserti32x8 m20, m28, ym6, 1 ; e60 e61 g60 g61 + vshufi32x4 m28, m6, q3232 ; e62 e63 g62 g63 + vshufi32x4 m2, m3, m15, q3131 ; 8 + vshufi32x4 m0, m3, m15, q2020 ; 0 + vshufi32x4 m6, m23, m22, q3131 ; 24 + vshufi32x4 m4, m23, m22, q2020 ; 16 + vshufi32x4 m3, m1, m18, q3131 ; 12 + vshufi32x4 m1, m18, q2020 ; 4 + vshufi32x4 m7, m27, m26, q3131 ; 28 + vshufi32x4 m5, m27, m26, q2020 ; 20 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + vshufi32x4 m16, m14, m17, q3131 ; 10 + vshufi32x4 m14, m17, q2020 ; 2 + vshufi32x4 m17, m19, m20, q3131 ; 14 + vshufi32x4 m15, m19, m20, q2020 ; 6 + vshufi32x4 m20, m25, m24, q3131 ; 26 + vshufi32x4 m18, m25, m24, q2020 ; 18 + vshufi32x4 m21, m29, m28, q3131 ; 30 + vshufi32x4 m19, m29, m28, q2020 ; 22 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + pmulhrsw m22, m13, [cq+64*16] ; a1 + pmulhrsw m23, m13, [cq+64*20] ; c1 + pmulhrsw m24, m13, [cq+64*24] ; e1 + pmulhrsw m25, m13, [cq+64*28] ; g1 + pmulhrsw m26, m13, [cq+64*17] ; a3 + pmulhrsw m27, m13, [cq+64*21] ; c3 + pmulhrsw m28, m13, [cq+64*25] ; e3 + pmulhrsw m29, m13, [cq+64*29] ; g3 + mova [cq+64* 8], m14 + mova [cq+64* 9], m15 + mova [cq+64*10], m16 + mova [cq+64*11], m17 + mova [cq+64*12], m18 + mova [cq+64*13], m19 + mova [cq+64*14], m20 + mova [cq+64*15], m21 + pmulhrsw m14, m13, [cq+64*18] ; a5 + pmulhrsw m15, m13, [cq+64*22] ; c5 + pmulhrsw m16, m13, [cq+64*26] ; e5 + pmulhrsw m17, m13, [cq+64*30] ; g5 + pmulhrsw m18, m13, [cq+64*19] ; a7 + pmulhrsw m19, m13, [cq+64*23] ; c7 + pmulhrsw m20, m13, [cq+64*27] ; e7 + pmulhrsw m21, m13, [cq+64*31] ; g7 + vinserti32x8 m8, m22, ym23, 1 ; a10 a11 c10 c11 + vshufi32x4 m22, m23, q3232 ; a12 a13 c12 c13 + vinserti32x8 m9, m24, ym25, 1 ; e10 e11 g10 g11 + vshufi32x4 m24, m25, q3232 ; e12 e13 g12 g13 + vinserti32x8 m23, m26, ym27, 1 ; a30 a31 c30 c31 + vshufi32x4 m26, m27, q3232 ; a32 a33 c32 c33 + vinserti32x8 m11, m28, ym29, 1 ; e30 e31 g30 g31 + vshufi32x4 m28, m29, q3232 ; e32 e33 g32 g33 + mova [cq+64* 0], m0 + mova [cq+64* 1], m1 + mova [cq+64* 2], m2 + mova [cq+64* 3], m3 + mova [cq+64* 4], m4 + mova [cq+64* 5], m5 + mova [cq+64* 6], m6 + mova [cq+64* 7], m7 + vinserti32x8 m12, m14, ym15, 1 ; a50 a51 c50 c51 + vshufi32x4 m14, m15, q3232 ; a52 a53 c52 c53 + vinserti32x8 m13, m16, ym17, 1 ; e50 e51 g50 g51 + vshufi32x4 m16, m17, q3232 ; e52 e53 g52 g53 + vinserti32x8 m25, m18, ym19, 1 ; a70 a71 c70 c71 + vshufi32x4 m18, m19, q3232 ; a72 a73 c72 c73 + vinserti32x8 m17, m20, ym21, 1 ; e70 e71 g70 g71 + vshufi32x4 m20, m21, q3232 ; e72 e73 g72 g73 + vshufi32x4 m27, m23, m11, q3131 ; 11 m27 + vshufi32x4 m23, m11, q2020 ; 3 m23 + vshufi32x4 m19, m26, m28, q3131 ; 27 m19 + vshufi32x4 m15, m26, m28, q2020 ; 19 m15 + vshufi32x4 m29, m25, m17, q3131 ; 15 m29 + vshufi32x4 m25, m17, q2020 ; 7 m25 + vshufi32x4 m21, m18, m20, q3131 ; 31 m21 + vshufi32x4 m17, m18, m20, q2020 ; 23 m17 + vshufi32x4 m20, m14, m16, q3131 ; 29 m20 + vshufi32x4 m16, m14, m16, q2020 ; 21 m16 + vshufi32x4 m18, m22, m24, q3131 ; 25 m18 + vshufi32x4 m14, m22, m24, q2020 ; 17 m14 + vshufi32x4 m26, m8, m9, q3131 ; 9 m26 + vshufi32x4 m22, m8, m9, q2020 ; 1 m22 + vshufi32x4 m28, m12, m13, q3131 ; 13 m28 + vshufi32x4 m24, m12, m13, q2020 ; 5 m24 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf + vpbroadcastd m13, [o(pw_16384)] + pmulhrsw m0, m13, [r4-64*21] + pmulhrsw m1, m13, [r4-64*22] + pmulhrsw m2, m13, [r4-64*23] + pmulhrsw m3, m13, [r4-64*24] + pmulhrsw m4, m13, [r4-64*25] + pmulhrsw m5, m13, [r4-64*26] + pmulhrsw m6, m13, [r4-64*27] + pmulhrsw m7, m13, [r4-64*28] + mova [cq+64*16], m14 + mova [cq+64*17], m15 + mova [cq+64*18], m16 + mova [cq+64*19], m17 + mova [cq+64*20], m18 + mova [cq+64*21], m19 + mova [cq+64*22], m20 + mova [cq+64*23], m21 + pmulhrsw m14, m13, [r4-64*12] + pmulhrsw m15, m13, [r4-64*11] + pmulhrsw m16, m13, [r4-64*10] + pmulhrsw m17, m13, [r4-64* 9] + pmulhrsw m18, m13, [r4-64* 8] + pmulhrsw m19, m13, [r4-64* 7] + pmulhrsw m20, m13, [r4-64* 6] + pmulhrsw m21, m13, [r4-64* 5] + mova [cq+64*24], m22 + mova [cq+64*25], m23 + mova [cq+64*26], m24 + mova [cq+64*27], m25 + mova [cq+64*28], m26 + mova [cq+64*29], m27 + mova [cq+64*30], m28 + mova [cq+64*31], m29 + call .transpose_2x8x8_lo + mova [r4-64*12], m1 + mova [r4-64*11], m3 + mova [r4-64*10], m5 + mova [r4-64* 9], m7 + mova [r4-64* 8], m15 + mova [r4-64* 7], m17 + mova [r4-64* 6], m19 + mova [r4-64* 5], m21 + vinserti32x8 m22, m0, ym14, 1 ; f00 f01 h00 h01 + vshufi32x4 m23, m0, m14, q3232 ; f02 f03 h02 h03 + vinserti32x8 m24, m2, ym16, 1 ; f20 f21 h20 h21 + vshufi32x4 m25, m2, m16, q3232 ; f22 f23 h22 h23 + vinserti32x8 m26, m4, ym18, 1 ; f40 f41 h40 h41 + vshufi32x4 m27, m4, m18, q3232 ; f42 f43 h42 h43 + vinserti32x8 m28, m6, ym20, 1 ; f60 f61 h60 h61 + vshufi32x4 m29, m6, m20, q3232 ; f62 f63 h62 h63 + pmulhrsw m0, m13, [r4-64*20] + pmulhrsw m1, m13, [r4-64*19] + pmulhrsw m2, m13, [r4-64*18] + pmulhrsw m3, m13, [r4-64*17] + pmulhrsw m4, m13, [r4-64*16] + pmulhrsw m5, m13, [r4-64*15] + pmulhrsw m6, m13, [r4-64*14] + pmulhrsw m7, m13, [r4-64*13] + pmulhrsw m14, m13, [r4-64*29] + pmulhrsw m15, m13, [r4-64*30] + pmulhrsw m16, m13, [r4-64*31] + pmulhrsw m17, m13, [r4-64*32] + pmulhrsw m18, m13, [r4-64*33] + pmulhrsw m19, m13, [r4-64*34] + pmulhrsw m20, m13, [r4-64*35] + pmulhrsw m21, m13, [r4-64*36] + call .transpose_2x8x8_lo + mova [r4-64*20], m1 + mova [r4-64*19], m3 + mova [r4-64*18], m5 + mova [r4-64*17], m7 + mova [r4-64*16], m15 + mova [r4-64*15], m17 + mova [r4-64*14], m19 + mova [r4-64*13], m21 + vinserti32x8 m1, m4, ym18, 1 ; b40 b41 d40 d41 + vshufi32x4 m5, m4, m18, q3232 ; b42 b43 d42 d43 + vshufi32x4 m4, m0, m14, q3232 ; b02 b03 d02 d03 + vinserti32x8 m0, ym14, 1 ; b00 b01 d00 d01 + vinserti32x8 m14, m2, ym16, 1 ; b20 b21 d20 d21 + vshufi32x4 m18, m2, m16, q3232 ; b22 b23 d22 d23 + vinserti32x8 m15, m6, ym20, 1 ; b60 b61 d60 d61 + vshufi32x4 m19, m6, m20, q3232 ; b62 b63 d62 d63 + vshufi32x4 m2, m0, m22, q3131 ; 8 + vshufi32x4 m0, m22, q2020 ; 0 + vshufi32x4 m3, m1, m26, q3131 ; 12 + vshufi32x4 m1, m26, q2020 ; 4 + vshufi32x4 m6, m4, m23, q3131 ; 24 + vshufi32x4 m4, m23, q2020 ; 16 + vshufi32x4 m7, m5, m27, q3131 ; 28 + vshufi32x4 m5, m27, q2020 ; 20 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + vshufi32x4 m16, m14, m24, q3131 ; 10 + vshufi32x4 m14, m24, q2020 ; 2 + vshufi32x4 m17, m15, m28, q3131 ; 14 + vshufi32x4 m15, m28, q2020 ; 6 + vshufi32x4 m20, m18, m25, q3131 ; 26 + vshufi32x4 m18, m25, q2020 ; 18 + vshufi32x4 m21, m19, m29, q3131 ; 30 + vshufi32x4 m19, m29, q2020 ; 22 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova m22, [r4-64*20] + mova m26, [r4-64*16] + mova m23, [r4-64*19] + mova m27, [r4-64*15] + mova m24, [r4-64*18] + mova m28, [r4-64*14] + mova m25, [r4-64*17] + mova m29, [r4-64*13] + mova [r4-64*20], m14 + mova [r4-64*19], m15 + mova [r4-64*18], m16 + mova [r4-64*17], m17 + mova [r4-64*16], m18 + mova [r4-64*15], m19 + mova [r4-64*14], m20 + mova [r4-64*13], m21 + mova m19, [r4-64*12] + mova m11, [r4-64* 8] + mova m20, [r4-64*11] + mova m12, [r4-64* 7] + mova m21, [r4-64*10] + mova m8, [r4-64* 6] + mova m9, [r4-64* 9] + mova m18, [r4-64* 5] + vshufi32x4 m14, m22, m26, q3232 ; b12 b13 d12 d13 + vinserti32x8 m22, ym26, 1 ; b10 b11 d10 d11 + vshufi32x4 m15, m23, m27, q3232 ; b32 b33 d32 d33 + vinserti32x8 m23, ym27, 1 ; b30 b31 d30 d31 + vshufi32x4 m16, m24, m28, q3232 ; b52 b53 d52 d53 + vinserti32x8 m24, ym28, 1 ; b50 b51 d50 d51 + vshufi32x4 m17, m25, m29, q3232 ; b72 b73 d72 d73 + vinserti32x8 m25, ym29, 1 ; b70 b71 d70 d71 + vinserti32x8 m27, m19, ym11, 1 ; f10 f11 h10 h11 + vshufi32x4 m19, m11, q3232 ; f12 f13 h12 h13 + vinserti32x8 m28, m20, ym12, 1 ; f30 f31 h30 h31 + vshufi32x4 m20, m12, q3232 ; f32 f33 h32 h33 + vinserti32x8 m29, m21, ym8, 1 ; f50 f51 h50 h51 + vshufi32x4 m21, m8, q3232 ; f52 f53 h52 h53 + vinserti32x8 m8, m9, ym18, 1 ; f70 f71 h70 h71 + vshufi32x4 m9, m18, q3232 ; f72 f73 h72 h73 + vshufi32x4 m26, m22, m27, q3131 ; 9 + vshufi32x4 m22, m27, q2020 ; 1 + vshufi32x4 m27, m23, m28, q3131 ; 11 + vshufi32x4 m23, m28, q2020 ; 3 + vshufi32x4 m28, m24, m29, q3131 ; 13 + vshufi32x4 m24, m29, q2020 ; 5 + vshufi32x4 m29, m25, m8, q3131 ; 15 + vshufi32x4 m25, m8, q2020 ; 7 + vshufi32x4 m18, m14, m19, q3131 ; 25 + vshufi32x4 m14, m19, q2020 ; 17 + vshufi32x4 m19, m15, m20, q3131 ; 27 + vshufi32x4 m15, m20, q2020 ; 19 + vshufi32x4 m20, m16, m21, q3131 ; 29 + vshufi32x4 m16, m21, q2020 ; 21 + vshufi32x4 m21, m17, m9, q3131 ; 31 + vshufi32x4 m17, m9, q2020 ; 23 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf + jmp .end +.fast: ; bottom/right halves are zero + {evex}vpmulhrsw ym8, ym23, [cq+64* 4] + {evex}vpmulhrsw xm1, xm23, [cq+64*12] + mova m28, [o(dup16_perm)] + {evex}vpmulhrsw ym7, ym23, [cq+64* 8] + vpmulhrsw ym22, ym23, [cq+64* 0] + vpermb m8, m28, m8 + vpermb ym1, ym28, ym1 + vpermb m7, m28, m7 + pmovzxwd m9, ym22 + pslld m9, 16 + call m(idct_16x16_internal_8bpc).main_fast2 + {evex}vpmulhrsw ym21, ym23, [cq+64* 2] + {evex}vpmulhrsw xm15, xm23, [cq+64*14] + {evex}vpmulhrsw xm18, xm23, [cq+64*10] + {evex}vpmulhrsw ym14, ym23, [cq+64* 6] + vpermb m21, m28, m21 + punpcklwd xm15, xm15 + vpermb ym18, ym28, ym18 + vpermb m14, m28, m14 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + vpmulhrsw ym22, ym23, [cq+64* 1] + {evex}vpmulhrsw xm29, xm23, [cq+64*15] + {evex}vpmulhrsw xm26, xm23, [cq+64* 9] + {evex}vpmulhrsw ym25, ym23, [cq+64* 7] + {evex}vpmulhrsw ym24, ym23, [cq+64* 5] + {evex}vpmulhrsw xm27, xm23, [cq+64*11] + {evex}vpmulhrsw xm8, xm23, [cq+64*13] + {evex}vpmulhrsw ym23, [cq+64* 3] + vpermb m22, m28, m22 + punpcklwd xm29, xm29 + vpermb ym26, ym28, ym26 + vpermb m25, m28, m25 + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + REPX {vpermb x, m28, x}, m24, m27, m23 + punpcklwd xm28, xm8, xm8 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast + mov r4, rsp + vpbroadcastd m13, [o(pw_16384)] + mova [r4+64*16], m4 + mova [r4+64*17], m5 + mova [r4+64*18], m6 + mova [r4+64*19], m7 + mova [r4+64*28], m26 + mova [r4+64*29], m27 + mova [r4+64*30], m28 + mova [r4+64*31], m29 + call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end + mova [r4+64*20], m22 + mova [r4+64*21], m23 + mova [r4+64*22], m24 + mova [r4+64*23], m25 + mova [r4+64*24], m26 + mova [r4+64*25], m27 + mova [r4+64*26], m28 + mova [r4+64*27], m29 + call .pass2_fast + mova [cq+64* 8], m14 + mova [cq+64* 9], m15 + mova [cq+64*10], m16 + mova [cq+64*11], m17 + mova [cq+64*12], m18 + mova [cq+64*13], m19 + mova [cq+64*14], m20 + mova [cq+64*15], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + mova [cq+64* 0], m0 + mova [cq+64* 1], m1 + mova [cq+64* 2], m2 + mova [cq+64* 3], m3 + mova [cq+64* 4], m4 + mova [cq+64* 5], m5 + mova [cq+64* 6], m6 + mova [cq+64* 7], m7 + pmulhrsw m0, m13, [r4+64*16] + pmulhrsw m1, m13, [r4+64*17] + pmulhrsw m2, m13, [r4+64*18] + pmulhrsw m3, m13, [r4+64*19] + pmulhrsw m4, m13, [r4+64*20] + pmulhrsw m5, m13, [r4+64*21] + pmulhrsw m6, m13, [r4+64*22] + pmulhrsw m7, m13, [r4+64*23] + mova [cq+64*16], m14 + mova [cq+64*17], m15 + mova [cq+64*18], m16 + mova [cq+64*19], m17 + mova [cq+64*20], m18 + mova [cq+64*21], m19 + mova [cq+64*22], m20 + mova [cq+64*23], m21 + pmulhrsw m14, m13, [r4+64*24] + pmulhrsw m15, m13, [r4+64*25] + pmulhrsw m16, m13, [r4+64*26] + pmulhrsw m17, m13, [r4+64*27] + pmulhrsw m18, m13, [r4+64*28] + pmulhrsw m19, m13, [r4+64*29] + pmulhrsw m20, m13, [r4+64*30] + pmulhrsw m21, m13, [r4+64*31] + mova [cq+64*24], m22 + mova [cq+64*25], m23 + mova [cq+64*26], m24 + mova [cq+64*27], m25 + mova [cq+64*28], m26 + mova [cq+64*29], m27 + mova [cq+64*30], m28 + mova [cq+64*31], m29 + call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round + call .pass2_fast + mova [r4+64*16], m14 + mova [r4+64*17], m15 + mova [r4+64*18], m16 + mova [r4+64*19], m17 + mova [r4+64*20], m18 + mova [r4+64*21], m19 + mova [r4+64*22], m20 + mova [r4+64*23], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast +.end: + vpbroadcastd m13, [o(pw_2048)] + lea r5, [strideq*3] + pxor m12, m12 + lea r3, [dstq+r5*8] + lea r6, [strideq+r5] ; stride*4 + add r3, r6 ; dst+stride*28 +%macro IDCT_64x32_END 5 ; src16, src32, mem, off_lo, off_hi + mova m11, [cq+64*( %3)] ; 0 + mova m9, [cq+64*(31-%3)] ; 31 +%if %3 >= 8 + mova m%1, [rsp+64*(%1+16)] +%endif + mova m10, [dstq+%4] + paddsw m8, m11, m9 + psubsw m11, m9 + paddsw m9, m%1, m%2 + psubsw m%1, m%2 + punpcklbw m%2, m10, m12 + punpckhbw m10, m12 + pmulhrsw m8, m13 + pmulhrsw m9, m13 + paddw m8, m%2 + paddw m9, m10 + mova m10, [r3+%5] + pmulhrsw m11, m13 + pmulhrsw m%1, m13 + mova [cq+64*( %3)], m12 + mova [cq+64*(31-%3)], m12 + punpcklbw m%2, m10, m12 + punpckhbw m10, m12 + packuswb m8, m9 + paddw m11, m%2 + paddw m%1, m10 + packuswb m11, m%1 + mova [dstq+%4], m8 + mova [r3 +%5], m11 +%if %3 == 3 || %3 == 7 || %3 == 11 + add dstq, r6 + sub r3, r6 +%endif +%endmacro + IDCT_64x32_END 0, 29, 0, strideq*0, r5 + IDCT_64x32_END 1, 28, 1, strideq*1, strideq*2 + IDCT_64x32_END 2, 27, 2, strideq*2, strideq*1 + IDCT_64x32_END 3, 26, 3, r5 , strideq*0 + IDCT_64x32_END 4, 25, 4, strideq*0, r5 + IDCT_64x32_END 5, 24, 5, strideq*1, strideq*2 + IDCT_64x32_END 6, 23, 6, strideq*2, strideq*1 + IDCT_64x32_END 7, 22, 7, r5 , strideq*0 + IDCT_64x32_END 0, 21, 8, strideq*0, r5 + IDCT_64x32_END 1, 20, 9, strideq*1, strideq*2 + IDCT_64x32_END 2, 19, 10, strideq*2, strideq*1 + IDCT_64x32_END 3, 18, 11, r5 , strideq*0 + IDCT_64x32_END 4, 17, 12, strideq*0, r5 + IDCT_64x32_END 5, 16, 13, strideq*1, strideq*2 + IDCT_64x32_END 6, 15, 14, strideq*2, strideq*1 + IDCT_64x32_END 7, 14, 15, r5 , strideq*0 + RET +ALIGN function_align +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 32 + imul r6d, 181 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 128+256 + sar r6d, 8+1 + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly2 +ALIGN function_align +.pass1_end_part1: +%macro IDCT_64x32_PASS1_END 3 ; src16, src32, src64 +%if %1 != %3 + mova m%1, [cq+64*%1] +%endif + mova m9, [r4+64*(%3-36)] ; idct64 32+n + mova m11, [r4+64*(-5-%3)] ; idct64 63-n + psubsw m8, m%1, m%2 ; idct32 31-n + paddsw m%1, m%2 ; idct32 0+n +%if %1 == %3 + psubsw m%2, m8, m9 ; out 32+n e + paddsw m8, m9 ; out 31-n d + psubsw m9, m%1, m11 ; out 63-n h + paddsw m%1, m11 ; out 0+n a +%else + paddsw m%2, m8, m9 ; out 23-n c + psubsw m8, m9 ; out 40+n f + paddsw m9, m%1, m11 ; out 8+n b + psubsw m%1, m11 ; out 55-n g +%endif + mova [r4+64*(%3-36)], m8 + mova [r4+64*(-5-%3)], m9 +%endmacro + IDCT_64x32_PASS1_END 0, 29, 0 + IDCT_64x32_PASS1_END 1, 28, 1 + IDCT_64x32_PASS1_END 2, 27, 2 + IDCT_64x32_PASS1_END 3, 26, 3 + IDCT_64x32_PASS1_END 4, 25, 4 + IDCT_64x32_PASS1_END 5, 24, 5 + IDCT_64x32_PASS1_END 6, 23, 6 + IDCT_64x32_PASS1_END 7, 22, 7 +.transpose_2x8x8_hi: ; m0-m7 + m22-m29 (inverted) + punpcklwd m8, m25, m24 ; e0 f0 e1 f1 e2 f2 e3 f3 + punpckhwd m25, m24 ; e4 f4 e5 f5 e6 f6 e7 f7 + punpcklwd m24, m23, m22 ; g0 h0 g1 h1 g2 h2 g3 h3 + punpckhwd m23, m22 ; g4 h4 g5 h5 g6 h6 g7 h7 + punpcklwd m22, m29, m28 ; a0 b0 a1 b1 a2 b2 a3 b3 + punpckhwd m29, m28 ; a4 b4 a5 b5 a6 b6 a7 b7 + punpcklwd m28, m27, m26 ; c0 d0 c1 d1 c2 d2 c3 d3 + punpckhwd m27, m26 ; c4 d4 c5 d5 c6 d6 c7 d7 + punpckldq m26, m29, m27 ; a4 b4 c4 d4 a5 b5 c5 d5 + punpckhdq m29, m27 ; a6 b6 c6 d6 a7 b7 c7 d7 + punpckldq m27, m8, m24 ; e0 f0 g0 h0 e1 f1 g1 h1 + punpckhdq m8, m24 ; e2 f2 g2 h2 e3 f3 g3 h3 + punpckhdq m24, m22, m28 ; a2 b2 c2 d2 a3 b3 c3 d3 + punpckldq m22, m28 ; a0 b0 c0 d0 a1 b1 c1 d1 + punpckldq m28, m25, m23 ; e4 f4 g4 h4 e5 f5 g5 h5 + punpckhdq m25, m23 ; e6 f6 g6 h6 e7 f7 g7 h7 + punpckhqdq m23, m22, m27 ; 1 23 + punpcklqdq m22, m27 ; 0 22 + punpckhqdq m27, m26, m28 ; 5 27 + punpcklqdq m26, m28 ; 4 26 + punpcklqdq m28, m29, m25 ; 6 28 + punpckhqdq m29, m25 ; 7 29 + punpckhqdq m25, m24, m8 ; 3 25 + punpcklqdq m24, m8 ; 2 24 +.transpose_8x8: + punpckhwd m8, m4, m5 + punpcklwd m4, m5 + punpckhwd m5, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m6, m7 + punpcklwd m6, m7 + punpckhwd m7, m2, m3 + punpcklwd m2, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckldq m2, m4, m6 + punpckhdq m4, m6 + punpckhdq m6, m5, m7 + punpckldq m5, m7 + punpckldq m7, m8, m1 + punpckhdq m8, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m8 + punpcklqdq m6, m8 + ret +.pass1_end_part2: + IDCT_64x32_PASS1_END 0, 21, 8 + IDCT_64x32_PASS1_END 1, 20, 9 + IDCT_64x32_PASS1_END 2, 19, 10 + IDCT_64x32_PASS1_END 3, 18, 11 + IDCT_64x32_PASS1_END 4, 17, 12 + IDCT_64x32_PASS1_END 5, 16, 13 + IDCT_64x32_PASS1_END 6, 15, 14 + IDCT_64x32_PASS1_END 7, 14, 15 +.transpose_2x8x8_lo: ; m0-m7 (inverted) + m14-m21 + punpcklwd m8, m3, m2 + punpckhwd m3, m2 + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m7, m6 + punpckhwd m7, m6 + punpcklwd m6, m5, m4 + punpckhwd m5, m4 + punpckldq m4, m7, m5 + punpckhdq m7, m5 + punpckldq m5, m8, m2 + punpckhdq m8, m2 + punpckhdq m2, m0, m6 + punpckldq m0, m6 + punpckldq m6, m3, m1 + punpckhdq m3, m1 + punpckhqdq m1, m0, m5 + punpcklqdq m0, m5 + punpckhqdq m5, m4, m6 + punpcklqdq m4, m6 + punpcklqdq m6, m7, m3 + punpckhqdq m7, m3 + punpckhqdq m3, m2, m8 + punpcklqdq m2, m8 + punpckhwd m8, m18, m19 + punpcklwd m18, m19 + punpckhwd m19, m14, m15 + punpcklwd m14, m15 + punpckhwd m15, m20, m21 + punpcklwd m20, m21 + punpckhwd m21, m16, m17 + punpcklwd m16, m17 + punpckhdq m17, m14, m16 + punpckldq m14, m16 + punpckldq m16, m18, m20 + punpckhdq m18, m20 + punpckhdq m20, m19, m21 + punpckldq m19, m21 + punpckldq m21, m8, m15 + punpckhdq m8, m15 + punpckhqdq m15, m14, m16 + punpcklqdq m14, m16 + punpcklqdq m16, m17, m18 + punpckhqdq m17, m18 + punpcklqdq m18, m19, m21 + punpckhqdq m19, m21 + punpckhqdq m21, m20, m8 + punpcklqdq m20, m8 + ret +.pass2_fast: + vshufi32x4 m24, m9, m15, q3131 ; 5 + vshufi32x4 m22, m9, m15, q2020 ; 1 + vshufi32x4 m15, m1, m16, q3131 ; 6 + vshufi32x4 m14, m1, m16, q2020 ; 2 + vshufi32x4 m1, m0, m3, q3131 ; 4 + vshufi32x4 m0, m3, q2020 ; 0 + vshufi32x4 m3, m8, m2, q3131 ; 12 + vshufi32x4 m2, m8, m2, q2020 ; 8 + vshufi32x4 m25, m11, m17, q3131 ; 7 + vshufi32x4 m23, m11, m17, q2020 ; 3 + vshufi32x4 m17, m5, m19, q3131 ; 14 + vshufi32x4 m16, m5, m19, q2020 ; 10 + vshufi32x4 m29, m6, m20, q3131 ; 15 + vshufi32x4 m27, m6, m20, q2020 ; 11 + vshufi32x4 m28, m4, m18, q3131 ; 13 + vshufi32x4 m26, m4, m18, q2020 ; 9 + jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + +cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 7, 0, dst, stride, c, eob + lea r5, [o_base] + test eobd, eobd + jz .dconly + PROLOGUE 0, 7, 30, 64*96, dst, stride, c, eob +%undef cmp + cmp eobd, 136 + jb .fast + mova m0, [cq+64* 1] + mova m1, [cq+64*31] + mova m2, [cq+64*17] + mova m3, [cq+64*15] + vpbroadcastd m10, [o(pd_2048)] + mov r4, rsp + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + mova m0, [cq+64* 7] + mova m1, [cq+64*25] + mova m2, [cq+64*23] + mova m3, [cq+64* 9] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + mova m0, [cq+64* 5] + mova m1, [cq+64*27] + mova m2, [cq+64*21] + mova m3, [cq+64*11] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + mova m0, [cq+64* 3] + mova m1, [cq+64*29] + mova m2, [cq+64*19] + mova m3, [cq+64*13] + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 + mova m0, [cq+64* 0] + mova m1, [cq+64* 8] + mova m2, [cq+64*16] + mova m3, [cq+64*24] + mova m14, [cq+64* 4] + mova m15, [cq+64*12] + mova m16, [cq+64*20] + mova m17, [cq+64*28] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + mova m22, [cq+64* 2] + mova m29, [cq+64*30] + mova m26, [cq+64*18] + mova m25, [cq+64*14] + mova m24, [cq+64*10] + mova m27, [cq+64*22] + mova m28, [cq+64*26] + mova m23, [cq+64* 6] + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + vpbroadcastd m13, [o(pw_8192)] + call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part1 + mova [r4+64*36], m1 + mova [r4+64*37], m3 + mova [r4+64*38], m5 + mova [r4+64*39], m7 + mova [r4+64*44], m23 + mova [r4+64*45], m25 + mova [r4+64*46], m27 + mova [r4+64*47], m29 + pmulhrsw m23, m13, m0 ; a0 + pmulhrsw m25, m13, m2 ; a2 + pmulhrsw m27, m13, m4 ; a4 + pmulhrsw m29, m13, m6 ; a6 + call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part2 + lea r6, [r4-64*4] + add r4, 64*28 + call .pass2_end + mov r4, rsp + mova m0, [r4+64*23] + mova m1, [r4+64*22] + mova m2, [r4+64*21] + mova m3, [r4+64*20] + mova m4, [r4+64*19] + mova m5, [r4+64*18] + mova m6, [r4+64*17] + mova m7, [r4+64*16] + mova m22, [r4+64*15] + mova m23, [r4+64*14] + mova m24, [r4+64*13] + mova m25, [r4+64*12] + mova m26, [r4+64*11] + mova m27, [r4+64*10] + mova m28, [r4+64* 9] + mova m29, [r4+64* 8] + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_hi + vpbroadcastd m13, [o(pw_8192)] + mova [r4+64* 8], m1 + mova [r4+64* 9], m3 + mova [r4+64*10], m5 + mova [r4+64*11], m7 + mova [r4+64*16], m23 + mova [r4+64*17], m25 + mova [r4+64*18], m27 + mova [r4+64*19], m29 + pmulhrsw m23, m13, m0 ; b0 + pmulhrsw m25, m13, m2 ; b2 + pmulhrsw m27, m13, m4 ; b4 + pmulhrsw m29, m13, m6 ; b6 + mova m0, [r4+64*31] + mova m1, [r4+64*30] + mova m2, [r4+64*29] + mova m3, [r4+64*28] + mova m4, [r4+64*27] + mova m5, [r4+64*26] + mova m6, [r4+64*25] + mova m7, [r4+64*24] + mova m14, [r4+64* 7] + mova m15, [r4+64* 6] + mova m16, [r4+64* 5] + mova m17, [r4+64* 4] + mova m18, [r4+64* 3] + mova m19, [r4+64* 2] + mova m20, [r4+64* 1] + mova m21, [r4+64* 0] + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_lo + mov r6, cq + call .pass2_end + jmp .end +.fast: ; bottom/right halves are zero + mova m28, [o(dup16_perm)] + pmovzxwd m9, [cq+64* 0] + vpermb m8, m28, [cq+64* 4] + vpermb ym1, ym28, [cq+64*12] + vpermb m7, m28, [cq+64* 8] + pslld m9, 16 + call m(idct_16x16_internal_8bpc).main_fast2 + vpermb m21, m28, [cq+64* 2] + vpermb ym15, ym28, [cq+64*14] + vpermb ym18, ym28, [cq+64*10] + vpermb m14, m28, [cq+64* 6] + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 + vpermb m22, m28, [cq+64* 1] + vpermb ym29, ym28, [cq+64*15] + vpermb ym26, ym28, [cq+64* 9] + vpermb m25, m28, [cq+64* 7] + vpermb m24, m28, [cq+64* 5] + vpermb ym27, ym28, [cq+64*11] + vpermb m23, m28, [cq+64* 3] + vpermb ym28, ym28, [cq+64*13] + mova [cq+64* 0], m14 + mova [cq+64* 1], m15 + mova [cq+64* 2], m16 + mova [cq+64* 3], m17 + mova [cq+64* 4], m18 + mova [cq+64* 5], m19 + mova [cq+64* 6], m20 + mova [cq+64* 7], m21 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast + vpbroadcastd m13, [o(pw_8192)] + mova [cq+64*16], m4 + mova [cq+64*17], m5 + mova [cq+64*18], m6 + mova [cq+64*19], m7 + mova [cq+64*28], m26 + mova [cq+64*29], m27 + mova [cq+64*30], m28 + mova [cq+64*31], m29 + call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end + mova [cq+64*20], m22 + mova [cq+64*21], m23 + mova [cq+64*22], m24 + mova [cq+64*23], m25 + mova [cq+64*24], m26 + mova [cq+64*25], m27 + mova [cq+64*26], m28 + mova [cq+64*27], m29 + lea r4, [rsp+64*64] + lea r3, [rsp+64*32] + call .pass2_fast + pmulhrsw m0, m13, [cq+64*16] + pmulhrsw m1, m13, [cq+64*17] + pmulhrsw m2, m13, [cq+64*18] + pmulhrsw m3, m13, [cq+64*19] + pmulhrsw m4, m13, [cq+64*20] + pmulhrsw m5, m13, [cq+64*21] + pmulhrsw m6, m13, [cq+64*22] + pmulhrsw m7, m13, [cq+64*23] + pmulhrsw m14, m13, [cq+64*24] + pmulhrsw m15, m13, [cq+64*25] + pmulhrsw m16, m13, [cq+64*26] + pmulhrsw m17, m13, [cq+64*27] + pmulhrsw m18, m13, [cq+64*28] + pmulhrsw m19, m13, [cq+64*29] + pmulhrsw m20, m13, [cq+64*30] + pmulhrsw m21, m13, [cq+64*31] + call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round + mov r4, rsp + mov r3, cq + call .pass2_fast +.end: + vpbroadcastd m17, [o(pw_2048)] + lea r5, [strideq*8] + mov r3, dstq + pxor m16, m16 + sub r4, 64*5 ; rsp+64*31 + mov r6, rsp +.end_loop: + mova m2, [r6+64*32] ; idct16 0+n lo + mova m7, [r6+64*48] ; idct32 31-n lo + mova m6, [cq+64* 0] ; idct16 0+n hi + mova m0, [cq+64*16] ; idct32 31-n hi + mova m4, [r4+64*64] ; idct64 63-n lo + mova m1, [r4+64* 0] ; idct64 63-n hi + mova m5, [r6+64*64] ; idct64 32+n lo + mova m8, [r6+64* 0] ; idct64 32+n hi + sub r3, strideq + paddsw m3, m2, m7 ; idct32 0+n lo + mova m12, [dstq+r5*0] + psubsw m2, m7 ; idct32 31-n lo + mova m15, [r3 +r5*8] + paddsw m7, m6, m0 ; idct32 0+n hi + mova m13, [r3 +r5*4] + psubsw m6, m0 ; idct32 31-n hi + mova m14, [dstq+r5*4] + paddsw m0, m3, m4 ; out 0+n lo + add r6, 64 + psubsw m3, m4 ; out 63-n lo + sub r4, 64 + paddsw m4, m7, m1 ; out 0+n hi + mova [cq+64* 0], m16 + psubsw m7, m1 ; out 63-n hi + mova [cq+64*16], m16 + paddsw m1, m2, m5 ; out 31-n lo + add cq, 64 + psubsw m2, m5 ; out 32+n lo + paddsw m5, m6, m8 ; out 31-n hi + psubsw m6, m8 ; out 32+n hi + pmulhrsw m0, m17 + punpcklbw m8, m12, m16 + pmulhrsw m4, m17 + punpckhbw m12, m16 + pmulhrsw m3, m17 + punpcklbw m11, m15, m16 + pmulhrsw m7, m17 + punpckhbw m15, m16 + pmulhrsw m1, m17 + punpcklbw m9, m13, m16 + pmulhrsw m5, m17 + punpckhbw m13, m16 + pmulhrsw m2, m17 + punpcklbw m10, m14, m16 + pmulhrsw m6, m17 + punpckhbw m14, m16 + paddw m0, m8 + paddw m4, m12 + packuswb m0, m4 + paddw m3, m11 + paddw m7, m15 + packuswb m3, m7 + paddw m1, m9 + paddw m5, m13 + packuswb m1, m5 + paddw m2, m10 + paddw m6, m14 + packuswb m2, m6 + mova [dstq+r5*0], m0 + mova [r3 +r5*8], m3 + mova [r3 +r5*4], m1 + mova [dstq+r5*4], m2 + add dstq, strideq + cmp r6, r4 + jb .end_loop + RET +.dconly: + movsx r6d, word [cq] + mov [cq], eobd + or r3d, 64 + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly +ALIGN function_align +.pass2_end: + REPX {pmulhrsw x, m13}, m22, m24, m26, m28, m14, m16, m18, m20, m0, m2, m4, m6 + mova [r4+64*20], m1 + mova [r4+64*21], m3 + mova [r4+64*22], m5 + mova [r4+64*23], m7 + vinserti32x8 m1, m23, ym14, 1 ; a00 a01 c00 c01 + vshufi32x4 m3, m23, m14, q3232 ; a02 a03 c02 c03 + vinserti32x8 m5, m22, ym0, 1 ; e00 e01 g00 g01 + vshufi32x4 m14, m22, m0, q3232 ; e02 e03 g02 g03 + mova [r4+64*12], m15 + mova [r4+64*13], m17 + mova [r4+64*14], m19 + mova [r4+64*15], m21 + vinserti32x8 m15, m27, ym18, 1 ; a40 a41 c40 c41 + vshufi32x4 m17, m27, m18, q3232 ; a42 a43 c42 c43 + vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41 + vshufi32x4 m19, m26, m4, q3232 ; e42 e43 g42 g43 + vinserti32x8 m22, m25, ym16, 1 ; a20 a21 c20 c21 + vshufi32x4 m26, m25, m16, q3232 ; a22 a23 c22 c23 + vinserti32x8 m25, m24, ym2, 1 ; e20 e21 g20 g21 + vshufi32x4 m27, m24, m2, q3232 ; e22 e23 g22 g23 + vinserti32x8 m23, m29, ym20, 1 ; a60 a61 c60 c61 + vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63 + vshufi32x4 m13, m28, m6, q3232 ; e62 e63 g62 g63 + vinserti32x8 m28, ym6, 1 ; e60 e61 g60 g61 + vshufi32x4 m0, m1, m5, q2020 ; 0 + vshufi32x4 m1, m5, q3131 ; 8 + vshufi32x4 m2, m3, m14, q2020 ; 16 + vshufi32x4 m3, m14, q3131 ; 24 + vshufi32x4 m14, m15, m18, q2020 ; 4 + vshufi32x4 m15, m18, q3131 ; 12 + vshufi32x4 m16, m17, m19, q2020 ; 20 + vshufi32x4 m17, m19, q3131 ; 28 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + vshufi32x4 m24, m22, m25, q3131 ; 10 + vshufi32x4 m22, m25, q2020 ; 2 + vshufi32x4 m25, m23, m28, q3131 ; 14 + vshufi32x4 m23, m28, q2020 ; 6 + vshufi32x4 m28, m26, m27, q3131 ; 26 + vshufi32x4 m26, m27, q2020 ; 18 + vshufi32x4 m27, m29, m13, q2020 ; 22 + vshufi32x4 m29, m13, q3131 ; 30 + mova [r6+64* 0], m0 + mova [r6+64* 1], m1 + mova [r6+64* 2], m2 + mova [r6+64* 3], m3 + mova [r6+64* 4], m4 + mova [r6+64* 5], m5 + mova [r6+64* 6], m6 + mova [r6+64* 7], m7 + mova [r6+64* 8], m14 + mova [r6+64* 9], m15 + mova [r6+64*10], m16 + mova [r6+64*11], m17 + mova [r6+64*12], m18 + mova [r6+64*13], m19 + mova [r6+64*14], m20 + mova [r6+64*15], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + vpbroadcastd m13, [o(pw_8192)] + mova [r6+64*16], m29 + mova [r6+64*17], m28 + mova [r6+64*18], m27 + mova [r6+64*19], m26 + mova [r6+64*20], m25 + mova [r6+64*21], m24 + mova [r6+64*22], m23 + mova [r6+64*23], m22 + mova [r6+64*24], m21 + mova [r6+64*25], m20 + mova [r6+64*26], m19 + mova [r6+64*27], m18 + mova [r6+64*28], m17 + mova [r6+64*29], m16 + mova [r6+64*30], m15 + mova [r6+64*31], m14 + pmulhrsw m15, m13, [r4+64* 8] ; 1 9 17 25 + pmulhrsw m16, m13, [r4+64*12] + pmulhrsw m17, m13, [r4+64*16] + pmulhrsw m18, m13, [r4+64*20] + pmulhrsw m19, m13, [r4+64*11] ; 7 15 23 31 + pmulhrsw m20, m13, [r4+64*15] + pmulhrsw m21, m13, [r4+64*19] + pmulhrsw m22, m13, [r4+64*23] + vinserti32x8 m14, m15, ym16, 1 ; a1 a9 c1 c9 + vshufi32x4 m15, m16, q3232 ; a17 a25 c17 c25 + vinserti32x8 m16, m17, ym18, 1 ; e1 e9 g1 g9 + vshufi32x4 m17, m18, q3232 ; e17 e25 g17 g25 + pmulhrsw m23, m13, [r4+64*10] ; 5 13 21 29 + pmulhrsw m24, m13, [r4+64*14] + pmulhrsw m25, m13, [r4+64*18] + pmulhrsw m26, m13, [r4+64*22] + vinserti32x8 m18, m19, ym20, 1 ; a7 a15 c7 c15 + vshufi32x4 m19, m20, q3232 ; a23 a31 c23 c31 + vinserti32x8 m20, m21, ym22, 1 ; e7 e15 g7 g15 + vshufi32x4 m21, m22, q3232 ; e23 e31 g23 g31 + pmulhrsw m27, m13, [r4+64* 9] ; 3 11 19 27 + pmulhrsw m28, m13, [r4+64*13] + pmulhrsw m29, m13, [r4+64*17] + pmulhrsw m13, [r4+64*21] + vshufi32x4 m0, m14, m16, q2020 ; 1 + vshufi32x4 m1, m19, m21, q3131 ; 31 + vshufi32x4 m2, m15, m17, q2020 ; 17 + vshufi32x4 m3, m18, m20, q3131 ; 15 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + vshufi32x4 m0, m18, m20, q2020 ; 7 + vshufi32x4 m1, m15, m17, q3131 ; 25 + vshufi32x4 m2, m19, m21, q2020 ; 23 + vshufi32x4 m3, m14, m16, q3131 ; 9 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + vinserti32x8 m22, m23, ym24, 1 ; a5 a13 c5 c13 + vshufi32x4 m23, m24, q3232 ; a21 a29 c21 c29 + vinserti32x8 m24, m25, ym26, 1 ; e5 e13 g5 g13 + vshufi32x4 m25, m26, q3232 ; e21 e29 g21 g29 + vinserti32x8 m26, m27, ym28, 1 ; a3 a11 c3 c11 + vshufi32x4 m27, m28, q3232 ; a19 a27 c19 c27 + vinserti32x8 m28, m29, ym13, 1 ; e3 e11 g3 g11 + vshufi32x4 m29, m13, q3232 ; e19 e17 g19 g27 + vshufi32x4 m0, m22, m24, q2020 ; 5 + vshufi32x4 m1, m27, m29, q3131 ; 27 + vshufi32x4 m2, m23, m25, q2020 ; 21 + vshufi32x4 m3, m26, m28, q3131 ; 11 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + vshufi32x4 m0, m26, m28, q2020 ; 3 + vshufi32x4 m1, m23, m25, q3131 ; 29 + vshufi32x4 m2, m27, m29, q2020 ; 19 + vshufi32x4 m3, m22, m24, q3131 ; 13 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 + jmp m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 +ALIGN function_align +.pass2_fast: + vshufi32x4 m23, m1, m16, q3131 ; 6 + vshufi32x4 m22, m1, m16, q2020 ; 2 + vshufi32x4 m14, m0, m3, q3131 ; 4 + vshufi32x4 m26, m0, m3, q2020 ; 0 + vshufi32x4 m28, m9, m15, q3131 ; 5 + vshufi32x4 m0, m9, m15, q2020 ; 1 + vshufi32x4 m16, m11, m17, q3131 ; 7 + vshufi32x4 m29, m11, m17, q2020 ; 3 + vshufi32x4 m15, m8, m2, q3131 ; 12 + vshufi32x4 m27, m8, m2, q2020 ; 8 + vshufi32x4 m25, m5, m19, q3131 ; 14 + vshufi32x4 m24, m5, m19, q2020 ; 10 + vshufi32x4 m3, m6, m20, q3131 ; 15 + vshufi32x4 m19, m6, m20, q2020 ; 11 + vshufi32x4 m17, m4, m18, q3131 ; 13 + vshufi32x4 m18, m4, m18, q2020 ; 9 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast + mova m0, m16 + mova m3, m18 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast + mova m0, m28 + mova m3, m19 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast + mova m0, m29 + mova m3, m17 + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast + call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 + mova m0, m26 + mova m1, m27 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 + mova [r3+64* 0], m0 + mova [r3+64* 1], m1 + mova [r3+64* 2], m2 + mova [r3+64* 3], m3 + mova [r3+64* 4], m4 + mova [r3+64* 5], m5 + mova [r3+64* 6], m6 + mova [r3+64* 7], m7 + mova [r3+64* 8], m14 + mova [r3+64* 9], m15 + mova [r3+64*10], m16 + mova [r3+64*11], m17 + mova [r3+64*12], m18 + mova [r3+64*13], m19 + mova [r3+64*14], m20 + mova [r3+64*15], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 + mova [r3+64*16], m29 + mova [r3+64*17], m28 + mova [r3+64*18], m27 + mova [r3+64*19], m26 + mova [r3+64*20], m25 + mova [r3+64*21], m24 + mova [r3+64*22], m23 + mova [r3+64*23], m22 + mova [r3+64*24], m21 + mova [r3+64*25], m20 + mova [r3+64*26], m19 + mova [r3+64*27], m18 + mova [r3+64*28], m17 + mova [r3+64*29], m16 + mova [r3+64*30], m15 + mova [r3+64*31], m14 + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/itx_sse.asm b/third_party/dav1d/src/x86/itx_sse.asm new file mode 100644 index 0000000000..ec7e3a52f4 --- /dev/null +++ b/third_party/dav1d/src/x86/itx_sse.asm @@ -0,0 +1,6533 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + + +SECTION_RODATA 16 + +deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 + +deint_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +deint_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 + +%macro COEF_PAIR 2-3 0 ; !0 = m%1_m%2, 2 = no %2_%1 +pw_%1_m%2: times 4 dw %1, -%2 +%if %3 != 2 +pw_%2_%1: times 4 dw %2, %1 +%endif +%if %3 +pw_m%1_m%2: times 4 dw -%1, -%2 +%endif +%endmacro + +;adst4 +pw_1321_3803: times 4 dw 1321, 3803 +pw_2482_m1321: times 4 dw 2482, -1321 +pw_3344_2482: times 4 dw 3344, 2482 +pw_3344_m3803: times 4 dw 3344, -3803 +pw_3344_m3344: times 4 dw 3344, -3344 +pw_0_3344 times 4 dw 0, 3344 +pw_m6688_m3803: times 4 dw -6688, -3803 + +COEF_PAIR 2896, 2896 +COEF_PAIR 1567, 3784 +COEF_PAIR 799, 4017 +COEF_PAIR 3406, 2276 +COEF_PAIR 401, 4076 +COEF_PAIR 1931, 3612 +COEF_PAIR 3166, 2598 +COEF_PAIR 3920, 1189 +COEF_PAIR 3784, 1567, 1 +COEF_PAIR 995, 3973 +COEF_PAIR 1751, 3703 +COEF_PAIR 3513, 2106 +COEF_PAIR 3857, 1380 +COEF_PAIR 4017, 799, 1 +COEF_PAIR 201, 4091 +COEF_PAIR 2440, 3290 +COEF_PAIR 3035, 2751 +COEF_PAIR 4052, 601 +COEF_PAIR 2276, 3406, 1 +COEF_PAIR 4076, 401, 2 +COEF_PAIR 2598, 3166, 2 +COEF_PAIR 3612, 1931, 2 +COEF_PAIR 1189, 3920, 2 + +pd_2048: times 4 dd 2048 +pw_2048: times 8 dw 2048 +pw_m2048: times 8 dw -2048 +pw_4096: times 8 dw 4096 +pw_16384: times 8 dw 16384 +pw_m16384: times 8 dw -16384 +pw_1697x16: times 8 dw 1697*16 +pw_1697x8: times 8 dw 1697*8 +pw_2896x8: times 8 dw 2896*8 +pw_3344x8: times 8 dw 3344*8 +pw_8192: times 8 dw 8192 +pw_m8192: times 8 dw -8192 +pw_5: times 8 dw 5 +pw_201x8: times 8 dw 201*8 +pw_4091x8: times 8 dw 4091*8 +pw_m2751x8: times 8 dw -2751*8 +pw_3035x8: times 8 dw 3035*8 +pw_1751x8: times 8 dw 1751*8 +pw_3703x8: times 8 dw 3703*8 +pw_m1380x8: times 8 dw -1380*8 +pw_3857x8: times 8 dw 3857*8 +pw_995x8: times 8 dw 995*8 +pw_3973x8: times 8 dw 3973*8 +pw_m2106x8: times 8 dw -2106*8 +pw_3513x8: times 8 dw 3513*8 +pw_2440x8: times 8 dw 2440*8 +pw_3290x8: times 8 dw 3290*8 +pw_m601x8: times 8 dw -601*8 +pw_4052x8: times 8 dw 4052*8 + +pw_4095x8: times 8 dw 4095*8 +pw_101x8: times 8 dw 101*8 +pw_2967x8: times 8 dw 2967*8 +pw_m2824x8: times 8 dw -2824*8 +pw_3745x8: times 8 dw 3745*8 +pw_1660x8: times 8 dw 1660*8 +pw_3822x8: times 8 dw 3822*8 +pw_m1474x8: times 8 dw -1474*8 +pw_3996x8: times 8 dw 3996*8 +pw_897x8: times 8 dw 897*8 +pw_3461x8: times 8 dw 3461*8 +pw_m2191x8: times 8 dw -2191*8 +pw_3349x8: times 8 dw 3349*8 +pw_2359x8: times 8 dw 2359*8 +pw_4036x8: times 8 dw 4036*8 +pw_m700x8: times 8 dw -700*8 +pw_4065x8: times 8 dw 4065*8 +pw_501x8: times 8 dw 501*8 +pw_3229x8: times 8 dw 3229*8 +pw_m2520x8: times 8 dw -2520*8 +pw_3564x8: times 8 dw 3564*8 +pw_2019x8: times 8 dw 2019*8 +pw_3948x8: times 8 dw 3948*8 +pw_m1092x8: times 8 dw -1092*8 +pw_3889x8: times 8 dw 3889*8 +pw_1285x8: times 8 dw 1285*8 +pw_3659x8: times 8 dw 3659*8 +pw_m1842x8: times 8 dw -1842*8 +pw_3102x8: times 8 dw 3102*8 +pw_2675x8: times 8 dw 2675*8 +pw_4085x8: times 8 dw 4085*8 +pw_m301x8: times 8 dw -301*8 + +SECTION .text + +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +%if ARCH_X86_64 +%define o(x) x +%else +%define o(x) r5-$$+x ; PIC +%endif + +%macro WRITE_4X4 9 ;src[1-2], tmp[1-3], row[1-4] + lea r2, [dstq+strideq*2] +%assign %%i 1 +%rotate 5 +%rep 4 + %if %1 & 2 + CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) + %else + CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) + %endif + %assign %%i %%i + 1 + %rotate 1 +%endrep + + movd m%3, [%%row_adr1] ;dst0 + movd m%5, [%%row_adr2] ;dst1 + punpckldq m%3, m%5 ;high: dst1 :low: dst0 + movd m%4, [%%row_adr3] ;dst2 + movd m%5, [%%row_adr4] ;dst3 + punpckldq m%4, m%5 ;high: dst3 :low: dst2 + + pxor m%5, m%5 + punpcklbw m%3, m%5 ;extend byte to word + punpcklbw m%4, m%5 ;extend byte to word + + paddw m%3, m%1 ;high: dst1 + out1 ;low: dst0 + out0 + paddw m%4, m%2 ;high: dst3 + out3 ;low: dst2 + out2 + + packuswb m%3, m%4 ;high->low: dst3 + out3, dst2 + out2, dst1 + out1, dst0 + out0 + + movd [%%row_adr1], m%3 ;store dst0 + out0 + pshuflw m%4, m%3, q1032 + movd [%%row_adr2], m%4 ;store dst1 + out1 + punpckhqdq m%3, m%3 + movd [%%row_adr3], m%3 ;store dst2 + out2 + psrlq m%3, 32 + movd [%%row_adr4], m%3 ;store dst3 + out3 +%endmacro + +%macro ITX4_END 4-5 2048 ; row[1-4], rnd +%if %5 + mova m2, [o(pw_%5)] + pmulhrsw m0, m2 + pmulhrsw m1, m2 +%endif + + WRITE_4X4 0, 1, 2, 3, 4, %1, %2, %3, %4 + ret +%endmacro + +; flags: 1 = swap, 2: coef_regs, 4: no_pack +%macro ITX_MUL2X_PACK 5-6 0 ; dst/src, tmp[1], rnd, coef[1-2], flags +%if %6 & 2 + pmaddwd m%2, m%4, m%1 + pmaddwd m%1, m%5 +%elif %6 & 1 + pmaddwd m%2, m%1, [o(pw_%5_%4)] + pmaddwd m%1, [o(pw_%4_m%5)] +%else + pmaddwd m%2, m%1, [o(pw_%4_m%5)] + pmaddwd m%1, [o(pw_%5_%4)] +%endif + paddd m%2, m%3 + paddd m%1, m%3 + psrad m%2, 12 + psrad m%1, 12 +%if %6 & 4 == 0 + packssdw m%1, m%2 +%endif +%endmacro + +%macro IDCT4_1D_PACKED 0-1 ;pw_2896x8 + mova m3, [o(pd_2048)] + punpckhwd m2, m0, m1 ;unpacked in1 in3 + punpcklwd m0, m1 ;unpacked in0 in2 + ITX_MUL2X_PACK 2, 1, 3, 1567, 3784 + ITX_MUL2X_PACK 0, 1, 3, 2896, 2896 + psubsw m1, m0, m2 ;high: out2 ;low: out3 + paddsw m0, m2 ;high: out1 ;low: out0 +%endmacro + +%macro INV_TXFM_FN 4+ ; type1, type2, size, xmm/stack +cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, %4, dst, stride, coeff, eob, tx2 + %define %%p1 m(i%1_%3_internal_8bpc) +%if ARCH_X86_32 + LEA r5, $$ +%endif +%if has_epilogue +%ifidn %1_%2, dct_dct + test eobd, eobd + jz %%end +%endif + lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] + call %%p1 + RET +%%end: +%else + lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x4, 6 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklqdq m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mov [coeffq], eobd ;0 + pmulhrsw m0, m1 + mova m1, m0 + TAIL_CALL m(iadst_4x4_internal_8bpc).end2 +%endif +%endmacro + +INIT_XMM ssse3 +; itx16 relies on dct_dct being the first function. If you change the order, adjust `itx8_start` in itx16. + +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst +INV_TXFM_4X4_FN dct, identity + +cglobal idct_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m0, [coeffq+16*0] ;high: in1 ;low: in0 + mova m1, [coeffq+16*1] ;high: in3 ;low in2 + + IDCT4_1D_PACKED + + mova m2, [o(deint_shuf)] + shufps m3, m0, m1, q1331 + shufps m0, m1, q0220 + pshufb m0, m2 ;high: in1 ;low: in0 + pshufb m1, m3, m2 ;high: in3 ;low :in2 + jmp tx2q + +.pass2: + IDCT4_1D_PACKED + + pxor m2, m2 + mova [coeffq+16*0], m2 + mova [coeffq+16*1], m2 ;memset(coeff, 0, sizeof(*coeff) * sh * sw); + + ITX4_END 0, 1, 3, 2 + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +cglobal iadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + call .main + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 ;high: in3 ;low :in2 + punpcklwd m0, m2 ;high: in1 ;low: in0 + jmp tx2q + +.pass2: + call .main + +.end: + pxor m2, m2 + mova [coeffq+16*0], m2 + mova [coeffq+16*1], m2 + +.end2: + ITX4_END 0, 1, 2, 3 + +ALIGN function_align +cglobal_label .main + punpcklwd m2, m0, m1 ;unpacked in0 in2 + punpckhwd m0, m1 ;unpacked in1 in3 + mova m3, m0 + pmaddwd m1, m2, [o(pw_3344_m3344)];3344 * in0 - 3344 * in2 + pmaddwd m0, [o(pw_0_3344)] ;3344 * in3 + paddd m1, m0 ;t2 + pmaddwd m0, m2, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 + pmaddwd m2, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 + pmaddwd m4, m3, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 + pmaddwd m5, m3, [o(pw_3344_m3803)];3344 * in1 - 3803 * in3 + paddd m4, m0 ;t0 + t3 + pmaddwd m3, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 + mova m0, [o(pd_2048)] + paddd m1, m0 ;t2 + 2048 + paddd m2, m0 + paddd m0, m4 ;t0 + t3 + 2048 + paddd m5, m2 ;t1 + t3 + 2048 + paddd m2, m4 + paddd m2, m3 ;t0 + t1 - t3 + 2048 + REPX {psrad x, 12}, m1, m0, m5, m2 + packssdw m0, m5 ;high: out1 ;low: out0 + packssdw m1, m2 ;high: out3 ;low: out3 + ret + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + call m(iadst_4x4_internal_8bpc).main + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 ;high: in3 ;low :in2 + punpckhwd m1, m2 ;high: in1 ;low: in0 + jmp tx2q + +.pass2: + call m(iadst_4x4_internal_8bpc).main + +.end: + pxor m2, m2 + mova [coeffq+16*0], m2 + mova [coeffq+16*1], m2 + +.end2: + ITX4_END 3, 2, 1, 0 + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + mova m3, [o(pw_1697x8)] + pmulhrsw m2, m0, m3 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 ;high: in3 ;low :in2 + punpcklwd m0, m2 ;high: in1 ;low: in0 + jmp tx2q + +.pass2: + mova m3, [o(pw_1697x8)] + pmulhrsw m2, m3, m0 + pmulhrsw m3, m1 + paddsw m0, m2 + paddsw m1, m3 + jmp m(iadst_4x4_internal_8bpc).end + +%macro IWHT4_1D_PACKED 0 + punpckhqdq m3, m0, m1 ;low: in1 high: in3 + punpcklqdq m0, m1 ;low: in0 high: in2 + psubw m2, m0, m3 ;low: in0 - in1 high: in2 - in3 + paddw m0, m3 ;low: in0 + in1 high: in2 + in3 + punpckhqdq m2, m2 ;t2 t2 + punpcklqdq m0, m0 ;t0 t0 + psubw m1, m0, m2 + psraw m1, 1 ;t4 t4 + psubw m1, m3 ;low: t1/out2 high: t3/out1 + psubw m0, m1 ;high: out0 + paddw m2, m1 ;low: out3 +%endmacro + +INIT_XMM sse2 +cglobal inv_txfm_add_wht_wht_4x4_8bpc, 3, 3, 4, dst, stride, coeff + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + pxor m2, m2 + mova [coeffq+16*0], m2 + mova [coeffq+16*1], m2 + psraw m0, 2 + psraw m1, 2 + IWHT4_1D_PACKED + punpckhwd m0, m1 + punpcklwd m3, m1, m2 + punpckhdq m1, m0, m3 + punpckldq m0, m3 + IWHT4_1D_PACKED + shufpd m0, m2, 0x01 + ITX4_END 0, 3, 2, 1, 0 + +%macro IDCT8_1D_PACKED 0 + mova m6, [o(pd_2048)] + punpckhwd m4, m0, m3 ;unpacked in1 in7 + punpcklwd m0, m2 ;unpacked in0 in4 + punpckhwd m2, m1 ;unpacked in5 in3 + punpcklwd m1, m3 ;unpacked in2 in6 + ITX_MUL2X_PACK 4, 3, 6, 799, 4017 ;low: t7a high: t4a + ITX_MUL2X_PACK 2, 3, 6, 3406, 2276 ;low: t6a high: t5a + ITX_MUL2X_PACK 1, 3, 6, 1567, 3784 ;low: t3 high: t2 + psubsw m3, m4, m2 ;low: t6a high: t5a + paddsw m4, m2 ;low: t7 high: t4 + pshufb m3, [o(deint_shuf1)] + ITX_MUL2X_PACK 0, 2, 6, 2896, 2896 ;low: t0 high: t1 + ITX_MUL2X_PACK 3, 2, 6, 2896, 2896 ;low: t6 high: t5 + psubsw m2, m0, m1 ;low: tmp3 high: tmp2 + paddsw m0, m1 ;low: tmp0 high: tmp1 + punpcklqdq m1, m4, m3 ;low: t7 high: t6 + punpckhqdq m4, m3 ;low: t4 high: t5 + psubsw m3, m0, m1 ;low: out7 high: out6 + paddsw m0, m1 ;low: out0 high: out1 + paddsw m1, m2, m4 ;low: out3 high: out2 + psubsw m2, m4 ;low: out4 high: out5 +%endmacro + +;dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +;dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +%macro ITX_MULSUB_2W 7-8 0 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2_in_tmp1 + punpckhwd m%4, m%1, m%2 + punpcklwd m%1, m%2 +%if %7 < 8 + pmaddwd m%2, m%7, m%1 + pmaddwd m%3, m%7, m%4 +%else + mova m%2, [o(pw_%7_%6)] +%if %8 + pmaddwd m%3, m%1, m%2 + pmaddwd m%2, m%4 +%else + pmaddwd m%3, m%4, m%2 + pmaddwd m%2, m%1 +%endif +%endif + paddd m%3, m%5 + paddd m%2, m%5 + psrad m%3, 12 + psrad m%2, 12 +%if %8 + packssdw m%3, m%2 +%else + packssdw m%2, m%3 ;dst2 +%endif +%if %7 < 8 + pmaddwd m%4, m%6 + pmaddwd m%1, m%6 +%elif %8 + mova m%2, [o(pw_%6_m%7)] + pmaddwd m%4, m%2 + pmaddwd m%1, m%2 +%else + mova m%3, [o(pw_%6_m%7)] + pmaddwd m%4, m%3 + pmaddwd m%1, m%3 +%endif + paddd m%4, m%5 + paddd m%1, m%5 + psrad m%4, 12 + psrad m%1, 12 + packssdw m%1, m%4 ;dst1 +%endmacro + +%macro IDCT4_1D 7 ; src[1-4], tmp[1-2], pd_2048 + ITX_MULSUB_2W %2, %4, %5, %6, %7, 1567, 3784, 1 ;t2, t3 + ITX_MULSUB_2W %1, %3, %4, %6, %7, 2896, 2896, 1 ;t1, t0 + psubsw m%3, m%1, m%2 ;out2 + paddsw m%2, m%1 ;out1 + paddsw m%1, m%5, m%4 ;out0 + psubsw m%4, m%5 ;out3 +%endmacro + +%macro WRITE_4X8 4 ;row[1-4] + WRITE_4X4 0, 1, 4, 5, 6, %1, %2, %3, %4 + lea dstq, [dstq+strideq*4] + WRITE_4X4 2, 3, 4, 5, 6, %1, %2, %3, %4 +%endmacro + +%macro INV_4X8 0 + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhdq m1, m0, m2 ;low: in2 high: in3 + punpckldq m0, m2 ;low: in0 high: in1 + punpckldq m2, m3, m4 ;low: in4 high: in5 + punpckhdq m3, m4 ;low: in6 high: in7 +%endmacro + +%macro INV_TXFM_4X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x8, 8 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklqdq m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mov [coeffq], eobd + pmulhrsw m0, m1 + pmulhrsw m0, m1 + pmulhrsw m0, [o(pw_2048)] + mova m1, m0 + mova m2, m0 + mova m3, m0 + TAIL_CALL m(iadst_4x8_internal_8bpc).end3 +%endif +%endmacro + +INIT_XMM ssse3 +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst +INV_TXFM_4X8_FN dct, identity + +cglobal idct_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + +.pass1: + call m(idct_8x4_internal_8bpc).main + jmp m(iadst_4x8_internal_8bpc).pass1_end + +.pass2: + call .main + shufps m1, m1, q1032 + shufps m3, m3, q1032 + mova m4, [o(pw_2048)] + jmp m(iadst_4x8_internal_8bpc).end2 + +ALIGN function_align +cglobal_label .main + IDCT8_1D_PACKED + ret + + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity + +cglobal iadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + +.pass1: + call m(iadst_8x4_internal_8bpc).main + +.pass1_end: + INV_4X8 + jmp tx2q + +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 + call .main + mova m4, [o(pw_2048)] + pxor m5, m5 + psubw m5, m4 + +.end: + punpcklqdq m4, m5 + +.end2: + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + pxor m5, m5 + mova [coeffq+16*0], m5 + mova [coeffq+16*1], m5 + mova [coeffq+16*2], m5 + mova [coeffq+16*3], m5 + +.end3: + WRITE_4X8 0, 1, 2, 3 + RET + +ALIGN function_align +cglobal_label .main + mova m6, [o(pd_2048)] + punpckhwd m4, m3, m0 ;unpacked in7 in0 + punpckhwd m5, m2, m1 ;unpacked in5 in2 + punpcklwd m1, m2 ;unpacked in3 in4 + punpcklwd m0, m3 ;unpacked in1 in6 + ITX_MUL2X_PACK 4, 2, 6, 401, 4076 ;low: t0a high: t1a + ITX_MUL2X_PACK 5, 2, 6, 1931, 3612 ;low: t2a high: t3a + ITX_MUL2X_PACK 1, 2, 6, 3166, 2598 ;low: t4a high: t5a + ITX_MUL2X_PACK 0, 2, 6, 3920, 1189 ;low: t6a high: t7a + + psubsw m3, m4, m1 ;low: t4 high: t5 + paddsw m4, m1 ;low: t0 high: t1 + psubsw m2, m5, m0 ;low: t6 high: t7 + paddsw m5, m0 ;low: t2 high: t3 + + shufps m1, m3, m2, q1032 + punpckhwd m2, m1 + punpcklwd m3, m1 + ITX_MUL2X_PACK 3, 0, 6, 1567, 3784, 1 ;low: t5a high: t4a + ITX_MUL2X_PACK 2, 0, 6, 3784, 1567 ;low: t7a high: t6a + + psubsw m1, m4, m5 ;low: t2 high: t3 + paddsw m4, m5 ;low: out0 high: -out7 + psubsw m5, m3, m2 ;low: t7 high: t6 + paddsw m3, m2 ;low: out6 high: -out1 + shufps m0, m4, m3, q3210 ;low: out0 high: -out1 + shufps m3, m4, q3210 ;low: out6 high: -out7 + + mova m2, [o(pw_2896_m2896)] + mova m7, [o(pw_2896_2896)] + shufps m4, m1, m5, q1032 ;low: t3 high: t7 + shufps m1, m5, q3210 ;low: t2 high: t6 + punpcklwd m5, m1, m4 + punpckhwd m1, m4 + pmaddwd m4, m2, m1 ;-out5 + pmaddwd m2, m5 ; out4 + pmaddwd m1, m7 ; out2 + pmaddwd m5, m7 ;-out3 + REPX {paddd x, m6}, m4, m2, m1, m5 + REPX {psrad x, 12}, m4, m2, m1, m5 + packssdw m1, m5 ;low: out2 high: -out3 + packssdw m2, m4 ;low: out4 high: -out5 + ret + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity + +cglobal iflipadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + +.pass1: + call m(iadst_8x4_internal_8bpc).main + + punpcklwd m4, m3, m2 + punpckhwd m3, m2 + punpcklwd m5, m1, m0 + punpckhwd m1, m0 + punpckldq m2, m3, m1 ;low: in4 high: in5 + punpckhdq m3, m1 ;low: in6 high: in7 + punpckldq m0, m4, m5 ;low: in0 high: in1 + punpckhdq m1, m4, m5 ;low: in2 high: in3 + jmp tx2q + +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 + call m(iadst_4x8_internal_8bpc).main + + mova m4, m0 + mova m5, m1 + pshufd m0, m3, q1032 + pshufd m1, m2, q1032 + pshufd m2, m5, q1032 + pshufd m3, m4, q1032 + mova m5, [o(pw_2048)] + pxor m4, m4 + psubw m4, m5 + jmp m(iadst_4x8_internal_8bpc).end + +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity + +cglobal iidentity_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + +.pass1: + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(iadst_4x8_internal_8bpc).pass1_end + +.pass2: + mova m4, [o(pw_4096)] + jmp m(iadst_4x8_internal_8bpc).end2 + + +%macro WRITE_8X2 5 ;coefs[1-2], tmp[1-3] + movq m%3, [dstq ] + movq m%4, [dstq+strideq] + pxor m%5, m%5 + punpcklbw m%3, m%5 ;extend byte to word + punpcklbw m%4, m%5 ;extend byte to word +%ifnum %1 + paddw m%3, m%1 +%else + paddw m%3, %1 +%endif +%ifnum %2 + paddw m%4, m%2 +%else + paddw m%4, %2 +%endif + packuswb m%3, m%4 + movq [dstq ], m%3 + punpckhqdq m%3, m%3 + movq [dstq+strideq], m%3 +%endmacro + +%macro WRITE_8X4 7 ;coefs[1-4], tmp[1-3] + WRITE_8X2 %1, %2, %5, %6, %7 + lea dstq, [dstq+strideq*2] + WRITE_8X2 %3, %4, %5, %6, %7 +%endmacro + +%macro INV_TXFM_8X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x4, 8 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklqdq m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + pmulhrsw m0, m1 + mova m2, [o(pw_2048)] + pmulhrsw m0, m1 + pmulhrsw m0, m2 + mova m1, m0 + mova m2, m0 + mova m3, m0 + TAIL_CALL m(iadst_8x4_internal_8bpc).end2 +%endif +%endmacro + +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst +INV_TXFM_8X4_FN dct, identity + +cglobal idct_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + + call m(idct_4x8_internal_8bpc).main + + mova m4, [o(deint_shuf1)] + mova m5, [o(deint_shuf2)] + pshufb m0, m4 + pshufb m1, m5 + pshufb m2, m4 + pshufb m3, m5 + punpckhdq m4, m0, m1 + punpckldq m0, m1 + punpckhdq m5, m2, m3 + punpckldq m2, m3 + punpckhqdq m1, m0, m2 ;in1 + punpcklqdq m0, m2 ;in0 + punpckhqdq m3, m4, m5 ;in3 + punpcklqdq m2 ,m4, m5 ;in2 + jmp tx2q + +.pass2: + call .main + jmp m(iadst_8x4_internal_8bpc).end + +ALIGN function_align +cglobal_label .main + mova m6, [o(pd_2048)] + IDCT4_1D 0, 1, 2, 3, 4, 5, 6 + ret + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + + shufps m0, m0, q1032 + shufps m1, m1, q1032 + call m(iadst_4x8_internal_8bpc).main + + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + pxor m5, m5 + psubsw m3, m5, m1 + psubsw m5, m4 + punpckhdq m4, m5, m3 + punpckldq m5, m3 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckhwd m1, m0, m5 ;in1 + punpcklwd m0, m5 ;in0 + punpcklwd m2, m3, m4 ;in2 + punpckhwd m3, m4 ;in3 + jmp tx2q + +.pass2: + call .main + +.end: + mova m4, [o(pw_2048)] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + +.end2: + pxor m6, m6 + mova [coeffq+16*0], m6 + mova [coeffq+16*1], m6 + mova [coeffq+16*2], m6 + mova [coeffq+16*3], m6 +.end3: + WRITE_8X4 0, 1, 2, 3, 4, 5, 6 + RET + +ALIGN function_align +cglobal_label .main + punpckhwd m6, m0, m2 ;unpacked in0 in2 + punpcklwd m0, m2 ;unpacked in0 in2 + punpckhwd m7, m1, m3 ;unpacked in1 in3 + punpcklwd m1, m3 ;unpacked in1 in3 + + mova m2, [o(pw_3344_m3344)] + mova m4, [o(pw_0_3344)] + pmaddwd m3, m2, m6 ;3344 * in0 - 3344 * in2 + pmaddwd m5, m4, m7 ;3344 * in3 + pmaddwd m2, m0 + pmaddwd m4, m1 + paddd m3, m5 + paddd m2, m4 + mova m4, [o(pd_2048)] + paddd m3, m4 ;t2 + 2048 + paddd m2, m4 + psrad m3, 12 + psrad m2, 12 + packssdw m2, m3 ;out2 + + pmaddwd m4, m0, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 + pmaddwd m0, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 + pmaddwd m3, m1, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 + pmaddwd m5, m1, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 + paddd m3, m4 ;t0 + t3 + + pmaddwd m1, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 + mova m4, [o(pd_2048)] + paddd m0, m4 + paddd m4, m3 ;t0 + t3 + 2048 + paddd m5, m0 ;t1 + t3 + 2048 + paddd m3, m0 + paddd m3, m1 ;t0 + t1 - t3 + 2048 + + psrad m4, 12 ;out0 + psrad m5, 12 ;out1 + psrad m3, 12 ;out3 + packssdw m0, m4, m5 ;low: out0 high: out1 + + pmaddwd m4, m6, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 + pmaddwd m6, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 + pmaddwd m1, m7, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 + pmaddwd m5, m7, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 + paddd m1, m4 ;t0 + t3 + pmaddwd m7, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 + + mova m4, [o(pd_2048)] + paddd m6, m4 + paddd m4, m1 ;t0 + t3 + 2048 + paddd m5, m6 ;t1 + t3 + 2048 + paddd m1, m6 + paddd m1, m7 ;t0 + t1 - t3 + 2048 + + psrad m4, 12 ;out0 + psrad m5, 12 ;out1 + psrad m1, 12 ;out3 + packssdw m3, m1 ;out3 + packssdw m4, m5 ;low: out0 high: out1 + + punpckhqdq m1, m0, m4 ;out1 + punpcklqdq m0, m4 ;out0 + ret + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + + shufps m0, m0, q1032 + shufps m1, m1, q1032 + call m(iadst_4x8_internal_8bpc).main + + punpckhwd m5, m3, m2 + punpcklwd m3, m2 + punpckhwd m2, m1, m0 + punpcklwd m1, m0 + + pxor m0, m0 + psubsw m4, m0, m2 + psubsw m0, m5 + punpckhdq m2, m0, m4 + punpckldq m0, m4 + punpckhdq m4, m3, m1 + punpckldq m3, m1 + punpckhwd m1, m0, m3 ;in1 + punpcklwd m0, m3 ;in0 + punpckhwd m3, m2, m4 ;in3 + punpcklwd m2, m4 ;in2 + jmp tx2q + +.pass2: + call m(iadst_8x4_internal_8bpc).main + mova m4, m0 + mova m5, m1 + mova m0, m3 + mova m1, m2 + mova m2, m5 + mova m3, m4 + jmp m(iadst_8x4_internal_8bpc).end + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [coeffq+16*0] + pmulhrsw m1, m3, [coeffq+16*1] + pmulhrsw m2, m3, [coeffq+16*2] + pmulhrsw m3, [coeffq+16*3] + paddsw m0, m0 + paddsw m1, m1 + paddsw m2, m2 + paddsw m3, m3 + + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhdq m5, m4, m1 + punpckldq m4, m1 + punpckhdq m3, m0, m2 + punpckldq m0, m2 + punpckhwd m1, m0, m4 ;in1 + punpcklwd m0, m4 ;in0 + punpcklwd m2, m3, m5 ;in2 + punpckhwd m3, m5 ;in3 + jmp tx2q + +.pass2: + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(iadst_8x4_internal_8bpc).end + +%macro INV_TXFM_8X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x8, 8, 16*4 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklwd m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mova m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m2 + psrlw m2, 3 + pmulhrsw m0, m1 + pmulhrsw m0, m2 +.end: + mov r3d, 2 + lea tx2q, [o(m(inv_txfm_add_dct_dct_8x8_8bpc).end3)] +.loop: + WRITE_8X4 0, 0, 0, 0, 1, 2, 3 + lea dstq, [dstq+strideq*2] + dec r3d + jg .loop + jmp tx2q +.end3: + RET +%endif +%endmacro + +%macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 +%if %3 + mova m7, [o(pw_2896x8)] + pmulhrsw m0, m7, [%1+%2*0] + pmulhrsw m1, m7, [%1+%2*1] + pmulhrsw m2, m7, [%1+%2*2] + pmulhrsw m3, m7, [%1+%2*3] + pmulhrsw m4, m7, [%1+%2*4] + pmulhrsw m5, m7, [%1+%2*5] + pmulhrsw m6, m7, [%1+%2*6] + pmulhrsw m7, [%1+%2*7] +%else + mova m0, [%1+%2*0] + mova m1, [%1+%2*1] + mova m2, [%1+%2*2] + mova m3, [%1+%2*3] + mova m4, [%1+%2*4] + mova m5, [%1+%2*5] + mova m6, [%1+%2*6] + mova m7, [%1+%2*7] +%endif +%endmacro + +%macro IDCT8_1D_ODDHALF 7 ; src[1-4], tmp[1-2], pd_2048 + ITX_MULSUB_2W %1, %4, %5, %6, %7, 799, 4017 ;t4a, t7a + ITX_MULSUB_2W %3, %2, %5, %6, %7, 3406, 2276, 1 ;t5a, t6a + psubsw m%2, m%4, m%5 ;t6a + paddsw m%4, m%5 ;t7 + psubsw m%5, m%1, m%3 ;t5a + paddsw m%1, m%3 ;t4 + ITX_MULSUB_2W %2, %5, %3, %6, %7, 2896, 2896, 1 ;t5, t6 +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst +INV_TXFM_8X8_FN dct, identity + +cglobal idct_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq, 16 + +.pass1: + call .main + +.pass1_end: + mova m7, [o(pw_16384)] + +.pass1_end1: + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova [rsp+gprsize+16*1], m6 + +.pass1_end2: + REPX {pmulhrsw x, m7}, m1, m3, m5 + pmulhrsw m7, [rsp+gprsize+16*0] + +cglobal_label .pass1_end3 + punpcklwd m6, m1, m5 ;10 50 11 51 12 52 13 53 + punpckhwd m1, m5 ;14 54 15 55 16 56 17 57 + punpckhwd m5, m0, m4 ;04 44 05 45 06 46 07 47 + punpcklwd m0, m4 ;00 40 01 41 02 42 03 43 + punpckhwd m4, m3, m7 ;34 74 35 75 36 76 37 77 + punpcklwd m3, m7 ;30 70 31 71 32 72 33 73 + punpckhwd m7, m1, m4 ;16 36 56 76 17 37 57 77 + punpcklwd m1, m4 ;14 34 54 74 15 35 55 75 + punpckhwd m4, m6, m3 ;12 32 52 72 13 33 53 73 + punpcklwd m6, m3 ;10 30 50 70 11 31 51 71 + mova [rsp+gprsize+16*2], m6 + mova m6, [rsp+gprsize+16*1] + punpckhwd m3, m2, m6 ;24 64 25 65 26 66 27 67 + punpcklwd m2, m6 ;20 60 21 61 22 62 23 63 + punpckhwd m6, m5, m3 ;06 26 46 66 07 27 47 67 + punpcklwd m5, m3 ;04 24 44 64 05 25 45 65 + punpckhwd m3, m0, m2 ;02 22 42 62 03 23 43 63 + punpcklwd m0, m2 ;00 20 40 60 01 21 41 61 + + punpckhwd m2, m6, m7 ;07 17 27 37 47 57 67 77 + punpcklwd m6, m7 ;06 16 26 36 46 56 66 76 + mova [rsp+gprsize+16*0], m2 + punpcklwd m2, m3, m4 ;02 12 22 32 42 52 62 72 + punpckhwd m3, m4 ;03 13 23 33 43 53 63 73 + punpcklwd m4, m5, m1 ;04 14 24 34 44 54 64 74 + punpckhwd m5, m1 ;05 15 25 35 45 55 65 75 + mova m7, [rsp+gprsize+16*2] + punpckhwd m1, m0, m7 ;01 11 21 31 41 51 61 71 + punpcklwd m0, m7 ;00 10 20 30 40 50 60 70 + mova m7, [rsp+gprsize+16*0] + jmp tx2q + +.pass2: + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + +.pass2_main: + call .main + +.end: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova [rsp+gprsize+16*1], m6 + +.end2: + REPX {pmulhrsw x, m7}, m1, m3, m5 + pmulhrsw m7, [rsp+gprsize+16*0] + mova [rsp+gprsize+16*2], m5 + mova [rsp+gprsize+16*0], m7 + +.end3: + WRITE_8X4 0, 1, 2, 3, 5, 6, 7 + lea dstq, [dstq+strideq*2] + WRITE_8X4 4, [rsp+gprsize+16*2], [rsp+gprsize+16*1], [rsp+gprsize+16*0], 5, 6, 7 + jmp tx2q + +.end4: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + ret + +ALIGN function_align +cglobal_label .main + mova [rsp+gprsize*2+16*0], m7 + mova [rsp+gprsize*2+16*1], m3 + mova [rsp+gprsize*2+16*2], m1 + mova m7, [o(pd_2048)] + IDCT4_1D 0, 2, 4, 6, 1, 3, 7 + mova m3, [rsp+gprsize*2+16*2] + mova [rsp+gprsize*2+16*2], m2 + mova m2, [rsp+gprsize*2+16*1] + mova [rsp+gprsize*2+16*1], m4 + mova m4, [rsp+gprsize*2+16*0] + mova [rsp+gprsize*2+16*0], m6 + IDCT8_1D_ODDHALF 3, 2, 5, 4, 1, 6, 7 + mova m6, [rsp+gprsize*2+16*0] + psubsw m7, m0, m4 ;out7 + paddsw m0, m4 ;out0 + mova [rsp+gprsize*2+16*0], m7 + mova m1, [rsp+gprsize*2+16*2] + psubsw m4, m6, m3 ;out4 + paddsw m3, m6 ;out3 + mova m7, [rsp+gprsize*2+16*1] + psubsw m6, m1, m5 ;out6 + paddsw m1, m5 ;out1 + psubsw m5, m7, m2 ;out5 + paddsw m2, m7 ;out2 + ret + + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity + +cglobal iadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq, 16 + +.pass1: + call .main + call .main_pass1_end + +.pass1_end: + mova m7, [o(pw_16384)] + +.pass1_end1: + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova [rsp+gprsize+16*1], m6 + pxor m6, m6 + psubw m6, m7 + mova m7, m6 + jmp m(idct_8x8_internal_8bpc).pass1_end2 + +ALIGN function_align +.pass2: + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + +.pass2_main: + call .main + call .main_pass2_end + +.end: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova [rsp+gprsize+16*1], m6 + pxor m6, m6 + psubw m6, m7 + mova m7, m6 + jmp m(idct_8x8_internal_8bpc).end2 + +ALIGN function_align +cglobal_label .main + mova [rsp+gprsize*2+16*0], m7 + mova [rsp+gprsize*2+16*1], m3 + mova [rsp+gprsize*2+16*2], m4 + mova m7, [o(pd_2048)] + ITX_MULSUB_2W 5, 2, 3, 4, 7, 1931, 3612 ;t3a, t2a + ITX_MULSUB_2W 1, 6, 3, 4, 7, 3920, 1189 ;t7a, t6a + paddsw m3, m2, m6 ;t2 + psubsw m2, m6 ;t6 + paddsw m4, m5, m1 ;t3 + psubsw m5, m1 ;t7 + ITX_MULSUB_2W 5, 2, 1, 6, 7, 3784, 1567 ;t6a, t7a + + mova m6, [rsp+gprsize*2+16*2] + mova [rsp+gprsize*2+16*2], m5 + mova m1, [rsp+gprsize*2+16*1] + mova [rsp+gprsize*2+16*1], m2 + mova m5, [rsp+gprsize*2+16*0] + mova [rsp+gprsize*2+16*0], m3 + ITX_MULSUB_2W 5, 0, 2, 3, 7, 401, 4076 ;t1a, t0a + ITX_MULSUB_2W 1, 6, 2, 3, 7, 3166, 2598 ;t5a, t4a + psubsw m2, m0, m6 ;t4 + paddsw m0, m6 ;t0 + paddsw m3, m5, m1 ;t1 + psubsw m5, m1 ;t5 + ITX_MULSUB_2W 2, 5, 1, 6, 7, 1567, 3784 ;t5a, t4a + + mova m7, [rsp+gprsize*2+16*0] + paddsw m1, m3, m4 ;-out7 + psubsw m3, m4 ;t3 + mova [rsp+gprsize*2+16*0], m1 + psubsw m4, m0, m7 ;t2 + paddsw m0, m7 ;out0 + mova m6, [rsp+gprsize*2+16*2] + mova m7, [rsp+gprsize*2+16*1] + paddsw m1, m5, m6 ;-out1 + psubsw m5, m6 ;t6 + paddsw m6, m2, m7 ;out6 + psubsw m2, m7 ;t7 + ret +ALIGN function_align +.main_pass1_end: + mova [rsp+gprsize*2+16*1], m1 + mova [rsp+gprsize*2+16*2], m6 + punpckhwd m1, m4, m3 + punpcklwd m4, m3 + punpckhwd m7, m5, m2 + punpcklwd m5, m2 + mova m2, [o(pw_2896_2896)] + mova m6, [o(pd_2048)] + pmaddwd m3, m2, m7 + pmaddwd m2, m5 + paddd m3, m6 + paddd m2, m6 + psrad m3, 12 + psrad m2, 12 + packssdw m2, m3 ;out2 + mova m3, [o(pw_2896_m2896)] + pmaddwd m7, m3 + pmaddwd m5, m3 + paddd m7, m6 + paddd m5, m6 + psrad m7, 12 + psrad m5, 12 + packssdw m5, m7 ;-out5 + mova m3, [o(pw_2896_2896)] + pmaddwd m7, m3, m1 + pmaddwd m3, m4 + paddd m7, m6 + paddd m3, m6 + psrad m7, 12 + psrad m3, 12 + packssdw m3, m7 ;-out3 + mova m7, [o(pw_2896_m2896)] + pmaddwd m1, m7 + pmaddwd m4, m7 + paddd m1, m6 + paddd m4, m6 + psrad m1, 12 + psrad m4, 12 + packssdw m4, m1 ;-out5 + mova m1, [rsp+gprsize*2+16*1] + mova m6, [rsp+gprsize*2+16*2] + ret +ALIGN function_align +cglobal_label .main_pass2_end + paddsw m7, m4, m3 ;t2 + t3 + psubsw m4, m3 ;t2 - t3 + paddsw m3, m5, m2 ;t6 + t7 + psubsw m5, m2 ;t6 - t7 + mova m2, [o(pw_2896x8)] + pmulhrsw m4, m2 ;out4 + pmulhrsw m5, m2 ;-out5 + pmulhrsw m7, m2 ;-out3 + pmulhrsw m2, m3 ;out2 + mova m3, m7 + ret + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity + +cglobal iflipadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq, 16 + +.pass1: + call m(iadst_8x8_internal_8bpc).main + call m(iadst_8x8_internal_8bpc).main_pass1_end + +.pass1_end: + mova m7, [o(pw_m16384)] + +.pass1_end1: + pmulhrsw m1, m7 + mova [rsp+gprsize+16*1], m1 + mova m1, m6 + mova m6, m2 + pmulhrsw m2, m5, m7 + mova m5, m6 + mova m6, m4 + pmulhrsw m4, m3, m7 + mova m3, m6 + mova m6, m0 + mova m0, m7 + pxor m7, m7 + psubw m7, m0 + pmulhrsw m0, [rsp+gprsize+16*0] + REPX {pmulhrsw x, m7}, m1, m3, m5 + pmulhrsw m7, m6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +ALIGN function_align +.pass2: + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + +.pass2_main: + call m(iadst_8x8_internal_8bpc).main + call m(iadst_8x8_internal_8bpc).main_pass2_end + +.end: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova [rsp+gprsize+16*2], m2 + mova m2, m0 + pxor m0, m0 + psubw m0, m7 + mova m7, m2 + pmulhrsw m1, m0 + pmulhrsw m2, m5, m0 + mova [rsp+gprsize+16*1], m1 + mova m5, m4 + mova m1, m6 + pmulhrsw m4, m3, m0 + pmulhrsw m0, [rsp+gprsize+16*0] + mova m3, m5 + mova [rsp+gprsize+16*0], m7 + jmp m(idct_8x8_internal_8bpc).end3 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq, 16 + mova [rsp+gprsize+16*1], m6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +ALIGN function_align +.pass2: + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + +.end: + pmulhrsw m7, [o(pw_4096)] + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_4096)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + mova [rsp+gprsize+16*2], m5 + mova [rsp+gprsize+16*1], m6 + jmp m(idct_8x8_internal_8bpc).end3 + + +%macro INV_TXFM_4X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x16, 8 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklwd m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mov [coeffq], eobd + pmulhrsw m0, [o(pw_16384)] + pmulhrsw m0, m1 + pmulhrsw m0, [o(pw_2048)] +.end: + WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 + lea dstq, [dstq+strideq*4] + WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 + lea dstq, [dstq+strideq*4] + WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 + lea dstq, [dstq+strideq*4] + WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 + RET +%endif +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst +INV_TXFM_4X16_FN dct, identity + +cglobal idct_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(idct_4x8_internal_8bpc).pass1)] + +.pass1: + mova m0, [coeffq+16*1] + mova m1, [coeffq+16*3] + mova m2, [coeffq+16*5] + mova m3, [coeffq+16*7] + push tx2q + lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_2)] + jmp r3 + +.pass1_2: + mova [coeffq+16*1], m0 + mova [coeffq+16*3], m1 + mova [coeffq+16*5], m2 + mova [coeffq+16*7], m3 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*2] + mova m2, [coeffq+16*4] + mova m3, [coeffq+16*6] + lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_end)] + jmp r3 + +.pass1_end: + pop tx2q + + mova m4, [coeffq+16*1] + mova m5, [coeffq+16*3] + mova m6, [coeffq+16*5] + mova m7, [o(pw_16384)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + + pmulhrsw m7, [coeffq+16*7] + mova [coeffq+16*7], m7 + jmp tx2q + +.pass2: + call m(idct_16x4_internal_8bpc).main + +.end: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [coeffq+16*7] + mova [coeffq+16*4], m4 + +.end1: + mova [coeffq+16*5], m5 + mova [coeffq+16*6], m6 + mov r3, coeffq + WRITE_4X8 0, 1, 3, 2 + + mova m0, [r3+16*4] + mova m1, [r3+16*5] + mova m2, [r3+16*6] + mova m3, m7 + lea dstq, [dstq+strideq*4] + WRITE_4X8 0, 1, 3, 2 + +.end2: + pxor m7, m7 + REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + ret + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity + +cglobal iadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iadst_4x8_internal_8bpc).pass1)] + jmp m(idct_4x16_internal_8bpc).pass1 + +.pass2: + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass2_end + + punpcklqdq m6, m5, m4 ;low: -out5 high: -out7 + punpckhqdq m4, m5 ;low: out8 high: out10 + punpcklqdq m5, m7, m2 ;low: out4 high: out6 + punpckhqdq m2, m7 ;low: -out9 high: -out11 + mova [coeffq+16*4], m2 + mova [coeffq+16*5], m6 + mova m2, [coeffq+16*6] + mova m6, [coeffq+16*7] + punpckhqdq m1, m6, m0 ;low: -out13 high: -out15 + punpcklqdq m0, m6 ;low: out0 high: out2 + punpckhqdq m6, m3, m2 ;low: out12 high: out14 + punpcklqdq m2, m3 ;low: -out1 high: -out3 + + mova m7, [o(pw_2048)] + +.end1: + REPX {pmulhrsw x, m7}, m0, m5, m4, m6 + pxor m3, m3 + psubw m3, m7 + mova m7, [coeffq+16*4] + REPX {pmulhrsw x, m3}, m2, m7, m1 + pmulhrsw m3, [coeffq+16*5] + mova [coeffq+16*7], m5 + + punpckhqdq m5, m4, m7 ;low: out10 high: out11 + punpcklqdq m4, m7 ;low: out8 high: out9 + punpckhqdq m7, m6, m1 ;low: out14 high: out15 + punpcklqdq m6, m1 ;low: out12 high: out13 + punpckhqdq m1, m0, m2 ;low: out2 high: out3 + punpcklqdq m0, m2 ;low: out0 high: out1 + mova [coeffq+16*4], m4 + mova m4, [coeffq+16*7] + punpcklqdq m2, m4, m3 ;low: out4 high: out5 + punpckhqdq m4, m3 ;low: out6 high: out7 + mova m3, m4 + +.end2: + mova [coeffq+16*5], m5 + mova [coeffq+16*6], m6 + mov r3, coeffq + WRITE_4X8 0, 1, 2, 3 + + mova m0, [r3+16*4] + mova m1, [r3+16*5] + mova m2, [r3+16*6] + mova m3, m7 + lea dstq, [dstq+strideq*4] + WRITE_4X8 0, 1, 2, 3 + +.end3: + pxor m7, m7 + REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + ret + + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity + +cglobal iflipadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iflipadst_4x8_internal_8bpc).pass1)] + jmp m(idct_4x16_internal_8bpc).pass1 + +.pass2: + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass2_end + + punpckhqdq m6, m5, m4 ;low: out5 high: out7 + punpcklqdq m4, m5 ;low: -out8 high: -out10 + punpckhqdq m5, m7, m2 ;low: -out4 high: -out6 + punpcklqdq m2, m7 ;low: out9 high: out11 + mova [coeffq+16*4], m2 + mova [coeffq+16*5], m6 + mova m2, [coeffq+16*6] + mova m6, [coeffq+16*7] + punpcklqdq m1, m6, m0 ;low: out13 high: out15 + punpckhqdq m0, m6 ;low: -out0 high: -out2 + punpcklqdq m6, m3, m2 ;low: -out12 high: -out14 + punpckhqdq m2, m3 ;low: out1 high: out3 + + mova m7, [o(pw_m2048)] + jmp m(iadst_4x16_internal_8bpc).end1 + + +INV_TXFM_4X16_FN identity, dct +INV_TXFM_4X16_FN identity, adst +INV_TXFM_4X16_FN identity, flipadst +INV_TXFM_4X16_FN identity, identity + +%macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] + pmulhrsw m%2, m%3, m%1 +%if %0 == 4 ; if downshifting by 1 + pmulhrsw m%2, m%4 +%else + paddsw m%1, m%1 +%endif + paddsw m%1, m%2 +%endmacro + +cglobal iidentity_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m0, [coeffq+16*1] + mova m6, [o(pw_1697x8)] + mova m1, [coeffq+16*3] + mova m2, [coeffq+16*5] + mova m3, [coeffq+16*7] + pcmpeqw m7, m7 + mov r3, tx2q + lea tx2q, [o(.pass1_2)] +.pass1: + pmulhrsw m4, m6, m0 + pmulhrsw m5, m6, m1 + pavgw m4, m0 + pcmpeqw m0, m7 + pavgw m5, m1 + pcmpeqw m1, m7 + pandn m0, m4 + pmulhrsw m4, m6, m2 + pandn m1, m5 + pmulhrsw m5, m6, m3 + pavgw m4, m2 + pcmpeqw m2, m7 + pavgw m5, m3 + pcmpeqw m3, m7 + pandn m2, m4 + pandn m3, m5 + jmp m(iadst_4x8_internal_8bpc).pass1_end +.pass1_2: + mova [coeffq+16*1], m0 + mova [coeffq+16*3], m1 + mova [coeffq+16*5], m2 + mova [coeffq+16*7], m3 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*2] + mova m2, [coeffq+16*4] + mova m3, [coeffq+16*6] + lea tx2q, [o(.pass1_end)] + jmp .pass1 +.pass1_end: + mova m4, [coeffq+16*1] + mova m5, [coeffq+16*3] + mova m6, [coeffq+16*5] + jmp r3 +.pass2: + mova m7, [o(pw_1697x16)] + mova [coeffq+16*6], m6 + REPX {IDTX16 x, 6, 7}, 0, 1, 2, 3, 4, 5 + mova m6, [coeffq+16*7] + IDTX16 6, 7, 7 + mova [coeffq+16*7], m6 + mova m6, [coeffq+16*6] + pmulhrsw m7, m6, [o(pw_1697x16)] + paddsw m6, m6 + paddsw m6, m7 + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [coeffq+16*7] + mova [coeffq+16*4], m4 + jmp m(iadst_4x16_internal_8bpc).end2 + + +%macro INV_TXFM_16X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x4, 8 +%ifidn %1_%2, dct_dct + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + mov [coeffq], eobd + mov r2d, 2 + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4_8bpc).end)] +.dconly: + pmulhrsw m0, m2 + movd m2, [o(pw_2048)] ;intentionally rip-relative + pmulhrsw m0, m1 + pmulhrsw m0, m2 + pshuflw m0, m0, q0000 + punpcklwd m0, m0 + pxor m5, m5 +.dconly_loop: + mova m1, [dstq] + mova m3, [dstq+strideq] + punpckhbw m2, m1, m5 + punpcklbw m1, m5 + punpckhbw m4, m3, m5 + punpcklbw m3, m5 + paddw m2, m0 + paddw m1, m0 + paddw m4, m0 + paddw m3, m0 + packuswb m1, m2 + packuswb m3, m4 + mova [dstq], m1 + mova [dstq+strideq], m3 + lea dstq, [dstq+strideq*2] + dec r2d + jg .dconly_loop + jmp tx2q +.end: + RET +%endif +%endmacro + +%macro LOAD_7ROWS 2 ;src, stride + mova m0, [%1+%2*0] + mova m1, [%1+%2*1] + mova m2, [%1+%2*2] + mova m3, [%1+%2*3] + mova m4, [%1+%2*4] + mova m5, [%1+%2*5] + mova m6, [%1+%2*6] +%endmacro + +%macro SAVE_7ROWS 2 ;src, stride + mova [%1+%2*0], m0 + mova [%1+%2*1], m1 + mova [%1+%2*2], m2 + mova [%1+%2*3], m3 + mova [%1+%2*4], m4 + mova [%1+%2*5], m5 + mova [%1+%2*6], m6 +%endmacro + +%macro IDCT16_1D_PACKED_ODDHALF 7 ;src[1-4], tmp[1-3] + punpckhwd m%5, m%4, m%1 ;packed in13 in3 + punpcklwd m%1, m%4 ;packed in1 in15 + punpcklwd m%4, m%3, m%2 ;packed in9 in7 + punpckhwd m%2, m%3 ;packed in5 in11 + mova m%7, [o(pd_2048)] + ITX_MUL2X_PACK %1, %6, %7, 401, 4076, 1 ;low: t8a high: t15a + ITX_MUL2X_PACK %4, %6, %7, 3166, 2598, 1 ;low: t9a high: t14a + ITX_MUL2X_PACK %2, %6, %7, 1931, 3612, 1 ;low: t10a high: t13a + ITX_MUL2X_PACK %5, %6, %7, 3920, 1189, 1 ;low: t11a high: t12a + psubsw m%6, m%1, m%4 ;low: t9 high: t14 + paddsw m%1, m%4 ;low: t8 high: t15 + psubsw m%4, m%5, m%2 ;low: t10 high: t13 + paddsw m%5, m%2 ;low: t11 high: t12 + mova m%2, [o(deint_shuf2)] + pshufb m%6, m%2 + pshufb m%4, m%2 + ITX_MUL2X_PACK %6, %3, %7, 1567, 3784, 1 ;low: t9a high: t14a + ITX_MUL2X_PACK %4, %3, %7, m3784, 1567, 1 ;low: t10a high: t13a + psubsw m%3, m%1, m%5 ;low: t11a high: t12a + paddsw m%1, m%5 ;low: t8a high: t15a + psubsw m%5, m%6, m%4 ;low: t10 high: t13 + paddsw m%6, m%4 ;low: t9 high: t14 + pshufb m%3, m%2 + pshufb m%5, m%2 + ITX_MUL2X_PACK %3, %2, %7, 2896, 2896, 4 ;t12, t11 + ITX_MUL2X_PACK %5, %4, %7, 2896, 2896, 4 ;t13a, t10a + packssdw m%2, m%4 ;low: t11 high: t10a + packssdw m%3, m%5 ;low: t12 high: t13a + punpckhqdq m%4, m%1, m%6 ;low: t15a high: t14 + punpcklqdq m%1, m%6 ;low: t8a high: t9 +%endmacro + +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst +INV_TXFM_16X4_FN dct, identity + +cglobal idct_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_7ROWS coeffq, 16 + call .main + +.pass1_end: + punpckhwd m7, m0, m2 ;packed out1, out5 + punpcklwd m0, m2 ;packed out0, out4 + punpcklwd m2, m1, m3 ;packed out3, out7 + punpckhwd m1, m3 ;packed out2, out6 + mova [coeffq+16*6], m7 + mova m7, [coeffq+16*7] + punpckhwd m3, m4, m6 ;packed out9, out13 + punpcklwd m4, m6 ;packed out8, out12 + punpcklwd m6, m5, m7 ;packed out11, out15 + punpckhwd m5, m7 ;packed out10, out14 + +.pass1_end2: + mova m7, [o(pw_16384)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [coeffq+16*6] + mova [coeffq+16*6], m7 + +.pass1_end3: + punpckhwd m7, m3, m6 ;packed 9, 11, 13, 15 high + punpcklwd m3, m6 ;packed 9, 10, 13, 15 low + punpckhwd m6, m4, m5 ;packed 8, 10, 12, 14 high + punpcklwd m4, m5 ;packed 8, 10, 12, 14 low + punpckhwd m5, m4, m3 ;8, 9, 10, 11, 12, 13, 14, 15(1) + punpcklwd m4, m3 ;8, 9, 10, 11, 12, 13, 14, 15(0) + punpckhwd m3, m6, m7 ;8, 9, 10, 11, 12, 13, 14, 15(3) + punpcklwd m6, m7 ;8, 9, 10, 11, 12, 13, 14, 15(2) + mova [coeffq+16*7], m3 + mova m3, [coeffq+16*6] + punpckhwd m7, m3, m2 ;packed 1, 3, 5, 7 high + punpcklwd m3, m2 ;packed 1, 3, 5, 7 low + punpckhwd m2, m0, m1 ;packed 0, 2, 4, 6 high + punpcklwd m0, m1 ;packed 0, 2, 4, 6 low + punpckhwd m1, m0, m3 ;0, 1, 2, 3, 4, 5, 6, 7(1) + punpcklwd m0, m3 ;0, 1, 2, 3, 4, 5, 6, 7(0) + punpckhwd m3, m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(3) + punpcklwd m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(2) + jmp tx2q + +.pass2: + lea tx2q, [o(m(idct_8x4_internal_8bpc).pass2)] + +.pass2_end: + mova [coeffq+16*4], m4 + mova [coeffq+16*5], m5 + mova [coeffq+16*6], m6 + lea r3, [dstq+8] + call tx2q + + add coeffq, 16*4 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + mova m2, [coeffq+16*2] + mova m3, [coeffq+16*3] + mov dstq, r3 + jmp tx2q + +ALIGN function_align +cglobal_label .main + punpckhqdq m7, m0, m1 ;low:in1 high:in3 + punpcklqdq m0, m1 + punpcklqdq m1, m2, m3 + punpckhqdq m3, m2 ;low:in7 high:in5 + mova [coeffq+16*4], m7 + mova [coeffq+16*5], m3 + mova m7, [coeffq+16*7] + punpcklqdq m2, m4, m5 + punpckhqdq m4, m5 ;low:in9 high:in11 + punpcklqdq m3, m6, m7 + punpckhqdq m7, m6 ;low:in15 high:in13 + mova [coeffq+16*6], m4 + IDCT8_1D_PACKED + mova m6, [coeffq+16*4] + mova m4, [coeffq+16*5] + mova m5, [coeffq+16*6] + mova [coeffq+16*4], m1 + mova [coeffq+16*5], m2 + mova [coeffq+16*6], m3 + + IDCT16_1D_PACKED_ODDHALF 6, 4, 5, 7, 1, 2, 3 + + mova m1, [coeffq+16*4] + psubsw m3, m0, m7 ;low:out15 high:out14 + paddsw m0, m7 ;low:out0 high:out1 + psubsw m7, m1, m5 ;low:out12 high:out13 + paddsw m1, m5 ;low:out3 high:out2 + mova [coeffq+16*7], m3 + mova m2, [coeffq+16*5] + mova m3, [coeffq+16*6] + psubsw m5, m2, m4 ;low:out11 high:out10 + paddsw m2, m4 ;low:out4 high:out5 + psubsw m4, m3, m6 ;low:out8 high:out9 + paddsw m3, m6 ;low:out7 high:out6 + mova m6, m7 + ret + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_7ROWS coeffq, 16 + call .main + call .main_pass1_end + + punpckhwd m6, m7, m0 ;packed -out11, -out15 + punpcklwd m0, m7 ;packed out0, out4 + punpcklwd m7, m3, m4 ;packed -out3, -out7 + punpckhwd m4, m3 ;packed out8, out12 + mova m1, [coeffq+16*6] + punpcklwd m3, m1, m5 ;packed -out1, -out5 + punpckhwd m5, m1 ;packed out10, out14 + mova m1, [coeffq+16*7] + mova [coeffq+16*6], m3 + mova [coeffq+16*7], m7 + punpckhwd m3, m2, m1 ;packed -out9, -out13 + punpcklwd m1, m2 ;packed out2, out6 + + mova m7, [o(pw_16384)] + +.pass1_end: + REPX {pmulhrsw x, m7}, m0, m1, m4, m5 + pxor m2, m2 + psubw m2, m7 + mova m7, [coeffq+16*6] + REPX {pmulhrsw x, m2}, m7, m3, m6 + pmulhrsw m2, [coeffq+16*7] + mova [coeffq+16*6], m7 + jmp m(idct_16x4_internal_8bpc).pass1_end3 + +.pass2: + lea tx2q, [o(m(iadst_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end + +ALIGN function_align +cglobal_label .main + mova [coeffq+16*6], m0 + pshufd m0, m1, q1032 + pshufd m2, m2, q1032 + punpckhwd m1, m6, m0 ;packed in13, in2 + punpcklwd m0, m6 ;packed in3, in12 + punpckhwd m7, m5, m2 ;packed in11, in4 + punpcklwd m2, m5 ;packed in5, in10 + mova m6, [o(pd_2048)] + ITX_MUL2X_PACK 1, 5, 6, 995, 3973 ;low:t2 high:t3 + ITX_MUL2X_PACK 7, 5, 6, 1751, 3703 ;low:t4 high:t5 + ITX_MUL2X_PACK 2, 5, 6, 3513, 2106 ;low:t10 high:t11 + ITX_MUL2X_PACK 0, 5, 6, 3857, 1380 ;low:t12 high:t13 + psubsw m5, m1, m2 ;low:t10a high:t11a + paddsw m1, m2 ;low:t2a high:t3a + psubsw m2, m7, m0 ;low:t12a high:t13a + paddsw m7, m0 ;low:t4a high:t5a + punpcklqdq m0, m5 + punpckhwd m0, m5 ;packed t10a, t11a + punpcklqdq m5, m2 + punpckhwd m2, m5 ;packed t13a, t12a + ITX_MUL2X_PACK 0, 5, 6, 3406, 2276 ;low:t10 high:t11 + ITX_MUL2X_PACK 2, 5, 6, 4017, 799, 1 ;low:t12 high:t13 + mova [coeffq+16*4], m1 + mova [coeffq+16*5], m7 + mova m1, [coeffq+16*6] + mova m7, [coeffq+16*7] + pshufd m1, m1, q1032 + pshufd m3, m3, q1032 + punpckhwd m5, m7, m1 ;packed in15, in0 + punpcklwd m1, m7 ;packed in1, in14 + punpckhwd m7, m4, m3 ;packed in9, in6 + punpcklwd m3, m4 ;packed in7, in8 + ITX_MUL2X_PACK 5, 4, 6, 201, 4091 ;low:t0 high:t1 + ITX_MUL2X_PACK 7, 4, 6, 2440, 3290 ;low:t6 high:t7 + ITX_MUL2X_PACK 3, 4, 6, 3035, 2751 ;low:t8 high:t9 + ITX_MUL2X_PACK 1, 4, 6, 4052, 601 ;low:t14 high:t15 + psubsw m4, m5, m3 ;low:t8a high:t9a + paddsw m5, m3 ;low:t0a high:t1a + psubsw m3, m7, m1 ;low:t14a high:t15a + paddsw m7, m1 ;low:t6a high:t7a + punpcklqdq m1, m4 + punpckhwd m1, m4 ;packed t8a, t9a + punpcklqdq m4, m3 + punpckhwd m3, m4 ;packed t15a, t14a + ITX_MUL2X_PACK 1, 4, 6, 799, 4017 ;low:t8 high:t9 + ITX_MUL2X_PACK 3, 4, 6, 2276, 3406, 1 ;low:t14 high:t15 + paddsw m4, m1, m2 ;low:t12a high:t13a + psubsw m1, m2 ;low:t8a high:t9a + psubsw m2, m0, m3 ;low:t14a high:t15a + paddsw m0, m3 ;low:t10a high:t11a + punpcklqdq m3, m1 + punpckhwd m3, m1 ;packed t12a, t13a + punpcklqdq m1, m2 + punpckhwd m2, m1 ;packed t15a, t14a + ITX_MUL2X_PACK 3, 1, 6, 1567, 3784 ;low:t12 high:t13 + ITX_MUL2X_PACK 2, 1, 6, 3784, 1567, 1 ;low:t14 high:t15 + psubsw m1, m3, m2 ;low:t14a high:t15a + paddsw m3, m2 ;low:out2 high:-out13 + psubsw m2, m4, m0 ;low:t10 high:t11 + paddsw m0, m4 ;low:-out1 high:out14 + mova [coeffq+16*6], m0 + mova [coeffq+16*7], m3 + mova m0, [coeffq+16*4] + mova m3, [coeffq+16*5] + psubsw m4, m5, m3 ;low:t4 high:t5 + paddsw m5, m3 ;low:t0 high:t1 + psubsw m3, m0, m7 ;low:t6 high:t7 + paddsw m0, m7 ;low:t2 high:t3 + punpcklqdq m7, m4 + punpckhwd m7, m4 ;packed t4, t5 + punpcklqdq m4, m3 + punpckhwd m3, m4 ;packed t7, t6 + ITX_MUL2X_PACK 7, 4, 6, 1567, 3784 ;low:t4a high:t5a + ITX_MUL2X_PACK 3, 4, 6, 3784, 1567, 1 ;low:t6a high:t7a + psubsw m4, m5, m0 ;low:t2a high:t3a + paddsw m0, m5 ;low:out0 high:-out15 + psubsw m5, m7, m3 ;low:t6 high:t7 + paddsw m3, m7 ;low:-out3 high:out12 + ret +ALIGN function_align +.main_pass1_end: + mova m7, [o(deint_shuf1)] + mova [coeffq+16*4], m0 + mova [coeffq+16*5], m3 + mova m0, [o(pw_2896_m2896)] + mova m3, [o(pw_2896_2896)] + pshufb m1, m7 ;t14a t15a + pshufb m2, m7 ;t10 t11 + pshufb m4, m7 ;t2a t3a + pshufb m5, m7 ;t6 t7 + pmaddwd m7, m0, m2 + pmaddwd m2, m3 + paddd m7, m6 + paddd m2, m6 + psrad m7, 12 + psrad m2, 12 + packssdw m2, m7 ;low:out6 high:-out9 + pmaddwd m7, m0, m4 + pmaddwd m4, m3 + paddd m7, m6 + paddd m4, m6 + psrad m7, 12 + psrad m4, 12 + packssdw m4, m7 ;low:-out7 high:out8 + pmaddwd m7, m3, m5 + pmaddwd m5, m0 + paddd m7, m6 + paddd m5, m6 + psrad m7, 12 + psrad m5, 12 + packssdw m7, m5 ;low:out4 high:-out11 + pmaddwd m5, m3, m1 + pmaddwd m1, m0 + paddd m5, m6 + paddd m1, m6 + psrad m5, 12 + psrad m1, 12 + packssdw m5, m1 ;low:-out5 high:out10 + mova m0, [coeffq+16*4] + mova m3, [coeffq+16*5] + ret +ALIGN function_align +cglobal_label .main_pass2_end + mova m7, [o(pw_2896x8)] + punpckhqdq m6, m2, m1 ;low:t11 high:t15a + punpcklqdq m2, m1 ;low:t10 high:t14a + psubsw m1, m2, m6 + paddsw m2, m6 + punpckhqdq m6, m4, m5 ;low:t3a high:t7 + punpcklqdq m4, m5 ;low:t2a high:t6 + psubsw m5, m4, m6 + paddsw m4, m6 + pmulhrsw m1, m7 ;low:-out9 high:out10 + pmulhrsw m2, m7 ;low:out6 high:-out5 + pmulhrsw m5, m7 ;low:out8 high:-out11 + pmulhrsw m4, m7 ;low:-out7 high:out4 + punpckhqdq m7, m4, m5 ;low:out4 high:-out11 + punpcklqdq m4, m5 ;low:-out7 high:out8 + punpckhqdq m5, m2, m1 ;low:-out5 high:out10 + punpcklqdq m2, m1 ;low:out6 high:-out9 + ret + + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_7ROWS coeffq, 16 + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass1_end + + punpcklwd m6, m7, m0 ;packed out11, out15 + punpckhwd m0, m7 ;packed -out0, -out4 + punpckhwd m7, m3, m4 ;packed out3, out7 + punpcklwd m4, m3 ;packed -out8, -out12 + mova m1, [coeffq+16*6] + punpckhwd m3, m1, m5 ;packed out1, out5 + punpcklwd m5, m1 ;packed -out10, -out14 + mova m1, [coeffq+16*7] + mova [coeffq+16*6], m3 + mova [coeffq+16*7], m7 + punpcklwd m3, m2, m1 ;packed out9, out13 + punpckhwd m1, m2 ;packed -out2, -out6 + + mova m7, [o(pw_m16384)] + jmp m(iadst_16x4_internal_8bpc).pass1_end + +.pass2: + lea tx2q, [o(m(iflipadst_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end + + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m1, [coeffq+16*6] + mova m0, [coeffq+16*5] + mova m2, [coeffq+16*7] + mova m6, [o(pw_1697x16)] + mova m7, [o(pw_16384)] + pmulhrsw m4, m6, m1 + pmulhrsw m3, m6, m0 + pmulhrsw m5, m6, m2 + pmulhrsw m4, m7 + pmulhrsw m3, m7 + pmulhrsw m5, m7 + paddsw m1, m4 + paddsw m0, m3 + paddsw m5, m2 + mova m2, [coeffq+16*2] + mova m3, [coeffq+16*3] + mova m4, [coeffq+16*4] + mova [coeffq+16*6], m1 + mova [coeffq+16*5], m0 + mova [coeffq+16*7], m5 + pmulhrsw m0, m6, m2 + pmulhrsw m1, m6, m3 + pmulhrsw m5, m6, m4 + pmulhrsw m0, m7 + pmulhrsw m1, m7 + pmulhrsw m5, m7 + paddsw m2, m0 + paddsw m3, m1 + paddsw m4, m5 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + pmulhrsw m5, m6, m0 + pmulhrsw m6, m1 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + paddsw m0, m5 + paddsw m1, m6 + mova m6, [coeffq+16*6] + mova m5, [coeffq+16*5] + punpckhwd m7, m0, m2 ;packed out1, out5 + punpcklwd m0, m2 ;packed out0, out4 + punpckhwd m2, m1, m3 ;packed out3, out7 + punpcklwd m1, m3 ;packed out2, out6 + mova [coeffq+16*6], m7 + mova m7, [coeffq+16*7] + punpckhwd m3, m4, m6 ;packed out9, out13 + punpcklwd m4, m6 ;packed out8, out12 + punpckhwd m6, m5, m7 ;packed out11, out15 + punpcklwd m5, m7 ;packed out10, out14 + jmp m(idct_16x4_internal_8bpc).pass1_end3 + +.pass2: + lea tx2q, [o(m(iidentity_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end + + +%macro SAVE_8ROWS 2 ;src, stride + mova [%1+%2*0], m0 + mova [%1+%2*1], m1 + mova [%1+%2*2], m2 + mova [%1+%2*3], m3 + mova [%1+%2*4], m4 + mova [%1+%2*5], m5 + mova [%1+%2*6], m6 + mova [%1+%2*7], m7 +%endmacro + +%macro INV_TXFM_8X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x16, 8, 16*16 +%ifidn %1_%2, dct_dct + pshuflw m0, [coeffq], q0000 + punpcklwd m0, m0 + mova m1, [o(pw_2896x8)] + pmulhrsw m0, m1 + mova m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m1 + pmulhrsw m0, m2 + psrlw m2, 3 ; pw_2048 + pmulhrsw m0, m1 + pmulhrsw m0, m2 + mov r3d, 4 + lea tx2q, [o(m(inv_txfm_add_dct_dct_8x16_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop +.end: + RET +%endif +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst +INV_TXFM_8X16_FN dct, identity + +cglobal idct_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(idct_8x8_internal_8bpc).pass1)] + +.pass1: + LOAD_8ROWS coeffq+16*1, 32, 1 + mov [rsp+gprsize+16*11], tx2q + lea tx2q, [o(m(idct_8x16_internal_8bpc).pass1_end)] + jmp r3 + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS coeffq+16*0, 32, 1 + mov tx2q, [rsp+gprsize+16*11] + jmp r3 + +.pass2: + lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] + +.pass2_pre: + mova [coeffq+16*2 ], m1 + mova [coeffq+16*6 ], m3 + mova [coeffq+16*10], m5 + mova [coeffq+16*14], m7 + mova m1, m2 + mova m2, m4 + mova m3, m6 + mova m4, [coeffq+16*1 ] + mova m5, [coeffq+16*5 ] + mova m6, [coeffq+16*9 ] + mova m7, [coeffq+16*13] + +.pass2_main: + call m(idct_8x8_internal_8bpc).main + + SAVE_7ROWS rsp+gprsize+16*3, 16 + mova m0, [coeffq+16*2 ] + mova m1, [coeffq+16*6 ] + mova m2, [coeffq+16*10] + mova m3, [coeffq+16*14] + mova m4, [coeffq+16*3 ] + mova m5, [coeffq+16*7 ] + mova m6, [coeffq+16*11] + mova m7, [coeffq+16*15] + call m(idct_16x8_internal_8bpc).main + + mov r3, dstq + lea dstq, [dstq+strideq*8] + jmp m(idct_8x8_internal_8bpc).end + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(idct_8x8_internal_8bpc).end + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ret + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity + +cglobal iadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iadst_8x8_internal_8bpc).pass1)] + jmp m(idct_8x16_internal_8bpc).pass1 + +.pass2: + lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] + +.pass2_pre: + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*5], m6 + mova [rsp+gprsize+16*6], m7 + mova m0, m2 + mova m1, m3 + mova m2, m4 + mova m3, m5 + +.pass2_main: + mova m4, [coeffq+16*1 ] + mova m5, [coeffq+16*3 ] + mova m6, [coeffq+16*13] + mova m7, [coeffq+16*15] + mova [rsp+gprsize+16*3], m4 + mova [rsp+gprsize+16*4], m5 + mova [rsp+gprsize+16*9], m6 + mova [rsp+gprsize+32*5], m7 + mova m4, [coeffq+16*5 ] + mova m5, [coeffq+16*7 ] + mova m6, [coeffq+16*9 ] + mova m7, [coeffq+16*11] + + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + + mov r3, dstq + lea dstq, [dstq+strideq*8] + jmp m(iadst_8x8_internal_8bpc).end + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(iadst_8x8_internal_8bpc).end + + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity + +cglobal iflipadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iflipadst_8x8_internal_8bpc).pass1)] + jmp m(idct_8x16_internal_8bpc).pass1 + +.pass2: + lea tx2q, [o(m(iflipadst_8x16_internal_8bpc).end)] + lea r3, [dstq+strideq*8] + +.pass2_pre: + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*5], m6 + mova [rsp+gprsize+16*6], m7 + mova m0, m2 + mova m1, m3 + mova m2, m4 + mova m3, m5 + +.pass2_main: + mova m4, [coeffq+16*1 ] + mova m5, [coeffq+16*3 ] + mova m6, [coeffq+16*13] + mova m7, [coeffq+16*15] + mova [rsp+gprsize+16*3], m4 + mova [rsp+gprsize+16*4], m5 + mova [rsp+gprsize+16*9], m6 + mova [rsp+gprsize+32*5], m7 + mova m4, [coeffq+16*5 ] + mova m5, [coeffq+16*7 ] + mova m6, [coeffq+16*9 ] + mova m7, [coeffq+16*11] + + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + jmp m(iflipadst_8x8_internal_8bpc).end + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(iflipadst_8x8_internal_8bpc).end + + +INV_TXFM_8X16_FN identity, dct +INV_TXFM_8X16_FN identity, adst +INV_TXFM_8X16_FN identity, flipadst +INV_TXFM_8X16_FN identity, identity + +cglobal iidentity_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq+16*1, 32, 1 + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + mova [rsp+gprsize+16*1], m6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS coeffq+16*0, 32, 1 + mov tx2q, r3 + mova [rsp+gprsize+16*1], m6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +.pass2: + lea tx2q, [o(.end1)] + +.end: + mova [rsp+gprsize+16*0], m7 + mova [rsp+gprsize+16*1], m6 + mova m7, [o(pw_1697x16)] + REPX {IDTX16 x, 6, 7}, 0, 1, 2, 3, 4, 5 + mova m6, [rsp+gprsize+16*1] + mova [rsp+gprsize+16*2], m5 + IDTX16 6, 5, 7 + mova m5, [rsp+gprsize+16*0] + IDTX16 5, 7, 7 + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [rsp+gprsize+16*2] + mova [rsp+gprsize+16*0], m5 + mova [rsp+gprsize+16*1], m6 + mova [rsp+gprsize+16*2], m7 + jmp m(idct_8x8_internal_8bpc).end3 + +.end1: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + lea dstq, [dstq+strideq*2] + jmp .end + + +%macro INV_TXFM_16X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x8, 8, 16*16 +%ifidn %1_%2, dct_dct + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m1 + mov r2d, 4 + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +.end: + RET +%endif +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst +INV_TXFM_16X8_FN dct, identity + +cglobal idct_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq+16*0, 32, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+16*1, 32, 1 + call .main + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass2: + lea tx2q, [o(.end)] + lea r3, [dstq+8] + jmp m(idct_8x8_internal_8bpc).pass2_main + +.end: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(idct_8x8_internal_8bpc).pass2_main + + +ALIGN function_align +cglobal_label .main + mova [rsp+gprsize*2+16*1], m2 + mova [rsp+gprsize*2+16*2], m6 + mova [rsp+gprsize*2+32*5], m5 + + mova m6, [o(pd_2048)] + ITX_MULSUB_2W 0, 7, 2, 5, 6, 401, 4076 ;t8a, t15a + ITX_MULSUB_2W 4, 3, 2, 5, 6, 3166, 2598 ;t9a, t14a + psubsw m2, m0, m4 ;t9 + paddsw m0, m4 ;t8 + psubsw m4, m7, m3 ;t14 + paddsw m7, m3 ;t15 + ITX_MULSUB_2W 4, 2, 3, 5, 6, 1567, 3784 ;t9a, t14a + mova m3, [rsp+gprsize*2+16*1] + mova m5, [rsp+gprsize*2+32*5] + mova [rsp+gprsize*2+16*1], m2 + mova [rsp+gprsize*2+32*5], m4 + mova m2, [rsp+gprsize*2+16*2] + mova [rsp+gprsize*2+16*2], m7 + ITX_MULSUB_2W 3, 5, 7, 4, 6, 1931, 3612 ;t10a, t13a + ITX_MULSUB_2W 2, 1, 7, 4, 6, 3920, 1189 ;t11a, t12a + psubsw m4, m2, m3 ;t10 + paddsw m2, m3 ;t11 + psubsw m3, m1, m5 ;t13 + paddsw m1, m5 ;t12 + ITX_MULSUB_2W 3, 4, 7, 5, 6, m3784, 1567 ;t10a, t13a + mova m7, [rsp+gprsize*2+32*5] + psubsw m6, m0, m2 ;t11a + paddsw m0, m2 ;t8a + paddsw m2, m7, m3 ;t9 + psubsw m7, m3 ;t10 + mova m5, [rsp+gprsize*2+16*0] + psubsw m3, m5, m0 ;out8 + paddsw m0, m5 ;out7 + mova [rsp+gprsize*2+32*5], m0 + mova m5, [rsp+gprsize*2+16*9] + psubsw m0, m5, m2 ;out9 + paddsw m2, m5 ;out6 + mova [rsp+gprsize*2+16*0], m0 + mova [rsp+gprsize*2+16*9], m2 + mova m0, [rsp+gprsize*2+16*1] + mova m2, [rsp+gprsize*2+16*2] + mova [rsp+gprsize*2+16*1], m3 + psubsw m5, m0, m4 ;t13 + paddsw m0, m4 ;t14 + mova m3, [o(pd_2048)] + psubsw m4, m2, m1 ;t12a + paddsw m1, m2 ;t15a + mova [rsp+gprsize*2+16*2], m1 + ITX_MULSUB_2W 5, 7, 1, 2, 3, 2896, 2896 ;t10a, t13a + ITX_MULSUB_2W 4, 6, 1, 2, 3, 2896, 2896 ;t11, t12 + mova m3, [rsp+gprsize*2+16*8] + psubsw m2, m3, m5 ;out10 + paddsw m3, m5 ;out5 + mova m5, [rsp+gprsize*2+16*7] + mova [rsp+gprsize*2+16*8], m3 + psubsw m3, m5, m4 ;out11 + paddsw m5, m4 ;out4 + mova m4, [rsp+gprsize*2+16*6] + mova [rsp+gprsize*2+16*7], m5 + paddsw m5, m4, m6 ;out3 + psubsw m4, m6 ;out12 + mova m6, [rsp+gprsize*2+16*5] + mova [rsp+gprsize*2+16*6], m5 + psubsw m5, m6, m7 ;out13 + paddsw m6, m7 ;out2 + mova m7, [rsp+gprsize*2+16*4] + mova [rsp+gprsize*2+16*5], m6 + psubsw m6, m7, m0 ;out14 + paddsw m7, m0 ;out1 + mova m1, [rsp+gprsize*2+16*2] + mova m0, [rsp+gprsize*2+16*3] + mova [rsp+gprsize*2+16*4], m7 + psubsw m7, m0, m1 ;out15 + paddsw m0, m1 ;out0 + mova [rsp+gprsize*2+16*3], m0 + mova m1, [rsp+gprsize*2+16*0] + mova m0, [rsp+gprsize*2+16*1] + mova [rsp+gprsize*2+16*0], m7 + ret + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity + +cglobal iadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m7, [o(pw_2896x8)] + pmulhrsw m0, m7, [coeffq+16*0 ] + pmulhrsw m1, m7, [coeffq+16*1 ] + pmulhrsw m2, m7, [coeffq+16*14] + pmulhrsw m3, m7, [coeffq+16*15] + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*9], m2 + mova [rsp+gprsize+32*5], m3 + pmulhrsw m0, m7, [coeffq+16*6 ] + pmulhrsw m1, m7, [coeffq+16*7 ] + pmulhrsw m2, m7, [coeffq+16*8 ] + pmulhrsw m3, m7, [coeffq+16*9 ] + mova [rsp+gprsize+16*3], m2 + mova [rsp+gprsize+16*4], m3 + mova [rsp+gprsize+16*5], m0 + mova [rsp+gprsize+16*6], m1 + pmulhrsw m0, m7, [coeffq+16*2 ] + pmulhrsw m1, m7, [coeffq+16*3 ] + pmulhrsw m2, m7, [coeffq+16*4 ] + pmulhrsw m3, m7, [coeffq+16*5 ] + pmulhrsw m4, m7, [coeffq+16*10] + pmulhrsw m5, m7, [coeffq+16*11] + pmulhrsw m6, m7, [coeffq+16*12] + pmulhrsw m7, [coeffq+16*13] + + call .main + call .main_pass1_end + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + jmp m(iadst_8x8_internal_8bpc).pass1_end + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + jmp m(iadst_8x8_internal_8bpc).pass1_end + +.pass2: + lea tx2q, [o(.end)] + lea r3, [dstq+8] + jmp m(iadst_8x8_internal_8bpc).pass2_main + +.end: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(iadst_8x8_internal_8bpc).pass2_main + +ALIGN function_align +cglobal_label .main + mova [rsp+gprsize*2+16*0], m1 + mova [rsp+gprsize*2+16*1], m2 + mova [rsp+gprsize*2+16*2], m6 + + mova m6, [o(pd_2048)] + ITX_MULSUB_2W 7, 0, 1, 2, 6, 995, 3973 ;t3, t2 + ITX_MULSUB_2W 3, 4, 1, 2, 6, 3513, 2106 ;t11, t10 + psubsw m1, m0, m4 ;t10a + paddsw m0, m4 ;t2a + psubsw m4, m7, m3 ;t11a + paddsw m3, m7 ;t3a + ITX_MULSUB_2W 1, 4, 7, 2, 6, 3406, 2276 ;t11, t10 + mova m2, [rsp+gprsize*2+16*0] ;in3 + mova m7, [rsp+gprsize*2+16*1] ;in4 + mova [rsp+gprsize*2+16*0], m1 ;t11 + mova [rsp+gprsize*2+16*1], m4 ;t10 + mova m1, [rsp+gprsize*2+16*2] ;in12 + mova [rsp+gprsize*2+16*2], m0 ;t2a + ITX_MULSUB_2W 5, 7, 0, 4, 6, 1751, 3703 ;t5, t4 + ITX_MULSUB_2W 2, 1, 0, 4, 6, 3857, 1380 ;t13, t12 + psubsw m0, m7, m1 ;t12a + paddsw m1, m7 ;t4a + psubsw m4, m5, m2 ;t13a + paddsw m5, m2 ;t5a + ITX_MULSUB_2W 4, 0, 7, 2, 6, 4017, 799 ;t12, t13 + mova m2, [rsp+gprsize*2+16*8] ;in1 + mova m7, [rsp+gprsize*2+16*9] ;in14 + mova [rsp+gprsize*2+16*8], m4 ;t12 + mova [rsp+gprsize*2+16*9], m0 ;t13 + mova m4, [rsp+gprsize*2+16*4] ;in9 + mova m0, [rsp+gprsize*2+16*5] ;in6 + mova [rsp+gprsize*2+16*4], m1 ;t4a + mova [rsp+gprsize*2+16*5], m5 ;t5a + ITX_MULSUB_2W 2, 7, 1, 5, 6, 4052, 601 ;t15, t14 + ITX_MULSUB_2W 4, 0, 1, 5, 6, 2440, 3290 ;t7, t6 + psubsw m1, m0, m7 ;t14a + paddsw m0, m7 ;t6a + psubsw m5, m4, m2 ;t15a + paddsw m4, m2 ;t7a + ITX_MULSUB_2W 5, 1, 7, 2, 6, 2276, 3406 ;t14, t15 + mova m2, [rsp+gprsize*2+16*2] ;t2a + mova [rsp+gprsize*2+16*2], m5 ;t14 + psubsw m7, m2, m0 ;t6 + paddsw m2, m0 ;t2 + psubsw m0, m3, m4 ;t7 + paddsw m3, m4 ;t3 + ITX_MULSUB_2W 0, 7, 4, 5, 6, 3784, 1567 ;t6a, t7a + mova m4, [rsp+gprsize*2+16*7] ;in0 + mova m5, [rsp+gprsize*2+32*5] ;in15 + mova [rsp+gprsize*2+16*7], m3 ;t3 + mova [rsp+gprsize*2+32*5], m1 ;t15 + mova m1, [rsp+gprsize*2+16*6] ;in7 + mova m3, [rsp+gprsize*2+16*3] ;in8 + mova [rsp+gprsize*2+16*6], m7 ;t7a + mova [rsp+gprsize*2+16*3], m0 ;t6a + ITX_MULSUB_2W 5, 4, 0, 7, 6, 201, 4091 ;t1, t0 + ITX_MULSUB_2W 1, 3, 0, 7, 6, 3035, 2751 ;t9, t8 + psubsw m0, m4, m3 ;t8a + paddsw m4, m3 ;t0a + psubsw m3, m5, m1 ;t9a + paddsw m5, m1 ;t1a + ITX_MULSUB_2W 0, 3, 1, 7, 6, 799, 4017 ;t9, t8 + mova m1, [rsp+gprsize*2+16*4] ;t4a + mova m7, [rsp+gprsize*2+16*5] ;t5a + mova [rsp+gprsize*2+16*4], m3 ;t8 + mova [rsp+gprsize*2+16*5], m0 ;t9 + psubsw m0, m4, m1 ;t4 + paddsw m4, m1 ;t0 + psubsw m3, m5, m7 ;t5 + paddsw m5, m7 ;t1 + ITX_MULSUB_2W 0, 3, 1, 7, 6, 1567, 3784 ;t5a, t4a + mova m7, [rsp+gprsize*2+16*3] ;t6a + psubsw m1, m4, m2 ;t2a + paddsw m4, m2 ;out0 + mova [rsp+gprsize*2+16*3], m4 ;out0 + mova m4, [rsp+gprsize*2+16*6] ;t7a + psubsw m2, m3, m7 ;t6 + paddsw m3, m7 ;-out3 + mova [rsp+gprsize*2+16*6], m3 ;-out3 + psubsw m3, m0, m4 ;t7 + paddsw m0, m4 ;out12 + mova [rsp+gprsize*2+16*12], m3 + mova m3, [rsp+gprsize*2+16*7] ;t3 + mova [rsp+gprsize*2+16* 7], m2 ;out4 + psubsw m2, m5, m3 ;t3a + paddsw m5, m3 ;-out15 + mova [rsp+gprsize*2+16*11], m2 + mova m2, [rsp+gprsize*2+32*5] ;t15 + mova [rsp+gprsize*2+16*10], m1 ;-out7 + mova m1, [rsp+gprsize*2+16*0] ;t11 + mova [rsp+gprsize*2+16*0 ], m5 ;-out15 + mova m3, [rsp+gprsize*2+16*1] ;t10 + mova [rsp+gprsize*2+16*1 ], m4 ;-out11 + mova m4, [rsp+gprsize*2+16*2] ;t14 + mova [rsp+gprsize*2+16*2 ], m0 ;out12 + psubsw m0, m3, m4 ;t14a + paddsw m3, m4 ;t10a + psubsw m5, m1, m2 ;t15a + paddsw m1, m2 ;t11a + ITX_MULSUB_2W 5, 0, 2, 4, 6, 3784, 1567 ;t14, t15 + mova m2, [rsp+gprsize*2+16*4] ;t8 + mova m4, [rsp+gprsize*2+16*5] ;t9 + mova [rsp+gprsize*2+16*4], m3 ;t10a + mova [rsp+gprsize*2+16*5], m1 ;t11a + mova m3, [rsp+gprsize*2+16*8] ;t12 + mova m1, [rsp+gprsize*2+16*9] ;t13 + mova [rsp+gprsize*2+16*8], m5 ;t14 + mova [rsp+gprsize*2+16*9], m0 ;t15 + psubsw m5, m2, m3 ;t12a + paddsw m2, m3 ;t8a + psubsw m0, m4, m1 ;t13a + paddsw m4, m1 ;t9a + ITX_MULSUB_2W 5, 0, 1, 3, 6, 1567, 3784 ;t13, t12 + mova m6, [rsp+gprsize*2+16*4] ;t10a + mova m1, [rsp+gprsize*2+16*5] ;t11a + psubsw m3, m2, m6 ;t10 + paddsw m2, m6 ;-out1 + paddsw m6, m4, m1 ;out14 + psubsw m4, m1 ;t11 + mova [rsp+gprsize*2+16*14], m4 + mova [rsp+gprsize*2+16* 4], m2 ;-out1 + mova m4, [rsp+gprsize*2+16*8] ;t14 + mova m2, [rsp+gprsize*2+16*9] ;t15 + mova [rsp+gprsize*2+16* 9], m3 ;out6 + psubsw m3, m0, m4 ;t14a + paddsw m0, m4 ;out2 + psubsw m4, m5, m2 ;t15a + paddsw m5, m2 ;-out13 + mova [rsp+gprsize*2+16* 5], m0 ;out2 + ret +ALIGN function_align +.main_pass1_end: + mova m0, [rsp+gprsize*2+16*14] + mova [rsp+gprsize*2+16*14], m5 + mova [rsp+gprsize*2+16*15], m6 + mova m5, [o(pw_2896_2896)] + mova m6, [o(pw_2896_m2896)] + mova m7, [o(pd_2048)] + punpcklwd m2, m3, m4 + punpckhwd m3, m4 + pmaddwd m4, m5, m2 + pmaddwd m2, m6 + pmaddwd m1, m5, m3 + pmaddwd m3, m6 + REPX {paddd x, m7}, m4, m2, m1, m3 + REPX {psrad x, 12}, m4, m1, m2, m3 + packssdw m4, m1 ;-out5 + packssdw m2, m3 ;out10 + mova [rsp+gprsize*2+16* 8], m4 + mova m3, [rsp+gprsize*2+16* 9] + punpcklwd m1, m3, m0 + punpckhwd m3, m0 + pmaddwd m0, m5, m1 + pmaddwd m1, m6 + pmaddwd m4, m5, m3 + pmaddwd m3, m6 + REPX {paddd x, m7}, m0, m1, m4, m3 + REPX {psrad x, 12}, m0, m4, m1, m3 + packssdw m0, m4 ;out6 + packssdw m1, m3 ;-out9 + mova [rsp+gprsize*2+16* 9], m0 + mova m0, [rsp+gprsize*2+16* 7] + mova m4, [rsp+gprsize*2+16*12] + punpcklwd m3, m0, m4 + punpckhwd m0, m4 + pmaddwd m4, m5, m3 + pmaddwd m3, m6 + pmaddwd m5, m0 + pmaddwd m0, m6 + REPX {paddd x, m7}, m4, m3, m5, m0 + REPX {psrad x, 12}, m4, m5, m3, m0 + packssdw m4, m5 ;out4 + packssdw m3, m0 ;-out11 + mova [rsp+gprsize*2+16* 7], m4 + mova m4, [rsp+gprsize*2+16*10] + mova m5, [rsp+gprsize*2+16*11] + punpcklwd m0, m4, m5 + punpckhwd m4, m5 + pmaddwd m5, m0, [o(pw_2896_2896)] + pmaddwd m0, m6 + pmaddwd m6, m4 + pmaddwd m4, [o(pw_2896_2896)] + REPX {paddd x, m7}, m5, m0, m6, m4 + REPX {psrad x, 12}, m0, m6, m5, m4 + packssdw m0, m6 ;out8 + packssdw m5, m4 ;-out7 + mova [rsp+gprsize*2+16*10], m5 + mova m4, [rsp+gprsize*2+16* 2] ;out12 + mova m5, [rsp+gprsize*2+16*14] ;-out13 + mova m6, [rsp+gprsize*2+16*15] ;out14 + ret +ALIGN function_align +cglobal_label .main_pass2_end + mova m7, [o(pw_2896x8)] + mova m1, [rsp+gprsize*2+16* 9] + mova m2, [rsp+gprsize*2+16*14] + paddsw m0, m1, m2 + psubsw m1, m2 + pmulhrsw m0, m7 ;out6 + pmulhrsw m1, m7 ;-out9 + mova [rsp+gprsize*2+16* 9], m0 + psubsw m2, m3, m4 + paddsw m3, m4 + pmulhrsw m2, m7 ;out10 + pmulhrsw m3, m7 ;-out5 + mova [rsp+gprsize*2+16* 8], m3 + mova m3, [rsp+gprsize*2+16* 7] + mova m4, [rsp+gprsize*2+16*12] + paddsw m0, m3, m4 + psubsw m3, m4 + pmulhrsw m0, m7 ;out4 + pmulhrsw m3, m7 ;-out11 + mova [rsp+gprsize*2+16* 7], m0 + mova m0, [rsp+gprsize*2+16*10] + paddsw m4, m0, [rsp+gprsize*2+16*11] + psubsw m0, [rsp+gprsize*2+16*11] + pmulhrsw m4, m7 ;-out7 + pmulhrsw m0, m7 ;out8 + mova [rsp+gprsize*2+16*10], m4 + mova m4, [rsp+gprsize*2+16*2 ] ;out12 + ret + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity + +cglobal iflipadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mova m7, [o(pw_2896x8)] + pmulhrsw m0, m7, [coeffq+16*0 ] + pmulhrsw m1, m7, [coeffq+16*1 ] + pmulhrsw m2, m7, [coeffq+16*14] + pmulhrsw m3, m7, [coeffq+16*15] + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*9], m2 + mova [rsp+gprsize+32*5], m3 + pmulhrsw m0, m7, [coeffq+16*6 ] + pmulhrsw m1, m7, [coeffq+16*7 ] + pmulhrsw m2, m7, [coeffq+16*8 ] + pmulhrsw m3, m7, [coeffq+16*9 ] + mova [rsp+gprsize+16*3], m2 + mova [rsp+gprsize+16*4], m3 + mova [rsp+gprsize+16*5], m0 + mova [rsp+gprsize+16*6], m1 + pmulhrsw m0, m7, [coeffq+16*2 ] + pmulhrsw m1, m7, [coeffq+16*3 ] + pmulhrsw m2, m7, [coeffq+16*4 ] + pmulhrsw m3, m7, [coeffq+16*5 ] + pmulhrsw m4, m7, [coeffq+16*10] + pmulhrsw m5, m7, [coeffq+16*11] + pmulhrsw m6, m7, [coeffq+16*12] + pmulhrsw m7, [coeffq+16*13] + + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS coeffq+16*0, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS coeffq+16*0, 32 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + jmp m(iflipadst_8x8_internal_8bpc).pass1_end + +.pass2: + lea tx2q, [o(.end)] + lea r3, [dstq+8] + jmp m(iflipadst_8x8_internal_8bpc).pass2_main + +.end: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(iflipadst_8x8_internal_8bpc).pass2_main + + +INV_TXFM_16X8_FN identity, dct +INV_TXFM_16X8_FN identity, adst +INV_TXFM_16X8_FN identity, flipadst +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + add coeffq, 16*16 + mova m4, [coeffq-16*7] + mova m5, [coeffq-16*5] + mova m6, [coeffq-16*3] + mova m7, [coeffq-16*1] + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + +.pass1: + mova m0, [o(pw_2896x8)] + mova m2, [o(pw_1697x16)] + mova m3, [o(pw_16384)] + sub coeffq, 8*16 + REPX {pmulhrsw x, m0}, m4, m5, m6, m7 + pmulhrsw m1, m2, m4 + pmulhrsw m1, m3 + paddsw m1, m4 ; 1 + pmulhrsw m4, m2, m5 + pmulhrsw m4, m3 + paddsw m4, m5 ; 3 + pmulhrsw m5, m2, m6 + pmulhrsw m5, m3 + paddsw m5, m6 ; 5 + pmulhrsw m6, m2, m7 + pmulhrsw m6, m3 + paddsw m7, m6 ; 7 + pmulhrsw m6, m0, [coeffq+16*6] + mova [rsp+gprsize+16*0], m4 + pmulhrsw m4, m2, m6 + pmulhrsw m4, m3 + paddsw m6, m4 ; 6 + pmulhrsw m4, m0, [coeffq+16*4] + mova [rsp+gprsize+16*1], m6 + pmulhrsw m6, m2, m4 + pmulhrsw m6, m3 + paddsw m4, m6 ; 4 + pmulhrsw m6, m0, [coeffq+16*2] + pmulhrsw m0, [coeffq+16*0] + pmulhrsw m2, m6 + pmulhrsw m2, m3 + paddsw m2, m6 ; 2 + pmulhrsw m6, m0, [o(pw_1697x16)] + pmulhrsw m6, m3 + mova m3, [rsp+gprsize+16*0] + paddsw m0, m6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +.pass1_end: + mova [coeffq+16*1], m4 + mova [coeffq+16*3], m5 + mova [coeffq+16*5], m6 + mova [coeffq+16*7], m7 + mova m4, [coeffq-16*7] + mova m5, [coeffq-16*5] + mova m6, [coeffq-16*3] + mova m7, [coeffq-16*1] + mova [coeffq-16*7], m0 + mova [coeffq-16*5], m1 + mova [coeffq-16*3], m2 + mova [coeffq-16*1], m3 + mov tx2q, r3 + jmp .pass1 + +.pass2: + lea tx2q, [o(.end)] + lea r3, [dstq+8] + jmp m(iidentity_8x8_internal_8bpc).end + +.end: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + mov dstq, r3 + jmp m(iidentity_8x8_internal_8bpc).end + + +%macro INV_TXFM_16X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x16, 8, 16*16 +%ifidn %1_%2, dct_dct + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r2d, 8 + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x16_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly +.end: + RET +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst +INV_TXFM_16X16_FN dct, identity + +cglobal idct_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq+16*1, 64 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*3, 64 + call m(idct_16x8_internal_8bpc).main + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + mova m7, [o(pw_8192)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+16*17, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + mova m7, [o(pw_8192)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS coeffq+16*0, 64 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*2, 64 + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(.pass1_end2)] + mova m7, [o(pw_8192)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+16*16, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + mova m7, [o(pw_8192)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass2: + lea tx2q, [o(.end)] + jmp m(idct_8x16_internal_8bpc).pass2_pre + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.end1)] + mov dstq, r3 + lea r3, [dstq+8] + jmp m(idct_8x8_internal_8bpc).end + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + + add coeffq, 32*8 + mov dstq, r3 + + mova m0, [coeffq+16*0 ] + mova m1, [coeffq+16*4 ] + mova m2, [coeffq+16*8 ] + mova m3, [coeffq+16*12] + mova m4, [coeffq+16*1 ] + mova m5, [coeffq+16*5 ] + mova m6, [coeffq+16*9 ] + mova m7, [coeffq+16*13] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] + jmp m(idct_8x16_internal_8bpc).pass2_main + + +%macro ITX_16X16_ADST_LOAD_ODD_COEFS 0 + mova m0, [coeffq+16*1 ] + mova m1, [coeffq+16*3 ] + mova m2, [coeffq+16*29] + mova m3, [coeffq+16*31] + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*9], m2 + mova [rsp+gprsize+32*5], m3 + mova m0, [coeffq+16*13] + mova m1, [coeffq+16*15] + mova m2, [coeffq+16*17] + mova m3, [coeffq+16*19] + mova [rsp+gprsize+16*3], m2 + mova [rsp+gprsize+16*4], m3 + mova [rsp+gprsize+16*5], m0 + mova [rsp+gprsize+16*6], m1 + mova m0, [coeffq+16*5 ] + mova m1, [coeffq+16*7 ] + mova m2, [coeffq+16*9 ] + mova m3, [coeffq+16*11] + mova m4, [coeffq+16*21] + mova m5, [coeffq+16*23] + mova m6, [coeffq+16*25] + mova m7, [coeffq+16*27] +%endmacro + +%macro ITX_16X16_ADST_LOAD_EVEN_COEFS 0 + mova m0, [coeffq+16*0 ] + mova m1, [coeffq+16*2 ] + mova m2, [coeffq+16*28] + mova m3, [coeffq+16*30] + mova [rsp+gprsize+16*7], m0 + mova [rsp+gprsize+16*8], m1 + mova [rsp+gprsize+16*9], m2 + mova [rsp+gprsize+32*5], m3 + mova m0, [coeffq+16*12] + mova m1, [coeffq+16*14] + mova m2, [coeffq+16*16] + mova m3, [coeffq+16*18] + mova [rsp+gprsize+16*3], m2 + mova [rsp+gprsize+16*4], m3 + mova [rsp+gprsize+16*5], m0 + mova [rsp+gprsize+16*6], m1 + mova m0, [coeffq+16*4 ] + mova m1, [coeffq+16*6 ] + mova m2, [coeffq+16*8 ] + mova m3, [coeffq+16*10] + mova m4, [coeffq+16*20] + mova m5, [coeffq+16*22] + mova m6, [coeffq+16*24] + mova m7, [coeffq+16*26] +%endmacro + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + ITX_16X16_ADST_LOAD_ODD_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + mova m7, [o(pw_8192)] + jmp m(iadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+16*17, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + mova m7, [o(pw_8192)] + jmp m(iadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+16*1, 32 + ITX_16X16_ADST_LOAD_EVEN_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + lea tx2q, [o(.pass1_end2)] + mova m7, [o(pw_8192)] + jmp m(iadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+16*16, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + mova m7, [o(pw_8192)] + jmp m(iadst_8x8_internal_8bpc).pass1_end1 + +.pass2: + lea tx2q, [o(.end)] + jmp m(iadst_8x16_internal_8bpc).pass2_pre + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.end1)] + mov dstq, r3 + lea r3, [dstq+8] + jmp m(iadst_8x8_internal_8bpc).end + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + + add coeffq, 32*8 + mov dstq, r3 + + mova m4, [coeffq+16*0 ] + mova m5, [coeffq+16*2 ] + mova m0, [coeffq+16*4 ] + mova m1, [coeffq+16*6 ] + mova m2, [coeffq+16*8 ] + mova m3, [coeffq+16*10] + mova m6, [coeffq+16*12] + mova m7, [coeffq+16*14] + mova [rsp+gprsize+16*7], m4 + mova [rsp+gprsize+16*8], m5 + mova [rsp+gprsize+16*5], m6 + mova [rsp+gprsize+16*6], m7 + lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] + jmp m(iadst_8x16_internal_8bpc).pass2_main + + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + ITX_16X16_ADST_LOAD_ODD_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + mova m7, [o(pw_m8192)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+16*1, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + mova m7, [o(pw_m8192)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+16*17, 32 + ITX_16X16_ADST_LOAD_EVEN_COEFS + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS coeffq+16*0, 32 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end2)] + mova m7, [o(pw_m8192)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+16*16, 32 + LOAD_8ROWS coeffq+16* 0, 32 + mova [rsp+gprsize+16*0], m7 + mov tx2q, r3 + mova m7, [o(pw_m8192)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 + +.pass2: + lea tx2q, [o(.end)] + lea r3, [dstq+8] + jmp m(iflipadst_8x16_internal_8bpc).pass2_pre + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.end1)] + lea dstq, [dstq+strideq*2] + jmp m(iflipadst_8x8_internal_8bpc).end + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + + add coeffq, 32*8 + + mova m4, [coeffq+16*0 ] + mova m5, [coeffq+16*2 ] + mova m0, [coeffq+16*4 ] + mova m1, [coeffq+16*6 ] + mova m2, [coeffq+16*8 ] + mova m3, [coeffq+16*10] + mova m6, [coeffq+16*12] + mova m7, [coeffq+16*14] + mova [rsp+gprsize+16*7], m4 + mova [rsp+gprsize+16*8], m5 + mova [rsp+gprsize+16*5], m6 + mova [rsp+gprsize+16*6], m7 + + lea tx2q, [o(.end2)] + mov dstq, r3 + jmp m(iflipadst_8x16_internal_8bpc).pass2_main + +.end2: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + lea dstq, [dstq+strideq*2] + jmp m(iflipadst_8x8_internal_8bpc).end + + +%macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 + pmulhrsw m%2, m%3, m%1 + psraw m%2, 1 + pavgw m%1, m%2 +%endmacro + +INV_TXFM_16X16_FN identity, dct +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + add coeffq, 16*17 + mov r3, tx2q + lea tx2q, [o(.pass1_end)] + +.pass1: + mova m6, [o(pw_1697x16)] + mova m7, [coeffq+32*6] + mova m0, [coeffq+32*0] + mova m1, [coeffq+32*1] + mova m2, [coeffq+32*2] + mova m3, [coeffq+32*3] + mova m4, [coeffq+32*4] + REPX {IDTX16B x, 5, 6}, 7, 0, 1, 2, 3, 4 + mova m5, [coeffq+32*5] + mova [rsp+gprsize+16*1], m7 + IDTX16B 5, 7, 6 + mova m7, [coeffq+32*7] + IDTX16B 7, 6, 6 + jmp m(idct_8x8_internal_8bpc).pass1_end3 + +.pass1_end: + SAVE_8ROWS coeffq, 32 + sub coeffq, 16 + lea tx2q, [o(.pass1_end1)] + jmp .pass1 + +.pass1_end1: + SAVE_8ROWS coeffq, 32 + sub coeffq, 15*16 + lea tx2q, [o(.pass1_end2)] + jmp .pass1 + +.pass1_end2: + SAVE_8ROWS coeffq, 32 + sub coeffq, 16 + mov tx2q, r3 + jmp .pass1 + +.pass2: + lea r3, [dstq+8] + lea tx2q, [o(.end1)] + +.end: + mova [rsp+gprsize+16*0], m7 + mova [rsp+gprsize+16*1], m4 + mova m7, [o(pw_1697x16)] + REPX {IDTX16 x, 4, 7}, 5, 6, 0, 1, 2, 3 + mova m4, [o(pw_2048)] + pmulhrsw m5, m4 + pmulhrsw m6, m4 + mova [rsp+gprsize+16*2], m5 + mova m5, [rsp+gprsize+16*1] + mova [rsp+gprsize+16*1], m6 + IDTX16 5, 6, 7 + mova m6, [rsp+gprsize+16*0] + IDTX16 6, 7, 7 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3, m6 + pmulhrsw m4, m5 + mova [rsp+gprsize+16*0], m6 + jmp m(idct_8x8_internal_8bpc).end3 + +.end1: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(.end2)] + lea dstq, [dstq+strideq*2] + jmp .end + +.end2: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + + add coeffq, 32*8 + LOAD_8ROWS coeffq, 32 + lea tx2q, [o(.end3)] + mov dstq, r3 + jmp .end + +.end3: + LOAD_8ROWS coeffq+16*1, 32 + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] + lea dstq, [dstq+strideq*2] + jmp .end + + +cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_8x32_internal_8bpc) + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + pmulhrsw m0, m2 + psrlw m2, 2 ;pw_2048 + pmulhrsw m0, m1 + pmulhrsw m0, m2 + pshuflw m0, m0, q0000 + punpcklwd m0, m0 + mov r3d, 8 + lea tx2q, [o(.end)] + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop + +.end: + RET + + + +cglobal idct_8x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + cmp eobd, 106 + jle .fast + + LOAD_8ROWS coeffq+16*3, 64 + call m(idct_8x8_internal_8bpc).main + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1: + mova [rsp+gprsize+16*9 ], m0 ;in24 + mova [rsp+gprsize+16*10], m4 ;in28 + mova [rsp+gprsize+16*17], m2 ;in26 + mova [rsp+gprsize+16*18], m6 ;in30 + mova [rsp+gprsize+16*31], m1 ;in25 + mova [rsp+gprsize+16*30], m3 ;in27 + mova [rsp+gprsize+16*27], m5 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + LOAD_8ROWS coeffq+16*2, 64 + call m(idct_8x8_internal_8bpc).main + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_1: + mova [rsp+gprsize+16*7 ], m0 ;in16 + mova [rsp+gprsize+16*8 ], m4 ;in20 + mova [rsp+gprsize+16*15], m2 ;in18 + mova [rsp+gprsize+16*16], m6 ;in22 + mova [rsp+gprsize+16*33], m1 ;in17 + mova [rsp+gprsize+16*28], m3 ;in19 + mova [rsp+gprsize+16*29], m5 ;in21 + mova [rsp+gprsize+16*32], m7 ;in23 + +.fast: + LOAD_8ROWS coeffq+16*1, 64 + call m(idct_8x8_internal_8bpc).main + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + mova [rsp+gprsize+16*5 ], m0 ;in8 + mova [rsp+gprsize+16*6 ], m4 ;in12 + mova [rsp+gprsize+16*13], m2 ;in10 + mova [rsp+gprsize+16*14], m6 ;in14 + mova [rsp+gprsize+16*21], m1 ;in9 + mova [rsp+gprsize+16*24], m3 ;in11 + mova [rsp+gprsize+16*25], m5 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + LOAD_8ROWS coeffq+16*0, 64 + call m(idct_8x8_internal_8bpc).main + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + mova [rsp+gprsize+16*11], m2 ;in2 + mova [rsp+gprsize+16*12], m6 ;in6 + mova [rsp+gprsize+16*19], m1 ;in1 + mova [rsp+gprsize+16*26], m3 ;in3 + mova [rsp+gprsize+16*23], m5 ;in5 + mova [rsp+gprsize+16*22], m7 ;in7 + mova m1, m4 ;in4 + mova m2, [rsp+gprsize+16*5 ] ;in8 + mova m3, [rsp+gprsize+16*6 ] ;in12 + + cmp eobd, 106 + jg .full + + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3 , 16 + mova m0, [rsp+gprsize+16*11] + mova m1, [rsp+gprsize+16*12] + mova m2, [rsp+gprsize+16*13] + mova m3, [rsp+gprsize+16*14] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call .main_fast + jmp .pass2 + +.full: + mova m4, [rsp+gprsize+16*7 ] ;in16 + mova m5, [rsp+gprsize+16*8 ] ;in20 + mova m6, [rsp+gprsize+16*9 ] ;in24 + mova m7, [rsp+gprsize+16*10] ;in28 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3 , 16 + LOAD_8ROWS rsp+gprsize+16*11, 16 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + call .main + +.pass2: + lea r3, [o(.end6)] + +.end: + mova [rsp+gprsize+16*0 ], m7 + lea tx2q, [o(.end2)] + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + + jmp tx2q + +.end2: + lea tx2q, [o(.end3)] + jmp m(idct_8x8_internal_8bpc).end + +.end3: + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0 ], m7 + lea dstq, [dstq+strideq*2] + lea tx2q, [o(.end4)] + jmp m(idct_8x8_internal_8bpc).end + +.end4: + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0 ], m7 + lea dstq, [dstq+strideq*2] + lea tx2q, [o(.end5)] + jmp m(idct_8x8_internal_8bpc).end + +.end5: + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0 ], m7 + lea dstq, [dstq+strideq*2] + mov tx2q, r3 + jmp m(idct_8x8_internal_8bpc).end + +.end6: + ret + +ALIGN function_align +cglobal_label .main_veryfast + mova m0, [rsp+gprsize*2+16*19] ;in1 + pmulhrsw m3, m0, [o(pw_4091x8)] ;t30,t31 + pmulhrsw m0, [o(pw_201x8)] ;t16,t17 + mova m7, [o(pd_2048)] + mova [rsp+gprsize*2+16*19], m0 ;t16 + mova [rsp+gprsize*2+16*34], m3 ;t31 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 799, 4017 ;t17a, t30a + mova [rsp+gprsize*2+16*20], m3 ;t17a + mova [rsp+gprsize*2+16*33], m0 ;t30a + mova m1, [rsp+gprsize*2+16*22] ;in7 + pmulhrsw m2, m1, [o(pw_3857x8)] ;t28,t29 + pmulhrsw m1, [o(pw_m1380x8)] ;t18,t19 + mova [rsp+gprsize*2+16*22], m1 ;t19 + mova [rsp+gprsize*2+16*31], m2 ;t28 + ITX_MULSUB_2W 2, 1, 0, 3, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m2 ;t18a + mova [rsp+gprsize*2+16*32], m1 ;t29a + mova m0, [rsp+gprsize*2+16*23] ;in5 + pmulhrsw m3, m0, [o(pw_3973x8)] ;t26, t27 + pmulhrsw m0, [o(pw_995x8)] ;t20, t21 + mova [rsp+gprsize*2+16*23], m0 ;t20 + mova [rsp+gprsize*2+16*30], m3 ;t27 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 3406, 2276 ;t21a, t26a + mova [rsp+gprsize*2+16*24], m3 ;t21a + mova [rsp+gprsize*2+16*29], m0 ;t26a + mova m2, [rsp+gprsize*2+16*26] ;in3 + pxor m0, m0 + mova m3, m0 + pmulhrsw m1, m2, [o(pw_4052x8)] + pmulhrsw m2, [o(pw_m601x8)] + jmp .main2 + +ALIGN function_align +cglobal_label .main_fast ;bottom half is zero + mova m0, [rsp+gprsize*2+16*19] ;in1 + mova m1, [rsp+gprsize*2+16*20] ;in15 + pmulhrsw m3, m0, [o(pw_4091x8)] ;t31a + pmulhrsw m0, [o(pw_201x8)] ;t16a + pmulhrsw m2, m1, [o(pw_3035x8)] ;t30a + pmulhrsw m1, [o(pw_m2751x8)] ;t17a + mova m7, [o(pd_2048)] + psubsw m4, m0, m1 ;t17 + paddsw m0, m1 ;t16 + psubsw m5, m3, m2 ;t30 + paddsw m3, m2 ;t31 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 799, 4017 ;t17a, t30a + mova [rsp+gprsize*2+16*19], m0 ;t16 + mova [rsp+gprsize*2+16*20], m5 ;t17a + mova [rsp+gprsize*2+16*33], m4 ;t30a + mova [rsp+gprsize*2+16*34], m3 ;t31 + mova m0, [rsp+gprsize*2+16*21] ;in9 + mova m1, [rsp+gprsize*2+16*22] ;in7 + pmulhrsw m3, m0, [o(pw_3703x8)] + pmulhrsw m0, [o(pw_1751x8)] + pmulhrsw m2, m1, [o(pw_3857x8)] + pmulhrsw m1, [o(pw_m1380x8)] + psubsw m4, m1, m0 ;t18 + paddsw m0, m1 ;t19 + psubsw m5, m2, m3 ;t29 + paddsw m3, m2 ;t28 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m5 ;t18a + mova [rsp+gprsize*2+16*22], m0 ;t19 + mova [rsp+gprsize*2+16*31], m3 ;t28 + mova [rsp+gprsize*2+16*32], m4 ;t29a + mova m0, [rsp+gprsize*2+16*23] ;in5 + mova m1, [rsp+gprsize*2+16*24] ;in11 + pmulhrsw m3, m0, [o(pw_3973x8)] + pmulhrsw m0, [o(pw_995x8)] + pmulhrsw m2, m1, [o(pw_3513x8)] + pmulhrsw m1, [o(pw_m2106x8)] + psubsw m4, m0, m1 ;t21 + paddsw m0, m1 ;t20 + psubsw m5, m3, m2 ;t26 + paddsw m3, m2 ;t27 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 3406, 2276 ;t21a, t26a + mova [rsp+gprsize*2+16*23], m0 ;t20 + mova [rsp+gprsize*2+16*24], m5 ;t21a + mova [rsp+gprsize*2+16*29], m4 ;t26a + mova [rsp+gprsize*2+16*30], m3 ;t27 + mova m0, [rsp+gprsize*2+16*25] ;in13 + mova m2, [rsp+gprsize*2+16*26] ;in3 + pmulhrsw m3, m0, [o(pw_3290x8)] + pmulhrsw m0, [o(pw_2440x8)] + pmulhrsw m1, m2, [o(pw_4052x8)] + pmulhrsw m2, [o(pw_m601x8)] + jmp .main2 + +ALIGN function_align +cglobal_label .main + mova m7, [o(pd_2048)] + mova m0, [rsp+gprsize*2+16*19] ;in1 + mova m1, [rsp+gprsize*2+16*20] ;in15 + mova m2, [rsp+gprsize*2+16*33] ;in17 + mova m3, [rsp+gprsize*2+16*34] ;in31 + ITX_MULSUB_2W 0, 3, 4, 5, 7, 201, 4091 ;t16a, t31a + ITX_MULSUB_2W 2, 1, 4, 5, 7, 3035, 2751 ;t17a, t30a + psubsw m4, m0, m2 ;t17 + paddsw m0, m2 ;t16 + psubsw m5, m3, m1 ;t30 + paddsw m3, m1 ;t31 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 799, 4017 ;t17a, t30a + mova [rsp+gprsize*2+16*19], m0 ;t16 + mova [rsp+gprsize*2+16*20], m5 ;t17a + mova [rsp+gprsize*2+16*33], m4 ;t30a + mova [rsp+gprsize*2+16*34], m3 ;t31 + mova m0, [rsp+gprsize*2+16*21] ;in9 + mova m1, [rsp+gprsize*2+16*22] ;in7 + mova m2, [rsp+gprsize*2+16*31] ;in25 + mova m3, [rsp+gprsize*2+16*32] ;in23 + ITX_MULSUB_2W 0, 3, 4, 5, 7, 1751, 3703 ;t18a, t29a + ITX_MULSUB_2W 2, 1, 4, 5, 7, 3857, 1380 ;t19a, t28a + psubsw m4, m2, m0 ;t18 + paddsw m0, m2 ;t19 + psubsw m5, m1, m3 ;t29 + paddsw m3, m1 ;t28 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4017, 799 ;t18a, t29a + mova [rsp+gprsize*2+16*21], m5 ;t18a + mova [rsp+gprsize*2+16*22], m0 ;t19 + mova [rsp+gprsize*2+16*31], m3 ;t28 + mova [rsp+gprsize*2+16*32], m4 ;t29a + mova m0, [rsp+gprsize*2+16*23] ;in5 + mova m1, [rsp+gprsize*2+16*24] ;in11 + mova m2, [rsp+gprsize*2+16*29] ;in21 + mova m3, [rsp+gprsize*2+16*30] ;in27 + ITX_MULSUB_2W 0, 3, 4, 5, 7, 995, 3973 ;t20a, t27a + ITX_MULSUB_2W 2, 1, 4, 5, 7, 3513, 2106 ;t21a, t26a + psubsw m4, m0, m2 ;t21 + paddsw m0, m2 ;t20 + psubsw m5, m3, m1 ;t26 + paddsw m3, m1 ;t27 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 3406, 2276 ;t21a, t26a + mova [rsp+gprsize*2+16*23], m0 ;t20 + mova [rsp+gprsize*2+16*24], m5 ;t21a + mova [rsp+gprsize*2+16*29], m4 ;t26a + mova [rsp+gprsize*2+16*30], m3 ;t27 + mova m0, [rsp+gprsize*2+16*25] ;in13 + mova m1, [rsp+gprsize*2+16*26] ;in3 + mova m2, [rsp+gprsize*2+16*27] ;in29 + mova m3, [rsp+gprsize*2+16*28] ;in19 + ITX_MULSUB_2W 0, 3, 4, 5, 7, 2440, 3290 ;t22a, t25a + ITX_MULSUB_2W 2, 1, 4, 5, 7, 4052, 601 ;t23a, t24a + +.main2: + psubsw m4, m2, m0 ;t22 + paddsw m0, m2 ;t23 + psubsw m5, m1, m3 ;t25 + paddsw m3, m1 ;t24 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m2276, 3406 ;t22a, t25a + mova m2, [rsp+gprsize*2+16*24] ;t21a + psubsw m1, m5, m2 ;t21 + paddsw m5, m2 ;t22 + mova [rsp+gprsize*2+16*25], m5 ;t22 + mova m2, [rsp+gprsize*2+16*29] ;t26a + psubsw m5, m4, m2 ;t26 + paddsw m4, m2 ;t25 + mova [rsp+gprsize*2+16*28], m4 ;t25 + ITX_MULSUB_2W 5, 1, 2, 4, 7, m3784, 1567 ;t21a, t26a + mova [rsp+gprsize*2+16*24], m5 ;t21a + mova [rsp+gprsize*2+16*29], m1 ;t26a + + mova m1, [rsp+gprsize*2+16*23] ;t20 + mova m5, [rsp+gprsize*2+16*30] ;t27 + psubsw m2, m0, m1 ;t20a + paddsw m0, m1 ;t23a + psubsw m6, m3, m5 ;t27a + paddsw m3, m5 ;t24a + ITX_MULSUB_2W 6, 2, 1, 5, 7, m3784, 1567 ;t20, t27 + mova [rsp+gprsize*2+16*26], m0 ;t23a + mova [rsp+gprsize*2+16*27], m3 ;t24a + mova [rsp+gprsize*2+16*30], m2 ;t27 + + mova m0, [rsp+gprsize*2+16*20] ;t17a + mova m1, [rsp+gprsize*2+16*21] ;t18a + mova m2, [rsp+gprsize*2+16*32] ;t29a + mova m3, [rsp+gprsize*2+16*33] ;t30a + psubsw m4, m0, m1 ;t18 + paddsw m0, m1 ;t17 + psubsw m5, m3, m2 ;t29 + paddsw m3, m2 ;t30 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 1567, 3784 ;t18a, t29a + mova [rsp+gprsize*2+16*20], m0 ;t17 + mova [rsp+gprsize*2+16*21], m5 ;t18a + mova [rsp+gprsize*2+16*32], m4 ;t29a + mova [rsp+gprsize*2+16*33], m3 ;t30 + mova m0, [rsp+gprsize*2+16*19] ;t16 + mova m1, [rsp+gprsize*2+16*22] ;t19 + mova m2, [rsp+gprsize*2+16*31] ;t28 + mova m3, [rsp+gprsize*2+16*34] ;t31 + psubsw m4, m0, m1 ;t19a + paddsw m0, m1 ;t16a + psubsw m5, m3, m2 ;t28a + paddsw m3, m2 ;t31a + ITX_MULSUB_2W 5, 4, 1, 2, 7, 1567, 3784 ;t19, t28 + mova m2, [rsp+gprsize*2+16*15] ;tmp12 + psubsw m1, m5, m6 ;t20a + paddsw m5, m6 ;t19a + psubsw m6, m2, m5 ;out19 + paddsw m2, m5 ;out12 + mova m5, [rsp+gprsize*2+16*30] ;t27 + mova [rsp+gprsize*2+16*22], m6 ;out19 + mova [rsp+gprsize*2+16*15], m2 ;out12 + psubsw m6, m4, m5 ;t27a + paddsw m4, m5 ;t28a + ITX_MULSUB_2W 6, 1, 2, 5, 7, 2896, 2896 ;t20, t27 + mova m2, [rsp+gprsize*2+16*6 ] ;tmp3 + psubsw m5, m2, m4 ;out28 + paddsw m2, m4 ;out3 + mova m4, [rsp+gprsize*2+16*14] ;tmp11 + mova [rsp+gprsize*2+16*31], m5 ;out28 + mova [rsp+gprsize*2+16*6 ], m2 ;out3 + psubsw m5, m4, m6 ;out20 + paddsw m4, m6 ;out11 + mova m2, [rsp+gprsize*2+16*7 ] ;tmp4 + mova [rsp+gprsize*2+16*23], m5 ;out20 + mova [rsp+gprsize*2+16*14], m4 ;out11 + psubsw m5, m2, m1 ;out27 + paddsw m2, m1 ;out4 + mova m1, [rsp+gprsize*2+16*26] ;t23a + mova m4, [rsp+gprsize*2+16*27] ;t24a + mova [rsp+gprsize*2+16*30], m5 ;out27 + mova [rsp+gprsize*2+16*7 ], m2 ;out4 + psubsw m5, m0, m1 ;t23 + paddsw m0, m1 ;t16 + psubsw m2, m3, m4 ;t24 + paddsw m3, m4 ;t31 + ITX_MULSUB_2W 2, 5, 4, 6, 7, 2896, 2896 ;t23a, t24a + mova m6, [rsp+gprsize*2+16*18] ;tmp15 + psubsw m4, m6, m0 ;out16 + paddsw m6, m0 ;out15 + mova m0, [rsp+gprsize*2+16*3 ] ;tmp0 + mova m1, [rsp+gprsize*2+16*11] ;tmp8 + mova [rsp+gprsize*2+16*18], m6 ;out15 + mova [rsp+gprsize*2+16*19], m4 ;out16 + psubsw m6, m0, m3 ;out31 + paddsw m0, m3 ;out0 + psubsw m4, m1, m2 ;out23 + paddsw m1, m2 ;out8 + mova m3, [rsp+gprsize*2+16*10] ;tmp7 + mova [rsp+gprsize*2+16*34], m6 ;out31 + mova [rsp+gprsize*2+16*11], m1 ;out8 + mova [rsp+gprsize*2+16*26], m4 ;out23 + paddsw m6, m3, m5 ;out7 + psubsw m3, m5 ;out24 + mova m1, [rsp+gprsize*2+16*20] ;t17 + mova m5, [rsp+gprsize*2+16*25] ;t22 + mova m2, [rsp+gprsize*2+16*17] ;tmp14 + mova [rsp+gprsize*2+16*27], m3 ;out24 + psubsw m4, m1, m5 ;t22a + paddsw m1, m5 ;t17a + psubsw m3, m2, m1 ;out17 + paddsw m2, m1 ;out14 + mova m5, [rsp+gprsize*2+16*28] ;t25 + mova m1, [rsp+gprsize*2+16*33] ;t30 + mova [rsp+gprsize*2+16*17], m2 ;out14 + mova [rsp+gprsize*2+16*20], m3 ;out17 + psubsw m2, m1, m5 ;t25a + paddsw m1, m5 ;t30a + ITX_MULSUB_2W 2, 4, 3, 5, 7, 2896, 2896 ;t22, t25 + mova m5, [rsp+gprsize*2+16*4 ] ;tmp1 + psubsw m3, m5, m1 ;out30 + paddsw m5, m1 ;out1 + mova m1, [rsp+gprsize*2+16*12] ;tmp9 + mova [rsp+gprsize*2+16*33], m3 ;out30 + mova [rsp+gprsize*2+16*4 ], m5 ;out1 + psubsw m3, m1, m2 ;out22 + paddsw m1, m2 ;out9 + mova m5, [rsp+gprsize*2+16*9 ] ;tmp6 + mova [rsp+gprsize*2+16*25], m3 ;out22 + mova [rsp+gprsize*2+16*12], m1 ;out9 + psubsw m3, m5, m4 ;out25 + paddsw m5, m4 ;out6 + mova m4, [rsp+gprsize*2+16*21] ;t18a + mova m1, [rsp+gprsize*2+16*24] ;t21a + mova m2, [rsp+gprsize*2+16*16] ;tmp13 + mova [rsp+gprsize*2+16*28], m3 ;out25 + mova [rsp+gprsize*2+16*9 ], m5 ;out6 + paddsw m3, m4, m1 ;t18 + psubsw m4, m1 ;t21 + psubsw m5, m2, m3 ;out18 + paddsw m2, m3 ;out13 + mova m1, [rsp+gprsize*2+16*29] ;t26a + mova m3, [rsp+gprsize*2+16*32] ;t29a + mova [rsp+gprsize*2+16*21], m5 ;out18 + mova [rsp+gprsize*2+16*16], m2 ;out13 + psubsw m5, m3, m1 ;t26 + paddsw m3, m1 ;t29 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 2896, 2896 ;t21a, t26a + mova m2, [rsp+gprsize*2+16*5 ] ;tmp2 + psubsw m1, m2, m3 ;out29 + paddsw m2, m3 ;out2 + mova m3, [rsp+gprsize*2+16*13] ;tmp10 + mova [rsp+gprsize*2+16*32], m1 ;out29 + psubsw m7, m3, m5 ;out21 + paddsw m3, m5 ;out10 + mova m5, [rsp+gprsize*2+16*8 ] ;tmp5 + mova [rsp+gprsize*2+16*24], m7 ;out21 + mova [rsp+gprsize*2+16*13], m3 ;out10 + psubsw m1, m5, m4 ;out26 + paddsw m5, m4 ;out5 + mova m7, m6 ;out7 + mova m3, [rsp+gprsize*2+16*6 ] ;out3 + mova m4, [rsp+gprsize*2+16*7 ] ;out4 + mova [rsp+gprsize*2+16*29], m1 ;out26 + mova m6, [rsp+gprsize*2+16*9 ] ;out6 + mova m1, [rsp+gprsize*2+16*4 ] ;out1 + ret + + +cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_32x8_internal_8bpc) + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r3d, 8 + lea tx2q, [o(.end)] + +.body: + pmulhrsw m0, m2 + movd m2, [o(pw_2048)] ;intentionally rip-relative + pmulhrsw m0, m1 + pmulhrsw m0, m2 + pshuflw m0, m0, q0000 + punpcklwd m0, m0 + pxor m5, m5 + +.loop: + mova m1, [dstq+16*0] + mova m3, [dstq+16*1] + punpckhbw m2, m1, m5 + punpcklbw m1, m5 + punpckhbw m4, m3, m5 + punpcklbw m3, m5 + paddw m2, m0 + paddw m1, m0 + paddw m4, m0 + paddw m3, m0 + packuswb m1, m2 + packuswb m3, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m3 + add dstq, strideq + dec r3d + jg .loop + jmp tx2q + +.end: + RET + + +cglobal idct_32x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq+16*0, 64 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+16*2, 64 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+16*1, 32 + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + cmp eobd, 106 + jg .full + call m(idct_8x32_internal_8bpc).main_fast + jmp .pass2 + +.full: + LOAD_8ROWS coeffq+16*17, 32 + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + call m(idct_8x32_internal_8bpc).main + +.pass2: + mova [rsp+gprsize+16*0 ], m7 + lea tx2q, [o(.end)] + jmp m(idct_8x32_internal_8bpc).end1 + +.end: + mova m7, [o(pw_8192)] + lea tx2q, [o(.end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.end1: + lea r3, [dstq+8] + lea tx2q, [o(.end2)] + jmp m(idct_8x8_internal_8bpc).pass2_main + +.end2: + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0 ], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.end3: + mov dstq, r3 + add r3, 8 + lea tx2q, [o(.end4)] + jmp m(idct_8x8_internal_8bpc).pass2_main + +.end4: + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0 ], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.end5: + mov dstq, r3 + add r3, 8 + lea tx2q, [o(.end6)] + jmp m(idct_8x8_internal_8bpc).pass2_main + +.end6: + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0 ], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.end7: + mov dstq, r3 + lea tx2q, [o(.end8)] + jmp m(idct_8x8_internal_8bpc).pass2_main + +.end8: + ret + + +cglobal inv_txfm_add_identity_identity_8x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 + mov r5d, 4 + mov tx2d, 2 + cmp eobd, 107 + cmovns tx2d, r5d + mov r3d, tx2d +%if ARCH_X86_32 + LEA r5, $$ +%endif + lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] +.loop: + LOAD_8ROWS coeffq+16*0, 64 + paddsw m6, [o(pw_5)] + mova [rsp+16*1], m6 + mova m6, [o(pw_5)] + REPX {paddsw x, m6}, m0, m1, m2, m3, m4, m5, m7 + call m(idct_8x8_internal_8bpc).pass1_end3 + REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 + mova [rsp+16*2], m5 + mova [rsp+16*1], m6 + mova [rsp+16*0], m7 + call m(idct_8x8_internal_8bpc).end3 + lea dstq, [dstq+strideq*2] + pxor m7, m7 + REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + add coeffq, 16 + dec r3d + jg .loop + RET + +cglobal inv_txfm_add_identity_identity_32x8_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 + mov r5d, 4 + mov tx2d, 2 + cmp eobd, 107 + cmovns tx2d, r5d + mov r3d, tx2d +%if ARCH_X86_32 + LEA r5, $$ +%endif + +.loop: + LOAD_8ROWS coeffq+16*0, 16 + pmulhrsw m6, [o(pw_4096)] + mova [rsp+16*1], m6 + mova m6, [o(pw_4096)] + REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] + call m(idct_8x8_internal_8bpc).pass1_end3 + + mov [rsp+16*3], dstq + mova [rsp+16*2], m5 + mova [rsp+16*1], m6 + mova [rsp+16*0], m7 + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + call m(idct_8x8_internal_8bpc).end3 + + add coeffq, 16*8 + mov dstq, [rsp+16*3] + lea dstq, [dstq+8] + dec r3d + jg .loop + jnc .loop + RET + + +cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_16x32_internal_8bpc) +.end: + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m1 + mov r2d, 16 + lea tx2q, [o(.end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly + + +cglobal idct_16x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + LOAD_8ROWS coeffq+16*1, 128, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*5, 128, 1 + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end: + SAVE_8ROWS coeffq+16*33, 64 ;in8~in15 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end1: + mova [coeffq+16*1 ], m0 ;in8 + mova [coeffq+16*5 ], m4 ;in12 + mova [rsp+gprsize+16*13], m2 ;in10 + mova [rsp+gprsize+16*14], m6 ;in14 + mova [rsp+gprsize+16*21], m1 ;in9 + mova [rsp+gprsize+16*24], m3 ;in11 + mova [rsp+gprsize+16*25], m5 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + LOAD_8ROWS coeffq+16*0, 128, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*4, 128, 1 + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end2: + SAVE_8ROWS coeffq+16*32, 64 ;in0~in7 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end3: + mova [rsp+gprsize+16*11], m2 ;in2 + mova [rsp+gprsize+16*12], m6 ;in6 + mova [rsp+gprsize+16*19], m1 ;in1 + mova [rsp+gprsize+16*26], m3 ;in3 + mova [rsp+gprsize+16*23], m5 ;in5 + mova [rsp+gprsize+16*22], m7 ;in7 + + cmp eobd, 150 + jg .full + + mova m1, m4 ;in4 + mova m2, [coeffq+16*1 ] ;in8 + mova m3, [coeffq+16*5 ] ;in12 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + mova m0, [rsp+gprsize+16*11] ;in2 + mova m1, [rsp+gprsize+16*12] ;in6 + mova m2, [rsp+gprsize+16*13] ;in10 + mova m3, [rsp+gprsize+16*14] ;in14 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main_fast + jmp .pass2 + +.full: + mova [coeffq+16*0 ], m0 ;in0 + mova [coeffq+16*4 ], m4 ;in4 + + LOAD_8ROWS coeffq+16*2, 128, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*6, 128, 1 + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end4: + SAVE_8ROWS coeffq+16*34, 64 ;in16~in23 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end5: + mova [coeffq+16*2 ], m0 ;in16 + mova [coeffq+16*6 ], m4 ;in20 + mova [rsp+gprsize+16*15], m2 ;in18 + mova [rsp+gprsize+16*16], m6 ;in22 + mova [rsp+gprsize+16*33], m1 ;in17 + mova [rsp+gprsize+16*28], m3 ;in19 + mova [rsp+gprsize+16*29], m5 ;in21 + mova [rsp+gprsize+16*32], m7 ;in23 + + LOAD_8ROWS coeffq+16*3, 128, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+16*7, 128, 1 + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(.pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end6: + SAVE_8ROWS coeffq+16*35, 64 ;in24~in31 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end7: + mova [rsp+gprsize+16*17], m2 ;in26 + mova [rsp+gprsize+16*18], m6 ;in30 + mova [rsp+gprsize+16*31], m1 ;in25 + mova [rsp+gprsize+16*30], m3 ;in27 + mova [rsp+gprsize+16*27], m5 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + + mova m6, m0 ;in24 + mova m7, m4 ;in28 + mova m0, [coeffq+16*0 ] ;in0 + mova m1, [coeffq+16*4 ] ;in4 + mova m2, [coeffq+16*1 ] ;in8 + mova m3, [coeffq+16*5 ] ;in12 + mova m4, [coeffq+16*2 ] ;in16 + mova m5, [coeffq+16*6 ] ;in20 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3 , 16 + LOAD_8ROWS rsp+gprsize+16*11, 16 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main + +.pass2: + mov [rsp+gprsize*1+16*35], eobd + lea r3, [dstq+8] + mov [rsp+gprsize*2+16*35], r3 + lea r3, [o(.end)] + jmp m(idct_8x32_internal_8bpc).end + +.end: + mov dstq, [rsp+gprsize*2+16*35] + mov eobd, [rsp+gprsize*1+16*35] + add coeffq, 16*32 + + mova m0, [coeffq+16*4 ] ;in1 + mova m1, [coeffq+16*12] ;in3 + mova m2, [coeffq+16*20] ;in5 + mova m3, [coeffq+16*28] ;in7 + mova m4, [coeffq+16*5 ] ;in9 + mova m5, [coeffq+16*13] ;in11 + mova m6, [coeffq+16*21] ;in13 + mova m7, [coeffq+16*29] ;in15 + + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + mova m0, [coeffq+16*0 ] ;in0 + mova m1, [coeffq+16*16] ;in4 + mova m2, [coeffq+16*1 ] ;in8 + mova m3, [coeffq+16*17] ;in12 + + cmp eobd, 150 + jg .full1 + + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + mova m0, [coeffq+16*8 ] ;in2 + mova m1, [coeffq+16*24] ;in6 + mova m2, [coeffq+16*9 ] ;in10 + mova m3, [coeffq+16*25] ;in14 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main_fast + jmp m(idct_8x32_internal_8bpc).pass2 + +.full1: + mova m4, [coeffq+16*2 ] ;in16 + mova m5, [coeffq+16*18] ;in20 + mova m6, [coeffq+16*3 ] ;in24 + mova m7, [coeffq+16*19] ;in26 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + mova m0, [coeffq+16*8 ] ;in2 + mova m1, [coeffq+16*24] ;in6 + mova m2, [coeffq+16*9 ] ;in10 + mova m3, [coeffq+16*25] ;in14 + mova m4, [coeffq+16*10] ;in18 + mova m5, [coeffq+16*26] ;in22 + mova m6, [coeffq+16*11] ;in26 + mova m7, [coeffq+16*27] ;in30 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + mova m0, [coeffq+16*6 ] ;in17 + mova m1, [coeffq+16*14] ;in19 + mova m2, [coeffq+16*22] ;in21 + mova m3, [coeffq+16*30] ;in23 + mova m4, [coeffq+16*7 ] ;in25 + mova m5, [coeffq+16*15] ;in27 + mova m6, [coeffq+16*23] ;in29 + mova m7, [coeffq+16*31] ;in31 + + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + + call m(idct_8x32_internal_8bpc).main + jmp m(idct_8x32_internal_8bpc).pass2 + + +cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + + call m(idct_32x16_internal_8bpc) + call m(idct_8x16_internal_8bpc).pass2 + + add coeffq, 16*16 + lea dstq, [r3+8] + LOAD_8ROWS rsp+16*11, 16 + mova [rsp+16*0], m7 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 + + add coeffq, 16*16 + lea dstq, [r3+8] + LOAD_8ROWS rsp+16*19, 16 + mova [rsp+16*0], m7 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 + + add coeffq, 16*16 + lea dstq, [r3+8] + LOAD_8ROWS rsp+16*27, 16 + mova [rsp+16*0], m7 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m1 + mov r3d, 16 + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body + + +cglobal idct_32x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + add coeffq, 16 + lea r3, [o(.pass1_end1)] +.pass1: + LOAD_8ROWS coeffq+16*0, 128, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+16*4, 128, 1 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+16*2, 64, 1 + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + LOAD_8ROWS coeffq+16*34, 64, 1 + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + call m(idct_8x32_internal_8bpc).main + +.pass1_end: + mova [rsp+gprsize+16*0 ], m7 + mov tx2q, r3 + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end1: + SAVE_8ROWS coeffq+16*0, 32 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0 ], m7 + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end2: + SAVE_8ROWS coeffq+16*16, 32 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0 ], m7 + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end3: + SAVE_8ROWS coeffq+16*32, 32 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0 ], m7 + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end4: + SAVE_8ROWS coeffq+16*48, 32 + + sub coeffq, 16 + lea r3, [o(.end)] + jmp .pass1 + +.end: + ret + + +cglobal inv_txfm_add_identity_identity_16x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 + mov r4d, eobd + cmp eobd, 43 ;if (eob > 43) + sbb r3d, r3d ; iteration_count++ + cmp r4d, 150 ;if (eob > 150) + sbb r3d, 0 ; iteration_count++ + cmp r4d, 278 ;if (eob > 278) + sbb r3d, -4 ; iteration_count++ + +%if ARCH_X86_32 + LEA r5, $$ +%endif + lea r4, [dstq+8] + mov [rsp+16*3], r4 + mov [rsp+gprsize+16*3], r3d + mov [rsp+gprsize*2+16*3], coeffq + +.loop: + LOAD_8ROWS coeffq, 64, 1 + mova [rsp+16*1], m6 + pxor m6, m6 + REPX {mova [coeffq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 + mova [rsp+16*0], m2 + mova [rsp+16*1], m3 + mova [rsp+16*2], m4 + mova m3, [o(pw_1697x16)] + mova m4, [o(pw_16384)] + REPX {IDTX16 x, 2, 3, 4}, 5, 6, 7, 0, 1 + mova m2, [o(pw_8192)] + REPX {pmulhrsw x, m2}, m5, m6, m7, m0, m1 + mova m2, [rsp+16*0] + mova [rsp+16*0], m7 + IDTX16 2, 7, 3, 4 + mova m7, [rsp+16*2] + mova [rsp+16*2], m5 + IDTX16 7, 5, 3, 4 + mova m5, [rsp+16*1] + mova [rsp+16*1], m6 + pmulhrsw m3, m5 + pmulhrsw m3, m4 + psrlw m4, 1 ; pw_8192 + paddsw m3, m5 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + pmulhrsw m4, m7 + call m(idct_8x8_internal_8bpc).end3 + lea dstq, [dstq+strideq*2] + add coeffq, 16 + dec r3d + jg .loop + mov coeffq, [rsp+gprsize*2+16*3] + add coeffq, 64*8 + mov r3d, [rsp+gprsize+16*3] + xor dstq, dstq + mov [rsp+gprsize+16*3], dstq + mov dstq, [rsp+16*3] + test r3d, r3d + jnz .loop + RET + + +cglobal inv_txfm_add_identity_identity_32x16_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 + mov r4d, 12 ;0100b + mov r5d, 136 ;1000 1000b + cmp eobd, 44 ;if (eob > 43) + cmovns r4d, r5d ; iteration_count+2 + cmp eobd, 151 ;if (eob > 150) + mov r3d, 34952 ;1000 1000 1000 1000b + cmovs r3d, r4d ; iteration_count += 4 + +%if ARCH_X86_32 + LEA r5, $$ +%endif + lea r4, [dstq+8] + mov [rsp+16*3], r4 + +.loop: + LOAD_8ROWS coeffq, 32, 1 + REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 + mova [rsp+16*1], m6 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 + mova [rsp+16*1], m5 + mova [rsp+16*2], m6 + mova m6, [o(pw_1697x16)] + REPX {IDTX16 x, 5, 6}, 7, 0, 1, 2, 3, 4 + pmulhrsw m7, [o(pw_2048)] + mova m5, [rsp+16*1] + mova [rsp+16*0], m7 + IDTX16 5, 7, 6 + mova m7, [rsp+16*2] + IDTX16 7, 6, 6 + mova m6, [o(pw_2048)] + REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 + mova [rsp+16*2], m5 + mova [rsp+16*1], m7 + call m(idct_8x8_internal_8bpc).end3 + lea dstq, [dstq+strideq*2] + pxor m7, m7 + REPX {mova [coeffq+32*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + +.loop_end: + add coeffq, 16 + shr r3d, 2 + jz .ret + test r3d, 2 + jnz .loop + mov r4d, r3d + and r4d, 1 + lea coeffq, [coeffq+r4*8+32*7] + mov dstq, [rsp+16*3] + lea r4, [dstq+8] + mov [rsp+16*3], r4 + jmp .loop + +.ret: + RET + + +cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + + call m(idct_32x32_internal_8bpc) + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r3d, 32 + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body + + +cglobal idct_32x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r4d, 2 + sub eobd, 136 + mov [rsp+gprsize*1+16*35], eobd + mov r3d, 4 + cmovs r3d, r4d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + mov [rsp+gprsize*2+16*35], coeffq + +.pass1_loop: + LOAD_8ROWS coeffq+64*1, 64*2 + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + mov tx2d, [rsp+gprsize*1+16*35] + test tx2d, tx2d + jl .fast + +.full: + LOAD_8ROWS coeffq+64*0, 64*4 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+64*2, 64*4 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+64*17, 64*2 + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + + call m(idct_8x32_internal_8bpc).main + jmp .pass1_end + +.fast: + mova m0, [coeffq+256*0] + mova m1, [coeffq+256*1] + mova m2, [coeffq+256*2] + mova m3, [coeffq+256*3] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + + SAVE_7ROWS rsp+gprsize+16*3, 16 + mova m0, [coeffq+128*1] + mova m1, [coeffq+128*3] + mova m2, [coeffq+128*5] + mova m3, [coeffq+128*7] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main_fast + +.pass1_end: + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end3: + SAVE_8ROWS coeffq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end4: + SAVE_8ROWS coeffq+64*24, 64 + + add coeffq, 16 + dec r3d + jg .pass1_loop + + +.pass2: + mov coeffq, [rsp+gprsize*2+16*35] + mov r3d, 4 + lea tx2q, [o(.pass2_end)] + +.pass2_loop: + mov [rsp+gprsize*3+16*35], r3d + lea r3, [dstq+8] + mov [rsp+gprsize*2+16*35], r3 + + mova m0, [coeffq+16*4 ] + mova m1, [coeffq+16*12] + mova m2, [coeffq+16*20] + mova m3, [coeffq+16*28] + mova m4, [coeffq+16*5 ] + mova m5, [coeffq+16*13] + mova m6, [coeffq+16*21] + mova m7, [coeffq+16*29] + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + mov eobd, [rsp+gprsize*1+16*35] + test eobd, eobd + jl .fast1 + +.full1: + mova m0, [coeffq+16*0 ] + mova m1, [coeffq+16*16] + mova m2, [coeffq+16*1 ] + mova m3, [coeffq+16*17] + mova m4, [coeffq+16*2 ] + mova m5, [coeffq+16*18] + mova m6, [coeffq+16*3 ] + mova m7, [coeffq+16*19] + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + mova m0, [coeffq+16*8 ] + mova m1, [coeffq+16*24] + mova m2, [coeffq+16*9 ] + mova m3, [coeffq+16*25] + mova m4, [coeffq+16*10] + mova m5, [coeffq+16*26] + mova m6, [coeffq+16*11] + mova m7, [coeffq+16*27] + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + mova m0, [coeffq+16*6 ] + mova m1, [coeffq+16*14] + mova m2, [coeffq+16*22] + mova m3, [coeffq+16*30] + mova m4, [coeffq+16*7 ] + mova m5, [coeffq+16*15] + mova m6, [coeffq+16*23] + mova m7, [coeffq+16*31] + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + + call m(idct_8x32_internal_8bpc).main + jmp tx2q + +.fast1: + mova m0, [coeffq+16*0 ] + mova m1, [coeffq+16*16] + mova m2, [coeffq+16*1 ] + mova m3, [coeffq+16*17] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + mova m0, [coeffq+16*8 ] + mova m1, [coeffq+16*24] + mova m2, [coeffq+16*9 ] + mova m3, [coeffq+16*25] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main_fast + jmp tx2q + +.pass2_end: + lea r3, [o(.pass2_end1)] + jmp m(idct_8x32_internal_8bpc).end + +.pass2_end1: + lea tx2q, [o(.pass2_end)] + add coeffq, 16*32 + mov dstq, [rsp+gprsize*2+16*35] + mov r3d, [rsp+gprsize*3+16*35] + dec r3d + jg .pass2_loop + + ret + + +cglobal inv_txfm_add_identity_identity_32x32_8bpc, 4, 6, 8, 16*5, dst, stride, coeff, eob, tx2 + mov r4d, 2 + cmp eobd, 136 + mov r3d, 4 + cmovs r3d, r4d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + lea r4, [dstq+8] + mov [rsp+gprsize*0+16*3], r4 + mov [rsp+gprsize*1+16*3], r3d + mov [rsp+gprsize*2+16*3], r3d + mov [rsp+gprsize*3+16*3], coeffq + +.loop: + LOAD_8ROWS coeffq, 64 + mova [rsp+16*1], m6 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 + pmulhrsw m7, [o(pw_8192)] + mova [rsp+16*0], m7 + mova m7, [o(pw_8192)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + mova [rsp+16*1], m6 + mova [rsp+16*2], m5 + call m(idct_8x8_internal_8bpc).end3 + lea dstq, [dstq+strideq*2] + + pxor m7, m7 + REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + + add coeffq, 16 + dec r3d + jg .loop + + mov r4d, [rsp+gprsize*2+16*3] + dec r4d + jle .ret + + mov dstq, [rsp+gprsize*0+16*3] + mov coeffq, [rsp+gprsize*3+16*3] + mov [rsp+gprsize*2+16*3], r4 + lea r3, [dstq+8] + add coeffq, 64*8 + mov [rsp+gprsize*0+16*3], r3 + mov r3d, [rsp+gprsize*1+16*3] + mov [rsp+gprsize*3+16*3], coeffq + jmp .loop + +.ret: + RET + + +cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_16x64_internal_8bpc) +.end: + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r2d, 32 + lea tx2q, [o(.end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly + + +cglobal idct_16x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r4d, 2 + sub eobd, 151 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + mov [rsp+gprsize*2+16*67], coeffq + +.pass1_loop: + LOAD_8ROWS coeffq+64*0, 64*2 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+64*1, 64*2 + call m(idct_16x8_internal_8bpc).main + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+64*0, 64 + + add coeffq, 16 + dec r3d + jg .pass1_loop + + mov coeffq, [rsp+gprsize*2+16*67] + mov r3d, 2 + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(.end1)] + +.pass2_loop: + mov [rsp+gprsize*3+16*67], r3d + mov eobd, [rsp+gprsize*1+16*67] + + mova m0, [coeffq+16*4 ] ;in1 + mova m1, [coeffq+16*12] ;in3 + mova m2, [coeffq+16*20] ;in5 + mova m3, [coeffq+16*28] ;in7 + mova m4, [coeffq+16*5 ] ;in9 + mova m5, [coeffq+16*13] ;in11 + mova m6, [coeffq+16*21] ;in13 + mova m7, [coeffq+16*29] ;in15 + mova [rsp+gprsize+16*35], m0 ;in1 + mova [rsp+gprsize+16*49], m1 ;in3 + mova [rsp+gprsize+16*43], m2 ;in5 + mova [rsp+gprsize+16*41], m3 ;in7 + mova [rsp+gprsize+16*39], m4 ;in9 + mova [rsp+gprsize+16*45], m5 ;in11 + mova [rsp+gprsize+16*47], m6 ;in13 + mova [rsp+gprsize+16*37], m7 ;in15 + + pxor m4, m4 + mova m0, [coeffq+16*0] + mova m1, [coeffq+16*1] + + test eobd, eobd + jl .fast + +.full: + mova m2, [coeffq+16*2] + mova m3, [coeffq+16*3] + + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + pxor m4, m4 + mova m0, [coeffq+16*16] + mova m1, [coeffq+16*17] + mova m2, [coeffq+16*18] + mova m3, [coeffq+16*19] + + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + mova m0, [coeffq+16*8 ] + mova m1, [coeffq+16*24] + mova m2, [coeffq+16*9 ] + mova m3, [coeffq+16*25] + mova m4, [coeffq+16*10] + mova m5, [coeffq+16*26] + mova m6, [coeffq+16*11] + mova m7, [coeffq+16*27] + mova [rsp+gprsize+16*19], m0 + mova [rsp+gprsize+16*26], m1 + mova [rsp+gprsize+16*23], m2 + mova [rsp+gprsize+16*22], m3 + mova [rsp+gprsize+16*21], m4 + mova [rsp+gprsize+16*24], m5 + mova [rsp+gprsize+16*25], m6 + mova [rsp+gprsize+16*20], m7 + + call m(idct_8x32_internal_8bpc).main_fast + SAVE_8ROWS rsp+gprsize+16*3, 16 + + mova m0, [coeffq+16*6 ] ;in17 + mova m1, [coeffq+16*14] ;in19 + mova m2, [coeffq+16*22] ;in21 + mova m3, [coeffq+16*30] ;in23 + mova m4, [coeffq+16*7 ] ;in25 + mova m5, [coeffq+16*15] ;in27 + mova m6, [coeffq+16*23] ;in29 + mova m7, [coeffq+16*31] ;in31 + mova [rsp+gprsize+16*63], m0 ;in17 + mova [rsp+gprsize+16*53], m1 ;in19 + mova [rsp+gprsize+16*55], m2 ;in21 + mova [rsp+gprsize+16*61], m3 ;in23 + mova [rsp+gprsize+16*59], m4 ;in25 + mova [rsp+gprsize+16*57], m5 ;in27 + mova [rsp+gprsize+16*51], m6 ;in29 + mova [rsp+gprsize+16*65], m7 ;in31 + + call .main + jmp .end + +.fast: + REPX {mova x, m4}, m2, m3, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + pxor m4, m4 + mova m0, [coeffq+16*16] + mova m1, [coeffq+16*17] + + REPX {mova x, m4}, m2, m3, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + mova m0, [coeffq+16*8 ] + mova m1, [coeffq+16*24] + mova m2, [coeffq+16*9 ] + mova m3, [coeffq+16*25] + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + + call m(idct_8x32_internal_8bpc).main_veryfast + SAVE_8ROWS rsp+gprsize+16*3, 16 + + call .main_fast + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mov r3, r4 + jmp m(idct_8x32_internal_8bpc).end2 + +.end1: + LOAD_8ROWS rsp+gprsize+16*35, 16 + lea dstq, [dstq+strideq*2] + lea r3, [rsp+16*32+gprsize] + call .write + mov dstq, [rsp+gprsize*2+16*67] + mov r3d, [rsp+gprsize*3+16*67] + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(.end1)] + + dec r3d + jg .pass2_loop + ret +.write: + mova [r3+16*0], m7 + mov r4, -16*32 + pxor m7, m7 + sub coeffq, r4 +.zero_loop: + mova [coeffq+r4+16*0], m7 + mova [coeffq+r4+16*1], m7 + add r4, 16*2 + jl .zero_loop + call .write_main2 + LOAD_8ROWS r3+16*11, 16 + call .write_main + LOAD_8ROWS r3+16*19, 16 + call .write_main + LOAD_8ROWS r3+16*27, 16 +.write_main: + mova [r3+16*0], m7 +.write_main2: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [r3+16*0] + mova [r3+16*2], m5 + mova [r3+16*1], m6 + mova [r3+16*0], m7 + WRITE_8X4 0, 1, 2, 3, 5, 6, 7 + lea dstq, [dstq+strideq*2] + WRITE_8X4 4, [r3+16*2], [r3+16*1], [r3+16*0], 5, 6, 7 + lea dstq, [dstq+strideq*2] + ret + + +ALIGN function_align +cglobal_label .main_fast + mova m0, [rsp+gprsize*2+16*35] ;in1 + pmulhrsw m3, m0, [o(pw_4095x8)] ;t62,t63 + pmulhrsw m0, [o(pw_101x8)] ;t32,t33 + mova m7, [o(pd_2048)] + mova [rsp+gprsize*2+16*35], m0 ;t32 + mova [rsp+gprsize*2+16*66], m3 ;t63 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 401, 4076 ;t33a, t62a + mova [rsp+gprsize*2+16*36], m3 ;t33a + mova [rsp+gprsize*2+16*65], m0 ;t62a + + mova m1, [rsp+gprsize*2+16*37] ;in15 + pmulhrsw m2, m1, [o(pw_3822x8)] ;t60,t61 + pmulhrsw m1, [o(pw_m1474x8)] ;t34,t35 + mova [rsp+gprsize*2+16*38], m1 ;t35 + mova [rsp+gprsize*2+16*63], m2 ;t60 + ITX_MULSUB_2W 2, 1, 0, 3, 7, m4076, 401 ;t34a, t61a + mova [rsp+gprsize*2+16*37], m2 ;t34a + mova [rsp+gprsize*2+16*64], m1 ;t61a + + mova m0, [rsp+gprsize*2+16*39] ;in9 + pmulhrsw m3, m0, [o(pw_3996x8)] ;t58,t59 + pmulhrsw m0, [o(pw_897x8)] ;t36,t37 + mova [rsp+gprsize*2+16*39], m0 ;t36 + mova [rsp+gprsize*2+16*62], m3 ;t59 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 3166, 2598 ;t37a, t58a + mova [rsp+gprsize*2+16*40], m3 ;t37a + mova [rsp+gprsize*2+16*61], m0 ;t58a + + mova m1, [rsp+gprsize*2+16*41] ;in7 + pmulhrsw m2, m1, [o(pw_4036x8)] ;t56,t57 + pmulhrsw m1, [o(pw_m700x8)] ;t38,t39 + mova [rsp+gprsize*2+16*42], m1 ;t39 + mova [rsp+gprsize*2+16*59], m2 ;t56 + ITX_MULSUB_2W 2, 1, 0, 3, 7, m2598, 3166 ;t38a, t57a + mova [rsp+gprsize*2+16*41], m2 ;t38a + mova [rsp+gprsize*2+16*60], m1 ;t57a + + mova m0, [rsp+gprsize*2+16*43] ;in5 + pmulhrsw m3, m0, [o(pw_4065x8)] ;t54,t55 + pmulhrsw m0, [o(pw_501x8)] ;t40,t41 + mova [rsp+gprsize*2+16*43], m0 ;t40 + mova [rsp+gprsize*2+16*58], m3 ;t55 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 1931, 3612 ;t41a, t54a + mova [rsp+gprsize*2+16*44], m3 ;t41a + mova [rsp+gprsize*2+16*57], m0 ;t54a + + mova m1, [rsp+gprsize*2+16*45] ;in11 + pmulhrsw m2, m1, [o(pw_3948x8)] ;t52,t53 + pmulhrsw m1, [o(pw_m1092x8)] ;t42,t43 + mova [rsp+gprsize*2+16*46], m1 ;t43 + mova [rsp+gprsize*2+16*55], m2 ;t52 + ITX_MULSUB_2W 2, 1, 0, 3, 7, m3612, 1931 ;t42a, t53a + mova [rsp+gprsize*2+16*45], m2 ;t42a + mova [rsp+gprsize*2+16*56], m1 ;t53a + + mova m0, [rsp+gprsize*2+16*47] ;in13 + pmulhrsw m3, m0, [o(pw_3889x8)] ;t50,t51 + pmulhrsw m0, [o(pw_1285x8)] ;t44,t45 + mova m6, m0 + mova [rsp+gprsize*2+16*54], m3 ;t51 + ITX_MULSUB_2W 3, 0, 1, 2, 7, 3920, 1189 ;t45a, t50a + mova [rsp+gprsize*2+16*48], m3 ;t45a + mova [rsp+gprsize*2+16*53], m0 ;t50a + + mova m0, [rsp+gprsize*2+16*49] ;in3 + pmulhrsw m3, m0, [o(pw_4085x8)] ;t48,t49 + pmulhrsw m0, [o(pw_m301x8)] ;t46,t47 + mova m4, m3 + mova m5, m0 + + jmp .main2 + +ALIGN function_align +cglobal_label .main + mova m0, [rsp+gprsize*2+16*35] ;in1 + mova m1, [rsp+gprsize*2+16*65] ;in31 + pmulhrsw m3, m0, [o(pw_4095x8)] ;t63a + pmulhrsw m0, [o(pw_101x8)] ;t32a + pmulhrsw m2, m1, [o(pw_2967x8)] ;t62a + pmulhrsw m1, [o(pw_m2824x8)] ;t33a + mova m7, [o(pd_2048)] + psubsw m4, m0, m1 ;t33 + paddsw m0, m1 ;t32 + psubsw m5, m3, m2 ;t62 + paddsw m3, m2 ;t63 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 401, 4076 ;t33a, t62a + mova [rsp+gprsize*2+16*35], m0 ;t32 + mova [rsp+gprsize*2+16*36], m5 ;t33a + mova [rsp+gprsize*2+16*65], m4 ;t62a + mova [rsp+gprsize*2+16*66], m3 ;t63 + + mova m0, [rsp+gprsize*2+16*63] ;in17 + mova m1, [rsp+gprsize*2+16*37] ;in15 + pmulhrsw m3, m0, [o(pw_3745x8)] ;t61a + pmulhrsw m0, [o(pw_1660x8)] ;t34a + pmulhrsw m2, m1, [o(pw_3822x8)] ;t60a + pmulhrsw m1, [o(pw_m1474x8)] ;t35a + psubsw m4, m1, m0 ;t34 + paddsw m0, m1 ;t35 + psubsw m5, m2, m3 ;t61 + paddsw m3, m2 ;t60 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m4076, 401 ;t34a, t61a + mova [rsp+gprsize*2+16*37], m5 ;t34a + mova [rsp+gprsize*2+16*38], m0 ;t35 + mova [rsp+gprsize*2+16*63], m3 ;t60 + mova [rsp+gprsize*2+16*64], m4 ;t61a + + mova m0, [rsp+gprsize*2+16*39] ;in9 + mova m1, [rsp+gprsize*2+16*61] ;in23 + pmulhrsw m3, m0, [o(pw_3996x8)] ;t59a + pmulhrsw m0, [o(pw_897x8)] ;t36a + pmulhrsw m2, m1, [o(pw_3461x8)] ;t58a + pmulhrsw m1, [o(pw_m2191x8)] ;t37a + psubsw m4, m0, m1 ;t37 + paddsw m0, m1 ;t36 + psubsw m5, m3, m2 ;t58 + paddsw m3, m2 ;t59 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 3166, 2598 ;t37a, t58a + mova [rsp+gprsize*2+16*39], m0 ;t36 + mova [rsp+gprsize*2+16*40], m5 ;t37a + mova [rsp+gprsize*2+16*61], m4 ;t58a + mova [rsp+gprsize*2+16*62], m3 ;t59 + + mova m0, [rsp+gprsize*2+16*59] ;in25 + mova m1, [rsp+gprsize*2+16*41] ;in7 + pmulhrsw m3, m0, [o(pw_3349x8)] ;t57a + pmulhrsw m0, [o(pw_2359x8)] ;t38a + pmulhrsw m2, m1, [o(pw_4036x8)] ;t56a + pmulhrsw m1, [o(pw_m700x8)] ;t39a + psubsw m4, m1, m0 ;t38 + paddsw m0, m1 ;t39 + psubsw m5, m2, m3 ;t57 + paddsw m3, m2 ;t56 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m2598, 3166 ;t38a, t57a + mova [rsp+gprsize*2+16*41], m5 ;t38a + mova [rsp+gprsize*2+16*42], m0 ;t39 + mova [rsp+gprsize*2+16*59], m3 ;t56 + mova [rsp+gprsize*2+16*60], m4 ;t57a + + mova m0, [rsp+gprsize*2+16*43] ;in5 + mova m1, [rsp+gprsize*2+16*57] ;in27 + pmulhrsw m3, m0, [o(pw_4065x8)] ;t55a + pmulhrsw m0, [o(pw_501x8)] ;t40a + pmulhrsw m2, m1, [o(pw_3229x8)] ;t54a + pmulhrsw m1, [o(pw_m2520x8)] ;t41a + psubsw m4, m0, m1 ;t41 + paddsw m0, m1 ;t40 + psubsw m5, m3, m2 ;t54 + paddsw m3, m2 ;t55 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 1931, 3612 ;t41a, t54a + mova [rsp+gprsize*2+16*43], m0 ;t40 + mova [rsp+gprsize*2+16*44], m5 ;t41a + mova [rsp+gprsize*2+16*57], m4 ;t54a + mova [rsp+gprsize*2+16*58], m3 ;t55 + + mova m0, [rsp+gprsize*2+16*55] ;in21 + mova m1, [rsp+gprsize*2+16*45] ;in11 + pmulhrsw m3, m0, [o(pw_3564x8)] ;t53a + pmulhrsw m0, [o(pw_2019x8)] ;t42a + pmulhrsw m2, m1, [o(pw_3948x8)] ;t52a + pmulhrsw m1, [o(pw_m1092x8)] ;t43a + psubsw m4, m1, m0 ;t42 + paddsw m0, m1 ;t43 + psubsw m5, m2, m3 ;t53 + paddsw m3, m2 ;t52 + ITX_MULSUB_2W 5, 4, 1, 2, 7, m3612, 1931 ;t42a, t53a + mova [rsp+gprsize*2+16*45], m5 ;t42a + mova [rsp+gprsize*2+16*46], m0 ;t43 + mova [rsp+gprsize*2+16*55], m3 ;t52 + mova [rsp+gprsize*2+16*56], m4 ;t53a + + mova m0, [rsp+gprsize*2+16*47] ;in13 + mova m1, [rsp+gprsize*2+16*53] ;in19 + pmulhrsw m3, m0, [o(pw_3889x8)] ;t51a + pmulhrsw m0, [o(pw_1285x8)] ;t44a + pmulhrsw m2, m1, [o(pw_3659x8)] ;t50a + pmulhrsw m1, [o(pw_m1842x8)] ;t45a + psubsw m4, m0, m1 ;t45 + paddsw m0, m1 ;t44 + psubsw m5, m3, m2 ;t50 + paddsw m3, m2 ;t51 + ITX_MULSUB_2W 5, 4, 1, 2, 7, 3920, 1189 ;t45a, t50a + mova m6, m0 + mova [rsp+gprsize*2+16*48], m5 ;t45a + mova [rsp+gprsize*2+16*53], m4 ;t50a + mova [rsp+gprsize*2+16*54], m3 ;t51 + + mova m0, [rsp+gprsize*2+16*51] ;in29 + mova m1, [rsp+gprsize*2+16*49] ;in3 + pmulhrsw m3, m0, [o(pw_3102x8)] ;t49a + pmulhrsw m0, [o(pw_2675x8)] ;t46a + pmulhrsw m2, m1, [o(pw_4085x8)] ;t48a + pmulhrsw m1, [o(pw_m301x8)] ;t47a + psubsw m5, m1, m0 ;t46 + paddsw m0, m1 ;t47 + psubsw m4, m2, m3 ;t49 + paddsw m3, m2 ;t48 + +ALIGN function_align +.main2: + ITX_MULSUB_2W 4, 5, 1, 2, 7, m1189, 3920 ;t46a, t49a + mova m1, [rsp+gprsize*2+16*54] ;t51 + psubsw m2, m0, m6 ;t44a + paddsw m0, m6 ;t47a + psubsw m6, m3, m1 ;t51a + paddsw m3, m1 ;t48a + mova [rsp+gprsize*2+16*50], m0 ;t47a + mova [rsp+gprsize*2+16*51], m3 ;t48a + ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t44, t51 + mova [rsp+gprsize*2+16*47], m6 ;t44 + mova [rsp+gprsize*2+16*54], m2 ;t51 + + mova m0, [rsp+gprsize*2+16*48] ;t45a + mova m3, [rsp+gprsize*2+16*53] ;t50a + psubsw m2, m4, m0 ;t45 + paddsw m4, m0 ;t46 + psubsw m6, m5, m3 ;t50 + paddsw m5, m3 ;t49 + ITX_MULSUB_2W 6, 2, 0, 3, 7, m2276, 3406 ;t45a, t50a + mova [rsp+gprsize*2+16*48], m6 ;t45a + mova [rsp+gprsize*2+16*49], m4 ;t46 + mova [rsp+gprsize*2+16*52], m5 ;t49 + mova [rsp+gprsize*2+16*53], m2 ;t50a + + mova m0, [rsp+gprsize*2+16*43] ;t40 + mova m2, [rsp+gprsize*2+16*46] ;t43 + mova m3, [rsp+gprsize*2+16*55] ;t52 + mova m1, [rsp+gprsize*2+16*58] ;t55 + psubsw m4, m0, m2 ;t43a + paddsw m0, m2 ;t40a + psubsw m5, m1, m3 ;t52a + paddsw m1, m3 ;t55a + ITX_MULSUB_2W 5, 4, 2, 3, 7, 3406, 2276 ;t43, t52 + mova [rsp+gprsize*2+16*43], m0 ;t40a + mova [rsp+gprsize*2+16*46], m5 ;t43 + mova [rsp+gprsize*2+16*55], m4 ;t52 + mova [rsp+gprsize*2+16*58], m1 ;t55a + + mova m0, [rsp+gprsize*2+16*44] ;t41a + mova m2, [rsp+gprsize*2+16*45] ;t42a + mova m3, [rsp+gprsize*2+16*56] ;t53a + mova m1, [rsp+gprsize*2+16*57] ;t54a + psubsw m4, m0, m2 ;t42 + paddsw m0, m2 ;t41 + psubsw m5, m1, m3 ;t53 + paddsw m1, m3 ;t54 + ITX_MULSUB_2W 5, 4, 2, 3, 7, 3406, 2276 ;t42a, t53a + mova [rsp+gprsize*2+16*44], m0 ;t41 + mova [rsp+gprsize*2+16*45], m5 ;t42a + mova [rsp+gprsize*2+16*56], m4 ;t53a + mova [rsp+gprsize*2+16*57], m1 ;t54 + + mova m0, [rsp+gprsize*2+16*41] ;t38a + mova m2, [rsp+gprsize*2+16*40] ;t37a + mova m3, [rsp+gprsize*2+16*61] ;t58a + mova m1, [rsp+gprsize*2+16*60] ;t57a + psubsw m4, m0, m2 ;t37 + paddsw m0, m2 ;t38 + psubsw m5, m1, m3 ;t58 + paddsw m1, m3 ;t57 + ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t37a, t58a + mova [rsp+gprsize*2+16*41], m0 ;t38 + mova [rsp+gprsize*2+16*40], m5 ;t37a + mova [rsp+gprsize*2+16*61], m4 ;t58a + mova [rsp+gprsize*2+16*60], m1 ;t57 + + mova m0, [rsp+gprsize*2+16*42] ;t39 + mova m2, [rsp+gprsize*2+16*39] ;t36 + mova m3, [rsp+gprsize*2+16*62] ;t59 + mova m1, [rsp+gprsize*2+16*59] ;t56 + psubsw m4, m0, m2 ;t36a + paddsw m0, m2 ;t39a + psubsw m5, m1, m3 ;t59a + paddsw m1, m3 ;t56a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m4017, 799 ;t36, t59 + mova [rsp+gprsize*2+16*42], m0 ;t39a + mova [rsp+gprsize*2+16*39], m5 ;t36 + mova [rsp+gprsize*2+16*62], m4 ;t59 + mova [rsp+gprsize*2+16*59], m1 ;t56a + + mova m0, [rsp+gprsize*2+16*35] ;t32 + mova m2, [rsp+gprsize*2+16*38] ;t35 + mova m3, [rsp+gprsize*2+16*63] ;t60 + mova m1, [rsp+gprsize*2+16*66] ;t63 + psubsw m4, m0, m2 ;t35a + paddsw m0, m2 ;t32a + psubsw m5, m1, m3 ;t60a + paddsw m1, m3 ;t63a + ITX_MULSUB_2W 5, 4, 2, 3, 7, 799, 4017 ;t35, t60 + mova [rsp+gprsize*2+16*35], m0 ;t32a + mova [rsp+gprsize*2+16*38], m5 ;t35 + mova [rsp+gprsize*2+16*63], m4 ;t60 + mova [rsp+gprsize*2+16*66], m1 ;t63a + + mova m0, [rsp+gprsize*2+16*36] ;t33a + mova m2, [rsp+gprsize*2+16*37] ;t34a + mova m3, [rsp+gprsize*2+16*64] ;t61a + mova m1, [rsp+gprsize*2+16*65] ;t62a + psubsw m4, m0, m2 ;t34 + paddsw m0, m2 ;t33 + psubsw m5, m1, m3 ;t61 + paddsw m1, m3 ;t62 + ITX_MULSUB_2W 5, 4, 2, 3, 7, 799, 4017 ;t34a, t61a + + mova m2, [rsp+gprsize*2+16*41] ;t38 + mova m3, [rsp+gprsize*2+16*60] ;t57 + psubsw m6, m0, m2 ;t38a + paddsw m0, m2 ;t33a + psubsw m2, m1, m3 ;t57a + paddsw m1, m3 ;t62a + mova [rsp+gprsize*2+16*36], m0 ;t33a + mova [rsp+gprsize*2+16*65], m1 ;t62a + ITX_MULSUB_2W 2, 6, 0, 3, 7, 1567, 3784 ;t38, t57 + mova [rsp+gprsize*2+16*41], m2 ;t38 + mova [rsp+gprsize*2+16*60], m6 ;t57 + + mova m2, [rsp+gprsize*2+16*40] ;t37 + mova m3, [rsp+gprsize*2+16*61] ;t58 + psubsw m0, m5, m2 ;t37 + paddsw m5, m2 ;t34 + psubsw m1, m4, m3 ;t58 + paddsw m4, m3 ;t61 + ITX_MULSUB_2W 1, 0, 2, 3, 7, 1567, 3784 ;t37a, t58a + mova [rsp+gprsize*2+16*37], m5 ;t34 + mova [rsp+gprsize*2+16*64], m4 ;t61 + mova [rsp+gprsize*2+16*40], m1 ;t37a + mova [rsp+gprsize*2+16*61], m0 ;t58a + + mova m0, [rsp+gprsize*2+16*38] ;t35 + mova m2, [rsp+gprsize*2+16*39] ;t36 + mova m3, [rsp+gprsize*2+16*62] ;t59 + mova m1, [rsp+gprsize*2+16*63] ;t60 + psubsw m4, m0, m2 ;t36a + paddsw m0, m2 ;t35a + psubsw m5, m1, m3 ;t59a + paddsw m1, m3 ;t60a + ITX_MULSUB_2W 5, 4, 2, 3, 7, 1567, 3784 ;t36, t59 + mova [rsp+gprsize*2+16*38], m0 ;t35a + mova [rsp+gprsize*2+16*39], m5 ;t36 + mova [rsp+gprsize*2+16*62], m4 ;t59 + mova [rsp+gprsize*2+16*63], m1 ;t60a + + mova m0, [rsp+gprsize*2+16*35] ;t32a + mova m2, [rsp+gprsize*2+16*42] ;t39a + mova m3, [rsp+gprsize*2+16*59] ;t56a + mova m1, [rsp+gprsize*2+16*66] ;t63a + psubsw m4, m0, m2 ;t39 + paddsw m0, m2 ;t32 + psubsw m5, m1, m3 ;t56 + paddsw m1, m3 ;t63 + ITX_MULSUB_2W 5, 4, 2, 3, 7, 1567, 3784 ;t39a, t56a + mova [rsp+gprsize*2+16*35], m0 ;t32 + mova [rsp+gprsize*2+16*42], m5 ;t39a + mova [rsp+gprsize*2+16*59], m4 ;t56a + mova [rsp+gprsize*2+16*66], m1 ;t63 + + mova m0, [rsp+gprsize*2+16*50] ;t47a + mova m2, [rsp+gprsize*2+16*43] ;t40a + mova m3, [rsp+gprsize*2+16*58] ;t55a + mova m1, [rsp+gprsize*2+16*51] ;t48a + psubsw m4, m0, m2 ;t40 + paddsw m0, m2 ;t47 + psubsw m5, m1, m3 ;t55 + paddsw m1, m3 ;t48 + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t40a, t55a + mova [rsp+gprsize*2+16*50], m0 ;t47 + mova [rsp+gprsize*2+16*43], m5 ;t40a + mova [rsp+gprsize*2+16*58], m4 ;t55a + mova [rsp+gprsize*2+16*51], m1 ;t48 + + mova m0, [rsp+gprsize*2+16*49] ;t46 + mova m2, [rsp+gprsize*2+16*44] ;t41 + mova m3, [rsp+gprsize*2+16*57] ;t54 + mova m1, [rsp+gprsize*2+16*52] ;t49 + psubsw m4, m0, m2 ;t41a + paddsw m0, m2 ;t46a + psubsw m5, m1, m3 ;t54a + paddsw m1, m3 ;t49a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t41, t54 + mova [rsp+gprsize*2+16*49], m0 ;t46a + mova [rsp+gprsize*2+16*44], m5 ;t41 + mova [rsp+gprsize*2+16*57], m4 ;t54 + mova [rsp+gprsize*2+16*52], m1 ;t49a + + mova m0, [rsp+gprsize*2+16*48] ;t45a + mova m2, [rsp+gprsize*2+16*45] ;t42a + mova m3, [rsp+gprsize*2+16*56] ;t53a + mova m1, [rsp+gprsize*2+16*53] ;t50a + psubsw m4, m0, m2 ;t42 + paddsw m0, m2 ;t45 + psubsw m5, m1, m3 ;t53 + paddsw m1, m3 ;t50 + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t42a, t53a + mova [rsp+gprsize*2+16*48], m0 ;t45 + mova [rsp+gprsize*2+16*45], m5 ;t42a + mova [rsp+gprsize*2+16*56], m4 ;t53a + mova [rsp+gprsize*2+16*53], m1 ;t50 + + mova m0, [rsp+gprsize*2+16*47] ;t44 + mova m2, [rsp+gprsize*2+16*46] ;t43 + mova m3, [rsp+gprsize*2+16*55] ;t52 + mova m1, [rsp+gprsize*2+16*54] ;t51 + psubsw m4, m0, m2 ;t43a + paddsw m0, m2 ;t44a + psubsw m5, m1, m3 ;t52a + paddsw m1, m3 ;t51a + ITX_MULSUB_2W 5, 4, 2, 3, 7, m3784, 1567 ;t43, t52 + + mova m2, [rsp+gprsize*2+16*38] ;t35a + mova m3, [rsp+gprsize*2+16*31] ;tmp[28] + psubsw m6, m2, m0 ;t44 + paddsw m2, m0 ;t35 + psubsw m0, m3, m2 ;out35 + paddsw m2, m3 ;out28 + mova m3, [rsp+gprsize*2+16*63] ;t60a + mova [rsp+gprsize*2+16*38], m0 ;out35 + mova [rsp+gprsize*2+16*31], m2 ;out28 + psubsw m0, m3, m1 ;t51 + paddsw m3, m1 ;t60 + ITX_MULSUB_2W 0, 6, 1, 2, 7, 2896, 2896 ;t44a, t51a + mova m2, [rsp+gprsize*2+16*6 ] ;tmp[3] + psubsw m1, m2, m3 ;out60 + paddsw m2, m3 ;out3 + mova m3, [rsp+gprsize*2+16*22] ;tmp[19] + mova [rsp+gprsize*2+16*63], m1 ;out60 + mova [rsp+gprsize*2+16*6 ], m2 ;out3 + psubsw m1, m3, m0 ;out44 + paddsw m3, m0 ;out19 + mova m2, [rsp+gprsize*2+16*15] ;tmp[12] + + mova m0, [rsp+gprsize*2+16*39] ;t36 + mova [rsp+gprsize*2+16*47], m1 ;out44 + mova [rsp+gprsize*2+16*22], m3 ;out19 + mova m1, [rsp+gprsize*2+16*62] ;t59 + psubsw m3, m2, m6 ;out51 + paddsw m2, m6 ;out12 + mova [rsp+gprsize*2+16*54], m3 ;out51 + mova [rsp+gprsize*2+16*15], m2 ;out12 + psubsw m2, m0, m5 ;t43a + paddsw m0, m5 ;t36a + mova m5, [rsp+gprsize*2+16*30] ;tmp[27] + psubsw m3, m1, m4 ;t52a + paddsw m1, m4 ;t59a + ITX_MULSUB_2W 3, 2, 4, 6, 7, 2896, 2896 ;t43, t52 + mova m4, [rsp+gprsize*2+16*7 ] ;tmp[4 ] + psubsw m6, m5, m0 ;out36 + paddsw m5, m0 ;out27 + psubsw m0, m4, m1 ;out59 + paddsw m4, m1 ;out4 + mova [rsp+gprsize*2+16*39], m6 ;out36 + mova [rsp+gprsize*2+16*30], m5 ;out27 + mova [rsp+gprsize*2+16*62], m0 ;out59 + mova [rsp+gprsize*2+16*7 ], m4 ;out4 + mova m0, [rsp+gprsize*2+16*23] ;tmp[20] + mova m5, [rsp+gprsize*2+16*14] ;tmp[11] + psubsw m4, m0, m3 ;out43 + paddsw m0, m3 ;out20 + psubsw m6, m5, m2 ;out52 + paddsw m5, m2 ;out11 + mova [rsp+gprsize*2+16*46], m4 ;out43 + mova [rsp+gprsize*2+16*23], m0 ;out20 + mova [rsp+gprsize*2+16*55], m6 ;out52 + mova [rsp+gprsize*2+16*14], m5 ;out11 + + mova m0, [rsp+gprsize*2+16*40] ;t37a + mova m5, [rsp+gprsize*2+16*45] ;t42a + mova m3, [rsp+gprsize*2+16*56] ;t53a + mova m1, [rsp+gprsize*2+16*61] ;t58a + mova m2, [rsp+gprsize*2+16*29] ;tmp[26] + psubsw m4, m0, m5 ;t42 + paddsw m0, m5 ;t37 + psubsw m5, m1, m3 ;t53 + paddsw m1, m3 ;t58 + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t43, t52 + mova m3, [rsp+gprsize*2+16*8 ] ;tmp[5 ] + psubsw m6, m2, m0 ;out37 + paddsw m2, m0 ;out26 + psubsw m0, m3, m1 ;out58 + paddsw m3, m1 ;out5 + mova [rsp+gprsize*2+16*40], m6 ;out37 + mova [rsp+gprsize*2+16*29], m2 ;out26 + mova [rsp+gprsize*2+16*61], m0 ;out58 + mova [rsp+gprsize*2+16*8 ], m3 ;out5 + mova m0, [rsp+gprsize*2+16*24] ;tmp[21] + mova m1, [rsp+gprsize*2+16*13] ;tmp[10] + psubsw m2, m0, m5 ;out42 + paddsw m0, m5 ;out21 + psubsw m3, m1, m4 ;out53 + paddsw m1, m4 ;out10 + mova [rsp+gprsize*2+16*45], m2 ;out42 + mova [rsp+gprsize*2+16*24], m0 ;out21 + mova [rsp+gprsize*2+16*56], m3 ;out53 + mova [rsp+gprsize*2+16*13], m1 ;out10 + + mova m0, [rsp+gprsize*2+16*41] ;t38 + mova m5, [rsp+gprsize*2+16*44] ;t41 + mova m3, [rsp+gprsize*2+16*57] ;t54 + mova m1, [rsp+gprsize*2+16*60] ;t57 + mova m2, [rsp+gprsize*2+16*28] ;tmp[25] + psubsw m4, m0, m5 ;t41a + paddsw m0, m5 ;t38a + psubsw m5, m1, m3 ;t54a + paddsw m1, m3 ;t57a + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t41a, t54a + mova m3, [rsp+gprsize*2+16*9 ] ;tmp[6 ] + psubsw m6, m2, m0 ;out38 + paddsw m2, m0 ;out25 + psubsw m0, m3, m1 ;out57 + paddsw m3, m1 ;out6 + mova [rsp+gprsize*2+16*41], m6 ;out38 + mova [rsp+gprsize*2+16*28], m2 ;out25 + mova [rsp+gprsize*2+16*60], m0 ;out57 + mova [rsp+gprsize*2+16*9 ], m3 ;out6 + mova m0, [rsp+gprsize*2+16*25] ;tmp[22] + mova m1, [rsp+gprsize*2+16*12] ;tmp[9 ] + psubsw m2, m0, m5 ;out41 + paddsw m0, m5 ;out22 + psubsw m3, m1, m4 ;out54 + paddsw m1, m4 ;out9 + mova [rsp+gprsize*2+16*44], m2 ;out41 + mova [rsp+gprsize*2+16*25], m0 ;out22 + mova [rsp+gprsize*2+16*57], m3 ;out54 + mova [rsp+gprsize*2+16*12], m1 ;out9 + + mova m0, [rsp+gprsize*2+16*42] ;t39a + mova m5, [rsp+gprsize*2+16*43] ;t40a + mova m3, [rsp+gprsize*2+16*58] ;t55a + mova m1, [rsp+gprsize*2+16*59] ;t56a + mova m2, [rsp+gprsize*2+16*27] ;tmp[24] + psubsw m4, m0, m5 ;t40 + paddsw m0, m5 ;t39 + psubsw m5, m1, m3 ;t55 + paddsw m1, m3 ;t56 + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t40a, t55a + mova m3, [rsp+gprsize*2+16*10] ;tmp[7 ] + psubsw m6, m2, m0 ;out39 + paddsw m2, m0 ;out24 + psubsw m0, m3, m1 ;out56 + paddsw m3, m1 ;out7 + mova [rsp+gprsize*2+16*42], m6 ;out39 + mova [rsp+gprsize*2+16*27], m2 ;out24 + mova [rsp+gprsize*2+16*59], m0 ;out56 + mova [rsp+gprsize*2+16*10], m3 ;out7 + mova m0, [rsp+gprsize*2+16*26] ;tmp[23] + mova m1, [rsp+gprsize*2+16*11] ;tmp[8 ] + psubsw m2, m0, m5 ;out40 + paddsw m0, m5 ;out23 + psubsw m3, m1, m4 ;out55 + paddsw m1, m4 ;out8 + mova [rsp+gprsize*2+16*43], m2 ;out40 + mova [rsp+gprsize*2+16*26], m0 ;out23 + mova [rsp+gprsize*2+16*58], m3 ;out55 + mova [rsp+gprsize*2+16*11], m1 ;out8 + + mova m0, [rsp+gprsize*2+16*37] ;t34 + mova m5, [rsp+gprsize*2+16*48] ;t45 + mova m3, [rsp+gprsize*2+16*53] ;t50 + mova m1, [rsp+gprsize*2+16*64] ;t61 + mova m2, [rsp+gprsize*2+16*32] ;tmp[29] + psubsw m4, m0, m5 ;t45a + paddsw m0, m5 ;t34a + psubsw m5, m1, m3 ;t50a + paddsw m1, m3 ;t61a + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t45, t50 + mova m3, [rsp+gprsize*2+16*5 ] ;tmp[2 ] + psubsw m6, m2, m0 ;out34 + paddsw m2, m0 ;out29 + psubsw m0, m3, m1 ;out61 + paddsw m3, m1 ;out2 + mova [rsp+gprsize*2+16*37], m6 ;out34 + mova [rsp+gprsize*2+16*32], m2 ;out29 + mova [rsp+gprsize*2+16*64], m0 ;out61 + mova [rsp+gprsize*2+16*5 ], m3 ;out2 + mova m0, [rsp+gprsize*2+16*21] ;tmp[18] + mova m1, [rsp+gprsize*2+16*16] ;tmp[13] + psubsw m2, m0, m5 ;out45 + paddsw m0, m5 ;out18 + psubsw m3, m1, m4 ;out50 + paddsw m1, m4 ;out13 + mova [rsp+gprsize*2+16*48], m2 ;out45 + mova [rsp+gprsize*2+16*21], m0 ;out18 + mova [rsp+gprsize*2+16*53], m3 ;out50 + mova [rsp+gprsize*2+16*16], m1 ;out13 + + mova m0, [rsp+gprsize*2+16*36] ;t33a + mova m5, [rsp+gprsize*2+16*49] ;t46a + mova m3, [rsp+gprsize*2+16*52] ;t49a + mova m1, [rsp+gprsize*2+16*65] ;t62a + mova m2, [rsp+gprsize*2+16*33] ;tmp[30] + psubsw m4, m0, m5 ;t46 + paddsw m0, m5 ;t33 + psubsw m5, m1, m3 ;t49 + paddsw m1, m3 ;t62 + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t45, t50 + mova m3, [rsp+gprsize*2+16*4 ] ;tmp[1 ] + psubsw m6, m2, m0 ;out33 + paddsw m2, m0 ;out30 + psubsw m0, m3, m1 ;out62 + paddsw m3, m1 ;out1 + mova [rsp+gprsize*2+16*36], m6 ;out33 + mova [rsp+gprsize*2+16*33], m2 ;out30 + mova [rsp+gprsize*2+16*65], m0 ;out62 + mova [rsp+gprsize*2+16*4 ], m3 ;out1 + mova m0, [rsp+gprsize*2+16*20] ;tmp[17] + mova m1, [rsp+gprsize*2+16*17] ;tmp[14] + psubsw m2, m0, m5 ;out46 + paddsw m0, m5 ;out17 + psubsw m3, m1, m4 ;out49 + paddsw m1, m4 ;out14 + mova [rsp+gprsize*2+16*49], m2 ;out46 + mova [rsp+gprsize*2+16*20], m0 ;out17 + mova [rsp+gprsize*2+16*52], m3 ;out49 + mova [rsp+gprsize*2+16*17], m1 ;out14 + + mova m0, [rsp+gprsize*2+16*35] ;t32 + mova m5, [rsp+gprsize*2+16*50] ;t47 + mova m3, [rsp+gprsize*2+16*51] ;t48 + mova m1, [rsp+gprsize*2+16*66] ;t63 + mova m2, [rsp+gprsize*2+16*34] ;tmp[31] + psubsw m4, m0, m5 ;t47a + paddsw m0, m5 ;t32a + psubsw m5, m1, m3 ;t48a + paddsw m1, m3 ;t63a + ITX_MULSUB_2W 5, 4, 3, 6, 7, 2896, 2896 ;t47, t48 + mova m3, [rsp+gprsize*2+16*3 ] ;tmp[0 ] + psubsw m6, m2, m0 ;out32 + paddsw m2, m0 ;out31 + psubsw m0, m3, m1 ;out63 + paddsw m3, m1 ;out0 + mova [rsp+gprsize*2+16*35], m6 ;out32 + mova [rsp+gprsize*2+16*34], m2 ;out31 + mova [rsp+gprsize*2+16*66], m0 ;out63 + mova [rsp+gprsize*2+16*3 ], m3 ;out0 + mova m0, [rsp+gprsize*2+16*19] ;tmp[16] + mova m1, [rsp+gprsize*2+16*18] ;tmp[15] + psubsw m2, m0, m5 ;out47 + paddsw m0, m5 ;out16 + psubsw m3, m1, m4 ;out48 + paddsw m1, m4 ;out15 + mova [rsp+gprsize*2+16*50], m2 ;out47 + mova [rsp+gprsize*2+16*19], m0 ;out16 + mova [rsp+gprsize*2+16*51], m3 ;out48 + mova [rsp+gprsize*2+16*18], m1 ;out15 + ret + + +cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 6, 8, 16*132, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + + call m(idct_64x16_internal_8bpc) + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r3d, 16 + lea tx2q, [o(.end)] + +.body: + pmulhrsw m0, m2 + movd m2, [o(pw_2048)] ;intentionally rip-relative + pmulhrsw m0, m1 + pmulhrsw m0, m2 + pshuflw m0, m0, q0000 + punpcklwd m0, m0 + pxor m7, m7 + +.loop: + mova m1, [dstq+16*0] + mova m3, [dstq+16*1] + mova m5, [dstq+16*2] + mova m6, [dstq+16*3] + punpckhbw m2, m1, m7 + punpcklbw m1, m7 + punpckhbw m4, m3, m7 + punpcklbw m3, m7 + paddw m2, m0 + paddw m1, m0 + paddw m4, m0 + paddw m3, m0 + packuswb m1, m2 + packuswb m3, m4 + punpckhbw m2, m5, m7 + punpcklbw m5, m7 + punpckhbw m4, m6, m7 + punpcklbw m6, m7 + paddw m2, m0 + paddw m5, m0 + paddw m4, m0 + paddw m6, m0 + packuswb m5, m2 + packuswb m6, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m3 + mova [dstq+16*2], m5 + mova [dstq+16*3], m6 + add dstq, strideq + dec r3d + jg .loop + jmp tx2q + +.end: + RET + + +%macro LOAD_4ROWS 2-3 0 ;src, stride, is_rect2 + +%if %3 + mova m3, [o(pw_2896x8)] + pmulhrsw m0, m3, [%1+%2*0] + pmulhrsw m1, m3, [%1+%2*1] + pmulhrsw m2, m3, [%1+%2*2] + pmulhrsw m3, [%1+%2*3] +%else + mova m0, [%1+%2*0] + mova m1, [%1+%2*1] + mova m2, [%1+%2*2] + mova m3, [%1+%2*3] +%endif +%endmacro + +%macro LOAD_4ROWS_H 2 ;src, stride + mova m4, [%1+%2*0] + mova m5, [%1+%2*1] + mova m6, [%1+%2*2] + mova m7, [%1+%2*3] +%endmacro + +cglobal idct_64x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r3d, 2 + mov [rsp+gprsize*2+16*67], dstq + lea dstq, [rsp+gprsize+16*68] + +.pass1_loop: + LOAD_4ROWS coeffq+32*0, 32*8 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + pxor m4, m4 + LOAD_4ROWS coeffq+32*4, 32*8 + + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+32*2, 32*4 + mova [rsp+gprsize+16*19], m0 + mova [rsp+gprsize+16*26], m1 + mova [rsp+gprsize+16*23], m2 + mova [rsp+gprsize+16*22], m3 + mova [rsp+gprsize+16*21], m4 + mova [rsp+gprsize+16*24], m5 + mova [rsp+gprsize+16*25], m6 + mova [rsp+gprsize+16*20], m7 + + call m(idct_8x32_internal_8bpc).main_fast + SAVE_8ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+32*1, 32*2 + mova [rsp+gprsize+16*35], m0 ;in1 + mova [rsp+gprsize+16*49], m1 ;in3 + mova [rsp+gprsize+16*43], m2 ;in5 + mova [rsp+gprsize+16*41], m3 ;in7 + mova [rsp+gprsize+16*39], m4 ;in9 + mova [rsp+gprsize+16*45], m5 ;in11 + mova [rsp+gprsize+16*47], m6 ;in13 + mova [rsp+gprsize+16*37], m7 ;in15 + + LOAD_8ROWS coeffq+32*17, 32*2 + mova [rsp+gprsize+16*63], m0 ;in17 + mova [rsp+gprsize+16*53], m1 ;in19 + mova [rsp+gprsize+16*55], m2 ;in21 + mova [rsp+gprsize+16*61], m3 ;in23 + mova [rsp+gprsize+16*59], m4 ;in25 + mova [rsp+gprsize+16*57], m5 ;in27 + mova [rsp+gprsize+16*51], m6 ;in29 + mova [rsp+gprsize+16*65], m7 ;in31 + + call m(idct_16x64_internal_8bpc).main + + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+32*0, 32 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+32*8, 32 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+32*16, 32 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end3: + SAVE_8ROWS coeffq+32*24, 32 + LOAD_8ROWS rsp+gprsize+16*35, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end4: + SAVE_8ROWS dstq+32*0, 32 + LOAD_8ROWS rsp+gprsize+16*43, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end5: + SAVE_8ROWS dstq+32*8, 32 + LOAD_8ROWS rsp+gprsize+16*51, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end6: + SAVE_8ROWS dstq+32*16, 32 + LOAD_8ROWS rsp+gprsize+16*59, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end7: + SAVE_8ROWS dstq+32*24, 32 + + add coeffq, 16 + add dstq, 16 + dec r3d + jg .pass1_loop + +.pass2: + mov dstq, [rsp+gprsize*2+16*67] + sub coeffq, 32 + mov r3d, 4 + +.pass2_loop: + mov [rsp+gprsize*1+16*67], r3d + + LOAD_4ROWS coeffq+16*0, 32*2 + LOAD_4ROWS_H coeffq+16*1, 32*2 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_4ROWS coeffq+16*2, 32*2 + LOAD_4ROWS_H coeffq+16*3, 32*2 + call m(idct_16x8_internal_8bpc).main + + mov r3, dstq + lea tx2q, [o(.end)] + lea dstq, [dstq+strideq*8] + jmp m(idct_8x8_internal_8bpc).end + +.end: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.end1)] + mov dstq, r3 + jmp m(idct_8x8_internal_8bpc).end + +.end1: + pxor m7, m7 + REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + + add coeffq, 16*16 + mov r3d, [rsp+gprsize*1+16*67] + mov dstq, [rsp+gprsize*2+16*67] + add dstq, 8 + mov [rsp+gprsize*2+16*67], dstq + dec r3d + jg .pass2_loop + + mov r3d, 4 + lea coeffq, [rsp+gprsize+16*68] +.pass2_loop2: + mov [rsp+gprsize*1+16*67], r3d + + LOAD_4ROWS coeffq+16*0, 32*2 + LOAD_4ROWS_H coeffq+16*1, 32*2 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_4ROWS coeffq+16*2, 32*2 + LOAD_4ROWS_H coeffq+16*3, 32*2 + call m(idct_16x8_internal_8bpc).main + + mov r3, dstq + lea tx2q, [o(.end2)] + lea dstq, [dstq+strideq*8] + jmp m(idct_8x8_internal_8bpc).end + +.end2: + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.end3)] + mov dstq, r3 + jmp m(idct_8x8_internal_8bpc).end + +.end3: + + add coeffq, 16*16 + mov r3d, [rsp+gprsize*1+16*67] + mov dstq, [rsp+gprsize*2+16*67] + add dstq, 8 + mov [rsp+gprsize*2+16*67], dstq + dec r3d + jg .pass2_loop2 + ret + + +cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_32x64_internal_8bpc) +.end: + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + mov [coeffq], eobd + pmulhrsw m0, m1 + mov r3d, 64 + lea tx2q, [o(.end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body + + +cglobal idct_32x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r4d, 2 + sub eobd, 136 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + mov [rsp+gprsize*2+16*67], coeffq + +.pass1_loop: + LOAD_8ROWS coeffq+64*1, 64*2, 1 + mova [rsp+gprsize+16*19], m0 ;in1 + mova [rsp+gprsize+16*26], m1 ;in3 + mova [rsp+gprsize+16*23], m2 ;in5 + mova [rsp+gprsize+16*22], m3 ;in7 + mova [rsp+gprsize+16*21], m4 ;in9 + mova [rsp+gprsize+16*24], m5 ;in11 + mova [rsp+gprsize+16*25], m6 ;in13 + mova [rsp+gprsize+16*20], m7 ;in15 + + mov tx2d, [rsp+gprsize*1+16*67] + test tx2d, tx2d + jl .fast + +.full: + LOAD_8ROWS coeffq+64*0, 64*4, 1 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_8ROWS coeffq+64*2, 64*4, 1 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+64*17, 64*2, 1 + mova [rsp+gprsize+16*33], m0 ;in17 + mova [rsp+gprsize+16*28], m1 ;in19 + mova [rsp+gprsize+16*29], m2 ;in21 + mova [rsp+gprsize+16*32], m3 ;in23 + mova [rsp+gprsize+16*31], m4 ;in25 + mova [rsp+gprsize+16*30], m5 ;in27 + mova [rsp+gprsize+16*27], m6 ;in29 + mova [rsp+gprsize+16*34], m7 ;in31 + + call m(idct_8x32_internal_8bpc).main + jmp .pass1_end + +.fast: + LOAD_4ROWS coeffq, 256, 1 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + + SAVE_7ROWS rsp+gprsize+16*3, 16 + LOAD_4ROWS coeffq+128*1, 256, 1 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + call m(idct_8x32_internal_8bpc).main_fast + +.pass1_end: + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end1: + SAVE_8ROWS coeffq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end2: + SAVE_8ROWS coeffq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end3: + SAVE_8ROWS coeffq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end4: + SAVE_8ROWS coeffq+64*24, 64 + + add coeffq, 16 + dec r3d + jg .pass1_loop + +.pass2: + mov coeffq, [rsp+gprsize*2+16*67] + mov r3d, 4 + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] + jmp m(idct_16x64_internal_8bpc).pass2_loop + + +cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + call m(idct_64x32_internal_8bpc) +.end: + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_16384)] + pmulhrsw m0, m1 + mov [coeffq], eobd + mov r3d, 32 + lea tx2q, [o(.end)] + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body + + +cglobal idct_64x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r4d, 2 + sub eobd, 136 + mov [rsp+gprsize*1+16*67], eobd + mov r3d, 4 + cmovs r3d, r4d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + mov [rsp+gprsize*2+16*67], coeffq + mov [rsp+gprsize*3+16*67], dstq + lea dstq, [rsp+gprsize+16*69] + mov [rsp+gprsize*4+16*67], dstq + +.pass1_loop: + LOAD_4ROWS coeffq+64*0, 64*8, 1 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + pxor m4, m4 + LOAD_4ROWS coeffq+64*4, 64*8, 1 + + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+64*2, 64*4, 1 + mova [rsp+gprsize+16*19], m0 + mova [rsp+gprsize+16*26], m1 + mova [rsp+gprsize+16*23], m2 + mova [rsp+gprsize+16*22], m3 + mova [rsp+gprsize+16*21], m4 + mova [rsp+gprsize+16*24], m5 + mova [rsp+gprsize+16*25], m6 + mova [rsp+gprsize+16*20], m7 + + call m(idct_8x32_internal_8bpc).main_fast + SAVE_8ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+64*1, 64*2, 1 + mova [rsp+gprsize+16*35], m0 ;in1 + mova [rsp+gprsize+16*49], m1 ;in3 + mova [rsp+gprsize+16*43], m2 ;in5 + mova [rsp+gprsize+16*41], m3 ;in7 + mova [rsp+gprsize+16*39], m4 ;in9 + mova [rsp+gprsize+16*45], m5 ;in11 + mova [rsp+gprsize+16*47], m6 ;in13 + mova [rsp+gprsize+16*37], m7 ;in15 + + LOAD_8ROWS coeffq+64*17, 64*2, 1 + mova [rsp+gprsize+16*63], m0 ;in17 + mova [rsp+gprsize+16*53], m1 ;in19 + mova [rsp+gprsize+16*55], m2 ;in21 + mova [rsp+gprsize+16*61], m3 ;in23 + mova [rsp+gprsize+16*59], m4 ;in25 + mova [rsp+gprsize+16*57], m5 ;in27 + mova [rsp+gprsize+16*51], m6 ;in29 + mova [rsp+gprsize+16*65], m7 ;in31 + + call m(idct_16x64_internal_8bpc).main + + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end: + SAVE_8ROWS coeffq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end1: + SAVE_8ROWS coeffq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end2: + SAVE_8ROWS coeffq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end3: + SAVE_8ROWS coeffq+64*24, 64 + LOAD_8ROWS rsp+gprsize+16*35, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end4: + SAVE_8ROWS dstq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*43, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end5: + SAVE_8ROWS dstq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*51, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end6: + SAVE_8ROWS dstq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*59, 16 + mova [rsp+gprsize+16*0], m7 + lea tx2q, [o(.pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end + +.pass1_end7: + SAVE_8ROWS dstq+64*24, 64 + + add coeffq, 16 + add dstq, 16 + dec r3d + jg .pass1_loop + +.pass2: + mov coeffq, [rsp+gprsize*4+16*67] + mov dstq, [rsp+gprsize*3+16*67] + mov eobd, [rsp+gprsize*1+16*67] + lea dstq, [dstq+32] + mov [rsp+gprsize*1+16*35], eobd + lea tx2q, [o(.pass2_end)] + mov r3d, 4 + jmp m(idct_32x32_internal_8bpc).pass2_loop + +.pass2_end: + mova [rsp+gprsize+16*0], m7 + lea r3, [o(.pass2_end1)] + jmp m(idct_8x32_internal_8bpc).end2 + +.pass2_end1: + lea tx2q, [o(.pass2_end)] + add coeffq, 16*32 + mov dstq, [rsp+gprsize*2+16*35] + mov r3d, [rsp+gprsize*3+16*35] + dec r3d + jg m(idct_32x32_internal_8bpc).pass2_loop + +.pass2_end2: + mov dstq, [rsp+gprsize*3+16*67] + mov coeffq, [rsp+gprsize*2+16*67] + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass2_end)] + mov r3d, 4 + jmp m(idct_32x32_internal_8bpc).pass2_loop + + +cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 +%if ARCH_X86_32 + LEA r5, $$ +%endif + test eobd, eobd + jz .dconly + + call m(idct_64x64_internal_8bpc) + RET + +.dconly: + movd m1, [o(pw_2896x8)] + pmulhrsw m0, m1, [coeffq] + movd m2, [o(pw_8192)] + mov [coeffq], eobd + mov r3d, 64 + lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body + +cglobal idct_64x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + mov r5d, 4 + mov r4d, 2 + sub eobd, 136 + cmovns r4d, r5d + +%if ARCH_X86_32 + LEA r5, $$ +%endif + + mov [rsp+gprsize*1+16*67], eobd + mov r3d, r4d + mov [rsp+gprsize*4+16*67], coeffq + mov [rsp+gprsize*3+16*67], dstq + lea dstq, [rsp+gprsize+16*69] + mov [rsp+gprsize*2+16*67], dstq + +.pass1_loop: + LOAD_4ROWS coeffq+64*0, 64*8 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_8bpc).main + SAVE_7ROWS rsp+gprsize+16*3, 16 + + pxor m4, m4 + LOAD_4ROWS coeffq+64*4, 64*8 + + REPX {mova x, m4}, m5, m6, m7 + call m(idct_16x8_internal_8bpc).main + mova m7, [rsp+gprsize+16*0] + SAVE_8ROWS rsp+gprsize+16*11, 16 + + LOAD_8ROWS coeffq+64*2, 64*4 + mova [rsp+gprsize+16*19], m0 + mova [rsp+gprsize+16*26], m1 + mova [rsp+gprsize+16*23], m2 + mova [rsp+gprsize+16*22], m3 + mova [rsp+gprsize+16*21], m4 + mova [rsp+gprsize+16*24], m5 + mova [rsp+gprsize+16*25], m6 + mova [rsp+gprsize+16*20], m7 + + call m(idct_8x32_internal_8bpc).main_fast + SAVE_8ROWS rsp+gprsize+16*3, 16 + + LOAD_8ROWS coeffq+64*1, 64*2 + mova [rsp+gprsize+16*35], m0 ;in1 + mova [rsp+gprsize+16*49], m1 ;in3 + mova [rsp+gprsize+16*43], m2 ;in5 + mova [rsp+gprsize+16*41], m3 ;in7 + mova [rsp+gprsize+16*39], m4 ;in9 + mova [rsp+gprsize+16*45], m5 ;in11 + mova [rsp+gprsize+16*47], m6 ;in13 + mova [rsp+gprsize+16*37], m7 ;in15 + + LOAD_8ROWS coeffq+64*17, 64*2 + mova [rsp+gprsize+16*63], m0 ;in17 + mova [rsp+gprsize+16*53], m1 ;in19 + mova [rsp+gprsize+16*55], m2 ;in21 + mova [rsp+gprsize+16*61], m3 ;in23 + mova [rsp+gprsize+16*59], m4 ;in25 + mova [rsp+gprsize+16*57], m5 ;in27 + mova [rsp+gprsize+16*51], m6 ;in29 + mova [rsp+gprsize+16*65], m7 ;in31 + + call m(idct_16x64_internal_8bpc).main + + LOAD_8ROWS rsp+gprsize+16*3, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end: + SAVE_8ROWS coeffq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*11, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end1: + SAVE_8ROWS coeffq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*19, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end2: + SAVE_8ROWS coeffq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*27, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end3: + SAVE_8ROWS coeffq+64*24, 64 + LOAD_8ROWS rsp+gprsize+16*35, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end4: + SAVE_8ROWS dstq+64*0, 64 + LOAD_8ROWS rsp+gprsize+16*43, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end5: + SAVE_8ROWS dstq+64*8, 64 + LOAD_8ROWS rsp+gprsize+16*51, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end6: + SAVE_8ROWS dstq+64*16, 64 + LOAD_8ROWS rsp+gprsize+16*59, 16 + mova [rsp+gprsize+16*0], m7 + mova m7, [o(pw_8192)] + lea tx2q, [o(.pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 + +.pass1_end7: + SAVE_8ROWS dstq+64*24, 64 + + add coeffq, 16 + add dstq, 16 + dec r3d + jg .pass1_loop + +.pass2: + mov dstq, [rsp+gprsize*3+16*67] + mov coeffq, [rsp+gprsize*2+16*67] + lea dstq, [dstq+32] + mov r3d, 4 + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(.pass2_end)] + jmp m(idct_16x64_internal_8bpc).pass2_loop + +.pass2_end: + LOAD_8ROWS rsp+gprsize+16*35, 16 + lea dstq, [dstq+strideq*2] + lea r3, [rsp+16*32+gprsize] + mova [rsp+gprsize+16*0], m7 + call m(idct_16x64_internal_8bpc).write + mov dstq, [rsp+gprsize*2+16*67] + mov r3d, [rsp+gprsize*3+16*67] + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(.pass2_end)] + + dec r3d + jg m(idct_16x64_internal_8bpc).pass2_loop + +.pass2_end2: + mov coeffq, [rsp+gprsize*4+16*67] + mov dstq, [rsp+gprsize*2+16*67] + mov r3d, 4 + sub dstq, 72 + lea r4, [dstq+8] + mov [rsp+gprsize*2+16*67], r4 + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] + jmp m(idct_16x64_internal_8bpc).pass2_loop diff --git a/third_party/dav1d/src/x86/loopfilter.h b/third_party/dav1d/src/x86/loopfilter.h new file mode 100644 index 0000000000..33c842a9ce --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter.h @@ -0,0 +1,66 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/loopfilter.h" + +#define decl_loopfilter_sb_fns(ext) \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_y, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_y, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_uv, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_uv, ext)) + +decl_loopfilter_sb_fns(ssse3); +decl_loopfilter_sb_fns(avx2); +decl_loopfilter_sb_fns(avx512icl); + +static ALWAYS_INLINE void loop_filter_dsp_init_x86(Dav1dLoopFilterDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, ssse3); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, ssse3); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, ssse3); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, ssse3); + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, avx2); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, avx2); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, avx2); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, avx2); + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, avx512icl); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, avx512icl); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, avx512icl); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, avx512icl); +#endif +} diff --git a/third_party/dav1d/src/x86/loopfilter16_avx2.asm b/third_party/dav1d/src/x86/loopfilter16_avx2.asm new file mode 100644 index 0000000000..ed83000ac2 --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter16_avx2.asm @@ -0,0 +1,1161 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +pb_mask: dd 1, 1, 2, 2, 4, 4, 8, 8 +pb_4x1_4x5_4x9_4x13: times 4 db 0, 1 + times 4 db 8, 9 + times 4 db 0, 1 + times 4 db 8, 9 + +pw_1: times 16 dw 1 +pw_2: times 16 dw 2 +pw_3: times 16 dw 3 +pw_4096: times 2 dw 4096 + +; 10bpc/12bpc: +pw_4: times 2 dw 4 + times 2 dw 16 +clip_max: times 2 dw 511 + times 2 dw 2047 +clip_min: times 2 dw -512 + times 2 dw -2048 + +SECTION .text + +; in: out: +; mm%1 a b c d a e i m +; mm%2 e f g h b f j n +; mm%3 i j k l -> c g k o +; mm%4 m n o p d h l p +%macro TRANSPOSE4X4W 5 + punpcklwd m%5, m%1, m%2 + punpckhwd m%1, m%2 + punpcklwd m%2, m%3, m%4 + punpckhwd m%3, m%4 + punpckldq m%4, m%5, m%2 + punpckhdq m%5, m%2 + punpckldq m%2, m%1, m%3 + punpckhdq m%1, m%3 + + SWAP %1, %4 + SWAP %2, %5, %3 +%endmacro + +; in: out: +; xmm%1 a b c d e f g h a i q y 6 E M U +; xmm%2 i j k l m n o p b j r z 7 F N V +; xmm%3 q r s t u v w x c k s 0 8 G O W +; xmm%4 y z 0 1 2 3 4 5 d l t 1 9 H P X +; xmm%5 6 7 8 9 A B C D -> e m u 2 A I Q Y +; xmm%6 E F G H I J K L f n v 3 B J R Z +; xmm%7 M N O P Q R S T g o w 4 C K S + +; xmm%8 U V W X Y Z + = h p x 5 D L T = +%macro TRANSPOSE8X8W 9 + ; xmm%1 a b c d e f g h a i q y b j r z + ; xmm%2 i j k l m n o p c k s 0 d l t 1 + ; xmm%3 q r s t u v w x -> e m u 2 f n v 3 + ; xmm%4 y z 0 1 2 3 4 5 g o w 4 h p x 5 + TRANSPOSE4X4W %1, %2, %3, %4, %9 + + ; xmm%5 6 7 8 9 A B C D 6 E M U 7 F N V + ; xmm%6 E F G H I J K L 8 G O W 9 H P X + ; xmm%7 M N O P Q R S T -> A I Q Y B J R Z + ; xmm%8 U V W X Y Z + = C K S + D L T = + TRANSPOSE4X4W %5, %6, %7, %8, %9 + + ; xmm%1 a i q y b j r z a i q y 6 E M U + ; xmm%2 c k s 0 d l t 1 b j r z 7 F N V + ; xmm%3 e m u 2 f n v 3 c k s 0 8 G O W + ; xmm%4 g o w 4 h p x 5 d l t 1 9 H P X + ; xmm%5 6 E M U 7 F N V -> e m u 2 A I Q Y + ; xmm%6 8 G O W 9 H P X f n v 3 B J R Z + ; xmm%7 A I Q Y B J R Z g o w 4 C K S + + ; xmm%8 C K S + D L T = h p x 5 D L T = + punpckhqdq m%9, m%1, m%5 + punpcklqdq m%1, m%5 + punpckhqdq m%5, m%2, m%6 + punpcklqdq m%2, m%6 + punpckhqdq m%6, m%3, m%7 + punpcklqdq m%3, m%7 + punpckhqdq m%7, m%4, m%8 + punpcklqdq m%4, m%8 + + SWAP %8, %7, %4, %5, %3, %2, %9 +%endmacro + +; transpose and write m3-6, everything else is scratch +%macro TRANSPOSE_8x4_AND_WRITE_4x16 0 + ; transpose 8x4 + punpcklwd m0, m3, m4 + punpckhwd m3, m4 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + punpckldq m6, m0, m4 + punpckhdq m0, m4 + punpckldq m4, m3, m5 + punpckhdq m3, m5 + + ; write out + movq [dstq+strideq*0-4], xm6 + movhps [dstq+strideq*1-4], xm6 + movq [dstq+strideq*2-4], xm0 + movhps [dstq+stride3q -4], xm0 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm4 + movhps [dstq+strideq*1-4], xm4 + movq [dstq+strideq*2-4], xm3 + movhps [dstq+stride3q -4], xm3 + lea dstq, [dstq+strideq*4] + + vextracti128 xm6, m6, 1 + vextracti128 xm0, m0, 1 + vextracti128 xm4, m4, 1 + vextracti128 xm3, m3, 1 + + movq [dstq+strideq*0-4], xm6 + movhps [dstq+strideq*1-4], xm6 + movq [dstq+strideq*2-4], xm0 + movhps [dstq+stride3q -4], xm0 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm4 + movhps [dstq+strideq*1-4], xm4 + movq [dstq+strideq*2-4], xm3 + movhps [dstq+stride3q -4], xm3 + lea dstq, [dstq+strideq*4] +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] + ; load data +%ifidn %2, v +%if %1 == 4 + lea tmpq, [dstq+mstrideq*2] + mova m3, [tmpq+strideq*0] ; p1 + mova m4, [tmpq+strideq*1] ; p0 + mova m5, [tmpq+strideq*2] ; q0 + mova m6, [tmpq+stride3q] ; q1 +%else + ; load 6-8 pixels, remainder (for wd=16) will be read inline + lea tmpq, [dstq+mstrideq*4] + ; we load p3 later + mova m13, [tmpq+strideq*1] + mova m3, [tmpq+strideq*2] + mova m4, [tmpq+stride3q] + mova m5, [dstq+strideq*0] + mova m6, [dstq+strideq*1] + mova m14, [dstq+strideq*2] +%if %1 != 6 + mova m15, [dstq+stride3q] +%endif +%endif +%else + ; load lines +%if %1 == 4 + movq xm3, [dstq+strideq*0-4] + movq xm4, [dstq+strideq*1-4] + movq xm5, [dstq+strideq*2-4] + movq xm6, [dstq+stride3q -4] + lea tmpq, [dstq+strideq*4] + movq xm11, [tmpq+strideq*0-4] + movq xm13, [tmpq+strideq*1-4] + movq xm14, [tmpq+strideq*2-4] + movq xm15, [tmpq+stride3q -4] + lea tmpq, [tmpq+strideq*4] + ; this overreads by 8 bytes but the buffers are padded + ; so that should be ok + vinserti128 m3, [tmpq+strideq*0-4], 1 + vinserti128 m4, [tmpq+strideq*1-4], 1 + vinserti128 m5, [tmpq+strideq*2-4], 1 + vinserti128 m6, [tmpq+stride3q -4], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m11, [tmpq+strideq*0-4], 1 + vinserti128 m13, [tmpq+strideq*1-4], 1 + vinserti128 m14, [tmpq+strideq*2-4], 1 + vinserti128 m15, [tmpq+stride3q -4], 1 + + ; transpose 4x8 + ; xm3: A-D0,A-D4 + ; xm4: A-D1,A-D5 + ; xm5: A-D2,A-D6 + ; xm6: A-D3,A-D7 + punpcklwd m7, m3, m4 + punpcklwd m3, m11, m13 + punpcklwd m4, m5, m6 + punpcklwd m5, m14, m15 + ; xm7: A0-1,B0-1,C0-1,D0-1 + ; xm3: A4-5,B4-5,C4-5,D4-5 + ; xm4: A2-3,B2-3,C2-3,D2-3 + ; xm5: A6-7,B6-7,C6-7,D6-7 + punpckldq m6, m7, m4 + punpckhdq m7, m4 + punpckldq m8, m3, m5 + punpckhdq m5, m3, m5 + ; xm6: A0-3,B0-3 + ; xm7: C0-3,D0-3 + ; xm8: A4-7,B4-7 + ; xm5: C4-7,D4-7 + punpcklqdq m3, m6, m8 + punpckhqdq m4, m6, m8 + punpckhqdq m6, m7, m5 + punpcklqdq m5, m7, m5 + ; xm3: A0-7 + ; xm4: B0-7 + ; xm5: C0-7 + ; xm6: D0-7 +%elif %1 == 6 || %1 == 8 + movu xm3, [dstq+strideq*0-8] + movu xm4, [dstq+strideq*1-8] + movu xm5, [dstq+strideq*2-8] + movu xm6, [dstq+stride3q -8] + lea tmpq, [dstq+strideq*4] + movu xm11, [tmpq+strideq*0-8] + movu xm13, [tmpq+strideq*1-8] + movu xm14, [tmpq+strideq*2-8] + movu xm15, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + vinserti128 m3, [tmpq+strideq*0-8], 1 + vinserti128 m4, [tmpq+strideq*1-8], 1 + vinserti128 m5, [tmpq+strideq*2-8], 1 + vinserti128 m6, [tmpq+stride3q -8], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m11, [tmpq+strideq*0-8], 1 + vinserti128 m13, [tmpq+strideq*1-8], 1 + vinserti128 m14, [tmpq+strideq*2-8], 1 + vinserti128 m15, [tmpq+stride3q -8], 1 + + ; transpose 8x16 + ; xm3: A-H0,A-H8 + ; xm4: A-H1,A-H9 + ; xm5: A-H2,A-H10 + ; xm6: A-H3,A-H11 + ; xm11: A-H4,A-H12 + ; xm13: A-H5,A-H13 + ; xm14: A-H6,A-H14 + ; xm15: A-H7,A-H15 + punpcklwd m7, m3, m4 + punpckhwd m3, m4 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + punpcklwd m6, m11, m13 + punpckhwd m11, m13 + punpcklwd m13, m14, m15 + punpckhwd m14, m15 + ; xm7: A0-1,B0-1,C0-1,D0-1 + ; xm3: E0-1,F0-1,G0-1,H0-1 + ; xm4: A2-3,B2-3,C2-3,D2-3 + ; xm5: E2-3,F2-3,G2-3,H2-3 + ; xm6: A4-5,B4-5,C4-5,D4-5 + ; xm11: E4-5,F4-5,G4-5,H4-5 + ; xm13: A6-7,B6-7,C6-7,D6-7 + ; xm14: E6-7,F6-7,G6-7,H6-7 + punpckldq m15, m7, m4 + punpckhdq m7, m4 + punpckldq m9, m3, m5 + punpckhdq m8, m3, m5 + punpckldq m3, m6, m13 + punpckhdq m6, m13 + punpckldq m10, m11, m14 + punpckhdq m11, m14 + ; xm15: A0-3,B0-3 + ; xm7: C0-3,D0-3 + ; xm9: E0-3,F0-3 + ; xm8: G0-3,H0-3 + ; xm3: A4-7,B4-7 + ; xm6: C4-7,D4-7 + ; xm10: E4-7,F4-7 + ; xm11: G4-7,H4-7 +%if %1 != 6 + punpcklqdq m0, m15, m3 +%endif + punpckhqdq m13, m15, m3 + punpcklqdq m3, m7, m6 + punpckhqdq m4, m7, m6 + punpcklqdq m5, m9, m10 + punpckhqdq m6, m9, m10 + punpcklqdq m14, m8, m11 +%if %1 != 6 + punpckhqdq m15, m8, m11 + mova [rsp+5*32], m0 +%endif +%else + ; We only use 14 pixels but we'll need the remainder at the end for + ; the second transpose + mova xm0, [dstq+strideq*0-16] + mova xm1, [dstq+strideq*1-16] + mova xm2, [dstq+strideq*2-16] + mova xm3, [dstq+stride3q -16] + lea tmpq, [dstq+strideq*4] + mova xm4, [tmpq+strideq*0-16] + mova xm5, [tmpq+strideq*1-16] + mova xm6, [tmpq+strideq*2-16] + mova xm7, [tmpq+stride3q -16] + lea tmpq, [tmpq+strideq*4] + vinserti128 m0, m0, [tmpq+strideq*0-16], 1 + vinserti128 m1, m1, [tmpq+strideq*1-16], 1 + vinserti128 m2, m2, [tmpq+strideq*2-16], 1 + vinserti128 m3, m3, [tmpq+stride3q -16], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m4, m4, [tmpq+strideq*0-16], 1 + vinserti128 m5, m5, [tmpq+strideq*1-16], 1 + vinserti128 m6, m6, [tmpq+strideq*2-16], 1 + vinserti128 m7, m7, [tmpq+stride3q -16], 1 + + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, 8 + + mova [rsp+6*32], m0 + mova [rsp+7*32], m1 + mova [rsp+8*32], m2 + mova [rsp+9*32], m3 + mova [rsp+5*32], m4 + + mova xm0, [dstq+strideq*0] + mova xm1, [dstq+strideq*1] + mova xm2, [dstq+strideq*2] + mova xm3, [dstq+stride3q ] + lea tmpq, [dstq+strideq*4] + mova xm8, [tmpq+strideq*0] + mova xm9, [tmpq+strideq*1] + mova xm10, [tmpq+strideq*2] + mova xm11, [tmpq+stride3q ] + lea tmpq, [tmpq+strideq*4] + vinserti128 m0, m0, [tmpq+strideq*0], 1 + vinserti128 m1, m1, [tmpq+strideq*1], 1 + vinserti128 m2, m2, [tmpq+strideq*2], 1 + vinserti128 m3, m3, [tmpq+stride3q ], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m8, m8, [tmpq+strideq*0], 1 + vinserti128 m9, m9, [tmpq+strideq*1], 1 + vinserti128 m10, m10, [tmpq+strideq*2], 1 + vinserti128 m11, m11, [tmpq+stride3q ], 1 + + TRANSPOSE8X8W 0, 1, 2, 3, 8, 9, 10, 11, 4 + + mova [rsp+10*32], m8 + mova [rsp+11*32], m9 + mova [rsp+12*32], m10 + mova [rsp+13*32], m11 + + ; 5,6,7,0,1,2,3 -> 13,3,4,5,6,14,15 + SWAP 13, 5, 0 + SWAP 3, 6, 1, 15 + SWAP 4, 7 + SWAP 2, 14 +%endif +%endif + + ; load L/E/I/H +%ifidn %2, v + pmovzxbw m1, [lq] + pmovzxbw m0, [lq+l_strideq] + pxor m2, m2 +%else + vpbroadcastq m0, [lq] ; l0, l1 + vpbroadcastq m1, [lq+l_strideq] ; l2, l3 + vpbroadcastq m2, [lq+l_strideq*2] ; l4, l5 + vpbroadcastq m10, [lq+l_stride3q] ; l6, l7 + punpckldq m0, m1 ; l0, l2, l1, l3 [2x] + punpckldq m2, m10 ; l4, l6, l5, l7 [2x] + vpblendd m0, m0, m2, 11110000b ; l0, l2, l1, l3, l4, l6, l5, l7 + pxor m2, m2 + punpcklbw m1, m0, m2 ; l0, l2, l4, l6 + punpckhbw m0, m2 ; l1, l3, l5, l7 +%endif + pcmpeqw m10, m2, m0 + pand m1, m10 + por m0, m1 ; l[x][] ? l[x][] : l[x-stride][] + pshufb m0, [pb_4x1_4x5_4x9_4x13] ; l[x][1] + pcmpeqw m10, m2, m0 ; !L + psrlw m10, 1 + psrlw m2, m0, [lutq+128] + vpbroadcastw m1, [lutq+136] + pminuw m2, m1 + pmaxuw m2, [pw_1] ; I + psrlw m1, m0, 4 ; H + paddw m0, [pw_2] + vpbroadcastd m8, [r11] + paddw m0, m0 + paddw m0, m2 ; E + REPX {pmullw x, m8}, m0, m1, m2 + + psubw m8, m3, m4 ; p1-p0 + psubw m9, m5, m6 ; q1-q0 + REPX {pabsw x, x}, m8, m9 + pmaxuw m8, m10 + pmaxuw m8, m9 + pcmpgtw m7, m8, m1 ; hev +%if %1 != 4 + psubw m9, m13, m4 ; p2-p0 + pabsw m9, m9 + pmaxuw m9, m8 +%if %1 != 6 +%ifidn %2, v + mova m11, [tmpq+strideq*0] ; p3 +%else + mova m11, [rsp+5*32] ; p3 +%endif + psubw m10, m11, m4 ; p3-p0 + pabsw m10, m10 + pmaxuw m9, m10 +%endif + psubw m10, m5, m14 ; q2-q0 + pabsw m10, m10 + pmaxuw m9, m10 +%if %1 != 6 + psubw m10, m5, m15 ; q3-q0 + pabsw m10, m10 + pmaxuw m9, m10 +%endif + vpbroadcastd m10, [r11] + pcmpgtw m9, m10 ; !flat8in + + psubw m10, m13, m3 ; p2-p1 + pabsw m10, m10 +%if %1 != 6 + psubw m11, m13 ; p3-p2 + pabsw m11, m11 + pmaxuw m10, m11 + psubw m11, m14, m15 ; q3-q2 + pabsw m11, m11 + pmaxuw m10, m11 +%endif + psubw m11, m14, m6 ; q2-q1 + pabsw m11, m11 + pmaxuw m10, m11 + +%if %1 == 16 + vpbroadcastd m11, [maskq+8] + vpbroadcastd m1, [maskq+4] + por m11, m1 + pand m11, m12 + pcmpeqd m11, m12 + pand m10, m11 +%else + vpbroadcastd m11, [maskq+4] + pand m11, m12 + pcmpeqd m11, m12 + pand m10, m11 ; only apply fm-wide to wd>4 blocks +%endif + pmaxuw m8, m10 +%endif + pcmpgtw m8, m2 + + psubw m10, m3, m6 ; p1-q1 + psubw m11, m4, m5 ; p0-q0 + REPX {pabsw x, x}, m10, m11 + paddw m11, m11 + psrlw m10, 1 + paddw m10, m11 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + pcmpgtw m10, m0 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E + por m8, m10 + +%if %1 == 16 + +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] + mova m1, [tmpq+strideq*2] + mova m2, [tmpq+stride3q] +%else + mova m0, [rsp+7*32] + mova m1, [rsp+8*32] + mova m2, [rsp+9*32] +%endif + REPX {psubw x, m4}, m0, m1, m2 + REPX {pabsw x, x}, m0, m1, m2 + pmaxuw m1, m0 + pmaxuw m1, m2 +%ifidn %2, v + lea tmpq, [dstq+strideq*4] + mova m0, [tmpq+strideq*0] + mova m2, [tmpq+strideq*1] + mova m10, [tmpq+strideq*2] +%else + mova m0, [rsp+10*32] + mova m2, [rsp+11*32] + mova m10, [rsp+12*32] +%endif + REPX {psubw x, m5}, m0, m2, m10 + REPX {pabsw x, x}, m0, m2, m10 + pmaxuw m0, m2 + pmaxuw m1, m10 + pmaxuw m1, m0 + vpbroadcastd m0, [r11] + pcmpgtw m1, m0 ; !flat8out + por m1, m9 ; !flat8in | !flat8out + vpbroadcastd m2, [maskq+8] + pand m10, m2, m12 + pcmpeqd m10, m12 + pandn m1, m10 ; flat16 + pandn m1, m8, m1 ; flat16 & fm + + vpbroadcastd m10, [maskq+4] + por m10, m2 + pand m2, m10, m12 + pcmpeqd m2, m12 + pandn m9, m2 ; flat8in + pandn m9, m8, m9 + vpbroadcastd m2, [maskq+0] + por m2, m10 + pand m2, m12 + pcmpeqd m2, m12 + pandn m8, m2 + pandn m8, m9, m8 ; fm & !flat8 & !flat16 + pandn m9, m1, m9 ; flat8 & !flat16 +%elif %1 != 4 + vpbroadcastd m0, [maskq+4] + pand m2, m0, m12 + pcmpeqd m2, m12 + pandn m9, m2 + pandn m9, m8, m9 ; flat8 & fm + vpbroadcastd m2, [maskq+0] + por m0, m2 + pand m0, m12 + pcmpeqd m0, m12 + pandn m8, m0 + pandn m8, m9, m8 ; fm & !flat8 +%else + vpbroadcastd m0, [maskq+0] + pand m0, m12 + pcmpeqd m0, m12 + pandn m8, m0 ; fm +%endif + + ; short filter + vpbroadcastd m0, [r11+8*1] ; 511 or 2047 + vpbroadcastd m2, [r11+8*2] ; -512 or -2048 + psubw m10, m5, m4 + paddw m11, m10, m10 + paddw m11, m10 + psubw m10, m3, m6 ; iclip_diff(p1-q1) + pminsw m10, m0 + pmaxsw m10, m2 + pand m10, m7 ; f=iclip_diff(p1-q1)&hev + paddw m10, m11 ; f=iclip_diff(3*(q0-p0)+f) + pminsw m10, m0 + pmaxsw m10, m2 + pand m8, m10 ; f&=fm + vpbroadcastd m10, [pw_4] + paddw m10, m8 + paddw m8, [pw_3] + REPX {pminsw x, m0}, m10, m8 + psraw m10, 3 ; f2 + psraw m8, 3 ; f1 + psubw m5, m10 + paddw m4, m8 + + paddw m10, [pw_1] + psraw m10, 1 ; f=(f1+1)>>1 + pandn m8, m7, m10 ; f&=!hev + paddw m3, m8 + psubw m6, m8 + pxor m8, m8 + psubw m0, m2 ; 1023 or 4095 + REPX {pminsw x, m0}, m3, m4, m5, m6 + REPX {pmaxsw x, m8}, m3, m4, m5, m6 + +%if %1 == 16 + +; m3-6 = p1/p0/q0/q1, m9=flat8, m1=flat16 +; m12=filter bits mask +; m13-15=p2/q2/q3 +; m0,2,7-8,10-11 = free + + ; flat16 filter +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] ; p6 + mova m2, [tmpq+strideq*2] ; p5 + mova m7, [tmpq+stride3q] ; p4 + mova m11, [tmpq+strideq*4] ; p3 +%else + mova m0, [rsp+7*32] + mova m2, [rsp+8*32] + mova m7, [rsp+9*32] + mova m11, [rsp+5*32] +%endif + + mova [rsp+ 0*32], m9 + + ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + paddw m8, m0, [pw_1] + psllw m8, 3 ; p6*8+8 + paddw m10, m2, m7 ; p5+p4 + psubw m8, m0 + paddw m10, m10 ; (p5+p4)*2 + paddw m8, m11 ; p6*7+p3 + paddw m10, m13 ; (p5+p4)*2+p2 + paddw m8, m3 ; p6*7+p3+p1 + paddw m10, m4 ; (p5+p4)*2+p2+p0 + paddw m8, m5 ; p6*7+p3+p1+q0 + paddw m8, m10 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + psrlw m10, m8, 4 + vpblendvb m10, m2, m10, m1 +%ifidn %2, v + mova [tmpq+strideq*2], m10 ; p5 +%else + mova [rsp+8*32], m10 +%endif + + ; sub p6*2, add p3/q1 + paddw m8, m11 + paddw m10, m0, m0 + paddw m8, m6 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m7, m10, m1 +%ifidn %2, v + mova [tmpq+stride3q], m10 ; p4 +%else + mova [rsp+9*32], m10 +%endif + + ; sub p6/p5, add p2/q2 + psubw m8, m0 + paddw m10, m13, m14 + psubw m8, m2 + paddw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m11, m10, m1 +%ifidn %2, v + mova [tmpq+strideq*4], m10 ; p3 + lea tmpq, [dstq+strideq*4] +%else + mova [rsp+5*32], m10 +%endif + + ; sub p6/p4, add p1/q3 + paddw m8, m3 + paddw m10, m0, m7 + paddw m8, m15 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m13, m10, m1 + mova [rsp+1*32], m10 ; don't clobber p2/m13 + + ; sub p6/p3, add p0/q4 + paddw m8, m4 + paddw m10, m0, m11 +%ifidn %2, v + paddw m8, [tmpq+strideq*0] +%else + paddw m8, [rsp+10*32] +%endif + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m3, m10, m1 + mova [rsp+2*32], m10 ; don't clobber p1/m3 + + ; sub p6/p2, add q0/q5 + paddw m8, m5 + paddw m10, m0, m13 +%ifidn %2, v + paddw m8, [tmpq+strideq*1] +%else + paddw m8, [rsp+11*32] +%endif + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m4, m10, m1 + mova [rsp+3*32], m10 ; don't clobber p0/m4 + + ; sub p6/p1, add q1/q6 + paddw m8, m6 + paddw m10, m0, m3 +%ifidn %2, v + mova m0, [tmpq+strideq*2] ; q6 +%else + mova m0, [rsp+12*32] ; q6 +%endif + paddw m8, m0 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m5, m10, m1 + mova [rsp+4*32], m10 ; don't clobber q0/m5 + + ; sub p5/p0, add q2/q6 + paddw m8, m14 + paddw m10, m2, m4 + paddw m8, m0 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m2, m6, m10, m1 ; don't clobber q1/m6 + + ; sub p4/q0, add q3/q6 + paddw m8, m15 + paddw m10, m7, m5 + paddw m8, m0 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m7, m14, m10, m1 ; don't clobber q2/m14 + + ; sub p3/q1, add q4/q6 +%ifidn %2, v + paddw m8, [tmpq+strideq*0] +%else + paddw m8, [rsp+10*32] +%endif + paddw m10, m11, m6 + paddw m8, m0 + psubw m8, m10 + psrlw m10, m8, 4 + vpblendvb m10, m15, m10, m1 +%ifidn %2, v + mova [tmpq+mstrideq], m10 ; q3 +%else + mova [rsp+14*32], m10 +%endif + + ; sub p2/q2, add q5/q6 +%ifidn %2, v + paddw m8, [tmpq+strideq*1] +%else + paddw m8, [rsp+11*32] +%endif + paddw m10, m13, m14 + paddw m8, m0 + psubw m8, m10 + psrlw m10, m8, 4 +%ifidn %2, v + mova m9, [tmpq+strideq*0] +%else + mova m9, [rsp+10*32] +%endif + vpblendvb m10, m9, m10, m1 +%ifidn %2, v + mova [tmpq+strideq*0], m10 ; q4 +%else + mova [rsp+10*32], m10 +%endif + + ; sub p1/q3, add q6*2 + psubw m8, m3 + paddw m0, m0 + psubw m8, m15 + paddw m8, m0 + psrlw m10, m8, 4 +%ifidn %2, v + mova m9, [tmpq+strideq*1] +%else + mova m9, [rsp+11*32] +%endif + vpblendvb m10, m9, m10, m1 +%ifidn %2, v + mova [tmpq+strideq*1], m10 ; q5 +%else + mova [rsp+11*32], m10 +%endif + + mova m9, [rsp+0*32] + mova m13, [rsp+1*32] + mova m3, [rsp+2*32] + mova m4, [rsp+3*32] + mova m5, [rsp+4*32] + SWAP 2, 6 + SWAP 7, 14 +%ifidn %2, v + lea tmpq, [dstq+mstrideq*4] +%else + mova m15, [rsp+14*32] +%endif +%endif + +%if %1 >= 8 + ; flat8 filter + vpbroadcastd m7, [pw_4096] +%ifidn %2, v + mova m0, [tmpq+strideq*0] ; p3 +%else + mova m0, [rsp+5*32] ; p3 +%endif + paddw m1, m0, m13 ; p3+p2 + paddw m2, m3, m4 ; p1+p0 + paddw m8, m1, m1 ; 2*(p3+p2) + paddw m2, m0 ; p1+p0+p3 + paddw m8, m5 ; 2*(p3+p2)+q0 + paddw m2, m8 ; 3*p3+2*p2+p1+p0+q0 + pmulhrsw m10, m2, m7 + + paddw m8, m3, m6 + psubw m2, m1 + paddw m2, m8 + pmulhrsw m8, m2, m7 + + paddw m11, m0, m3 + paddw m1, m4, m14 + psubw m2, m11 + paddw m2, m1 + pmulhrsw m1, m2, m7 + + paddw m11, m0, m4 + pblendvb m4, m1, m9 + paddw m1, m5, m15 + psubw m2, m11 + paddw m2, m1 + pmulhrsw m11, m2, m7 + + paddw m2, m6 + paddw m2, m15 + paddw m1, m13, m5 + pblendvb m5, m11, m9 + pblendvb m13, m10, m9 + psubw m2, m1 + pmulhrsw m1, m2, m7 + + psubw m2, m3 + pblendvb m3, m8, m9 + psubw m2, m6 + pblendvb m6, m1, m9 + paddw m1, m15, m14 + paddw m2, m1 + pmulhrsw m2, m7 + + pblendvb m14, m2, m9 + +%ifidn %2, v + mova [tmpq+strideq*1], m13 ; p2 + mova [tmpq+strideq*2], m3 ; p1 + mova [tmpq+stride3q ], m4 ; p0 + mova [dstq+strideq*0], m5 ; q0 + mova [dstq+strideq*1], m6 ; q1 + mova [dstq+strideq*2], m14 ; q2 +%elif %1 == 8 + TRANSPOSE8X8W 0, 13, 3, 4, 5, 6, 14, 15, 1 + + ; write 8x16 + movu [dstq+strideq*0-8], xm0 + movu [dstq+strideq*1-8], xm13 + movu [dstq+strideq*2-8], xm3 + movu [dstq+stride3q -8], xm4 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm5 + movu [dstq+strideq*1-8], xm6 + movu [dstq+strideq*2-8], xm14 + movu [dstq+stride3q -8], xm15 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m0, 1 + vextracti128 [dstq+strideq*1-8], m13, 1 + vextracti128 [dstq+strideq*2-8], m3, 1 + vextracti128 [dstq+stride3q -8], m4, 1 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m5, 1 + vextracti128 [dstq+strideq*1-8], m6, 1 + vextracti128 [dstq+strideq*2-8], m14, 1 + vextracti128 [dstq+stride3q -8], m15, 1 + lea dstq, [dstq+strideq*4] +%else + mova m8, [rsp+6*32] + mova m1, [rsp+7*32] + mova m2, [rsp+8*32] + mova m7, [rsp+9*32] + TRANSPOSE8X8W 8, 1, 2, 7, 0, 13, 3, 4, 9 + + mova [dstq+strideq*0-16], xm8 + mova [dstq+strideq*1-16], xm1 + mova [dstq+strideq*2-16], xm2 + mova [dstq+stride3q -16], xm7 + lea tmpq, [dstq+strideq*4] + mova [tmpq+strideq*0-16], xm0 + mova [tmpq+strideq*1-16], xm13 + mova [tmpq+strideq*2-16], xm3 + mova [tmpq+stride3q -16], xm4 + lea tmpq, [tmpq+strideq*4] + vextracti128 [tmpq+strideq*0-16], m8, 1 + vextracti128 [tmpq+strideq*1-16], m1, 1 + vextracti128 [tmpq+strideq*2-16], m2, 1 + vextracti128 [tmpq+stride3q -16], m7, 1 + lea tmpq, [tmpq+strideq*4] + vextracti128 [tmpq+strideq*0-16], m0, 1 + vextracti128 [tmpq+strideq*1-16], m13, 1 + vextracti128 [tmpq+strideq*2-16], m3, 1 + vextracti128 [tmpq+stride3q -16], m4, 1 + + mova m0, [rsp+10*32] + mova m1, [rsp+11*32] + mova m2, [rsp+12*32] + mova m3, [rsp+13*32] + TRANSPOSE8X8W 5, 6, 14, 15, 0, 1, 2, 3, 4 + mova [dstq+strideq*0], xm5 + mova [dstq+strideq*1], xm6 + mova [dstq+strideq*2], xm14 + mova [dstq+stride3q ], xm15 + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + mova [dstq+strideq*2], xm2 + mova [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0], m5, 1 + vextracti128 [dstq+strideq*1], m6, 1 + vextracti128 [dstq+strideq*2], m14, 1 + vextracti128 [dstq+stride3q ], m15, 1 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0], m0, 1 + vextracti128 [dstq+strideq*1], m1, 1 + vextracti128 [dstq+strideq*2], m2, 1 + vextracti128 [dstq+stride3q ], m3, 1 + lea dstq, [dstq+strideq*4] +%endif +%elif %1 == 6 + ; flat6 filter + vpbroadcastd m7, [pw_4096] + paddw m8, m3, m4 + paddw m8, m13 ; p2+p1+p0 + paddw m11, m13, m5 + paddw m8, m8 + paddw m8, m11 ; p2+2*(p2+p1+p0)+q0 + pmulhrsw m2, m8, m7 + + paddw m8, m5 + paddw m11, m13, m13 + paddw m8, m6 + psubw m8, m11 + pmulhrsw m10, m8, m7 + + paddw m8, m6 + paddw m11, m13, m3 + paddw m8, m14 + psubw m8, m11 + pmulhrsw m11, m8, m7 + + psubw m8, m3 + paddw m14, m14 + psubw m8, m4 + paddw m8, m14 + pmulhrsw m8, m7 + + pblendvb m3, m2, m9 + pblendvb m4, m10, m9 + pblendvb m5, m11, m9 + pblendvb m6, m8, m9 + +%ifidn %2, v + mova [tmpq+strideq*2], m3 ; p1 + mova [tmpq+stride3q ], m4 ; p0 + mova [dstq+strideq*0], m5 ; q0 + mova [dstq+strideq*1], m6 ; q1 +%else + TRANSPOSE_8x4_AND_WRITE_4x16 +%endif +%else +%ifidn %2, v + mova [tmpq+strideq*0], m3 ; p1 + mova [tmpq+strideq*1], m4 ; p0 + mova [tmpq+strideq*2], m5 ; q0 + mova [tmpq+stride3q ], m6 ; q1 +%else + TRANSPOSE_8x4_AND_WRITE_4x16 +%endif +%endif +%endmacro + +INIT_YMM avx2 +cglobal lpf_v_sb_y_16bpc, 6, 12, 16, 32 * 5, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits + mov r6d, r7m + lea r11, [pw_4] + shr r6d, 11 ; is_12bpc + lea r11, [r11+r6*4] + mov wd, wm + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + mov mask_bitsd, 0xf + mova m12, [pb_mask] + +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + jz .no_flat16 + + FILTER 16, v + jmp .end + +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + + FILTER 8, v + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .end + + call .v4 + +.end: + pslld m12, 4 + add lq, 16 + add dstq, 32 + shl mask_bitsd, 4 + sub wd, 4 + jg .loop + RET +ALIGN function_align +.v4: + FILTER 4, v + ret + +INIT_YMM avx2 +cglobal lpf_h_sb_y_16bpc, 6, 12, 16, 32 * 15, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp, mask_bits + mov r6d, r7m + lea r11, [pw_4] + shr r6d, 11 ; is_12bpc + lea r11, [r11+r6*4] + mov hd, hm + shl l_strideq, 2 + sub lq, 4 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] + mov mask_bitsd, 0xf + mova m12, [pb_mask] + +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + jz .no_flat16 + + FILTER 16, h + jmp .end + +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + + FILTER 8, h + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .no_filter + + call .h4 + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] + lea dstq, [dstq+strideq*8] +.end: + pslld m12, 4 + lea lq, [lq+l_strideq*4] + shl mask_bitsd, 4 + sub hd, 4 + jg .loop + RET +ALIGN function_align +.h4: + FILTER 4, h + ret + +INIT_YMM avx2 +cglobal lpf_v_sb_uv_16bpc, 6, 12, 16, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits + mov r6d, r7m + lea r11, [pw_4] + shr r6d, 11 ; is_12bpc + lea r11, [r11+r6*4] + mov wd, wm + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + mov mask_bitsd, 0xf + mova m12, [pb_mask] + +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + + FILTER 6, v + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .end + + call mangle(private_prefix %+ _lpf_v_sb_y_16bpc_avx2).v4 + +.end: + pslld m12, 4 + add lq, 16 + add dstq, 32 + shl mask_bitsd, 4 + sub wd, 4 + jg .loop + RET + +INIT_YMM avx2 +cglobal lpf_h_sb_uv_16bpc, 6, 12, 16, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp, mask_bits + mov r6d, r7m + lea r11, [pw_4] + shr r6d, 11 ; is_12bpc + lea r11, [r11+r6*4] + mov hd, hm + shl l_strideq, 2 + sub lq, 4 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] + mov mask_bitsd, 0xf + mova m12, [pb_mask] + +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + + FILTER 6, h + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .no_filter + + call mangle(private_prefix %+ _lpf_h_sb_y_16bpc_avx2).h4 + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] + lea dstq, [dstq+strideq*8] +.end: + pslld m12, 4 + lea lq, [lq+l_strideq*4] + shl mask_bitsd, 4 + sub hd, 4 + jg .loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/loopfilter16_avx512.asm b/third_party/dav1d/src/x86/loopfilter16_avx512.asm new file mode 100644 index 0000000000..b7bc3aa106 --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter16_avx512.asm @@ -0,0 +1,912 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +l_shuf_v: times 2 db 0, 32 +pw_1: times 2 dw 1 + times 2 db 4, 36 +pw_3: times 2 dw 3 + times 2 db 8, 40 +pw_4: times 2 dw 4 + times 2 db 12, 44 +pw_16: times 2 dw 16 + times 2 db 16, 48 +pw_4096: times 2 dw 4096 + times 2 db 20, 52 +pw_16384: times 2 dw 16384 + times 2 db 24, 56 +pw_32767: times 2 dw 32767 + times 2 db 28, 60 + times 2 dw 0 +filter_mask: dd 1, 2, 4, 8, 16, 32, 64,128 +stride_mul: dd 0, 1, 8, 9, 16, 17, 24, 25 +l_shuf_h: db 4, -1, 4, -1, 4, -1, 4, -1, 12, -1, 12, -1, 12, -1, 12, -1 +clip_max: dw 511, 511, 2047, 2047 +clip_min: dw -512, -512, -2048, -2048 + +SECTION .text + +%macro TRANSPOSE8X8W 9 ; src/dst[1-8], tmp + punpckhwd m%9, m%5, m%6 + punpcklwd m%5, m%6 + punpckhwd m%6, m%1, m%2 + punpcklwd m%1, m%2 + punpckhwd m%2, m%7, m%8 + punpcklwd m%7, m%8 + punpckhwd m%8, m%3, m%4 + punpcklwd m%3, m%4 + punpckhdq m%4, m%1, m%3 + punpckldq m%1, m%3 + punpckldq m%3, m%5, m%7 + punpckhdq m%5, m%7 + punpckhdq m%7, m%6, m%8 + punpckldq m%6, m%8 + punpckldq m%8, m%9, m%2 + punpckhdq m%9, m%2 + punpckhqdq m%2, m%1, m%3 + punpcklqdq m%1, m%3 + punpcklqdq m%3, m%4, m%5 + punpckhqdq m%4, m%5 + punpcklqdq m%5, m%6, m%8 + punpckhqdq m%6, m%8 + punpckhqdq m%8, m%7, m%9 + punpcklqdq m%7, m%9 +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] +%ifidn %2, v +%if %1 == 16 + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1 ] + mova m1, [tmpq+strideq*2 ] ; p5 + mova m2, [tmpq+stride3q ] ; p4 + mova m3, [tmpq+strideq*4 ] ; p3 + mova m4, [tmpq+stride5q ] ; p2 +%elif %1 == 6 || %1 == 8 + lea tmpq, [dstq+mstrideq*4] +%if %1 == 8 + mova m3, [tmpq+strideq*0 ] +%endif + mova m4, [tmpq+strideq*1 ] +%endif + mova m5, [dstq+mstrideq*2] ; p1 + mova m6, [dstq+mstrideq*1] ; p0 + mova m7, [dstq+strideq*0 ] ; q0 + mova m8, [dstq+strideq*1 ] ; q1 +%if %1 != 4 + mova m9, [dstq+strideq*2 ] ; q2 +%endif +%if %1 == 8 || %1 == 16 + mova m10, [dstq+stride3q ] ; q3 +%endif +%if %1 == 16 + mova m11, [dstq+strideq*4 ] ; q4 + mova m22, [dstq+stride5q ] ; q5 + mova m23, [dstq+stride3q*2] +%endif +%else ; h +%if %1 == 16 + movu ym16, [dstq+strideq*0 -16] + movu ym17, [dstq+strideq*1 -16] + movu ym18, [dstq+strideq*2 -16] + movu ym19, [dstq+stride3q -16] + movu ym20, [dstq+strideq*4 -16] + movu ym22, [dstq+stride5q -16] + movu ym23, [dstq+stride3q*2-16] + movu ym28, [dstq+stride7q -16] + lea tmpq, [dstq+strideq*8 -16] + vinserti32x8 m7, m16, [tmpq+strideq*0 ], 1 + vinserti32x8 m8, m17, [tmpq+strideq*1 ], 1 + vinserti32x8 m9, m18, [tmpq+strideq*2 ], 1 + vinserti32x8 m10, m19, [tmpq+stride3q ], 1 + vinserti32x8 m11, m20, [tmpq+strideq*4 ], 1 + vinserti32x8 m22, m22, [tmpq+stride5q ], 1 + vinserti32x8 m23, m23, [tmpq+stride3q*2], 1 + vinserti32x8 m28, m28, [tmpq+stride7q ], 1 + lea tmpq, [tmpq+strideq*8] + TRANSPOSE8X8W 7, 8, 9, 10, 11, 22, 23, 28, 27 + movu ym16, [tmpq+strideq*0 ] + movu ym17, [tmpq+strideq*1 ] + movu ym18, [tmpq+strideq*2 ] + movu ym19, [tmpq+stride3q ] + movu ym24, [tmpq+strideq*4 ] + movu ym25, [tmpq+stride5q ] + movu ym26, [tmpq+stride3q*2] + movu ym20, [tmpq+stride7q ] + lea tmpq, [tmpq+strideq*8] + vinserti32x8 m0, m16, [tmpq+strideq*0 ], 1 + vinserti32x8 m1, m17, [tmpq+strideq*1 ], 1 + vinserti32x8 m2, m18, [tmpq+strideq*2 ], 1 + vinserti32x8 m3, m19, [tmpq+stride3q ], 1 + vinserti32x8 m4, m24, [tmpq+strideq*4 ], 1 + vinserti32x8 m5, m25, [tmpq+stride5q ], 1 + vinserti32x8 m6, m26, [tmpq+stride3q*2], 1 + vinserti32x8 m20, m20, [tmpq+stride7q ], 1 + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 20, 27 + vshufi32x4 m27, m7, m0, q2020 + vshufi32x4 m7, m0, q3131 + vshufi32x4 m0, m8, m1, q2020 + vshufi32x4 m8, m1, q3131 + vshufi32x4 m1, m9, m2, q2020 + vshufi32x4 m9, m2, q3131 + vshufi32x4 m2, m10, m3, q2020 + vshufi32x4 m10, m3, q3131 + vshufi32x4 m3, m11, m4, q2020 + vshufi32x4 m11, m4, q3131 + vshufi32x4 m4, m22, m5, q2020 + vshufi32x4 m22, m5, q3131 + vshufi32x4 m5, m23, m6, q2020 + vshufi32x4 m23, m6, q3131 + vshufi32x4 m6, m28, m20, q2020 + vshufi32x4 m28, m20, q3131 +%elif %1 == 6 || %1 == 8 +%if %1 == 8 + sub dstq, 8 + movu xm16, [dstq+strideq*0 ] + movu xm17, [dstq+strideq*1 ] + movu xm18, [dstq+strideq*2 ] + movu xm19, [dstq+stride3q ] + movu xm24, [dstq+strideq*4 ] + movu xm25, [dstq+stride5q ] + movu xm26, [dstq+stride3q*2] + movu xm27, [dstq+stride7q ] + lea tmpq, [dstq+strideq*8 ] + vinserti128 ym16, [tmpq+strideq*0 ], 1 + vinserti128 ym17, [tmpq+strideq*1 ], 1 + vinserti128 ym18, [tmpq+strideq*2 ], 1 + vinserti128 ym19, [tmpq+stride3q ], 1 + vinserti128 ym24, [tmpq+strideq*4 ], 1 + vinserti128 ym25, [tmpq+stride5q ], 1 + vinserti128 ym26, [tmpq+stride3q*2], 1 + vinserti128 ym27, [tmpq+stride7q ], 1 + lea tmpq, [tmpq+strideq*8 ] + vinserti32x4 m10, m16, [tmpq+strideq*0 ], 2 + vinserti32x4 m8, m17, [tmpq+strideq*1 ], 2 + vinserti32x4 m5, m18, [tmpq+strideq*2 ], 2 + vinserti32x4 m7, m19, [tmpq+stride3q ], 2 + vinserti32x4 m2, m24, [tmpq+strideq*4 ], 2 + vinserti32x4 m9, m25, [tmpq+stride5q ], 2 + vinserti32x4 m3, m26, [tmpq+stride3q*2], 2 + vinserti32x4 m4, m27, [tmpq+stride7q ], 2 + lea tmpq, [tmpq+strideq*8 ] + vinserti32x4 m10, [tmpq+strideq*0 ], 3 + vinserti32x4 m8, [tmpq+strideq*1 ], 3 + vinserti32x4 m5, [tmpq+strideq*2 ], 3 + vinserti32x4 m7, [tmpq+stride3q ], 3 + vinserti32x4 m2, [tmpq+strideq*4 ], 3 + vinserti32x4 m9, [tmpq+stride5q ], 3 + vinserti32x4 m3, [tmpq+stride3q*2], 3 + vinserti32x4 m4, [tmpq+stride7q ], 3 +%else ; %1 == 6 + movu xm16, [dstq+strideq*0-8] + movu xm17, [dstq+strideq*1-8] + movu xm18, [dstq+strideq*2-8] + movu xm19, [dstq+stride3q -8] + lea tmpq, [dstq+strideq*4-8] + movu xm2, [tmpq+strideq*0] + movu xm9, [tmpq+strideq*1] + movu xm3, [tmpq+strideq*2] + movu xm4, [tmpq+stride3q ] + lea tmpq, [tmpq+strideq*4] + vinserti128 ym16, [tmpq+strideq*0], 1 + vinserti128 ym17, [tmpq+strideq*1], 1 + vinserti128 ym18, [tmpq+strideq*2], 1 + vinserti128 ym19, [tmpq+stride3q ], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 ym2, [tmpq+strideq*0], 1 + vinserti128 ym9, [tmpq+strideq*1], 1 + vinserti128 ym3, [tmpq+strideq*2], 1 + vinserti128 ym4, [tmpq+stride3q ], 1 + lea tmpq, [tmpq+strideq*4] + vinserti32x4 m10, m16, [tmpq+strideq*0], 2 + vinserti32x4 m8, m17, [tmpq+strideq*1], 2 + vinserti32x4 m5, m18, [tmpq+strideq*2], 2 + vinserti32x4 m7, m19, [tmpq+stride3q ], 2 + lea tmpq, [tmpq+strideq*4] + vinserti32x4 m2, [tmpq+strideq*0], 2 + vinserti32x4 m9, [tmpq+strideq*1], 2 + vinserti32x4 m3, [tmpq+strideq*2], 2 + vinserti32x4 m4, [tmpq+stride3q ], 2 + lea tmpq, [tmpq+strideq*4] + vinserti32x4 m10, [tmpq+strideq*0], 3 + vinserti32x4 m8, [tmpq+strideq*1], 3 + vinserti32x4 m5, [tmpq+strideq*2], 3 + vinserti32x4 m7, [tmpq+stride3q ], 3 + lea tmpq, [tmpq+strideq*4] + vinserti32x4 m2, [tmpq+strideq*0], 3 + vinserti32x4 m9, [tmpq+strideq*1], 3 + vinserti32x4 m3, [tmpq+strideq*2], 3 + vinserti32x4 m4, [tmpq+stride3q ], 3 +%endif + punpcklwd m6, m10, m8 + punpckhwd m10, m8 + punpcklwd m8, m5, m7 + punpckhwd m5, m7 + punpcklwd m7, m2, m9 + punpckhwd m2, m9 + punpcklwd m9, m3, m4 + punpckhwd m3, m4 + punpckldq m4, m6, m8 + punpckhdq m6, m8 + punpckldq m8, m10, m5 + punpckhdq m10, m5 + punpckldq m5, m7, m9 + punpckhdq m7, m9 + punpckldq m9, m2, m3 + punpckhdq m2, m3 +%if %1 == 8 + punpcklqdq m3, m4, m5 +%endif + punpckhqdq m4, m5 + punpcklqdq m5, m6, m7 + punpckhqdq m6, m7 + punpcklqdq m7, m8, m9 + punpckhqdq m8, m9 + punpcklqdq m9, m10, m2 +%if %1 == 8 + punpckhqdq m10, m2 +%endif +%else ; %1 == 4 + kxnorb k1, k1, k1 + kmovb k2, k1 + vpgatherdq m7{k1}, [dstq+ym12-4] + lea tmpq, [dstq+strideq*2-4] + kmovb k1, k2 + vpgatherdq m4{k2}, [tmpq+ym12] + lea tmpq, [tmpq+strideq*2] + kmovb k2, k1 + vpgatherdq m5{k1}, [tmpq+ym12] + lea tmpq, [tmpq+strideq*2] + vpgatherdq m6{k2}, [tmpq+ym12] + punpcklwd m8, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 + punpcklwd m6, m8, m7 + punpckhwd m8, m7 + punpcklwd m7, m4, m5 + punpckhwd m4, m5 + punpcklqdq m5, m6, m7 + punpckhqdq m6, m7 + punpcklqdq m7, m8, m4 + punpckhqdq m8, m4 +%endif +%endif + + ; load L/E/I/H +%ifidn %2, v + movu ym16, [lq+l_strideq*1] + movsldup m17, [l_shuf_v] + vptestnmb k1, ym16, ym16 + vmovdqu8 ym16{k1}, [lq+l_strideq*0] ; l[x][] ? l[x][] : l[x-stride][] + vpermb m16, m17, m16 ; l[x][1] +%else + movq xm16, [lq+l_strideq*0] + movq xm17, [lq+l_strideq*1] + vinserti128 ym16, [lq+l_strideq*2], 1 + vinserti128 ym17, [lq+l_stride3q ], 1 + lea tmpq, [lq+l_strideq*4] + vinserti32x4 m16, [tmpq+l_strideq*0], 2 + vinserti32x4 m17, [tmpq+l_strideq*1], 2 + vinserti32x4 m16, [tmpq+l_strideq*2], 3 + vinserti32x4 m17, [tmpq+l_stride3q ], 3 + punpcklqdq m16, m17 + vbroadcasti32x4 m17, [l_shuf_h] + vptestnmb k1, m16, m16 + vpalignr m16{k1}, m16, 12 + pshufb m16, m17 ; l[x][1] +%endif + vpbroadcastd m20, [pw_32767] + psubw m17, m5, m6 ; p1-p0 + psubw m18, m7, m8 ; q1-q0 + vptestmw k1, m16, m16 ; L + pabsw m17, m17 + pabsw m18, m18 + vpmaxuw m20{k1}, m17, m18 + vpbroadcastw m17, [lutq+136] + psrlw m18, m16, [lutq+128] + vpbroadcastd m19, [pw_1] + pminuw m18, m17 + psrlw m17, m16, 4 ; H + paddw m16, m16 + pmaxuw m18, m19 ; I + vpaddd m16, [pw_4] {1to16} + paddw m16, m18 ; E + REPX {pmullw x, m13}, m17, m18, m16 + vpcmpw k4, m20, m17, 6 ; hev +%if %1 != 4 + psubw m19, m4, m5 ; p2-p1 + pabsw m19, m19 +%if %1 == 8 || %1 == 16 + psubw m17, m3, m4 ; p3-p2 + pabsw m17, m17 + pmaxuw m19, m17 + psubw m17, m9, m10 ; q3-q2 + pabsw m17, m17 + pmaxuw m19, m17 +%endif + psubw m17, m9, m8 ; q2-q1 + pabsw m17, m17 + pmaxuw m19, m17 +%if %1 == 16 + vpbroadcastd ym17, [maskq+4] + vpord ym17, [maskq+8] {1to8} + vptestmd k1, ym17, ym21 +%else + vptestmd k1, ym21, [maskq+4] {1to8} +%endif + pmaxuw m19, m20 + psubw m17, m4, m6 ; p2-p0 + pabsw m17, m17 + pmaxuw m17, m20 + vmovdqa64 m20{k1}, m19 ; only apply fm-wide to wd>4 blocks +%if %1 == 8 || %1 == 16 + psubw m19, m3, m6 ; p3-p0 + pabsw m19, m19 + pmaxuw m17, m19 + psubw m19, m7, m10 ; q3-q0 + pabsw m19, m19 + pmaxuw m17, m19 +%endif + psubw m19, m7, m9 ; q2-q0 + pabsw m19, m19 + pmaxuw m17, m19 +%endif + vpcmpw k1, m20, m18, 2 + psubw m18, m5, m8 ; p1-q1 + psubw m19, m6, m7 ; p0-q0 + pabsw m18, m18 + pabsw m19, m19 + psrlw m18, 1 + paddw m19, m19 + paddw m18, m19 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + vpcmpw k1{k1}, m18, m16, 2 ; abs(p0-q0)*2+(abs(p1-q1)>>1) <= E +%if %1 != 4 + vpcmpw k2{k1}, m17, m13, 2 ; flat8in +%endif +%if %1 == 16 + psubw m20, m0, m6 + psubw m16, m1, m6 + pabsw m20, m20 + psubw m17, m2, m6 + pabsw m16, m16 + psubw m18, m11, m7 + pabsw m17, m17 + psubw m19, m22, m7 + pabsw m18, m18 + pmaxuw m20, m16 + psubw m16, m23, m7 + pabsw m19, m19 + pmaxuw m17, m18 + pabsw m16, m16 + vpandd ym18, ym21, [maskq+8] {1to8} + pmaxuw m20, m17 + pmaxuw m19, m16 + pcmpeqd ym16, ym21, ym18 + vpternlogd ym18, ym21, [maskq+4] {1to8}, 0xc8 + pmaxuw m20, m19 + pcmpeqd ym17, ym21, ym18 + vpternlogd ym18, ym21, [maskq+0] {1to8}, 0xc8 + vpcmpw k3{k2}, m20, m13, 2 ; flat8in & flat8out + pcmpeqd ym18, ym21 + vptestmb k3{k3}, ym16, ym16 ; flat8 & fm + vptestmb k2{k2}, ym17, ym17 ; flat8in + vptestmb k1{k1}, ym18, ym18 + kandnd k1, k2, k1 ; fm & !flat8 & !flat16 + kandnd k2, k3, k2 ; flat8 & !flat16 +%elif %1 == 6 || %1 == 8 + vpandd ym17, ym21, [maskq+4] {1to8} + pcmpeqd ym16, ym21, ym17 + vpternlogd ym17, ym21, [maskq+0] {1to8}, 0xc8 + pcmpeqd ym17, ym21 + vptestmb k2{k2}, ym16, ym16 ; flat8 & fm + vptestmb k1{k1}, ym17, ym17 + kandnd k1, k2, k1 ; fm & !flat8 +%else ; %1 == 4 + vpandd ym16, ym21, [maskq+0] {1to8} + pcmpeqd ym16, ym21 + vptestmb k1{k1}, ym16, ym16 +%endif + + ; short filter + psubw m16, m7, m6 + vpbroadcastd m17, [pw_3] + paddw m18, m16, m16 + paddw m18, m16 + psubw m16, m5, m8 ; iclip_diff(p1-q1) + pminsw m16, m14 + vpmaxsw m16{k4}{z}, m15 ; f=iclip_diff(p1-q1)&hev + knotd k4, k4 ; !hev + paddw m16, m18 ; f=iclip_diff(3*(q0-p0)+f) + vpbroadcastd m18, [pw_4] + pminsw m16, m14 + vpmaxsw m16{k1}{z}, m15 ; f&=fm + paddw m17, m16 + paddw m16, m18 + vpbroadcastd m18, [pw_16384] + pminsw m17, m14 + pminsw m16, m14 + psraw m17, 3 ; f2 + psraw m16, 3 ; f1 + paddw m6, m17 + psubw m7, m16 + vpmulhrsw m16{k4}{z}, m18 ; (f=(f1+1)>>1) & !hev + psubw m17, m14, m15 ; 1023 or 4095 + pxor m18, m18 + paddw m5, m16 + psubw m8, m16 + REPX {pminsw x, m17}, m6, m7, m5, m8 + REPX {pmaxsw x, m18}, m6, m7, m5, m8 + +%if %1 == 16 ; flat16 filter + vpaddd m19, m0, [pw_1] {1to16} + paddw m16, m1, m2 ; p5+p4 + paddw m26, m1, m6 ; p5+p0 + paddw m24, m2, m7 ; p4+q0 + paddw m16, m4 ; p5+p4+p3 + paddw m17, m3, m5 ; p2+p1 + psllw m19, 3 + paddw m16, m26 ; p5*2+p4+p3+p0 + paddw m17, m24 ; p4+p2+p1+q0 + psubw m19, m0 ; p6*7+8 + paddw m16, m17 ; p5*2+p4*2+p3+p2+p1+q0 + paddw m18, m3, m8 + paddw m19, m16 ; p6*7+p5+p4*2+p3+p2+p1+p0+q0 + paddw m25, m1, m0 + paddw m16, m0, m0 + psrlw m1{k3}, m19, 4 + paddw m19, m18 + psubw m19, m16 ; +p3+q1-p6*2 + paddw m16, m2, m0 + psrlw m2{k3}, m19, 4 + psubw m19, m25 + paddw m25, m4, m9 + paddw m20, m10, m5 + paddw m19, m25 ; +p2+q2-p6-p5 + paddw m17, m0, m3 + psubw m16, m20, m16 + psrlw m3{k3}, m19, 4 + paddw m19, m16 ; +p1+q3-p6-p4 + paddw m16, m11, m6 + psubw m16, m17 + paddw m17, m0, m4 + psrlw m4{k3}, m19, 4 + paddw m19, m16 ; +p0+q4-p6-p3 + paddw m16, m22, m7 + psubw m16, m17 + paddw m17, m0, m5 + psrlw m5{k3}, m19, 4 + paddw m19, m16 ; +q0+q5-p6-p2 + paddw m16, m23, m8 + psrlw m6{k3}, m19, 4 + psubw m16, m17 + paddw m19, m16 ; +q1+q6-p6-p1 + paddw m16, m23, m9 + psrlw m7{k3}, m19, 4 + psubw m16, m26 + paddw m19, m16 ; +q2+q6-p5-p0 + paddw m16, m23, m10 + psrlw m8{k3}, m19, 4 + psubw m16, m24 + paddw m19, m16 ; +q3+q6-p4-p0 + paddw m16, m23, m11 + psrlw m9{k3}, m19, 4 + psubw m16, m18 + paddw m19, m16 ; +q4+q6-p3-q1 + paddw m16, m23, m22 + psrlw m10{k3}, m19, 4 + psubw m16, m25 + paddw m19, m16 ; +q5+q6-p2-q2 + paddw m16, m23, m23 + psrlw m11{k3}, m19, 4 + psubw m16, m20 + paddw m19, m16 ; +q6*2-p1-q3 + psrlw m22{k3}, m19, 4 +%endif +%if %1 == 8 || %1 == 16 ; flat8 filter + vpbroadcastd m20, [pw_4096] + paddw m16, m3, m4 ; p3+p2 + paddw m19, m5, m6 ; p1+p0 + paddw m17, m16, m16 ; 2*(p3+p2) + paddw m19, m3 ; p1+p0+p3 + paddw m17, m7 ; 2*(p3+p2)+q0 + paddw m19, m17 ; 3*p3+2*p2+p1+p0+q0 + paddw m18, m4, m7 + pmulhrsw m4{k2}, m19, m20 + psubw m19, m16 + paddw m17, m5, m8 + paddw m16, m3, m5 + paddw m19, m17 + pmulhrsw m5{k2}, m19, m20 + psubw m19, m16 + paddw m16, m6, m9 + paddw m19, m16 + paddw m16, m3, m6 + pmulhrsw m6{k2}, m19, m20 + paddw m19, m10 + psubw m16, m7, m16 + paddw m19, m16 + psubw m16, m10, m18 + pmulhrsw m7{k2}, m19, m20 + paddw m16, m8 + paddw m19, m16 + psubw m16, m10, m17 + pmulhrsw m8{k2}, m19, m20 + paddw m16, m9 + paddw m19, m16 + pmulhrsw m9{k2}, m19, m20 +%elif %1 == 6 ; flat6 filter + vpbroadcastd m10, [pw_4096] + paddw m2, m5, m6 + paddw m0, m4, m7 + paddw m1, m2, m4 ; p2+p1+p0 + paddw m3, m4, m4 + paddw m1, m1 + paddw m4, m5 + paddw m1, m0 ; p2+2*(p2+p1+p0)+q0 + psubw m3, m7, m3 + pmulhrsw m5{k2}, m1, m10 + paddw m3, m8 + psubw m4, m8, m4 + paddw m1, m3 + pmulhrsw m6{k2}, m1, m10 + paddw m4, m9 + paddw m9, m9 + paddw m1, m4 + pmulhrsw m7{k2}, m1, m10 + psubw m9, m2 + paddw m1, m9 + pmulhrsw m8{k2}, m1, m10 +%endif + +%ifidn %2, v +%if %1 == 16 + mova [tmpq+strideq*2 ], m1 ; p5 + mova [tmpq+stride3q ], m2 ; p4 + mova [tmpq+strideq*4 ], m3 ; p3 + mova [tmpq+stride5q ], m4 ; p2 +%elif %1 == 8 + mova [tmpq+strideq*1 ], m4 ; p2 +%endif + mova [dstq+mstrideq*2], m5 ; p1 + mova [dstq+mstrideq ], m6 ; p0 + mova [dstq+strideq*0 ], m7 ; q0 + mova [dstq+strideq*1 ], m8 ; q1 +%if %1 == 8 || %1 == 16 + mova [dstq+strideq*2 ], m9 ; q2 +%endif +%if %1 == 16 + mova [dstq+stride3q ], m10 ; q3 + mova [dstq+strideq*4 ], m11 ; q4 + mova [dstq+stride5q ], m22 ; q5 +%endif +%else +%if %1 == 16 + TRANSPOSE8X8W 27, 0, 1, 2, 3, 4, 5, 6, 20 + TRANSPOSE8X8W 7, 8, 9, 10, 11, 22, 23, 28, 20 + mova [dstq+strideq*0 -16], xm27 + mova [dstq+strideq*0 ], xm7 + mova [dstq+strideq*1 -16], xm0 + mova [dstq+strideq*1 ], xm8 + mova [dstq+strideq*2 -16], xm1 + mova [dstq+strideq*2 ], xm9 + mova [dstq+stride3q -16], xm2 + mova [dstq+stride3q ], xm10 + mova [dstq+strideq*4 -16], xm3 + mova [dstq+strideq*4 ], xm11 + mova [dstq+stride5q -16], xm4 + mova [dstq+stride5q ], xm22 + mova [dstq+stride3q*2-16], xm5 + mova [dstq+stride3q*2 ], xm23 + mova [dstq+stride7q -16], xm6 + mova [dstq+stride7q ], xm28 + lea dstq, [dstq+strideq*8] + vextracti128 [dstq+strideq*0 -16], ym27, 1 + vextracti128 [dstq+strideq*0 ], ym7, 1 + vextracti128 [dstq+strideq*1 -16], ym0, 1 + vextracti128 [dstq+strideq*1 ], ym8, 1 + vextracti128 [dstq+strideq*2 -16], ym1, 1 + vextracti128 [dstq+strideq*2 ], ym9, 1 + vextracti128 [dstq+stride3q -16], ym2, 1 + vextracti128 [dstq+stride3q ], ym10, 1 + vextracti128 [dstq+strideq*4 -16], ym3, 1 + vextracti128 [dstq+strideq*4 ], ym11, 1 + vextracti128 [dstq+stride5q -16], ym4, 1 + vextracti128 [dstq+stride5q ], ym22, 1 + vextracti128 [dstq+stride3q*2-16], ym5, 1 + vextracti128 [dstq+stride3q*2 ], ym23, 1 + vextracti128 [dstq+stride7q -16], ym6, 1 + vextracti128 [dstq+stride7q ], ym28, 1 + lea dstq, [dstq+strideq*8] + vextracti32x4 [dstq+strideq*0 -16], m27, 2 + vextracti32x4 [dstq+strideq*0 ], m7, 2 + vextracti32x4 [dstq+strideq*1 -16], m0, 2 + vextracti32x4 [dstq+strideq*1 ], m8, 2 + vextracti32x4 [dstq+strideq*2 -16], m1, 2 + vextracti32x4 [dstq+strideq*2 ], m9, 2 + vextracti32x4 [dstq+stride3q -16], m2, 2 + vextracti32x4 [dstq+stride3q ], m10, 2 + vextracti32x4 [dstq+strideq*4 -16], m3, 2 + vextracti32x4 [dstq+strideq*4 ], m11, 2 + vextracti32x4 [dstq+stride5q -16], m4, 2 + vextracti32x4 [dstq+stride5q ], m22, 2 + vextracti32x4 [dstq+stride3q*2-16], m5, 2 + vextracti32x4 [dstq+stride3q*2 ], m23, 2 + vextracti32x4 [dstq+stride7q -16], m6, 2 + vextracti32x4 [dstq+stride7q ], m28, 2 + lea dstq, [dstq+strideq*8] + vextracti32x4 [dstq+strideq*0 -16], m27, 3 + vextracti32x4 [dstq+strideq*0 ], m7, 3 + vextracti32x4 [dstq+strideq*1 -16], m0, 3 + vextracti32x4 [dstq+strideq*1 ], m8, 3 + vextracti32x4 [dstq+strideq*2 -16], m1, 3 + vextracti32x4 [dstq+strideq*2 ], m9, 3 + vextracti32x4 [dstq+stride3q -16], m2, 3 + vextracti32x4 [dstq+stride3q ], m10, 3 + vextracti32x4 [dstq+strideq*4 -16], m3, 3 + vextracti32x4 [dstq+strideq*4 ], m11, 3 + vextracti32x4 [dstq+stride5q -16], m4, 3 + vextracti32x4 [dstq+stride5q ], m22, 3 + vextracti32x4 [dstq+stride3q*2-16], m5, 3 + vextracti32x4 [dstq+stride3q*2 ], m23, 3 + vextracti32x4 [dstq+stride7q -16], m6, 3 + vextracti32x4 [dstq+stride7q ], m28, 3 +%elif %1 == 8 + TRANSPOSE8X8W 3, 4, 5, 6, 7, 8, 9, 10, 2 + movu [dstq+strideq*0 ], xm3 + movu [dstq+strideq*1 ], xm4 + movu [dstq+strideq*2 ], xm5 + movu [dstq+stride3q ], xm6 + movu [dstq+strideq*4 ], xm7 + movu [dstq+stride5q ], xm8 + movu [dstq+stride3q*2], xm9 + movu [dstq+stride7q ], xm10 + lea dstq, [dstq+strideq*8] + vextracti128 [dstq+strideq*0 ], ym3, 1 + vextracti128 [dstq+strideq*1 ], ym4, 1 + vextracti128 [dstq+strideq*2 ], ym5, 1 + vextracti128 [dstq+stride3q ], ym6, 1 + vextracti128 [dstq+strideq*4 ], ym7, 1 + vextracti128 [dstq+stride5q ], ym8, 1 + vextracti128 [dstq+stride3q*2], ym9, 1 + vextracti128 [dstq+stride7q ], ym10, 1 + lea dstq, [dstq+strideq*8] + vextracti32x4 [dstq+strideq*0 ], m3, 2 + vextracti32x4 [dstq+strideq*1 ], m4, 2 + vextracti32x4 [dstq+strideq*2 ], m5, 2 + vextracti32x4 [dstq+stride3q ], m6, 2 + vextracti32x4 [dstq+strideq*4 ], m7, 2 + vextracti32x4 [dstq+stride5q ], m8, 2 + vextracti32x4 [dstq+stride3q*2], m9, 2 + vextracti32x4 [dstq+stride7q ], m10, 2 + lea dstq, [dstq+strideq*8] + vextracti32x4 [dstq+strideq*0 ], m3, 3 + vextracti32x4 [dstq+strideq*1 ], m4, 3 + vextracti32x4 [dstq+strideq*2 ], m5, 3 + vextracti32x4 [dstq+stride3q ], m6, 3 + vextracti32x4 [dstq+strideq*4 ], m7, 3 + vextracti32x4 [dstq+stride5q ], m8, 3 + vextracti32x4 [dstq+stride3q*2], m9, 3 + vextracti32x4 [dstq+stride7q ], m10, 3 + lea dstq, [dstq+strideq*8+8] +%else ; %1 == 4 || %1 == 6 + punpcklwd m9, m5, m6 + punpckhwd m5, m6 + kxnorb k1, k1, k1 + punpcklwd m6, m7, m8 + punpckhwd m7, m8 + kmovb k2, k1 + punpckldq m8, m9, m6 + vpscatterdq [dstq+ym12-4]{k1}, m8 + punpckhdq m9, m6 + lea tmpq, [dstq+strideq*2-4] + kmovb k1, k2 + vpscatterdq [tmpq+ym12]{k2}, m9 + punpckldq m6, m5, m7 + lea tmpq, [tmpq+strideq*2] + kmovb k2, k1 + vpscatterdq [tmpq+ym12]{k1}, m6 + punpckhdq m5, m7 + lea tmpq, [tmpq+strideq*2] + vpscatterdq [tmpq+ym12]{k2}, m5 +%endif +%endif +%endmacro + +INIT_ZMM avx512icl +cglobal lpf_v_sb_y_16bpc, 6, 12, 26, dst, stride, mask, l, l_stride, \ + lut, w, stride3, mstride, tmp, \ + mask_bits, stride5 +%define base tmpq-filter_mask + SWAP 12, 26 ; avoids clobbering xmm10 on WIN64 + lea tmpq, [filter_mask] + mov r6d, r7m ; bitdepth_max + lea stride3q, [strideq*3] + shl l_strideq, 2 + lea stride5q, [strideq*5] + shr r6d, 11 ; is_12bpc + mova ym21, [base+filter_mask] + mov mstrideq, strideq + vpbroadcastd m13, [base+pw_4+r6*8] + mov mask_bitsd, 0xff + vpbroadcastd m14, [base+clip_max+r6*4] + sub lq, l_strideq + vpbroadcastd m15, [base+clip_min+r6*4] + neg mstrideq + mov wd, wm +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + jz .no_flat16 + FILTER 16, v + jmp .end +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + FILTER 8, v + jmp .end +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .end + call .v4 +.end: + shl mask_bitsd, 8 + add dstq, 64 + pslld ym21, 8 + add lq, 32 + sub wd, 8 + jg .loop + RET +ALIGN function_align +.v4: ; called by both luma and chroma + FILTER 4, v + ret + +cglobal lpf_h_sb_y_16bpc, 6, 13, 29, dst, stride, mask, l, l_stride, \ + lut, h, stride3, l_stride3, tmp, \ + mask_bits, stride5, stride7 + lea tmpq, [filter_mask] + mov r6d, r7m ; bitdepth_max + lea stride3q, [strideq*3] + vpbroadcastd ym12, strided + shl l_strideq, 2 + lea stride5q, [strideq*5] + shr r6d, 11 ; is_12bpc + pmulld ym12, [base+stride_mul] + lea stride7q, [strideq+stride3q*2] + mova ym21, [base+filter_mask] + mov mask_bitsd, 0xff + vpbroadcastd m13, [base+pw_4+r6*8] + sub lq, 4 + vpbroadcastd m14, [base+clip_max+r6*4] + lea l_stride3q, [l_strideq*3] + vpbroadcastd m15, [base+clip_min+r6*4] + mov hd, hm +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + jz .no_flat16 + FILTER 16, h + jmp .end +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + FILTER 8, h + jmp .end2 +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .no_filter + call .h4 +.no_filter: + lea dstq, [dstq+stride3q*8] +.end: + lea dstq, [dstq+strideq*8] +.end2: + shl mask_bitsd, 8 + pslld ym21, 8 + lea lq, [lq+l_strideq*8] + sub hd, 8 + jg .loop + RET +ALIGN function_align +.h4: ; called by both luma and chroma + FILTER 4, h + ret + +cglobal lpf_v_sb_uv_16bpc, 6, 11, 22, dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits + lea tmpq, [filter_mask] + mov r6d, r7m ; bitdepth_max + shl l_strideq, 2 + lea stride3q, [strideq*3] + shr r6d, 11 ; is_12bpc + mova ym21, [base+filter_mask] + mov mstrideq, strideq + vpbroadcastd m13, [base+pw_4+r6*8] + mov mask_bitsd, 0xff + vpbroadcastd m14, [base+clip_max+r6*4] + sub lq, l_strideq + vpbroadcastd m15, [base+clip_min+r6*4] + neg mstrideq + mov wd, wm +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + FILTER 6, v + jmp .end +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .end + call mangle(private_prefix %+ _lpf_v_sb_y_16bpc_avx512icl).v4 +.end: + shl mask_bitsd, 8 + add dstq, 64 + pslld ym21, 8 + add lq, 32 + sub wd, 8 + jg .loop + RET + +cglobal lpf_h_sb_uv_16bpc, 6, 11, 22, dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp, mask_bits + lea tmpq, [filter_mask] + mov r6d, r7m ; bitdepth_max + vpbroadcastd ym12, strided + shl l_strideq, 2 + shr r6d, 11 ; is_12bpc + pmulld ym12, [base+stride_mul] + lea stride3q, [strideq*3] + mova ym21, [base+filter_mask] + mov mask_bitsd, 0xff + vpbroadcastd m13, [base+pw_4+r6*8] + sub lq, 4 + vpbroadcastd m14, [base+clip_max+r6*4] + lea l_stride3q, [l_strideq*3] + vpbroadcastd m15, [base+clip_min+r6*4] + mov hd, hm +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + jz .no_flat + FILTER 6, h + jmp .end +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + jz .end + call mangle(private_prefix %+ _lpf_h_sb_y_16bpc_avx512icl).h4 +.end: + lea tmpq, [strideq+stride3q] + shl mask_bitsd, 8 + pslld ym21, 8 + lea dstq, [dstq+tmpq*8] + lea lq, [lq+l_strideq*8] + sub hd, 8 + jg .loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/loopfilter16_sse.asm b/third_party/dav1d/src/x86/loopfilter16_sse.asm new file mode 100644 index 0000000000..c486b57a21 --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter16_sse.asm @@ -0,0 +1,1793 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +%if ARCH_X86_64 +%define PIC_sym(a) a +%else +%define PIC_base $$ +%define PIC_sym(a) pic_regq+a-PIC_base +%endif + +pb_4x1_4x5_4x9_4x13: times 4 db 0, 1 + times 4 db 8, 9 + +pw_1: times 8 dw 1 +pw_2: times 8 dw 2 +pw_3: times 8 dw 3 +; 4 and 16 need to be next to each other since they are used as alternates +; depending on whether bitdepth is 10 or 12 +pw_4: times 8 dw 4 +pw_16: times 8 dw 16 +pw_8: times 8 dw 8 +pw_4096: times 8 dw 4096 + +pb_mask: dd 1, 1, 2, 2 + +SECTION .text + +%if ARCH_X86_32 +%if STACK_ALIGNMENT < 16 +%define extra_stack 2 +%else +%define extra_stack 0 +%endif +%endif + +%macro RELOC_ARGS 2 ; h/v, off +ASSERT ARCH_X86_32 +%if STACK_ALIGNMENT < 16 + mov r5d, [rstk + stack_offset + 4*4 + 4] +%define lstridem [esp+%2+0*gprsize] + mov lstridem, r5d + mov r5d, [rstk + stack_offset + 4*5 + 4] +%define lutm [esp+%2+1*gprsize] + mov lutm, r5d + mov r5d, [rstk + stack_offset + 4*6 + 4] +%ifidn %1, v +%define wm [esp+%2+2*gprsize] + mov wm, r5d + mov r5d, [rstk + stack_offset + 4*3 + 4] +%define lm [esp+%2+3*gprsize] + mov lm, r5d +%else ; %1 == h +%define hm [esp+%2+2*gprsize] + mov hm, r5d +%endif ; %1==v + mov r5d, r7m +%define bdmulm [esp+%2+4*gprsize] + mov bdmulm, r5d +%else +%define lstridem r4m +%define lutm r5m +%ifidn %1, v +%define wm r6m +%define lm r3m +%else +%define hm r6m +%endif +%define bdmulm r7m +%endif ; STACK_ALIGNMENT +%endmacro + +%macro UNRELOC_ARGS 0 +%if ARCH_X86_32 +%undef lm +%undef lstridem +%undef wm +%undef hm +%undef lutm +%endif +%endmacro + +%macro SPLATD 2 + movd %1, %2 + pshufd %1, %1, q0000 +%endmacro + +%macro SPLATW 2 + movd %1, %2 + pshuflw %1, %1, q0000 + punpcklqdq %1, %1 +%endmacro + +; in: out: +; mm%1 a b c d a e i m +; mm%2 e f g h b f j n +; mm%3 i j k l -> c g k o +; mm%4 m n o p d h l p +%macro TRANSPOSE4X4W 5 + punpcklwd m%5, m%1, m%2 + punpckhwd m%1, m%2 + punpcklwd m%2, m%3, m%4 + punpckhwd m%3, m%4 + punpckldq m%4, m%5, m%2 + punpckhdq m%5, m%2 + punpckldq m%2, m%1, m%3 + punpckhdq m%1, m%3 + + SWAP %1, %4 + SWAP %2, %5, %3 +%endmacro + +; in: out: +; m%1 a b c d e f g h a i q y 6 E M U +; m%2 i j k l m n o p b j r z 7 F N V +; m%3 q r s t u v w x c k s 0 8 G O W +; m%4 y z 0 1 2 3 4 5 d l t 1 9 H P X +; m%5 6 7 8 9 A B C D -> e m u 2 A I Q Y +; m%6 E F G H I J K L f n v 3 B J R Z +; m%7 M N O P Q R S T g o w 4 C K S + +; m%8 U V W X Y Z + = h p x 5 D L T = +%if ARCH_X86_64 +%macro TRANSPOSE8X8W 9 + ; m%1 a b c d e f g h a i q y b j r z + ; m%2 i j k l m n o p c k s 0 d l t 1 + ; m%3 q r s t u v w x -> e m u 2 f n v 3 + ; m%4 y z 0 1 2 3 4 5 g o w 4 h p x 5 + TRANSPOSE4X4W %1, %2, %3, %4, %9 + + ; m%5 6 7 8 9 A B C D 6 E M U 7 F N V + ; m%6 E F G H I J K L 8 G O W 9 H P X + ; m%7 M N O P Q R S T -> A I Q Y B J R Z + ; m%8 U V W X Y Z + = C K S + D L T = + TRANSPOSE4X4W %5, %6, %7, %8, %9 + + ; m%1 a i q y b j r z a i q y 6 E M U + ; m%2 c k s 0 d l t 1 b j r z 7 F N V + ; m%3 e m u 2 f n v 3 c k s 0 8 G O W + ; m%4 g o w 4 h p x 5 d l t 1 9 H P X + ; m%5 6 E M U 7 F N V -> e m u 2 A I Q Y + ; m%6 8 G O W 9 H P X f n v 3 B J R Z + ; m%7 A I Q Y B J R Z g o w 4 C K S + + ; m%8 C K S + D L T = h p x 5 D L T = + punpckhqdq m%9, m%1, m%5 + punpcklqdq m%1, m%5 + punpckhqdq m%5, m%2, m%6 + punpcklqdq m%2, m%6 + punpckhqdq m%6, m%3, m%7 + punpcklqdq m%3, m%7 + punpckhqdq m%7, m%4, m%8 + punpcklqdq m%4, m%8 + + SWAP %8, %7, %4, %5, %3, %2, %9 +%endmacro +%else ; x86-32 +; input: 1-7 in registers, 8 in first memory [read-only] +; second memory is scratch, and may overlap with first or third memory +; output: 1-5,7-8 in registers, 6 in third memory [write-only] +%macro TRANSPOSE8X8W 13 ; regs [8x], mem [3x], a/u [in/out alignment [2x] + TRANSPOSE4X4W %1, %2, %3, %4, %8 +%ifnidn %9, "" + mov%12 m%8, %9 +%else + mova m%8, %10 +%endif + mova %10, m%4 + TRANSPOSE4X4W %5, %6, %7, %8, %4 + punpckhqdq m%4, m%1, m%5 + punpcklqdq m%1, m%5 + punpckhqdq m%5, m%2, m%6 + punpcklqdq m%2, m%6 + punpckhqdq m%6, m%3, m%7 + punpcklqdq m%3, m%7 + mova m%7, %10 +%ifnidn %11, "" + mov%13 %11, m%6 +%else + mova %10, m%6 +%endif + punpckhqdq m%6, m%7, m%8 + punpcklqdq m%7, m%8 + + ; 1,4,2,5,3,8,7,6 -> 1,2,3,4,5,6,7,8 + SWAP %2, %4, %5, %3 + SWAP %6, %8 +%endmacro +%endif ; x86-32/64 + +; transpose and write m8-11, everything else is scratch +%macro TRANSPOSE_8x4_AND_WRITE_4x8 5 ; p1, p0, q0, q1, tmp + ; transpose 8x4 + punpcklwd %5, %1, %2 + punpckhwd %1, %2 + punpcklwd %2, %3, %4 + punpckhwd %3, %4 + punpckldq %4, %5, %2 + punpckhdq %5, %2 + punpckldq %2, %1, %3 + punpckhdq %1, %3 + + ; write out + movq [dstq+strideq*0-4], %4 + movhps [dstq+strideq*1-4], %4 + movq [dstq+strideq*2-4], %5 + movhps [dstq+stride3q -4], %5 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], %2 + movhps [dstq+strideq*1-4], %2 + movq [dstq+strideq*2-4], %1 + movhps [dstq+stride3q -4], %1 + lea dstq, [dstq+strideq*4] +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] + ; load data +%ifidn %2, v +%if %1 == 4 +%if ARCH_X86_64 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 + mova P1, [dstq+mstrideq*2] ; p1 + mova P0, [dstq+mstrideq*1] ; p0 + mova Q0, [dstq+strideq*0] ; q0 + mova Q1, [dstq+strideq*1] ; q1 +%else ; x86-32 +%define P1 [dstq+mstrideq*2] +%define P0 [dstq+mstrideq*1] +%define Q0 [dstq+strideq*0] +%define Q1 [dstq+strideq*1] +%endif ; x86-32/64 +%else ; %1 != 4 + ; load 6-8 pixels, remainder (for wd=16) will be read inline + lea tmpq, [dstq+mstrideq*4] +%if ARCH_X86_64 + ; we load p3 later +%define P2 m13 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 + mova P2, [tmpq+strideq*1] + mova P1, [tmpq+strideq*2] + mova P0, [tmpq+stride3q] + mova Q0, [dstq+strideq*0] + mova Q1, [dstq+strideq*1] + mova Q2, [dstq+strideq*2] +%if %1 != 6 +%define P3 [tmpq+strideq*0] +%define Q3 m15 + mova Q3, [dstq+stride3q] +%endif ; %1 != 6 +%else ; x86-32 +%define P2 [tmpq+strideq*1] +%define P1 [dstq+mstrideq*2] +%define P0 [dstq+mstrideq*1] +%define Q0 [dstq+strideq*0] +%define Q1 [dstq+strideq*1] +%define Q2 [dstq+strideq*2] +%if %1 != 6 +%define P3 [dstq+mstrideq*4] +%define Q3 [dstq+stride3q] +%endif ; %1 != 6 +%endif ; x86-32/64 +%endif ; %1 ==/!= 4 +%else ; %2 != v + ; load lines +%if %1 == 4 + movq m0, [dstq+strideq*0-4] + movq m2, [dstq+strideq*1-4] + movq m4, [dstq+strideq*2-4] + movq m5, [dstq+stride3q -4] + lea tmpq, [dstq+strideq*4] + movq m3, [tmpq+strideq*0-4] + movq m6, [tmpq+strideq*1-4] + movq m1, [tmpq+strideq*2-4] + movq m7, [tmpq+stride3q -4] + + ; transpose 4x8 + ; m0: A-D0 + ; m2: A-D1 + ; m4: A-D2 + ; m5: A-D3 + ; m3: A-D4 + ; m6: A-D5 + ; m1: A-D6 + ; m7: A-D7 + punpcklwd m0, m2 + punpcklwd m4, m5 + punpcklwd m3, m6 + punpcklwd m1, m7 + ; m0: A0-1,B0-1,C0-1,D0-1 + ; m4: A2-3,B2-3,C2-3,D2-3 + ; m3: A4-5,B4-5,C4-5,D4-5 + ; m1: A6-7,B6-7,C6-7,D6-7 + punpckhdq m2, m0, m4 + punpckldq m0, m4 + punpckhdq m4, m3, m1 + punpckldq m3, m1 + ; m0: A0-3,B0-3 + ; m2: C0-3,D0-3 + ; m3: A4-7,B4-7 + ; m4: C4-7,D4-7 + punpckhqdq m1, m0, m3 + punpcklqdq m0, m3 + punpckhqdq m3, m2, m4 + punpcklqdq m2, m4 + ; m0: A0-7 + ; m1: B0-7 + ; m2: C0-7 + ; m3: D0-7 +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%else +%define P1 [esp+3*mmsize] +%define P0 [esp+4*mmsize] +%define Q0 [esp+5*mmsize] +%define Q1 [esp+6*mmsize] + mova P1, m0 + mova P0, m1 + mova Q0, m2 + mova Q1, m3 +%endif +%elif %1 == 6 || %1 == 8 + movu m0, [dstq+strideq*0-8] + movu m1, [dstq+strideq*1-8] + movu m2, [dstq+strideq*2-8] + movu m3, [dstq+stride3q -8] + lea tmpq, [dstq+strideq*4] + movu m4, [tmpq+strideq*0-8] + movu m5, [tmpq+strideq*1-8] + movu m6, [tmpq+strideq*2-8] +%if ARCH_X86_64 + movu m7, [tmpq+stride3q -8] +%endif + + ; transpose 8x16 + ; m0: A-H0,A-H8 + ; m1: A-H1,A-H9 + ; m2: A-H2,A-H10 + ; m3: A-H3,A-H11 + ; m4: A-H4,A-H12 + ; m5: A-H5,A-H13 + ; m6: A-H6,A-H14 + ; m7: A-H7,A-H15 +%if ARCH_X86_64 + punpcklwd m8, m0, m1 +%else + punpcklwd m7, m0, m1 +%endif + punpckhwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + punpcklwd m3, m4, m5 + punpckhwd m4, m5 +%if ARCH_X86_64 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 +%else + mova [rsp+3*16], m4 + movu m4, [tmpq+stride3q -8] + punpcklwd m5, m6, m4 + punpckhwd m6, m4 +%endif + ; m8: A0-1,B0-1,C0-1,D0-1 [m7 on x86-32] + ; m0: E0-1,F0-1,G0-1,H0-1 + ; m1: A2-3,B2-3,C2-3,D2-3 + ; m2: E2-3,F2-3,G2-3,H2-3 + ; m3: A4-5,B4-5,C4-5,D4-5 + ; m4: E4-5,F4-5,G4-5,H4-5 [r3 on x86-32] + ; m5: A6-7,B6-7,C6-7,D6-7 + ; m6: E6-7,F6-7,G6-7,H6-7 +%if ARCH_X86_64 + punpckldq m7, m8, m1 + punpckhdq m8, m1 +%else + punpckldq m4, m7, m1 + punpckhdq m7, m1 +%endif + punpckldq m1, m0, m2 + punpckhdq m0, m2 + punpckldq m2, m3, m5 + punpckhdq m3, m5 +%if ARCH_X86_64 + punpckldq m5, m4, m6 + punpckhdq m4, m6 +%else + mova [rsp+4*16], m3 + mova m3, [rsp+3*16] + punpckldq m5, m3, m6 + punpckhdq m3, m6 +%endif + ; m7: A0-3,B0-3 [m4 on x86-32] + ; m8: C0-3,D0-3 [m7 on x86-32] + ; m1: E0-3,F0-3 + ; m0: G0-3,H0-3 + ; m2: A4-7,B4-7 + ; m3: C4-7,D4-7 [r4 on x86-32] + ; m5: E4-7,F4-7 + ; m4: G4-7,H4-7 [m3 on x86-32] +%if ARCH_X86_64 +%if %1 != 6 + punpcklqdq m6, m7, m2 +%endif + punpckhqdq m7, m2 + punpcklqdq m2, m8, m3 + punpckhqdq m8, m3 + punpcklqdq m3, m1, m5 + punpckhqdq m1, m5 +%if %1 != 6 + punpckhqdq m5, m0, m4 +%endif + punpcklqdq m0, m4 +%if %1 == 8 + mova [rsp+1*16], m6 +%define P3 [rsp+1*16] +%endif + ; 7,2,8,3,1,0,5 -> 13,8,9,10,11,14,15 + SWAP 7, 13 + SWAP 8, 2, 9 + SWAP 3, 10 + SWAP 1, 11 + SWAP 0, 14 + SWAP 5, 15 +%define P2 m13 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 +%if %1 == 8 +%define Q3 m15 +%endif +%else ; x86-32 +%if %1 == 8 +%define P3 [rsp+ 6*16] + punpcklqdq m6, m4, m2 + mova P3, m6 +%endif + mova m6, [rsp+4*16] + punpckhqdq m4, m2 + punpcklqdq m2, m7, m6 + punpckhqdq m7, m6 + punpcklqdq m6, m1, m5 + punpckhqdq m1, m5 +%if %1 == 8 +%define Q3 [rsp+24*16] + punpckhqdq m5, m0, m3 + mova Q3, m5 +%endif + punpcklqdq m0, m3 +%if %1 == 8 +%define P2 [rsp+18*16] +%define P1 [rsp+19*16] +%define P0 [rsp+20*16] +%define Q0 [rsp+21*16] +%define Q1 [rsp+22*16] +%define Q2 [rsp+23*16] +%else +%define P2 [rsp+3*16] +%define P1 [rsp+4*16] +%define P0 [rsp+5*16] +%define Q0 [rsp+6*16] +%define Q1 [rsp+7*16] +%define Q2 [rsp+8*16] +%endif + mova P2, m4 + mova P1, m2 + mova P0, m7 + mova Q0, m6 + mova Q1, m1 + mova Q2, m0 +%endif ; x86-32/64 +%else ; %1 == 16 + ; We only use 14 pixels but we'll need the remainder at the end for + ; the second transpose + mova m0, [dstq+strideq*0-16] + mova m1, [dstq+strideq*1-16] + mova m2, [dstq+strideq*2-16] + mova m3, [dstq+stride3q -16] + lea tmpq, [dstq+strideq*4] + mova m4, [tmpq+strideq*0-16] + mova m5, [tmpq+strideq*1-16] + mova m6, [tmpq+strideq*2-16] +%if ARCH_X86_64 + mova m7, [tmpq+stride3q -16] + + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, 8 + SWAP 5, 13 + SWAP 6, 8 + SWAP 7, 9 +%define P2 m13 +%define P1 m8 +%define P0 m9 +%else ; x86-32 +%define P2 [esp+18*16] +%define P1 [esp+19*16] +%define P0 [esp+20*16] + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, \ + [tmpq+stride3q -16], P2, "", a, a + mova P1, m6 + mova P0, m7 +%endif ; x86-32/64 + mova [rsp+ 7*16], m0 + mova [rsp+ 8*16], m1 + mova [rsp+ 9*16], m2 + mova [rsp+10*16], m3 +%define P3 [rsp+6*16] + mova P3, m4 + + mova m0, [dstq+strideq*0] + mova m1, [dstq+strideq*1] + mova m2, [dstq+strideq*2] + mova m3, [dstq+stride3q ] + lea tmpq, [dstq+strideq*4] + mova m4, [tmpq+strideq*0] + mova m5, [tmpq+strideq*1] + mova m6, [tmpq+strideq*2] +%if ARCH_X86_64 + mova m7, [tmpq+stride3q ] + + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, 10 + SWAP 0, 10 + SWAP 1, 11 + SWAP 2, 14 + SWAP 3, 15 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 +%define Q3 m15 +%else ; x86-32 + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, \ + [tmpq+stride3q ], [rsp+12*16], "", a, a +%define Q0 [esp+21*16] +%define Q1 [esp+22*16] +%define Q2 [esp+23*16] +%define Q3 [esp+24*16] + mova Q0, m0 + mova Q1, m1 + mova Q2, m2 + mova Q3, m3 +%endif ; x86-32/64 + + mova [rsp+11*16], m4 +%if ARCH_X86_64 + mova [rsp+12*16], m5 +%endif + mova [rsp+13*16], m6 + mova [rsp+14*16], m7 +%endif ; %1 == 4/6/8/16 +%endif ; %2 ==/!= v + + ; load L/E/I/H +%if ARCH_X86_32 +%define l_strideq r5 + mov l_strideq, dword lstridem +%ifidn %2, v +%define lq r3 + mov lq, dword lm +%endif +%endif +%ifidn %2, v +%if cpuflag(sse4) + pmovzxbw m1, [lq] + pmovzxbw m0, [lq+l_strideq] + pxor m2, m2 +%else ; ssse3 + movq m1, [lq] + movq m0, [lq+l_strideq] + pxor m2, m2 + REPX {punpcklbw x, m2}, m1, m0 +%endif ; ssse3/sse4 +%else ; %2 != v + movq m0, [lq] ; l0, l1 + movq m1, [lq+l_strideq] ; l2, l3 + punpckldq m0, m1 ; l0, l2, l1, l3 + pxor m2, m2 + punpcklbw m1, m0, m2 ; l0, l2 + punpckhbw m0, m2 ; l1, l3 +%endif ; %2==/!=v +%if ARCH_X86_32 +%ifidn %2, v +%undef lq + mov mstrideq, mstridem +%endif +%endif + pcmpeqw m5, m2, m0 + pand m1, m5 + por m0, m1 ; l[x][] ? l[x][] : l[x-stride][] + pshufb m0, [PIC_sym(pb_4x1_4x5_4x9_4x13)] ; l[x][1] + pcmpeqw m5, m2, m0 ; !L + psrlw m5, 1 +%if ARCH_X86_64 + psrlw m2, m0, [lutq+128] + SPLATW m1, [lutq+136] +%else ; x86-32 + mov r5, lutm + psrlw m2, m0, [r5+128] + SPLATW m1, [r5+136] +%endif ; x86-32/64 + pminsw m2, m1 + pmaxsw m2, [PIC_sym(pw_1)] ; I + psrlw m1, m0, 4 ; H + paddw m0, [PIC_sym(pw_2)] + paddw m0, m0 + paddw m0, m2 ; E + REPX {pmullw x, [bdmulq]}, m0, m1, m2 +%if ARCH_X86_32 +%undef l_strideq + lea stride3q, [strideq*3] +%endif + + psubw m3, P1, P0 ; p1-p0 + psubw m4, Q0, Q1 ; q0-q1 + REPX {pabsw x, x}, m3, m4 + pmaxsw m3, m5 + pmaxsw m3, m4 + pcmpgtw m7, m3, m1 ; hev +%if %1 != 4 + psubw m4, P2, P0 ; p2-p0 + pabsw m4, m4 + pmaxsw m4, m3 +%if %1 != 6 + mova m6, P3 ; p3 + psubw m5, m6, P0 ; p3-p0 + pabsw m5, m5 + pmaxsw m4, m5 +%endif ; %1 != 6 + psubw m5, Q0, Q2 ; q0-q2 + pabsw m5, m5 + pmaxsw m4, m5 +%if %1 != 6 + psubw m5, Q0, Q3 ; q0-q3 + pabsw m5, m5 + pmaxsw m4, m5 +%endif ; %1 != 6 + pcmpgtw m4, [bdmulq] ; !flat8in + + psubw m5, P2, P1 ; p2-p1 + pabsw m5, m5 +%if %1 != 6 + psubw m6, P2 ; p3-p2 + pabsw m6, m6 + pmaxsw m5, m6 + psubw m6, Q2, Q3 ; q2-q3 + pabsw m6, m6 + pmaxsw m5, m6 +%endif ; %1 != 6 + psubw m6, Q2, Q1 ; q2-q1 + pabsw m6, m6 + pmaxsw m5, m6 + +%if %1 == 16 + SPLATD m6, [maskq+8] + SPLATD m1, [maskq+4] + por m6, m1 + pand m6, m12 + pcmpeqd m6, m12 + pand m5, m6 +%else ; %1 != 16 + SPLATD m6, [maskq+4] + pand m6, m12 + pcmpeqd m6, m12 + pand m5, m6 ; only apply fm-wide to wd>4 blocks +%endif ; %1==/!=16 + pmaxsw m3, m5 +%endif ; %1 != 4 + pcmpgtw m3, m2 + + psubw m5, P1, Q1 ; p1-q1 + psubw m6, P0, Q0 ; p0-q0 + REPX {pabsw x, x}, m5, m6 + paddw m6, m6 + psrlw m5, 1 + paddw m5, m6 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + pcmpgtw m5, m0 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E + por m3, m5 + +%if %1 == 16 + +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] + mova m1, [tmpq+strideq*2] + mova m2, [tmpq+stride3q] +%else ; %2 != v + mova m0, [rsp+ 8*16] + mova m1, [rsp+ 9*16] + mova m2, [rsp+10*16] +%endif ; %2==/!=v + REPX {psubw x, P0}, m0, m1, m2 + REPX {pabsw x, x}, m0, m1, m2 + pmaxsw m1, m0 + pmaxsw m1, m2 +%ifidn %2, v + lea tmpq, [dstq+strideq*4] + mova m0, [tmpq+strideq*0] + mova m2, [tmpq+strideq*1] + mova m5, [tmpq+strideq*2] +%else ; %2 != v + mova m0, [rsp+11*16] + mova m2, [rsp+12*16] + mova m5, [rsp+13*16] +%endif ; %2==/!=v + REPX {psubw x, Q0}, m0, m2, m5 + REPX {pabsw x, x}, m0, m2, m5 + pmaxsw m0, m2 + pmaxsw m1, m5 + pmaxsw m1, m0 + pcmpgtw m1, [bdmulq] ; !flat8out + por m1, m4 ; !flat8in | !flat8out + SPLATD m2, [maskq+8] + pand m5, m2, m12 + pcmpeqd m5, m12 + pandn m1, m5 ; flat16 + pandn m5, m3, m1 ; flat16 & fm + SWAP 1, 5 + + SPLATD m5, [maskq+4] + por m5, m2 + pand m2, m5, m12 + pcmpeqd m2, m12 + pandn m4, m2 ; flat8in + pandn m2, m3, m4 + SWAP 2, 4 + SPLATD m2, [maskq+0] + por m2, m5 + pand m2, m12 + pcmpeqd m2, m12 + pandn m3, m2 + pandn m0, m4, m3 ; fm & !flat8 & !flat16 + SWAP 0, 3 + pandn m0, m1, m4 ; flat8 & !flat16 + SWAP 0, 4 +%elif %1 != 4 + SPLATD m0, [maskq+4] + pand m2, m0, m12 + pcmpeqd m2, m12 + pandn m4, m2 + pandn m2, m3, m4 ; flat8 & fm + SWAP 2, 4 + SPLATD m2, [maskq+0] + por m0, m2 + pand m0, m12 + pcmpeqd m0, m12 + pandn m3, m0 + pandn m0, m4, m3 ; fm & !flat8 + SWAP 0, 3 +%else ; %1 == 4 + SPLATD m0, [maskq+0] + pand m0, m12 + pcmpeqd m0, m12 + pandn m3, m0 ; fm +%endif ; %1==/!=4 + + ; short filter +%if ARCH_X86_64 + SPLATW m0, r7m +%else + SPLATW m0, bdmulm +%endif + pcmpeqw m2, m2 + psrlw m0, 1 ; 511 or 2047 + pxor m2, m0 ; -512 or -2048 + + psubw m5, Q0, P0 ; q0-p0 + paddw m6, m5, m5 + paddw m6, m5 ; 3*(q0-p0) + psubw m5, P1, Q1 ; iclip_diff(p1-q1) + pminsw m5, m0 + pmaxsw m5, m2 + pand m5, m7 ; f=iclip_diff(p1-q1)&hev + paddw m5, m6 ; f=iclip_diff(3*(q0-p0)+f) + pminsw m5, m0 + pmaxsw m5, m2 + pand m3, m5 ; f&=fm + paddw m5, m3, [PIC_sym(pw_3)] + paddw m3, [PIC_sym(pw_4)] + REPX {pminsw x, m0}, m5, m3 + psraw m5, 3 ; f2 + psraw m3, 3 ; f1 + psubw m0, m2 ; 1023 or 4095 + pxor m2, m2 +%if ARCH_X86_64 + paddw P0, m5 + psubw Q0, m3 +%else + paddw m5, P0 + psubw m6, Q0, m3 + REPX {pminsw x, m0}, m5, m6 + REPX {pmaxsw x, m2}, m5, m6 +%endif + + paddw m3, [PIC_sym(pw_1)] + psraw m3, 1 ; f=(f1+1)>>1 + pandn m7, m3 ; f&=!hev + SWAP 7, 3 +%if ARCH_X86_64 + paddw P1, m3 + psubw Q1, m3 + REPX {pminsw x, m0}, P1, P0, Q0, Q1 + REPX {pmaxsw x, m2}, P1, P0, Q0, Q1 +%else + psubw m7, Q1, m3 + paddw m3, P1 + REPX {pminsw x, m0}, m7, m3 + REPX {pmaxsw x, m2}, m7, m3 +%if %1 > 4 + mova P1, m3 + mova P0, m5 + mova Q0, m6 + mova Q1, m7 +%endif +%endif + +%if %1 == 16 + +; m8-11 = p1/p0/q0/q1, m4=flat8, m1=flat16 +; m12=filter bits mask +; m13-15=p2/q2/q3 +; m0,2-3,5-7 = free + + ; flat16 filter +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] ; p6 + mova m2, [tmpq+strideq*2] ; p5 + mova m7, [tmpq+stride3q] ; p4 + mova m6, [tmpq+strideq*4] ; p3 + lea tmpq, [dstq+mstrideq*4] +%else ; %2 != v + mova m0, [rsp+ 8*16] + mova m2, [rsp+ 9*16] + mova m7, [rsp+10*16] + mova m6, [rsp+ 6*16] +%endif ; %2==/!=v + + mova [rsp+ 0*16], m4 + + ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + psllw m3, m0, 3 ; p6*8 + paddw m3, [PIC_sym(pw_8)] + paddw m5, m2, m7 ; p5+p4 + psubw m3, m0 + paddw m5, m5 ; (p5+p4)*2 + paddw m3, m6 ; p6*7+p3 + paddw m5, P2 ; (p5+p4)*2+p2 + paddw m3, P1 ; p6*7+p3+p1 + paddw m5, P0 ; (p5+p4)*2+p2+p0 + paddw m3, Q0 ; p6*7+p3+p1+q0 + paddw m3, m5 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m2 + por m5, m4 +%ifidn %2, v + mova [tmpq+mstrideq*2], m5 ; p5 +%else ; %2 != v + mova [rsp+9*16], m5 +%endif ; %2==/!=v + + ; sub p6*2, add p3/q1 + paddw m3, m6 + paddw m5, m0, m0 + paddw m3, Q1 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m7 + por m5, m4 +%ifidn %2, v + mova [tmpq+mstrideq*1], m5 ; p4 +%else ; %2 != v + mova [rsp+10*16], m5 +%endif ; %2==/!=v + + ; sub p6/p5, add p2/q2 + psubw m3, m0 + paddw m5, P2, Q2 + psubw m3, m2 + paddw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m6 + por m5, m4 +%ifidn %2, v + mova [tmpq+strideq*0], m5 ; p3 +%else ; %2 != v + mova [rsp+6*16], m5 +%endif ; %2==/!=v + +%define WRITE_IN_PLACE 0 +%ifidn %2, v +%if ARCH_X86_64 +%define WRITE_IN_PLACE 1 +%endif +%endif + + ; sub p6/p4, add p1/q3 + paddw m3, P1 + paddw m5, m0, m7 + paddw m3, Q3 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P2 + por m5, m4 +%if WRITE_IN_PLACE + mova [tmpq+strideq*1], m5 +%else + mova [rsp+1*16], m5 ; don't clobber p2/m13 +%endif + + ; sub p6/p3, add p0/q4 + paddw m3, P0 + paddw m5, m0, m6 +%ifidn %2, v + paddw m3, [dstq+strideq*4] +%else ; %2 != v + paddw m3, [rsp+11*16] +%endif ; %2==/!=v + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P1 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq+mstrideq*2], m5 +%else + mova [rsp+2*16], m5 ; don't clobber p1/m3 +%endif + + ; sub p6/p2, add q0/q5 + paddw m3, Q0 + paddw m5, m0, P2 +%ifidn %2, v +%if ARCH_X86_32 + lea r4, P2 +%endif + lea tmpq, [dstq+strideq*4] + paddw m3, [tmpq+strideq*1] +%else ; %2 != v + paddw m3, [rsp+12*16] +%endif ; %2==/!=v + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P0 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq+mstrideq*1], m5 +%else + mova [rsp+3*16], m5 ; don't clobber p0/m4 +%endif + + ; sub p6/p1, add q1/q6 + paddw m3, Q1 + paddw m5, m0, P1 +%ifidn %2, v + mova m0, [tmpq+strideq*2] ; q6 +%else ; %2 != v + mova m0, [rsp+13*16] ; q6 +%endif ; %2==/!=v + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, Q0 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq], m5 +%else + mova [rsp+4*16], m5 ; don't clobber q0/m5 +%endif + + ; sub p5/p0, add q2/q6 + paddw m3, Q2 + paddw m5, m2, P0 + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, Q1 + por m2, m5, m4 ; don't clobber q1/m6 + + ; sub p4/q0, add q3/q6 + paddw m3, Q3 + paddw m7, Q0 + paddw m3, m0 + psubw m3, m7 + psrlw m7, m3, 4 + pand m7, m1 + pandn m4, m1, Q2 + por m7, m4 ; don't clobber q2/m14 + + ; sub p3/q1, add q4/q6 +%ifidn %2, v + paddw m3, [tmpq+strideq*0] +%else ; %2 != v + paddw m3, [rsp+11*16] +%endif ; %2==/!=v + paddw m6, Q1 + paddw m3, m0 + psubw m3, m6 + psrlw m6, m3, 4 + pand m6, m1 + pandn m4, m1, Q3 + por m6, m4 +%if WRITE_IN_PLACE + mova [tmpq+mstrideq], m6 ; q3 +%else ; %2 != v + mova [rsp+5*16], m6 +%endif ; %2==/!=v + + ; sub p2/q2, add q5/q6 +%ifidn %2, v + paddw m3, [tmpq+strideq*1] +%if ARCH_X86_64 + paddw m5, P2, Q2 +%else + ; because tmpq is clobbered, so we use a backup pointer for P2 instead + paddw m5, [r4], Q2 + mov pic_regq, pic_regm +%endif +%else ; %2 != v + paddw m3, [rsp+12*16] + paddw m5, P2, Q2 +%endif ; %2==/!=v + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 +%ifidn %2, v + pandn m4, m1, [tmpq+strideq*0] +%else ; %2 != v + pandn m4, m1, [rsp+11*16] +%endif ; %2==/!=v + por m5, m4 +%ifidn %2, v + mova [tmpq+strideq*0], m5 ; q4 +%else ; %2 != v + mova [rsp+11*16], m5 +%endif ; %2==/!=v + + ; sub p1/q3, add q6*2 + psubw m3, P1 + paddw m0, m0 + psubw m3, Q3 + paddw m3, m0 + psrlw m5, m3, 4 + pand m5, m1 +%ifidn %2, v + pandn m4, m1, [tmpq+strideq*1] +%else ; %2 != v + pandn m4, m1, [rsp+12*16] +%endif ; %2==/!=v + por m5, m4 +%ifidn %2, v + mova [tmpq+strideq*1], m5 ; q5 +%else ; %2 != v + mova [rsp+12*16], m5 +%endif ; %2==/!=v + + mova m4, [rsp+0*16] +%ifidn %2, v + lea tmpq, [dstq+mstrideq*4] +%endif +%if ARCH_X86_64 + SWAP 2, 11 + SWAP 7, 14 + SWAP 6, 15 +%else ; x86-32 + mova Q1, m2 + mova Q2, m7 +%endif ; x86-32/64 +%if WRITE_IN_PLACE + mova P2, [tmpq+strideq*1] + mova P1, [tmpq+strideq*2] + mova P0, [tmpq+stride3q] + mova Q0, [dstq] +%elif ARCH_X86_64 + mova P2, [rsp+1*16] + mova P1, [rsp+2*16] + mova P0, [rsp+3*16] + mova Q0, [rsp+4*16] +%else ; !WRITE_IN_PLACE & x86-32 + mova m0, [rsp+1*16] + mova m1, [rsp+2*16] + mova m2, [rsp+3*16] + mova m3, [rsp+4*16] + mova m7, [rsp+5*16] + mova P2, m0 + mova P1, m1 + mova P0, m2 + mova Q0, m3 + mova Q3, m7 +%endif ; WRITE_IN_PLACE / x86-32/64 +%undef WRITE_IN_PLACE +%endif ; %1 == 16 + +%if %1 >= 8 + + ; flat8 filter + mova m0, P3 ; p3 + paddw m1, m0, P2 ; p3+p2 + paddw m2, P1, P0 ; p1+p0 + paddw m3, m1, m1 ; 2*(p3+p2) + paddw m2, m0 ; p1+p0+p3 + paddw m3, Q0 ; 2*(p3+p2)+q0 + paddw m2, m3 ; 3*p3+2*p2+p1+p0+q0 + pmulhrsw m7, m2, [PIC_sym(pw_4096)] + psubw m7, P2 + pand m7, m4 + + paddw m3, P1, Q1 ; p1+q1 + psubw m2, m1 ; 2*p3+p2+p1+p0+q0 + paddw m2, m3 ; 2*p3+p2+2*p1+p0+q0+q1 + pmulhrsw m3, m2, [PIC_sym(pw_4096)] + psubw m3, P1 + pand m3, m4 + + paddw m5, m0, P1 ; p3+p1 + paddw m6, P0, Q2 ; p0+q2 + psubw m2, m5 ; p3+p2+p1+p0+q0+q1 + paddw m2, m6 ; p3+p2+p1+2*p0+q0+q1+q2 + pmulhrsw m5, m2, [PIC_sym(pw_4096)] + psubw m5, P0 + pand m5, m4 + + paddw m6, m0, P0 ; p3+p0 + paddw m1, Q0, Q3 ; q0+q3 + psubw m2, m6 ; p2+p1+p0+q0+q1+q2 + paddw m2, m1 ; p2+p1+p0+2*q0+q1+q2+q3 + pmulhrsw m6, m2, [PIC_sym(pw_4096)] + psubw m6, Q0 + pand m6, m4 + + paddw m2, Q1 ; p2+p1+p0+2*q0+2*q1+q2+q3 + paddw m2, Q3 ; p2+p1+p0+2*q0+2*q1+q2+2*q3 + paddw m1, P2, Q0 ; p2+q0 + psubw m2, m1 ; p1+p0+q0+2*q1+q2+2*q3 + pmulhrsw m1, m2, [PIC_sym(pw_4096)] + psubw m1, Q1 + pand m1, m4 + + psubw m2, P1 ; p0+q0+2*q1+q2+2*q3 + psubw m2, Q1 ; p0+q0+q1+q2+2*q3 + paddw m0, Q3, Q2 ; q3+q2 + paddw m2, m0 ; p0+q0+q1+2*q2+3*q3 + pmulhrsw m2, [PIC_sym(pw_4096)] + psubw m2, Q2 + pand m2, m4 + + paddw m7, P2 + paddw m3, P1 + paddw m5, P0 + paddw m6, Q0 + paddw m1, Q1 + paddw m2, Q2 + +%ifidn %2, v + mova [tmpq+strideq*1], m7 ; p2 + mova [tmpq+strideq*2], m3 ; p1 + mova [tmpq+stride3q ], m5 ; p0 + mova [dstq+strideq*0], m6 ; q0 + mova [dstq+strideq*1], m1 ; q1 + mova [dstq+strideq*2], m2 ; q2 +%else ; %2 != v + mova m0, P3 + +%if %1 == 8 + lea tmpq, [dstq+strideq*4] +%if ARCH_X86_64 + SWAP 4, 15 + TRANSPOSE8X8W 0, 7, 3, 5, 6, 1, 2, 4, 8 +%else + TRANSPOSE8X8W 0, 7, 3, 5, 6, 1, 2, 4, "", \ + Q3, [tmpq+strideq*1-8], a, u +%endif + + ; write 8x8 + movu [dstq+strideq*0-8], m0 + movu [dstq+strideq*1-8], m7 + movu [dstq+strideq*2-8], m3 + movu [dstq+stride3q -8], m5 + movu [tmpq+strideq*0-8], m6 +%if ARCH_X86_64 + movu [tmpq+strideq*1-8], m1 +%endif + movu [tmpq+strideq*2-8], m2 + movu [tmpq+stride3q -8], m4 + lea dstq, [dstq+strideq*8] +%else ; %1 != 8 +%if ARCH_X86_64 + SWAP 6, 8 + SWAP 1, 9 + SWAP 2, 10 +%else + mova [rsp+1*16], m6 + mova [rsp+2*16], m1 + mova [rsp+3*16], m2 +%endif + + mova m1, [rsp+ 7*16] + mova m2, [rsp+ 8*16] + mova m4, [rsp+ 9*16] + mova m6, [rsp+10*16] + lea tmpq, [dstq+strideq*4] +%if ARCH_X86_64 + TRANSPOSE8X8W 1, 2, 4, 6, 0, 7, 3, 5, 11 +%else + mova [rsp+7*16], m5 + TRANSPOSE8X8W 1, 2, 4, 6, 0, 7, 3, 5, "", \ + [rsp+7*16], [tmpq+strideq*1-16], a, a +%endif + + mova [dstq+strideq*0-16], m1 + mova [dstq+strideq*1-16], m2 + mova [dstq+strideq*2-16], m4 + mova [dstq+stride3q -16], m6 + mova [tmpq+strideq*0-16], m0 +%if ARCH_X86_64 + mova [tmpq+strideq*1-16], m7 +%endif + mova [tmpq+strideq*2-16], m3 + mova [tmpq+stride3q -16], m5 + +%if ARCH_X86_64 + SWAP 6, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 4, 15 +%else + mova m6, [rsp+1*16] + mova m1, [rsp+2*16] + mova m2, [rsp+3*16] + mova m4, Q3 +%endif + mova m0, [rsp+11*16] + mova m3, [rsp+12*16] + mova m5, [rsp+13*16] +%if ARCH_X86_64 + mova m7, [rsp+14*16] + TRANSPOSE8X8W 6, 1, 2, 4, 0, 3, 5, 7, 8 +%else + TRANSPOSE8X8W 6, 1, 2, 4, 0, 3, 5, 7, "", \ + [rsp+14*16], [tmpq+strideq*1], a, a +%endif + mova [dstq+strideq*0], m6 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m4 + mova [tmpq+strideq*0], m0 +%if ARCH_X86_64 + mova [tmpq+strideq*1], m3 +%endif + mova [tmpq+strideq*2], m5 + mova [tmpq+stride3q ], m7 + lea dstq, [dstq+strideq*8] +%endif ; %1==/!=8 +%endif ; %2==/!=v +%elif %1 == 6 + ; flat6 filter + paddw m3, P1, P0 ; p1+p0 + paddw m3, P2 ; p2+p1+p0 + paddw m6, P2, Q0 ; p2+q0 + paddw m3, m3 ; 2*(p2+p1+p0) + paddw m3, m6 ; p2+2*(p2+p1+p0)+q0 + pmulhrsw m2, m3, [PIC_sym(pw_4096)] + psubw m2, P1 + pand m2, m4 + + paddw m3, Q0 ; p2+2*(p2+p1+p0+q0) + paddw m6, P2, P2 ; 2*p2 + paddw m3, Q1 ; p2+2*(p2+p1+p0+q0)+q1 + psubw m3, m6 ; p2+2*(p1+p0+q0)+q1 + pmulhrsw m5, m3, [PIC_sym(pw_4096)] + psubw m5, P0 + pand m5, m4 + + paddw m3, Q1 ; p2+2*(p1+p0+q0+q1) + paddw m6, P2, P1 ; p2+p1 + paddw m3, Q2 ; p2+2*(p1+p0+q0+q1)+q2 + psubw m3, m6 ; p1+2*(p0+q0+q1)+q2 + pmulhrsw m6, m3, [PIC_sym(pw_4096)] + psubw m6, Q0 + pand m6, m4 + + psubw m3, P1 ; 2*(p0+q0+q1)+q2 +%if ARCH_X86_64 + paddw Q2, Q2 ; q2*2 +%else + mova m0, Q2 + paddw m0, m0 +%endif + psubw m3, P0 ; p0+2*(q0+q1)+q2 +%if ARCH_X86_64 + paddw m3, Q2 ; p0+q*(q0+q1+q2)+q2 +%else + paddw m3, m0 +%endif + pmulhrsw m3, [PIC_sym(pw_4096)] + psubw m3, Q1 + pand m3, m4 + + paddw m2, P1 + paddw m5, P0 + paddw m6, Q0 + paddw m3, Q1 + +%ifidn %2, v + mova [dstq+mstrideq*2], m2 ; p1 + mova [dstq+mstrideq*1], m5 ; p0 + mova [dstq+strideq*0], m6 ; q0 + mova [dstq+strideq*1], m3 ; q1 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 m2, m5, m6, m3, m0 +%endif ; %2==/!=v +%else ; %1 == 4 +%if ARCH_X86_64 +%ifidn %2, v + mova [dstq+mstrideq*2], P1 ; p1 + mova [dstq+mstrideq*1], P0 ; p0 + mova [dstq+strideq*0], Q0 ; q0 + mova [dstq+strideq*1], Q1 ; q1 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 P1, P0, Q0, Q1, m0 +%endif ; %2==/!=v +%else ; x86-32 +%ifidn %2, v + mova [dstq+mstrideq*2], m3 + mova [dstq+mstrideq*1], m5 + mova [dstq+strideq*0], m6 + mova [dstq+strideq*1], m7 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 m3, m5, m6, m7, m0 +%endif ; %2==/!=v +%endif ; x86-32/64 +%endif ; %1 +%undef P3 +%undef P2 +%undef P1 +%undef P0 +%undef Q0 +%undef Q1 +%undef Q2 +%undef Q3 +%endmacro + +INIT_XMM ssse3 +; stack layout: +; r0 - flat8 backup inside flat16 code +%if ARCH_X86_64 +cglobal lpf_v_sb_y_16bpc, 6, 12, 16, -16 * 1, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits, bdmul + mov r6d, r7m + sar r6d, 7 + and r6d, 16 ; 0 for 10bpc, 16 for 12bpc + lea bdmulq, [pw_4] + add bdmulq, r6 + mov wd, wm + shl l_strideq, 2 + sub lq, l_strideq +%else +; stack layout [32bit only]: +; r1-4 - p2-q0 post-filter16 +; r5 - p3 +; r6 - q3 post-filter16 +; r7 - GPRs [mask_bitsm, mstridem] +; r8 - m12/pb_mask +; r9 - bdmulq +cglobal lpf_v_sb_y_16bpc, 4, 7, 8, -16 * (10 + extra_stack), \ + dst, stride, mask, mstride, pic_reg, stride3, tmp + RELOC_ARGS v, 10*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base +%define pic_regm dword [esp+7*16+2*gprsize] + mov pic_regm, pic_regq + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+9*16 + mova [bdmulq], m0 + shl dword lstridem, 2 + sub r3, dword lstridem + mov dword lm, r3 +%endif + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] +%if ARCH_X86_64 + mov mask_bitsd, 0x3 + mova m12, [pb_mask] +%else +%define mstridem dword [esp+7*16+1*gprsize] + mov mstridem, mstrideq +%define mask_bitsm dword [esp+7*16+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+8*16] + mova m12, m0 +%endif + +.loop: +%if ARCH_X86_64 + test [maskq+8], mask_bitsd ; vmask[2] +%else + mov r6d, mask_bitsm + test [maskq+8], r6d +%endif + jz .no_flat16 + + FILTER 16, v + jmp .end + +.no_flat16: +%if ARCH_X86_64 + test [maskq+4], mask_bitsd ; vmask[1] +%else + test [maskq+4], r6d +%endif + jz .no_flat + + FILTER 8, v + jmp .end + +.no_flat: +%if ARCH_X86_64 + test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif + jz .end + + FILTER 4, v + +.end: +%if ARCH_X86_64 + pslld m12, 2 + add lq, 8 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add dword lm, 8 +%endif + add dstq, 16 +%if ARCH_X86_64 + shl mask_bitsd, 2 + sub wd, 2 +%else + shl mask_bitsm, 2 + sub dword wm, 2 +%endif + jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS + RET + +INIT_XMM ssse3 +; stack layout: +; r0 - flat8 backup inside flat16 +; r1-4 - p2-q0 post-filter16 backup +; r5 - q3 post-filter16 backup +; r6 - p3 +; r7-10 - p7-4 +; r11-14 - q4-7 +%if ARCH_X86_64 +cglobal lpf_h_sb_y_16bpc, 6, 11, 16, -16 * 15, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, tmp, mask_bits, bdmul + mov r6d, r7m + sar r6d, 7 + and r6d, 16 ; 0 for 10bpc, 16 for 12bpc + lea bdmulq, [pw_4] + add bdmulq, r6 + mov hd, hm + shl l_strideq, 2 +%else +; stack layout [32bit only]: +; r15 - GPRs [mask_bitsm] +; r16 - m12/pb_mask +; r17 - bdmulq +; r18-24 - p2-q3 +cglobal lpf_h_sb_y_16bpc, 4, 7, 8, -16 * (25 + extra_stack), \ + dst, stride, mask, l, pic_reg, stride3, tmp + RELOC_ARGS h, 25*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+17*16 + mova [bdmulq], m0 + shl dword lstridem, 2 +%endif + sub lq, 4 + lea stride3q, [strideq*3] +%if ARCH_X86_64 + mov mask_bitsd, 0x3 + mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+15*16+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+16*16] + mova m12, m0 +%endif + +.loop: +%if ARCH_X86_64 + test [maskq+8], mask_bitsd ; vmask[2] +%else + mov r6d, mask_bitsm + test [maskq+8], r6d +%endif + jz .no_flat16 + + FILTER 16, h + jmp .end + +.no_flat16: +%if ARCH_X86_64 + test [maskq+4], mask_bitsd ; vmask[1] +%else + test [maskq+4], r6d +%endif + jz .no_flat + + FILTER 8, h + jmp .end + +.no_flat: +%if ARCH_X86_64 + test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif + jz .no_filter + + FILTER 4, h + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] +.end: +%if ARCH_X86_64 + pslld m12, 2 + lea lq, [lq+l_strideq*2] + shl mask_bitsd, 2 + sub hd, 2 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add lq, dword lstridem + add lq, dword lstridem + shl mask_bitsm, 2 + sub dword hm, 2 +%endif + jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS + RET + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_v_sb_uv_16bpc, 6, 12, 16, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits, bdmul + mov r6d, r7m + sar r6d, 7 + and r6d, 16 ; 0 for 10bpc, 16 for 12bpc + lea bdmulq, [pw_4] + add bdmulq, r6 + mov wd, wm + shl l_strideq, 2 + sub lq, l_strideq +%else +; stack layout [32bit only]: +; r0 - GPRs [mask_bitsm, mstridem] +; r1 - m12/pb_mask +; r2 - bdmulq +cglobal lpf_v_sb_uv_16bpc, 4, 7, 8, -16 * (3 + extra_stack), \ + dst, stride, mask, mstride, pic_reg, stride3, tmp + RELOC_ARGS v, 3*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+2*16 + mova [bdmulq], m0 + shl dword lstridem, 2 + sub r3, dword lstridem + mov dword lm, r3 +%endif + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] +%if ARCH_X86_64 + mov mask_bitsd, 0x3 + mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+0*gprsize] +%define mstridem dword [esp+1*gprsize] + mov mask_bitsm, 0x3 + mov mstridem, mstrideq + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+1*16] + mova m12, m0 +%endif + +.loop: +%if ARCH_X86_64 + test [maskq+4], mask_bitsd ; vmask[1] +%else + mov r6d, mask_bitsm + test [maskq+4], r6d +%endif + jz .no_flat + + FILTER 6, v + jmp .end + +.no_flat: +%if ARCH_X86_64 + test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif + jz .end + + FILTER 4, v + +.end: +%if ARCH_X86_64 + pslld m12, 2 + add lq, 8 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add dword lm, 8 +%endif + add dstq, 16 +%if ARCH_X86_64 + shl mask_bitsd, 2 + sub wd, 2 +%else + shl mask_bitsm, 2 + sub dword wm, 2 +%endif + jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS + RET + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_h_sb_uv_16bpc, 6, 11, 16, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, tmp, mask_bits, bdmul + mov r6d, r7m + sar r6d, 7 + and r6d, 16 ; 0 for 10bpc, 16 for 12bpc + lea bdmulq, [pw_4] + add bdmulq, r6 + mov hd, hm + shl l_strideq, 2 +%else +; stack layout [32bit only]: +; r0 - GPRs [mask_bitsm] +; r1 - m12/pb_mask +; r2 - bdmulq +; r3-8 - p2-q2 +cglobal lpf_h_sb_uv_16bpc, 4, 7, 8, -16 * (9 + extra_stack), \ + dst, stride, mask, l, pic_reg, stride3, tmp + RELOC_ARGS h, 9*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+2*16 + mova [bdmulq], m0 + shl dword lstridem, 2 +%endif + sub lq, 4 + lea stride3q, [strideq*3] +%if ARCH_X86_64 + mov mask_bitsd, 0x3 + mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+1*16] + mova m12, m0 +%endif + +.loop: +%if ARCH_X86_64 + test [maskq+4], mask_bitsd ; vmask[1] +%else + mov r6d, mask_bitsm + test [maskq+4], r6d +%endif + jz .no_flat + + FILTER 6, h + jmp .end + +.no_flat: +%if ARCH_X86_64 + test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif + jz .no_filter + + FILTER 4, h + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] +.end: +%if ARCH_X86_64 + pslld m12, 2 + lea lq, [lq+l_strideq*2] + shl mask_bitsd, 2 + sub hd, 2 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add lq, dword lstridem + add lq, dword lstridem + shl mask_bitsm, 2 + sub dword hm, 2 +%endif + jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS + RET diff --git a/third_party/dav1d/src/x86/loopfilter_avx2.asm b/third_party/dav1d/src/x86/loopfilter_avx2.asm new file mode 100644 index 0000000000..84696c758a --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter_avx2.asm @@ -0,0 +1,1569 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +pb_4x1_4x5_4x9_4x13: times 2 db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 +pb_7_1: times 16 db 7, 1 +pb_3_1: times 16 db 3, 1 +pb_2_1: times 16 db 2, 1 +pb_m1_0: times 16 db -1, 0 +pb_m1_1: times 16 db -1, 1 +pb_m1_2: times 16 db -1, 2 +pb_1: times 32 db 1 +pb_2: times 32 db 2 +pb_3: times 32 db 3 +pb_4: times 32 db 4 +pb_16: times 32 db 16 +pb_63: times 32 db 63 +pb_64: times 32 db 64 +pb_128: times 32 db 0x80 +pb_129: times 32 db 0x81 +pb_240: times 32 db 0xf0 +pb_248: times 32 db 0xf8 +pb_254: times 32 db 0xfe + +pw_2048: times 16 dw 2048 +pw_4096: times 16 dw 4096 + +pb_mask: dd 1, 2, 4, 8, 16, 32, 64, 128 + +SECTION .text + +%macro ABSSUB 4 ; dst, a, b, tmp + psubusb %1, %2, %3 + psubusb %4, %3, %2 + por %1, %4 +%endmacro + +%macro TRANSPOSE_16x4_AND_WRITE_4x32 5 + ; transpose 16x4 + punpcklbw m%5, m%1, m%2 + punpckhbw m%1, m%2 + punpcklbw m%2, m%3, m%4 + punpckhbw m%3, m%4 + punpcklwd m%4, m%5, m%2 + punpckhwd m%5, m%2 + punpcklwd m%2, m%1, m%3 + punpckhwd m%1, m%3 + + ; write out + movd [dstq+strideq*0-2], xm%4 + pextrd [dstq+strideq*1-2], xm%4, 1 + pextrd [dstq+strideq*2-2], xm%4, 2 + pextrd [dstq+stride3q-2], xm%4, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%5 + pextrd [dstq+strideq*1-2], xm%5, 1 + pextrd [dstq+strideq*2-2], xm%5, 2 + pextrd [dstq+stride3q-2], xm%5, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%2 + pextrd [dstq+strideq*1-2], xm%2, 1 + pextrd [dstq+strideq*2-2], xm%2, 2 + pextrd [dstq+stride3q-2], xm%2, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%1 + pextrd [dstq+strideq*1-2], xm%1, 1 + pextrd [dstq+strideq*2-2], xm%1, 2 + pextrd [dstq+stride3q-2], xm%1, 3 + lea dstq, [dstq+strideq*4] + + vextracti128 xm%4, m%4, 1 + vextracti128 xm%5, m%5, 1 + vextracti128 xm%2, m%2, 1 + vextracti128 xm%1, m%1, 1 + + movd [dstq+strideq*0-2], xm%4 + pextrd [dstq+strideq*1-2], xm%4, 1 + pextrd [dstq+strideq*2-2], xm%4, 2 + pextrd [dstq+stride3q-2], xm%4, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%5 + pextrd [dstq+strideq*1-2], xm%5, 1 + pextrd [dstq+strideq*2-2], xm%5, 2 + pextrd [dstq+stride3q-2], xm%5, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%2 + pextrd [dstq+strideq*1-2], xm%2, 1 + pextrd [dstq+strideq*2-2], xm%2, 2 + pextrd [dstq+stride3q-2], xm%2, 3 + lea dstq, [dstq+strideq*4] + movd [dstq+strideq*0-2], xm%1 + pextrd [dstq+strideq*1-2], xm%1, 1 + pextrd [dstq+strideq*2-2], xm%1, 2 + pextrd [dstq+stride3q-2], xm%1, 3 + lea dstq, [dstq+strideq*4] +%endmacro + +%macro TRANSPOSE_16X16B 3 ; in_load_15_from_mem, out_store_0_in_mem, mem +%if %1 == 0 + mova %3, m15 +%endif + + ; input in m0-15 + punpcklbw m15, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + punpcklbw m3, m4, m5 + punpckhbw m4, m5 + punpcklbw m5, m6, m7 + punpckhbw m6, m7 + punpcklbw m7, m8, m9 + punpckhbw m8, m9 + punpcklbw m9, m10, m11 + punpckhbw m10, m11 + punpcklbw m11, m12, m13 + punpckhbw m12, m13 + mova m13, %3 + mova %3, m12 + punpcklbw m12, m14, m13 + punpckhbw m13, m14, m13 + + ; interleaved in m15,0,1,2,3,4,5,6,7,8,9,10,11,rsp%3,12,13 + punpcklwd m14, m15, m1 + punpckhwd m15, m1 + punpcklwd m1, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m6 + punpckhwd m4, m6 + punpcklwd m6, m7, m9 + punpckhwd m7, m9 + punpcklwd m9, m8, m10 + punpckhwd m8, m10 + punpcklwd m10, m11, m12 + punpckhwd m11, m12 + mova m12, %3 + mova %3, m11 + punpcklwd m11, m12, m13 + punpckhwd m12, m13 + + ; interleaved in m14,15,1,0,2,3,5,4,6,7,9,8,10,rsp%3,11,12 + punpckldq m13, m14, m2 + punpckhdq m14, m2 + punpckldq m2, m15, m3 + punpckhdq m15, m3 + punpckldq m3, m1, m5 + punpckhdq m1, m5 + punpckldq m5, m0, m4 + punpckhdq m0, m4 + punpckldq m4, m6, m10 + punpckhdq m6, m10 + punpckldq m10, m9, m11 + punpckhdq m9, m11 + punpckldq m11, m8, m12 + punpckhdq m8, m12 + mova m12, %3 + mova %3, m8 + punpckldq m8, m7, m12 + punpckhdq m7, m12 + + ; interleaved in m13,14,2,15,3,1,5,0,4,6,8,7,10,9,11,rsp%3 + punpcklqdq m12, m13, m4 + punpckhqdq m13, m4 + punpcklqdq m4, m14, m6 + punpckhqdq m14, m6 + punpcklqdq m6, m2, m8 + punpckhqdq m2, m8 + punpcklqdq m8, m15, m7 + punpckhqdq m15, m7 + punpcklqdq m7, m3, m10 + punpckhqdq m3, m10 + punpcklqdq m10, m1, m9 + punpckhqdq m1, m9 + punpcklqdq m9, m5, m11 + punpckhqdq m5, m11 + mova m11, %3 + mova %3, m12 + punpcklqdq m12, m0, m11 + punpckhqdq m0, m11 +%if %2 == 0 + mova m11, %3 +%endif + + ; interleaved m11,13,4,14,6,2,8,15,7,3,10,1,9,5,12,0 + SWAP 0, 11, 1, 13, 5, 2, 4, 6, 8, 7, 15 + SWAP 3, 14, 12, 9 +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] + ; load data +%ifidn %2, v +%if %1 == 4 + lea tmpq, [dstq+mstrideq*2] + mova m3, [tmpq+strideq*0] ; p1 + mova m4, [tmpq+strideq*1] ; p0 + mova m5, [tmpq+strideq*2] ; q0 + mova m6, [tmpq+stride3q] ; q1 +%else + ; load 6-8 pixels, remainder (for wd=16) will be read inline + lea tmpq, [dstq+mstrideq*4] +%if %1 != 6 + mova m12, [tmpq+strideq*0] +%endif + mova m13, [tmpq+strideq*1] + mova m3, [tmpq+strideq*2] + mova m4, [tmpq+stride3q] + mova m5, [dstq+strideq*0] + mova m6, [dstq+strideq*1] + mova m14, [dstq+strideq*2] +%if %1 != 6 + mova m15, [dstq+stride3q] +%endif +%endif +%else + ; load lines +%if %1 == 4 + movd xm3, [dstq+strideq*0-2] + movd xm4, [dstq+strideq*1-2] + movd xm5, [dstq+strideq*2-2] + movd xm6, [dstq+stride3q -2] + lea tmpq, [dstq+strideq*4] + pinsrd xm3, [tmpq+strideq*0-2], 2 + pinsrd xm4, [tmpq+strideq*1-2], 2 + pinsrd xm5, [tmpq+strideq*2-2], 2 + pinsrd xm6, [tmpq+stride3q -2], 2 + lea tmpq, [tmpq+strideq*4] + pinsrd xm3, [tmpq+strideq*0-2], 1 + pinsrd xm4, [tmpq+strideq*1-2], 1 + pinsrd xm5, [tmpq+strideq*2-2], 1 + pinsrd xm6, [tmpq+stride3q -2], 1 + lea tmpq, [tmpq+strideq*4] + pinsrd xm3, [tmpq+strideq*0-2], 3 + pinsrd xm4, [tmpq+strideq*1-2], 3 + pinsrd xm5, [tmpq+strideq*2-2], 3 + pinsrd xm6, [tmpq+stride3q -2], 3 + lea tmpq, [tmpq+strideq*4] + movd xm12, [tmpq+strideq*0-2] + movd xm13, [tmpq+strideq*1-2] + movd xm14, [tmpq+strideq*2-2] + movd xm15, [tmpq+stride3q -2] + lea tmpq, [tmpq+strideq*4] + pinsrd xm12, [tmpq+strideq*0-2], 2 + pinsrd xm13, [tmpq+strideq*1-2], 2 + pinsrd xm14, [tmpq+strideq*2-2], 2 + pinsrd xm15, [tmpq+stride3q -2], 2 + lea tmpq, [tmpq+strideq*4] + pinsrd xm12, [tmpq+strideq*0-2], 1 + pinsrd xm13, [tmpq+strideq*1-2], 1 + pinsrd xm14, [tmpq+strideq*2-2], 1 + pinsrd xm15, [tmpq+stride3q -2], 1 + lea tmpq, [tmpq+strideq*4] + pinsrd xm12, [tmpq+strideq*0-2], 3 + pinsrd xm13, [tmpq+strideq*1-2], 3 + pinsrd xm14, [tmpq+strideq*2-2], 3 + pinsrd xm15, [tmpq+stride3q -2], 3 + vinserti128 m3, xm12, 1 + vinserti128 m4, xm13, 1 + vinserti128 m5, xm14, 1 + vinserti128 m6, xm15, 1 + + ; transpose 4x16 + ; xm3: A-D0,A-D8,A-D4,A-D12 + ; xm4: A-D1,A-D9,A-D5,A-D13 + ; xm5: A-D2,A-D10,A-D6,A-D14 + ; xm6: A-D3,A-D11,A-D7,A-D15 + punpcklbw m7, m3, m4 + punpckhbw m3, m4 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + ; xm7: A0-1,B0-1,C0-1,D0-1,A8-9,B8-9,C8-9,D8-9 + ; xm3: A4-5,B4-5,C4-5,D4-5,A12-13,B12-13,C12-13,D12-13 + ; xm4: A2-3,B2-3,C2-3,D2-3,A10-11,B10-11,C10-11,D10-11 + ; xm5: A6-7,B6-7,C6-7,D6-7,A14-15,B14-15,C14-15,D14-15 + punpcklwd m6, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m3, m5 + punpckhwd m3, m5 + ; xm6: A0-3,B0-3,C0-3,D0-3 + ; xm7: A8-11,B8-11,C8-11,D8-11 + ; xm4: A4-7,B4-7,C4-7,D4-7 + ; xm3: A12-15,B12-15,C12-15,D12-15 + punpckldq m5, m6, m4 + punpckhdq m6, m4 + punpckldq m4, m7, m3 + punpckhdq m7, m3 + ; xm5: A0-7,B0-7 + ; xm6: C0-7,D0-7 + ; xm4: A8-15,B8-15 + ; xm7: C8-15,D8-15 + punpcklqdq m3, m5, m4 + punpckhqdq m4, m5, m4 + punpcklqdq m5, m6, m7 + punpckhqdq m6, m7 + ; xm3: A0-15 + ; xm5: B0-15 + ; xm4: C0-15 + ; xm6: D0-15 +%elif %1 == 6 || %1 == 8 + movq xm3, [dstq+strideq*0-%1/2] + movq xm4, [dstq+strideq*1-%1/2] + movq xm5, [dstq+strideq*2-%1/2] + movq xm6, [dstq+stride3q -%1/2] + lea tmpq, [dstq+strideq*8] + movhps xm3, [tmpq+strideq*0-%1/2] + movhps xm4, [tmpq+strideq*1-%1/2] + movhps xm5, [tmpq+strideq*2-%1/2] + movhps xm6, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + movq xm7, [tmpq+strideq*0-%1/2] + movq xm8, [tmpq+strideq*1-%1/2] + movq xm9, [tmpq+strideq*2-%1/2] + movq xm11, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + movhps xm7, [tmpq+strideq*0-%1/2] + movhps xm8, [tmpq+strideq*1-%1/2] + movhps xm9, [tmpq+strideq*2-%1/2] + movhps xm11, [tmpq+stride3q -%1/2] + vinserti128 m3, xm7, 1 + vinserti128 m4, xm8, 1 + vinserti128 m5, xm9, 1 + vinserti128 m6, xm11, 1 + lea tmpq, [dstq+strideq*4] + movq xm12, [tmpq+strideq*0-%1/2] + movq xm13, [tmpq+strideq*1-%1/2] + movq xm14, [tmpq+strideq*2-%1/2] + movq xm15, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + movhps xm12, [tmpq+strideq*0-%1/2] + movhps xm13, [tmpq+strideq*1-%1/2] + movhps xm14, [tmpq+strideq*2-%1/2] + movhps xm15, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + movq xm7, [tmpq+strideq*0-%1/2] + movq xm8, [tmpq+strideq*1-%1/2] + movq xm9, [tmpq+strideq*2-%1/2] + movq xm11, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + movhps xm7, [tmpq+strideq*0-%1/2] + movhps xm8, [tmpq+strideq*1-%1/2] + movhps xm9, [tmpq+strideq*2-%1/2] + movhps xm11, [tmpq+stride3q -%1/2] + vinserti128 m12, xm7, 1 + vinserti128 m13, xm8, 1 + vinserti128 m14, xm9, 1 + vinserti128 m15, xm11, 1 + + ; transpose 8x16 + ; xm3: A-H0,A-H8 + ; xm4: A-H1,A-H9 + ; xm5: A-H2,A-H10 + ; xm6: A-H3,A-H11 + ; xm12: A-H4,A-H12 + ; xm13: A-H5,A-H13 + ; xm14: A-H6,A-H14 + ; xm15: A-H7,A-H15 + punpcklbw m7, m3, m4 + punpckhbw m3, m4 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + punpcklbw m6, m12, m13 + punpckhbw m12, m13 + punpcklbw m13, m14, m15 + punpckhbw m14, m15 + ; xm7: A0-1,B0-1,C0-1,D0-1,E0-1,F0-1,G0-1,H0-1 + ; xm3: A8-9,B8-9,C8-9,D8-9,E8-9,F8-9,G8-9,H8-9 + ; xm4: A2-3,B2-3,C2-3,D2-3,E2-3,F2-3,G2-3,H2-3 + ; xm5: A10-11,B10-11,C10-11,D10-11,E10-11,F10-11,G10-11,H10-11 + ; xm6: A4-5,B4-5,C4-5,D4-5,E4-5,F4-5,G4-5,H4-5 + ; xm12: A12-13,B12-13,C12-13,D12-13,E12-13,F12-13,G12-13,H12-13 + ; xm13: A6-7,B6-7,C6-7,D6-7,E6-7,F6-7,G6-7,H6-7 + ; xm14: A14-15,B14-15,C14-15,D14-15,E14-15,F14-15,G14-15,H14-15 + punpcklwd m15, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m6, m13 + punpckhwd m6, m13 + punpcklwd m13, m12, m14 + punpckhwd m12, m14 + ; xm15: A0-3,B0-3,C0-3,D0-3 + ; xm7: E0-3,F0-3,G0-3,H0-3 + ; xm4: A8-11,B8-11,C8-11,D8-11 + ; xm3: E8-11,F8-11,G8-11,H8-11 + ; xm5: A4-7,B4-7,C4-7,D4-7 + ; xm6: E4-7,F4-7,G4-7,H4-7 + ; xm13: A12-15,B12-15,C12-15,D12-15 + ; xm12: E12-15,F12-15,G12-15,H12-15 + punpckldq m14, m15, m5 + punpckhdq m15, m5 + punpckldq m5, m7, m6 +%if %1 != 6 + punpckhdq m7, m6 +%endif + punpckldq m6, m4, m13 + punpckhdq m4, m13 + punpckldq m13, m3, m12 +%if %1 != 6 + punpckhdq m12, m3, m12 +%endif + ; xm14: A0-7,B0-7 + ; xm15: C0-7,D0-7 + ; xm5: E0-7,F0-7 + ; xm7: G0-7,H0-7 + ; xm6: A8-15,B8-15 + ; xm4: C8-15,D8-15 + ; xm13: E8-15,F8-15 + ; xm12: G8-15,H8-15 + punpcklqdq m3, m14, m6 + punpckhqdq m14, m6 + punpckhqdq m6, m15, m4 + punpcklqdq m15, m4 + punpcklqdq m4, m5, m13 + punpckhqdq m13, m5, m13 +%if %1 == 8 + punpcklqdq m5, m7, m12 + punpckhqdq m12, m7, m12 + ; xm3: A0-15 + ; xm14: B0-15 + ; xm15: C0-15 + ; xm6: D0-15 + ; xm4: E0-15 + ; xm13: F0-15 + ; xm5: G0-15 + ; xm12: H0-15 + SWAP 12, 3, 15 + SWAP 13, 14, 5, 4, 6 + ; 3,14,15,6,4,13,5,12 -> 12,13,3,4,5,6,14,15 +%else + SWAP 13, 3, 14 + SWAP 6, 4, 15, 5 + ; 3,14,15,6,4,13 -> 13,3,4,5,6,14 +%endif +%else + ; load and 16x16 transpose. We only use 14 pixels but we'll need the + ; remainder at the end for the second transpose + movu xm0, [dstq+strideq*0-8] + movu xm1, [dstq+strideq*1-8] + movu xm2, [dstq+strideq*2-8] + movu xm3, [dstq+stride3q -8] + lea tmpq, [dstq+strideq*4] + movu xm4, [tmpq+strideq*0-8] + movu xm5, [tmpq+strideq*1-8] + movu xm6, [tmpq+strideq*2-8] + movu xm7, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + movu xm8, [tmpq+strideq*0-8] + movu xm9, [tmpq+strideq*1-8] + movu xm10, [tmpq+strideq*2-8] + movu xm11, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + movu xm12, [tmpq+strideq*0-8] + movu xm13, [tmpq+strideq*1-8] + movu xm14, [tmpq+strideq*2-8] + movu xm15, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + vinserti128 m0, [tmpq+strideq*0-8], 1 + vinserti128 m1, [tmpq+strideq*1-8], 1 + vinserti128 m2, [tmpq+strideq*2-8], 1 + vinserti128 m3, [tmpq+stride3q -8], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m4, [tmpq+strideq*0-8], 1 + vinserti128 m5, [tmpq+strideq*1-8], 1 + vinserti128 m6, [tmpq+strideq*2-8], 1 + vinserti128 m7, [tmpq+stride3q -8], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m8, [tmpq+strideq*0-8], 1 + vinserti128 m9, [tmpq+strideq*1-8], 1 + vinserti128 m10, [tmpq+strideq*2-8], 1 + vinserti128 m11, [tmpq+stride3q -8], 1 + lea tmpq, [tmpq+strideq*4] + vinserti128 m12, [tmpq+strideq*0-8], 1 + vinserti128 m13, [tmpq+strideq*1-8], 1 + vinserti128 m14, [tmpq+strideq*2-8], 1 + vinserti128 m15, [tmpq+stride3q -8], 1 + + TRANSPOSE_16X16B 0, 1, [rsp+11*32] + mova [rsp+12*32], m1 + mova [rsp+13*32], m2 + mova [rsp+14*32], m3 + mova [rsp+15*32], m12 + mova [rsp+16*32], m13 + mova [rsp+17*32], m14 + mova [rsp+18*32], m15 + ; 4,5,6,7,8,9,10,11 -> 12,13,3,4,5,6,14,15 + SWAP 12, 4, 7 + SWAP 13, 5, 8 + SWAP 3, 6, 9 + SWAP 10, 14 + SWAP 11, 15 +%endif +%endif + + ; load L/E/I/H +%ifidn %2, v + movu m1, [lq] + movu m0, [lq+l_strideq] +%else + movq xm1, [lq] + movq xm2, [lq+l_strideq*2] + movhps xm1, [lq+l_strideq] + movhps xm2, [lq+l_stride3q] + lea lq, [lq+l_strideq*4] + movq xm10, [lq] + movq xm0, [lq+l_strideq*2] + movhps xm10, [lq+l_strideq] + movhps xm0, [lq+l_stride3q] + lea lq, [lq+l_strideq*4] + vinserti128 m1, xm10, 1 + vinserti128 m2, xm0, 1 + shufps m0, m1, m2, q3131 + shufps m1, m2, q2020 +%endif + pxor m2, m2 + pcmpeqb m10, m2, m0 + pand m1, m10 + por m0, m1 ; l[x][] ? l[x][] : l[x-stride][] + pshufb m0, [pb_4x1_4x5_4x9_4x13] ; l[x][1] + pcmpeqb m10, m2, m0 ; !L + psrlq m2, m0, [lutq+128] + pand m2, [pb_63] + vpbroadcastb m1, [lutq+136] + pminub m2, m1 + pmaxub m2, [pb_1] ; I + pand m1, m0, [pb_240] + psrlq m1, 4 ; H + paddb m0, [pb_2] + paddb m0, m0 + paddb m0, m2 ; E + pxor m1, [pb_128] + pxor m2, [pb_128] + pxor m0, [pb_128] + + ABSSUB m8, m3, m4, m9 ; abs(p1-p0) + pmaxub m8, m10 + ABSSUB m9, m5, m6, m10 ; abs(q1-q0) + pmaxub m8, m9 +%if %1 == 4 + pxor m8, [pb_128] + pcmpgtb m7, m8, m1 ; hev +%else + pxor m7, m8, [pb_128] + pcmpgtb m7, m1 ; hev + +%if %1 == 6 + ABSSUB m9, m13, m4, m10 ; abs(p2-p0) + pmaxub m9, m8 +%else + ABSSUB m9, m12, m4, m10 ; abs(p3-p0) + pmaxub m9, m8 + ABSSUB m10, m13, m4, m11 ; abs(p2-p0) + pmaxub m9, m10 +%endif + ABSSUB m10, m5, m14, m11 ; abs(q2-q0) + pmaxub m9, m10 +%if %1 != 6 + ABSSUB m10, m5, m15, m11 ; abs(q3-q0) + pmaxub m9, m10 +%endif + pxor m9, [pb_128] + pcmpgtb m9, [pb_129] ; !flat8in + +%if %1 == 6 + ABSSUB m10, m13, m3, m1 ; abs(p2-p1) +%else + ABSSUB m10, m12, m13, m11 ; abs(p3-p2) + ABSSUB m11, m13, m3, m1 ; abs(p2-p1) + pmaxub m10, m11 + ABSSUB m11, m14, m15, m1 ; abs(q3-q2) + pmaxub m10, m11 +%endif + ABSSUB m11, m14, m6, m1 ; abs(q2-q1) + pmaxub m10, m11 +%if %1 == 16 + vpbroadcastd m11, [maskq+8] + vpbroadcastd m1, [maskq+4] + por m11, m1 + pand m11, [pb_mask] + pcmpeqd m11, [pb_mask] + pand m10, m11 +%else + vpbroadcastd m11, [maskq+4] + pand m11, [pb_mask] + pcmpeqd m11, [pb_mask] + pand m10, m11 ; only apply fm-wide to wd>4 blocks +%endif + pmaxub m8, m10 + + pxor m8, [pb_128] +%endif + pcmpgtb m8, m2 + + ABSSUB m10, m3, m6, m11 ; abs(p1-q1) + ABSSUB m11, m4, m5, m2 ; abs(p0-q0) + paddusb m11, m11 + pand m10, [pb_254] + psrlq m10, 1 + paddusb m10, m11 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + pxor m10, [pb_128] + pcmpgtb m10, m0 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E + por m8, m10 + +%if %1 == 16 +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] +%else + mova m0, [rsp+12*32] +%endif + ABSSUB m1, m0, m4, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*2] +%else + mova m0, [rsp+13*32] +%endif + ABSSUB m2, m0, m4, m10 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+stride3q] +%else + mova m0, [rsp+14*32] +%endif + ABSSUB m2, m0, m4, m10 + pmaxub m1, m2 +%ifidn %2, v + lea tmpq, [dstq+strideq*4] + mova m0, [tmpq+strideq*0] +%else + mova m0, [rsp+15*32] +%endif + ABSSUB m2, m0, m5, m10 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*1] +%else + mova m0, [rsp+16*32] +%endif + ABSSUB m2, m0, m5, m10 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*2] +%else + mova m0, [rsp+17*32] +%endif + ABSSUB m2, m0, m5, m10 + pmaxub m1, m2 + pxor m1, [pb_128] + pcmpgtb m1, [pb_129] ; !flat8out + por m1, m9 ; !flat8in | !flat8out + vpbroadcastd m2, [maskq+8] + pand m10, m2, [pb_mask] + pcmpeqd m10, [pb_mask] + pandn m1, m10 ; flat16 + pandn m1, m8, m1 ; flat16 & fm + + vpbroadcastd m10, [maskq+4] + por m10, m2 + pand m2, m10, [pb_mask] + pcmpeqd m2, [pb_mask] + pandn m9, m2 ; flat8in + pandn m9, m8, m9 + vpbroadcastd m2, [maskq+0] + por m2, m10 + pand m2, [pb_mask] + pcmpeqd m2, [pb_mask] + pandn m8, m2 + pandn m8, m9, m8 ; fm & !flat8 & !flat16 + pandn m9, m1, m9 ; flat8 & !flat16 +%elif %1 != 4 + vpbroadcastd m0, [maskq+4] + pand m2, m0, [pb_mask] + pcmpeqd m2, [pb_mask] + pandn m9, m2 + pandn m9, m8, m9 ; flat8 & fm + vpbroadcastd m2, [maskq+0] + por m0, m2 + pand m0, [pb_mask] + pcmpeqd m0, [pb_mask] + pandn m8, m0 + pandn m8, m9, m8 ; fm & !flat8 +%else + vpbroadcastd m0, [maskq+0] + pand m0, [pb_mask] + pcmpeqd m0, [pb_mask] + pandn m8, m0 ; fm +%endif + + ; short filter + + pxor m3, [pb_128] + pxor m6, [pb_128] + psubsb m10, m3, m6 ; iclip_diff(p1-q1) + pand m10, m7 ; f=iclip_diff(p1-q1)&hev + pxor m4, [pb_128] + pxor m5, [pb_128] + psubsb m11, m5, m4 + paddsb m10, m11 + paddsb m10, m11 + paddsb m10, m11 ; f=iclip_diff(3*(q0-p0)+f) + pand m8, m10 ; f&=fm + paddsb m10, m8, [pb_3] + paddsb m8, [pb_4] + pand m10, [pb_248] + pand m8, [pb_248] + psrlq m10, 3 + psrlq m8, 3 + pxor m10, [pb_16] + pxor m8, [pb_16] + psubb m10, [pb_16] ; f2 + psubb m8, [pb_16] ; f1 + paddsb m4, m10 + psubsb m5, m8 + pxor m4, [pb_128] + pxor m5, [pb_128] + + pxor m8, [pb_128] + pxor m10, m10 + pavgb m8, m10 ; f=(f1+1)>>1 + psubb m8, [pb_64] + pandn m8, m7, m8 ; f&=!hev + paddsb m3, m8 + psubsb m6, m8 + pxor m3, [pb_128] + pxor m6, [pb_128] + +%if %1 == 16 + ; flat16 filter +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] ; p6 + mova m2, [tmpq+strideq*2] ; p5 + mova m7, [tmpq+stride3q] ; p4 +%else + mova m0, [rsp+12*32] + mova m2, [rsp+13*32] + mova m7, [rsp+14*32] +%endif + + mova [rsp+0*32], m9 + mova [rsp+1*32], m14 + mova [rsp+2*32], m15 + + ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 [p5/p4/p2/p1/p0/q0][p6/p3] A + ; write -6 + punpcklbw m14, m0, m12 + punpckhbw m15, m0, m12 + pmaddubsw m10, m14, [pb_7_1] + pmaddubsw m11, m15, [pb_7_1] ; p6*7+p3 + punpcklbw m8, m2, m7 + punpckhbw m9, m2, m7 + pmaddubsw m8, [pb_2] + pmaddubsw m9, [pb_2] + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3 + punpcklbw m8, m13, m3 + punpckhbw m9, m13, m3 + pmaddubsw m8, [pb_1] + pmaddubsw m9, [pb_1] + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3+p2+p1 + punpcklbw m8, m4, m5 + punpckhbw m9, m4, m5 + pmaddubsw m8, [pb_1] + pmaddubsw m9, [pb_1] + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + pand m8, m1 + pandn m9, m1, m2 + por m8, m9 +%ifidn %2, v + mova [tmpq+strideq*2], m8 ; p5 +%else + mova [rsp+13*32], m8 +%endif + + ; sub p6*2, add p3/q1 [reuse p6/p3 from A][-p6,+q1|save] B + ; write -5 + pmaddubsw m14, [pb_m1_1] + pmaddubsw m15, [pb_m1_1] + paddw m10, m14 + paddw m11, m15 ; p6*6+p5*2+p4*2+p3*2+p2+p1+p0+q0 + punpcklbw m8, m0, m6 + punpckhbw m9, m0, m6 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + mova [rsp+3*32], m8 + mova [rsp+4*32], m9 + paddw m10, m8 + paddw m11, m9 ; p6*5+p5*2+p4*2+p3*2+p2+p1+p0+q0+q1 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m7, m8, m1 +%ifidn %2, v + mova [tmpq+stride3q], m8 ; p4 +%else + mova [rsp+14*32], m8 +%endif + + ; sub p6/p5, add p2/q2 [-p6,+p2][-p5,+q2|save] C + ; write -4 + mova m14, [rsp+1*32] + punpcklbw m8, m0, m13 + punpckhbw m9, m0, m13 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p6*4+p5*2+p4*2+p3*2+p2*2+p1+p0+q0+q1 + punpcklbw m8, m2, m14 + punpckhbw m2, m14 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m2, [pb_m1_1] + mova [rsp+1*32], m8 + paddw m10, m8 + paddw m11, m2 ; p6*4+p5+p4*2+p3*2+p2*2+p1+p0+q0+q1+q2 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m12, m8, m1 +%ifidn %2, v + mova [tmpq+strideq*4], m8 ; p3 +%else + mova [rsp+19*32], m8 +%endif + + ; sub p6/p4, add p1/q3 [-p6,+p1][-p4,+q3|save] D + ; write -3 + mova m15, [rsp+2*32] + punpcklbw m8, m0, m3 + punpckhbw m9, m0, m3 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p6*3+p5+p4*2+p3*2+p2*2+p1*2+p0+q0+q1+q2 + punpcklbw m8, m7, m15 + punpckhbw m7, m15 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m7, [pb_m1_1] + mova [rsp+2*32], m8 + paddw m10, m8 + paddw m11, m7 ; p6*3+p5+p4+p3*2+p2*2+p1*2+p0+q0+q1+q2+q3 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m13, m8, m1 + mova [rsp+6*32], m8 ; don't clobber p2/m13 since we need it in F + + ; sub p6/p3, add p0/q4 [-p6,+p0][-p3,+q4|save] E + ; write -2 +%ifidn %2, v + lea tmpq, [dstq+strideq*4] +%endif + punpcklbw m8, m0, m4 + punpckhbw m9, m0, m4 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p6*2+p5+p4+p3*2+p2*2+p1*2+p0*2+q0+q1+q2+q3 +%ifidn %2, v + mova m9, [tmpq+strideq*0] ; q4 +%else + mova m9, [rsp+15*32] +%endif + punpcklbw m8, m12, m9 + punpckhbw m9, m12, m9 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + mova [rsp+7*32], m8 + mova [rsp+5*32], m9 + paddw m10, m8 + paddw m11, m9 ; p6*2+p5+p4+p3+p2*2+p1*2+p0*2+q0+q1+q2+q3+q4 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m3, m8, m1 + mova [rsp+8*32], m8 ; don't clobber p1/m3 since we need it in G + + ; sub p6/p2, add q0/q5 [-p6,+q0][-p2,+q5|save] F + ; write -1 +%ifidn %2, v + mova m9, [tmpq+strideq*1] ; q5 +%else + mova m9, [rsp+16*32] +%endif + punpcklbw m8, m0, m5 + punpckhbw m0, m5 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m0, [pb_m1_1] + paddw m10, m8 + paddw m11, m0 ; p6+p5+p4+p3+p2*2+p1*2+p0*2+q0*2+q1+q2+q3+q4 + punpcklbw m0, m13, m9 + punpckhbw m9, m13, m9 + mova m13, [rsp+6*32] + pmaddubsw m0, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + mova [rsp+ 9*32], m0 + mova [rsp+10*32], m9 + paddw m10, m0 + paddw m11, m9 ; p6+p5+p4+p3+p2+p1*2+p0*2+q0*2+q1+q2+q3+q4+q5 + pmulhrsw m0, m10, [pw_2048] + pmulhrsw m8, m11, [pw_2048] + packuswb m0, m8 + vpblendvb m0, m4, m0, m1 + mova [rsp+6*32], m0 ; don't clobber p0/m4 since we need it in H + + ; sub p6/p1, add q1/q6 [reuse -p6,+q1 from B][-p1,+q6|save] G + ; write +0 +%ifidn %2, v + mova m0, [tmpq+strideq*2] ; q6 +%else + mova m0, [rsp+17*32] +%endif + paddw m10, [rsp+3*32] + paddw m11, [rsp+4*32] ; p5+p4+p3+p2+p1*2+p0*2+q0*2+q1*2+q2+q3+q4+q5 + punpcklbw m8, m3, m0 + punpckhbw m9, m3, m0 + mova m3, [rsp+8*32] + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + mova [rsp+3*32], m8 + mova [rsp+4*32], m9 + paddw m10, m8 + paddw m11, m9 ; p5+p4+p3+p2+p1+p0*2+q0*2+q1*2+q2+q3+q4+q5+q6 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m5, m8, m1 + mova [rsp+8*32], m8 ; don't clobber q0/m5 since we need it in I + + ; sub p5/p0, add q2/q6 [reuse -p5,+q2 from C][-p0,+q6] H + ; write +1 + paddw m10, [rsp+1*32] + paddw m11, m2 ; p4+p3+p2+p1+p0*2+q0*2+q1*2+q2*2+q3+q4+q5+q6 + punpcklbw m8, m4, m0 + punpckhbw m2, m4, m0 + mova m4, [rsp+6*32] + pmaddubsw m8, [pb_m1_1] + pmaddubsw m2, [pb_m1_1] + paddw m10, m8 + paddw m11, m2 ; p4+p3+p2+p1+p0+q0*2+q1*2+q2*2+q3+q4+q5+q6*2 + pmulhrsw m2, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m2, m9 + vpblendvb m2, m6, m2, m1 ; don't clobber q1/m6 since we need it in K + + ; sub p4/q0, add q3/q6 [reuse -p4,+q3 from D][-q0,+q6] I + ; write +2 + paddw m10, [rsp+2*32] + paddw m11, m7 ; p3+p2+p1+p0+q0*2+q1*2+q2*2+q3*2+q4+q5+q6*2 + punpcklbw m8, m5, m0 + punpckhbw m9, m5, m0 + mova m5, [rsp+8*32] + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p3+p2+p1+p0+q0+q1*2+q2*2+q3*2+q4+q5+q6*3 + pmulhrsw m7, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m7, m9 + vpblendvb m7, m14, m7, m1 ; don't clobber q2/m14 since we need it in K + + ; sub p3/q1, add q4/q6 [reuse -p3,+q4 from E][-q1,+q6] J + ; write +3 + paddw m10, [rsp+7*32] + paddw m11, [rsp+5*32] ; p2+p1+p0+q0+q1*2+q2*2+q3*2+q4*2+q5+q6*3 + punpcklbw m8, m6, m0 + punpckhbw m9, m6, m0 + SWAP 2, 6 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p2+p1+p0+q0+q1+q2*2+q3*2+q4*2+q5+q6*4 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 + vpblendvb m8, m15, m8, m1 +%ifidn %2, v + mova [tmpq+mstrideq], m8 ; q3 +%else + mova [rsp+20*32], m8 +%endif + + ; sub p2/q2, add q5/q6 [reuse -p2,+q5 from F][-q2,+q6] K + ; write +4 + paddw m10, [rsp+ 9*32] + paddw m11, [rsp+10*32] ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + punpcklbw m8, m14, m0 + punpckhbw m9, m14, m0 + SWAP 14, 7 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m8, m10, [pw_2048] + pmulhrsw m9, m11, [pw_2048] + packuswb m8, m9 +%ifidn %2, v + mova m9, [tmpq+strideq*0] +%else + mova m9, [rsp+15*32] +%endif + vpblendvb m8, m9, m8, m1 +%ifidn %2, v + mova [tmpq+strideq*0], m8 ; q4 +%else + mova [rsp+15*32], m8 +%endif + + ; sub p1/q3, add q6*2 [reuse -p1,+q6 from G][-q3,+q6] L + ; write +5 + paddw m10, [rsp+3*32] + paddw m11, [rsp+4*32] ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + punpcklbw m8, m15, m0 + punpckhbw m9, m15, m0 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m9, [pb_m1_1] + paddw m10, m8 + paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m10, [pw_2048] + pmulhrsw m11, [pw_2048] + packuswb m10, m11 +%ifidn %2, v + mova m11, [tmpq+strideq*1] +%else + mova m11, [rsp+16*32] +%endif + vpblendvb m10, m11, m10, m1 +%ifidn %2, v + mova [tmpq+strideq*1], m10 ; q5 +%else + mova [rsp+16*32], m10 +%endif + + mova m9, [rsp+0*32] +%ifidn %2, v + lea tmpq, [dstq+mstrideq*4] +%endif +%endif +%if %1 >= 8 + ; flat8 filter + punpcklbw m0, m12, m3 + punpckhbw m1, m12, m3 + pmaddubsw m2, m0, [pb_3_1] + pmaddubsw m7, m1, [pb_3_1] ; 3 * p3 + p1 + punpcklbw m8, m13, m4 + punpckhbw m11, m13, m4 + pmaddubsw m8, [pb_2_1] + pmaddubsw m11, [pb_2_1] + paddw m2, m8 + paddw m7, m11 ; 3 * p3 + 2 * p2 + p1 + p0 + punpcklbw m8, m5, [pb_4] + punpckhbw m11, m5, [pb_4] + pmaddubsw m8, [pb_1] + pmaddubsw m11, [pb_1] + paddw m2, m8 + paddw m7, m11 ; 3 * p3 + 2 * p2 + p1 + p0 + q0 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendvb m10, m13, m8, m9 ; p2 +%ifidn %2, v + mova [tmpq+strideq*1], m10 ; p2 +%endif + + pmaddubsw m8, m0, [pb_m1_1] + pmaddubsw m11, m1, [pb_m1_1] + paddw m2, m8 + paddw m7, m11 + punpcklbw m8, m13, m6 + punpckhbw m11, m13, m6 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m11, [pb_m1_1] + paddw m2, m8 + paddw m7, m11 ; 2 * p3 + p2 + 2 * p1 + p0 + q0 + q1 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendvb m8, m3, m8, m9 ; p1 +%ifidn %2, v + mova [tmpq+strideq*2], m8 ; p1 +%else + mova [rsp+0*32], m8 +%endif + + pmaddubsw m0, [pb_1] + pmaddubsw m1, [pb_1] + psubw m2, m0 + psubw m7, m1 + punpcklbw m8, m4, m14 + punpckhbw m11, m4, m14 + pmaddubsw m8, [pb_1] + pmaddubsw m11, [pb_1] + paddw m2, m8 + paddw m7, m11 ; p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendvb m8, m4, m8, m9 ; p0 +%ifidn %2, v + mova [tmpq+stride3q ], m8 ; p0 +%else + mova [rsp+1*32], m8 +%endif + + punpcklbw m0, m5, m15 + punpckhbw m1, m5, m15 + pmaddubsw m8, m0, [pb_1] + pmaddubsw m11, m1, [pb_1] + paddw m2, m8 + paddw m7, m11 + punpcklbw m8, m4, m12 + punpckhbw m11, m4, m12 + pmaddubsw m8, [pb_1] + pmaddubsw m11, [pb_1] + psubw m2, m8 + psubw m7, m11 ; p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendvb m11, m5, m8, m9 ; q0 +%ifidn %2, v + mova [dstq+strideq*0], m11 ; q0 +%endif + + pmaddubsw m0, [pb_m1_1] + pmaddubsw m1, [pb_m1_1] + paddw m2, m0 + paddw m7, m1 + punpcklbw m8, m13, m6 + punpckhbw m13, m6 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m13, [pb_m1_1] + paddw m2, m8 + paddw m7, m13 ; p1 + p0 + q0 + 2 * q1 + q2 + 2 * q3 + 4 + psrlw m8, m2, 3 + psrlw m13, m7, 3 + packuswb m8, m13 + vpblendvb m13, m6, m8, m9 ; q1 +%ifidn %2, v + mova [dstq+strideq*1], m13 ; q1 +%endif + + punpcklbw m0, m3, m6 + punpckhbw m1, m3, m6 + pmaddubsw m0, [pb_1] + pmaddubsw m1, [pb_1] + psubw m2, m0 + psubw m7, m1 + punpcklbw m0, m14, m15 + punpckhbw m1, m14, m15 + pmaddubsw m0, [pb_1] + pmaddubsw m1, [pb_1] + paddw m2, m0 + paddw m7, m1 ; p0 + q0 + q1 + q2 + 2 * q2 + 3 * q3 + 4 + psrlw m2, 3 + psrlw m7, 3 + packuswb m2, m7 + vpblendvb m2, m14, m2, m9 ; q2 +%ifidn %2, v + mova [dstq+strideq*2], m2 ; q2 +%else + mova m0, [rsp+0*32] + mova m1, [rsp+1*32] +%if %1 == 8 + ; 16x8 transpose + punpcklbw m3, m12, m10 + punpckhbw m12, m10 + punpcklbw m10, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m11, m13 + punpckhbw m11, m13 + punpcklbw m13, m2, m15 + punpckhbw m2, m15 + + punpcklwd m15, m3, m10 + punpckhwd m3, m10 + punpcklwd m10, m12, m0 + punpckhwd m12, m0 + punpcklwd m0, m1, m13 + punpckhwd m1, m13 + punpcklwd m13, m11, m2 + punpckhwd m11, m2 + + punpckldq m2, m15, m0 + punpckhdq m15, m0 + punpckldq m0, m3, m1 + punpckhdq m3, m1 + punpckldq m1, m10, m13 + punpckhdq m10, m13 + punpckldq m13, m12, m11 + punpckhdq m12, m11 + + ; write 8x32 + movq [dstq+strideq*0-4], xm2 + movhps [dstq+strideq*1-4], xm2 + movq [dstq+strideq*2-4], xm15 + movhps [dstq+stride3q -4], xm15 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm0 + movhps [dstq+strideq*1-4], xm0 + movq [dstq+strideq*2-4], xm3 + movhps [dstq+stride3q -4], xm3 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm1 + movhps [dstq+strideq*1-4], xm1 + movq [dstq+strideq*2-4], xm10 + movhps [dstq+stride3q -4], xm10 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm13 + movhps [dstq+strideq*1-4], xm13 + movq [dstq+strideq*2-4], xm12 + movhps [dstq+stride3q -4], xm12 + lea dstq, [dstq+strideq*4] + + vextracti128 xm2, m2, 1 + vextracti128 xm15, m15, 1 + vextracti128 xm0, m0, 1 + vextracti128 xm3, m3, 1 + vextracti128 xm1, m1, 1 + vextracti128 xm10, m10, 1 + vextracti128 xm13, m13, 1 + vextracti128 xm12, m12, 1 + + movq [dstq+strideq*0-4], xm2 + movhps [dstq+strideq*1-4], xm2 + movq [dstq+strideq*2-4], xm15 + movhps [dstq+stride3q -4], xm15 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm0 + movhps [dstq+strideq*1-4], xm0 + movq [dstq+strideq*2-4], xm3 + movhps [dstq+stride3q -4], xm3 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm1 + movhps [dstq+strideq*1-4], xm1 + movq [dstq+strideq*2-4], xm10 + movhps [dstq+stride3q -4], xm10 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm13 + movhps [dstq+strideq*1-4], xm13 + movq [dstq+strideq*2-4], xm12 + movhps [dstq+stride3q -4], xm12 + lea dstq, [dstq+strideq*4] +%else + ; 16x16 transpose and store + SWAP 5, 10, 2 + SWAP 6, 0 + SWAP 7, 1 + SWAP 8, 11 + SWAP 9, 13 + mova m0, [rsp+11*32] + mova m1, [rsp+12*32] + mova m2, [rsp+13*32] + mova m3, [rsp+14*32] + mova m4, [rsp+19*32] + mova m11, [rsp+20*32] + mova m12, [rsp+15*32] + mova m13, [rsp+16*32] + mova m14, [rsp+17*32] + TRANSPOSE_16X16B 1, 0, [rsp+18*32] + movu [dstq+strideq*0-8], xm0 + movu [dstq+strideq*1-8], xm1 + movu [dstq+strideq*2-8], xm2 + movu [dstq+stride3q -8], xm3 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm4 + movu [dstq+strideq*1-8], xm5 + movu [dstq+strideq*2-8], xm6 + movu [dstq+stride3q -8], xm7 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm8 + movu [dstq+strideq*1-8], xm9 + movu [dstq+strideq*2-8], xm10 + movu [dstq+stride3q -8], xm11 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm12 + movu [dstq+strideq*1-8], xm13 + movu [dstq+strideq*2-8], xm14 + movu [dstq+stride3q -8], xm15 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m0, 1 + vextracti128 [dstq+strideq*1-8], m1, 1 + vextracti128 [dstq+strideq*2-8], m2, 1 + vextracti128 [dstq+stride3q -8], m3, 1 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m4, 1 + vextracti128 [dstq+strideq*1-8], m5, 1 + vextracti128 [dstq+strideq*2-8], m6, 1 + vextracti128 [dstq+stride3q -8], m7, 1 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m8, 1 + vextracti128 [dstq+strideq*1-8], m9, 1 + vextracti128 [dstq+strideq*2-8], m10, 1 + vextracti128 [dstq+stride3q -8], m11, 1 + lea dstq, [dstq+strideq*4] + vextracti128 [dstq+strideq*0-8], m12, 1 + vextracti128 [dstq+strideq*1-8], m13, 1 + vextracti128 [dstq+strideq*2-8], m14, 1 + vextracti128 [dstq+stride3q -8], m15, 1 + lea dstq, [dstq+strideq*4] +%endif +%endif +%elif %1 == 6 + ; flat6 filter + + punpcklbw m8, m13, m5 + punpckhbw m11, m13, m5 + pmaddubsw m0, m8, [pb_3_1] + pmaddubsw m1, m11, [pb_3_1] + punpcklbw m7, m4, m3 + punpckhbw m10, m4, m3 + pmaddubsw m2, m7, [pb_2] + pmaddubsw m12, m10, [pb_2] + paddw m0, m2 + paddw m1, m12 + pmulhrsw m2, m0, [pw_4096] + pmulhrsw m12, m1, [pw_4096] + packuswb m2, m12 + vpblendvb m2, m3, m2, m9 +%ifidn %2, v + mova [tmpq+strideq*2], m2 ; p1 +%endif + + pmaddubsw m8, [pb_m1_1] + pmaddubsw m11, [pb_m1_1] + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m13, m6 + punpckhbw m11, m13, m6 + pmaddubsw m8, [pb_m1_1] + pmaddubsw m11, [pb_m1_1] + paddw m0, m8 + paddw m1, m11 + pmulhrsw m12, m0, [pw_4096] + pmulhrsw m13, m1, [pw_4096] + packuswb m12, m13 + vpblendvb m12, m4, m12, m9 +%ifidn %2, v + mova [tmpq+stride3q], m12 ; p0 +%endif + + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m3, m14 + punpckhbw m11, m3, m14 + pmaddubsw m14, m8, [pb_m1_1] + pmaddubsw m13, m11, [pb_m1_1] + paddw m0, m14 + paddw m1, m13 + pmulhrsw m14, m0, [pw_4096] + pmulhrsw m13, m1, [pw_4096] + packuswb m14, m13 + vpblendvb m14, m5, m14, m9 +%ifidn %2, v + mova [dstq+strideq*0], m14 ; q0 +%endif + + pmaddubsw m8, [pb_m1_2] + pmaddubsw m11, [pb_m1_2] + paddw m0, m8 + paddw m1, m11 + pmaddubsw m7, [pb_m1_0] + pmaddubsw m10, [pb_m1_0] + paddw m0, m7 + paddw m1, m10 + pmulhrsw m0, [pw_4096] + pmulhrsw m1, [pw_4096] + packuswb m0, m1 + vpblendvb m0, m6, m0, m9 +%ifidn %2, v + mova [dstq+strideq*1], m0 ; q1 +%else + TRANSPOSE_16x4_AND_WRITE_4x32 2, 12, 14, 0, 1 +%endif +%else +%ifidn %2, v + mova [tmpq+strideq*0], m3 ; p1 + mova [tmpq+strideq*1], m4 ; p0 + mova [tmpq+strideq*2], m5 ; q0 + mova [tmpq+stride3q ], m6 ; q1 +%else + TRANSPOSE_16x4_AND_WRITE_4x32 3, 4, 5, 6, 7 +%endif +%endif +%endmacro + +INIT_YMM avx2 +cglobal lpf_v_sb_y_8bpc, 7, 10, 16, 32 * 11, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + +.loop: + cmp byte [maskq+8], 0 ; vmask[2] + je .no_flat16 + + FILTER 16, v + jmp .end + +.no_flat16: + cmp byte [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 8, v + jmp .end + +.no_flat: + cmp byte [maskq+0], 0 ; vmask[0] + je .end + + call .v4 + +.end: + add lq, 32 + add dstq, 32 + add maskq, 1 + sub wd, 8 + jg .loop + RET +ALIGN function_align +.v4: + FILTER 4, v + ret + +INIT_YMM avx2 +cglobal lpf_h_sb_y_8bpc, 7, 10, 16, 32 * 21, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp + shl l_strideq, 2 + sub lq, 4 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] + +.loop: + cmp byte [maskq+8], 0 ; vmask[2] + je .no_flat16 + + FILTER 16, h + jmp .end + +.no_flat16: + cmp byte [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 8, h + jmp .end + +.no_flat: + cmp byte [maskq+0], 0 ; vmask[0] + je .no_filter + + call .h4 + jmp .end + +.no_filter: + lea dstq, [dstq+stride3q*8] + lea lq, [lq+l_strideq*8] + lea dstq, [dstq+strideq*8] +.end: + add maskq, 1 + sub hd, 8 + jg .loop + RET +ALIGN function_align +.h4: + FILTER 4, h + ret + +INIT_YMM avx2 +cglobal lpf_v_sb_uv_8bpc, 7, 10, 16, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + +.loop: + cmp byte [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 6, v + jmp .end + +.no_flat: + cmp byte [maskq+0], 0 ; vmask[0] + je .end + + call mangle(private_prefix %+ _lpf_v_sb_y_8bpc_avx2).v4 + +.end: + add lq, 32 + add dstq, 32 + add maskq, 1 + sub wd, 8 + jg .loop + RET + +INIT_YMM avx2 +cglobal lpf_h_sb_uv_8bpc, 7, 10, 16, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp + shl l_strideq, 2 + sub lq, 4 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] + +.loop: + cmp byte [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 6, h + jmp .end + +.no_flat: + cmp byte [maskq+0], 0 ; vmask[0] + je .no_filter + + call mangle(private_prefix %+ _lpf_h_sb_y_8bpc_avx2).h4 + jmp .end + +.no_filter: + lea dstq, [dstq+stride3q*8] + lea lq, [lq+l_strideq*8] + lea dstq, [dstq+strideq*8] +.end: + add maskq, 1 + sub hd, 8 + jg .loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/loopfilter_avx512.asm b/third_party/dav1d/src/x86/loopfilter_avx512.asm new file mode 100644 index 0000000000..0218b624d3 --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter_avx512.asm @@ -0,0 +1,1534 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +pb_4x0_4x4_4x8_4x12: times 4 db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 + +pb_mask: dd 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080 + dd 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000 + +hmulA: dd 0, 8, 16, 24, 32, 40, 48, 56, 4, 12, 20, 28, 36, 44, 52, 60 +hmulB: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +hmulC: dd 0, 1, 2, 3, 16, 17, 18, 19, 32, 33, 34, 35, 48, 49, 50, 51 +hmulD: dd 0, 1, 16, 17, 32, 33, 48, 49 +hshuf4:db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 + +pb_1: times 4 db 1 +pb_2: times 4 db 2 +pb_3: times 4 db 3 +pb_4: times 4 db 4 +pb_16: times 4 db 16 +pb_63: times 4 db 63 +pb_64: times 4 db 64 +pb_128: times 4 db 0x80 +pb_240: times 4 db 0xf0 +pb_248: times 4 db 0xf8 +pb_254: times 4 db 0xfe +pb_2_1: times 2 db 2, 1 +pb_3_1: times 2 db 3, 1 +pb_7_1: times 2 db 7, 1 +pb_m1_0: times 2 db -1, 0 +pb_m1_1: times 2 db -1, 1 +pb_m1_2: times 2 db -1, 2 +pw_2048: times 2 dw 2048 +pw_4096: times 2 dw 4096 + +SECTION .text + +%macro ABSSUB 4 ; dst, a, b, tmp + psubusb %1, %2, %3 + psubusb %4, %3, %2 + por %1, %4 +%endmacro + +%macro TRANSPOSE_16x4_AND_WRITE_4x32 5 + punpcklbw m%5, m%1, m%2 + punpckhbw m%1, m%2 + punpcklbw m%2, m%3, m%4 + punpckhbw m%3, m%4 + punpcklwd m%4, m%5, m%2 + punpckhwd m%5, m%2 + punpcklwd m%2, m%1, m%3 + punpckhwd m%1, m%3 + kmovw k1, k6 + lea t0, [dstq+strideq*4] + vpscatterdd [dstq+m19-2]{k1}, m%4 + kmovw k1, k6 + lea t1, [dstq+strideq*8] + vpscatterdd [t0 +m19-2]{k1}, m%5 + kmovw k1, k6 + lea t2, [t0 +strideq*8] + vpscatterdd [t1 +m19-2]{k1}, m%2 + kmovw k1, k6 + vpscatterdd [t2 +m19-2]{k1}, m%1 +%endmacro + +%macro TRANSPOSE_16X16B 3 ; in_load_15_from_mem, out_store_0_in_mem, mem +%if %1 == 0 + SWAP m16, m22 +%endif + punpcklbw m22, m24, m26 + punpckhbw m24, m26 + punpcklbw m26, m2, m3 + punpckhbw m2, m3 + punpcklbw m3, m4, m5 + punpckhbw m4, m5 + punpcklbw m5, m6, m7 + punpckhbw m6, m7 + punpcklbw m7, m8, m9 + punpckhbw m8, m9 + punpcklbw m9, m10, m11 + punpckhbw m10, m11 + punpcklbw m11, m25, m13 + punpckhbw m25, m13 +%if %1 == 0 + SWAP m13, m16 +%else + mova m13, %3 +%endif + SWAP m16, m25 + punpcklbw m25, m14, m13 + punpckhbw m13, m14, m13 + ; interleaved in m22,24,26,2,3,4,5,6,7,8,9,10,11,rsp%3,25,13 + punpcklwd m14, m22, m26 + punpckhwd m22, m26 + punpcklwd m26, m24, m2 + punpckhwd m24, m2 + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m6 + punpckhwd m4, m6 + punpcklwd m6, m7, m9 + punpckhwd m7, m9 + punpcklwd m9, m8, m10 + punpckhwd m8, m10 + punpcklwd m10, m11, m25 + punpckhwd m11, m25 + SWAP m25, m16, m11 + punpcklwd m11, m25, m13 + punpckhwd m25, m13 + ; interleaved in m14,15,26,24,2,3,5,4,6,7,9,8,10,rsp%3,11,25 + punpckldq m13, m14, m2 + punpckhdq m14, m2 + punpckldq m2, m22, m3 + punpckhdq m22, m3 + punpckldq m3, m26, m5 + punpckhdq m26, m5 + punpckldq m5, m24, m4 + punpckhdq m24, m4 + punpckldq m4, m6, m10 + punpckhdq m6, m10 + punpckldq m10, m9, m11 + punpckhdq m9, m11 + punpckldq m11, m8, m25 + punpckhdq m8, m25 + SWAP m25, m16, m8 + punpckldq m8, m7, m25 + punpckhdq m7, m25 + ; interleaved in m13,14,2,15,3,26,5,24,4,6,8,7,10,9,11,rsp%3 + punpcklqdq m25, m13, m4 + punpckhqdq m13, m4 + punpcklqdq m4, m14, m6 + punpckhqdq m14, m6 + punpcklqdq m6, m2, m8 + punpckhqdq m2, m8 + punpcklqdq m8, m22, m7 + punpckhqdq m22, m7 + punpcklqdq m7, m3, m10 + punpckhqdq m3, m10 + punpcklqdq m10, m26, m9 + punpckhqdq m26, m9 + punpcklqdq m9, m5, m11 + punpckhqdq m5, m11 + SWAP m11, m16 +%if %2 == 0 + SWAP m16, m25 +%else + mova %3, m25 +%endif + punpcklqdq m25, m24, m11 + punpckhqdq m24, m11 +%if %2 == 0 + SWAP m11, m16 +%endif + ; interleaved m11,13,4,14,6,2,8,15,7,3,10,26,9,5,25,24 + SWAP 24, 11, 26, 13, 5, 2, 4, 6, 8, 7, 22 + SWAP 3, 14, 25, 9 +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] + ; load data +%ifidn %2, v +%define is_h 0 +%if %1 == 4 + lea t0, [dstq+mstrideq*2] + mova m3, [t0 +strideq*0] ; p1 + mova m4, [t0 +strideq*1] ; p0 + mova m5, [t0 +strideq*2] ; q0 + mova m6, [t0 +stride3q ] ; q1 +%else + ; load 6-8 pixels, remainder (for wd=16) will be read inline +%if %1 == 16 + lea t0, [dstq+mstrideq*8] + mova m16, [t0 +strideq*1] + mova m17, [t0 +strideq*2] + mova m18, [t0 +stride3q ] +%endif + lea t0, [dstq+mstrideq*4] +%if %1 != 6 + mova m25, [t0 +strideq*0] +%endif + mova m13, [t0 +strideq*1] + mova m3, [t0 +strideq*2] + mova m4, [t0 +stride3q ] + mova m5, [dstq+strideq*0] + mova m6, [dstq+strideq*1] + mova m14, [dstq+strideq*2] +%if %1 != 6 + mova m22, [dstq+stride3q ] +%endif +%if %1 == 16 + lea t0, [dstq+strideq*4] + mova m29, [t0 +strideq*0] + mova m30, [t0 +strideq*1] + mova m31, [t0 +strideq*2] +%endif +%endif +%else ; h +%define is_h 1 + ; load lines +%if %1 == 4 + vbroadcasti32x4 m0, [hshuf4] + kmovw k1, k6 + lea t0, [dstq+strideq*4] + vpgatherdd m3{k1}, [dstq+m19-2] + kmovw k1, k6 + lea t1, [dstq+strideq*8] + vpgatherdd m4{k1}, [t0 +m19-2] + kmovw k1, k6 + lea t2, [t0 +strideq*8] + vpgatherdd m5{k1}, [t1 +m19-2] + kmovw k1, k6 + vpgatherdd m6{k1}, [t2 +m19-2] + pshufb m3, m0 + pshufb m4, m0 + pshufb m5, m0 + pshufb m6, m0 + punpckldq m7, m3, m4 + punpckhdq m3, m4 + punpckldq m4, m5, m6 + punpckhdq m5, m6 + punpcklqdq m6, m7, m4 + punpckhqdq m7, m4 + punpcklqdq m4, m3, m5 + punpckhqdq m3, m5 + SWAP 3, 6 + SWAP 5, 4, 7 + ; 6,7,4,3 -> 3,4,5,6 +%elif %1 == 6 || %1 == 8 + kmovb k1, k7 + lea t0, [dstq+strideq*1] + vpgatherdq m3{k1}, [dstq+ym21-%1/2] + kmovb k1, k7 + lea t1, [dstq+strideq*2] + vpgatherdq m4{k1}, [t0 +ym21-%1/2] + kmovb k1, k7 + lea t2, [dstq+stride3q ] + vpgatherdq m5{k1}, [t1 +ym21-%1/2] + kmovb k1, k7 + vextracti32x8 ym0, m21, 1 + vpgatherdq m6{k1}, [t2 +ym21-%1/2] + kmovb k1, k7 + vpgatherdq m12{k1}, [dstq+ym0 -%1/2] + kmovb k1, k7 + vpgatherdq m13{k1}, [t0 +ym0 -%1/2] + kmovb k1, k7 + vpgatherdq m14{k1}, [t1 +ym0 -%1/2] + kmovb k1, k7 + vpgatherdq m15{k1}, [t2 +ym0 -%1/2] + ; transpose 8x16 + ; xm3: A-H0,A-H8 + ; xm4: A-H1,A-H9 + ; xm5: A-H2,A-H10 + ; xm6: A-H3,A-H11 + ; xm12: A-H4,A-H12 + ; xm13: A-H5,A-H13 + ; xm14: A-H6,A-H14 + ; xm15: A-H7,A-H15 + punpcklbw m7, m3, m4 + punpckhbw m3, m4 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + punpcklbw m6, m12, m13 + punpckhbw m12, m13 + punpcklbw m13, m14, m15 + punpckhbw m14, m15 + ; xm7: A0-1,B0-1,C0-1,D0-1,E0-1,F0-1,G0-1,H0-1 + ; xm3: A8-9,B8-9,C8-9,D8-9,E8-9,F8-9,G8-9,H8-9 + ; xm4: A2-3,B2-3,C2-3,D2-3,E2-3,F2-3,G2-3,H2-3 + ; xm5: A10-11,B10-11,C10-11,D10-11,E10-11,F10-11,G10-11,H10-11 + ; xm6: A4-5,B4-5,C4-5,D4-5,E4-5,F4-5,G4-5,H4-5 + ; xm12: A12-13,B12-13,C12-13,D12-13,E12-13,F12-13,G12-13,H12-13 + ; xm13: A6-7,B6-7,C6-7,D6-7,E6-7,F6-7,G6-7,H6-7 + ; xm14: A14-15,B14-15,C14-15,D14-15,E14-15,F14-15,G14-15,H14-15 + punpcklwd m15, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m6, m13 + punpckhwd m6, m13 + punpcklwd m13, m12, m14 + punpckhwd m12, m14 + ; xm15: A0-3,B0-3,C0-3,D0-3 + ; xm7: E0-3,F0-3,G0-3,H0-3 + ; xm4: A8-11,B8-11,C8-11,D8-11 + ; xm3: E8-11,F8-11,G8-11,H8-11 + ; xm5: A4-7,B4-7,C4-7,D4-7 + ; xm6: E4-7,F4-7,G4-7,H4-7 + ; xm13: A12-15,B12-15,C12-15,D12-15 + ; xm12: E12-15,F12-15,G12-15,H12-15 + punpckldq m14, m15, m5 + punpckhdq m15, m5 + punpckldq m5, m7, m6 + %if %1 != 6 + punpckhdq m7, m6 + %endif + punpckldq m6, m4, m13 + punpckhdq m4, m13 + punpckldq m13, m3, m12 + %if %1 != 6 + punpckhdq m12, m3, m12 + %endif + ; xm14: A0-7,B0-7 + ; xm15: C0-7,D0-7 + ; xm5: E0-7,F0-7 + ; xm7: G0-7,H0-7 + ; xm6: A8-15,B8-15 + ; xm4: C8-15,D8-15 + ; xm13: E8-15,F8-15 + ; xm12: G8-15,H8-15 + punpcklqdq m3, m14, m6 + punpckhqdq m14, m6 + punpckhqdq m6, m15, m4 + punpcklqdq m15, m4 + punpcklqdq m4, m5, m13 + punpckhqdq m13, m5, m13 + %if %1 == 8 + punpcklqdq m5, m7, m12 + punpckhqdq m25, m7, m12 + ; xm3: A0-15 + ; xm14: B0-15 + ; xm15: C0-15 + ; xm6: D0-15 + ; xm4: E0-15 + ; xm13: F0-15 + ; xm5: G0-15 + ; xm25: H0-15 + SWAP 25, 3, 15 + SWAP 13, 14, 5, 4, 6 + SWAP 15, 22 + ; 3,14,15,6,4,13,5,12 -> 12,13,3,4,5,6,14,22 + %else + SWAP 13, 3, 14 + SWAP 6, 4, 15, 5 + ; 3,14,15,6,4,13 -> 13,3,4,5,6,14 + %endif +%else ; 16, h + ; load and 16x16 transpose. We only use 14 pixels but we'll need the + ; remainder at the end for the second transpose + movu xm24, [dstq+strideq*0-8] + movu xm26, [dstq+strideq*1-8] + movu xm2, [dstq+strideq*2-8] + movu xm3, [dstq+stride3q -8] + lea t0, [dstq+strideq*4] + movu xm4, [t0 +strideq*0-8] + movu xm5, [t0 +strideq*1-8] + movu xm6, [t0 +strideq*2-8] + movu xm7, [t0 +stride3q -8] + lea t0, [t0 +strideq*4] + movu xm8, [t0 +strideq*0-8] + movu xm9, [t0 +strideq*1-8] + movu xm10, [t0 +strideq*2-8] + movu xm11, [t0 +stride3q -8] + lea t0, [t0 +strideq*4] + movu xm25, [t0 +strideq*0-8] + movu xm13, [t0 +strideq*1-8] + movu xm14, [t0 +strideq*2-8] + movu xm22, [t0 +stride3q -8] + lea t0, [t0 +strideq*4] + vinserti32x4 ym24, [t0 +strideq*0-8], 1 + vinserti32x4 ym26, [t0 +strideq*1-8], 1 + vinserti32x4 ym2, [t0 +strideq*2-8], 1 + vinserti32x4 ym3, [t0 +stride3q -8], 1 + lea t0, [t0 +strideq*4] + vinserti32x4 ym4, [t0 +strideq*0-8], 1 + vinserti32x4 ym5, [t0 +strideq*1-8], 1 + vinserti32x4 ym6, [t0 +strideq*2-8], 1 + vinserti32x4 ym7, [t0 +stride3q -8], 1 + lea t0, [t0 +strideq*4] + vinserti32x4 ym8, [t0 +strideq*0-8], 1 + vinserti32x4 ym9, [t0 +strideq*1-8], 1 + vinserti32x4 ym10, [t0 +strideq*2-8], 1 + vinserti32x4 ym11, [t0 +stride3q -8], 1 + lea t0, [t0 +strideq*4] + vinserti32x4 ym25, [t0 +strideq*0-8], 1 + vinserti32x4 ym13, [t0 +strideq*1-8], 1 + vinserti32x4 ym14, [t0 +strideq*2-8], 1 + vinserti32x4 ym22, [t0 +stride3q -8], 1 + lea t0, [t0 +strideq*4] + vinserti32x4 m24, [t0 +strideq*0-8], 2 + vinserti32x4 m26, [t0 +strideq*1-8], 2 + vinserti32x4 m2, [t0 +strideq*2-8], 2 + vinserti32x4 m3, [t0 +stride3q -8], 2 + lea t0, [t0 +strideq*4] + vinserti32x4 m4, [t0 +strideq*0-8], 2 + vinserti32x4 m5, [t0 +strideq*1-8], 2 + vinserti32x4 m6, [t0 +strideq*2-8], 2 + vinserti32x4 m7, [t0 +stride3q -8], 2 + lea t0, [t0 +strideq*4] + vinserti32x4 m8, [t0 +strideq*0-8], 2 + vinserti32x4 m9, [t0 +strideq*1-8], 2 + vinserti32x4 m10, [t0 +strideq*2-8], 2 + vinserti32x4 m11, [t0 +stride3q -8], 2 + lea t0, [t0 +strideq*4] + vinserti32x4 m25, [t0 +strideq*0-8], 2 + vinserti32x4 m13, [t0 +strideq*1-8], 2 + vinserti32x4 m14, [t0 +strideq*2-8], 2 + vinserti32x4 m22, [t0 +stride3q -8], 2 + lea t0, [t0 +strideq*4] + vinserti32x4 m24, [t0 +strideq*0-8], 3 + vinserti32x4 m26, [t0 +strideq*1-8], 3 + vinserti32x4 m2, [t0 +strideq*2-8], 3 + vinserti32x4 m3, [t0 +stride3q -8], 3 + lea t0, [t0 +strideq*4] + vinserti32x4 m4, [t0 +strideq*0-8], 3 + vinserti32x4 m5, [t0 +strideq*1-8], 3 + vinserti32x4 m6, [t0 +strideq*2-8], 3 + vinserti32x4 m7, [t0 +stride3q -8], 3 + lea t0, [t0 +strideq*4] + vinserti32x4 m8, [t0 +strideq*0-8], 3 + vinserti32x4 m9, [t0 +strideq*1-8], 3 + vinserti32x4 m10, [t0 +strideq*2-8], 3 + vinserti32x4 m11, [t0 +stride3q -8], 3 + lea t0, [t0 +strideq*4] + vinserti32x4 m25, [t0 +strideq*0-8], 3 + vinserti32x4 m13, [t0 +strideq*1-8], 3 + vinserti32x4 m14, [t0 +strideq*2-8], 3 + vinserti32x4 m22, [t0 +stride3q -8], 3 + ; + TRANSPOSE_16X16B 0, 1, [rsp+0*64] + SWAP m16, m26 + SWAP m17, m2 + SWAP m18, m3 + SWAP m29, m25 + SWAP m30, m13 + SWAP m31, m14 + mova [rsp+4*64], m22 + ; 4,5,6,7,8,9,10,11 -> 25,13,3,4,5,6,14,22 + SWAP 25, 4, 7 + SWAP 13, 5, 8 + SWAP 3, 6, 9 + SWAP 10, 14 + SWAP 11, 22 +%endif +%endif + + ; load L/E/I/H + vpbroadcastd m15, [pb_1] +%ifidn %2, v + movu m1, [lq] + movu m0, [lq+l_strideq] +%else + kmovw k1, k6 + vpgatherdd m0{k1}, [lq+m20+4] + kmovw k1, k6 + vpgatherdd m1{k1}, [lq+m20+0] +%endif + pxor m2, m2 + pcmpeqb k1, m0, m2 + vmovdqu8 m0{k1}, m1 ; l[x][] ? l[x][] : l[x-stride][] + pshufb m0, pbshuf ; l[x][0] + vpcmpub k3, m0, m2, 4 ; neq ; L + psrlq m2, m0, [lutq+128] + pand m2, [pb_63]{bcstd} + vpbroadcastb m1, [lutq+136] + pminub m2, m1 + pmaxub m2, m15 ; I + pand m1, m0, [pb_240]{bcstd} + psrlq m1, 4 ; H + paddd m0, [pb_2]{bcstd} + paddb m0, m0 + paddb m0, m2 ; E + + ABSSUB m8, m3, m4, m9 ; abs(p1-p0) + ABSSUB m9, m5, m6, m10 ; abs(q1-q0) + pmaxub m8, m9 + vpcmpub k1, m8, m1, 6 ; gt ; hev +%if %1 != 4 + %if %1 == 6 + ABSSUB m9, m13, m4, m10 ; abs(p2-p0) + pmaxub m9, m8 + %else + ABSSUB m9, m25, m4, m10 ; abs(p3-p0) + pmaxub m9, m8 + ABSSUB m10, m13, m4, m11 ; abs(p2-p0) + pmaxub m9, m10 + %endif + ABSSUB m10, m5, m14, m11 ; abs(q2-q0) + pmaxub m9, m10 + %if %1 != 6 + ABSSUB m10, m5, m22, m11 ; abs(q3-q0) + pmaxub m9, m10 + %endif + vpcmpub k2{k3}, m9, m15, 2 ; le ; flat8in + %if %1 == 6 + ABSSUB m10, m13, m3, m1 ; abs(p2-p1) + %else + ABSSUB m10, m25, m13, m11 ; abs(p3-p2) + ABSSUB m11, m13, m3, m1 ; abs(p2-p1) + pmaxub m10, m11 + ABSSUB m11, m14, m22, m1 ; abs(q3-q2) + pmaxub m10, m11 + %endif + ABSSUB m11, m14, m6, m1 ; abs(q2-q1) + pmaxub m10, m11 + %if %1 == 16 + vpbroadcastd m11, [maskq+8] + por m11, [maskq+4]{bcstd} + %else + vpbroadcastd m11, [maskq+4] + %endif + vptestmd k4, m11, pbmask + vmovdqa32 m10{k4}{z}, m10 ; only apply fm-wide to wd>4 blocks + pmaxub m8, m10 +%endif + vpcmpub k3{k3}, m8, m2, 2 ; le + ABSSUB m10, m3, m6, m11 ; abs(p1-q1) + ABSSUB m11, m4, m5, m2 ; abs(p0-q0) + paddusb m11, m11 + pand m10, [pb_254]{bcstd} + psrlq m10, 1 + paddusb m10, m11 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + vpcmpub k3{k3}, m10, m0, 2 ; abs(p0-q0)*2+(abs(p1-q1)>>1) <= E + +%if %1 == 16 + ABSSUB m1, m16, m4, m2 + ABSSUB m2, m17, m4, m10 + pmaxub m1, m2 + ABSSUB m2, m18, m4, m10 + pmaxub m1, m2 + ABSSUB m2, m29, m5, m10 + pmaxub m1, m2 + ABSSUB m2, m30, m5, m10 + pmaxub m1, m2 + ABSSUB m2, m31, m5, m10 + pmaxub m1, m2 + kandq k2, k2, k3 + vpcmpub k4{k2}, m1, m15, 2 ; flat8in & flat8out + vpbroadcastd m2, [maskq+8] + vptestmd k5, m2, pbmask + vpmovm2d m7, k5 + vptestmb k4{k4}, m7, m7 ; flat16 & fm + por m10, m2, [maskq+4]{bcstd} + vptestmd k5, m10, pbmask + vpmovm2d m7, k5 + vptestmb k2{k2}, m7, m7 ; flat8in + por m2, m10, [maskq+0]{bcstd} + vptestmd k5, m2, pbmask + vpmovm2d m7, k5 + vptestmb k3{k3}, m7, m7 + kandnq k3, k2, k3 ; fm & !flat8 & !flat16 + kandnq k2, k4, k2 ; flat8 & !flat16 +%elif %1 != 4 + vpbroadcastd m0, [maskq+4] + vptestmd k4, m0, pbmask + vpmovm2d m7, k4 + vptestmb k2{k2}, m7, m7 + kandq k2, k2, k3 ; flat8 & fm + por m0, [maskq+0]{bcstd} + vptestmd k4, m0, pbmask + vpmovm2d m7, k4 + vptestmb k3{k3}, m7, m7 + kandnq k3, k2, k3 ; fm & !flat8 +%else + %ifidn %2, v + vptestmd k4, pbmask, [maskq+0]{bcstd} + %else + vpbroadcastd m0, [maskq+0] + vptestmd k4, m0, pbmask + %endif + vpmovm2d m7, k4 + vptestmb k3{k3}, m7, m7 ; fm +%endif + + ; short filter +%if %1 >= 8 + SWAP m23, m15 +%endif + vpbroadcastd m15, [pb_3] + vpbroadcastd m0, [pb_4] + vpbroadcastd m12, [pb_16] + vpbroadcastd m1, [pb_64] + pxor m3, pb128 + pxor m6, pb128 + psubsb m10{k1}{z}, m3, m6 ; f=iclip_diff(p1-q1)&hev + pxor m4, pb128 + pxor m5, pb128 + psubsb m11, m5, m4 + paddsb m10, m11 + paddsb m10, m11 + paddsb m10{k3}{z}, m10, m11 ; f=iclip_diff(3*(q0-p0)+f)&fm + paddsb m8, m10, m15 + paddsb m10, m0 + pand m8, [pb_248]{bcstd} + pand m10, [pb_248]{bcstd} + psrlq m8, 3 + psrlq m10, 3 + pxor m8, m12 + pxor m10, m12 + psubb m8, m12 ; f2 + psubb m10, m12 ; f1 + paddsb m4, m8 + psubsb m5, m10 + pxor m4, pb128 + pxor m5, pb128 + ; + pxor m10, pb128 + pxor m8, m8 + pavgb m8, m10 ; f=(f1+1)>>1 + psubb m8, m1 + knotq k1, k1 + paddsb m3{k1}, m3, m8 + psubsb m6{k1}, m6, m8 + pxor m3, pb128 + pxor m6, pb128 + +%if %1 == 16 + ; flat16 filter +%ifidn %2, v + lea t0, [dstq+mstrideq*8] +%endif + SWAP m24, m16, m14 + SWAP m2, m17, m22 + SWAP m7, m18 + + ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 [p5/p4/p2/p1/p0/q0][p6/p3] A + ; write -6 + vpbroadcastd m1, [pb_7_1] + vpbroadcastd m12, [pb_2] + punpcklbw m14, m24, m25 + punpckhbw m22, m24, m25 + pmaddubsw m10, m14, m1 + pmaddubsw m11, m22, m1 ; p6*7+p3 + punpcklbw m8, m2, m7 + punpckhbw m9, m2, m7 + pmaddubsw m8, m12 + pmaddubsw m9, m12 + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3 +%ifidn %2, h + vpbroadcastd m27, [pw_2048] + vpbroadcastd m1, [pb_m1_1] + %define pw2048 m27 + %define pbm1_1 m1 +%endif + punpcklbw m8, m13, m3 + punpckhbw m9, m13, m3 + pmaddubsw m8, m23 + pmaddubsw m9, m23 + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3+p2+p1 + punpcklbw m8, m4, m5 + punpckhbw m9, m4, m5 + pmaddubsw m8, m23 + pmaddubsw m9, m23 + paddw m10, m8 + paddw m11, m9 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 +%ifidn %2, v + vmovdqu8 [t0+strideq*2]{k4}, m8 ; p5 +%else + vpblendmb m8{k4}, m2, m8 + mova [rsp+1*64], m8 +%endif + + ; sub p6*2, add p3/q1 [reuse p6/p3 from A][-p6,+q1|save] B + ; write -5 + pmaddubsw m14, pbm1_1 + pmaddubsw m22, pbm1_1 + paddw m10, m14 + paddw m11, m22 ; p6*6+p5*2+p4*2+p3*2+p2+p1+p0+q0 + punpcklbw m8, m24, m6 + punpckhbw m9, m24, m6 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p6*5+p5*2+p4*2+p3*2+p2+p1+p0+q0+q1 + SWAP m18, m8 + SWAP m23, m9 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 +%ifidn %2, v + vmovdqu8 [t0+stride3q]{k4}, m8 ; p4 +%else + vpblendmb m8{k4}, m7, m8 + mova [rsp+2*64], m8 +%endif + + ; sub p6/p5, add p2/q2 [-p6,+p2][-p5,+q2|save] C + ; write -4 + SWAP m14, m16 + punpcklbw m8, m24, m13 + punpckhbw m9, m24, m13 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p6*4+p5*2+p4*2+p3*2+p2*2+p1+p0+q0+q1 + punpcklbw m8, m2, m14 + punpckhbw m2, m14 + pmaddubsw m8, pbm1_1 + pmaddubsw m2, pbm1_1 + paddw m10, m8 + paddw m11, m2 ; p6*4+p5+p4*2+p3*2+p2*2+p1+p0+q0+q1+q2 + SWAP m16, m8 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 +%ifidn %2, v + vmovdqu8 [t0+strideq*4]{k4}, m8 ; p3 +%else + vpblendmb m8{k4}, m25, m8 + mova [rsp+3*64], m8 +%endif + + ; sub p6/p4, add p1/q3 [-p6,+p1][-p4,+q3|save] D + ; write -3 + SWAP m22, m17 + punpcklbw m8, m24, m3 + punpckhbw m9, m24, m3 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p6*3+p5+p4*2+p3*2+p2*2+p1*2+p0+q0+q1+q2 + punpcklbw m8, m7, m22 + punpckhbw m7, m22 + pmaddubsw m8, pbm1_1 + pmaddubsw m7, pbm1_1 + paddw m10, m8 + paddw m11, m7 ; p6*3+p5+p4+p3*2+p2*2+p1*2+p0+q0+q1+q2+q3 + SWAP m17, m8 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 + vpblendmb m15{k4}, m13, m8 ; don't clobber p2/m13 since we need it in F + + ; sub p6/p3, add p0/q4 [-p6,+p0][-p3,+q4|save] E + ; write -2 +%ifidn %2, v + lea t0, [dstq+strideq*4] +%endif + punpcklbw m8, m24, m4 + punpckhbw m9, m24, m4 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p6*2+p5+p4+p3*2+p2*2+p1*2+p0*2+q0+q1+q2+q3 + punpcklbw m8, m25, m29 + punpckhbw m9, m25, m29 + SWAP m26, m29 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p6*2+p5+p4+p3+p2*2+p1*2+p0*2+q0+q1+q2+q3+q4 + SWAP m29, m8 + SWAP m0, m9 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 + vpblendmb m12{k4}, m3, m8 ; don't clobber p1/m3 since we need it in G + + ; sub p6/p2, add q0/q5 [-p6,+q0][-p2,+q5|save] F + ; write -1 +%ifidn %2, h + SWAP m28, m24 + punpcklbw m8, m28, m5 + punpckhbw m24, m28, m5 +%else + punpcklbw m8, m24, m5 + punpckhbw m24, m5 +%endif + pmaddubsw m8, pbm1_1 + pmaddubsw m24, pbm1_1 + paddw m10, m8 + paddw m11, m24 ; p6+p5+p4+p3+p2*2+p1*2+p0*2+q0*2+q1+q2+q3+q4 + punpcklbw m24, m13, m30 + punpckhbw m9, m13, m30 +%ifidn %2, h + SWAP m27, m30 +%endif + SWAP m13, m15 + pmaddubsw m24, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m24 + paddw m11, m9 ; p6+p5+p4+p3+p2+p1*2+p0*2+q0*2+q1+q2+q3+q4+q5 + SWAP m30, m24 + SWAP m15, m9 +%ifidn %2, h + SWAP m9, m24 + %define pw2048 m9 +%endif + pmulhrsw m24, m10, pw2048 + pmulhrsw m8, m11, pw2048 + paddw m10, m18 ; p5+p4+p3+p2+p1*2+p0*2+q0*2+q1*2+q2+q3+q4+q5 + paddw m11, m23 + packuswb m24, m8 + punpcklbw m8, m3, m31 + pmaddubsw m8, pbm1_1 + paddw m10, m8 ; p5+p4+p3+p2+p1+p0*2+q0*2+q1*2+q2+q3+q4+q5+q6 + SWAP m18, m8 + pmulhrsw m8, m10, pw2048 + paddw m10, m16 ; p4+p3+p2+p1+p0*2+q0*2+q1*2+q2*2+q3+q4+q5+q6 +%ifidn %2, h + SWAP m16, m9 + %define pw2048 m16 +%endif + punpckhbw m9, m3, m31 + SWAP m3, m12 + pmaddubsw m9, pbm1_1 + paddw m11, m9 ; p5+p4+p3+p2+p1+p0*2+q0*2+q1*2+q2+q3+q4+q5+q6 + SWAP m23, m9 + pmulhrsw m9, m11, pw2048 + paddw m11, m2 ; p4+p3+p2+p1+p0*2+q0*2+q1*2+q2*2+q3+q4+q5+q6 +%ifidn %2, h + SWAP m2, m1 + %define pbm1_1 m2 +%endif + vpblendmb m1{k4}, m4, m24 ; don't clobber p0/m4 since we need it in H + + ; sub p6/p1, add q1/q6 [reuse -p6,+q1 from B][-p1,+q6|save] G + ; write +0 + SWAP m24, m31 ; q6 + packuswb m8, m9 +%ifidn %2, h + SWAP m31, m2 + %define pbm1_1 m31 +%endif + vpblendmb m12{k4}, m5, m8 ; don't clobber q0/m5 since we need it in I + + ; sub p5/p0, add q2/q6 [reuse -p5,+q2 from C][-p0,+q6] H + ; write +1 + punpcklbw m8, m4, m24 + punpckhbw m2, m4, m24 + SWAP m4, m1 + pmaddubsw m8, pbm1_1 + pmaddubsw m2, pbm1_1 + paddw m10, m8 + paddw m11, m2 ; p4+p3+p2+p1+p0+q0*2+q1*2+q2*2+q3+q4+q5+q6*2 + pmulhrsw m2, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m2, m9 + vpblendmb m2{k4}, m6, m2 ; don't clobber q1/m6 since we need it in K + + ; sub p4/q0, add q3/q6 [reuse -p4,+q3 from D][-q0,+q6] I + ; write +2 + paddw m10, m17 ; p3+p2+p1+p0+q0*2+q1*2+q2*2+q3*2+q4+q5+q6*2 + paddw m11, m7 + punpcklbw m8, m5, m24 + punpckhbw m9, m5, m24 + SWAP m5, m12 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p3+p2+p1+p0+q0+q1*2+q2*2+q3*2+q4+q5+q6*3 + pmulhrsw m7, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m7, m9 + vpblendmb m7{k4}, m14, m7 ; don't clobber q2/m14 since we need it in K + + ; sub p3/q1, add q4/q6 [reuse -p3,+q4 from E][-q1,+q6] J + ; write +3 + paddw m10, m29 ; p2+p1+p0+q0+q1*2+q2*2+q3*2+q4*2+q5+q6*3 + paddw m11, m0 + punpcklbw m8, m6, m24 + punpckhbw m9, m6, m24 + SWAP 2, 6 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p2+p1+p0+q0+q1+q2*2+q3*2+q4*2+q5+q6*4 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 +%ifidn %2, v + vmovdqu8 [t0+mstrideq]{k4}, m8 +%else + SWAP m29, m16 + %define pw2048 m29 + vpblendmb m16{k4}, m22, m8 +%endif + + ; sub p2/q2, add q5/q6 [reuse -p2,+q5 from F][-q2,+q6] K + ; write +4 + paddw m10, m30 ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + paddw m11, m15 +%ifidn %2, h + SWAP m15, m8 +%endif + punpcklbw m8, m14, m24 + punpckhbw m9, m14, m24 + SWAP 14, 7 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m8, m10, pw2048 + pmulhrsw m9, m11, pw2048 + packuswb m8, m9 +%ifidn %2, v + vmovdqu8 [t0+strideq*0]{k4}, m8 ; q4 +%else + vpblendmb m17{k4}, m26, m8 +%endif + + ; sub p1/q3, add q6*2 [reuse -p1,+q6 from G][-q3,+q6] L + ; write +5 + paddw m10, m18 ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + paddw m11, m23 + punpcklbw m8, m22, m24 + punpckhbw m9, m22, m24 + SWAP m30, m24 + pmaddubsw m8, pbm1_1 + pmaddubsw m9, pbm1_1 + paddw m10, m8 + paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m10, pw2048 + pmulhrsw m11, pw2048 + packuswb m10, m11 +%ifidn %2, v + vmovdqu8 [t0+strideq*1]{k4}, m10 ; q5 +%else + vmovdqu8 m27{k4}, m10 +%endif + +%ifidn %2, v + lea t0, [dstq+mstrideq*4] +%endif +%endif + +%if %1 >= 8 + ; flat8 filter + vpbroadcastd m9, [pb_3_1] + vpbroadcastd m10, [pb_2_1] +%if %1 == 16 + vpbroadcastd m23, [pb_1] + vpbroadcastd m0, [pb_4] +%elifidn %2, h + vpbroadcastd m31, [pb_m1_1] + %define pbm1_1 m31 +%endif + punpcklbw m24, m25, m3 + punpckhbw m26, m25, m3 + pmaddubsw m2, m24, m9 + pmaddubsw m7, m26, m9 ; 3 * p3 + p1 + punpcklbw m8, m13, m4 + punpckhbw m11, m13, m4 + pmaddubsw m8, m10 + pmaddubsw m11, m10 + paddw m2, m8 + paddw m7, m11 ; 3 * p3 + 2 * p2 + p1 + p0 + punpcklbw m8, m5, m0 + punpckhbw m11, m5, m0 + pmaddubsw m8, m23 + pmaddubsw m11, m23 + paddw m2, m8 + paddw m7, m11 ; 3 * p3 + 2 * p2 + p1 + p0 + q0 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 +%if is_h || %1 == 16 + vpblendmb m10{k2}, m13, m8 ; p2 +%endif +%ifidn %2, v + %if %1 == 8 + vmovdqu8 [t0+strideq*1]{k2}, m8 + %else + mova [t0+strideq*1], m10 + %endif +%endif + + pmaddubsw m8, m24, pbm1_1 + pmaddubsw m11, m26, pbm1_1 + paddw m2, m8 + paddw m7, m11 + punpcklbw m8, m13, m6 + punpckhbw m11, m13, m6 + pmaddubsw m8, pbm1_1 + pmaddubsw m11, pbm1_1 + paddw m2, m8 + paddw m7, m11 ; 2 * p3 + p2 + 2 * p1 + p0 + q0 + q1 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendmb m8{k2}, m3, m8 ; p1 +%ifidn %2, v + mova [t0+strideq*2], m8 +%else + SWAP m18, m8 +%endif + + pmaddubsw m24, m23 + pmaddubsw m26, m23 + psubw m2, m24 + psubw m7, m26 + punpcklbw m8, m4, m14 + punpckhbw m11, m4, m14 + pmaddubsw m8, m23 + pmaddubsw m11, m23 + paddw m2, m8 + paddw m7, m11 ; p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendmb m8{k2}, m4, m8 ; p0 +%ifidn %2, v + mova [t0+stride3q], m8 +%else + SWAP m29, m8 +%endif + + punpcklbw m24, m5, m22 + punpckhbw m26, m5, m22 + pmaddubsw m8, m24, m23 + pmaddubsw m11, m26, m23 + paddw m2, m8 + paddw m7, m11 + punpcklbw m8, m4, m25 + punpckhbw m11, m4, m25 + pmaddubsw m8, m23 + pmaddubsw m11, m23 + psubw m2, m8 + psubw m7, m11 ; p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + 4 + psrlw m8, m2, 3 + psrlw m11, m7, 3 + packuswb m8, m11 + vpblendmb m11{k2}, m5, m8 ; q0 +%ifidn %2, v + mova [dstq+strideq*0], m11 +%endif + + pmaddubsw m24, pbm1_1 + pmaddubsw m26, pbm1_1 + paddw m2, m24 + paddw m7, m26 + punpcklbw m8, m13, m6 + punpckhbw m13, m6 + pmaddubsw m8, pbm1_1 + pmaddubsw m13, pbm1_1 + paddw m2, m8 + paddw m7, m13 ; p1 + p0 + q0 + 2 * q1 + q2 + 2 * q3 + 4 + psrlw m8, m2, 3 + psrlw m13, m7, 3 + packuswb m8, m13 + vpblendmb m13{k2}, m6, m8 ; q1 +%ifidn %2, v + mova [dstq+strideq*1], m13 +%endif + + punpcklbw m24, m3, m6 + punpckhbw m26, m3, m6 + pmaddubsw m24, m23 + pmaddubsw m26, m23 + psubw m2, m24 + psubw m7, m26 + punpcklbw m24, m14, m22 + punpckhbw m26, m14, m22 + pmaddubsw m24, m23 + pmaddubsw m26, m23 + paddw m2, m24 + paddw m7, m26 ; p0 + q0 + q1 + q2 + 2 * q2 + 3 * q3 + 4 + psrlw m2, 3 + psrlw m7, 3 + packuswb m2, m7 +%if is_h || %1 == 16 + vpblendmb m2{k2}, m14, m2 ; q2 +%endif +%ifidn %2, v + %if %1 == 8 + vmovdqu8 [dstq+strideq*2]{k2}, m2 + %else + mova [dstq+strideq*2], m2 + %endif +%endif + +%ifidn %2, h + SWAP m24, m18 + SWAP m26, m29 +%if %1 == 8 + ; 16x8 transpose + punpcklbw m3, m25, m10 + punpckhbw m25, m10 + punpcklbw m10, m24, m26 + punpckhbw m24, m26 + punpcklbw m26, m11, m13 + punpckhbw m11, m13 + punpcklbw m13, m2, m22 + punpckhbw m2, m22 + ; + punpcklwd m22, m3, m10 + punpckhwd m3, m10 + punpcklwd m10, m25, m24 + punpckhwd m25, m24 + punpcklwd m24, m26, m13 + punpckhwd m26, m13 + punpcklwd m13, m11, m2 + punpckhwd m11, m2 + ; + punpckldq m2, m22, m24 + punpckhdq m22, m24 + punpckldq m24, m3, m26 + punpckhdq m3, m26 + punpckldq m26, m10, m13 + punpckhdq m10, m13 + punpckldq m13, m25, m11 + punpckhdq m25, m11 + ; write 8x32 + vpbroadcastd ym16, strided + pmulld ym16, [hmulD] + lea t1, [dstq+strideq*2] + lea t2, [dstq+strideq*4] + lea t3, [t1 +strideq*4] + lea t0, [dstq+strideq*8] + kmovb k1, k6 + kmovb k2, k6 + kmovb k3, k6 + kmovb k4, k6 + vpscatterdq [dstq+ym16-4]{k1}, m2 + vpscatterdq [t1 +ym16-4]{k2}, m22 + vpscatterdq [t2 +ym16-4]{k3}, m24 + vpscatterdq [t3 +ym16-4]{k4}, m3 + lea t1, [t0+strideq*2] + lea t2, [t0+strideq*4] + lea t3, [t1+strideq*4] + kmovb k1, k6 + kmovb k2, k6 + kmovb k3, k6 + kmovb k4, k6 + vpscatterdq [t0+ym16-4]{k1}, m26 + vpscatterdq [t1+ym16-4]{k2}, m10 + vpscatterdq [t2+ym16-4]{k3}, m13 + vpscatterdq [t3+ym16-4]{k4}, m25 +%else + ; 16x16 transpose and store + SWAP 5, 10, 2 + SWAP 6, 24 + SWAP 7, 26 + SWAP 8, 11 + SWAP 9, 13 + mova m24, [rsp+0*64] + SWAP m26, m28 + mova m2, [rsp+1*64] + mova m3, [rsp+2*64] + mova m4, [rsp+3*64] + SWAP m11, m16 + SWAP m25, m17 + SWAP m13, m27 + SWAP m14, m30 + TRANSPOSE_16X16B 1, 0, [rsp+4*64] + movu [dstq+strideq*0-8], xm24 + movu [dstq+strideq*1-8], xm26 + movu [dstq+strideq*2-8], xm2 + movu [dstq+stride3q -8], xm3 + lea t0, [dstq+strideq*4] + movu [t0+strideq*0-8], xm4 + movu [t0+strideq*1-8], xm5 + movu [t0+strideq*2-8], xm6 + movu [t0+stride3q -8], xm7 + lea t0, [t0+strideq*4] + movu [t0+strideq*0-8], xm8 + movu [t0+strideq*1-8], xm9 + movu [t0+strideq*2-8], xm10 + movu [t0+stride3q -8], xm11 + lea t0, [t0+strideq*4] + movu [t0+strideq*0-8], xm25 + movu [t0+strideq*1-8], xm13 + movu [t0+strideq*2-8], xm14 + movu [t0+stride3q -8], xm22 + lea t0, [t0+strideq*4] + vextracti128 [t0+strideq*0-8], ym24, 1 + vextracti128 [t0+strideq*1-8], ym26, 1 + vextracti128 [t0+strideq*2-8], ym2, 1 + vextracti128 [t0+stride3q -8], ym3, 1 + lea t0, [t0+strideq*4] + vextracti128 [t0+strideq*0-8], ym4, 1 + vextracti128 [t0+strideq*1-8], ym5, 1 + vextracti128 [t0+strideq*2-8], ym6, 1 + vextracti128 [t0+stride3q -8], ym7, 1 + lea t0, [t0+strideq*4] + vextracti128 [t0+strideq*0-8], ym8, 1 + vextracti128 [t0+strideq*1-8], ym9, 1 + vextracti128 [t0+strideq*2-8], ym10, 1 + vextracti128 [t0+stride3q -8], ym11, 1 + lea t0, [t0+strideq*4] + vextracti128 [t0+strideq*0-8], ym25, 1 + vextracti128 [t0+strideq*1-8], ym13, 1 + vextracti128 [t0+strideq*2-8], ym14, 1 + vextracti128 [t0+stride3q -8], ym22, 1 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m24, 2 + vextracti32x4 [t0+strideq*1-8], m26, 2 + vextracti32x4 [t0+strideq*2-8], m2, 2 + vextracti32x4 [t0+stride3q -8], m3, 2 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m4, 2 + vextracti32x4 [t0+strideq*1-8], m5, 2 + vextracti32x4 [t0+strideq*2-8], m6, 2 + vextracti32x4 [t0+stride3q -8], m7, 2 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m8, 2 + vextracti32x4 [t0+strideq*1-8], m9, 2 + vextracti32x4 [t0+strideq*2-8], m10, 2 + vextracti32x4 [t0+stride3q -8], m11, 2 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m25, 2 + vextracti32x4 [t0+strideq*1-8], m13, 2 + vextracti32x4 [t0+strideq*2-8], m14, 2 + vextracti32x4 [t0+stride3q -8], m22, 2 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m24, 3 + vextracti32x4 [t0+strideq*1-8], m26, 3 + vextracti32x4 [t0+strideq*2-8], m2, 3 + vextracti32x4 [t0+stride3q -8], m3, 3 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m4, 3 + vextracti32x4 [t0+strideq*1-8], m5, 3 + vextracti32x4 [t0+strideq*2-8], m6, 3 + vextracti32x4 [t0+stride3q -8], m7, 3 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m8, 3 + vextracti32x4 [t0+strideq*1-8], m9, 3 + vextracti32x4 [t0+strideq*2-8], m10, 3 + vextracti32x4 [t0+stride3q -8], m11, 3 + lea t0, [t0+strideq*4] + vextracti32x4 [t0+strideq*0-8], m25, 3 + vextracti32x4 [t0+strideq*1-8], m13, 3 + vextracti32x4 [t0+strideq*2-8], m14, 3 + vextracti32x4 [t0+stride3q -8], m22, 3 +%endif +%endif + +%elif %1 == 6 + ; flat6 filter + vpbroadcastd m15, [pb_3_1] + vpbroadcastd m12, [pb_2] + punpcklbw m8, m13, m5 + punpckhbw m11, m13, m5 + pmaddubsw m0, m8, m15 + pmaddubsw m1, m11, m15 + punpcklbw m7, m4, m3 + punpckhbw m10, m4, m3 + pmaddubsw m2, m7, m12 + pmaddubsw m12, m10, m12 +%ifidn %2, h + vpbroadcastd m15, [pb_m1_1] + %define pbm1_1 m15 +%endif + paddw m0, m2 + paddw m1, m12 + pmulhrsw m2, m0, m16 + pmulhrsw m12, m1, m16 + packuswb m2, m12 + vpblendmb m2{k2}, m3, m2 ; p1 +%ifidn %2, v + mova [t0+strideq*2], m2 +%endif + + pmaddubsw m8, pbm1_1 + pmaddubsw m11, pbm1_1 + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m13, m6 + punpckhbw m11, m13, m6 + pmaddubsw m8, pbm1_1 + pmaddubsw m11, pbm1_1 + paddw m0, m8 + paddw m1, m11 + pmulhrsw m12, m0, m16 + pmulhrsw m13, m1, m16 + packuswb m12, m13 + vpblendmb m12{k2}, m4, m12 ; p0 +%ifidn %2, v + mova [t0+stride3q], m12 +%endif + + vpbroadcastd m9, [pb_m1_2] + vpbroadcastd m4, [pb_m1_0] + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m3, m14 + punpckhbw m11, m3, m14 + pmaddubsw m14, m8, pbm1_1 + pmaddubsw m13, m11, pbm1_1 + paddw m0, m14 + paddw m1, m13 + pmulhrsw m14, m0, m16 + pmulhrsw m13, m1, m16 + packuswb m14, m13 + vpblendmb m14{k2}, m5, m14 ; q0 +%ifidn %2, v + mova [dstq+strideq*0], m14 +%endif + + pmaddubsw m8, m9 + pmaddubsw m11, m9 + paddw m0, m8 + paddw m1, m11 + pmaddubsw m7, m4 + pmaddubsw m10, m4 + paddw m0, m7 + paddw m1, m10 + pmulhrsw m0, m16 + pmulhrsw m1, m16 + packuswb m0, m1 + vpblendmb m0{k2}, m6, m0 ; q1 +%ifidn %2, v + mova [dstq+strideq*1], m0 +%else + TRANSPOSE_16x4_AND_WRITE_4x32 2, 12, 14, 0, 1 +%endif +%else ; %1 == 4 +%ifidn %2, v + mova [t0+strideq*0], m3 ; p1 + mova [t0+strideq*1], m4 ; p0 + mova [t0+strideq*2], m5 ; q0 + mova [t0+stride3q ], m6 ; q1 +%else + TRANSPOSE_16x4_AND_WRITE_4x32 3, 4, 5, 6, 7 +%endif +%endif +%endmacro + +%define k7 k6 + +INIT_ZMM avx512icl +cglobal lpf_v_sb_y_8bpc, 7, 10, 32, dst, stride, mask, l, l_stride, \ + lut, w, stride3, mstride + DECLARE_REG_TMP 9 + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + mova m21, [pb_4x0_4x4_4x8_4x12] + mova m20, [pb_mask] + vpbroadcastd m19, [pb_128] + vpbroadcastd m28, [pb_m1_1] + vpbroadcastd m27, [pw_2048] + %define pbshuf m21 + %define pbmask m20 + %define pb128 m19 + %define pbm1_1 m28 + %define pw2048 m27 + +.loop: + cmp word [maskq+8], 0 ; vmask[2] + je .no_flat16 + + FILTER 16, v + jmp .end + +.no_flat16: + cmp word [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 8, v + jmp .end + +.no_flat: + cmp word [maskq+0], 0 ; vmask[0] + je .end + + call .v4 + +.end: + add lq, 64 + add dstq, 64 + add maskq, 2 + sub wd, 16 + jg .loop + RET +ALIGN function_align +RESET_MM_PERMUTATION +.v4: + FILTER 4, v + ret + +cglobal lpf_h_sb_y_8bpc, 7, 13, 32, 5*64, dst, stride, mask, l, l_stride, \ + lut, h, stride3, stride8 + DECLARE_REG_TMP 9, 10, 11, 12 + shl l_strideq, 2 + sub lq, 4 + lea stride3q, [strideq*3] + lea stride8q, [strideq*8] + kxnorw k6, k6, k6 + vpbroadcastd m19, strided + vpbroadcastd m20, l_strided + pmulld m21, m19, [hmulA] + pmulld m20, [hmulB] + pmulld m19, [hmulC] + %define pbshuf [pb_4x0_4x4_4x8_4x12] + %define pbmask [pb_mask] + %define pb128 [pb_128]{bcstd} + shl l_strideq, 1 + +.loop: + cmp word [maskq+8], 0 ; vmask[2] + je .no_flat16 + + FILTER 16, h + jmp .end + +.no_flat16: + cmp word [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 8, h + jmp .end + +.no_flat: + cmp word [maskq+0], 0 ; vmask[0] + je .end + + call .h4 + +.end: + lea lq, [lq+l_strideq*8] + lea dstq, [dstq+stride8q*8] + add maskq, 2 + sub hd, 16 + jg .loop + RET +ALIGN function_align +RESET_MM_PERMUTATION +.h4: + FILTER 4, h + ret + +cglobal lpf_v_sb_uv_8bpc, 7, 10, 22, dst, stride, mask, l, l_stride, \ + lut, w, stride3, mstride + DECLARE_REG_TMP 9 + shl l_strideq, 2 + sub lq, l_strideq + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] + mova m21, [pb_4x0_4x4_4x8_4x12] + mova m20, [pb_mask] + vpbroadcastd m19, [pb_128] + vpbroadcastd m17, [pb_m1_1] + vpbroadcastd m16, [pw_4096] + %define pbshuf m21 + %define pbmask m20 + %define pb128 m19 + %define pbm1_1 m17 + +.loop: + cmp word [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 6, v + jmp .end + +.no_flat: + cmp word [maskq+0], 0 ; vmask[0] + je .end + + call mangle(private_prefix %+ _lpf_v_sb_y_8bpc_avx512icl).v4 + +.end: + add lq, 64 + add dstq, 64 + add maskq, 2 + sub wd, 16 + jg .loop + RET + +%undef k7 +cglobal lpf_h_sb_uv_8bpc, 7, 12, 22, dst, stride, mask, l, l_stride, \ + lut, h, stride3, stride8 + DECLARE_REG_TMP 9, 10, 11 + mov r7d, 0xffff + movzx r8d, r7b + cmp hd, 9 + cmovb r7d, r8d + kmovw k6, r7d ; h > 8 ? 0xffff : 0x00ff + shl l_strideq, 2 + sub lq, 4 + kshiftrw k7, k6, 4 ; h > 8 ? 0xff : 0xf0 + lea stride3q, [strideq*3] + lea stride8q, [strideq*8] + vpbroadcastd m19, strided + vpbroadcastd m20, l_strided + pmulld m21, m19, [hmulA] + pmulld m20, [hmulB] + pmulld m19, [hmulC] + mova m18, [pb_mask] + vpbroadcastd m17, [pb_128] + vpbroadcastd m16, [pw_4096] + %define pbshuf [pb_4x0_4x4_4x8_4x12] + %define pbmask m18 + %define pb128 m17 + add l_strideq, l_strideq + +.loop: + cmp word [maskq+4], 0 ; vmask[1] + je .no_flat + + FILTER 6, h + jmp .end + +.no_flat: + cmp word [maskq+0], 0 ; vmask[0] + je .end + + call mangle(private_prefix %+ _lpf_h_sb_y_8bpc_avx512icl).h4 + +.end: + lea lq, [lq+l_strideq*8] + lea dstq, [dstq+stride8q*8] + add maskq, 2 + sub hd, 16 + jg .loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/loopfilter_sse.asm b/third_party/dav1d/src/x86/loopfilter_sse.asm new file mode 100644 index 0000000000..cd0eb54702 --- /dev/null +++ b/third_party/dav1d/src/x86/loopfilter_sse.asm @@ -0,0 +1,2348 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +pb_4x0_4x4_4x8_4x12: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 +pb_7_1: times 8 db 7, 1 +pb_3_1: times 8 db 3, 1 +pb_2_1: times 8 db 2, 1 +pb_m1_0: times 8 db -1, 0 +pb_m1_1: times 8 db -1, 1 +pb_m1_2: times 8 db -1, 2 +pb_1: times 16 db 1 +pb_2: times 16 db 2 +pb_3: times 16 db 3 +pb_4: times 16 db 4 +pb_16: times 16 db 16 +pb_63: times 16 db 63 +pb_64: times 16 db 64 +pb_128: times 16 db 0x80 +pb_129: times 16 db 0x81 +pb_240: times 16 db 0xf0 +pb_248: times 16 db 0xf8 +pb_254: times 16 db 0xfe + +pw_2048: times 8 dw 2048 +pw_4096: times 8 dw 4096 + +pd_mask: dd 1, 2, 4, 8 + +SECTION .text + +%macro ABSSUB 4 ; dst, a, b, tmp + psubusb %1, %2, %3 + psubusb %4, %3, %2 + por %1, %4 +%endmacro + +%macro TRANSPOSE_16x4_AND_WRITE_4x16 5 + ; transpose 16x4 + punpcklbw m%5, m%1, m%2 + punpckhbw m%1, m%2 + punpcklbw m%2, m%3, m%4 + punpckhbw m%3, m%4 + punpcklwd m%4, m%5, m%2 + punpckhwd m%5, m%2 + punpcklwd m%2, m%1, m%3 + punpckhwd m%1, m%3 + + ; write out +%assign %%n 0 +%rep 4 + movd [dstq+strideq *0-2], xm%4 + movd [dstq+strideq *4-2], xm%5 + movd [dstq+strideq *8-2], xm%2 + movd [dstq+stride3q*4-2], xm%1 + add dstq, strideq +%if %%n < 3 + psrldq xm%4, 4 + psrldq xm%5, 4 + psrldq xm%2, 4 + psrldq xm%1, 4 +%endif +%assign %%n (%%n+1) +%endrep + lea dstq, [dstq+stride3q*4] +%endmacro + +%macro TRANSPOSE_16X16B 2 ; output_transpose, mem +%if %1 == 0 + mova %2, m15 ; m7 in 32-bit +%endif + + ; input in m0-7 + punpcklbw m15, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + punpcklbw m3, m4, m5 + punpckhbw m4, m5 +%if ARCH_X86_64 + SWAP 4, 5, 7 +%else + %if %1 == 0 + mova m5, %2 + %else + mova m5, [esp+1*16] + %endif + mova %2, m4 +%endif + punpcklbw m4, m6, m5 + punpckhbw m6, m5 + + ; interleaved in m15,0,1,2,3,7,4,6 + punpcklwd m5, m15, m1 + punpckhwd m15, m1 + punpcklwd m1, m0, m2 + punpckhwd m0, m2 + punpcklwd m2, m3, m4 + punpckhwd m3, m4 +%if ARCH_X86_64 + SWAP 3, 4, 7 +%else + mova m4, %2 + mova %2, m3 +%endif + punpcklwd m3, m4, m6 + punpckhwd m4, m6 + + ; interleaved in m5,15,1,0,2,7,3,4 + punpckldq m6, m5, m2 + punpckhdq m5, m2 +%if ARCH_X86_64 + SWAP 2, 7, 5 +%else + mova m2, %2 + mova [esp+1*16], m5 +%endif + punpckldq m5, m15, m2 + punpckhdq m15, m2 + punpckldq m2, m1, m3 + punpckhdq m1, m3 + punpckldq m3, m0, m4 + punpckhdq m0, m4 + +%if ARCH_X86_32 + mova [esp+0*16], m6 + mova [esp+2*16], m5 + mova [esp+3*16], m15 + mova [esp+4*16], m2 + mova [esp+5*16], m1 + mova [esp+6*16], m3 + mova [esp+7*16], m0 + mova m8, [esp+ 8*16] + mova m9, [esp+ 9*16] + mova m10, [esp+10*16] + %if %1 == 0 + mova m11, [esp+11*16] + mova m12, [esp+12*16] + mova m13, [esp+13*16] + mova m14, [esp+14*16] + %else + mova m11, [esp+20*16] + mova m12, [esp+15*16] + mova m13, [esp+16*16] + mova m14, [esp+17*16] + %endif +%endif + + ; input in m8-m15 +%if ARCH_X86_64 + SWAP 7, 4 +%endif + punpcklbw m7, m8, m9 + punpckhbw m8, m9 + punpcklbw m9, m10, m11 + punpckhbw m10, m11 + punpcklbw m11, m12, m13 + punpckhbw m12, m13 +%if ARCH_X86_64 + mova m13, %2 +%else + %if %1 == 0 + mova m13, [esp+15*16] + %else + mova m13, [esp+18*16] + %endif +%endif + mova %2, m12 + punpcklbw m12, m14, m13 + punpckhbw m14, m14, m13 + + ; interleaved in m7,8,9,10,11,rsp%2,12,14 + punpcklwd m13, m7, m9 + punpckhwd m7, m9 + punpcklwd m9, m8, m10 + punpckhwd m8, m10 + punpcklwd m10, m11, m12 + punpckhwd m11, m12 + mova m12, %2 + mova %2, m11 + punpcklwd m11, m12, m14 + punpckhwd m12, m14 + + ; interleaved in m13,7,9,8,10,rsp%2,11,12 + punpckldq m14, m13, m10 + punpckhdq m13, m10 + punpckldq m10, m9, m11 + punpckhdq m9, m11 + punpckldq m11, m8, m12 + punpckhdq m8, m12 + mova m12, %2 + mova %2, m8 + punpckldq m8, m7, m12 + punpckhdq m7, m12 + +%if ARCH_X86_32 + mova [esp+ 8*16], m10 + mova [esp+ 9*16], m9 + mova [esp+10*16], m11 + SWAP 6, 1 + SWAP 4, 2 + SWAP 5, 3 + mova m6, [esp+0*16] + mova m4, [esp+1*16] + mova m5, [esp+2*16] +%endif + + ; interleaved in m6,7,5,15,2,1,3,0,14,13,10,9,11,rsp%2,8,7 + punpcklqdq m12, m6, m14 + punpckhqdq m6, m14 + punpcklqdq m14, m4, m13 + punpckhqdq m4, m13 + punpcklqdq m13, m5, m8 + punpckhqdq m5, m8 +%if ARCH_X86_64 + SWAP 8, 5 +%else + mova m8, [esp+3*16] + mova [esp+27*16], m5 + %define m15 m8 +%endif + punpcklqdq m5, m15, m7 + punpckhqdq m15, m7 + +%if ARCH_X86_32 + mova [esp+11*16], m12 + mova [esp+12*16], m6 + mova [esp+13*16], m14 + mova [esp+14*16], m4 + mova [esp+26*16], m13 + mova [esp+ 0*16], m5 + mova [esp+ 1*16], m15 + mova m2, [esp+ 4*16] + mova m10, [esp+ 8*16] + mova m1, [esp+ 5*16] + mova m9, [esp+ 9*16] + mova m3, [esp+ 6*16] + mova m11, [esp+10*16] + mova m0, [esp+ 7*16] +%endif + + punpcklqdq m7, m2, m10 + punpckhqdq m2, m10 + punpcklqdq m10, m1, m9 + punpckhqdq m1, m9 + punpcklqdq m9, m3, m11 + punpckhqdq m3, m11 + mova m11, %2 +%if ARCH_X86_32 + %define m12 m3 +%endif + mova %2, m12 + punpcklqdq m12, m0, m11 + punpckhqdq m0, m11 +%if %1 == 1 + mova m11, %2 +%endif + +%if ARCH_X86_64 + ; interleaved m11,6,14,4,13,8,5,15,7,2,10,1,9,3,12,0 + SWAP 0, 11, 1, 6, 5, 8, 7, 15 + SWAP 2, 14, 12, 9 + SWAP 3, 4, 13 +%else + %if %1 == 0 + mova [esp+15*16], m9 + mova [esp+17*16], m12 + mova [esp+18*16], m0 + mova [esp+28*16], m10 + mova [esp+29*16], m1 + mova m3, [esp+0*16] + mova m4, [esp+1*16] + SWAP m5, m7 + SWAP m6, m2 + %else + SWAP 0, 7 + SWAP 3, 1, 2, 4, 6 + %endif +%endif +%endmacro + +%macro FILTER 2 ; width [4/6/8/16], dir [h/v] +%if ARCH_X86_64 + %define %%flat8mem [rsp+0*16] + %define %%q2mem [rsp+1*16] + %define %%q3mem [rsp+2*16] +%else + %if %1 == 4 || %1 == 6 + %define %%p2mem [esp+ 8*16] + %define %%q2mem [esp+ 9*16] + %define %%flat8mem [esp+10*16] + %else + %ifidn %2, v + %define %%p2mem [esp+16*16] + %define %%q2mem [esp+ 1*16] + %define %%q3mem [esp+18*16] + %define %%flat8mem [esp+ 0*16] + %define %%flat16mem [esp+20*16] + %else + %define %%p2mem [esp+27*16] + %define %%q2mem [esp+28*16] + %define %%q3mem [esp+29*16] + %define %%flat8mem [esp+21*16] + %define %%flat16mem [esp+30*16] + %endif + %endif + %xdefine m12reg m12 +%endif + +%if ARCH_X86_32 + lea stride3q, [strideq*3] +%endif + ; load data +%ifidn %2, v +%if ARCH_X86_32 + mov mstrideq, strideq + neg mstrideq +%endif +%if %1 == 4 + lea tmpq, [dstq+mstrideq*2] + mova m3, [tmpq+strideq*0] ; p1 + mova m4, [tmpq+strideq*1] ; p0 + mova m5, [tmpq+strideq*2] ; q0 + mova m6, [tmpq+stride3q] ; q1 +%else + ; load 6-8 pixels, remainder (for wd=16) will be read inline + lea tmpq, [dstq+mstrideq*4] + ; we load p3 later +%define %%p3mem [dstq+mstrideq*4] + %if ARCH_X86_32 + %define m13 m0 + %define m14 m1 + %define m15 m2 + %endif + mova m13, [tmpq+strideq*1] + mova m3, [tmpq+strideq*2] + mova m4, [tmpq+stride3q] + mova m5, [dstq+strideq*0] + mova m6, [dstq+strideq*1] + mova m14, [dstq+strideq*2] +%if %1 != 6 + mova m15, [dstq+stride3q] +%endif + %if ARCH_X86_32 + mova %%p2mem, m13 + mova %%q2mem, m14 + %define m13 %%p2mem + %define m14 %%q2mem + %if %1 != 6 + mova %%q3mem, m15 + %define m15 %%q3mem + %endif + %endif +%endif +%else ; %2 == h + ; load lines +%if %1 == 4 + ; transpose 4x16 + movd m7, [dstq+strideq*0-2] + movd m3, [dstq+strideq*1-2] + movd m4, [dstq+strideq*2-2] + movd m5, [dstq+stride3q -2] + lea tmpq, [dstq+strideq*4] + punpcklbw m7, m3 + punpcklbw m4, m5 + movd m3, [tmpq+strideq*0-2] + movd m1, [tmpq+strideq*1-2] + movd m5, [tmpq+strideq*2-2] + movd m6, [tmpq+stride3q -2] + lea tmpq, [tmpq+strideq*4] + punpcklbw m3, m1 + punpcklbw m5, m6 + movd m0, [tmpq+strideq*0-2] + movd m1, [tmpq+strideq*1-2] + punpcklbw m0, m1 + movd m1, [tmpq+strideq*2-2] + movd m2, [tmpq+stride3q -2] + punpcklbw m1, m2 + punpcklqdq m7, m0 + punpcklqdq m4, m1 + lea tmpq, [tmpq+strideq*4] + movd m0, [tmpq+strideq*0-2] + movd m1, [tmpq+strideq*1-2] + punpcklbw m0, m1 + movd m1, [tmpq+strideq*2-2] + movd m2, [tmpq+stride3q -2] + punpcklbw m1, m2 + punpcklqdq m3, m0 + punpcklqdq m5, m1 + ; xm7: A0-1,B0-1,C0-1,D0-1,A8-9,B8-9,C8-9,D8-9 + ; xm3: A4-5,B4-5,C4-5,D4-5,A12-13,B12-13,C12-13,D12-13 + ; xm4: A2-3,B2-3,C2-3,D2-3,A10-11,B10-11,C10-11,D10-11 + ; xm5: A6-7,B6-7,C6-7,D6-7,A14-15,B14-15,C14-15,D14-15 + punpcklwd m6, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m3, m5 + punpckhwd m3, m5 + ; xm6: A0-3,B0-3,C0-3,D0-3 + ; xm7: A8-11,B8-11,C8-11,D8-11 + ; xm4: A4-7,B4-7,C4-7,D4-7 + ; xm3: A12-15,B12-15,C12-15,D12-15 + punpckldq m5, m6, m4 + punpckhdq m6, m4 + punpckldq m4, m7, m3 + punpckhdq m7, m3 + ; xm5: A0-7,B0-7 + ; xm6: C0-7,D0-7 + ; xm4: A8-15,B8-15 + ; xm7: C8-15,D8-15 + punpcklqdq m3, m5, m4 + punpckhqdq m5, m5, m4 + punpcklqdq m4, m6, m7 + punpckhqdq m6, m7 + ; xm3: A0-15 + ; xm5: B0-15 + ; xm4: C0-15 + ; xm6: D0-15 + SWAP 4, 5 +%elif %1 == 6 || %1 == 8 + ; transpose 8x16 + movq m7, [dstq+strideq*0-%1/2] + movq m3, [dstq+strideq*1-%1/2] + movq m4, [dstq+strideq*2-%1/2] + movq m5, [dstq+stride3q -%1/2] + lea tmpq, [dstq+strideq*8] + punpcklbw m7, m3 + punpcklbw m4, m5 + movq m3, [tmpq+strideq*0-%1/2] + movq m1, [tmpq+strideq*1-%1/2] + movq m5, [tmpq+strideq*2-%1/2] + movq m6, [tmpq+stride3q -%1/2] + lea tmpq, [dstq+strideq*4] + punpcklbw m3, m1 + punpcklbw m5, m6 + movq m6, [tmpq+strideq*0-%1/2] + movq m0, [tmpq+strideq*1-%1/2] + movq m1, [tmpq+strideq*2-%1/2] + movq m2, [tmpq+stride3q -%1/2] + lea tmpq, [tmpq+strideq*8] + punpcklbw m6, m0 + punpcklbw m1, m2 + movq m2, [tmpq+strideq*2-%1/2] + movq m0, [tmpq+stride3q -%1/2] + punpcklbw m2, m0 +%if ARCH_X86_64 + SWAP m15, m2 +%else + %define m15 [esp+3*16] + mova m15, m2 +%endif + movq m0, [tmpq+strideq*0-%1/2] + movq m2, [tmpq+strideq*1-%1/2] + punpcklbw m0, m2 + ; xm7: A0-1,B0-1,C0-1,D0-1,E0-1,F0-1,G0-1,H0-1 + ; xm3: A8-9,B8-9,C8-9,D8-9,E8-9,F8-9,G8-9,H8-9 + ; xm4: A2-3,B2-3,C2-3,D2-3,E2-3,F2-3,G2-3,H2-3 + ; xm5: A10-11,B10-11,C10-11,D10-11,E10-11,F10-11,G10-11,H10-11 + ; xm6: A4-5,B4-5,C4-5,D4-5,E4-5,F4-5,G4-5,H4-5 + ; xm0: A12-13,B12-13,C12-13,D12-13,E12-13,F12-13,G12-13,H12-13 + ; xm1: A6-7,B6-7,C6-7,D6-7,E6-7,F6-7,G6-7,H6-7 + ; xm2: A14-15,B14-15,C14-15,D14-15,E14-15,F14-15,G14-15,H14-15 + punpcklwd m2, m7, m4 + punpckhwd m7, m4 + punpcklwd m4, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m6, m1 + punpckhwd m6, m1 + punpcklwd m1, m0, m15 + punpckhwd m0, m15 +%if ARCH_X86_64 + SWAP m15, m0 +%else + mova m15, m0 +%endif + ; xm2: A0-3,B0-3,C0-3,D0-3 + ; xm7: E0-3,F0-3,G0-3,H0-3 + ; xm4: A8-11,B8-11,C8-11,D8-11 + ; xm3: E8-11,F8-11,G8-11,H8-11 + ; xm5: A4-7,B4-7,C4-7,D4-7 + ; xm6: E4-7,F4-7,G4-7,H4-7 + ; xm1: A12-15,B12-15,C12-15,D12-15 + ; xm0: E12-15,F12-15,G12-15,H12-15 + punpckldq m0, m2, m5 + punpckhdq m2, m5 + punpckldq m5, m7, m6 +%if %1 != 6 + punpckhdq m7, m6 +%endif + punpckldq m6, m4, m1 + punpckhdq m4, m1 + punpckldq m1, m3, m15 +%if %1 != 6 + punpckhdq m3, m15 + %if ARCH_X86_64 + SWAP m15, m3 + %else + mova m15, m3 + %endif +%endif + ; xm0: A0-7,B0-7 + ; xm2: C0-7,D0-7 + ; xm5: E0-7,F0-7 + ; xm7: G0-7,H0-7 + ; xm6: A8-15,B8-15 + ; xm4: C8-15,D8-15 + ; xm1: E8-15,F8-15 + ; xm3: G8-15,H8-15 + punpcklqdq m3, m0, m6 + punpckhqdq m0, m6 + punpckhqdq m6, m2, m4 + punpcklqdq m2, m4 + punpcklqdq m4, m5, m1 + punpckhqdq m5, m1 +%if %1 == 8 + punpcklqdq m1, m7, m15 + punpckhqdq m7, m15 + ; xm3: A0-15 + ; xm0: B0-15 + ; xm2: C0-15 + ; xm6: D0-15 + ; xm4: E0-15 + ; xm5: F0-15 + ; xm1: G0-15 + ; xm7: H0-15 +%if ARCH_X86_64 + SWAP 11, 3, 2 + SWAP 13, 0 + SWAP 6, 5, 4 + SWAP 14, 1 + SWAP 15, 7 + ; 3,0,2,6,4,5,1,7 -> 11,13,3,4,5,6,14,15 + mova [rsp+21*16], m11 + %define %%p3mem [rsp+21*16] +%else + %define m11 [esp+26*16] + %define m13 [esp+27*16] + %define m14 [esp+28*16] + %define m15 [esp+29*16] + mova m11, m3 + mova m13, m0 + SWAP 3, 2 + SWAP 6, 5, 4 + mova m14, m1 + mova m15, m7 + %define %%p3mem [esp+26*16] +%endif +%else + %if ARCH_X86_64 + SWAP 13, 3, 0 + SWAP 14, 5, 6, 4, 2 + ; 3,0,2,6,4,5 -> 13,3,4,5,6,14 + %else + %define m13 %%p2mem + %define m14 %%q2mem + mova m13, m3 + mova m14, m5 + SWAP 3, 0 + SWAP 5, 6, 4, 2 + ; 0,2,6,4 -> 3,4,5,6 + %endif +%endif +%else +%if ARCH_X86_64 + mova [rsp+20*16], m12 +%endif + ; load and 16x16 transpose. We only use 14 pixels but we'll need the + ; remainder at the end for the second transpose +%if ARCH_X86_32 + %xdefine m8 m0 + %xdefine m9 m1 + %xdefine m10 m2 + %xdefine m11 m3 + %xdefine m12 m4 + %xdefine m13 m5 + %xdefine m14 m6 + %xdefine m15 m7 + lea tmpq, [dstq+strideq*8] + movu m8, [tmpq+strideq*0-8] + movu m9, [tmpq+strideq*1-8] + movu m10, [tmpq+strideq*2-8] + movu m11, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + movu m12, [tmpq+strideq*0-8] + movu m13, [tmpq+strideq*1-8] + movu m14, [tmpq+strideq*2-8] + movu m15, [tmpq+stride3q -8] + mova [esp+ 8*16], m8 + mova [esp+ 9*16], m9 + mova [esp+10*16], m10 + mova [esp+11*16], m11 + mova [esp+12*16], m12 + mova [esp+13*16], m13 + mova [esp+14*16], m14 + mova [esp+15*16], m15 +%endif + movu m0, [dstq+strideq*0-8] + movu m1, [dstq+strideq*1-8] + movu m2, [dstq+strideq*2-8] + movu m3, [dstq+stride3q -8] + lea tmpq, [dstq+strideq*4] + movu m4, [tmpq+strideq*0-8] + movu m5, [tmpq+strideq*1-8] + movu m6, [tmpq+strideq*2-8] + movu m7, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] +%if ARCH_X86_64 + movu m8, [tmpq+strideq*0-8] + movu m9, [tmpq+strideq*1-8] + movu m10, [tmpq+strideq*2-8] + movu m11, [tmpq+stride3q -8] + lea tmpq, [tmpq+strideq*4] + movu m12, [tmpq+strideq*0-8] + movu m13, [tmpq+strideq*1-8] + movu m14, [tmpq+strideq*2-8] + movu m15, [tmpq+stride3q -8] +%endif + +%if ARCH_X86_64 + TRANSPOSE_16X16B 0, [rsp+11*16] + mova [rsp+12*16], m1 + mova [rsp+13*16], m2 + mova [rsp+14*16], m3 + mova [rsp+15*16], m12 + mova [rsp+16*16], m13 + mova [rsp+17*16], m14 + mova [rsp+18*16], m15 + ; 4,5,6,7,8,9,10,11 -> 12,13,3,4,5,6,14,15 + SWAP 12, 4, 7 + SWAP 13, 5, 8 + SWAP 3, 6, 9 + SWAP 10, 14 + SWAP 11, 15 + mova [rsp+21*16], m12 + %define %%p3mem [rsp+21*16] + mova m12, [rsp+20*16] +%else + TRANSPOSE_16X16B 0, [esp+16*16] + %define %%p3mem [esp+26*16] + %define m11 %%p3mem + %define m13 %%p2mem + %define m14 %%q2mem + %define m15 %%q3mem +%endif +%endif ; if 4 elif 6 or 8 else 16 +%endif ; if v else h + + ; load L/E/I/H +%if ARCH_X86_32 + mov l_strideq, l_stridem +%endif +%ifidn %2, v + movu m1, [lq] + movu m0, [lq+l_strideq] +%else + %if ARCH_X86_32 + lea l_stride3q, [l_strideq*3] + %endif + movq xm1, [lq] + movq xm2, [lq+l_strideq*2] + movhps xm1, [lq+l_strideq] + movhps xm2, [lq+l_stride3q] + shufps m0, m1, m2, q3131 + shufps m1, m2, q2020 + %if ARCH_X86_32 + lea stride3q, [strideq*3] + %endif +%endif + +%if ARCH_X86_32 + %ifidn %2, v + mov lutd, lutm + %endif +%endif + pxor m2, m2 + pcmpeqb m7, m2, m0 + pand m1, m7 + por m0, m1 ; l[x][] ? l[x][] : l[x-stride][] + pshufb m0, [PIC_sym(pb_4x0_4x4_4x8_4x12)] ; l[x][1] + pcmpeqb m2, m0 ; !L + psrlq m7, m0, [lutq+128] + pand m7, [PIC_sym(pb_63)] + pminub m7, minlvl + pmaxub m7, [PIC_sym(pb_1)] ; I + pand m1, m0, [PIC_sym(pb_240)] + psrlq m1, 4 ; H + paddb m0, [PIC_sym(pb_2)] + paddb m0, m0 + paddb m0, m7 ; E + pxor m1, [PIC_sym(pb_128)] + pxor m7, [PIC_sym(pb_128)] + pxor m0, [PIC_sym(pb_128)] + SWAP 2, 7 + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 2, 10 +%else + %ifidn %2, v + mov mstrideq, strideq + neg mstrideq + %if %1 == 4 + lea tmpq, [dstq+mstrideq*2] + %elif %1 == 6 || %1 == 8 + lea tmpq, [dstq+mstrideq*4] + %endif + %endif + mova [esp+3*16], m0 + mova [esp+4*16], m2 +%endif + + ABSSUB m0, m3, m4, m2 ; abs(p1-p0) + pmaxub m0, m7 + ABSSUB m2, m5, m6, m7 ; abs(q1-q0) + pmaxub m0, m2 +%if %1 == 4 + pxor m0, [PIC_sym(pb_128)] + pcmpgtb m7, m0, m1 ; hev + %if ARCH_X86_64 + SWAP 7, 11 + %else + mova [esp+5*16], m7 + %endif +%else + pxor m7, m0, [PIC_sym(pb_128)] + pcmpgtb m7, m1 ; hev +%if ARCH_X86_64 + SWAP 7, 11 +%else + mova [esp+5*16], m7 +%endif + +%if %1 == 6 + ABSSUB m1, m13, m4, m7 ; abs(p2-p0) + pmaxub m1, m0 +%else + mova m2, %%p3mem + ABSSUB m1, m2, m4, m7 ; abs(p3-p0) + pmaxub m1, m0 + ABSSUB m7, m13, m4, m2 ; abs(p2-p0) + pmaxub m1, m7 +%endif + ABSSUB m7, m5, m14, m2 ; abs(p2-p0) + pmaxub m1, m7 +%if %1 != 6 + ABSSUB m7, m5, m15, m2 ; abs(q3-q0) + pmaxub m1, m7 +%endif + pxor m1, [PIC_sym(pb_128)] + pcmpgtb m1, [PIC_sym(pb_129)] ; !flat8in +%if ARCH_X86_64 + SWAP 1, 9 +%else + mova [esp+6*16], m1 +%endif + +%if %1 == 6 + ABSSUB m7, m13, m3, m1 ; abs(p2-p1) +%else + mova m2, %%p3mem + ABSSUB m7, m2, m13, m1 ; abs(p3-p2) + ABSSUB m2, m13, m3, m1 ; abs(p2-p1) + pmaxub m7, m2 + ABSSUB m2, m14, m15, m1 ; abs(q3-q2) + pmaxub m7, m2 +%endif + ABSSUB m2, m14, m6, m1 ; abs(q2-q1) + pmaxub m7, m2 +%if ARCH_X86_32 + %define m12 m1 + mova m12, maskmem +%endif + pand m2, m12, mask1 + pcmpeqd m2, m12 + pand m7, m2 ; only apply fm-wide to wd>4 blocks + pmaxub m0, m7 + + pxor m0, [PIC_sym(pb_128)] +%endif ; %if %1 == 4 else +%if ARCH_X86_64 + SWAP 2, 10 + pcmpgtb m0, m2 +%else + pcmpgtb m0, [esp+4*16] +%endif + + ABSSUB m1, m3, m6, m7 ; abs(p1-q1) + ABSSUB m7, m4, m5, m2 ; abs(p0-q0) + paddusb m7, m7 + pand m1, [PIC_sym(pb_254)] + psrlq m1, 1 + paddusb m1, m7 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + pxor m1, [PIC_sym(pb_128)] +%if ARCH_X86_64 + pcmpgtb m1, m8 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E +%else + pcmpgtb m1, [esp+3*16] +%endif + por m0, m1 + +%if %1 == 16 +%if ARCH_X86_64 + SWAP 0, 8 +%else + mova [esp+3*16], m0 +%endif +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] +%else + mova m0, [rsp+12*16] +%endif + ABSSUB m1, m0, m4, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*2] +%else + mova m0, [rsp+13*16] +%endif + ABSSUB m2, m0, m4, m7 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+stride3q] +%else + mova m0, [rsp+14*16] +%endif + ABSSUB m2, m0, m4, m7 + pmaxub m1, m2 +%ifidn %2, v + lea tmpq, [dstq+strideq*4] + mova m0, [tmpq+strideq*0] +%else + mova m0, [rsp+15*16] +%endif + ABSSUB m2, m0, m5, m7 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*1] +%else + mova m0, [rsp+16*16] +%endif + ABSSUB m2, m0, m5, m7 + pmaxub m1, m2 +%ifidn %2, v + mova m0, [tmpq+strideq*2] +%else + mova m0, [rsp+17*16] +%endif + ABSSUB m2, m0, m5, m7 + pmaxub m1, m2 + pxor m1, [PIC_sym(pb_128)] + pcmpgtb m1, [PIC_sym(pb_129)] ; !flat8out +%if ARCH_X86_64 + por m1, m9 ; !flat8in | !flat8out +%else + por m1, [esp+6*16] + %define m12 m7 + mova m12, maskmem +%endif + pand m2, m12, mask2 + pcmpeqd m2, m12 + pandn m1, m2 ; flat16 +%if ARCH_X86_64 + pandn m2, m8, m1 ; flat16 & fm +%else + pandn m2, [esp+3*16], m1 ; flat16 & fm + mova %%flat16mem, m2 +%endif + SWAP 1, 2 + + pand m2, m12, mask1 + pcmpeqd m2, m12 +%if ARCH_X86_64 + pandn m9, m2 ; flat8in + pandn m2, m8, m9 + SWAP 2, 9 +%else + pandn m0, [esp+6*16], m2 + pandn m2, [esp+3*16], m0 + mova [esp+6*16], m2 +%endif + pand m2, m12, mask0 + pcmpeqd m2, m12 +%if ARCH_X86_64 + pandn m8, m2 + pandn m2, m9, m8 ; fm & !flat8 & !flat16 + SWAP 2, 8 + pandn m2, m1, m9 ; flat8 & !flat16 + SWAP 2, 9 + SWAP 0, 8 + SWAP 1, 10 +%else + pandn m0, [esp+3*16], m2 + pandn m2, [esp+6*16], m0 + SWAP 2, 0 + pandn m2, m1, [esp+6*16] + mova %%flat8mem, m2 +%endif +%elif %1 != 4 + %if ARCH_X86_64 + SWAP 1, 9 + %else + %define m12 m7 + mova m12, maskmem + mova m1, [esp+6*16] + %endif + pand m2, m12, mask1 + pcmpeqd m2, m12 + pandn m1, m2 + pandn m2, m0, m1 ; flat8 & fm + pand m1, m12, mask0 + pcmpeqd m1, m12 + pandn m0, m1 + pandn m1, m2, m0 ; fm & !flat8 + SWAP 1, 2, 0 + %if ARCH_X86_64 + SWAP 1, 9 + %else + mova %%flat8mem, m1 + %endif +%else +%if ARCH_X86_32 + %define m12 m1 + mova m12, maskmem +%endif + pand m2, m12, mask0 + pcmpeqd m2, m12 + pandn m0, m2 ; fm +%endif + + ; short filter + + mova m1, [PIC_sym(pb_128)] +%if ARCH_X86_64 + SWAP 7, 11 +%else + mova m7, [esp+5*16] +%endif + pxor m3, m1 + pxor m6, m1 + pxor m4, m1 + pxor m5, m1 + psubsb m1, m3, m6 ; iclip_diff(p1-q1) + pand m1, m7 ; f=iclip_diff(p1-q1)&hev + psubsb m2, m5, m4 + paddsb m1, m2 + paddsb m1, m2 + paddsb m1, m2 ; f=iclip_diff(3*(q0-p0)+f) + mova m2, [PIC_sym(pb_16)] + pand m0, m1 ; f&=fm + paddsb m1, m0, [PIC_sym(pb_3)] + paddsb m0, [PIC_sym(pb_4)] + pand m1, [PIC_sym(pb_248)] + pand m0, [PIC_sym(pb_248)] + psrlq m1, 3 + psrlq m0, 3 + pxor m1, m2 + pxor m0, m2 + psubb m1, m2 ; f2 + psubb m0, m2 ; f1 + mova m2, [PIC_sym(pb_128)] + paddsb m4, m1 + psubsb m5, m0 + pxor m4, m2 + pxor m5, m2 + + pxor m0, m2 + pxor m1, m1 + pavgb m0, m1 ; f=(f1+1)>>1 + psubb m0, [PIC_sym(pb_64)] + pandn m7, m0 ; f&=!hev + paddsb m3, m7 + psubsb m6, m7 + pxor m3, m2 + pxor m6, m2 + +%if %1 == 16 + ; flat16 filter +%ifidn %2, v + lea tmpq, [dstq+mstrideq*8] + mova m0, [tmpq+strideq*1] ; p6 + mova m2, [tmpq+strideq*2] ; p5 + mova m7, [tmpq+stride3q] ; p4 +%else + mova m0, [rsp+12*16] + mova m2, [rsp+13*16] + mova m7, [rsp+14*16] +%endif + +%if ARCH_X86_64 + SWAP 1, 10 + mova %%flat8mem, m9 + mova %%q2mem, m14 + mova %%q3mem, m15 + SWAP 0, 8 + SWAP 1, 9 +%else + %ifidn %2, v + mova [esp+17*16], m0 + mova [esp+19*16], m3 + mova [esp+21*16], m4 + mova [esp+22*16], m5 + mova [esp+23*16], m6 + %xdefine m11 m3 + %xdefine m14 m4 + %xdefine m15 m5 + %xdefine m10 m6 + %define m13 %%p2mem + %define m8 [esp+17*16] + %define m9 %%flat16mem + %define m3 [esp+19*16] + %define m4 [esp+21*16] + %define m5 [esp+22*16] + %define m6 [esp+23*16] + %else + mova [esp+31*16], m0 + mova [esp+32*16], m3 + mova [esp+33*16], m4 + mova [esp+34*16], m5 + mova [esp+35*16], m6 + %xdefine m11 m3 + %xdefine m14 m4 + %xdefine m15 m5 + %xdefine m10 m6 + %define m13 %%p2mem + %define m8 [esp+31*16] + %define m9 %%flat16mem + %define m3 [esp+32*16] + %define m4 [esp+33*16] + %define m5 [esp+34*16] + %define m6 [esp+35*16] + %endif +%endif + + ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 [p5/p4/p2/p1/p0/q0][p6/p3] A + ; write -6 + mova m11, %%p3mem +%if ARCH_X86_64 + punpcklbw m14, m8, m11 + punpckhbw m15, m8, m11 +%else + punpcklbw m14, m0, m11 + punpckhbw m15, m0, m11 +%endif +%ifidn %2, v + mova [rsp+5*16], m11 +%endif + pmaddubsw m10, m14, [PIC_sym(pb_7_1)] + pmaddubsw m11, m15, [PIC_sym(pb_7_1)] ; p6*7+p3 + punpcklbw m0, m2, m7 + punpckhbw m1, m2, m7 + pmaddubsw m0, [PIC_sym(pb_2)] + pmaddubsw m1, [PIC_sym(pb_2)] + paddw m10, m0 + paddw m11, m1 ; p6*7+p5*2+p4*2+p3 + punpcklbw m0, m13, m3 + punpckhbw m1, m13, m3 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + paddw m10, m0 + paddw m11, m1 ; p6*7+p5*2+p4*2+p3+p2+p1 + punpcklbw m0, m4, m5 + punpckhbw m1, m4, m5 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + paddw m10, m0 + paddw m11, m1 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m2 + por m0, m1 +%ifidn %2, v + mova [tmpq+strideq*2], m0 ; p5 +%else + mova [rsp+13*16], m0 +%endif + + ; sub p6*2, add p3/q1 [reuse p6/p3 from A][-p6,+q1|save] B + ; write -5 + pmaddubsw m14, [PIC_sym(pb_m1_1)] + pmaddubsw m15, [PIC_sym(pb_m1_1)] + paddw m10, m14 + paddw m11, m15 ; p6*6+p5*2+p4*2+p3*2+p2+p1+p0+q0 + punpcklbw m0, m8, m6 + punpckhbw m1, m8, m6 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + mova [rsp+3*16], m0 + mova [rsp+4*16], m1 + paddw m10, m0 + paddw m11, m1 ; p6*5+p5*2+p4*2+p3*2+p2+p1+p0+q0+q1 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m7 + por m0, m1 +%ifidn %2, v + mova [tmpq+stride3q], m0 ; p4 +%else + mova [rsp+14*16], m0 +%endif + + ; sub p6/p5, add p2/q2 [-p6,+p2][-p5,+q2|save] C + ; write -4 + mova m14, %%q2mem + punpcklbw m0, m8, m13 + punpckhbw m1, m8, m13 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p6*4+p5*2+p4*2+p3*2+p2*2+p1+p0+q0+q1 + punpcklbw m0, m2, m14 + punpckhbw m2, m14 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m2, [PIC_sym(pb_m1_1)] + mova [rsp+1*16], m0 + paddw m10, m0 + paddw m11, m2 ; p6*4+p5+p4*2+p3*2+p2*2+p1+p0+q0+q1+q2 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, %%p3mem + por m0, m1 +%ifidn %2, v + mova [tmpq+strideq*4], m0 ; p3 +%else + mova [rsp+19*16], m0 +%endif + + ; sub p6/p4, add p1/q3 [-p6,+p1][-p4,+q3|save] D + ; write -3 + mova m15, %%q3mem + punpcklbw m0, m8, m3 + punpckhbw m1, m8, m3 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p6*3+p5+p4*2+p3*2+p2*2+p1*2+p0+q0+q1+q2 + punpcklbw m0, m7, m15 + punpckhbw m7, m15 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m7, [PIC_sym(pb_m1_1)] + mova [rsp+2*16], m0 +%if ARCH_X86_32 + %ifidn %2, v + mova [esp+24*16], m7 + %else + mova [esp+36*16], m7 + %endif +%endif + paddw m10, m0 + paddw m11, m7 ; p6*3+p5+p4+p3*2+p2*2+p1*2+p0+q0+q1+q2+q3 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m13 + por m0, m1 + mova [rsp+6*16], m0 ; don't clobber p2/m13 since we need it in F + + ; sub p6/p3, add p0/q4 [-p6,+p0][-p3,+q4|save] E + ; write -2 + punpcklbw m0, m8, m4 + punpckhbw m1, m8, m4 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p6*2+p5+p4+p3*2+p2*2+p1*2+p0*2+q0+q1+q2+q3 +%if ARCH_X86_64 + SWAP 7, 8 +%endif +%ifidn %2, v + mova m1, [dstq+strideq*4] ; q4 + mova m7, [rsp+5*16] ; (pre-filter) p3 +%else + mova m1, [rsp+15*16] + mova m7, %%p3mem ; (pre-filter) p3 +%endif + punpcklbw m0, m1, m7 + punpckhbw m1, m1, m7 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + mova [rsp+7*16], m0 + mova [rsp+5*16], m1 + psubw m10, m0 + psubw m11, m1 ; p6*2+p5+p4+p3+p2*2+p1*2+p0*2+q0+q1+q2+q3+q4 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m3 + por m0, m1 + mova [rsp+8*16], m0 ; don't clobber p1/m3 since we need it in G + + ; sub p6/p2, add q0/q5 [-p6,+q0][-p2,+q5|save] F + ; write -1 +%ifidn %2, v + mova m7, [tmpq+strideq*1] ; p6 + lea tmpq, [dstq+strideq*4] + mova m1, [tmpq+strideq*1] ; q5 +%else + mova m7, [rsp+12*16] ; p6 + mova m1, [rsp+16*16] +%endif + punpcklbw m0, m7, m5 + punpckhbw m7, m5 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m7, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m7 ; p6+p5+p4+p3+p2*2+p1*2+p0*2+q0*2+q1+q2+q3+q4 + punpcklbw m7, m13, m1 + pmaddubsw m7, [PIC_sym(pb_m1_1)] + mova [rsp+9*16], m7 + paddw m10, m7 +%if ARCH_X86_64 + punpckhbw m13, m1 + mova m1, [rsp+6*16] + SWAP 1, 13 +%else + punpckhbw m7, m13, m1 + mova m1, [esp+6*16] + mova m13, m1 + SWAP 1, 7 +%endif + pmaddubsw m1, [PIC_sym(pb_m1_1)] + mova [rsp+10*16], m1 + paddw m11, m1 ; p6+p5+p4+p3+p2+p1*2+p0*2+q0*2+q1+q2+q3+q4+q5 + pmulhrsw m7, m10, [PIC_sym(pw_2048)] + pmulhrsw m0, m11, [PIC_sym(pw_2048)] + packuswb m7, m0 + pand m7, m9 + pandn m0, m9, m4 + por m7, m0 + mova [rsp+6*16], m7 ; don't clobber p0/m4 since we need it in H + + ; sub p6/p1, add q1/q6 [reuse -p6,+q1 from B][-p1,+q6|save] G + ; write +0 +%ifidn %2, v + mova m7, [tmpq+strideq*2] ; q6 +%else + mova m7, [rsp+17*16] +%endif + paddw m10, [rsp+3*16] + paddw m11, [rsp+4*16] ; p5+p4+p3+p2+p1*2+p0*2+q0*2+q1*2+q2+q3+q4+q5 + punpcklbw m0, m3, m7 + punpckhbw m1, m3, m7 +%if ARCH_X86_64 + mova m3, [rsp+8*16] +%endif + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + mova [rsp+3*16], m0 + mova [rsp+4*16], m1 + paddw m10, m0 + paddw m11, m1 ; p5+p4+p3+p2+p1+p0*2+q0*2+q1*2+q2+q3+q4+q5+q6 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m5 + por m0, m1 +%if ARCH_X86_32 + mova m1, [esp+8*16] + mova m3, m1 +%endif + mova [rsp+8*16], m0 ; don't clobber q0/m5 since we need it in I + + ; sub p5/p0, add q2/q6 [reuse -p5,+q2 from C][-p0,+q6] H + ; write +1 + paddw m10, [rsp+1*16] + paddw m11, m2 ; p4+p3+p2+p1+p0*2+q0*2+q1*2+q2*2+q3+q4+q5+q6 + punpcklbw m0, m4, m7 + punpckhbw m2, m4, m7 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m2, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m2 ; p4+p3+p2+p1+p0+q0*2+q1*2+q2*2+q3+q4+q5+q6*2 +%if ARCH_X86_64 + mova m4, [rsp+6*16] +%else + %define m4 [esp+6*16] +%endif + pmulhrsw m2, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m2, m1 + pand m2, m9 + pandn m1, m9, m6 + por m2, m1 ; don't clobber q1/m6 since we need it in K + + ; sub p4/q0, add q3/q6 [reuse -p4,+q3 from D][-q0,+q6] I + ; write +2 + paddw m10, [rsp+2*16] +%if ARCH_X86_64 + SWAP 7, 8 + paddw m11, m7 +%else + mova m8, m7 + %ifidn %2, v + paddw m11, [esp+24*16] ; p3+p2+p1+p0+q0*2+q1*2+q2*2+q3*2+q4+q5+q6*2 + %else + paddw m11, [esp+36*16] ; p3+p2+p1+p0+q0*2+q1*2+q2*2+q3*2+q4+q5+q6*2 + %endif +%endif + punpcklbw m0, m5, m8 + punpckhbw m1, m5, m8 +%if ARCH_X86_64 + mova m5, [rsp+8*16] +%else + %define m5 [esp+8*16] +%endif + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p3+p2+p1+p0+q0+q1*2+q2*2+q3*2+q4+q5+q6*3 + pmulhrsw m7, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m7, m1 + pand m7, m9 + pandn m1, m9, m14 + por m7, m1 ; don't clobber q2/m14 since we need it in K + + ; sub p3/q1, add q4/q6 [reuse -p3,+q4 from E][-q1,+q6] J + ; write +3 + psubw m10, [rsp+7*16] + psubw m11, [rsp+5*16] ; p2+p1+p0+q0+q1*2+q2*2+q3*2+q4*2+q5+q6*3 + punpcklbw m0, m6, m8 + punpckhbw m1, m6, m8 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p2+p1+p0+q0+q1+q2*2+q3*2+q4*2+q5+q6*4 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m15 + por m0, m1 +%ifidn %2, v + mova [tmpq+mstrideq], m0 ; q3 +%else + mova [rsp+20*16], m0 +%endif + + ; sub p2/q2, add q5/q6 [reuse -p2,+q5 from F][-q2,+q6] K + ; write +4 + paddw m10, [rsp+ 9*16] + paddw m11, [rsp+10*16] ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + punpcklbw m0, m14, m8 + punpckhbw m1, m14, m8 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m0, m10, [PIC_sym(pw_2048)] + pmulhrsw m1, m11, [PIC_sym(pw_2048)] + packuswb m0, m1 + pand m0, m9 +%ifidn %2, v + pandn m1, m9, [tmpq+strideq*0] +%else + pandn m1, m9, [rsp+15*16] +%endif + por m0, m1 +%ifidn %2, v + mova [tmpq+strideq*0], m0 ; q4 +%else + mova [rsp+15*16], m0 +%endif + + ; sub p1/q3, add q6*2 [reuse -p1,+q6 from G][-q3,+q6] L + ; write +5 + paddw m10, [rsp+3*16] + paddw m11, [rsp+4*16] ; p1+p0+q0+q1+q2*2+q3*2+q4*2+q5*2+q6*4 + punpcklbw m0, m15, m8 + punpckhbw m1, m15, m8 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m10, m0 + paddw m11, m1 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 + pmulhrsw m10, [PIC_sym(pw_2048)] + pmulhrsw m11, [PIC_sym(pw_2048)] + packuswb m10, m11 + pand m10, m9 +%ifidn %2, v + pandn m11, m9, [tmpq+strideq*1] +%else + pandn m11, m9, [rsp+16*16] +%endif + por m10, m11 +%ifidn %2, v + mova [tmpq+strideq*1], m10 ; q5 +%else + mova [rsp+16*16], m10 +%endif + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 14, 7 +%else + %xdefine m3 m11 + %xdefine m4 m14 + %xdefine m5 m15 + %xdefine m6 m10 + mova %%q2mem, m7 + %ifidn %2, v + mova m3, [esp+19*16] + %else + mova m3, [esp+32*16] + %endif + mova m4, [esp+ 6*16] + mova m5, [esp+ 8*16] +%endif + SWAP m6, m2 + +%if ARCH_X86_64 + mova m9, %%flat8mem +%endif +%ifidn %2, v + lea tmpq, [dstq+mstrideq*4] +%endif +%endif ; if %1 == 16 +%if %1 >= 8 + ; flat8 filter +%if ARCH_X86_32 + %define m9 %%flat8mem + %define m11 m1 + %define m13 %%p2mem + %define m14 %%q2mem + %define m15 %%q3mem +%endif + mova m11, %%p3mem + punpcklbw m0, m11, m3 + punpcklbw m7, m13, m4 + pmaddubsw m2, m0, [PIC_sym(pb_3_1)] ; 3 * p3 + p1 + pmaddubsw m7, [PIC_sym(pb_2_1)] + paddw m2, m7 ; 3 * p3 + 2 * p2 + p1 + p0 + punpcklbw m7, m5, [PIC_sym(pb_4)] + pmaddubsw m7, [PIC_sym(pb_1)] + paddw m2, m7 ; 3 * p3 + 2 * p2 + p1 + p0 + q0 + 4 + punpckhbw m1, m11, m3 + pmaddubsw m7, m1, [PIC_sym(pb_3_1)] ; 3 * p3 + p1 + punpckhbw m0, m13, m4 + pmaddubsw m0, [PIC_sym(pb_2_1)] + paddw m7, m0 ; 3 * p3 + 2 * p2 + p1 + p0 + punpckhbw m0, m5, [PIC_sym(pb_4)] + pmaddubsw m0, [PIC_sym(pb_1)] + paddw m7, m0 ; 3 * p3 + 2 * p2 + p1 + p0 + q0 + 4 + psrlw m0, m2, 3 + psrlw m1, m7, 3 + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m13 + por m0, m1 ; p2 +%ifidn %2, v + mova [tmpq+strideq*1], m0 +%else + %if ARCH_X86_64 + SWAP 0, 10 + %else + mova [esp+2*16], m0 + %endif +%endif + +%if ARCH_X86_32 + mova m11, %%p3mem +%endif + punpcklbw m0, m11, m3 + punpckhbw m1, m11, m3 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m2, m0 + paddw m7, m1 + punpcklbw m0, m13, m6 + punpckhbw m1, m13, m6 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m2, m0 + paddw m7, m1 ; 2 * p3 + p2 + 2 * p1 + p0 + q0 + q1 + 4 + psrlw m0, m2, 3 + psrlw m1, m7, 3 + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m3 + por m0, m1 ; p1 +%ifidn %2, v + mova [tmpq+strideq*2], m0 +%else + mova [rsp+0*16], m0 +%endif + +%if ARCH_X86_32 + mova m11, %%p3mem +%endif + punpcklbw m0, m11, m3 + punpckhbw m1, m11, m3 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + psubw m2, m0 + psubw m7, m1 + punpcklbw m0, m4, m14 + punpckhbw m1, m4, m14 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + paddw m2, m0 + paddw m7, m1 ; p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + 4 + psrlw m0, m2, 3 + psrlw m1, m7, 3 + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m4 + por m0, m1 ; p0 +%ifidn %2, v + mova [tmpq+stride3q], m0 +%else + mova [rsp+1*16], m0 +%endif + + punpcklbw m0, m5, m15 + punpckhbw m1, m5, m15 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + paddw m2, m0 + paddw m7, m1 +%if ARCH_X86_32 + mova m11, %%p3mem +%endif + punpcklbw m0, m11, m4 + punpckhbw m11, m11, m4 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m11, [PIC_sym(pb_1)] + psubw m2, m0 + psubw m7, m11 ; p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + 4 + psrlw m0, m2, 3 + psrlw m11, m7, 3 + packuswb m0, m11 + pand m0, m9 + pandn m11, m9, m5 + por m11, m0 ; q0 +%ifidn %2, v + mova [dstq+strideq*0], m11 +%elif ARCH_X86_32 + mova [esp+8*16], m11 +%endif + + punpcklbw m0, m5, m15 + punpckhbw m1, m5, m15 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m2, m0 + paddw m7, m1 + punpcklbw m0, m13, m6 + punpckhbw m1, m13, m6 + pmaddubsw m0, [PIC_sym(pb_m1_1)] + pmaddubsw m1, [PIC_sym(pb_m1_1)] + paddw m2, m0 + paddw m7, m1 ; p1 + p0 + q0 + 2 * q1 + q2 + 2 * q3 + 4 + psrlw m0, m2, 3 + psrlw m1, m7, 3 + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m6 + por m0, m1 ; q1 +%ifidn %2, v + mova [dstq+strideq*1], m0 +%else + %if ARCH_X86_64 + SWAP 0, 13 + %else + mova [esp+9*16], m0 + %endif +%endif + + punpcklbw m0, m3, m6 + punpckhbw m1, m3, m6 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + psubw m2, m0 + psubw m7, m1 + punpcklbw m0, m14, m15 + punpckhbw m1, m14, m15 + pmaddubsw m0, [PIC_sym(pb_1)] + pmaddubsw m1, [PIC_sym(pb_1)] + paddw m2, m0 + paddw m7, m1 ; p0 + q0 + q1 + q2 + 2 * q2 + 3 * q3 + 4 + psrlw m2, 3 + psrlw m7, 3 + packuswb m2, m7 + pand m2, m9 + pandn m7, m9, m14 + por m2, m7 ; q2 +%ifidn %2, v + mova [dstq+strideq*2], m2 +%else + mova m0, [rsp+0*16] +%if %1 == 8 + mova m1, [rsp+1*16] + mova m4, %%p3mem + +%if ARCH_X86_32 + %define m10 [esp+2*16] + %define m11 [esp+8*16] + %define m13 [esp+9*16] +%endif + + ; 16x8 transpose + punpcklbw m3, m4, m10 + punpckhbw m4, m10 + punpcklbw m5, m0, m1 + punpckhbw m0, m1 + punpcklbw m1, m11, m13 + punpckhbw m6, m11, m13 + punpcklbw m7, m2, m15 + punpckhbw m2, m15 +%if ARCH_X86_64 + SWAP 2, 15 +%else + mova m15, m2 +%endif + + punpcklwd m2, m3, m5 + punpckhwd m3, m5 + punpcklwd m5, m4, m0 + punpckhwd m4, m0 + punpcklwd m0, m1, m7 + punpckhwd m1, m7 + punpcklwd m7, m6, m15 + punpckhwd m6, m15 +%if ARCH_X86_64 + SWAP 6, 15 +%else + mova m15, m6 +%endif + + punpckldq m6, m2, m0 + punpckhdq m2, m0 + punpckldq m0, m3, m1 + punpckhdq m3, m1 + punpckldq m1, m5, m7 + punpckhdq m5, m7 + punpckldq m7, m4, m15 + punpckhdq m4, m15 + + ; write 8x16 + movq [dstq+strideq*0-4], xm6 + movhps [dstq+strideq*1-4], xm6 + movq [dstq+strideq*2-4], xm2 + movhps [dstq+stride3q -4], xm2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm0 + movhps [dstq+strideq*1-4], xm0 + movq [dstq+strideq*2-4], xm3 + movhps [dstq+stride3q -4], xm3 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm1 + movhps [dstq+strideq*1-4], xm1 + movq [dstq+strideq*2-4], xm5 + movhps [dstq+stride3q -4], xm5 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0-4], xm7 + movhps [dstq+strideq*1-4], xm7 + movq [dstq+strideq*2-4], xm4 + movhps [dstq+stride3q -4], xm4 + lea dstq, [dstq+strideq*4] +%else + ; 16x16 transpose and store + SWAP 6, 0 + SWAP 7, 1 + %if ARCH_X86_64 + SWAP 5, 10, 2 + SWAP 8, 11 + SWAP 9, 13 + mova [rsp+21*16], m12 + %else + mova [esp+10*16], m2 + %xdefine m8 m0 + %xdefine m9 m1 + %xdefine m10 m2 + %xdefine m11 m3 + %xdefine m12 m4 + %xdefine m13 m5 + %xdefine m14 m6 + %xdefine m15 m7 + %endif + mova m0, [rsp+11*16] + mova m1, [rsp+12*16] + mova m2, [rsp+13*16] + mova m3, [rsp+14*16] + mova m4, [rsp+19*16] +%if ARCH_X86_64 + mova m7, [rsp+ 1*16] + mova m11, [rsp+20*16] + mova m12, [rsp+15*16] + mova m13, [rsp+16*16] + mova m14, [rsp+17*16] + TRANSPOSE_16X16B 1, [rsp+18*16] +%else + mova m5, [esp+ 2*16] + TRANSPOSE_16X16B 1, [esp+32*16] + mov tmpq, dstq + lea dstq, [dstq+strideq*8] +%endif + movu [dstq+strideq*0-8], xm0 + movu [dstq+strideq*1-8], xm1 + movu [dstq+strideq*2-8], xm2 + movu [dstq+stride3q -8], xm3 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm4 + movu [dstq+strideq*1-8], xm5 + movu [dstq+strideq*2-8], xm6 + movu [dstq+stride3q -8], xm7 +%if ARCH_X86_64 + lea dstq, [dstq+strideq*4] +%else + %xdefine m8 m0 + %xdefine m9 m1 + %xdefine m10 m2 + %xdefine m11 m3 + %xdefine m12 m4 + %xdefine m13 m5 + %xdefine m14 m6 + %xdefine m15 m7 + mova m8, [esp+11*16] + mova m9, [esp+12*16] + mova m10, [esp+13*16] + mova m11, [esp+14*16] + mova m12, [esp+26*16] + mova m13, [esp+27*16] + mova m14, [esp+ 0*16] + mova m15, [esp+ 1*16] + mov dstq, tmpq +%endif + movu [dstq+strideq*0-8], xm8 + movu [dstq+strideq*1-8], xm9 + movu [dstq+strideq*2-8], xm10 + movu [dstq+stride3q -8], xm11 + lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], xm12 + movu [dstq+strideq*1-8], xm13 + movu [dstq+strideq*2-8], xm14 + movu [dstq+stride3q -8], xm15 + lea dstq, [dstq+strideq*4] +%if ARCH_X86_32 + lea dstq, [dstq+strideq*8] +%else + mova m12, [rsp+21*16] +%endif + +%endif ; if %1 == 8 +%endif ; ifidn %2, v +%elif %1 == 6 + ; flat6 filter +%if ARCH_X86_32 + mova [esp+3*16], m3 + mova [esp+4*16], m4 + mova [esp+5*16], m5 + mova [esp+6*16], m6 + %xdefine m8 m3 + %xdefine m10 m4 + %xdefine m11 m5 + %xdefine m15 m6 + %define m3 [esp+3*16] + %define m4 [esp+4*16] + %define m5 [esp+5*16] + %define m6 [esp+6*16] + %define m9 %%flat8mem + %define m13 %%p2mem + %define m14 %%q2mem +%endif + + punpcklbw m8, m13, m5 + punpckhbw m11, m13, m5 + pmaddubsw m0, m8, [PIC_sym(pb_3_1)] + pmaddubsw m1, m11, [PIC_sym(pb_3_1)] + punpcklbw m7, m4, m3 + punpckhbw m10, m4, m3 + pmaddubsw m2, m7, [PIC_sym(pb_2)] + pmaddubsw m15, m10, [PIC_sym(pb_2)] + paddw m0, m2 + paddw m1, m15 + pmulhrsw m2, m0, [PIC_sym(pw_4096)] + pmulhrsw m15, m1, [PIC_sym(pw_4096)] + packuswb m2, m15 + pand m2, m9 + pandn m15, m9, m3 + por m2, m15 +%ifidn %2, v + mova [tmpq+strideq*2], m2 ; p1 +%elif ARCH_X86_32 + mova [esp+11*16], m2 +%endif + + pmaddubsw m8, [PIC_sym(pb_m1_1)] + pmaddubsw m11, [PIC_sym(pb_m1_1)] + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m13, m6 + punpckhbw m11, m13, m6 +%if ARCH_X86_64 + SWAP 2, 13 +%endif + pmaddubsw m8, [PIC_sym(pb_m1_1)] + pmaddubsw m11, [PIC_sym(pb_m1_1)] + paddw m0, m8 + paddw m1, m11 + pmulhrsw m2, m0, [PIC_sym(pw_4096)] + pmulhrsw m15, m1, [PIC_sym(pw_4096)] + packuswb m2, m15 + pand m2, m9 + pandn m15, m9, m4 + por m2, m15 +%ifidn %2, v + mova [tmpq+stride3q], m2 ; p0 +%elif ARCH_X86_32 + mova [esp+8*16], m2 +%endif + + paddw m0, m8 + paddw m1, m11 + punpcklbw m8, m3, m14 + punpckhbw m11, m3, m14 +%if ARCH_X86_64 + SWAP 2, 14 +%endif + pmaddubsw m2, m8, [PIC_sym(pb_m1_1)] + pmaddubsw m15, m11, [PIC_sym(pb_m1_1)] + paddw m0, m2 + paddw m1, m15 + pmulhrsw m2, m0, [PIC_sym(pw_4096)] + pmulhrsw m15, m1, [PIC_sym(pw_4096)] + packuswb m2, m15 + pand m2, m9 + pandn m15, m9, m5 + por m2, m15 +%ifidn %2, v + mova [dstq+strideq*0], m2 ; q0 +%endif + + pmaddubsw m8, [PIC_sym(pb_m1_2)] + pmaddubsw m11, [PIC_sym(pb_m1_2)] + paddw m0, m8 + paddw m1, m11 + pmaddubsw m7, [PIC_sym(pb_m1_0)] + pmaddubsw m10, [PIC_sym(pb_m1_0)] + paddw m0, m7 + paddw m1, m10 + pmulhrsw m0, [PIC_sym(pw_4096)] + pmulhrsw m1, [PIC_sym(pw_4096)] + packuswb m0, m1 + pand m0, m9 + pandn m1, m9, m6 + por m0, m1 +%if ARCH_X86_32 + %xdefine m3 m8 + %xdefine m4 m10 + %xdefine m5 m11 + %xdefine m6 m15 +%endif +%ifidn %2, v + mova [dstq+strideq*1], m0 ; q1 +%else + %if ARCH_X86_64 + SWAP 3, 13 + SWAP 4, 14 + %else + mova m3, [esp+11*16] + mova m4, [esp+ 8*16] + %endif + SWAP 5, 2 + SWAP 6, 0 + TRANSPOSE_16x4_AND_WRITE_4x16 3, 4, 5, 6, 7 +%endif +%else ; if %1 == 4 +%ifidn %2, v + mova [tmpq+strideq*0], m3 ; p1 + mova [tmpq+strideq*1], m4 ; p0 + mova [tmpq+strideq*2], m5 ; q0 + mova [tmpq+stride3q ], m6 ; q1 +%else + TRANSPOSE_16x4_AND_WRITE_4x16 3, 4, 5, 6, 7 +%endif +%endif +%if ARCH_X86_32 + %define m12 m12reg +%endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; 32-bit PIC helpers ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%if ARCH_X86_32 + %define PIC_base_offset $$ + + %macro SETUP_PIC 0 ; PIC_reg + %define PIC_reg r2 + %assign PIC_reg_stk_offset stack_size-gprsize*(1+copy_args*4) + LEA PIC_reg, $$ + %endmacro + + %macro XCHG_PIC_REG 1 ; 0=mask 1=PIC_base + %if %1 == 0 + mov [esp+PIC_reg_stk_offset], PIC_reg + mov PIC_reg, maskm + %else + mov PIC_reg, [esp+PIC_reg_stk_offset] + %endif + %endmacro + + %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) + +%else + %macro XCHG_PIC_REG 1 + %endmacro + %define PIC_sym(sym) (sym) +%endif + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < required_stack_alignment + %assign copy_args 1 + %else + %assign copy_args 0 + %endif +%endif + +%macro RELOC_ARGS 1 + %if copy_args + %define maskm [esp+stack_size-gprsize*1] + %define l_stridem [esp+stack_size-gprsize*2] + %define lutm [esp+stack_size-gprsize*3] + %define %1m [esp+stack_size-gprsize*4] + mov r6d, r6m + mov maskm, maskd + mov lutm, lutd + mov %1m, r6d + %else + %define %1m r6m + %endif +%endmacro + +%if ARCH_X86_32 + %define tmpq r4 + %define mstrideq r5 + %define stride3q r6 + %define l_stride3q r6 +%endif + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_v_sb_y_8bpc, 7, 11, 16, 16 * 15, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits +%else +cglobal lpf_v_sb_y_8bpc, 6, 7, 8, -16 * (26 + copy_args), \ + dst, stride, mask, l, l_stride, lut, mask_bits + RELOC_ARGS w + SETUP_PIC + %define m12 m5 +%endif + shl l_strideq, 2 + sub lq, l_strideq +%if ARCH_X86_64 + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] +%else + mov l_stridem, l_strided +%endif + mov mask_bitsd, 0xf + mova m12, [PIC_sym(pd_mask)] + XCHG_PIC_REG 0 + movu m0, [maskq] + pxor m4, m4 + movd m3, [lutq+136] + pshufb m3, m4 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + por m1, m2 + por m0, m1 + mova [rsp+11*16], m0 + mova [rsp+12*16], m1 + mova [rsp+13*16], m2 + mova [rsp+14*16], m3 + +%define maskmem [esp+15*16] +%define mask0 [rsp+11*16] +%define mask1 [rsp+12*16] +%define mask2 [rsp+13*16] +%define minlvl [rsp+14*16] + +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + je .no_flat16 + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+25*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 16, v + jmp .end + +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + je .no_flat + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+25*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 8, v + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + XCHG_PIC_REG 1 + je .no_filter + +%if ARCH_X86_32 + mov [esp+25*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 4, v + +.end: +%if ARCH_X86_32 + mova m12, maskmem + mov mask_bitsd, [esp+25*16] +%endif +.no_filter: + pslld m12, 4 + shl mask_bitsd, 4 + add lq, 16 + add dstq, 16 +%if ARCH_X86_64 + sub wd, 4 +%else + sub dword wm, 4 +%endif + XCHG_PIC_REG 0 + jg .loop + RET + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_h_sb_y_8bpc, 7, 11, 16, 16 * 26, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp, mask_bits +%else +cglobal lpf_h_sb_y_8bpc, 6, 7, 8, -16 * (39 + copy_args), \ + dst, stride, mask, l, l_stride, lut, mask_bits + RELOC_ARGS h + SETUP_PIC + %define m12 m5 +%endif + sub lq, 4 + shl l_strideq, 2 +%if ARCH_X86_64 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] +%else + mov l_stridem, l_strided +%endif + mov mask_bitsd, 0xf + mova m12, [PIC_sym(pd_mask)] + XCHG_PIC_REG 0 + movu m0, [maskq] + pxor m4, m4 + movd m3, [lutq+136] + pshufb m3, m4 + pshufd m2, m0, q2222 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + por m1, m2 + por m0, m1 + mova [rsp+22*16], m0 + mova [rsp+23*16], m1 + mova [rsp+24*16], m2 + mova [rsp+25*16], m3 + +%define maskmem [esp+37*16] +%define mask0 [rsp+22*16] +%define mask1 [rsp+23*16] +%define mask2 [rsp+24*16] +%define minlvl [rsp+25*16] + +.loop: + test [maskq+8], mask_bitsd ; vmask[2] + je .no_flat16 + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+38*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 16, h + jmp .end + +.no_flat16: + test [maskq+4], mask_bitsd ; vmask[1] + je .no_flat + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+38*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 8, h + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[0] + XCHG_PIC_REG 1 + je .no_filter + +%if ARCH_X86_32 + mov [esp+38*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 4, h + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] + lea dstq, [dstq+strideq*8] +%if ARCH_X86_32 + jmp .end_noload +.end: + mova m12, maskmem + mov l_strideq, l_stridem + mov mask_bitsd, [esp+38*16] +.end_noload: +%else +.end: +%endif + lea lq, [lq+l_strideq*4] + pslld m12, 4 + shl mask_bitsd, 4 +%if ARCH_X86_64 + sub hd, 4 +%else + sub dword hm, 4 +%endif + XCHG_PIC_REG 0 + jg .loop + RET + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_v_sb_uv_8bpc, 7, 11, 16, 3 * 16, \ + dst, stride, mask, l, l_stride, lut, \ + w, stride3, mstride, tmp, mask_bits +%else +cglobal lpf_v_sb_uv_8bpc, 6, 7, 8, -16 * (12 + copy_args), \ + dst, stride, mask, l, l_stride, lut, mask_bits + RELOC_ARGS w + SETUP_PIC + %define m12 m4 +%endif + shl l_strideq, 2 + sub lq, l_strideq +%if ARCH_X86_64 + mov mstrideq, strideq + neg mstrideq + lea stride3q, [strideq*3] +%else + mov l_stridem, l_strided +%endif + mov mask_bitsd, 0xf + mova m12, [PIC_sym(pd_mask)] + XCHG_PIC_REG 0 + movq m0, [maskq] + pxor m3, m3 + movd m2, [lutq+136] + pshufb m2, m3 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + por m0, m1 + mova [rsp+0*16], m0 + mova [rsp+1*16], m1 + mova [rsp+2*16], m2 + +%define maskmem [esp+7*16] +%define mask0 [rsp+0*16] +%define mask1 [rsp+1*16] +%define minlvl [rsp+2*16] + +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + je .no_flat + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+11*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 6, v + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[1] + XCHG_PIC_REG 1 + je .no_filter + +%if ARCH_X86_32 + mov [esp+11*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 4, v + +.end: +%if ARCH_X86_32 + mova m12, maskmem + mov mask_bitsd, [esp+11*16] +%endif +.no_filter: + pslld m12, 4 + shl mask_bitsd, 4 + add lq, 16 + add dstq, 16 +%if ARCH_X86_64 + sub wd, 4 +%else + sub dword wm, 4 +%endif + XCHG_PIC_REG 0 + jg .loop + RET + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal lpf_h_sb_uv_8bpc, 7, 11, 16, 16 * 3, \ + dst, stride, mask, l, l_stride, lut, \ + h, stride3, l_stride3, tmp, mask_bits +%else +cglobal lpf_h_sb_uv_8bpc, 6, 7, 8, -16 * (13 + copy_args), \ + dst, stride, mask, l, l_stride, lut, mask_bits + RELOC_ARGS h + SETUP_PIC + %define m12 m4 +%endif + sub lq, 4 + shl l_strideq, 2 +%if ARCH_X86_64 + lea stride3q, [strideq*3] + lea l_stride3q, [l_strideq*3] +%else + mov l_stridem, l_strided +%endif + mov mask_bitsd, 0xf + mova m12, [PIC_sym(pd_mask)] + XCHG_PIC_REG 0 + movq m0, [maskq] + pxor m3, m3 + movd m2, [lutq+136] + pshufb m2, m3 + pshufd m1, m0, q1111 + pshufd m0, m0, q0000 + por m0, m1 + mova [rsp+0*16], m0 + mova [rsp+1*16], m1 + mova [rsp+2*16], m2 + +%define maskmem [esp+7*16] +%define mask0 [rsp+0*16] +%define mask1 [rsp+1*16] +%define minlvl [rsp+2*16] + +.loop: + test [maskq+4], mask_bitsd ; vmask[1] + je .no_flat + +%if ARCH_X86_32 + XCHG_PIC_REG 1 + mov [esp+12*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 6, h + jmp .end + +.no_flat: + test [maskq+0], mask_bitsd ; vmask[1] + XCHG_PIC_REG 1 + je .no_filter + +%if ARCH_X86_32 + mov [esp+12*16], mask_bitsd + mova maskmem, m12 +%endif + FILTER 4, h + jmp .end + +.no_filter: + lea dstq, [dstq+strideq*8] + lea dstq, [dstq+strideq*8] +%if ARCH_X86_32 + jmp .end_noload +.end: + mova m12, maskmem + mov l_strided, l_stridem + mov mask_bitsd, [esp+12*16] +.end_noload: +%else +.end: +%endif + lea lq, [lq+l_strideq*4] + pslld m12, 4 + shl mask_bitsd, 4 +%if ARCH_X86_64 + sub hd, 4 +%else + sub dword hm, 4 +%endif + XCHG_PIC_REG 0 + jg .loop + RET diff --git a/third_party/dav1d/src/x86/looprestoration.h b/third_party/dav1d/src/x86/looprestoration.h new file mode 100644 index 0000000000..de23be8866 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration.h @@ -0,0 +1,94 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/looprestoration.h" + +#include "common/intops.h" + +#define decl_wiener_filter_fns(ext) \ +decl_lr_filter_fn(BF(dav1d_wiener_filter7, ext)); \ +decl_lr_filter_fn(BF(dav1d_wiener_filter5, ext)) + +#define decl_sgr_filter_fns(ext) \ +decl_lr_filter_fn(BF(dav1d_sgr_filter_5x5, ext)); \ +decl_lr_filter_fn(BF(dav1d_sgr_filter_3x3, ext)); \ +decl_lr_filter_fn(BF(dav1d_sgr_filter_mix, ext)) + +decl_wiener_filter_fns(sse2); +decl_wiener_filter_fns(ssse3); +decl_wiener_filter_fns(avx2); +decl_wiener_filter_fns(avx512icl); +decl_sgr_filter_fns(ssse3); +decl_sgr_filter_fns(avx2); +decl_sgr_filter_fns(avx512icl); + +static ALWAYS_INLINE void loop_restoration_dsp_init_x86(Dav1dLoopRestorationDSPContext *const c, const int bpc) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; +#if BITDEPTH == 8 + c->wiener[0] = BF(dav1d_wiener_filter7, sse2); + c->wiener[1] = BF(dav1d_wiener_filter5, sse2); +#endif + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + c->wiener[0] = BF(dav1d_wiener_filter7, ssse3); + c->wiener[1] = BF(dav1d_wiener_filter5, ssse3); + if (BITDEPTH == 8 || bpc == 10) { + c->sgr[0] = BF(dav1d_sgr_filter_5x5, ssse3); + c->sgr[1] = BF(dav1d_sgr_filter_3x3, ssse3); + c->sgr[2] = BF(dav1d_sgr_filter_mix, ssse3); + } + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->wiener[0] = BF(dav1d_wiener_filter7, avx2); + c->wiener[1] = BF(dav1d_wiener_filter5, avx2); + if (BITDEPTH == 8 || bpc == 10) { + c->sgr[0] = BF(dav1d_sgr_filter_5x5, avx2); + c->sgr[1] = BF(dav1d_sgr_filter_3x3, avx2); + c->sgr[2] = BF(dav1d_sgr_filter_mix, avx2); + } + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + + c->wiener[0] = BF(dav1d_wiener_filter7, avx512icl); +#if BITDEPTH == 8 + /* With VNNI we don't need a 5-tap version. */ + c->wiener[1] = c->wiener[0]; +#else + c->wiener[1] = BF(dav1d_wiener_filter5, avx512icl); +#endif + if (BITDEPTH == 8 || bpc == 10) { + c->sgr[0] = BF(dav1d_sgr_filter_5x5, avx512icl); + c->sgr[1] = BF(dav1d_sgr_filter_3x3, avx512icl); + c->sgr[2] = BF(dav1d_sgr_filter_mix, avx512icl); + } +#endif +} diff --git a/third_party/dav1d/src/x86/looprestoration16_avx2.asm b/third_party/dav1d/src/x86/looprestoration16_avx2.asm new file mode 100644 index 0000000000..ef25c28474 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration16_avx2.asm @@ -0,0 +1,2540 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +sgr_lshuf3: db 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +sgr_lshuf5: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +wiener_shufA: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 +wiener_shufB: db 6, 7, 4, 5, 8, 9, 6, 7, 10, 11, 8, 9, 12, 13, 10, 11 +wiener_shufC: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 +wiener_shufD: db 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 +wiener_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +wiener_lshuf5: db 4, 5, 4, 5, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +wiener_lshuf7: db 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 12, 13, 14, 15 +pb_0to31: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + +wiener_hshift: dw 4, 4, 1, 1 +wiener_vshift: dw 1024, 1024, 4096, 4096 +wiener_round: dd 1049600, 1048832 + +pb_m10_m9: times 2 db -10, -9 +pb_m6_m5: times 2 db -6, -5 +pb_m2_m1: times 2 db -2, -1 +pb_2_3: times 2 db 2, 3 +pb_6_7: times 2 db 6, 7 +pw_1023: times 2 dw 1023 +pd_8: dd 8 +pd_25: dd 25 +pd_4096: dd 4096 +pd_34816: dd 34816 +pd_m262128: dd -262128 +pd_0xf00800a4: dd 0xf00800a4 +pd_0xf00801c7: dd 0xf00801c7 + +%define pw_256 sgr_lshuf5 + +cextern sgr_x_by_x_avx2 + +SECTION .text + +DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; wiener ring buffer pointers + +INIT_YMM avx2 +cglobal wiener_filter7_16bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt +%define base t4-wiener_hshift + mov fltq, r6mp + movifnidn wd, wm + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + vbroadcasti128 m6, [wiener_shufA] + vpbroadcastd m12, [fltq+ 0] ; x0 x1 + lea t4, [wiener_hshift] + vbroadcasti128 m7, [wiener_shufB] + add wd, wd + vpbroadcastd m13, [fltq+ 4] ; x2 x3 + shr t3d, 11 + vpbroadcastd m14, [fltq+16] ; y0 y1 + add lpfq, wq + vpbroadcastd m15, [fltq+20] ; y2 y3 + add dstq, wq + vbroadcasti128 m8, [wiener_shufC] + lea t1, [rsp+wq+16] + vbroadcasti128 m9, [wiener_shufD] + neg wq + vpbroadcastd m0, [base+wiener_hshift+t3*4] + vpbroadcastd m10, [base+wiener_round+t3*4] + vpbroadcastd m11, [base+wiener_vshift+t3*4] + pmullw m12, m0 ; upshift filter coefs to make the + pmullw m13, m0 ; horizontal downshift constant + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call .v + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call .v +.v2: + call .v + jmp .v1 +.extend_right: + movd xm1, r10d + vpbroadcastd m0, [pb_6_7] + movu m2, [pb_0to31] + vpbroadcastb m1, xm1 + psubb m0, m1 + pminub m0, m2 + pshufb m3, m0 + vpbroadcastd m0, [pb_m2_m1] + psubb m0, m1 + pminub m0, m2 + pshufb m4, m0 + vpbroadcastd m0, [pb_m10_m9] + psubb m0, m1 + pminub m0, m2 + pshufb m5, m0 + ret +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movq xm3, [leftq] + vpblendd m3, [lpfq+r10-8], 0xfc + add leftq, 8 + jmp .h_main +.h_extend_left: + vbroadcasti128 m3, [lpfq+r10] ; avoid accessing memory located + mova m4, [lpfq+r10] ; before the start of the buffer + shufpd m3, m4, 0x05 + pshufb m3, [wiener_lshuf7] + jmp .h_main2 +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+r10-8] +.h_main: + mova m4, [lpfq+r10+0] +.h_main2: + movu m5, [lpfq+r10+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -36 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m0, m3, m6 + pshufb m1, m4, m7 + paddw m0, m1 + pshufb m3, m8 + pmaddwd m0, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + pmaddwd m3, m13 + pshufb m2, m5, m7 + paddw m1, m2 + vpbroadcastd m2, [pd_m262128] ; (1 << 4) - (1 << 18) + pshufb m4, m8 + pmaddwd m1, m12 + pshufb m5, m9 + paddw m4, m5 + pmaddwd m4, m13 + paddd m0, m2 + paddd m1, m2 + paddd m0, m3 + paddd m1, m4 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+r10], m0 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movq xm3, [leftq] + vpblendd m3, [lpfq+r10-8], 0xfc + add leftq, 8 + jmp .hv_main +.hv_extend_left: + movu m3, [lpfq+r10-8] + pshufb m3, [wiener_lshuf7] + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+r10-8] +.hv_main: + mova m4, [lpfq+r10+0] + movu m5, [lpfq+r10+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -36 + jl .hv_have_right + call .extend_right +.hv_have_right: + pshufb m0, m3, m6 + pshufb m1, m4, m7 + paddw m0, m1 + pshufb m3, m8 + pmaddwd m0, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + pmaddwd m3, m13 + pshufb m2, m5, m7 + paddw m1, m2 + vpbroadcastd m2, [pd_m262128] + pshufb m4, m8 + pmaddwd m1, m12 + pshufb m5, m9 + paddw m4, m5 + pmaddwd m4, m13 + paddd m0, m2 + paddd m1, m2 + mova m2, [t4+r10] + paddw m2, [t2+r10] + mova m5, [t3+r10] + paddd m0, m3 + paddd m1, m4 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + mova m4, [t5+r10] + paddw m4, [t1+r10] + psraw m0, 1 + paddw m3, m0, [t6+r10] + mova [t0+r10], m0 + punpcklwd m0, m2, m5 + pmaddwd m0, m15 + punpckhwd m2, m5 + pmaddwd m2, m15 + punpcklwd m1, m3, m4 + pmaddwd m1, m14 + punpckhwd m3, m4 + pmaddwd m3, m14 + paddd m0, m10 + paddd m2, m10 + paddd m0, m1 + paddd m2, m3 + psrad m0, 5 + psrad m2, 5 + packusdw m0, m2 + pmulhuw m0, m11 + mova [dstq+r10], m0 + add r10, 32 + jl .hv_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m1, [t4+r10] + paddw m1, [t2+r10] + mova m2, [t3+r10] + mova m4, [t1+r10] + paddw m3, m4, [t6+r10] + paddw m4, [t5+r10] + punpcklwd m0, m1, m2 + pmaddwd m0, m15 + punpckhwd m1, m2 + pmaddwd m1, m15 + punpcklwd m2, m3, m4 + pmaddwd m2, m14 + punpckhwd m3, m4 + pmaddwd m3, m14 + paddd m0, m10 + paddd m1, m10 + paddd m0, m2 + paddd m1, m3 + psrad m0, 5 + psrad m1, 5 + packusdw m0, m1 + pmulhuw m0, m11 + mova [dstq+r10], m0 + add r10, 32 + jl .v_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq + ret + +cglobal wiener_filter5_16bpc, 4, 13, 16, 384*8+16, dst, stride, left, lpf, \ + w, h, edge, flt +%define base t4-wiener_hshift + mov fltq, r6mp + movifnidn wd, wm + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + vbroadcasti128 m5, [wiener_shufE] + vpbroadcastw m11, [fltq+ 2] ; x1 + vbroadcasti128 m6, [wiener_shufB] + lea t4, [wiener_hshift] + vbroadcasti128 m7, [wiener_shufD] + add wd, wd + vpbroadcastd m12, [fltq+ 4] ; x2 x3 + shr t3d, 11 + vpbroadcastd m8, [pd_m262128] ; (1 << 4) - (1 << 18) + add lpfq, wq + vpbroadcastw m13, [fltq+18] ; y1 + add dstq, wq + vpbroadcastd m14, [fltq+20] ; y2 y3 + lea t1, [rsp+wq+16] + neg wq + vpbroadcastd m0, [base+wiener_hshift+t3*4] + vpbroadcastd m9, [base+wiener_round+t3*4] + vpbroadcastd m10, [base+wiener_vshift+t3*4] + movu xm15, [wiener_lshuf5] + pmullw m11, m0 + vinserti128 m15, [pb_0to31], 1 + pmullw m12, m0 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t4, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t3, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 +.main: + mov t0, t4 +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v2 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.end: + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v2 + add t0, 384*6 + call .hv + dec hd + jnz .main +.v2: + call .v + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq +.v1: + call .v + jmp .end +.extend_right: + movd xm2, r10d + vpbroadcastd m0, [pb_2_3] + vpbroadcastd m1, [pb_m6_m5] + vpbroadcastb m2, xm2 + psubb m0, m2 + psubb m1, m2 + movu m2, [pb_0to31] + pminub m0, m2 + pminub m1, m2 + pshufb m3, m0 + pshufb m4, m1 + ret +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm3, [leftq+4] + vpblendd m3, [lpfq+r10-4], 0xfe + add leftq, 8 + jmp .h_main +.h_extend_left: + vbroadcasti128 m4, [lpfq+r10] ; avoid accessing memory located + mova m3, [lpfq+r10] ; before the start of the buffer + palignr m3, m4, 12 + pshufb m3, m15 + jmp .h_main +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+r10-4] +.h_main: + movu m4, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -34 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m0, m3, m5 + pmaddwd m0, m11 + pshufb m1, m4, m5 + pmaddwd m1, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + pmaddwd m2, m12 + pshufb m4, m7 + paddw m3, m4 + pmaddwd m3, m12 + paddd m0, m8 + paddd m1, m8 + paddd m0, m2 + paddd m1, m3 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+r10], m0 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm3, [leftq+4] + vpblendd m3, [lpfq+r10-4], 0xfe + add leftq, 8 + jmp .hv_main +.hv_extend_left: + movu m3, [lpfq+r10-4] + pshufb m3, m15 + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+r10-4] +.hv_main: + movu m4, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -34 + jl .hv_have_right + call .extend_right +.hv_have_right: + pshufb m0, m3, m5 + pmaddwd m0, m11 + pshufb m1, m4, m5 + pmaddwd m1, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + pmaddwd m2, m12 + pshufb m4, m7 + paddw m3, m4 + pmaddwd m3, m12 + paddd m0, m8 + paddd m1, m8 + paddd m0, m2 + mova m2, [t3+r10] + paddw m2, [t1+r10] + paddd m1, m3 + mova m4, [t2+r10] + punpckhwd m3, m2, m4 + pmaddwd m3, m14 + punpcklwd m2, m4 + mova m4, [t4+r10] + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + pmaddwd m2, m14 + psraw m0, 1 + mova [t0+r10], m0 + punpckhwd m1, m0, m4 + pmaddwd m1, m13 + punpcklwd m0, m4 + pmaddwd m0, m13 + paddd m3, m9 + paddd m2, m9 + paddd m1, m3 + paddd m0, m2 + psrad m1, 5 + psrad m0, 5 + packusdw m0, m1 + pmulhuw m0, m10 + mova [dstq+r10], m0 + add r10, 32 + jl .hv_loop + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t4 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m0, [t1+r10] + paddw m2, m0, [t3+r10] + mova m1, [t2+r10] + mova m4, [t4+r10] + punpckhwd m3, m2, m1 + pmaddwd m3, m14 + punpcklwd m2, m1 + pmaddwd m2, m14 + punpckhwd m1, m0, m4 + pmaddwd m1, m13 + punpcklwd m0, m4 + pmaddwd m0, m13 + paddd m3, m9 + paddd m2, m9 + paddd m1, m3 + paddd m0, m2 + psrad m1, 5 + psrad m0, 5 + packusdw m0, m1 + pmulhuw m0, m10 + mova [dstq+r10], m0 + add r10, 32 + jl .v_loop + ret + +cglobal sgr_filter_5x5_16bpc, 4, 14, 15, 400*24+16, dst, stride, left, lpf, \ + w, h, edge, params + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [sgr_x_by_x_avx2+256*4] + movifnidn hd, hm + mov edged, r7m + add wd, wd + vpbroadcastw m7, [paramsq+8] ; w0 + add lpfq, wq + vpbroadcastd m8, [pd_8] + add dstq, wq + vpbroadcastd m9, [pd_25] + lea t3, [rsp+wq*2+400*12+16] + vpbroadcastd m10, [paramsq+0] ; s0 + lea t4, [rsp+wq+400*20+16] + vpbroadcastd m11, [pd_0xf00800a4] + lea t1, [rsp+wq+20] + mova xm12, [sgr_lshuf5] + neg wq + vpbroadcastd m13, [pd_34816] ; (1 << 11) + (1 << 15) + pxor m6, m6 + vpbroadcastd m14, [pw_1023] + psllw m7, 4 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, strideq + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + test hd, hd + jz .odd_height + call .h + add lpfq, strideq + call .hv + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .h_top + add lpfq, strideq + call .hv_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + call .n0 + call .n1 +.odd_height_end: + call .v + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+400*6] + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + jmp .main +.no_top_height1: + call .v + call .prep_n + jmp .odd_height_end +.extend_right: + vpbroadcastw m0, [lpfq-2] + movu m1, [r13+r10+ 0] + movu m2, [r13+r10+16] + vpblendvb m4, m0, m1 + vpblendvb m5, m0, m2 + ret +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, xm12 + vinserti128 m4, [lpfq+wq+10], 1 + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+r10- 2] +.h_main: + movu m5, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -36 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m2, m5, m4, 2 + paddw m0, m4, m2 + palignr m3, m5, m4, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + shufpd m5, m4, m5, 0x05 + paddw m0, m5 + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + paddd m1, m3 + punpckhwd m3, m4, m5 + pmaddwd m3, m3 + shufps m4, m5, q2121 + paddw m0, m4 ; sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m2, m3 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+r10+400*0] + paddd m1, [t1+r10+400*2] + paddd m2, [t1+r10+400*4] +.h_loop_end: + paddd m1, m5 ; sumsq + paddd m2, m4 + mova [t1+r10+400*0], m0 + mova [t1+r10+400*2], m1 + mova [t1+r10+400*4], m2 + add r10, 32 + jl .h_loop + ret +.top_fixup: + lea r10, [wq-4] +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+r10+400*0] + mova m1, [t1+r10+400*2] + mova m2, [t1+r10+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m1 + mova [t2+r10+400*4], m2 + add r10, 32 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 10 + jmp .hv_main +.hv_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, xm12 + vinserti128 m4, [lpfq+wq+10], 1 + jmp .hv_main +.hv_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m4, [lpfq+r10- 2] +.hv_main: + movu m5, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -36 + jl .hv_have_right + call .extend_right +.hv_have_right: + palignr m3, m5, m4, 2 + paddw m0, m4, m3 + palignr m1, m5, m4, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + shufpd m5, m4, m5, 0x05 + paddw m0, m5 + punpcklwd m1, m4, m5 + pmaddwd m1, m1 + paddd m2, m1 + punpckhwd m1, m4, m5 + pmaddwd m1, m1 + shufps m4, m5, q2121 + paddw m0, m4 ; h sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m3, m1 + paddd m2, m5 ; h sumsq + paddd m3, m4 + paddw m1, m0, [t1+r10+400*0] + paddd m4, m2, [t1+r10+400*2] + paddd m5, m3, [t1+r10+400*4] + test hd, hd + jz .hv_last_row +.hv_main2: + paddw m1, [t2+r10+400*0] ; hv sum + paddd m4, [t2+r10+400*2] ; hv sumsq + paddd m5, [t2+r10+400*4] + mova [t0+r10+400*0], m0 + mova [t0+r10+400*2], m2 + mova [t0+r10+400*4], m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + psrld m4, 4 ; (a + 8) >> 4 + punpcklwd m2, m3, m6 + psrld m5, 4 + punpckhwd m3, m6 + pmulld m4, m9 ; a * 25 + pmulld m5, m9 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + pmaxud m5, m3 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m10 ; p * s + pmulld m5, m10 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+r10+4], m2 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+r10*2+ 8], xm0 + vextracti128 [t3+r10*2+40], m0, 1 + mova [t3+r10*2+24], xm1 + vextracti128 [t3+r10*2+56], m1, 1 + add r10, 32 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+r10+400*0], m1 + paddw m1, m0 + mova [t1+r10+400*2], m4 + paddd m4, m2 + mova [t1+r10+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab + lea r10, [wq-4] +.v_loop: + mova m0, [t1+r10+400*0] + mova m2, [t1+r10+400*2] + mova m3, [t1+r10+400*4] + paddw m1, m0, [t2+r10+400*0] + paddd m4, m2, [t2+r10+400*2] + paddd m5, m3, [t2+r10+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + paddd m5, m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + psrld m4, 4 ; (a + 8) >> 4 + punpcklwd m2, m3, m6 + psrld m5, 4 + punpckhwd m3, m6 + pmulld m4, m9 ; a * 25 + pmulld m5, m9 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + pmaxud m5, m3 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m10 ; p * s + pmulld m5, m10 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+r10+4], m2 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+r10*2+ 8], xm0 + vextracti128 [t3+r10*2+40], m0, 1 + mova [t3+r10*2+24], xm1 + vextracti128 [t3+r10*2+56], m1, 1 + add r10, 32 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t4+r10*1+ 2] + movu m1, [t3+r10*2+ 4] + movu m2, [t3+r10*2+36] + paddw m3, m0, [t4+r10*1+ 0] + paddd m4, m1, [t3+r10*2+ 0] + paddd m5, m2, [t3+r10*2+32] + paddw m3, [t4+r10*1+ 4] + paddd m4, [t3+r10*2+ 8] + paddd m5, [t3+r10*2+40] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + mova [t4+r10*1+400*2+ 0], m0 + mova [t3+r10*2+400*4+ 0], m1 + mova [t3+r10*2+400*4+32], m2 + add r10, 32 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m0, [t4+r10*1+ 2] + movu m1, [t3+r10*2+ 4] + movu m2, [t3+r10*2+36] + paddw m3, m0, [t4+r10*1+ 0] + paddd m4, m1, [t3+r10*2+ 0] + paddd m5, m2, [t3+r10*2+32] + paddw m3, [t4+r10*1+ 4] + paddd m4, [t3+r10*2+ 8] + paddd m5, [t3+r10*2+40] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + paddw m3, m0, [t4+r10*1+400*2+ 0] + paddd m4, m1, [t3+r10*2+400*4+ 0] + paddd m5, m2, [t3+r10*2+400*4+32] + mova [t4+r10*1+400*2+ 0], m0 + mova [t3+r10*2+400*4+ 0], m1 + mova [t3+r10*2+400*4+32], m2 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vinserti128 m1, m4, xm5, 1 + vperm2i128 m4, m5, 0x31 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 32 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m0, [dstq+r10] + mova m3, [t4+r10*1+400*2+ 0] + mova m4, [t3+r10*2+400*4+ 0] + mova m5, [t3+r10*2+400*4+32] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vinserti128 m1, m4, xm5, 1 + vperm2i128 m4, m5, 0x31 + psubd m1, m2 ; b - a * src + (1 << 7) + psubd m4, m3 + psrad m1, 8 + psrad m4, 8 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 32 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_3x3_16bpc, 4, 14, 14, 400*42+8, dst, stride, left, lpf, \ + w, h, edge, params + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [sgr_x_by_x_avx2+256*4] + add wd, wd + movifnidn hd, hm + mov edged, r7m + add lpfq, wq + vpbroadcastw m7, [paramsq+10] ; w1 + add dstq, wq + vpbroadcastd m9, [paramsq+ 4] ; s1 + lea t3, [rsp+wq*2+400*12+8] + vpbroadcastd m8, [pd_8] + lea t4, [rsp+wq+400*32+8] + vpbroadcastd m10, [pd_0xf00801c7] + lea t1, [rsp+wq+12] + vpbroadcastd m11, [pd_34816] + neg wq + mova xm12, [sgr_lshuf3] + pxor m6, m6 + vpbroadcastd m13, [pw_1023] + psllw m7, 4 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + add t1, 400*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea r10, [wq-4] + lea t2, [t1+400*6] +.top_fixup_loop: + mova m0, [t1+r10+400*0] + mova m1, [t1+r10+400*2] + mova m2, [t1+r10+400*4] + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m1 + mova [t2+r10+400*4], m2 + add r10, 32 + jl .top_fixup_loop + call .v0 + jmp .main +.extend_right: + vpbroadcastw m0, [lpfq-2] + movu m1, [r13+r10+ 2] + movu m2, [r13+r10+18] + vpblendvb m4, m0, m1 + vpblendvb m5, m0, m2 + ret +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 12 + jmp .h_main +.h_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, xm12 + vinserti128 m4, [lpfq+wq+12], 1 + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+r10+ 0] +.h_main: + movu m5, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -34 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + mova [t1+r10+400*0], m1 + mova [t1+r10+400*2], m2 + mova [t1+r10+400*4], m3 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 12 + jmp .hv0_main +.hv0_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, xm12 + vinserti128 m4, [lpfq+wq+12], 1 + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu m4, [lpfq+r10+ 0] +.hv0_main: + movu m5, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -34 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + paddw m0, m1, [t1+r10+400*0] + paddd m4, m2, [t1+r10+400*2] + paddd m5, m3, [t1+r10+400*4] + mova [t1+r10+400*0], m1 + mova [t1+r10+400*2], m2 + mova [t1+r10+400*4], m3 + paddw m1, m0, [t2+r10+400*0] + paddd m2, m4, [t2+r10+400*2] + paddd m3, m5, [t2+r10+400*4] + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m4 + mova [t2+r10+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + psubd m4, m2 ; p + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m9 ; p * s + pmulld m5, m9 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*0+ 4], m2 + mova [t3+r10*2+400*0+ 8], xm0 + vextracti128 [t3+r10*2+400*0+40], m0, 1 + mova [t3+r10*2+400*0+24], xm1 + vextracti128 [t3+r10*2+400*0+56], m1, 1 + add r10, 32 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 12 + jmp .hv1_main +.hv1_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, xm12 + vinserti128 m4, [lpfq+wq+12], 1 + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu m4, [lpfq+r10+ 0] +.hv1_main: + movu m5, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -34 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + palignr m1, m5, m4, 2 + paddw m0, m4, m1 + punpcklwd m2, m4, m1 + pmaddwd m2, m2 + punpckhwd m3, m4, m1 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m0, m5 ; h sum + punpcklwd m1, m5, m6 + pmaddwd m1, m1 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m1 ; h sumsq + paddd m3, m5 + paddw m1, m0, [t2+r10+400*0] + paddd m4, m2, [t2+r10+400*2] + paddd m5, m3, [t2+r10+400*4] + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m2 + mova [t2+r10+400*4], m3 + paddd m4, m8 + paddd m5, m8 + psrld m4, 4 ; (a + 8) >> 4 + psrld m5, 4 + pslld m2, m4, 3 + pslld m3, m5, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + psubd m4, m2 ; p + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m9 ; p * s + pmulld m5, m9 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*2 +4], m2 + mova [t3+r10*2+400*4+ 8], xm0 + vextracti128 [t3+r10*2+400*4+40], m0, 1 + mova [t3+r10*2+400*4+24], xm1 + vextracti128 [t3+r10*2+400*4+56], m1, 1 + add r10, 32 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) + lea r10, [wq-4] +.v0_loop: + mova m0, [t1+r10+400*0] + mova m4, [t1+r10+400*2] + mova m5, [t1+r10+400*4] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+r10+400*0] + paddd m2, m4, [t2+r10+400*2] + paddd m3, m5, [t2+r10+400*4] + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m4 + mova [t2+r10+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + psubd m4, m2 ; p + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m9 ; p * s + pmulld m5, m9 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*0+ 4], m2 + mova [t3+r10*2+400*0+ 8], xm0 + vextracti128 [t3+r10*2+400*0+40], m0, 1 + mova [t3+r10*2+400*0+24], xm1 + vextracti128 [t3+r10*2+400*0+56], m1, 1 + add r10, 32 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-4] +.v1_loop: + mova m0, [t1+r10+400*0] + mova m4, [t1+r10+400*2] + mova m5, [t1+r10+400*4] + paddw m1, m0, [t2+r10+400*0] + paddd m2, m4, [t2+r10+400*2] + paddd m3, m5, [t2+r10+400*4] + mova [t2+r10+400*0], m0 + mova [t2+r10+400*2], m4 + mova [t2+r10+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaxud m4, m2 + psubd m4, m2 ; p + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m9 ; p * s + pmulld m5, m9 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*2+ 4], m2 + mova [t3+r10*2+400*4+ 8], xm0 + vextracti128 [t3+r10*2+400*4+40], m0, 1 + mova [t3+r10*2+400*4+24], xm1 + vextracti128 [t3+r10*2+400*4+56], m1, 1 + add r10, 32 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + mova xm0, [t4+r10*1+400*0+0] + paddw xm0, [t4+r10*1+400*0+4] + paddw xm2, xm0, [t4+r10*1+400*0+2] + mova m1, [t3+r10*2+400*0+0] + paddd m1, [t3+r10*2+400*0+8] + paddd m3, m1, [t3+r10*2+400*0+4] + psllw xm2, 2 ; a[-1] 444 + pslld m3, 2 ; b[-1] 444 + psubw xm2, xm0 ; a[-1] 343 + psubd m3, m1 ; b[-1] 343 + mova [t4+r10*1+400* 4], xm2 + mova [t3+r10*2+400* 8], m3 + mova xm0, [t4+r10*1+400*2+0] + paddw xm0, [t4+r10*1+400*2+4] + paddw xm2, xm0, [t4+r10*1+400*2+2] + mova m1, [t3+r10*2+400*4+0] + paddd m1, [t3+r10*2+400*4+8] + paddd m3, m1, [t3+r10*2+400*4+4] + psllw xm2, 2 ; a[ 0] 444 + pslld m3, 2 ; b[ 0] 444 + mova [t4+r10*1+400* 6], xm2 + mova [t3+r10*2+400*12], m3 + psubw xm2, xm0 ; a[ 0] 343 + psubd m3, m1 ; b[ 0] 343 + mova [t4+r10*1+400* 8], xm2 + mova [t3+r10*2+400*16], m3 + add r10, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + mova m3, [t4+r10*1+400*0+0] + paddw m3, [t4+r10*1+400*0+4] + paddw m1, m3, [t4+r10*1+400*0+2] + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+r10*1+400*4] + paddw m3, [t4+r10*1+400*6] + mova [t4+r10*1+400*4], m2 + mova [t4+r10*1+400*6], m1 + mova m4, [t3+r10*2+400*0+0] + paddd m4, [t3+r10*2+400*0+8] + paddd m1, m4, [t3+r10*2+400*0+4] + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+r10*2+400* 8+ 0] + paddd m4, [t3+r10*2+400*12+ 0] + mova [t3+r10*2+400* 8+ 0], m2 + mova [t3+r10*2+400*12+ 0], m1 + mova m5, [t3+r10*2+400*0+32] + paddd m5, [t3+r10*2+400*0+40] + paddd m1, m5, [t3+r10*2+400*0+36] + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+r10*2+400* 8+32] + paddd m5, [t3+r10*2+400*12+32] + mova [t3+r10*2+400* 8+32], m2 + mova [t3+r10*2+400*12+32], m1 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vinserti128 m1, m4, xm5, 1 + vperm2i128 m4, m5, 0x31 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+r10], m0 + add r10, 32 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m3, [t4+r10*1+400*2+0] + paddw m3, [t4+r10*1+400*2+4] + paddw m1, m3, [t4+r10*1+400*2+2] + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+r10*1+400*6] + paddw m3, [t4+r10*1+400*8] + mova [t4+r10*1+400*6], m1 + mova [t4+r10*1+400*8], m2 + mova m4, [t3+r10*2+400*4+0] + paddd m4, [t3+r10*2+400*4+8] + paddd m1, m4, [t3+r10*2+400*4+4] + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+r10*2+400*12+ 0] + paddd m4, [t3+r10*2+400*16+ 0] + mova [t3+r10*2+400*12+ 0], m1 + mova [t3+r10*2+400*16+ 0], m2 + mova m5, [t3+r10*2+400*4+32] + paddd m5, [t3+r10*2+400*4+40] + paddd m1, m5, [t3+r10*2+400*4+36] + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+r10*2+400*12+32] + paddd m5, [t3+r10*2+400*16+32] + mova [t3+r10*2+400*12+32], m1 + mova [t3+r10*2+400*16+32], m2 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vinserti128 m1, m4, xm5, 1 + vperm2i128 m4, m5, 0x31 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+r10], m0 + add r10, 32 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_mix_16bpc, 4, 14, 16, 400*66+8, dst, stride, left, lpf, \ + w, h, edge, params + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [sgr_x_by_x_avx2+256*4] + add wd, wd + movifnidn hd, hm + mov edged, r7m + add lpfq, wq + vpbroadcastd m15, [paramsq+8] ; w0 w1 + add dstq, wq + vpbroadcastd m13, [paramsq+0] ; s0 + lea t3, [rsp+wq*2+400*24+8] + vpbroadcastd m14, [paramsq+4] ; s1 + lea t4, [rsp+wq+400*52+8] + vpbroadcastd m9, [pd_8] + lea t1, [rsp+wq+12] + vpbroadcastd m10, [pd_34816] + neg wq + vpbroadcastd m11, [pd_4096] + pxor m7, m7 + vpbroadcastd m12, [pd_0xf00801c7] + psllw m15, 2 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_avx2).top_fixup + add t1, 400*12 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea r10, [wq-4] + lea t2, [t1+400*12] +.top_fixup_loop: + mova m0, [t1+r10+400* 0] + mova m1, [t1+r10+400* 2] + mova m2, [t1+r10+400* 4] + paddw m0, m0 + mova m3, [t1+r10+400* 6] + paddd m1, m1 + mova m4, [t1+r10+400* 8] + paddd m2, m2 + mova m5, [t1+r10+400*10] + mova [t2+r10+400* 0], m0 + mova [t2+r10+400* 2], m1 + mova [t2+r10+400* 4], m2 + mova [t2+r10+400* 6], m3 + mova [t2+r10+400* 8], m4 + mova [t2+r10+400*10], m5 + add r10, 32 + jl .top_fixup_loop + call .v0 + jmp .main +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, [sgr_lshuf5] + vinserti128 m4, [lpfq+wq+10], 1 + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+r10- 2] +.h_main: + movu m5, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -36 + jl .h_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_avx2).extend_right +.h_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; sum3 + punpcklwd m6, m0, m7 + pmaddwd m6, m6 + punpckhwd m0, m7 + pmaddwd m0, m0 + paddd m2, m6 ; sumsq3 + shufpd m6, m4, m5, 0x05 + punpcklwd m5, m6, m4 + paddw m8, m4, m6 + pmaddwd m5, m5 + punpckhwd m6, m4 + pmaddwd m6, m6 + paddd m3, m0 + mova [t1+r10+400* 6], m1 + mova [t1+r10+400* 8], m2 + mova [t1+r10+400*10], m3 + paddw m8, m1 ; sum5 + paddd m5, m2 ; sumsq5 + paddd m6, m3 + mova [t1+r10+400* 0], m8 + mova [t1+r10+400* 2], m5 + mova [t1+r10+400* 4], m6 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 10 + jmp .hv0_main +.hv0_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, [sgr_lshuf5] + vinserti128 m4, [lpfq+wq+10], 1 + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu m4, [lpfq+r10- 2] +.hv0_main: + movu m5, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -36 + jl .hv0_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_avx2).extend_right +.hv0_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; h sum3 + punpcklwd m6, m0, m7 + pmaddwd m6, m6 + punpckhwd m0, m7 + pmaddwd m0, m0 + paddd m2, m6 ; h sumsq3 + shufpd m6, m4, m5, 0x05 + punpcklwd m5, m6, m4 + paddw m8, m4, m6 + pmaddwd m5, m5 + punpckhwd m6, m4 + pmaddwd m6, m6 + paddd m3, m0 + paddw m8, m1 ; h sum5 + paddd m5, m2 ; h sumsq5 + paddd m6, m3 + mova [t3+r10*2+400*8+ 8], m8 ; we need a clean copy of the last row TODO: t4? + mova [t3+r10*2+400*0+ 8], m5 ; in case height is odd + mova [t3+r10*2+400*0+40], m6 + paddw m8, [t1+r10+400* 0] + paddd m5, [t1+r10+400* 2] + paddd m6, [t1+r10+400* 4] + mova [t1+r10+400* 0], m8 + mova [t1+r10+400* 2], m5 + mova [t1+r10+400* 4], m6 + paddw m0, m1, [t1+r10+400* 6] + paddd m4, m2, [t1+r10+400* 8] + paddd m5, m3, [t1+r10+400*10] + mova [t1+r10+400* 6], m1 + mova [t1+r10+400* 8], m2 + mova [t1+r10+400*10], m3 + paddw m1, m0, [t2+r10+400* 6] + paddd m2, m4, [t2+r10+400* 8] + paddd m3, m5, [t2+r10+400*10] + mova [t2+r10+400* 6], m0 + mova [t2+r10+400* 8], m4 + mova [t2+r10+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pmaxud m4, m2 + psubd m4, m2 ; p3 + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m12 ; b3 * 455 + pmaddwd m1, m12 + paddusw m4, m12 + paddusw m5, m12 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x3 + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*2+ 4], m2 + mova [t3+r10*2+400*4+ 8], xm0 + vextracti128 [t3+r10*2+400*4+40], m0, 1 + mova [t3+r10*2+400*4+24], xm1 + vextracti128 [t3+r10*2+400*4+56], m1, 1 + add r10, 32 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + vpbroadcastq xm5, [leftq] + vinserti128 m5, [lpfq+wq], 1 + mova m4, [lpfq+wq] + add leftq, 8 + palignr m4, m5, 10 + jmp .hv1_main +.hv1_extend_left: + mova xm4, [lpfq+wq] + pshufb xm4, [sgr_lshuf5] + vinserti128 m4, [lpfq+wq+10], 1 + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu m4, [lpfq+r10- 2] +.hv1_main: + movu m5, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -36 + jl .hv1_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_avx2).extend_right +.hv1_have_right: + palignr m6, m5, m4, 2 + palignr m3, m5, m4, 4 + paddw m2, m6, m3 + punpcklwd m0, m6, m3 + pmaddwd m0, m0 + punpckhwd m6, m3 + pmaddwd m6, m6 + palignr m3, m5, m4, 6 + paddw m2, m3 ; h sum3 + punpcklwd m1, m3, m7 + pmaddwd m1, m1 + punpckhwd m3, m7 + pmaddwd m3, m3 + paddd m0, m1 ; h sumsq3 + shufpd m1, m4, m5, 0x05 + punpckhwd m5, m4, m1 + paddw m8, m4, m1 + pmaddwd m5, m5 + punpcklwd m4, m1 + pmaddwd m4, m4 + paddd m6, m3 + paddw m1, m2, [t2+r10+400* 6] + mova [t2+r10+400* 6], m2 + paddw m8, m2 ; h sum5 + paddd m2, m0, [t2+r10+400* 8] + paddd m3, m6, [t2+r10+400*10] + mova [t2+r10+400* 8], m0 + mova [t2+r10+400*10], m6 + paddd m4, m0 ; h sumsq5 + paddd m5, m6 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m0, m2, 3 + pslld m6, m3, 3 + paddd m2, m0 ; ((a3 + 8) >> 4) * 9 + paddd m3, m6 + psrlw m6, m1, 1 + pavgw m6, m7 ; (b3 + 2) >> 2 + punpcklwd m0, m6, m7 + pmaddwd m0, m0 + punpckhwd m6, m7 + pmaddwd m6, m6 + pmaxud m2, m0 + psubd m2, m0 ; p3 + pmaxud m3, m6 + psubd m3, m6 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pmulld m2, m14 ; p3 * s1 + pmulld m3, m14 + pmaddwd m0, m12 ; b3 * 455 + pmaddwd m1, m12 + paddusw m2, m12 + paddusw m3, m12 + psrad m7, m2, 20 ; min(z3, 255) - 256 + vpgatherdd m6, [r13+m7*4], m2 ; x3 + psrad m2, m3, 20 + vpgatherdd m7, [r13+m2*4], m3 + pmulld m0, m6 + packssdw m6, m7 + pmulld m7, m1 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m7, m10 + psrld m0, 12 + psrld m7, 12 + paddw m1, m8, [t2+r10+400*0] + paddd m2, m4, [t2+r10+400*2] + paddd m3, m5, [t2+r10+400*4] + paddw m1, [t1+r10+400*0] + paddd m2, [t1+r10+400*2] + paddd m3, [t1+r10+400*4] + mova [t2+r10+400*0], m8 + mova [t2+r10+400*2], m4 + mova [t2+r10+400*4], m5 + mova [t4+r10*1+400*4 +4], m6 + mova [t3+r10*2+400*8+ 8], xm0 + vextracti128 [t3+r10*2+400*8+40], m0, 1 + mova [t3+r10*2+400*8+24], xm7 + vextracti128 [t3+r10*2+400*8+56], m7, 1 + vpbroadcastd m4, [pd_25] + pxor m7, m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + pmulld m2, m4 ; ((a5 + 8) >> 4) * 25 + pmulld m3, m4 + psrlw m5, m1, 1 + pavgw m5, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m5, m7 + pmaddwd m4, m4 + punpckhwd m5, m7 + pmaddwd m5, m5 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 + pmaxud m2, m4 + psubd m2, m4 ; p5 + vpbroadcastd m4, [pd_0xf00800a4] + pmaxud m3, m5 + psubd m3, m5 + pmulld m2, m13 ; p5 * s0 + pmulld m3, m13 + pmaddwd m0, m4 ; b5 * 164 + pmaddwd m1, m4 + paddusw m2, m4 + paddusw m3, m4 + psrad m5, m2, 20 ; min(z5, 255) - 256 + vpgatherdd m4, [r13+m5*4], m2 ; x5 + psrad m2, m3, 20 + vpgatherdd m5, [r13+m2*4], m3 + pmulld m0, m4 + pmulld m1, m5 + packssdw m4, m5 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*0+ 4], m4 + mova [t3+r10*2+400*0+ 8], xm0 + vextracti128 [t3+r10*2+400*0+40], m0, 1 + mova [t3+r10*2+400*0+24], xm1 + vextracti128 [t3+r10*2+400*0+56], m1, 1 + add r10, 32 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) + lea r10, [wq-4] +.v0_loop: + mova m0, [t1+r10+400* 6] + mova m4, [t1+r10+400* 8] + mova m5, [t1+r10+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+r10+400* 6] + paddd m2, m4, [t2+r10+400* 8] + paddd m3, m5, [t2+r10+400*10] + mova [t2+r10+400* 6], m0 + mova [t2+r10+400* 8], m4 + mova [t2+r10+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pmaxud m4, m2 + psubd m4, m2 ; p3 + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m12 ; b3 * 455 + pmaddwd m1, m12 + paddusw m4, m12 + paddusw m5, m12 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x3 + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + psrld m0, 12 + psrld m1, 12 + mova m3, [t1+r10+400*0] + mova m4, [t1+r10+400*2] + mova m5, [t1+r10+400*4] + mova [t3+r10*2+400*8+ 8], m3 + mova [t3+r10*2+400*0+ 8], m4 + mova [t3+r10*2+400*0+40], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+r10+400*0], m3 + mova [t1+r10+400*2], m4 + mova [t1+r10+400*4], m5 + mova [t4+r10*1+400*2+ 4], m2 + mova [t3+r10*2+400*4+ 8], xm0 + vextracti128 [t3+r10*2+400*4+40], m0, 1 + mova [t3+r10*2+400*4+24], xm1 + vextracti128 [t3+r10*2+400*4+56], m1, 1 + add r10, 32 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-4] +.v1_loop: + mova m4, [t1+r10+400* 6] + mova m5, [t1+r10+400* 8] + mova m6, [t1+r10+400*10] + paddw m1, m4, [t2+r10+400* 6] + paddd m2, m5, [t2+r10+400* 8] + paddd m3, m6, [t2+r10+400*10] + mova [t2+r10+400* 6], m4 + mova [t2+r10+400* 8], m5 + mova [t2+r10+400*10], m6 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pmaxud m4, m2 + psubd m4, m2 ; p3 + pmaxud m5, m3 + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m12 ; b3 * 455 + pmaddwd m1, m12 + paddusw m4, m12 + paddusw m5, m12 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r13+m3*4], m4 ; x3 + psrad m4, m5, 20 + vpgatherdd m3, [r13+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + packssdw m2, m3 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + psrld m0, 12 + psrld m8, m1, 12 + mova [t4+r10*1+400*4+4], m2 + mova m4, [t3+r10*2+400*8+ 8] + mova m5, [t3+r10*2+400*0+ 8] + mova m6, [t3+r10*2+400*0+40] + paddw m1, m4, [t2+r10+400*0] + paddd m2, m5, [t2+r10+400*2] + paddd m3, m6, [t2+r10+400*4] + paddw m1, [t1+r10+400*0] + paddd m2, [t1+r10+400*2] + paddd m3, [t1+r10+400*4] + mova [t2+r10+400*0], m4 + mova [t2+r10+400*2], m5 + mova [t2+r10+400*4], m6 + vpbroadcastd m4, [pd_25] + mova [t3+r10*2+400*8+ 8], xm0 + vextracti128 [t3+r10*2+400*8+40], m0, 1 + mova [t3+r10*2+400*8+24], xm8 + vextracti128 [t3+r10*2+400*8+56], m8, 1 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + pmulld m2, m4 ; ((a5 + 8) >> 4) * 25 + pmulld m3, m4 + psrlw m5, m1, 1 + pavgw m5, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m5, m7 + pmaddwd m4, m4 + punpckhwd m5, m7 + pmaddwd m5, m5 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 + pmaxud m2, m4 + psubd m2, m4 ; p5 + vpbroadcastd m4, [pd_0xf00800a4] + pmaxud m3, m5 + psubd m3, m5 + pmulld m2, m13 ; p5 * s0 + pmulld m3, m13 + pmaddwd m0, m4 ; b5 * 164 + pmaddwd m1, m4 + paddusw m2, m4 + paddusw m3, m4 + psrad m5, m2, 20 ; min(z5, 255) - 256 + vpgatherdd m4, [r13+m5*4], m2 ; x5 + psrad m2, m3, 20 + vpgatherdd m5, [r13+m2*4], m3 + pmulld m0, m4 + pmulld m1, m5 + packssdw m4, m5 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + psrld m0, 12 + psrld m1, 12 + mova [t4+r10*1+400*0+ 4], m4 + mova [t3+r10*2+400*0+ 8], xm0 + vextracti128 [t3+r10*2+400*0+40], m0, 1 + mova [t3+r10*2+400*0+24], xm1 + vextracti128 [t3+r10*2+400*0+56], m1, 1 + add r10, 32 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu xm0, [t4+r10*1+400*0+2] + paddw xm2, xm0, [t4+r10*1+400*0+0] + paddw xm2, [t4+r10*1+400*0+4] + movu m1, [t3+r10*2+400*0+4] + paddd m3, m1, [t3+r10*2+400*0+0] + paddd m3, [t3+r10*2+400*0+8] + paddw xm0, xm2 + paddd m1, m3 + psllw xm2, 2 + pslld m3, 2 + paddw xm0, xm2 ; a5 565 + paddd m1, m3 ; b5 565 + mova [t4+r10*1+400* 6], xm0 + mova [t3+r10*2+400*12], m1 + mova xm0, [t4+r10*1+400*2+0] + paddw xm0, [t4+r10*1+400*2+4] + paddw xm2, xm0, [t4+r10*1+400*2+2] + mova m1, [t3+r10*2+400*4+0] + paddd m1, [t3+r10*2+400*4+8] + paddd m3, m1, [t3+r10*2+400*4+4] + psllw xm2, 2 ; a3[-1] 444 + pslld m3, 2 ; b3[-1] 444 + psubw xm2, xm0 ; a3[-1] 343 + psubd m3, m1 ; b3[-1] 343 + mova [t4+r10*1+400* 8], xm2 + mova [t3+r10*2+400*16], m3 + mova xm0, [t4+r10*1+400*4+0] + paddw xm0, [t4+r10*1+400*4+4] + paddw xm2, xm0, [t4+r10*1+400*4+2] + mova m1, [t3+r10*2+400*8+0] + paddd m1, [t3+r10*2+400*8+8] + paddd m3, m1, [t3+r10*2+400*8+4] + psllw xm2, 2 ; a3[ 0] 444 + pslld m3, 2 ; b3[ 0] 444 + mova [t4+r10*1+400*10], xm2 + mova [t3+r10*2+400*20], m3 + psubw xm2, xm0 ; a3[ 0] 343 + psubd m3, m1 ; b3[ 0] 343 + mova [t4+r10*1+400*12], xm2 + mova [t3+r10*2+400*24], m3 + add r10, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu xm2, [t4+r10*1+2] + paddw xm0, xm2, [t4+r10*1+0] + paddw xm0, [t4+r10*1+4] + paddw xm2, xm0 + psllw xm0, 2 + paddw xm0, xm2 ; a5 + movu m1, [t3+r10*2+4] + paddd m4, m1, [t3+r10*2+0] + paddd m4, [t3+r10*2+8] + paddd m1, m4 + pslld m4, 2 + paddd m4, m1 ; b5 + paddw xm2, xm0, [t4+r10*1+400* 6] + mova [t4+r10*1+400* 6], xm0 + paddd m0, m4, [t3+r10*2+400*12] + mova [t3+r10*2+400*12], m4 + mova xm3, [t4+r10*1+400*2+0] + paddw xm3, [t4+r10*1+400*2+4] + paddw xm5, xm3, [t4+r10*1+400*2+2] + psllw xm5, 2 ; a3[ 1] 444 + psubw xm4, xm5, xm3 ; a3[ 1] 343 + paddw xm3, xm4, [t4+r10*1+400* 8] + paddw xm3, [t4+r10*1+400*10] + mova [t4+r10*1+400* 8], xm4 + mova [t4+r10*1+400*10], xm5 + mova m1, [t3+r10*2+400*4+0] + paddd m1, [t3+r10*2+400*4+8] + paddd m5, m1, [t3+r10*2+400*4+4] + pslld m5, 2 ; b3[ 1] 444 + psubd m4, m5, m1 ; b3[ 1] 343 + paddd m1, m4, [t3+r10*2+400*16] + paddd m1, [t3+r10*2+400*20] + mova [t3+r10*2+400*16], m4 + mova [t3+r10*2+400*20], m5 + pmovzxwd m4, [dstq+r10] + pmovzxwd m2, xm2 ; a5 + pmovzxwd m3, xm3 ; a3 + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + pslld m4, 13 + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) + psrld m0, 9 + pslld m1, 7 + pblendw m0, m1, 0xaa + pmaddwd m0, m15 + paddd m4, m11 + paddd m0, m4 + psrad m0, 7 + vextracti128 xm1, m0, 1 + packusdw xm0, xm1 ; clip + psrlw xm0, 6 + mova [dstq+r10], xm0 + add r10, 16 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova xm3, [t4+r10*1+400*4+0] + paddw xm3, [t4+r10*1+400*4+4] + paddw xm5, xm3, [t4+r10*1+400*4+2] + psllw xm5, 2 ; a3[ 1] 444 + psubw xm4, xm5, xm3 ; a3[ 1] 343 + paddw xm3, xm4, [t4+r10*1+400*12] + paddw xm3, [t4+r10*1+400*10] + mova [t4+r10*1+400*10], xm5 + mova [t4+r10*1+400*12], xm4 + mova m1, [t3+r10*2+400*8+0] + paddd m1, [t3+r10*2+400*8+8] + paddd m5, m1, [t3+r10*2+400*8+4] + pslld m5, 2 ; b3[ 1] 444 + psubd m4, m5, m1 ; b3[ 1] 343 + paddd m1, m4, [t3+r10*2+400*24] + paddd m1, [t3+r10*2+400*20] + mova [t3+r10*2+400*20], m5 + mova [t3+r10*2+400*24], m4 + pmovzxwd m4, [dstq+r10] + pmovzxwd m2, [t4+r10*1+400* 6] + pmovzxwd m3, xm3 + mova m0, [t3+r10*2+400*12] + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + pslld m4, 13 + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) + psrld m0, 8 + pslld m1, 7 + pblendw m0, m1, 0xaa + pmaddwd m0, m15 + paddd m4, m11 + paddd m0, m4 + psrad m0, 7 + vextracti128 xm1, m0, 1 + packusdw xm0, xm1 ; clip + psrlw xm0, 6 + mova [dstq+r10], xm0 + add r10, 16 + jl .n1_loop + add dstq, strideq + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/looprestoration16_avx512.asm b/third_party/dav1d/src/x86/looprestoration16_avx512.asm new file mode 100644 index 0000000000..e560c54a40 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration16_avx512.asm @@ -0,0 +1,2524 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 16 + +wiener_shufA: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 +wiener_shufB: db 6, 7, 4, 5, 8, 9, 6, 7, 10, 11, 8, 9, 12, 13, 10, 11 +wiener_shufC: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 +wiener_shufD: db 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 +wiener_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +r_ext_mask: times 72 db -1 + times 8 db 0 +wiener_hshift: dw 4, 4, 1, 1 +wiener_vshift: dw 1024, 1024, 4096, 4096 +wiener_round: dd 1049600, 1048832 + +pw_164_455: dw 164, 455 +pw_1023: times 2 dw 1023 +pw_61448: times 2 dw 61448 +pd_m262128: dd -262128 +pd_m34816: dd -34816 +pd_m25: dd -25 +pd_m9: dd -9 +pd_8: dd 8 +pd_2147483648: dd 2147483648 + +cextern sgr_x_by_x + +SECTION .text + +DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers + +INIT_ZMM avx512icl +cglobal wiener_filter7_16bpc, 4, 15, 17, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt +%define base t4-wiener_hshift + mov fltq, r6mp + movifnidn wd, wm + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + vbroadcasti128 m6, [wiener_shufA] + vpbroadcastd m12, [fltq+ 0] ; x0 x1 + lea t4, [wiener_hshift] + vbroadcasti128 m7, [wiener_shufB] + add wd, wd + vpbroadcastd m13, [fltq+ 4] ; x2 x3 + shr t3d, 11 + vpbroadcastd m14, [fltq+16] ; y0 y1 + add lpfq, wq + vpbroadcastd m15, [fltq+20] ; y2 y3 + add dstq, wq + vbroadcasti128 m8, [wiener_shufC] + lea t1, [rsp+wq+16] + vbroadcasti128 m9, [wiener_shufD] + neg wq + vpbroadcastd m0, [base+wiener_hshift+t3*4] + mov r10d, 0xfe + vpbroadcastd m10, [base+wiener_round+t3*4] + kmovb k1, r10d + vpbroadcastd m11, [base+wiener_vshift+t3*4] + pmullw m12, m0 ; upshift filter coefs to make the + vpbroadcastd m16, [pd_m262128] + pmullw m13, m0 ; horizontal downshift constant + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call .v + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call .v +.v2: + call .v + jmp .v1 +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movq xm3, [leftq] + vmovdqu64 m3{k1}, [lpfq+r10-8] + add leftq, 8 + jmp .h_main +.h_extend_left: + mova m4, [lpfq+r10+0] + vpbroadcastw xm3, xm4 + vmovdqu64 m3{k1}, [lpfq+r10-8] + jmp .h_main2 +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+r10-8] +.h_main: + mova m4, [lpfq+r10+0] +.h_main2: + movu m5, [lpfq+r10+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -68 + jl .h_have_right + push r0 + lea r0, [r_ext_mask+66] + vpbroadcastw m0, [lpfq-2] + vpternlogd m3, m0, [r0+r10+ 0], 0xe4 ; c ? a : b + vpternlogd m4, m0, [r0+r10+ 8], 0xe4 + vpternlogd m5, m0, [r0+r10+16], 0xe4 + pop r0 +.h_have_right: + pshufb m2, m3, m6 + pshufb m1, m4, m7 + paddw m2, m1 + pshufb m3, m8 + mova m0, m16 + vpdpwssd m0, m2, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + vpdpwssd m0, m3, m13 + pshufb m2, m5, m7 + paddw m2, m1 + mova m1, m16 + pshufb m4, m8 + vpdpwssd m1, m2, m12 + pshufb m5, m9 + paddw m4, m5 + vpdpwssd m1, m4, m13 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+r10], m0 + add r10, 64 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movq xm3, [leftq] + vmovdqu64 m3{k1}, [lpfq+r10-8] + add leftq, 8 + jmp .hv_main +.hv_extend_left: + mova m4, [lpfq+r10+0] + vpbroadcastw xm3, xm4 + vmovdqu64 m3{k1}, [lpfq+r10-8] + jmp .hv_main2 +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+r10-8] +.hv_main: + mova m4, [lpfq+r10+0] +.hv_main2: + movu m5, [lpfq+r10+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -68 + jl .hv_have_right + push r0 + lea r0, [r_ext_mask+66] + vpbroadcastw m0, [lpfq-2] + vpternlogd m3, m0, [r0+r10+ 0], 0xe4 + vpternlogd m4, m0, [r0+r10+ 8], 0xe4 + vpternlogd m5, m0, [r0+r10+16], 0xe4 + pop r0 +.hv_have_right: + pshufb m2, m3, m6 + pshufb m1, m4, m7 + paddw m2, m1 + pshufb m3, m8 + mova m0, m16 + vpdpwssd m0, m2, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + vpdpwssd m0, m3, m13 + pshufb m2, m5, m7 + paddw m2, m1 + pshufb m4, m8 + mova m1, m16 + vpdpwssd m1, m2, m12 + pshufb m5, m9 + paddw m4, m5 + vpdpwssd m1, m4, m13 + mova m2, [t4+r10] + paddw m2, [t2+r10] + mova m5, [t3+r10] + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + mova m4, [t5+r10] + paddw m4, [t1+r10] + psraw m0, 1 + paddw m3, m0, [t6+r10] + mova [t0+r10], m0 + punpcklwd m1, m2, m5 + mova m0, m10 + vpdpwssd m0, m1, m15 + punpckhwd m2, m5 + mova m1, m10 + vpdpwssd m1, m2, m15 + punpcklwd m2, m3, m4 + vpdpwssd m0, m2, m14 + punpckhwd m3, m4 + vpdpwssd m1, m3, m14 + psrad m0, 5 + psrad m1, 5 + packusdw m0, m1 + pmulhuw m0, m11 + mova [dstq+r10], m0 + add r10, 64 + jl .hv_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m2, [t4+r10] + paddw m2, [t2+r10] + mova m3, [t3+r10] + punpcklwd m1, m2, m3 + mova m0, m10 + vpdpwssd m0, m1, m15 + punpckhwd m2, m3 + mova m1, m10 + vpdpwssd m1, m2, m15 + mova m4, [t1+r10] + paddw m3, m4, [t6+r10] + paddw m4, [t5+r10] + punpcklwd m2, m3, m4 + vpdpwssd m0, m2, m14 + punpckhwd m3, m4 + vpdpwssd m1, m3, m14 + psrad m0, 5 + psrad m1, 5 + packusdw m0, m1 + pmulhuw m0, m11 + mova [dstq+r10], m0 + add r10, 64 + jl .v_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq + ret + +cglobal wiener_filter5_16bpc, 4, 14, 15, 384*8+16, dst, stride, left, lpf, \ + w, h, edge, flt +%define base r13-r_ext_mask-70 + mov fltq, r6mp + movifnidn wd, wm + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + vbroadcasti128 m5, [wiener_shufE] + vpbroadcastw m11, [fltq+ 2] ; x1 + vbroadcasti128 m6, [wiener_shufB] + lea r13, [r_ext_mask+70] + vbroadcasti128 m7, [wiener_shufD] + add wd, wd + vpbroadcastd m12, [fltq+ 4] ; x2 x3 + shr t3d, 11 + vpbroadcastd m8, [pd_m262128] ; (1 << 4) - (1 << 18) + add lpfq, wq + vpbroadcastw m13, [fltq+18] ; y1 + add dstq, wq + vpbroadcastd m14, [fltq+20] ; y2 y3 + lea t1, [rsp+wq+16] + vpbroadcastd m0, [base+wiener_hshift+t3*4] + neg wq + vpbroadcastd m9, [base+wiener_round+t3*4] + mov r10d, 0xfffe + vpbroadcastd m10, [base+wiener_vshift+t3*4] + kmovw k1, r10d + pmullw m11, m0 + pmullw m12, m0 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t4, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t3, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 +.main: + mov t0, t4 +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v2 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.end: + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v2 + add t0, 384*6 + call .hv + dec hd + jnz .main +.v2: + call .v + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq +.v1: + call .v + jmp .end +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm3, [leftq+4] + vmovdqu32 m3{k1}, [lpfq+r10-4] + add leftq, 8 + jmp .h_main +.h_extend_left: + vpbroadcastw xm3, [lpfq+r10] + vmovdqu32 m3{k1}, [lpfq+r10-4] + jmp .h_main +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+r10-4] +.h_main: + movu m4, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -66 + jl .h_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m3, m0, [r13+r10+0], 0xe4 ; c ? a : b + vpternlogd m4, m0, [r13+r10+8], 0xe4 +.h_have_right: + pshufb m1, m3, m5 + mova m0, m8 + vpdpwssd m0, m1, m11 + pshufb m2, m4, m5 + mova m1, m8 + vpdpwssd m1, m2, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + vpdpwssd m0, m2, m12 + pshufb m4, m7 + paddw m3, m4 + vpdpwssd m1, m3, m12 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+r10], m0 + add r10, 64 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm3, [leftq+4] + vmovdqu32 m3{k1}, [lpfq+r10-4] + add leftq, 8 + jmp .hv_main +.hv_extend_left: + vpbroadcastw xm3, [lpfq+r10] + vmovdqu32 m3{k1}, [lpfq+r10-4] + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+r10-4] +.hv_main: + movu m4, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -66 + jl .hv_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m3, m0, [r13+r10+0], 0xe4 + vpternlogd m4, m0, [r13+r10+8], 0xe4 +.hv_have_right: + pshufb m1, m3, m5 + mova m0, m8 + vpdpwssd m0, m1, m11 + pshufb m2, m4, m5 + mova m1, m8 + vpdpwssd m1, m2, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + vpdpwssd m0, m2, m12 + pshufb m4, m7 + paddw m4, m3 + vpdpwssd m1, m4, m12 + mova m2, [t3+r10] + paddw m2, [t1+r10] + mova m3, [t2+r10] + punpcklwd m4, m2, m3 + punpckhwd m2, m3 + mova m3, m9 + vpdpwssd m3, m2, m14 + mova m2, m9 + vpdpwssd m2, m4, m14 + mova m4, [t4+r10] + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t0+r10], m0 + punpcklwd m1, m0, m4 + vpdpwssd m2, m1, m13 + punpckhwd m0, m4 + vpdpwssd m3, m0, m13 + psrad m2, 5 + psrad m3, 5 + packusdw m2, m3 + pmulhuw m2, m10 + mova [dstq+r10], m2 + add r10, 64 + jl .hv_loop + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t4 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m0, [t1+r10] + paddw m2, m0, [t3+r10] + mova m1, [t2+r10] + mova m4, [t4+r10] + punpckhwd m3, m2, m1 + pmaddwd m3, m14 + punpcklwd m2, m1 + pmaddwd m2, m14 + punpckhwd m1, m0, m4 + pmaddwd m1, m13 + punpcklwd m0, m4 + pmaddwd m0, m13 + paddd m3, m9 + paddd m2, m9 + paddd m1, m3 + paddd m0, m2 + psrad m1, 5 + psrad m0, 5 + packusdw m0, m1 + pmulhuw m0, m10 + mova [dstq+r10], m0 + add r10, 64 + jl .v_loop + ret + +cglobal sgr_filter_5x5_16bpc, 4, 14, 22, 416*24+8, dst, stride, left, lpf, \ + w, h, edge, params +%define base r13-r_ext_mask-72 + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [r_ext_mask+72] + mov edged, r7m + movifnidn hd, hm + pxor m6, m6 + vpbroadcastw m7, [paramsq+8] ; w0 + add wd, wd + vpbroadcastd m8, [base+pd_8] + add lpfq, wq + vpbroadcastd m9, [base+pd_m25] + add dstq, wq + vpsubd m10, m6, [paramsq+0] {1to16} ; -s0 + lea t3, [rsp+wq*2+416*12+8] + vpbroadcastd m11, [base+pw_164_455] + lea t4, [rsp+wq+416*20+8] + vpbroadcastd m12, [base+pw_61448] ; (15 << 12) + (1 << 3) + lea t1, [rsp+wq+12] + vpbroadcastd m13, [base+pd_m34816] ; -((1 << 11) + (1 << 15)) + neg wq + vpbroadcastd m14, [base+pw_1023] + psllw m7, 4 + mova m18, [sgr_x_by_x+64*0] + mov r10d, 0xfffffff8 + mova m19, [sgr_x_by_x+64*1] + kmovd k1, r10d + mova m20, [sgr_x_by_x+64*2] + mov r10, 0x3333333333333333 + mova m21, [sgr_x_by_x+64*3] + kmovq k2, r10 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call .top_fixup + add t1, 416*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, strideq + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + test hd, hd + jz .odd_height + call .h + add lpfq, strideq + call .hv + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .h_top + add lpfq, strideq + call .hv_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + call .n0 + call .n1 +.odd_height_end: + call .v + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+416*6] + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + jmp .main +.no_top_height1: + call .v + call .prep_n + jmp .odd_height_end +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movq xm16, [leftq+2] + vmovdqu16 m16{k1}, [lpfq+wq-6] + add leftq, 8 + jmp .h_main +.h_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-6] + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m16, [lpfq+r10- 2] +.h_main: + movu m17, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -68 + jl .h_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 ; c ? a : b + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.h_have_right: + palignr m2, m17, m16, 2 + paddw m0, m16, m2 + palignr m3, m17, m16, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + shufpd m17, m16, m17, 0x55 + paddw m0, m17 + punpcklwd m3, m16, m17 + vpdpwssd m1, m3, m3 + punpckhwd m3, m16, m17 + vpdpwssd m2, m3, m3 + shufps m16, m17, q2121 + paddw m0, m16 ; sum + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+r10+416*0] + paddd m1, [t1+r10+416*2] + paddd m2, [t1+r10+416*4] +.h_loop_end: + punpcklwd m17, m16, m6 + vpdpwssd m1, m17, m17 ; sumsq + punpckhwd m16, m6 + vpdpwssd m2, m16, m16 + mova [t1+r10+416*0], m0 + mova [t1+r10+416*2], m1 + mova [t1+r10+416*4], m2 + add r10, 64 + jl .h_loop + ret +.top_fixup: + lea r10, [wq-4] +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+r10+416*0] + mova m1, [t1+r10+416*2] + mova m2, [t1+r10+416*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m1 + mova [t2+r10+416*4], m2 + add r10, 64 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movq xm16, [leftq+2] + vmovdqu16 m16{k1}, [lpfq+wq-6] + add leftq, 8 + jmp .hv_main +.hv_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-6] + jmp .hv_main +.hv_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m16, [lpfq+r10- 2] +.hv_main: + movu m17, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -68 + jl .hv_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.hv_have_right: + palignr m3, m17, m16, 2 + paddw m0, m16, m3 + palignr m1, m17, m16, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + shufpd m17, m16, m17, 0x55 + paddw m0, m17 + punpcklwd m1, m16, m17 + vpdpwssd m2, m1, m1 + punpckhwd m1, m16, m17 + vpdpwssd m3, m1, m1 + shufps m16, m17, q2121 + paddw m0, m16 ; h sum + punpcklwd m17, m16, m6 + vpdpwssd m2, m17, m17 ; h sumsq + punpckhwd m16, m6 + vpdpwssd m3, m16, m16 + paddw m1, m0, [t1+r10+416*0] + paddd m16, m2, [t1+r10+416*2] + paddd m17, m3, [t1+r10+416*4] + test hd, hd + jz .hv_last_row +.hv_main2: + paddw m1, [t2+r10+416*0] ; hv sum + paddd m16, [t2+r10+416*2] ; hv sumsq + paddd m17, [t2+r10+416*4] + mova [t0+r10+416*0], m0 + mova [t0+r10+416*2], m2 + mova [t0+r10+416*4], m3 + psrlw m3, m1, 1 + paddd m16, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m17, m8 + psrld m16, 4 ; (a + 8) >> 4 + psrld m17, 4 + pmulld m16, m9 ; -a * 25 + pmulld m17, m9 + punpcklwd m2, m3, m6 + vpdpwssd m16, m2, m2 ; -p + punpckhwd m3, m6 + vpdpwssd m17, m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmulld m16, m10 ; p * s + pmulld m17, m10 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + pmaxsw m17, m6 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + packssdw m16, m17 + psubd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + psubd m1, m13 + mova [t4+r10+4], m16 + psrld m16, m0, 12 ; b + psrld m17, m1, 12 + mova [t3+r10*2+ 8], xm16 + mova [t3+r10*2+ 24], xm17 + vextracti128 [t3+r10*2+ 40], ym16, 1 + vextracti128 [t3+r10*2+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+ 72], m16, 2 + vextracti32x4 [t3+r10*2+ 88], m17, 2 + vextracti32x4 [t3+r10*2+104], m16, 3 + vextracti32x4 [t3+r10*2+120], m17, 3 + add r10, 64 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+r10+416*0], m1 + paddw m1, m0 + mova [t1+r10+416*2], m16 + paddd m16, m2 + mova [t1+r10+416*4], m17 + paddd m17, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab + lea r10, [wq-4] +.v_loop: + mova m2, [t1+r10+416*2] + mova m3, [t1+r10+416*4] + mova m0, [t1+r10+416*0] + paddd m16, m2, [t2+r10+416*2] + paddd m17, m3, [t2+r10+416*4] + paddw m1, m0, [t2+r10+416*0] + paddd m2, m2 + paddd m3, m3 + paddd m16, m2 ; hv sumsq + paddd m17, m3 + paddd m16, m8 + paddd m17, m8 + psrld m16, 4 ; (a + 8) >> 4 + psrld m17, 4 + pmulld m16, m9 ; -a * 25 + pmulld m17, m9 + paddw m0, m0 + paddw m1, m0 ; hv sum + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + vpdpwssd m16, m2, m2 ; -p + punpckhwd m3, m6 + vpdpwssd m17, m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmulld m16, m10 ; p * s + pmulld m17, m10 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + pmaxsw m17, m6 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + packssdw m16, m17 + psubd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + psubd m1, m13 + mova [t4+r10+4], m16 + psrld m16, m0, 12 ; b + psrld m17, m1, 12 + mova [t3+r10*2+ 8], xm16 + mova [t3+r10*2+ 24], xm17 + vextracti128 [t3+r10*2+ 40], ym16, 1 + vextracti128 [t3+r10*2+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+ 72], m16, 2 + vextracti32x4 [t3+r10*2+ 88], m17, 2 + vextracti32x4 [t3+r10*2+104], m16, 3 + vextracti32x4 [t3+r10*2+120], m17, 3 + add r10, 64 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t4+r10*1+ 2] + movu m1, [t3+r10*2+ 4] + movu m2, [t3+r10*2+68] + paddw m3, m0, [t4+r10*1+ 0] + paddd m16, m1, [t3+r10*2+ 0] + paddd m17, m2, [t3+r10*2+64] + paddw m3, [t4+r10*1+ 4] + paddd m16, [t3+r10*2+ 8] + paddd m17, [t3+r10*2+72] + paddw m0, m3 + psllw m3, 2 + paddd m1, m16 + pslld m16, 2 + paddd m2, m17 + pslld m17, 2 + paddw m0, m3 ; a 565 + paddd m1, m16 ; b 565 + paddd m2, m17 + mova [t4+r10*1+416*2+ 0], m0 + mova [t3+r10*2+416*4+ 0], m1 + mova [t3+r10*2+416*4+64], m2 + add r10, 64 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m0, [t4+r10*1+ 2] + movu m1, [t3+r10*2+ 4] + movu m2, [t3+r10*2+68] + paddw m3, m0, [t4+r10*1+ 0] + paddd m16, m1, [t3+r10*2+ 0] + paddd m17, m2, [t3+r10*2+64] + paddw m3, [t4+r10*1+ 4] + paddd m16, [t3+r10*2+ 8] + paddd m17, [t3+r10*2+72] + paddw m0, m3 + psllw m3, 2 + paddd m1, m16 + pslld m16, 2 + paddd m2, m17 + pslld m17, 2 + paddw m0, m3 ; a 565 + paddd m1, m16 ; b 565 + paddd m2, m17 + paddw m3, m0, [t4+r10*1+416*2+ 0] + paddd m16, m1, [t3+r10*2+416*4+ 0] + paddd m17, m2, [t3+r10*2+416*4+64] + mova [t4+r10*1+416*2+ 0], m0 + mova [t3+r10*2+416*4+ 0], m1 + mova [t3+r10*2+416*4+64], m2 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vshufi32x4 m1, m16, m17, q2020 + vshufi32x4 m16, m17, q3131 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m16, m3 + psrad m1, 9 + psrad m16, 9 + packssdw m1, m16 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 64 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m0, [dstq+r10] + mova m3, [t4+r10*1+416*2+ 0] + mova m16, [t3+r10*2+416*4+ 0] + mova m17, [t3+r10*2+416*4+64] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vshufi32x4 m1, m16, m17, q2020 + vshufi32x4 m16, m17, q3131 + psubd m1, m2 ; b - a * src + (1 << 7) + psubd m16, m3 + psrad m1, 8 + psrad m16, 8 + packssdw m1, m16 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 64 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_3x3_16bpc, 4, 14, 22, 416*42+8, dst, stride, left, lpf, \ + w, h, edge, params + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [r_ext_mask+72] + mov edged, r7m + movifnidn hd, hm + pxor m6, m6 + vpbroadcastw m7, [paramsq+10] ; w1 + add wd, wd + vpbroadcastd m8, [base+pd_8] + add lpfq, wq + vpbroadcastd m9, [base+pd_m9] + add dstq, wq + vpsubd m10, m6, [paramsq+4] {1to16} ; -s1 + lea t3, [rsp+wq*2+416*12+8] + vpbroadcastd m11, [base+pw_164_455] + lea t4, [rsp+wq+416*32+8] + vpbroadcastd m12, [base+pw_61448] + lea t1, [rsp+wq+12] + vpbroadcastd m13, [base+pd_m34816] + neg wq + vpbroadcastd m14, [base+pw_1023] + psllw m7, 4 + mova m18, [sgr_x_by_x+64*0] + mov r10d, 0xfffffffc + mova m19, [sgr_x_by_x+64*1] + kmovd k1, r10d + mova m20, [sgr_x_by_x+64*2] + mov r10, 0x3333333333333333 + mova m21, [sgr_x_by_x+64*3] + kmovq k2, r10 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + add t1, 416*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea r10, [wq-4] + lea t2, [t1+416*6] +.top_fixup_loop: + mova m0, [t1+r10+416*0] + mova m1, [t1+r10+416*2] + mova m2, [t1+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m1 + mova [t2+r10+416*4], m2 + add r10, 64 + jl .top_fixup_loop + call .v0 + jmp .main +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm16, [leftq+4] + vmovdqu16 m16{k1}, [lpfq+wq-4] + add leftq, 8 + jmp .h_main +.h_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-4] + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m16, [lpfq+r10+ 0] +.h_main: + movu m17, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -66 + jl .h_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.h_have_right: + palignr m0, m17, m16, 2 + paddw m1, m16, m0 + punpcklwd m2, m16, m0 + pmaddwd m2, m2 + punpckhwd m3, m16, m0 + pmaddwd m3, m3 + palignr m17, m16, 4 + paddw m1, m17 ; sum + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; sumsq + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + mova [t1+r10+416*0], m1 + mova [t1+r10+416*2], m2 + mova [t1+r10+416*4], m3 + add r10, 64 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movd xm16, [leftq+4] + vmovdqu16 m16{k1}, [lpfq+wq-4] + add leftq, 8 + jmp .hv0_main +.hv0_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-4] + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu m16, [lpfq+r10+ 0] +.hv0_main: + movu m17, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -66 + jl .hv0_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.hv0_have_right: + palignr m0, m17, m16, 2 + paddw m1, m16, m0 + punpcklwd m2, m16, m0 + pmaddwd m2, m2 + punpckhwd m3, m16, m0 + pmaddwd m3, m3 + palignr m17, m16, 4 + paddw m1, m17 ; sum + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; sumsq + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + paddw m0, m1, [t1+r10+416*0] + paddd m16, m2, [t1+r10+416*2] + paddd m17, m3, [t1+r10+416*4] + mova [t1+r10+416*0], m1 + mova [t1+r10+416*2], m2 + mova [t1+r10+416*4], m3 + paddw m1, m0, [t2+r10+416*0] + paddd m2, m16, [t2+r10+416*2] + paddd m3, m17, [t2+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m16 + mova [t2+r10+416*4], m17 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pmulld m2, m9 ; -((a + 8) >> 4) * 9 + pmulld m3, m9 + psrlw m17, m1, 1 + pavgw m17, m6 ; (b + 2) >> 2 + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; -p + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + punpcklwd m16, m6, m1 ; b + punpckhwd m17, m6, m1 + pminsd m2, m6 + pminsd m3, m6 + pmulld m2, m10 ; p * s + pmulld m3, m10 + pmaddwd m16, m11 ; b * 455 + pmaddwd m17, m11 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + paddusw m3, m12 + psraw m3, 4 ; min(z, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x + pandn m2, m13, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + psubd m16, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + psubd m17, m13 + mova [t4+r10*1+416*0+4], m2 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*0+ 8], xm16 + mova [t3+r10*2+416*0+ 24], xm17 + vextracti128 [t3+r10*2+416*0+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*0+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*0+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*0+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*0+104], m16, 3 + vextracti32x4 [t3+r10*2+416*0+120], m17, 3 + add r10, 64 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movd xm16, [leftq+4] + vmovdqu16 m16{k1}, [lpfq+wq-4] + add leftq, 8 + jmp .hv1_main +.hv1_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-4] + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu m16, [lpfq+r10+ 0] +.hv1_main: + movu m17, [lpfq+r10+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -66 + jl .hv1_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.hv1_have_right: + palignr m1, m17, m16, 2 + paddw m0, m16, m1 + punpcklwd m2, m16, m1 + pmaddwd m2, m2 + punpckhwd m3, m16, m1 + pmaddwd m3, m3 + palignr m17, m16, 4 + paddw m0, m17 ; h sum + punpcklwd m1, m17, m6 + vpdpwssd m2, m1, m1 ; h sumsq + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + paddw m1, m0, [t2+r10+416*0] + paddd m16, m2, [t2+r10+416*2] + paddd m17, m3, [t2+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m2 + mova [t2+r10+416*4], m3 + paddd m16, m8 + paddd m17, m8 + psrld m16, 4 ; (a + 8) >> 4 + psrld m17, 4 + pmulld m16, m9 ; -((a + 8) >> 4) * 9 + pmulld m17, m9 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + vpdpwssd m16, m2, m2 ; -p + punpckhwd m3, m6 + vpdpwssd m17, m3, m3 + punpcklwd m0, m6, m1 ; b + punpckhwd m1, m6, m1 + pminsd m16, m6 + pminsd m17, m6 + pmulld m16, m10 ; p * s + pmulld m17, m10 + pmaddwd m0, m11 ; b * 455 + pmaddwd m1, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + packssdw m16, m17 + psubd m0, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + psubd m1, m13 + mova [t4+r10*1+416*2+4], m16 + psrld m16, m0, 12 + psrld m17, m1, 12 + mova [t3+r10*2+416*4+ 8], xm16 + mova [t3+r10*2+416*4+ 24], xm17 + vextracti128 [t3+r10*2+416*4+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*4+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*4+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*4+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*4+104], m16, 3 + vextracti32x4 [t3+r10*2+416*4+120], m17, 3 + add r10, 64 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) + lea r10, [wq-4] +.v0_loop: + mova m0, [t1+r10+416*0] + mova m16, [t1+r10+416*2] + mova m17, [t1+r10+416*4] + paddw m0, m0 + paddd m16, m16 + paddd m17, m17 + paddw m1, m0, [t2+r10+416*0] + paddd m2, m16, [t2+r10+416*2] + paddd m3, m17, [t2+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m16 + mova [t2+r10+416*4], m17 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pmulld m2, m9 ; -((a + 8) >> 4) * 9 + pmulld m3, m9 + psrlw m17, m1, 1 + pavgw m17, m6 ; (b + 2) >> 2 + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; -p + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + punpcklwd m16, m6, m1 ; b + punpckhwd m17, m6, m1 + pminsd m2, m6 + pminsd m3, m6 + pmulld m2, m10 ; p * s + pmulld m3, m10 + pmaddwd m16, m11 ; b * 455 + pmaddwd m17, m11 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + paddusw m3, m12 + psraw m3, 4 ; min(z, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x + pandn m2, m13, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + psubd m16, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + psubd m17, m13 + mova [t4+r10*1+416*0+4], m2 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*0+ 8], xm16 + mova [t3+r10*2+416*0+ 24], xm17 + vextracti128 [t3+r10*2+416*0+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*0+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*0+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*0+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*0+104], m16, 3 + vextracti32x4 [t3+r10*2+416*0+120], m17, 3 + add r10, 64 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-4] +.v1_loop: + mova m0, [t1+r10+416*0] + mova m16, [t1+r10+416*2] + mova m17, [t1+r10+416*4] + paddw m1, m0, [t2+r10+416*0] + paddd m2, m16, [t2+r10+416*2] + paddd m3, m17, [t2+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m16 + mova [t2+r10+416*4], m17 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pmulld m2, m9 ; -((a + 8) >> 4) * 9 + pmulld m3, m9 + psrlw m17, m1, 1 + pavgw m17, m6 ; (b + 2) >> 2 + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; -p + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + punpcklwd m16, m6, m1 ; b + punpckhwd m17, m6, m1 + pminsd m2, m6 + pminsd m3, m6 + pmulld m2, m10 ; p * s + pmulld m3, m10 + pmaddwd m16, m11 ; b * 455 + pmaddwd m17, m11 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + paddusw m3, m12 + psraw m3, 4 ; min(z, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x + pandn m2, m13, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + psubd m16, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + psubd m17, m13 + mova [t4+r10*1+416*2+4], m2 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*4+ 8], xm16 + mova [t3+r10*2+416*4+ 24], xm17 + vextracti128 [t3+r10*2+416*4+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*4+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*4+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*4+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*4+104], m16, 3 + vextracti32x4 [t3+r10*2+416*4+120], m17, 3 + add r10, 64 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + mova ym16, [t4+r10*1+416*0+0] + paddw ym16, [t4+r10*1+416*0+4] + paddw ym17, ym16, [t4+r10*1+416*0+2] + mova m0, [t3+r10*2+416*0+0] + paddd m0, [t3+r10*2+416*0+8] + paddd m1, m0, [t3+r10*2+416*0+4] + psllw ym17, 2 ; a[-1] 444 + pslld m1, 2 ; b[-1] 444 + psubw ym17, ym16 ; a[-1] 343 + psubd m1, m0 ; b[-1] 343 + vmovdqa32 [t4+r10*1+416* 4], ym17 + vmovdqa32 [t3+r10*2+416* 8], m1 + mova ym16, [t4+r10*1+416*2+0] + paddw ym16, [t4+r10*1+416*2+4] + paddw ym17, ym16, [t4+r10*1+416*2+2] + mova m0, [t3+r10*2+416*4+0] + paddd m0, [t3+r10*2+416*4+8] + paddd m1, m0, [t3+r10*2+416*4+4] + psllw ym17, 2 ; a[ 0] 444 + pslld m1, 2 ; b[ 0] 444 + vmovdqa32 [t4+r10*1+416* 6], ym17 + vmovdqa32 [t3+r10*2+416*12], m1 + psubw ym17, ym16 ; a[ 0] 343 + psubd m1, m0 ; b[ 0] 343 + vmovdqa32 [t4+r10*1+416* 8], ym17 + vmovdqa32 [t3+r10*2+416*16], m1 + add r10, 32 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + mova m3, [t4+r10*1+416*0+0] + paddw m3, [t4+r10*1+416*0+4] + paddw m1, m3, [t4+r10*1+416*0+2] + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+r10*1+416*4] + paddw m3, [t4+r10*1+416*6] + mova [t4+r10*1+416*4], m2 + mova [t4+r10*1+416*6], m1 + mova m16, [t3+r10*2+416*0+0] + paddd m16, [t3+r10*2+416*0+8] + paddd m1, m16, [t3+r10*2+416*0+4] + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m16 ; b[ 1] 343 + paddd m16, m2, [t3+r10*2+416* 8+ 0] + paddd m16, [t3+r10*2+416*12+ 0] + mova [t3+r10*2+416* 8+ 0], m2 + mova [t3+r10*2+416*12+ 0], m1 + mova m17, [t3+r10*2+416*0+64] + paddd m17, [t3+r10*2+416*0+72] + paddd m1, m17, [t3+r10*2+416*0+68] + pslld m1, 2 + psubd m2, m1, m17 + paddd m17, m2, [t3+r10*2+416* 8+64] + paddd m17, [t3+r10*2+416*12+64] + mova [t3+r10*2+416* 8+64], m2 + mova [t3+r10*2+416*12+64], m1 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vshufi32x4 m1, m16, m17, q2020 + vshufi32x4 m16, m17, q3131 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m16, m3 + psrad m1, 9 + psrad m16, 9 + packssdw m1, m16 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 64 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m3, [t4+r10*1+416*2+0] + paddw m3, [t4+r10*1+416*2+4] + paddw m1, m3, [t4+r10*1+416*2+2] + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+r10*1+416*6] + paddw m3, [t4+r10*1+416*8] + mova [t4+r10*1+416*6], m1 + mova [t4+r10*1+416*8], m2 + mova m16, [t3+r10*2+416*4+0] + paddd m16, [t3+r10*2+416*4+8] + paddd m1, m16, [t3+r10*2+416*4+4] + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m16 ; b[ 1] 343 + paddd m16, m2, [t3+r10*2+416*12+ 0] + paddd m16, [t3+r10*2+416*16+ 0] + mova [t3+r10*2+416*12+ 0], m1 + mova [t3+r10*2+416*16+ 0], m2 + mova m17, [t3+r10*2+416*4+64] + paddd m17, [t3+r10*2+416*4+72] + paddd m1, m17, [t3+r10*2+416*4+68] + pslld m1, 2 + psubd m2, m1, m17 + paddd m17, m2, [t3+r10*2+416*12+64] + paddd m17, [t3+r10*2+416*16+64] + mova [t3+r10*2+416*12+64], m1 + mova [t3+r10*2+416*16+64], m2 + mova m0, [dstq+r10] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + vshufi32x4 m1, m16, m17, q2020 + vshufi32x4 m16, m17, q3131 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m16, m3 + psrad m1, 9 + psrad m16, 9 + packssdw m1, m16 + pmulhrsw m1, m7 + paddw m0, m1 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+r10], m0 + add r10, 64 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_mix_16bpc, 4, 14, 23, 416*66+8, dst, stride, left, lpf, \ + w, h, edge, params + movifnidn wd, wm + mov paramsq, r6mp + lea r13, [r_ext_mask+72] + mov edged, r7m + movifnidn hd, hm + vpbroadcastd m7, [paramsq+8] ; w0 w1 + pxor m6, m6 + vpbroadcastd m8, [base+pd_8] + add wd, wd + vpbroadcastd m9, [base+pd_m9] + add lpfq, wq + vpbroadcastd m10, [base+pd_m25] + add dstq, wq + vpsubd m11, m6, [paramsq+0] {1to16} ; -s0 + lea t3, [rsp+wq*2+416*24+8] + vpsubd m12, m6, [paramsq+4] {1to16} ; -s1 + lea t4, [rsp+wq+416*52+8] + vpbroadcastd m13, [base+pw_164_455] + lea t1, [rsp+wq+12] + vpbroadcastd m14, [base+pw_61448] + neg wq + vpbroadcastd m15, [base+pd_m34816] + psllw m7, 2 + vpbroadcastd m22, [base+pd_2147483648] + mov r10d, 0xfffffff8 + mova m18, [sgr_x_by_x+64*0] + kmovd k1, r10d + mova m19, [sgr_x_by_x+64*1] + mov r10, 0x3333333333333333 + mova m20, [sgr_x_by_x+64*2] + kmovq k2, r10 + mova m21, [sgr_x_by_x+64*3] + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_avx512icl).top_fixup + add t1, 416*12 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea r10, [wq-4] + lea t2, [t1+416*12] +.top_fixup_loop: + mova m0, [t1+r10+416* 0] + mova m1, [t1+r10+416* 2] + mova m2, [t1+r10+416* 4] + paddw m0, m0 + mova m3, [t1+r10+416* 6] + paddd m1, m1 + mova m4, [t1+r10+416* 8] + paddd m2, m2 + mova m5, [t1+r10+416*10] + mova [t2+r10+416* 0], m0 + mova [t2+r10+416* 2], m1 + mova [t2+r10+416* 4], m2 + mova [t2+r10+416* 6], m3 + mova [t2+r10+416* 8], m4 + mova [t2+r10+416*10], m5 + add r10, 64 + jl .top_fixup_loop + call .v0 + jmp .main +.h: ; horizontal boxsum + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movq xm16, [leftq+2] + vmovdqu16 m16{k1}, [lpfq+wq-6] + add leftq, 8 + jmp .h_main +.h_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-6] + jmp .h_main +.h_top: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m16, [lpfq+r10- 2] +.h_main: + movu m17, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -68 + jl .h_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.h_have_right: + palignr m3, m17, m16, 2 + palignr m0, m17, m16, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m17, m16, 6 + paddw m1, m0 ; sum3 + punpcklwd m4, m0, m6 + vpdpwssd m2, m4, m4 ; sumsq3 + punpckhwd m0, m6 + vpdpwssd m3, m0, m0 + shufpd m4, m16, m17, 0x55 + punpcklwd m17, m4, m16 + paddw m0, m16, m4 + punpckhwd m4, m16 + mova [t1+r10+416* 6], m1 + mova [t1+r10+416* 8], m2 + mova [t1+r10+416*10], m3 + paddw m1, m0 ; sum5 + vpdpwssd m2, m17, m17 ; sumsq5 + vpdpwssd m3, m4, m4 + mova [t1+r10+416* 0], m1 + mova [t1+r10+416* 2], m2 + mova [t1+r10+416* 4], m3 + add r10, 64 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movq xm16, [leftq+2] + vmovdqu16 m16{k1}, [lpfq+wq-6] + add leftq, 8 + jmp .hv0_main +.hv0_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-6] + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu m16, [lpfq+r10- 2] +.hv0_main: + movu m17, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -68 + jl .hv0_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.hv0_have_right: + palignr m3, m17, m16, 2 + palignr m0, m17, m16, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m17, m16, 6 + paddw m1, m0 ; h sum3 + punpcklwd m4, m0, m6 + vpdpwssd m2, m4, m4 ; h sumsq3 + punpckhwd m0, m6 + vpdpwssd m3, m0, m0 + shufpd m17, m16, m17, 0x55 + paddw m4, m1, [t1+r10+416* 6] + paddd m5, m2, [t1+r10+416* 8] + mova [t1+r10+416* 6], m1 + mova [t1+r10+416* 8], m2 + paddw m1, m16 + paddw m1, m17 ; h sum5 + punpcklwd m0, m17, m16 + vpdpwssd m2, m0, m0 ; h sumsq5 + paddd m0, m3, [t1+r10+416*10] + mova [t1+r10+416*10], m3 + punpckhwd m17, m16 + vpdpwssd m3, m17, m17 + mova [t3+r10*2+416*8+ 8], m1 ; we need a clean copy of the last row + mova [t3+r10*2+416*0+ 8], m2 ; in case height is odd + mova [t3+r10*2+416*0+72], m3 + paddw m1, [t1+r10+416* 0] + paddd m2, [t1+r10+416* 2] + paddd m3, [t1+r10+416* 4] + mova [t1+r10+416* 0], m1 + mova [t1+r10+416* 2], m2 + mova [t1+r10+416* 4], m3 + paddw m17, m4, [t2+r10+416* 6] + paddd m2, m5, [t2+r10+416* 8] + paddd m3, m0, [t2+r10+416*10] + mova [t2+r10+416* 6], m4 + mova [t2+r10+416* 8], m5 + mova [t2+r10+416*10], m0 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pmulld m2, m9 ; -((a3 + 8) >> 4) * 9 + pmulld m3, m9 + psrlw m5, m17, 1 + pavgw m5, m6 ; (b3 + 2) >> 2 + punpcklwd m4, m5, m6 + vpdpwssd m2, m4, m4 ; -p3 + punpckhwd m5, m6 + vpdpwssd m3, m5, m5 + punpcklwd m16, m6, m17 ; b3 + punpckhwd m17, m6, m17 + pminsd m2, m6 + pminsd m3, m6 + pmulld m2, m12 ; p3 * s1 + pmulld m3, m12 + pmaddwd m16, m13 ; b3 * 455 + pmaddwd m17, m13 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + paddusw m3, m14 + psraw m3, 4 ; min(z3, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x3 + pandn m2, m15, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + mova [t4+r10*1+416*2+4], m2 + psubd m16, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*4+ 8], xm16 + mova [t3+r10*2+416*4+ 24], xm17 + vextracti128 [t3+r10*2+416*4+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*4+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*4+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*4+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*4+104], m16, 3 + vextracti32x4 [t3+r10*2+416*4+120], m17, 3 + add r10, 64 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movq xm16, [leftq+2] + vmovdqu16 m16{k1}, [lpfq+wq-6] + add leftq, 8 + jmp .hv1_main +.hv1_extend_left: + vpbroadcastw xm16, [lpfq+wq] + vmovdqu16 m16{k1}, [lpfq+wq-6] + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-4] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu m16, [lpfq+r10- 2] +.hv1_main: + movu m17, [lpfq+r10+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -68 + jl .hv1_have_right + vpbroadcastw m0, [lpfq-2] + vpternlogd m16, m0, [r13+r10+ 0], 0xe4 + vpternlogd m17, m0, [r13+r10+16], 0xe4 +.hv1_have_right: + palignr m1, m17, m16, 2 + palignr m3, m17, m16, 4 + paddw m2, m1, m3 + punpcklwd m0, m1, m3 + pmaddwd m0, m0 + punpckhwd m1, m3 + pmaddwd m1, m1 + palignr m3, m17, m16, 6 + paddw m2, m3 ; h sum3 + punpcklwd m5, m3, m6 + vpdpwssd m0, m5, m5 ; h sumsq3 + punpckhwd m3, m6 + vpdpwssd m1, m3, m3 + shufpd m3, m16, m17, 0x55 + punpcklwd m5, m16, m3 + paddw m4, m16, m3 + punpckhwd m16, m3 + paddw m17, m2, [t2+r10+416* 6] + mova [t2+r10+416* 6], m2 + paddw m4, m2 ; h sum5 + paddd m2, m0, [t2+r10+416* 8] + paddd m3, m1, [t2+r10+416*10] + mova [t2+r10+416* 8], m0 + mova [t2+r10+416*10], m1 + vpdpwssd m0, m5, m5 ; h sumsq5 + vpdpwssd m1, m16, m16 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pmulld m2, m9 ; -((a3 + 8) >> 4) * 9 + pmulld m3, m9 + psrlw m16, m17, 1 + pavgw m16, m6 ; (b3 + 2) >> 2 + punpcklwd m5, m16, m6 + vpdpwssd m2, m5, m5 ; -p3 + punpckhwd m16, m6 + vpdpwssd m3, m16, m16 + punpcklwd m16, m6, m17 ; b3 + punpckhwd m17, m6, m17 + pminsd m2, m6 + pminsd m3, m6 + pmulld m2, m12 ; p3 * s1 + pmulld m3, m12 + pmaddwd m16, m13 ; b3 * 455 + pmaddwd m17, m13 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + paddusw m3, m14 + psraw m3, 4 ; min(z3, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x3 + pandn m2, m15, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + mova [t4+r10*1+416*4+4], m2 + psubd m16, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + paddw m5, m4, [t2+r10+416*0] + paddd m2, m0, [t2+r10+416*2] + paddd m3, m1, [t2+r10+416*4] + paddw m5, [t1+r10+416*0] + paddd m2, [t1+r10+416*2] + paddd m3, [t1+r10+416*4] + mova [t2+r10+416*0], m4 + mova [t2+r10+416*2], m0 + mova [t2+r10+416*4], m1 + mova [t3+r10*2+416*8+ 8], xm16 + mova [t3+r10*2+416*8+ 24], xm17 + vextracti128 [t3+r10*2+416*8+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*8+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*8+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*8+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*8+104], m16, 3 + vextracti32x4 [t3+r10*2+416*8+120], m17, 3 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + pmulld m2, m10 ; -((a5 + 8) >> 4) * 25 + pmulld m3, m10 + psrlw m17, m5, 1 + pavgw m17, m6 ; (b5 + 2) >> 2 + punpcklwd m16, m17, m6 + vpdpwssd m2, m16, m16 ; -p5 + punpckhwd m17, m6 + vpdpwssd m3, m17, m17 + punpcklwd m16, m5, m6 ; b5 + punpckhwd m17, m5, m6 + pmulld m2, m11 ; p5 * s0 + pmulld m3, m11 + pmaddwd m16, m13 ; b5 * 164 + pmaddwd m17, m13 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + pmaxsw m3, m6 + paddusw m3, m14 + psraw m3, 4 ; min(z5, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x5 + pandn m2, m15, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + mova [t4+r10*1+416*0+4], m2 + psubd m16, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*0+ 8], xm16 + mova [t3+r10*2+416*0+ 24], xm17 + vextracti128 [t3+r10*2+416*0+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*0+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*0+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*0+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*0+104], m16, 3 + vextracti32x4 [t3+r10*2+416*0+120], m17, 3 + add r10, 64 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) + lea r10, [wq-4] +.v0_loop: + mova m16, [t1+r10+416* 6] + mova m2, [t1+r10+416* 8] + mova m3, [t1+r10+416*10] + paddw m16, m16 + paddd m2, m2 + paddd m3, m3 + paddw m17, m16, [t2+r10+416* 6] + paddd m4, m2, [t2+r10+416* 8] + paddd m5, m3, [t2+r10+416*10] + mova [t2+r10+416* 6], m16 + mova [t2+r10+416* 8], m2 + mova [t2+r10+416*10], m3 + paddd m4, m8 + paddd m5, m8 + psrld m4, 4 ; (a3 + 8) >> 4 + psrld m5, 4 + pmulld m4, m9 ; -((a3 + 8) >> 4) * 9 + pmulld m5, m9 + psrlw m3, m17, 1 + pavgw m3, m6 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m6 + vpdpwssd m4, m2, m2 ; -p3 + punpckhwd m3, m6 + vpdpwssd m5, m3, m3 + punpcklwd m16, m6, m17 ; b3 + punpckhwd m17, m6, m17 + pminsd m4, m6 + pminsd m5, m6 + pmulld m4, m12 ; p3 * s1 + pmulld m5, m12 + pmaddwd m16, m13 ; b3 * 455 + pmaddwd m17, m13 + vpalignr m5{k2}, m4, m4, 2 + mova m4, m20 + paddusw m5, m14 + psraw m5, 4 ; min(z3, 255) - 256 + vpermt2b m4, m5, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m5 + vpermi2b m5, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m5{k3}, m4 ; x3 + pandn m4, m15, m5 + psrld m5, 16 + pmulld m16, m4 + pmulld m17, m5 + packssdw m4, m5 + mova [t4+r10*1+416*2+4], m4 + psubd m16, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + mova m3, [t1+r10+416*0] + mova m4, [t1+r10+416*2] + mova m5, [t1+r10+416*4] + mova [t3+r10*2+416*8+ 8], m3 + mova [t3+r10*2+416*0+ 8], m4 + mova [t3+r10*2+416*0+72], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+r10+416*0], m3 + mova [t1+r10+416*2], m4 + mova [t1+r10+416*4], m5 + mova [t3+r10*2+416*4+ 8], xm16 + mova [t3+r10*2+416*4+ 24], xm17 + vextracti128 [t3+r10*2+416*4+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*4+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*4+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*4+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*4+104], m16, 3 + vextracti32x4 [t3+r10*2+416*4+120], m17, 3 + add r10, 64 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-4] +.v1_loop: + mova m16, [t1+r10+416* 6] + mova m2, [t1+r10+416* 8] + mova m3, [t1+r10+416*10] + paddw m17, m16, [t2+r10+416* 6] + paddd m4, m2, [t2+r10+416* 8] + paddd m5, m3, [t2+r10+416*10] + mova [t2+r10+416* 6], m16 + mova [t2+r10+416* 8], m2 + mova [t2+r10+416*10], m3 + paddd m4, m8 + paddd m5, m8 + psrld m4, 4 ; (a3 + 8) >> 4 + psrld m5, 4 + pmulld m4, m9 ; -((a3 + 8) >> 4) * 9 + pmulld m5, m9 + psrlw m3, m17, 1 + pavgw m3, m6 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m6 + vpdpwssd m4, m2, m2 ; -p3 + punpckhwd m3, m6 + vpdpwssd m5, m3, m3 + punpcklwd m16, m6, m17 ; b3 + punpckhwd m17, m6, m17 + pminsd m4, m6 + pminsd m5, m6 + pmulld m4, m12 ; p3 * s1 + pmulld m5, m12 + pmaddwd m16, m13 ; b3 * 455 + pmaddwd m17, m13 + vpalignr m5{k2}, m4, m4, 2 + mova m4, m20 + paddusw m5, m14 + psraw m5, 4 ; min(z3, 255) - 256 + vpermt2b m4, m5, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m5 + vpermi2b m5, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m5{k3}, m4 ; x3 + pandn m4, m15, m5 + psrld m5, 16 + pmulld m16, m4 + pmulld m17, m5 + packssdw m4, m5 + mova [t4+r10*1+416*4+4], m4 + psubd m16, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + mova m0, [t3+r10*2+416*8+ 8] + mova m4, [t3+r10*2+416*0+ 8] + mova m5, [t3+r10*2+416*0+72] + paddw m1, m0, [t2+r10+416*0] + paddd m2, m4, [t2+r10+416*2] + paddd m3, m5, [t2+r10+416*4] + paddw m1, [t1+r10+416*0] + paddd m2, [t1+r10+416*2] + paddd m3, [t1+r10+416*4] + mova [t2+r10+416*0], m0 + mova [t2+r10+416*2], m4 + mova [t2+r10+416*4], m5 + mova [t3+r10*2+416*8+ 8], xm16 + mova [t3+r10*2+416*8+ 24], xm17 + vextracti128 [t3+r10*2+416*8+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*8+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*8+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*8+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*8+104], m16, 3 + vextracti32x4 [t3+r10*2+416*8+120], m17, 3 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + pmulld m2, m10 ; -((a5 + 8) >> 4) * 25 + pmulld m3, m10 + psrlw m5, m1, 1 + pavgw m5, m6 ; (b5 + 2) >> 2 + punpcklwd m4, m5, m6 + vpdpwssd m2, m4, m4 ; -p5 + punpckhwd m5, m6 + vpdpwssd m3, m5, m5 + punpcklwd m16, m1, m6 ; b5 + punpckhwd m17, m1, m6 + pmulld m2, m11 ; p5 * s0 + pmulld m3, m11 + pmaddwd m16, m13 ; b5 * 164 + pmaddwd m17, m13 + vpalignr m3{k2}, m2, m2, 2 + mova m2, m20 + pmaxsw m3, m6 + paddusw m3, m14 + psraw m3, 4 ; min(z5, 255) - 256 + vpermt2b m2, m3, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m3 + vpermi2b m3, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m3{k3}, m2 ; x5 + pandn m2, m15, m3 + psrld m3, 16 + pmulld m16, m2 + pmulld m17, m3 + packssdw m2, m3 + mova [t4+r10*1+416*0+4], m2 + psubd m16, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + psubd m17, m15 + psrld m16, 12 + psrld m17, 12 + mova [t3+r10*2+416*0+ 8], xm16 + mova [t3+r10*2+416*0+ 24], xm17 + vextracti128 [t3+r10*2+416*0+ 40], ym16, 1 + vextracti128 [t3+r10*2+416*0+ 56], ym17, 1 + vextracti32x4 [t3+r10*2+416*0+ 72], m16, 2 + vextracti32x4 [t3+r10*2+416*0+ 88], m17, 2 + vextracti32x4 [t3+r10*2+416*0+104], m16, 3 + vextracti32x4 [t3+r10*2+416*0+120], m17, 3 + add r10, 64 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu ym0, [t4+r10*1+416*0+2] + paddw ym2, ym0, [t4+r10*1+416*0+0] + paddw ym2, [t4+r10*1+416*0+4] + movu m1, [t3+r10*2+416*0+4] + paddd m3, m1, [t3+r10*2+416*0+0] + paddd m3, [t3+r10*2+416*0+8] + paddw ym0, ym2 + paddd m1, m3 + psllw ym2, 2 + pslld m3, 2 + paddw ym0, ym2 ; a5 565 + paddd m1, m3 ; b5 565 + mova [t4+r10*1+416* 6], ym0 + mova [t3+r10*2+416*12], m1 + mova ym0, [t4+r10*1+416*2+0] + paddw ym0, [t4+r10*1+416*2+4] + paddw ym2, ym0, [t4+r10*1+416*2+2] + mova m1, [t3+r10*2+416*4+0] + paddd m1, [t3+r10*2+416*4+8] + paddd m3, m1, [t3+r10*2+416*4+4] + psllw ym2, 2 ; a3[-1] 444 + pslld m3, 2 ; b3[-1] 444 + psubw ym2, ym0 ; a3[-1] 343 + psubd m3, m1 ; b3[-1] 343 + mova [t4+r10*1+416* 8], ym2 + mova [t3+r10*2+416*16], m3 + mova ym0, [t4+r10*1+416*4+0] + paddw ym0, [t4+r10*1+416*4+4] + paddw ym2, ym0, [t4+r10*1+416*4+2] + mova m1, [t3+r10*2+416*8+0] + paddd m1, [t3+r10*2+416*8+8] + paddd m3, m1, [t3+r10*2+416*8+4] + psllw ym2, 2 ; a3[ 0] 444 + pslld m3, 2 ; b3[ 0] 444 + mova [t4+r10*1+416*10], ym2 + mova [t3+r10*2+416*20], m3 + psubw ym2, ym0 ; a3[ 0] 343 + psubd m3, m1 ; b3[ 0] 343 + mova [t4+r10*1+416*12], ym2 + mova [t3+r10*2+416*24], m3 + add r10, 32 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu ym2, [t4+r10*1+2] + paddw ym0, ym2, [t4+r10*1+0] + paddw ym0, [t4+r10*1+4] + paddw ym2, ym0 + psllw ym0, 2 + paddw ym0, ym2 ; a5 + movu m1, [t3+r10*2+4] + paddd m4, m1, [t3+r10*2+0] + paddd m4, [t3+r10*2+8] + paddd m1, m4 + pslld m4, 2 + paddd m4, m1 ; b5 + paddw ym2, ym0, [t4+r10*1+416* 6] + mova [t4+r10*1+416* 6], ym0 + paddd m0, m4, [t3+r10*2+416*12] + mova [t3+r10*2+416*12], m4 + mova ym3, [t4+r10*1+416*2+0] + paddw ym3, [t4+r10*1+416*2+4] + paddw ym5, ym3, [t4+r10*1+416*2+2] + psllw ym5, 2 ; a3[ 1] 444 + psubw ym4, ym5, ym3 ; a3[ 1] 343 + paddw ym3, ym4, [t4+r10*1+416* 8] + paddw ym3, [t4+r10*1+416*10] + mova [t4+r10*1+416* 8], ym4 + mova [t4+r10*1+416*10], ym5 + mova m1, [t3+r10*2+416*4+0] + paddd m1, [t3+r10*2+416*4+8] + paddd m5, m1, [t3+r10*2+416*4+4] + pslld m5, 2 ; b3[ 1] 444 + psubd m4, m5, m1 ; b3[ 1] 343 + paddd m1, m4, [t3+r10*2+416*16] + paddd m1, [t3+r10*2+416*20] + mova [t3+r10*2+416*16], m4 + mova [t3+r10*2+416*20], m5 + pmovzxwd m4, [dstq+r10] + pmovzxwd m2, ym2 ; a5 + pmovzxwd m3, ym3 ; a3 + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + vpshldd m4, m22, 13 + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) + psrld m0, 9 + pslld m1, 7 + vpblendmb m0{k2}, m1, m0 + vpdpwssd m4, m0, m7 + psrad m4, 7 + pmaxsd m4, m6 + vpmovusdw ym16, m4 ; clip + psrlw ym16, 6 + mova [dstq+r10], ym16 + add r10, 32 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova ym3, [t4+r10*1+416*4+0] + paddw ym3, [t4+r10*1+416*4+4] + paddw ym5, ym3, [t4+r10*1+416*4+2] + psllw ym5, 2 ; a3[ 1] 444 + psubw ym4, ym5, ym3 ; a3[ 1] 343 + paddw ym3, ym4, [t4+r10*1+416*12] + paddw ym3, [t4+r10*1+416*10] + mova [t4+r10*1+416*10], ym5 + mova [t4+r10*1+416*12], ym4 + mova m0, [t3+r10*2+416*8+0] + paddd m0, [t3+r10*2+416*8+8] + paddd m5, m0, [t3+r10*2+416*8+4] + pslld m5, 2 ; b3[ 1] 444 + psubd m4, m5, m0 ; b3[ 1] 343 + paddd m0, m4, [t3+r10*2+416*24] + paddd m0, [t3+r10*2+416*20] + mova [t3+r10*2+416*20], m5 + mova [t3+r10*2+416*24], m4 + pmovzxwd m4, [dstq+r10] + pmovzxwd m2, [t4+r10*1+416* 6] + pmovzxwd m3, ym3 + mova m1, [t3+r10*2+416*12] + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + vpshldd m4, m22, 13 + psubd m1, m2 ; b5 - a5 * src + (1 << 8) + psubd m0, m3 ; b3 - a3 * src + (1 << 8) + pslld m0, 7 + vpalignr m0{k2}, m1, m1, 1 + vpdpwssd m4, m0, m7 + psrad m4, 7 + pmaxsd m4, m6 + vpmovusdw ym16, m4 ; clip + psrlw ym16, 6 + mova [dstq+r10], ym16 + add r10, 32 + jl .n1_loop + add dstq, strideq + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/looprestoration16_sse.asm b/third_party/dav1d/src/x86/looprestoration16_sse.asm new file mode 100644 index 0000000000..872e502982 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration16_sse.asm @@ -0,0 +1,3723 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +wiener_shufA: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 +wiener_shufB: db 6, 7, 4, 5, 8, 9, 6, 7, 10, 11, 8, 9, 12, 13, 10, 11 +wiener_shufC: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 +wiener_shufD: db 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 +wiener_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +wiener_lshuf5: db 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +wiener_lshuf7: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 +sgr_lshuf3: db 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +sgr_lshuf5: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +pb_0to15: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +pb_m14_m13: times 8 db -14,-13 +pb_m10_m9: times 8 db -10, -9 +pb_m6_m5: times 8 db -6, -5 +pb_m2_m1: times 8 db -2, -1 +pb_2_3: times 8 db 2, 3 +pb_6_7: times 8 db 6, 7 +pw_256: times 8 dw 256 +pw_1023: times 8 dw 1023 +pd_8: times 4 dd 8 +pd_4096: times 4 dd 4096 +pd_34816: times 4 dd 34816 +pd_m262128: times 4 dd -262128 +pd_0xffff: times 4 dd 0xffff +pd_0xf00800a4: times 4 dd 0xf00800a4 +pd_0xf00801c7: times 4 dd 0xf00801c7 +pd_0xfffffff0: times 4 dd 0xfffffff0 + +wiener_shifts: dw 4, 4, 2048, 2048, 1, 1, 8192, 8192 +wiener_round: dd 1049600, 1048832 + +cextern sgr_x_by_x + +SECTION .text + +%macro movif64 2 ; dst, src + %if ARCH_X86_64 + mov %1, %2 + %endif +%endmacro + +%macro movif32 2 ; dst, src + %if ARCH_X86_32 + mov %1, %2 + %endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_32 +DECLARE_REG_TMP 5, 6 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 13*16 + %else + %assign extra_stack 12*16 + %endif +cglobal wiener_filter7_16bpc, 4, 7, 8, -384*12-16-extra_stack, \ + dst, stride, left, lpf, w, flt + %if STACK_ALIGNMENT < 16 + %define lpfm dword [esp+calloff+16*12+ 0] + %define wm dword [esp+calloff+16*12+ 4] + %define hd dword [esp+calloff+16*12+ 8] + %define edgeb byte [esp+calloff+16*12+12] + %define edged dword [esp+calloff+16*12+12] + %else + %define hd dword r5m + %define edgeb byte r7m + %endif + %define PICmem dword [esp+calloff+4*0] + %define t0m dword [esp+calloff+4*1] ; wiener ring buffer pointers + %define t1m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define t5m dword [esp+calloff+4*6] + %define t6m dword [esp+calloff+4*7] + %define t2 t2m + %define t3 t3m + %define t4 t4m + %define t5 t5m + %define t6 t6m + %define m8 [esp+calloff+16*2] + %define m9 [esp+calloff+16*3] + %define m10 [esp+calloff+16*4] + %define m11 [esp+calloff+16*5] + %define m12 [esp+calloff+16*6] + %define m13 [esp+calloff+16*7] + %define m14 [esp+calloff+16*8] + %define m15 [esp+calloff+16*9] + %define r10 r4 + %define base t0-wiener_shifts + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov wd, [rstk+stack_offset+20] + mov wm, wd + mov r5, [rstk+stack_offset+24] + mov hd, r5 + mov r5, [rstk+stack_offset+32] + mov edged, r5 ; edge + %endif +%else +DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; wiener ring buffer pointers +cglobal wiener_filter7_16bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt + %define base +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov fltq, r6mp + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + movq m13, [fltq] + movq m15, [fltq+16] +%else + %if STACK_ALIGNMENT < 16 + mov t0, [rstk+stack_offset+28] + mov t1, [rstk+stack_offset+36] ; pixel_max + movq m1, [t0] ; fx + movq m3, [t0+16] ; fy + LEA t0, wiener_shifts + %else + mov fltq, r6m + movq m1, [fltq] + movq m3, [fltq+16] + LEA t0, wiener_shifts + mov t1, r8m ; pixel_max + %endif + mov PICmem, t0 +%endif + mova m6, [base+wiener_shufA] + mova m7, [base+wiener_shufB] +%if ARCH_X86_64 + lea t4, [wiener_shifts] + add wd, wd + pshufd m12, m13, q0000 ; x0 x1 + pshufd m13, m13, q1111 ; x2 x3 + pshufd m14, m15, q0000 ; y0 y1 + pshufd m15, m15, q1111 ; y2 y3 + mova m8, [wiener_shufC] + mova m9, [wiener_shufD] + add lpfq, wq + lea t1, [rsp+wq+16] + add dstq, wq + neg wq + shr t3d, 11 + %define base t4-wiener_shifts + movd m10, [base+wiener_round+t3*4] + movq m11, [base+wiener_shifts+t3*8] + pshufd m10, m10, q0000 + pshufd m0, m11, q0000 + pshufd m11, m11, q1111 + pmullw m12, m0 ; upshift filter coefs to make the + pmullw m13, m0 ; horizontal downshift constant + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] + %define base + %define wiener_lshuf7_mem [wiener_lshuf7] + %define pd_m262128_mem [pd_m262128] +%else + add wd, wd + mova m4, [base+wiener_shufC] + mova m5, [base+wiener_shufD] + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + pshufd m2, m3, q0000 + pshufd m3, m3, q1111 + mova m8, m4 + mova m9, m5 + mova m14, m2 + mova m15, m3 + shr t1, 11 + add lpfq, wq + mova m3, [base+pd_m262128] + movd m4, [base+wiener_round+t1*4] + movq m5, [base+wiener_shifts+t1*8] + lea t1, [esp+extra_stack+wq+16] + add dstq, wq + neg wq + pshufd m4, m4, q0000 + pshufd m2, m5, q0000 + pshufd m5, m5, q1111 + mov wm, wq + pmullw m0, m2 + pmullw m1, m2 + mova m2, [base+wiener_lshuf7] + %define pd_m262128_mem [esp+calloff+16*10] + mova pd_m262128_mem, m3 + mova m10, m4 + mova m11, m5 + mova m12, m0 + mova m13, m1 + %define wiener_lshuf7_mem [esp+calloff+16*11] + mova wiener_lshuf7_mem, m2 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 384*2 + add r10, strideq + mov lpfm, r10 ; below + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, lpfm + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call .v + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call .v + movif32 wq, wm +.v2: + call .v + movif32 wq, wm + jmp .v1 +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movif32 t0, PICmem + pxor m0, m0 + movd m1, wd + mova m2, [base+pb_0to15] + pshufb m1, m0 + mova m0, [base+pb_6_7] + psubb m0, m1 + pminub m0, m2 + pshufb m3, m0 + mova m0, [base+pb_m2_m1] + psubb m0, m1 + pminub m0, m2 + pshufb m4, m0 + mova m0, [base+pb_m10_m9] + psubb m0, m1 + pminub m0, m2 + pshufb m5, m0 + movif32 t0, t0m + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: + movif64 wq, r4 + movif32 wq, wm + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movq m3, [leftq] + movhps m3, [lpfq+wq] + add leftq, 8 + jmp .h_main +.h_extend_left: + mova m3, [lpfq+wq] ; avoid accessing memory located + pshufb m3, wiener_lshuf7_mem ; before the start of the buffer + jmp .h_main +.h_top: + movif64 wq, r4 + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+wq-8] +.h_main: + mova m4, [lpfq+wq+0] + movu m5, [lpfq+wq+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -20 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m0, m3, m6 + pshufb m1, m4, m7 + paddw m0, m1 + pshufb m3, m8 + pmaddwd m0, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + pmaddwd m3, m13 + pshufb m2, m5, m7 + paddw m1, m2 + mova m2, pd_m262128_mem ; (1 << 4) - (1 << 18) + pshufb m4, m8 + pmaddwd m1, m12 + pshufb m5, m9 + paddw m4, m5 + pmaddwd m4, m13 + paddd m0, m2 + paddd m1, m2 + paddd m0, m3 + paddd m1, m4 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+wq], m0 + add wq, 16 + jl .h_loop + movif32 wq, wm + ret +ALIGN function_align +.hv: + add lpfq, strideq + movif64 wq, r4 + movif32 t0m, t0 + movif32 t1m, t1 + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movq m3, [leftq] + movhps m3, [lpfq+wq] + add leftq, 8 + jmp .hv_main +.hv_extend_left: + mova m3, [lpfq+wq] + pshufb m3, wiener_lshuf7_mem + jmp .hv_main +.hv_bottom: + movif64 wq, r4 + movif32 t0m, t0 + movif32 t1m, t1 + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+wq-8] +.hv_main: + mova m4, [lpfq+wq+0] + movu m5, [lpfq+wq+8] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -20 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t1, t4m + movif32 t0, t2m + pshufb m0, m3, m6 + pshufb m1, m4, m7 + paddw m0, m1 + pshufb m3, m8 + pmaddwd m0, m12 + pshufb m1, m4, m9 + paddw m3, m1 + pshufb m1, m4, m6 + pmaddwd m3, m13 + pshufb m2, m5, m7 + paddw m1, m2 + mova m2, pd_m262128_mem + pshufb m4, m8 + pmaddwd m1, m12 + pshufb m5, m9 + paddw m4, m5 + pmaddwd m4, m13 + paddd m0, m2 + paddd m1, m2 +%if ARCH_X86_64 + mova m2, [t4+wq] + paddw m2, [t2+wq] + mova m5, [t3+wq] +%else + mova m2, [t1+wq] + paddw m2, [t0+wq] + mov t1, t3m + mov t0, t5m + mova m5, [t1+wq] + mov t1, t1m +%endif + paddd m0, m3 + paddd m1, m4 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 +%if ARCH_X86_64 + mova m4, [t5+wq] + paddw m4, [t1+wq] + psraw m0, 1 + paddw m3, m0, [t6+wq] +%else + mova m4, [t0+wq] + paddw m4, [t1+wq] + mov t0, t0m + mov t1, t6m + psraw m0, 1 + paddw m3, m0, [t1+wq] +%endif + mova [t0+wq], m0 + punpcklwd m0, m2, m5 + pmaddwd m0, m15 + punpckhwd m2, m5 + pmaddwd m2, m15 + punpcklwd m1, m3, m4 + pmaddwd m1, m14 + punpckhwd m3, m4 + pmaddwd m3, m14 + paddd m0, m10 + paddd m2, m10 + paddd m0, m1 + paddd m2, m3 + psrad m0, 6 + psrad m2, 6 + packssdw m0, m2 + pmulhw m0, m11 + pxor m1, m1 + pmaxsw m0, m1 + mova [dstq+wq], m0 + add wq, 16 + jl .hv_loop +%if ARCH_X86_64 + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 +%else + mov r4, t5m + mov t1, t4m + mov t6m, r4 + mov t5m, t1 + mov r4, t3m + mov t1, t2m + mov t4m, r4 + mov t3m, t1 + mov r4, t1m + mov t1, t0 + mov t2m, r4 + mov t0, t6m + mov wq, wm +%endif + add dstq, strideq + ret +.v: + movif64 wq, r4 + movif32 t0m, t0 + movif32 t1m, t1 +.v_loop: +%if ARCH_X86_64 + mova m1, [t4+wq] + paddw m1, [t2+wq] + mova m2, [t3+wq] + mova m4, [t1+wq] + paddw m3, m4, [t6+wq] + paddw m4, [t5+wq] +%else + mov t0, t4m + mov t1, t2m + mova m1, [t0+wq] + paddw m1, [t1+wq] + mov t0, t3m + mov t1, t1m + mova m2, [t0+wq] + mova m4, [t1+wq] + mov t0, t6m + mov t1, t5m + paddw m3, m4, [t0+wq] + paddw m4, [t1+wq] +%endif + punpcklwd m0, m1, m2 + pmaddwd m0, m15 + punpckhwd m1, m2 + pmaddwd m1, m15 + punpcklwd m2, m3, m4 + pmaddwd m2, m14 + punpckhwd m3, m4 + pmaddwd m3, m14 + paddd m0, m10 + paddd m1, m10 + paddd m0, m2 + paddd m1, m3 + psrad m0, 6 + psrad m1, 6 + packssdw m0, m1 + pmulhw m0, m11 + pxor m1, m1 + pmaxsw m0, m1 + mova [dstq+wq], m0 + add wq, 16 + jl .v_loop +%if ARCH_X86_64 + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 +%else + mov t0, t5m + mov t1, t4m + mov r4, t3m + mov t6m, t0 + mov t5m, t1 + mov t4m, r4 + mov r4, t2m + mov t1, t1m + mov t0, t0m + mov t3m, r4 + mov t2m, t1 +%endif + add dstq, strideq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign stack_size 12*16+384*8 + %else + %assign stack_size 11*16+384*8 + %endif +cglobal wiener_filter5_16bpc, 4, 7, 8, -stack_size, dst, stride, left, \ + lpf, w, flt + %if STACK_ALIGNMENT < 16 + %define lpfm dword [esp+calloff+4*6] + %define wm dword [esp+calloff+4*7] + %define hd dword [esp+calloff+16*10+0] + %define edgeb byte [esp+calloff+16*10+4] + %define edged dword [esp+calloff+16*10+4] + %else + %define hd dword r5m + %define edgeb byte r7m + %endif + %define PICmem dword [esp+calloff+4*0] + %define t0m dword [esp+calloff+4*1] ; wiener ring buffer pointers + %define t1m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define t2 t2m + %define t3 t3m + %define t4 t4m + %define m8 [esp+calloff+16*2] + %define m9 [esp+calloff+16*3] + %define m10 [esp+calloff+16*4] + %define m11 [esp+calloff+16*5] + %define m12 [esp+calloff+16*6] + %define m13 [esp+calloff+16*7] + %define m14 [esp+calloff+16*8] + %define m15 [esp+calloff+16*9] + %define base t0-wiener_shifts + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov wd, [rstk+stack_offset+20] + mov wm, wd + mov r5, [rstk+stack_offset+24] + mov hd, r5 + mov r5, [rstk+stack_offset+32] + mov edged, r5 ; edge + %endif +%else +cglobal wiener_filter5_16bpc, 4, 14, 16, 384*8+16, dst, stride, left, lpf, \ + w, h, edge, flt + %define base +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov fltq, r6mp + movifnidn hd, hm + mov edged, r7m + mov t3d, r8m ; pixel_max + movq m12, [fltq] + movq m14, [fltq+16] +%else + %if STACK_ALIGNMENT < 16 + mov t0, [rstk+stack_offset+28] + mov t1, [rstk+stack_offset+36] ; pixel_max + movq m1, [t0] ; fx + movq m3, [t0+16] ; fy + LEA t0, wiener_shifts + %else + mov fltq, r6m + movq m1, [fltq] + movq m3, [fltq+16] + LEA t0, wiener_shifts + mov t1, r8m ; pixel_max + %endif + mov PICmem, t0 +%endif + mova m5, [base+wiener_shufE] + mova m6, [base+wiener_shufB] + mova m7, [base+wiener_shufD] +%if ARCH_X86_64 + lea t4, [wiener_shifts] + add wd, wd + punpcklwd m11, m12, m12 + pshufd m11, m11, q1111 ; x1 + pshufd m12, m12, q1111 ; x2 x3 + punpcklwd m13, m14, m14 + pshufd m13, m13, q1111 ; y1 + pshufd m14, m14, q1111 ; y2 y3 + shr t3d, 11 + mova m8, [pd_m262128] ; (1 << 4) - (1 << 18) + add lpfq, wq + lea t1, [rsp+wq+16] + add dstq, wq + neg wq + %define base t4-wiener_shifts + movd m9, [base+wiener_round+t3*4] + movq m10, [base+wiener_shifts+t3*8] + pshufd m9, m9, q0000 + pshufd m0, m10, q0000 + pshufd m10, m10, q1111 + mova m15, [wiener_lshuf5] + pmullw m11, m0 + pmullw m12, m0 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] + %define base +%else + add wd, wd + punpcklwd m0, m1, m1 + pshufd m0, m0, q1111 ; x1 + pshufd m1, m1, q1111 ; x2 x3 + punpcklwd m2, m3, m3 + pshufd m2, m2, q1111 ; y1 + pshufd m3, m3, q1111 ; y2 y3 + mova m4, [base+pd_m262128] ; (1 << 4) - (1 << 18) + mova m13, m2 + mova m14, m3 + mova m8, m4 + shr t1, 11 + add lpfq, wq + movd m2, [base+wiener_round+t1*4] + movq m3, [base+wiener_shifts+t1*8] + %if STACK_ALIGNMENT < 16 + lea t1, [esp+16*11+wq+16] + %else + lea t1, [esp+16*10+wq+16] + %endif + add dstq, wq + neg wq + pshufd m2, m2, q0000 + pshufd m4, m3, q0000 + pshufd m3, m3, q1111 + mov wm, wq + pmullw m0, m4 + pmullw m1, m4 + mova m4, [base+wiener_lshuf5] + mova m9, m2 + mova m10, m3 + mova m11, m0 + mova m12, m1 + mova m15, m4 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t4, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t3, t1 + add t1, 384*2 + add r10, strideq + mov lpfm, r10 ; below + call .h + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 +.main: + mov t0, t4 +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v2 + mov lpfq, lpfm + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.end: + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v2 + add t0, 384*6 + call .hv + dec hd + jnz .main +.v2: + call .v +%if ARCH_X86_64 + mov t4, t3 + mov t3, t2 + mov t2, t1 +%else + mov t0, t3m + mov r4, t2m + mov t1, t1m + mov t4m, t0 + mov t3m, r4 + mov t2m, t1 + mov wq, wm +%endif + add dstq, strideq +.v1: + call .v + jmp .end +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movif32 t0, PICmem + pxor m1, m1 + movd m2, wd + mova m0, [base+pb_2_3] + pshufb m2, m1 + mova m1, [base+pb_m6_m5] + psubb m0, m2 + psubb m1, m2 + mova m2, [base+pb_0to15] + pminub m0, m2 + pminub m1, m2 + pshufb m3, m0 + pshufb m4, m1 + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: + movif64 wq, r4 + movif32 wq, wm + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + mova m4, [lpfq+wq] + movd m3, [leftq+4] + pslldq m4, 4 + por m3, m4 + add leftq, 8 + jmp .h_main +.h_extend_left: + mova m3, [lpfq+wq] ; avoid accessing memory located + pshufb m3, m15 ; before the start of the buffer + jmp .h_main +.h_top: + movif64 wq, r4 + movif32 wq, wm + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m3, [lpfq+wq-4] +.h_main: + movu m4, [lpfq+wq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -18 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m0, m3, m5 + pmaddwd m0, m11 + pshufb m1, m4, m5 + pmaddwd m1, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + pmaddwd m2, m12 + pshufb m4, m7 + paddw m3, m4 + pmaddwd m3, m12 + paddd m0, m8 + paddd m1, m8 + paddd m0, m2 + paddd m1, m3 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + psraw m0, 1 + mova [t1+wq], m0 + add wq, 16 + jl .h_loop + movif32 wq, wm + ret +ALIGN function_align +.hv: + add lpfq, strideq + movif64 wq, r4 + movif32 t0m, t0 + movif32 t1m, t1 + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + mova m4, [lpfq+wq] + movd m3, [leftq+4] + pslldq m4, 4 + por m3, m4 + add leftq, 8 + jmp .hv_main +.hv_extend_left: + mova m3, [lpfq+wq] + pshufb m3, m15 + jmp .hv_main +.hv_bottom: + movif64 wq, r4 + movif32 t0m, t0 + movif32 t1m, t1 + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m3, [lpfq+wq-4] +.hv_main: + movu m4, [lpfq+wq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -18 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t1, t1m + movif32 t0, t3m + pshufb m0, m3, m5 + pmaddwd m0, m11 + pshufb m1, m4, m5 + pmaddwd m1, m11 + pshufb m2, m3, m6 + pshufb m3, m7 + paddw m2, m3 + pshufb m3, m4, m6 + pmaddwd m2, m12 + pshufb m4, m7 + paddw m3, m4 + pmaddwd m3, m12 + paddd m0, m8 + paddd m1, m8 + paddd m0, m2 +%if ARCH_X86_64 + mova m2, [t3+wq] + paddw m2, [t1+wq] + paddd m1, m3 + mova m4, [t2+wq] +%else + mova m2, [t0+wq] + mov t0, t2m + paddw m2, [t1+wq] + mov t1, t4m + paddd m1, m3 + mova m4, [t0+wq] + mov t0, t0m +%endif + punpckhwd m3, m2, m4 + pmaddwd m3, m14 + punpcklwd m2, m4 +%if ARCH_X86_64 + mova m4, [t4+wq] +%else + mova m4, [t1+wq] +%endif + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + pmaddwd m2, m14 + psraw m0, 1 + mova [t0+wq], m0 + punpckhwd m1, m0, m4 + pmaddwd m1, m13 + punpcklwd m0, m4 + pmaddwd m0, m13 + paddd m3, m9 + paddd m2, m9 + paddd m1, m3 + paddd m0, m2 + psrad m1, 6 + psrad m0, 6 + packssdw m0, m1 + pmulhw m0, m10 + pxor m1, m1 + pmaxsw m0, m1 + mova [dstq+wq], m0 + add wq, 16 + jl .hv_loop +%if ARCH_X86_64 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t4 +%else + mov r4, t3m + mov t1, t2m + mov t4m, r4 + mov t3m, t1 + mov r4, t1m + mov t1, t0 + mov t2m, r4 + mov t0, t4m + mov wq, wm +%endif + add dstq, strideq + ret +.v: + movif64 wq, r4 + movif32 t1m, t1 +.v_loop: +%if ARCH_X86_64 + mova m0, [t1+wq] + paddw m2, m0, [t3+wq] + mova m1, [t2+wq] + mova m4, [t4+wq] +%else + mov t0, t3m + mova m0, [t1+wq] + mov t1, t2m + paddw m2, m0, [t0+wq] + mov t0, t4m + mova m1, [t1+wq] + mova m4, [t0+wq] +%endif + punpckhwd m3, m2, m1 + pmaddwd m3, m14 + punpcklwd m2, m1 + pmaddwd m2, m14 + punpckhwd m1, m0, m4 + pmaddwd m1, m13 + punpcklwd m0, m4 + pmaddwd m0, m13 + paddd m3, m9 + paddd m2, m9 + paddd m1, m3 + paddd m0, m2 + psrad m1, 6 + psrad m0, 6 + packssdw m0, m1 + pmulhw m0, m10 + pxor m1, m1 + pmaxsw m0, m1 + mova [dstq+wq], m0 + add wq, 16 +%if ARCH_X86_64 + jl .v_loop +%else + jge .v_end + mov t1, t1m + jmp .v_loop +.v_end: +%endif + ret + +%macro GATHERDD 3 ; dst, src, tmp + movd %3d, %2 + %if ARCH_X86_64 + movd %1, [r13+%3] + pextrw %3d, %2, 2 + pinsrw %1, [r13+%3+2], 3 + pextrw %3d, %2, 4 + pinsrw %1, [r13+%3+2], 5 + pextrw %3d, %2, 6 + pinsrw %1, [r13+%3+2], 7 + %else + movd %1, [base+sgr_x_by_x-0xf03+%3] + pextrw %3, %2, 2 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 3 + pextrw %3, %2, 4 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 5 + pextrw %3, %2, 6 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 7 + %endif +%endmacro + +%macro GATHER_X_BY_X 5 ; dst, src0, src1, tmp32, tmp32_restore + %if ARCH_X86_64 + %define tmp r14 + %else + %define tmp %4 + %endif + GATHERDD %1, %2, tmp + GATHERDD %2, %3, tmp + movif32 %4, %5 + psrld %1, 24 + psrld %2, 24 + packssdw %1, %2 +%endmacro + +%macro MAXSD 3-4 0 ; dst, src, restore_tmp + pcmpgtd %3, %1, %2 + pand %1, %3 + pandn %3, %2 + por %1, %3 + %if %4 == 1 + pxor %3, %3 + %endif +%endmacro + +%macro MULLD 3 ; dst, src, tmp + pmulhuw %3, %1, %2 + pmullw %1, %2 + pslld %3, 16 + paddd %1, %3 +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 0, 1, 2, 3, 5 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 5*16 + %else + %assign extra_stack 3*16 + %endif +cglobal sgr_filter_5x5_16bpc, 1, 7, 8, -400*24-16-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*0+4*6] + %define stridemp dword [esp+calloff+16*0+4*7] + %define leftm dword [esp+calloff+16*3+4*0] + %define lpfm dword [esp+calloff+16*3+4*1] + %define w0m dword [esp+calloff+16*3+4*2] + %define hd dword [esp+calloff+16*3+4*3] + %define edgeb byte [esp+calloff+16*3+4*4] + %define edged dword [esp+calloff+16*3+4*4] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t0m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define m8 [base+pd_8] + %define m9 [base+pd_0xfffffff0] + %define m10 [esp+calloff+16*2] + %define m11 [base+pd_0xf00800a4] + %define m12 [base+sgr_lshuf5] + %define m13 [base+pd_34816] + %define m14 [base+pw_1023] + %define r10 r4 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_5x5_16bpc, 4, 15, 15, -400*24-16, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + movifnidn hd, hm + add wd, wd + mov edged, r7m + movu m10, [paramsq] + mova m12, [sgr_lshuf5] + add lpfq, wq + mova m8, [pd_8] + lea t1, [rsp+wq+20] + mova m9, [pd_0xfffffff0] + add dstq, wq + lea t3, [rsp+wq*2+400*12+16] + mova m11, [pd_0xf00800a4] + lea t4, [rsp+wq+400*20+16] + pshufhw m7, m10, q0000 + pshufb m10, [pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + neg wq + mova m13, [pd_34816] ; (1 << 11) + (1 << 15) + pxor m6, m6 + mova m14, [pw_1023] + psllw m7, 4 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + add wd, wd + movu m1, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*20+16] + mov t3m, t3 + pshufhw m7, m1, q0000 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + psllw m7, 4 + neg wq + mova m10, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 4 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + movif32 t2m, t1 + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t0m, t2 + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, stridemp + movif32 t4, t4m + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp +%if ARCH_X86_64 + test hb, hb +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + call .h + add lpfq, stridemp + call .hv + movif32 dstq, dstm + call .n0 + call .n1 + sub hd, 2 + movif32 t0, t0m + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .h_top + add lpfq, stridemp + call .hv_bottom +.end: + movif32 dstq, dstm + call .n0 + call .n1 +.end2: + RET +.height1: + movif32 t4, t4m + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + movif32 dstq, dstm + call .n0 + call .n1 +.odd_height_end: + call .v + movif32 dstq, dstm + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h + lea t2, [t1+400*6] + movif32 t2m, t2 + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + movif32 t0m, t0 + jmp .main +.no_top_height1: + movif32 t3, t3m + movif32 t4, t4m + call .v + call .prep_n + jmp .odd_height_end +.extend_right: + movd m0, wd + movd m1, [lpfq-2] + mova m2, [base+pw_256] + mova m3, [base+pb_m14_m13] + pshufb m0, m6 + pshufb m1, m2 + psubb m2, m0 + psubb m3, m0 + mova m0, [base+pb_0to15] + pcmpgtb m2, m0 + pcmpgtb m3, m0 + pand m4, m2 + pand m5, m3 + pandn m2, m1 + pandn m3, m1 + por m4, m2 + por m5, m3 + ret +%assign stack_offset stack_offset+4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r4-4] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq- 2] +.h_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -20 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m2, m5, m4, 2 + paddw m0, m4, m2 + palignr m3, m5, m4, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + paddd m1, m3 + punpckhwd m3, m4, m5 + pmaddwd m3, m3 + shufps m4, m5, q2121 + paddw m0, m4 ; sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m2, m3 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+wq+400*0] + paddd m1, [t1+wq+400*2] + paddd m2, [t1+wq+400*4] +.h_loop_end: + paddd m1, m5 ; sumsq + paddd m2, m4 + mova [t1+wq+400*0], m0 + mova [t1+wq+400*2], m1 + mova [t1+wq+400*4], m2 + add wq, 16 + jl .h_loop + ret +.top_fixup: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+wq+400*0] + mova m1, [t1+wq+400*2] + mova m2, [t1+wq+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m1 + mova [t2+wq+400*4], m2 + add wq, 16 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv_main +.hv_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv_main +.hv_bottom: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv_loop_start +%endif +.hv_loop: + movif32 lpfq, hvsrcm +.hv_loop_start: + movu m4, [lpfq+wq- 2] +.hv_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -20 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t3, hd + palignr m3, m5, m4, 2 + paddw m0, m4, m3 + palignr m1, m5, m4, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m1, m4, m5 + pmaddwd m1, m1 + paddd m2, m1 + punpckhwd m1, m4, m5 + pmaddwd m1, m1 + shufps m4, m5, q2121 + paddw m0, m4 ; h sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m3, m1 + paddd m2, m5 ; h sumsq + paddd m3, m4 + paddw m1, m0, [t1+wq+400*0] + paddd m4, m2, [t1+wq+400*2] + paddd m5, m3, [t1+wq+400*4] +%if ARCH_X86_64 + test hd, hd +%else + test t3, t3 +%endif + jz .hv_last_row +.hv_main2: + paddw m1, [t2+wq+400*0] ; hv sum + paddd m4, [t2+wq+400*2] ; hv sumsq + paddd m5, [t2+wq+400*4] + mova [t0+wq+400*0], m0 + mova [t0+wq+400*2], m2 + mova [t0+wq+400*4], m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + pand m4, m9 ; ((a + 8) >> 4) << 4 + pand m5, m9 + psrld m2, m4, 4 + psrld m0, m5, 4 + paddd m2, m4 + psrld m4, 1 + paddd m0, m5 + psrld m5, 1 + paddd m4, m2 ; a * 25 + paddd m5, m0 + punpcklwd m2, m3, m6 + punpckhwd m3, m6 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m6 + MAXSD m5, m3, m6, 1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m10, m2 ; p * s + MULLD m5, m10, m2 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+wq+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + movif32 t2m, t2 + movif32 t0m, t0 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+wq+400*0], m1 + paddw m1, m0 + mova [t1+wq+400*2], m4 + paddd m4, m2 + mova [t1+wq+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.v_loop: + mova m0, [t1+wq+400*0] + mova m2, [t1+wq+400*2] + mova m3, [t1+wq+400*4] + paddw m1, m0, [t2+wq+400*0] + paddd m4, m2, [t2+wq+400*2] + paddd m5, m3, [t2+wq+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + paddd m5, m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + pand m4, m9 ; ((a + 8) >> 4) << 4 + pand m5, m9 + psrld m2, m4, 4 + psrld m0, m5, 4 + paddd m2, m4 + psrld m4, 1 + paddd m0, m5 + psrld m5, 1 + paddd m4, m2 ; a * 25 + paddd m5, m0 + punpcklwd m2, m3, m6 + punpckhwd m3, m6 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m6 + MAXSD m5, m3, m6, 1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m10, m2 ; p * s + MULLD m5, m10, m2 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+wq+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+ 2] + movu m3, [t4+wq*1+ 4] + movu m1, [t3+wq*2+ 4] + movu m4, [t3+wq*2+ 8] + movu m2, [t3+wq*2+20] + movu m5, [t3+wq*2+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*1+ 0] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + mova [t4+wq*1+400*2+ 0], m0 + mova [t3+wq*2+400*4+ 0], m1 + mova [t3+wq*2+400*4+16], m2 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*1+ 2] + movu m3, [t4+wq*1+ 4] + movu m1, [t3+wq*2+ 4] + movu m4, [t3+wq*2+ 8] + movu m2, [t3+wq*2+20] + movu m5, [t3+wq*2+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*1+ 0] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + paddw m3, m0, [t4+wq*1+400*2+ 0] + paddd m4, m1, [t3+wq*2+400*4+ 0] + paddd m5, m2, [t3+wq*2+400*4+16] + mova [t4+wq*1+400*2+ 0], m0 + mova [t3+wq*2+400*4+ 0], m1 + mova [t3+wq*2+400*4+16], m2 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + mova m0, [dstq+wq] + mova m3, [t4+wq*1+400*2+ 0] + mova m4, [t3+wq*2+400*4+ 0] + mova m5, [t3+wq*2+400*4+16] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 7) + psubd m5, m3 + psrad m4, 8 + psrad m5, 8 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 4*16 + %else + %assign extra_stack 2*16 + %endif +cglobal sgr_filter_3x3_16bpc, 1, 7, 8, -400*42-16-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*2+4*0] + %define stridemp dword [esp+calloff+16*2+4*1] + %define leftm dword [esp+calloff+16*2+4*2] + %define lpfm dword [esp+calloff+16*2+4*3] + %define w0m dword [esp+calloff+16*2+4*4] + %define hd dword [esp+calloff+16*2+4*5] + %define edgeb byte [esp+calloff+16*2+4*6] + %define edged dword [esp+calloff+16*2+4*6] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %define m8 [base+pd_8] + %define m9 [esp+calloff+16*1] + %define m10 [base+pd_0xf00801c7] + %define m11 [base+pd_34816] + %define m12 [base+sgr_lshuf3] + %define m13 [base+pw_1023] + %define m14 m6 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_3x3_16bpc, 4, 15, 15, -400*42-8, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + movifnidn hd, hm + add wd, wd + mov edged, r7m + movq m9, [paramsq+4] + add lpfq, wq + lea t1, [rsp+wq+12] + mova m8, [pd_8] + add dstq, wq + lea t3, [rsp+wq*2+400*12+8] + mova m10, [pd_0xf00801c7] + lea t4, [rsp+wq+400*32+8] + mova m11, [pd_34816] + pshuflw m7, m9, q3333 + pshufb m9, [pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + neg wq + pxor m6, m6 + mova m13, [pw_1023] + psllw m7, 4 + mova m12, [sgr_lshuf3] + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + add wd, wd + movq m1, [r1+4] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*32+16] + mov t3m, t3 + pshuflw m7, m1, q3333 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 4 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + mov t2, t1 + add t1, 400*6 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv0 +%if ARCH_X86_64 + test hb, hb +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*6] +.top_fixup_loop: + mova m0, [t1+wq+400*0] + mova m1, [t1+wq+400*2] + mova m2, [t1+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m1 + mova [t2+wq+400*4], m2 + add wq, 16 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: + movd m1, wd + movd m5, [lpfq-2] + mova m2, [base+pw_256] + mova m3, [base+pb_0to15] + pshufb m1, m6 + pshufb m5, m2 + psubb m2, m1 + pcmpgtb m2, m3 + pand m4, m2 + pandn m2, m5 + por m4, m2 + ret +%assign stack_offset stack_offset+4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r4-4] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq+ 0] +.h_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -18 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + mova [t1+wq+400*0], m1 + mova [t1+wq+400*2], m2 + mova [t1+wq+400*4], m3 + add wq, 16 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m4, [lpfq+wq+ 0] +.hv0_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -18 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + paddw m0, m1, [t1+wq+400*0] + paddd m4, m2, [t1+wq+400*2] + paddd m5, m3, [t1+wq+400*4] + mova [t1+wq+400*0], m1 + mova [t1+wq+400*2], m2 + mova [t1+wq+400*4], m3 + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m4, [lpfq+wq+ 0] +.hv1_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -18 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + palignr m1, m5, m4, 2 + paddw m0, m4, m1 + punpcklwd m2, m4, m1 + pmaddwd m2, m2 + punpckhwd m3, m4, m1 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m0, m5 ; h sum + punpcklwd m1, m5, m6 + pmaddwd m1, m1 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m1 ; h sumsq + paddd m3, m5 + paddw m1, m0, [t2+wq+400*0] + paddd m4, m2, [t2+wq+400*2] + paddd m5, m3, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m2 + mova [t2+wq+400*4], m3 + paddd m4, m8 + paddd m5, m8 + psrld m4, 4 ; (a + 8) >> 4 + psrld m5, 4 + pslld m2, m4, 3 + pslld m3, m5, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*2 +4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*0+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m1 + add wq, 16 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.v1_loop: + mova m0, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+400*0+ 4] + movu m1, [t3+wq*2+400*0+ 8] + movu m2, [t3+wq*2+400*0+24] + movu m3, [t4+wq*1+400*0+ 2] + movu m4, [t3+wq*2+400*0+ 4] + movu m5, [t3+wq*2+400*0+20] + paddw m0, [t4+wq*1+400*0+ 0] + paddd m1, [t3+wq*2+400*0+ 0] + paddd m2, [t3+wq*2+400*0+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[-1] 444 + pslld m4, 2 ; b[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a[-1] 343 + psubd m4, m1 ; b[-1] 343 + psubd m5, m2 + mova [t4+wq*1+400*4], m3 + mova [t3+wq*2+400*8+ 0], m4 + mova [t3+wq*2+400*8+16], m5 + movu m0, [t4+wq*1+400*2+ 4] + movu m1, [t3+wq*2+400*4+ 8] + movu m2, [t3+wq*2+400*4+24] + movu m3, [t4+wq*1+400*2+ 2] + movu m4, [t3+wq*2+400*4+ 4] + movu m5, [t3+wq*2+400*4+20] + paddw m0, [t4+wq*1+400*2+ 0] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m2, [t3+wq*2+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[ 0] 444 + pslld m4, 2 ; b[ 0] 444 + pslld m5, 2 + mova [t4+wq*1+400* 6], m3 + mova [t3+wq*2+400*12+ 0], m4 + mova [t3+wq*2+400*12+16], m5 + psubw m3, m0 ; a[ 0] 343 + psubd m4, m1 ; b[ 0] 343 + psubd m5, m2 + mova [t4+wq*1+400* 8], m3 + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m5 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m3, [t4+wq*1+400*0+4] + movu m1, [t4+wq*1+400*0+2] + paddw m3, [t4+wq*1+400*0+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*1+400*4] + paddw m3, [t4+wq*1+400*6] + mova [t4+wq*1+400*4], m2 + mova [t4+wq*1+400*6], m1 + movu m4, [t3+wq*2+400*0+8] + movu m1, [t3+wq*2+400*0+4] + paddd m4, [t3+wq*2+400*0+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*2+400* 8+ 0] + paddd m4, [t3+wq*2+400*12+ 0] + mova [t3+wq*2+400* 8+ 0], m2 + mova [t3+wq*2+400*12+ 0], m1 + movu m5, [t3+wq*2+400*0+24] + movu m1, [t3+wq*2+400*0+20] + paddd m5, [t3+wq*2+400*0+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*2+400* 8+16] + paddd m5, [t3+wq*2+400*12+16] + mova [t3+wq*2+400* 8+16], m2 + mova [t3+wq*2+400*12+16], m1 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*1+400*2+4] + movu m1, [t4+wq*1+400*2+2] + paddw m3, [t4+wq*1+400*2+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*1+400*6] + paddw m3, [t4+wq*1+400*8] + mova [t4+wq*1+400*6], m1 + mova [t4+wq*1+400*8], m2 + movu m4, [t3+wq*2+400*4+8] + movu m1, [t3+wq*2+400*4+4] + paddd m4, [t3+wq*2+400*4+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*2+400*12+ 0] + paddd m4, [t3+wq*2+400*16+ 0] + mova [t3+wq*2+400*12+ 0], m1 + mova [t3+wq*2+400*16+ 0], m2 + movu m5, [t3+wq*2+400*4+24] + movu m1, [t3+wq*2+400*4+20] + paddd m5, [t3+wq*2+400*4+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*2+400*12+16] + paddd m5, [t3+wq*2+400*16+16] + mova [t3+wq*2+400*12+16], m1 + mova [t3+wq*2+400*16+16], m2 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 10*16 + %else + %assign extra_stack 8*16 + %endif +cglobal sgr_filter_mix_16bpc, 1, 7, 8, -400*66-48-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*8+4*0] + %define stridemp dword [esp+calloff+16*8+4*1] + %define leftm dword [esp+calloff+16*8+4*2] + %define lpfm dword [esp+calloff+16*8+4*3] + %define w0m dword [esp+calloff+16*8+4*4] + %define hd dword [esp+calloff+16*8+4*5] + %define edgeb byte [esp+calloff+16*8+4*6] + %define edged dword [esp+calloff+16*8+4*6] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %xdefine m8 m6 + %define m9 [base+pd_8] + %define m10 [base+pd_34816] + %define m11 [base+pd_0xf00801c7] + %define m12 [base+pd_0xf00800a4] + %define m13 [esp+calloff+16*4] + %define m14 [esp+calloff+16*5] + %define m15 [esp+calloff+16*6] + %define m6 [esp+calloff+16*7] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_mix_16bpc, 4, 15, 16, -400*66-40, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + movifnidn hd, hm + add wd, wd + mov edged, r7m + mova m14, [paramsq] + add lpfq, wq + mova m9, [pd_8] + lea t1, [rsp+wq+44] + mova m10, [pd_34816] + add dstq, wq + mova m11, [pd_0xf00801c7] + lea t3, [rsp+wq*2+400*24+40] + mova m12, [pd_0xf00800a4] + lea t4, [rsp+wq+400*52+40] + neg wq + pshufd m15, m14, q2222 ; w0 w1 + punpcklwd m14, m14 + pshufd m13, m14, q0000 ; s0 + pshufd m14, m14, q2222 ; s1 + pxor m6, m6 + psllw m15, 2 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + add wd, wd + mova m2, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+52] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*24+48] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*52+48] + mov t3m, t3 + mov t4m, t4 + neg wq + pshuflw m0, m2, q0000 + pshuflw m1, m2, q2222 + pshufhw m2, m2, q1010 + punpcklqdq m0, m0 ; s0 + punpcklqdq m1, m1 ; s1 + punpckhqdq m2, m2 ; w0 w1 + mov w1m, wd + pxor m3, m3 + psllw m2, 2 + mova m13, m0 + mova m14, m1 + sub wd, 4 + mova m15, m2 + mova m6, m3 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + mov t2, t1 +%if ARCH_X86_64 + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).top_fixup +%else + mov wq, w0m + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).top_fixup_loop +%endif + add t1, 400*12 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv0 +%if ARCH_X86_64 + test hd, hd +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*12] +.top_fixup_loop: + mova m0, [t1+wq+400* 0] + mova m1, [t1+wq+400* 2] + mova m2, [t1+wq+400* 4] + paddw m0, m0 + mova m3, [t1+wq+400* 6] + paddd m1, m1 + mova m4, [t1+wq+400* 8] + paddd m2, m2 + mova m5, [t1+wq+400*10] + mova [t2+wq+400* 0], m0 + mova [t2+wq+400* 2], m1 + mova [t2+wq+400* 4], m2 + mova [t2+wq+400* 6], m3 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + add wq, 16 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.h: ; horizontal boxsum +%assign stack_offset stack_offset+4 +%assign calloff 4 +%if ARCH_X86_64 + lea wq, [r4-4] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq- 2] +.h_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -20 + jl .h_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.h_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; sum3 + punpcklwd m7, m0, m6 + pmaddwd m7, m7 + punpckhwd m0, m6 + pmaddwd m0, m0 + paddd m2, m7 ; sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + mova [t1+wq+400* 6], m1 + mova [t1+wq+400* 8], m2 + mova [t1+wq+400*10], m3 + paddw m8, m1 ; sum5 + paddd m7, m2 ; sumsq5 + paddd m5, m3 + mova [t1+wq+400* 0], m8 + mova [t1+wq+400* 2], m7 + mova [t1+wq+400* 4], m5 + add wq, 16 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m4, [lpfq+wq- 2] +.hv0_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -20 + jl .hv0_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.hv0_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + movif32 t3, t3m + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; h sum3 + punpcklwd m7, m0, m6 + pmaddwd m7, m7 + punpckhwd m0, m6 + pmaddwd m0, m0 + paddd m2, m7 ; h sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + paddw m8, m1 ; h sum5 + paddd m7, m2 ; h sumsq5 + paddd m5, m3 + mova [t3+wq*2+400*8+ 8], m8 + mova [t3+wq*2+400*0+ 8], m7 + mova [t3+wq*2+400*0+24], m5 + paddw m8, [t1+wq+400* 0] + paddd m7, [t1+wq+400* 2] + paddd m5, [t1+wq+400* 4] + mova [t1+wq+400* 0], m8 + mova [t1+wq+400* 2], m7 + mova [t1+wq+400* 4], m5 + paddw m0, m1, [t1+wq+400* 6] + paddd m4, m2, [t1+wq+400* 8] + paddd m5, m3, [t1+wq+400*10] + mova [t1+wq+400* 6], m1 + mova [t1+wq+400* 8], m2 + mova [t1+wq+400*10], m3 + paddw m1, m0, [t2+wq+400* 6] + paddd m2, m4, [t2+wq+400* 8] + paddd m3, m5, [t2+wq+400*10] + mova [t2+wq+400* 6], m0 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m4, [lpfq+wq- 2] +.hv1_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -20 + jl .hv1_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.hv1_have_right: + palignr m7, m5, m4, 2 + palignr m3, m5, m4, 4 + paddw m2, m7, m3 + punpcklwd m0, m7, m3 + pmaddwd m0, m0 + punpckhwd m7, m3 + pmaddwd m7, m7 + palignr m3, m5, m4, 6 + paddw m2, m3 ; h sum3 + punpcklwd m1, m3, m6 + pmaddwd m1, m1 + punpckhwd m3, m6 + pmaddwd m3, m3 + paddd m0, m1 ; h sumsq3 + palignr m5, m4, 8 + punpckhwd m1, m4, m5 + paddw m8, m4, m5 + pmaddwd m1, m1 + punpcklwd m4, m5 + pmaddwd m4, m4 + paddd m7, m3 + paddw m5, m2, [t2+wq+400* 6] + mova [t2+wq+400* 6], m2 + paddw m8, m2 ; h sum5 + paddd m2, m0, [t2+wq+400* 8] + paddd m3, m7, [t2+wq+400*10] + mova [t2+wq+400* 8], m0 + mova [t2+wq+400*10], m7 + paddd m4, m0 ; h sumsq5 + paddd m1, m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m0, m2, 3 + pslld m7, m3, 3 + paddd m2, m0 ; ((a3 + 8) >> 4) * 9 + paddd m3, m7 + psrlw m7, m5, 1 + pavgw m7, m6 ; (b3 + 2) >> 2 + punpcklwd m0, m7, m6 + pmaddwd m0, m0 + punpckhwd m7, m6 + pmaddwd m7, m7 +%if ARCH_X86_32 + mova [esp+20], m8 +%else + SWAP m8, m6 +%endif + MAXSD m2, m0, m8 + MAXSD m3, m7, m8 + pxor m8, m8 + psubd m2, m0 ; p3 + psubd m3, m7 + punpcklwd m0, m5, m8 ; b3 + punpckhwd m5, m8 + MULLD m2, m14, m8 ; p3 * s1 + MULLD m3, m14, m8 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m5, m11 + paddusw m2, m11 + paddusw m3, m11 + psrld m2, 20 ; min(z3, 255) + movif32 t3, t3m + psrld m3, 20 + GATHER_X_BY_X m8, m2, m3, r0, dstm + punpcklwd m2, m8, m8 + punpckhwd m3, m8, m8 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m5, m10 + psrld m0, 12 + psrld m5, 12 + mova [t4+wq*1+400*4+4], m8 + mova [t3+wq*2+400*8+ 8], m0 + mova [t3+wq*2+400*8+24], m5 +%if ARCH_X86_32 + mova m8, [esp+20] +%else + SWAP m6, m8 + pxor m6, m6 +%endif + paddw m5, m8, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m1, [t2+wq+400*4] + paddw m5, [t1+wq+400*0] + paddd m2, [t1+wq+400*2] + paddd m3, [t1+wq+400*4] + mova [t2+wq+400*0], m8 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + mova [t2+wq+400*2], m4 + pslld m8, m2, 4 + mova [t2+wq+400*4], m1 + pslld m4, m3, 4 + paddd m8, m2 + pslld m2, 3 + paddd m4, m3 + pslld m3, 3 + paddd m2, m8 ; ((a5 + 8) >> 4) * 25 + paddd m3, m4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + psrlw m1, m5, 1 + pavgw m1, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m1, m7 + pmaddwd m4, m4 + punpckhwd m1, m7 + pmaddwd m1, m1 + punpcklwd m0, m5, m7 ; b5 + punpckhwd m5, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m2, m4, m7 + psubd m2, m4 ; p5 + MAXSD m3, m1, m7 + psubd m3, m1 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m5, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m1, m2, m3, r0, dstm + punpcklwd m2, m1, m1 + punpckhwd m3, m1, m1 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m5, m10 + mova [t4+wq*1+400*0+ 4], m1 + psrld m0, 12 + psrld m5, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m5 + add wq, 16 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq+400* 6] + mova m4, [t1+wq+400* 8] + mova m5, [t1+wq+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq+400* 6] + paddd m2, m4, [t2+wq+400* 8] + paddd m3, m5, [t2+wq+400*10] + mova [t2+wq+400* 6], m0 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova m3, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + mova [t3+wq*2+400*8+ 8], m3 + mova [t3+wq*2+400*0+ 8], m4 + mova [t3+wq*2+400*0+24], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+wq+400*0], m3 + mova [t1+wq+400*2], m4 + mova [t1+wq+400*4], m5 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-4] +%else + mov wd, w0m +%endif +.v1_loop: + mova m4, [t1+wq+400* 6] + mova m5, [t1+wq+400* 8] + mova m7, [t1+wq+400*10] + paddw m1, m4, [t2+wq+400* 6] + paddd m2, m5, [t2+wq+400* 8] + paddd m3, m7, [t2+wq+400*10] + mova [t2+wq+400* 6], m4 + mova [t2+wq+400* 8], m5 + mova [t2+wq+400*10], m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*4+4], m3 + psrld m0, 12 + psrld m8, m1, 12 + mova m4, [t3+wq*2+400*8+ 8] + mova m5, [t3+wq*2+400*0+ 8] + mova m7, [t3+wq*2+400*0+24] + paddw m1, m4, [t2+wq+400*0] + paddd m2, m5, [t2+wq+400*2] + paddd m3, m7, [t2+wq+400*4] + paddw m1, [t1+wq+400*0] + paddd m2, [t1+wq+400*2] + paddd m3, [t1+wq+400*4] + mova [t2+wq+400*0], m4 + mova [t2+wq+400*2], m5 + mova [t2+wq+400*4], m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + mova [t3+wq*2+400*8+ 8], m0 + pslld m4, m2, 4 + mova [t3+wq*2+400*8+24], m8 + pslld m5, m3, 4 + paddd m4, m2 + pslld m2, 3 + paddd m5, m3 + pslld m3, 3 + paddd m2, m4 + paddd m3, m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + psrlw m5, m1, 1 + pavgw m5, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m5, m7 + pmaddwd m4, m4 + punpckhwd m5, m7 + pmaddwd m5, m5 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m2, m4, m7 + psubd m2, m4 ; p5 + MAXSD m3, m5, m7 + psubd m3, m5 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m1, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m4, m2, m3, r0, dstm + punpcklwd m2, m4, m4 + punpckhwd m3, m4, m4 + MULLD m0, m2, m7 + MULLD m1, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*0+ 4], m4 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m1 + add wq, 16 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+400*0+ 2] + movu m1, [t3+wq*2+400*0+ 4] + movu m2, [t3+wq*2+400*0+20] + movu m7, [t4+wq*1+400*0+ 4] + movu m8, [t3+wq*2+400*0+ 8] + paddw m3, m0, [t4+wq*1+400*0+ 0] + paddd m4, m1, [t3+wq*2+400*0+ 0] + paddd m5, m2, [t3+wq*2+400*0+16] + paddw m3, m7 + paddd m4, m8 + movu m7, [t3+wq*2+400*0+24] + paddw m0, m3 + paddd m1, m4 + psllw m3, 2 + pslld m4, 2 + paddd m5, m7 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a5 565 + paddd m1, m4 ; b5 565 + paddd m2, m5 + mova [t4+wq*1+400* 6+ 0], m0 + mova [t3+wq*2+400*12+ 0], m1 + mova [t3+wq*2+400*12+16], m2 + movu m0, [t4+wq*1+400*2+ 4] + movu m1, [t3+wq*2+400*4+ 8] + movu m2, [t3+wq*2+400*4+24] + movu m3, [t4+wq*1+400*2+ 2] + movu m4, [t3+wq*2+400*4+ 4] + movu m5, [t3+wq*2+400*4+20] + paddw m0, [t4+wq*1+400*2+ 0] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m2, [t3+wq*2+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[-1] 444 + pslld m4, 2 ; b3[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a3[-1] 343 + psubd m4, m1 ; b3[-1] 343 + psubd m5, m2 + mova [t4+wq*1+400* 8+ 0], m3 + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m5 + movu m0, [t4+wq*1+400*4+ 4] + movu m1, [t3+wq*2+400*8+ 8] + movu m2, [t3+wq*2+400*8+24] + movu m3, [t4+wq*1+400*4+ 2] + movu m4, [t3+wq*2+400*8+ 4] + movu m5, [t3+wq*2+400*8+20] + paddw m0, [t4+wq*1+400*4+ 0] + paddd m1, [t3+wq*2+400*8+ 0] + paddd m2, [t3+wq*2+400*8+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[ 0] 444 + pslld m4, 2 ; b3[ 0] 444 + pslld m5, 2 + mova [t4+wq*1+400*10+ 0], m3 + mova [t3+wq*2+400*20+ 0], m4 + mova [t3+wq*2+400*20+16], m5 + psubw m3, m0 ; a3[ 0] 343 + psubd m4, m1 ; b3[ 0] 343 + psubd m5, m2 + mova [t4+wq*1+400*12+ 0], m3 + mova [t3+wq*2+400*24+ 0], m4 + mova [t3+wq*2+400*24+16], m5 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*1+ 4] + movu m2, [t4+wq*1+ 2] + paddw m0, [t4+wq*1+ 0] + paddw m0, m2 + paddw m2, m0 + psllw m0, 2 + paddw m0, m2 ; a5 + movu m4, [t3+wq*2+ 8] + movu m5, [t3+wq*2+24] + movu m1, [t3+wq*2+ 4] + movu m3, [t3+wq*2+20] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddd m4, m1 + paddd m5, m3 + paddd m1, m4 + paddd m3, m5 + pslld m4, 2 + pslld m5, 2 + paddd m4, m1 ; b5 + paddd m5, m3 + movu m2, [t4+wq*1+400* 6] + paddw m2, m0 + mova [t4+wq*1+400* 6], m0 + paddd m0, m4, [t3+wq*2+400*12+ 0] + paddd m1, m5, [t3+wq*2+400*12+16] + mova [t3+wq*2+400*12+ 0], m4 + mova [t3+wq*2+400*12+16], m5 + mova [rsp+16+ARCH_X86_32*4], m1 + movu m3, [t4+wq*1+400*2+4] + movu m5, [t4+wq*1+400*2+2] + paddw m3, [t4+wq*1+400*2+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + movu m3, [t4+wq*1+400* 8] + paddw m3, [t4+wq*1+400*10] + paddw m3, m4 + mova [t4+wq*1+400* 8], m4 + mova [t4+wq*1+400*10], m5 + movu m1, [t3+wq*2+400*4+ 8] + movu m5, [t3+wq*2+400*4+ 4] + movu m7, [t3+wq*2+400*4+24] + movu m8, [t3+wq*2+400*4+20] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m7, [t3+wq*2+400*4+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 +%if ARCH_X86_32 + mova [esp+52], m8 + psubd m8, m7 +%else + psubd m6, m8, m7 + SWAP m8, m6 +%endif + paddd m1, m4, [t3+wq*2+400*16+ 0] + paddd m7, m8, [t3+wq*2+400*16+16] + paddd m1, [t3+wq*2+400*20+ 0] + paddd m7, [t3+wq*2+400*20+16] + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m8 + mova [t3+wq*2+400*20+ 0], m5 +%if ARCH_X86_32 + mova m8, [esp+52] +%else + SWAP m8, m6 + pxor m6, m6 +%endif + mova [t3+wq*2+400*20+16], m8 + mova [rsp+32+ARCH_X86_32*4], m7 + movu m5, [dstq+wq] + punpcklwd m4, m5, m6 + punpcklwd m7, m2, m6 + pmaddwd m7, m4 ; a5 * src + punpcklwd m8, m3, m6 + pmaddwd m8, m4 ; a3 * src + punpckhwd m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + pslld m4, 13 + pslld m5, 13 + psubd m0, m7 ; b5 - a5 * src + (1 << 8) + psubd m1, m8 ; b3 - a3 * src + (1 << 8) + mova m7, [base+pd_0xffff] + psrld m0, 9 + pslld m1, 7 + pand m0, m7 + pandn m8, m7, m1 + por m0, m8 + mova m1, [rsp+16+ARCH_X86_32*4] + mova m8, [rsp+32+ARCH_X86_32*4] + psubd m1, m2 + psubd m8, m3 + mova m2, [base+pd_4096] + psrld m1, 9 + pslld m8, 7 + pand m1, m7 + pandn m7, m8 + por m1, m7 + pmaddwd m0, m15 + pmaddwd m1, m15 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + paddd m4, m2 + paddd m5, m2 + paddd m0, m4 + paddd m1, m5 + psrad m0, 8 + psrad m1, 8 + packssdw m0, m1 ; clip + pmaxsw m0, m7 + psrlw m0, 5 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, stridemp + ret +%if ARCH_X86_64 + SWAP m6, m7 +%endif +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*1+400*4+4] + movu m5, [t4+wq*1+400*4+2] + paddw m3, [t4+wq*1+400*4+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + paddw m3, m4, [t4+wq*1+400*12] + paddw m3, [t4+wq*1+400*10] + mova [t4+wq*1+400*10], m5 + mova [t4+wq*1+400*12], m4 + movu m1, [t3+wq*2+400*8+ 8] + movu m5, [t3+wq*2+400*8+ 4] + movu m7, [t3+wq*2+400*8+24] + movu m8, [t3+wq*2+400*8+20] + paddd m1, [t3+wq*2+400*8+ 0] + paddd m7, [t3+wq*2+400*8+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 + psubd m0, m8, m7 + paddd m1, m4, [t3+wq*2+400*24+ 0] + paddd m7, m0, [t3+wq*2+400*24+16] + paddd m1, [t3+wq*2+400*20+ 0] + paddd m7, [t3+wq*2+400*20+16] + mova [t3+wq*2+400*20+ 0], m5 + mova [t3+wq*2+400*20+16], m8 + mova [t3+wq*2+400*24+ 0], m4 + mova [t3+wq*2+400*24+16], m0 + mova m5, [dstq+wq] + mova m2, [t4+wq*1+400* 6] + punpcklwd m4, m5, m6 + punpcklwd m8, m2, m6 + pmaddwd m8, m4 ; a5 * src + punpcklwd m0, m3, m6 + pmaddwd m0, m4 ; a3 * src + punpckhwd m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + psubd m1, m0 ; b3 - a3 * src + (1 << 8) + pslld m4, 13 + pslld m5, 13 + mova m0, [t3+wq*2+400*12+ 0] + psubd m0, m8 ; b5 - a5 * src + (1 << 8) + mova m8, [t3+wq*2+400*12+16] + psubd m8, m2 + psubd m7, m3 + mova m2, [base+pd_0xffff] + pslld m1, 7 + psrld m0, 8 + psrld m8, 8 + pslld m7, 7 + pand m0, m2 + pandn m3, m2, m1 + por m0, m3 + pand m8, m2 + pandn m2, m7 + por m2, m8 + mova m1, [base+pd_4096] + pmaddwd m0, m15 + pmaddwd m2, m15 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + pxor m7, m7 + paddd m4, m1 + paddd m5, m1 + paddd m0, m4 + paddd m2, m5 + psrad m0, 8 + psrad m2, 8 + packssdw m0, m2 ; clip + pmaxsw m0, m7 + psrlw m0, 5 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret diff --git a/third_party/dav1d/src/x86/looprestoration_avx2.asm b/third_party/dav1d/src/x86/looprestoration_avx2.asm new file mode 100644 index 0000000000..a73cb21882 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration_avx2.asm @@ -0,0 +1,2237 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +wiener_l_shuf: db 4, 4, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +pb_0to31: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +wiener_shufA: db 1, 7, 2, 8, 3, 9, 4, 10, 5, 11, 6, 12, 7, 13, 8, 14 +wiener_shufB: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +wiener_shufC: db 6, 5, 7, 6, 8, 7, 9, 8, 10, 9, 11, 10, 12, 11, 13, 12 +sgr_r_ext: times 16 db 1 + times 16 db 9 + +; dword version of dav1d_sgr_x_by_x[] for use with gathers, wastes a bit of +; cache but eliminates some shifts in the inner sgr loop which is overall a win +const sgr_x_by_x_avx2 + dd 255,128, 85, 64, 51, 43, 37, 32, 28, 26, 23, 21, 20, 18, 17, 16 + dd 15, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8 + dd 8, 8, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 5, 5 + dd 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4 + dd 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3 + dd 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 + dd 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 + dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1 + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 + + times 4 db -1 ; needed for 16-bit sgr +pb_m5: times 4 db -5 +pb_3: times 4 db 3 +pw_5_6: dw 5, 6 + +sgr_l_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +sgr_shuf: db 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1, 7, -1, 8, -1 + db 9, -1, 10, -1, 11, -1, 12, -1 + +pw_256: times 2 dw 256 +pw_2056: times 2 dw 2056 +pw_m16380: times 2 dw -16380 +pd_25: dd 25 +pd_34816: dd 34816 +pd_m4096: dd -4096 +pd_0xf00801c7: dd 0xf00801c7 +pd_0xf00800a4: dd 0xf00800a4 + +SECTION .text + +DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers + +INIT_YMM avx2 +cglobal wiener_filter7_8bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt + mov fltq, r6mp + movifnidn hd, hm + mov edged, r7m + mov wd, wm + vbroadcasti128 m6, [wiener_shufA] + vpbroadcastb m11, [fltq+ 0] ; x0 x0 + vbroadcasti128 m7, [wiener_shufB] + vpbroadcastd m12, [fltq+ 2] + vbroadcasti128 m8, [wiener_shufC] + packsswb m12, m12 ; x1 x2 + vpbroadcastw m13, [fltq+ 6] ; x3 + vbroadcasti128 m9, [sgr_shuf+6] + add lpfq, wq + vpbroadcastd m10, [pw_m16380] + vpbroadcastd m14, [fltq+16] ; y0 y1 + add dstq, wq + vpbroadcastd m15, [fltq+20] ; y2 y3 + lea t1, [rsp+wq*2+16] + psllw m14, 5 + neg wq + psllw m15, 5 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call .v + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call .v +.v2: + call .v + jmp .v1 +.extend_right: + movd xm2, r10d + vpbroadcastd m0, [pb_3] + vpbroadcastd m1, [pb_m5] + vpbroadcastb m2, xm2 + movu m3, [pb_0to31] + psubb m0, m2 + psubb m1, m2 + pminub m0, m3 + pminub m1, m3 + pshufb m4, m0 + pshufb m5, m1 + ret +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm4, [leftq] + vpblendd m4, [lpfq+r10-4], 0xfe + add leftq, 4 + jmp .h_main +.h_extend_left: + vbroadcasti128 m5, [lpfq+r10] ; avoid accessing memory located + mova m4, [lpfq+r10] ; before the start of the buffer + palignr m4, m5, 12 + pshufb m4, [wiener_l_shuf] + jmp .h_main +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+r10-4] +.h_main: + movu m5, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -34 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m0, m4, m6 + pmaddubsw m0, m11 + pshufb m1, m5, m6 + pmaddubsw m1, m11 + pshufb m2, m4, m7 + pmaddubsw m2, m12 + pshufb m3, m5, m7 + pmaddubsw m3, m12 + paddw m0, m2 + pshufb m2, m4, m8 + pmaddubsw m2, m12 + paddw m1, m3 + pshufb m3, m5, m8 + pmaddubsw m3, m12 + pshufb m4, m9 + paddw m0, m2 + pmullw m2, m4, m13 + pshufb m5, m9 + paddw m1, m3 + pmullw m3, m5, m13 + psllw m4, 7 + psllw m5, 7 + paddw m4, m10 + paddw m5, m10 + paddw m0, m2 + vpbroadcastd m2, [pw_2056] + paddw m1, m3 + paddsw m0, m4 + paddsw m1, m5 + psraw m0, 3 + psraw m1, 3 + paddw m0, m2 + paddw m1, m2 + mova [t1+r10*2+ 0], m0 + mova [t1+r10*2+32], m1 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm4, [leftq] + vpblendd m4, [lpfq+r10-4], 0xfe + add leftq, 4 + jmp .hv_main +.hv_extend_left: + movu m4, [lpfq+r10-4] + pshufb m4, [wiener_l_shuf] + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m4, [lpfq+r10-4] +.hv_main: + movu m5, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -34 + jl .hv_have_right + call .extend_right +.hv_have_right: + pshufb m0, m4, m6 + pmaddubsw m0, m11 + pshufb m1, m5, m6 + pmaddubsw m1, m11 + pshufb m2, m4, m7 + pmaddubsw m2, m12 + pshufb m3, m5, m7 + pmaddubsw m3, m12 + paddw m0, m2 + pshufb m2, m4, m8 + pmaddubsw m2, m12 + paddw m1, m3 + pshufb m3, m5, m8 + pmaddubsw m3, m12 + pshufb m4, m9 + paddw m0, m2 + pmullw m2, m4, m13 + pshufb m5, m9 + paddw m1, m3 + pmullw m3, m5, m13 + psllw m4, 7 + psllw m5, 7 + paddw m4, m10 + paddw m5, m10 + paddw m0, m2 + paddw m1, m3 + mova m2, [t4+r10*2] + paddw m2, [t2+r10*2] + mova m3, [t3+r10*2] + paddsw m0, m4 + vpbroadcastd m4, [pw_2056] + paddsw m1, m5 + mova m5, [t5+r10*2] + paddw m5, [t1+r10*2] + psraw m0, 3 + psraw m1, 3 + paddw m0, m4 + paddw m1, m4 + paddw m4, m0, [t6+r10*2] + mova [t0+r10*2], m0 + punpcklwd m0, m2, m3 + pmaddwd m0, m15 + punpckhwd m2, m3 + pmaddwd m2, m15 + punpcklwd m3, m4, m5 + pmaddwd m3, m14 + punpckhwd m4, m5 + pmaddwd m4, m14 + paddd m0, m3 + paddd m4, m2 + mova m2, [t4+r10*2+32] + paddw m2, [t2+r10*2+32] + mova m3, [t3+r10*2+32] + mova m5, [t5+r10*2+32] + paddw m5, [t1+r10*2+32] + packuswb m0, m4 + paddw m4, m1, [t6+r10*2+32] + mova [t0+r10*2+32], m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m15 + punpckhwd m2, m3 + pmaddwd m2, m15 + punpcklwd m3, m4, m5 + pmaddwd m3, m14 + punpckhwd m4, m5 + pmaddwd m4, m14 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+r10], m0 + add r10, 32 + jl .hv_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m2, [t4+r10*2+ 0] + paddw m2, [t2+r10*2+ 0] + mova m4, [t3+r10*2+ 0] + mova m6, [t1+r10*2+ 0] + paddw m8, m6, [t6+r10*2+ 0] + paddw m6, [t5+r10*2+ 0] + mova m3, [t4+r10*2+32] + paddw m3, [t2+r10*2+32] + mova m5, [t3+r10*2+32] + mova m7, [t1+r10*2+32] + paddw m9, m7, [t6+r10*2+32] + paddw m7, [t5+r10*2+32] + punpcklwd m0, m2, m4 + pmaddwd m0, m15 + punpckhwd m2, m4 + pmaddwd m2, m15 + punpcklwd m4, m8, m6 + pmaddwd m4, m14 + punpckhwd m6, m8, m6 + pmaddwd m6, m14 + punpcklwd m1, m3, m5 + pmaddwd m1, m15 + punpckhwd m3, m5 + pmaddwd m3, m15 + punpcklwd m5, m9, m7 + pmaddwd m5, m14 + punpckhwd m7, m9, m7 + pmaddwd m7, m14 + paddd m0, m4 + paddd m2, m6 + paddd m1, m5 + paddd m3, m7 + packuswb m0, m2 + packuswb m1, m3 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+r10], m0 + add r10, 32 + jl .v_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq + ret + +cglobal wiener_filter5_8bpc, 4, 13, 16, 384*8+16, dst, stride, left, lpf, \ + w, h, edge, flt + mov fltq, r6mp + movifnidn hd, hm + mov edged, r7m + mov wd, wm + vbroadcasti128 m6, [wiener_shufB] + vpbroadcastd m12, [fltq+ 2] + vbroadcasti128 m7, [wiener_shufC] + packsswb m12, m12 ; x1 x2 + vpbroadcastw m13, [fltq+ 6] ; x3 + vbroadcasti128 m8, [sgr_shuf+6] + add lpfq, wq + vpbroadcastd m9, [pw_m16380] + vpbroadcastd m10, [pw_2056] + mova m11, [wiener_l_shuf] + vpbroadcastd m14, [fltq+16] ; __ y1 + add dstq, wq + vpbroadcastd m15, [fltq+20] ; y2 y3 + lea t1, [rsp+wq*2+16] + psllw m14, 5 + neg wq + psllw m15, 5 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t4, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t3, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 +.main: + mov t0, t4 +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v2 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.end: + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v2 + add t0, 384*6 + call .hv + dec hd + jnz .main +.v2: + call .v + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq +.v1: + call .v + jmp .end +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm4, [leftq] + vpblendd m4, [lpfq+r10-4], 0xfe + add leftq, 4 + jmp .h_main +.h_extend_left: + vbroadcasti128 m5, [lpfq+r10] ; avoid accessing memory located + mova m4, [lpfq+r10] ; before the start of the buffer + palignr m4, m5, 12 + pshufb m4, m11 + jmp .h_main +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+r10-4] +.h_main: + movu m5, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -33 + jl .h_have_right + call mangle(private_prefix %+ _wiener_filter7_8bpc_avx2).extend_right +.h_have_right: + pshufb m0, m4, m6 + pmaddubsw m0, m12 + pshufb m1, m5, m6 + pmaddubsw m1, m12 + pshufb m2, m4, m7 + pmaddubsw m2, m12 + pshufb m3, m5, m7 + pmaddubsw m3, m12 + pshufb m4, m8 + paddw m0, m2 + pmullw m2, m4, m13 + pshufb m5, m8 + paddw m1, m3 + pmullw m3, m5, m13 + psllw m4, 7 + psllw m5, 7 + paddw m4, m9 + paddw m5, m9 + paddw m0, m2 + paddw m1, m3 + paddsw m0, m4 + paddsw m1, m5 + psraw m0, 3 + psraw m1, 3 + paddw m0, m10 + paddw m1, m10 + mova [t1+r10*2+ 0], m0 + mova [t1+r10*2+32], m1 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm4, [leftq] + vpblendd m4, [lpfq+r10-4], 0xfe + add leftq, 4 + jmp .hv_main +.hv_extend_left: + movu m4, [lpfq+r10-4] + pshufb m4, m11 + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m4, [lpfq+r10-4] +.hv_main: + movu m5, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -33 + jl .hv_have_right + call mangle(private_prefix %+ _wiener_filter7_8bpc_avx2).extend_right +.hv_have_right: + pshufb m0, m4, m6 + pmaddubsw m0, m12 + pshufb m1, m5, m6 + pmaddubsw m1, m12 + pshufb m2, m4, m7 + pmaddubsw m2, m12 + pshufb m3, m5, m7 + pmaddubsw m3, m12 + pshufb m4, m8 + paddw m0, m2 + pmullw m2, m4, m13 + pshufb m5, m8 + paddw m1, m3 + pmullw m3, m5, m13 + psllw m4, 7 + psllw m5, 7 + paddw m4, m9 + paddw m5, m9 + paddw m0, m2 + paddw m1, m3 + mova m2, [t3+r10*2] + paddw m2, [t1+r10*2] + mova m3, [t2+r10*2] + paddsw m0, m4 + paddsw m1, m5 + psraw m0, 3 + psraw m1, 3 + paddw m0, m10 + paddw m1, m10 + paddw m4, m0, [t4+r10*2] + mova [t0+r10*2], m0 + punpcklwd m0, m2, m3 + pmaddwd m0, m15 + punpckhwd m2, m3 + pmaddwd m2, m15 + punpcklwd m3, m4, m4 + pmaddwd m3, m14 + punpckhwd m4, m4 + pmaddwd m4, m14 + paddd m0, m3 + paddd m4, m2 + mova m2, [t3+r10*2+32] + paddw m2, [t1+r10*2+32] + mova m3, [t2+r10*2+32] + packuswb m0, m4 + paddw m4, m1, [t4+r10*2+32] + mova [t0+r10*2+32], m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m15 + punpckhwd m2, m3 + pmaddwd m2, m15 + punpcklwd m3, m4, m4 + pmaddwd m3, m14 + punpckhwd m4, m4 + pmaddwd m4, m14 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+r10], m0 + add r10, 32 + jl .hv_loop + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t4 + add dstq, strideq + ret +.v: + mov r10, wq + psrld m13, m14, 16 ; y1 __ +.v_loop: + mova m6, [t1+r10*2+ 0] + paddw m2, m6, [t3+r10*2+ 0] + mova m4, [t2+r10*2+ 0] + mova m7, [t1+r10*2+32] + paddw m3, m7, [t3+r10*2+32] + mova m5, [t2+r10*2+32] + paddw m6, [t4+r10*2+ 0] + paddw m7, [t4+r10*2+32] + punpcklwd m0, m2, m4 + pmaddwd m0, m15 + punpckhwd m2, m4 + pmaddwd m2, m15 + punpcklwd m1, m3, m5 + pmaddwd m1, m15 + punpckhwd m3, m5 + pmaddwd m3, m15 + punpcklwd m5, m7, m6 + pmaddwd m4, m5, m14 + punpckhwd m7, m6 + pmaddwd m6, m7, m14 + pmaddwd m5, m13 + pmaddwd m7, m13 + paddd m0, m4 + paddd m2, m6 + paddd m1, m5 + paddd m3, m7 + packuswb m0, m2 + packuswb m1, m3 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+r10], m0 + add r10, 32 + jl .v_loop + ret + +cglobal sgr_filter_5x5_8bpc, 4, 13, 16, 400*24+16, dst, stride, left, lpf, \ + w, h, edge, params +%define base r12-sgr_x_by_x_avx2-256*4 + lea r12, [sgr_x_by_x_avx2+256*4] + mov paramsq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + vbroadcasti128 m8, [base+sgr_shuf+0] + vbroadcasti128 m9, [base+sgr_shuf+8] + add lpfq, wq + vbroadcasti128 m10, [base+sgr_shuf+2] + add dstq, wq + vbroadcasti128 m11, [base+sgr_shuf+6] + lea t3, [rsp+wq*4+16+400*12] + vpbroadcastd m12, [paramsq+0] ; s0 + pxor m6, m6 + vpbroadcastw m7, [paramsq+8] ; w0 + lea t1, [rsp+wq*2+20] + vpbroadcastd m13, [base+pd_0xf00800a4] + neg wq + vpbroadcastd m14, [base+pd_34816] ; (1 << 11) + (1 << 15) + psllw m7, 4 + vpbroadcastd m15, [base+pd_m4096] + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, strideq + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + test hd, hd + jz .odd_height + call .h + add lpfq, strideq + call .hv + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .h_top + add lpfq, strideq + call .hv_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + call .n0 + call .n1 +.odd_height_end: + call .v + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+400*6] + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + jmp .main +.no_top_height1: + call .v + call .prep_n + jmp .odd_height_end +.extend_right: + movd xm2, r10d + mova m0, [sgr_r_ext] + vpbroadcastb m2, xm2 + psubb m0, m2 + pminub m0, [pb_0to31] + pshufb m5, m0 + ret +.h: ; horizontal boxsum + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .h_main +.h_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu xm5, [lpfq+r10-2] +.h_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -18 + jl .h_have_right + call .extend_right +.h_have_right: + pshufb m3, m5, m8 + pmullw m4, m3, m3 + pshufb m2, m5, m9 + paddw m0, m3, m2 + shufps m3, m2, q2121 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + punpcklwd m3, m4, m6 + paddd m1, m3 + punpckhwd m4, m6 + paddd m2, m4 + pshufb m4, m5, m10 + paddw m0, m4 + pshufb m5, m11 + paddw m0, m5 ; sum + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + punpckhwd m4, m5 + pmaddwd m4, m4 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+r10*2+400*0] + paddd m1, [t1+r10*2+400*2] + paddd m2, [t1+r10*2+400*4] +.h_loop_end: + paddd m1, m3 ; sumsq + paddd m2, m4 + mova [t1+r10*2+400*0], m0 + mova [t1+r10*2+400*2], m1 + mova [t1+r10*2+400*4], m2 + add r10, 16 + jl .h_loop + ret +.top_fixup: + lea r10, [wq-2] +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+r10*2+400*0] + mova m1, [t1+r10*2+400*2] + mova m2, [t1+r10*2+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+r10*2+400*0], m0 + mova [t2+r10*2+400*2], m1 + mova [t2+r10*2+400*4], m2 + add r10, 16 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .hv_main +.hv_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .hv_main +.hv_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu xm5, [lpfq+r10-2] +.hv_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -18 + jl .hv_have_right + call .extend_right +.hv_have_right: + pshufb m1, m5, m8 + pmullw m4, m1, m1 + pshufb m3, m5, m9 + paddw m0, m1, m3 + shufps m1, m3, q2121 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + punpcklwd m1, m4, m6 + paddd m2, m1 + punpckhwd m4, m6 + paddd m3, m4 + pshufb m1, m5, m10 + paddw m0, m1 + pshufb m5, m11 + paddw m0, m5 ; h sum + punpcklwd m4, m5, m1 + pmaddwd m4, m4 + punpckhwd m5, m1 + pmaddwd m5, m5 + paddw m1, m0, [t1+r10*2+400*0] + paddd m2, m4 ; h sumsq + paddd m3, m5 + paddd m4, m2, [t1+r10*2+400*2] + paddd m5, m3, [t1+r10*2+400*4] + test hd, hd + jz .hv_last_row +.hv_main2: + paddw m1, [t2+r10*2+400*0] ; hv sum + paddd m4, [t2+r10*2+400*2] ; hv sumsq + paddd m5, [t2+r10*2+400*4] + mova [t0+r10*2+400*0], m0 + mova [t0+r10*2+400*2], m2 + mova [t0+r10*2+400*4], m3 + vpbroadcastd m2, [pd_25] + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmulld m4, m2 ; a * 25 + pmulld m5, m2 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m12 ; p * s + pmulld m5, m12 + pmaddwd m0, m13 ; b * 164 + pmaddwd m1, m13 + paddusw m4, m13 + paddusw m5, m13 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r12+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r12+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m14 + pand m0, m15 + pand m1, m15 + por m0, m2 ; a | (b << 12) + por m1, m3 + mova [t3+r10*4+ 8], xm0 ; The neighbor calculations requires + vextracti128 [t3+r10*4+40], m0, 1 ; 13 bits for a and 21 bits for b. + mova [t3+r10*4+24], xm1 ; Packing them allows for 12+20, but + vextracti128 [t3+r10*4+56], m1, 1 ; that gets us most of the way. + add r10, 16 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+r10*2+400*0], m1 + paddw m1, m0 + mova [t1+r10*2+400*2], m4 + paddd m4, m2 + mova [t1+r10*2+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab + lea r10, [wq-2] +.v_loop: + mova m0, [t1+r10*2+400*0] + mova m2, [t1+r10*2+400*2] + mova m3, [t1+r10*2+400*4] + paddw m1, m0, [t2+r10*2+400*0] + paddd m4, m2, [t2+r10*2+400*2] + paddd m5, m3, [t2+r10*2+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + paddd m5, m3 + vpbroadcastd m2, [pd_25] + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmulld m4, m2 ; a * 25 + pmulld m5, m2 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m12 ; p * s + pmulld m5, m12 + pmaddwd m0, m13 ; b * 164 + pmaddwd m1, m13 + paddusw m4, m13 + paddusw m5, m13 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r12+m3*4], m4 ; x + psrad m4, m5, 20 + vpgatherdd m3, [r12+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m14 + pand m0, m15 + pand m1, m15 + por m0, m2 ; a | (b << 12) + por m1, m3 + mova [t3+r10*4+ 8], xm0 + vextracti128 [t3+r10*4+40], m0, 1 + mova [t3+r10*4+24], xm1 + vextracti128 [t3+r10*4+56], m1, 1 + add r10, 16 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t3+r10*4+ 4] + movu m1, [t3+r10*4+36] + paddd m2, m0, [t3+r10*4+ 0] + paddd m3, m1, [t3+r10*4+32] + paddd m2, [t3+r10*4+ 8] + paddd m3, [t3+r10*4+40] + paddd m0, m2 + pslld m2, 2 + paddd m1, m3 + pslld m3, 2 + paddd m2, m0 ; ab 565 + paddd m3, m1 + pandn m0, m15, m2 ; a + psrld m2, 12 ; b + pandn m1, m15, m3 + psrld m3, 12 + mova [t3+r10*4+400*4+ 0], m0 + mova [t3+r10*4+400*8+ 0], m2 + mova [t3+r10*4+400*4+32], m1 + mova [t3+r10*4+400*8+32], m3 + add r10, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m0, [t3+r10*4+ 4] + movu m1, [t3+r10*4+36] + paddd m2, m0, [t3+r10*4+ 0] + paddd m3, m1, [t3+r10*4+32] + paddd m2, [t3+r10*4+ 8] + paddd m3, [t3+r10*4+40] + paddd m0, m2 + pslld m2, 2 + paddd m1, m3 + pslld m3, 2 + paddd m2, m0 + paddd m3, m1 + pandn m0, m15, m2 + psrld m2, 12 + pandn m1, m15, m3 + psrld m3, 12 + paddd m4, m0, [t3+r10*4+400*4+ 0] ; a + paddd m5, m1, [t3+r10*4+400*4+32] + mova [t3+r10*4+400*4+ 0], m0 + mova [t3+r10*4+400*4+32], m1 + paddd m0, m2, [t3+r10*4+400*8+ 0] ; b + paddd m1, m3, [t3+r10*4+400*8+32] + mova [t3+r10*4+400*8+ 0], m2 + mova [t3+r10*4+400*8+32], m3 + pmovzxbd m2, [dstq+r10+0] + pmovzxbd m3, [dstq+r10+8] + pmaddwd m4, m2 ; a * src + pmaddwd m5, m3 + packssdw m2, m3 + psubd m0, m4 ; b - a * src + (1 << 8) + psubd m1, m5 + psrad m0, 9 + psrad m1, 9 + packssdw m0, m1 + pmulhrsw m0, m7 + paddw m0, m2 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + pshufd xm0, xm0, q3120 + mova [dstq+r10], xm0 + add r10, 16 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + pmovzxbd m2, [dstq+r10+0] + pmovzxbd m3, [dstq+r10+8] + pmaddwd m4, m2, [t3+r10*4+400*4+ 0] ; a * src + pmaddwd m5, m3, [t3+r10*4+400*4+32] + mova m0, [t3+r10*4+400*8+ 0] ; b + mova m1, [t3+r10*4+400*8+32] + packssdw m2, m3 + psubd m0, m4 ; b - a * src + (1 << 7) + psubd m1, m5 + psrad m0, 8 + psrad m1, 8 + packssdw m0, m1 + pmulhrsw m0, m7 + paddw m0, m2 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + pshufd xm0, xm0, q3120 + mova [dstq+r10], xm0 + add r10, 16 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_3x3_8bpc, 4, 15, 15, -400*28-16, dst, stride, left, lpf, \ + w, h, edge, params +%define base r14-sgr_x_by_x_avx2-256*4 + mov paramsq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + lea r14, [sgr_x_by_x_avx2+256*4] + vbroadcasti128 m8, [base+sgr_shuf+2] + add lpfq, wq + vbroadcasti128 m9, [base+sgr_shuf+4] + add dstq, wq + vbroadcasti128 m10, [base+sgr_shuf+6] + lea t3, [rsp+wq*4+16+400*12] + vpbroadcastd m11, [paramsq+ 4] ; s1 + pxor m6, m6 + vpbroadcastw m7, [paramsq+10] ; w1 + lea t1, [rsp+wq*2+20] + vpbroadcastd m12, [base+pd_0xf00801c7] + neg wq + vpbroadcastd m13, [base+pd_34816] ; (1 << 11) + (1 << 15) + psllw m7, 4 + vpbroadcastd m14, [base+pd_m4096] + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + add t1, 400*6 + call .h_top + lea t4, [lpfq+strideq*4] + mov lpfq, dstq + add t4, strideq + mov [rsp], t4 ; below + mov t0, t2 + call .hv +.main: + mov t5, t3 + add t3, 400*4 + dec hd + jz .height1 + add lpfq, strideq + call .hv + call .prep_n + dec hd + jz .extend_bottom +.main_loop: + add lpfq, strideq + call .hv + call .n + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv_bottom + call .n + add lpfq, strideq + call .hv_bottom +.end: + call .n + RET +.height1: + call .v + call .prep_n + mov t2, t1 + call .v + jmp .end +.extend_bottom: + call .v + call .n + mov t2, t1 + call .v + jmp .end +.no_top: + lea t4, [lpfq+strideq*4] + mov lpfq, dstq + lea t4, [t4+strideq*2] + mov [rsp], t4 + call .h + lea t0, [t1+400*6] + mov t2, t1 + call .v + jmp .main +.h: ; horizontal boxsum + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .h_main +.h_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu xm5, [lpfq+r10-2] +.h_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -17 + jl .h_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right +.h_have_right: + pshufb m0, m5, m8 + pmullw m2, m0, m0 + pshufb m4, m5, m9 + paddw m0, m4 + pshufb m5, m10 + paddw m0, m5 ; sum + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + punpckhwd m4, m5 + pmaddwd m4, m4 + punpcklwd m1, m2, m6 + punpckhwd m2, m6 + mova [t1+r10*2+400*0], m0 + paddd m1, m3 ; sumsq + paddd m2, m4 + mova [t1+r10*2+400*2], m1 + mova [t1+r10*2+400*4], m2 + add r10, 16 + jl .h_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .hv_main +.hv_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .hv_main +.hv_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu xm5, [lpfq+r10-2] +.hv_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -17 + jl .hv_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right +.hv_have_right: + pshufb m0, m5, m8 + pmullw m3, m0, m0 + pshufb m1, m5, m9 + paddw m0, m1 + pshufb m5, m10 + paddw m0, m5 ; h sum + punpcklwd m4, m5, m1 + pmaddwd m4, m4 + punpckhwd m5, m1 + pmaddwd m5, m5 + paddw m1, m0, [t2+r10*2+400*0] + paddw m1, [t1+r10*2+400*0] ; hv sum + punpcklwd m2, m3, m6 + punpckhwd m3, m6 + paddd m4, m2 ; h sumsq + paddd m5, m3 + paddd m2, m4, [t2+r10*2+400*2] + paddd m3, m5, [t2+r10*2+400*4] + paddd m2, [t1+r10*2+400*2] ; hv sumsq + paddd m3, [t1+r10*2+400*4] + mova [t0+r10*2+400*0], m0 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + mova [t0+r10*2+400*2], m4 + pslld m4, m2, 3 + mova [t0+r10*2+400*4], m5 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + pmaddwd m2, m0, m0 ; b * b + paddd m5, m3 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m11 ; p * s + pmulld m5, m11 + pmaddwd m0, m12 ; b * 455 + pmaddwd m1, m12 + paddusw m4, m12 + paddusw m5, m12 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r14+m3*4], m4 + psrad m4, m5, 20 + vpgatherdd m3, [r14+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + paddd m0, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m13 + pand m0, m14 + pand m1, m14 + por m0, m2 ; a | (b << 12) + por m1, m3 + mova [t3+r10*4+ 8], xm0 + vextracti128 [t3+r10*4+40], m0, 1 + mova [t3+r10*4+24], xm1 + vextracti128 [t3+r10*4+56], m1, 1 + add r10, 16 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.v: ; vertical boxsum + ab + lea r10, [wq-2] +.v_loop: + mova m1, [t1+r10*2+400*0] + paddw m1, m1 + paddw m1, [t2+r10*2+400*0] ; hv sum + mova m2, [t1+r10*2+400*2] + mova m3, [t1+r10*2+400*4] + paddd m2, m2 + paddd m3, m3 + paddd m2, [t2+r10*2+400*2] ; hv sumsq + paddd m3, [t2+r10*2+400*4] + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + pmaddwd m2, m0, m0 ; b * b + paddd m5, m3 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + pmulld m4, m11 ; p * s + pmulld m5, m11 + pmaddwd m0, m12 ; b * 455 + pmaddwd m1, m12 + paddusw m4, m12 + paddusw m5, m12 + psrad m3, m4, 20 ; min(z, 255) - 256 + vpgatherdd m2, [r14+m3*4], m4 + psrad m4, m5, 20 + vpgatherdd m3, [r14+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + paddd m0, m13 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m13 + pand m0, m14 + pand m1, m14 + por m0, m2 ; a | (b << 12) + por m1, m3 + mova [t3+r10*4+ 8], xm0 + vextracti128 [t3+r10*4+40], m0, 1 + mova [t3+r10*4+24], xm1 + vextracti128 [t3+r10*4+56], m1, 1 + add r10, 16 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq + mov t4, t3 + add t3, 400*4 +.prep_n_loop: + mova m2, [t5+r10*4+0] + mova m3, [t4+r10*4+0] + paddd m2, [t5+r10*4+8] + paddd m3, [t4+r10*4+8] + paddd m0, m2, [t5+r10*4+4] + paddd m1, m3, [t4+r10*4+4] + pslld m0, 2 + paddd m1, m1 ; ab[ 0] 222 + psubd m0, m2 ; ab[-1] 343 + mova [t3+r10*4+400*4], m1 + paddd m1, m1 + mova [t5+r10*4], m0 + psubd m1, m3 ; ab[ 0] 343 + mova [t4+r10*4], m1 + add r10, 8 + jl .prep_n_loop + ret +; a+b are packed together in a single dword, but we can't do the +; full neighbor calculations before splitting them since we don't +; have sufficient precision. The solution is to do the calculations +; in two equal halves and split a and b before doing the final sum. +ALIGN function_align +.n: ; neighbor + output + mov r10, wq +.n_loop: + mova m4, [t3+r10*4+ 0] + paddd m4, [t3+r10*4+ 8] + paddd m5, m4, [t3+r10*4+ 4] + paddd m5, m5 ; ab[+1] 222 + mova m2, [t3+r10*4+400*4+ 0] + paddd m0, m2, [t5+r10*4+ 0] ; ab[ 0] 222 + ab[-1] 343 + mova m3, [t3+r10*4+400*4+32] + paddd m1, m3, [t5+r10*4+32] + mova [t3+r10*4+400*4+ 0], m5 + paddd m5, m5 + psubd m5, m4 ; ab[+1] 343 + mova [t5+r10*4+ 0], m5 + paddd m2, m5 ; ab[ 0] 222 + ab[+1] 343 + mova m4, [t3+r10*4+32] + paddd m4, [t3+r10*4+40] + paddd m5, m4, [t3+r10*4+36] + paddd m5, m5 + mova [t3+r10*4+400*4+32], m5 + paddd m5, m5 + psubd m5, m4 + mova [t5+r10*4+32], m5 + pandn m4, m14, m0 + psrld m0, 12 + paddd m3, m5 + pandn m5, m14, m2 + psrld m2, 12 + paddd m4, m5 ; a + pandn m5, m14, m1 + psrld m1, 12 + paddd m0, m2 ; b + (1 << 8) + pandn m2, m14, m3 + psrld m3, 12 + paddd m5, m2 + pmovzxbd m2, [dstq+r10+0] + paddd m1, m3 + pmovzxbd m3, [dstq+r10+8] + pmaddwd m4, m2 ; a * src + pmaddwd m5, m3 + packssdw m2, m3 + psubd m0, m4 ; b - a * src + (1 << 8) + psubd m1, m5 + psrad m0, 9 + psrad m1, 9 + packssdw m0, m1 + pmulhrsw m0, m7 + paddw m0, m2 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + pshufd xm0, xm0, q3120 + mova [dstq+r10], xm0 + add r10, 16 + jl .n_loop + mov r10, t5 + mov t5, t4 + mov t4, r10 + add dstq, strideq + ret + +cglobal sgr_filter_mix_8bpc, 4, 13, 16, 400*56+8, dst, stride, left, lpf, \ + w, h, edge, params +%define base r12-sgr_x_by_x_avx2-256*4 + lea r12, [sgr_x_by_x_avx2+256*4] + mov paramsq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + vbroadcasti128 m9, [base+sgr_shuf+0] + vbroadcasti128 m10, [base+sgr_shuf+8] + add lpfq, wq + vbroadcasti128 m11, [base+sgr_shuf+2] + vbroadcasti128 m12, [base+sgr_shuf+6] + add dstq, wq + vpbroadcastd m15, [paramsq+8] ; w0 w1 + lea t3, [rsp+wq*4+400*24+8] + vpbroadcastd m13, [paramsq+0] ; s0 + pxor m7, m7 + vpbroadcastd m14, [paramsq+4] ; s1 + lea t1, [rsp+wq*2+12] + neg wq + psllw m15, 2 ; to reuse existing pd_m4096 register for rounding + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).top_fixup + add t1, 400*12 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+400*12] + lea r10, [wq-2] +.top_fixup_loop: + mova m0, [t1+r10*2+400* 0] + mova m1, [t1+r10*2+400* 2] + mova m2, [t1+r10*2+400* 4] + paddw m0, m0 + mova m3, [t1+r10*2+400* 6] + paddd m1, m1 + mova m4, [t1+r10*2+400* 8] + paddd m2, m2 + mova m5, [t1+r10*2+400*10] + mova [t2+r10*2+400* 0], m0 + mova [t2+r10*2+400* 2], m1 + mova [t2+r10*2+400* 4], m2 + mova [t2+r10*2+400* 6], m3 + mova [t2+r10*2+400* 8], m4 + mova [t2+r10*2+400*10], m5 + add r10, 16 + jl .top_fixup_loop + call .v0 + jmp .main +.h: ; horizontal boxsums + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .h_main +.h_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu xm5, [lpfq+r10-2] +.h_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -18 + jl .h_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right +.h_have_right: + pshufb m6, m5, m9 + pshufb m4, m5, m10 + paddw m8, m6, m4 + shufps m0, m6, m4, q2121 + pmullw m3, m0, m0 + pshufb m2, m5, m11 + paddw m0, m2 + pshufb m5, m12 + paddw m0, m5 ; sum3 + punpcklwd m1, m2, m5 + pmaddwd m1, m1 + punpckhwd m2, m5 + pmaddwd m2, m2 + punpcklwd m5, m6, m4 + pmaddwd m5, m5 + punpckhwd m6, m4 + pmaddwd m6, m6 + punpcklwd m4, m3, m7 + paddd m1, m4 ; sumsq3 + punpckhwd m3, m7 + paddd m2, m3 + mova [t1+r10*2+400* 6], m0 + mova [t1+r10*2+400* 8], m1 + mova [t1+r10*2+400*10], m2 + paddw m8, m0 ; sum5 + paddd m5, m1 ; sumsq5 + paddd m6, m2 + mova [t1+r10*2+400* 0], m8 + mova [t1+r10*2+400* 2], m5 + mova [t1+r10*2+400* 4], m6 + add r10, 16 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsums + vertical boxsum3 + ab3 (even rows) + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .hv0_main +.hv0_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu xm5, [lpfq+r10-2] +.hv0_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -18 + jl .hv0_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right +.hv0_have_right: + pshufb m6, m5, m9 + pshufb m4, m5, m10 + paddw m8, m6, m4 + shufps m1, m6, m4, q2121 + pmullw m0, m1, m1 + pshufb m3, m5, m11 + paddw m1, m3 + pshufb m5, m12 + paddw m1, m5 ; sum3 + punpcklwd m2, m3, m5 + pmaddwd m2, m2 + punpckhwd m3, m5 + pmaddwd m3, m3 + punpcklwd m5, m6, m4 + pmaddwd m5, m5 + punpckhwd m6, m4 + pmaddwd m6, m6 + punpcklwd m4, m0, m7 + paddd m2, m4 ; sumsq3 + punpckhwd m0, m7 + paddd m3, m0 + paddw m8, m1 ; sum5 + paddd m5, m2 ; sumsq5 + paddd m6, m3 + mova [t3+r10*4+400*8+ 8], m8 ; we need a clean copy of the last row + mova [t3+r10*4+400*0+ 8], m5 ; in case height is odd + mova [t3+r10*4+400*0+40], m6 + paddw m8, [t1+r10*2+400* 0] + paddd m5, [t1+r10*2+400* 2] + paddd m6, [t1+r10*2+400* 4] + mova [t1+r10*2+400* 0], m8 + mova [t1+r10*2+400* 2], m5 + mova [t1+r10*2+400* 4], m6 + paddw m0, m1, [t1+r10*2+400* 6] + paddd m4, m2, [t1+r10*2+400* 8] + paddd m5, m3, [t1+r10*2+400*10] + mova [t1+r10*2+400* 6], m1 + mova [t1+r10*2+400* 8], m2 + mova [t1+r10*2+400*10], m3 + paddw m1, m0, [t2+r10*2+400* 6] + paddd m2, m4, [t2+r10*2+400* 8] + paddd m3, m5, [t2+r10*2+400*10] + mova [t2+r10*2+400* 6], m0 + mova [t2+r10*2+400* 8], m4 + mova [t2+r10*2+400*10], m5 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + pmaddwd m2, m0, m0 ; b3 * b + paddd m5, m3 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + vpbroadcastd m2, [base+pd_0xf00801c7] + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m2 ; b3 * 455 + pmaddwd m1, m2 + paddusw m4, m2 + paddusw m5, m2 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r12+m3*4], m4 + psrad m4, m5, 20 + vpgatherdd m3, [r12+m4*4], m5 + vpbroadcastd m4, [base+pd_34816] + pmulld m0, m2 + vpbroadcastd m5, [base+pd_m4096] + pmulld m1, m3 + paddd m0, m4 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m4 + pand m0, m5 + pand m1, m5 + por m0, m2 ; a3 | (b3 << 12) + por m1, m3 + mova [t3+r10*4+400*4+ 8], xm0 + vextracti128 [t3+r10*4+400*4+40], m0, 1 + mova [t3+r10*4+400*4+24], xm1 + vextracti128 [t3+r10*4+400*4+56], m1, 1 + add r10, 16 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + vpbroadcastd xm0, [leftq] + mova xm5, [lpfq+wq] + palignr xm5, xm0, 12 + add leftq, 4 + jmp .hv1_main +.hv1_extend_left: + mova xm5, [lpfq+wq] + pshufb xm5, [base+sgr_l_shuf] + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu xm5, [lpfq+r10-2] +.hv1_main: + vinserti128 m5, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -18 + jl .hv1_have_right + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right +.hv1_have_right: + pshufb m6, m5, m9 + pshufb m3, m5, m10 + paddw m8, m6, m3 + shufps m2, m6, m3, q2121 + pmullw m1, m2, m2 + pshufb m0, m5, m11 + paddw m2, m0 + pshufb m5, m12 + paddw m2, m5 ; sum3 + punpcklwd m4, m5, m0 + pmaddwd m4, m4 + punpckhwd m5, m0 + pmaddwd m5, m5 + punpcklwd m0, m6, m3 + pmaddwd m0, m0 + punpckhwd m6, m3 + pmaddwd m6, m6 + punpcklwd m3, m1, m7 + paddd m4, m3 ; sumsq3 + punpckhwd m1, m7 + paddd m5, m1 + paddw m1, m2, [t2+r10*2+400* 6] + mova [t2+r10*2+400* 6], m2 + paddw m8, m2 ; sum5 + paddd m2, m4, [t2+r10*2+400* 8] + paddd m3, m5, [t2+r10*2+400*10] + mova [t2+r10*2+400* 8], m4 + mova [t2+r10*2+400*10], m5 + paddd m4, m0 ; sumsq5 + paddd m5, m6 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pslld m6, m2, 3 + pslld m7, m3, 3 + paddd m6, m2 ; a3 * 9 + pmaddwd m2, m0, m0 ; b3 * b3 + paddd m7, m3 + pmaddwd m3, m1, m1 + psubd m6, m2 ; p3 + vpbroadcastd m2, [base+pd_0xf00801c7] + psubd m7, m3 + pmulld m6, m14 ; p3 * s1 + pmulld m7, m14 + pmaddwd m0, m2 ; b3 * 455 + pmaddwd m1, m2 + paddusw m6, m2 + paddusw m7, m2 + psrad m3, m6, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r12+m3*4], m6 + psrad m6, m7, 20 + vpgatherdd m3, [r12+m6*4], m7 + vpbroadcastd m6, [base+pd_34816] ; x3 + pmulld m0, m2 + vpbroadcastd m7, [base+pd_m4096] + pmulld m1, m3 + paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m6 + pand m0, m7 + pand m7, m1 + por m0, m2 ; a3 | (b3 << 12) + por m7, m3 + paddw m1, m8, [t2+r10*2+400*0] + paddd m2, m4, [t2+r10*2+400*2] + paddd m3, m5, [t2+r10*2+400*4] + paddw m1, [t1+r10*2+400*0] + paddd m2, [t1+r10*2+400*2] + paddd m3, [t1+r10*2+400*4] + mova [t2+r10*2+400*0], m8 + mova [t2+r10*2+400*2], m4 + mova [t2+r10*2+400*4], m5 + mova [t3+r10*4+400*8+ 8], xm0 + vextracti128 [t3+r10*4+400*8+40], m0, 1 + mova [t3+r10*4+400*8+24], xm7 + vextracti128 [t3+r10*4+400*8+56], m7, 1 + vpbroadcastd m4, [base+pd_25] + pxor m7, m7 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 + pmulld m2, m4 ; a5 * 25 + pmulld m3, m4 + pmaddwd m4, m0, m0 ; b5 * b5 + pmaddwd m5, m1, m1 + psubd m2, m4 ; p5 + vpbroadcastd m4, [base+pd_0xf00800a4] + psubd m3, m5 + pmulld m2, m13 ; p5 * s0 + pmulld m3, m13 + pmaddwd m0, m4 ; b5 * 164 + pmaddwd m1, m4 + paddusw m2, m4 + paddusw m3, m4 + psrad m5, m2, 20 ; min(z5, 255) - 256 + vpgatherdd m4, [r12+m5*4], m2 ; x5 + psrad m2, m3, 20 + vpgatherdd m5, [r12+m2*4], m3 + pmulld m0, m4 + pmulld m1, m5 + paddd m0, m6 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m6 + vpbroadcastd m6, [base+pd_m4096] + pand m0, m6 + pand m1, m6 + por m0, m4 ; a5 | (b5 << 12) + por m1, m5 + mova [t3+r10*4+400*0+ 8], xm0 + vextracti128 [t3+r10*4+400*0+40], m0, 1 + mova [t3+r10*4+400*0+24], xm1 + vextracti128 [t3+r10*4+400*0+56], m1, 1 + add r10, 16 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) + lea r10, [wq-2] + vpbroadcastd m6, [base+pd_34816] + vpbroadcastd m8, [base+pd_m4096] +.v0_loop: + mova m0, [t1+r10*2+400* 6] + mova m4, [t1+r10*2+400* 8] + mova m5, [t1+r10*2+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+r10*2+400* 6] + paddd m2, m4, [t2+r10*2+400* 8] + paddd m3, m5, [t2+r10*2+400*10] + mova [t2+r10*2+400* 6], m0 + mova [t2+r10*2+400* 8], m4 + mova [t2+r10*2+400*10], m5 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + pmaddwd m2, m0, m0 ; b3 * b3 + paddd m5, m3 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + vpbroadcastd m2, [base+pd_0xf00801c7] + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m2 ; b3 * 455 + pmaddwd m1, m2 + paddusw m4, m2 + paddusw m5, m2 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r12+m3*4], m4 ; x3 + psrad m4, m5, 20 + vpgatherdd m3, [r12+m4*4], m5 + pmulld m0, m2 + pmulld m1, m3 + paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m6 + pand m0, m8 + pand m1, m8 + por m0, m2 ; a3 | (b3 << 12) + por m1, m3 + mova m2, [t1+r10*2+400*0] + mova m3, [t1+r10*2+400*2] + mova m4, [t1+r10*2+400*4] + mova [t3+r10*4+400*8+ 8], m2 + mova [t3+r10*4+400*0+ 8], m3 + mova [t3+r10*4+400*0+40], m4 + paddw m2, m2 ; cc5 + paddd m3, m3 + paddd m4, m4 + mova [t1+r10*2+400*0], m2 + mova [t1+r10*2+400*2], m3 + mova [t1+r10*2+400*4], m4 + mova [t3+r10*4+400*4+ 8], xm0 + vextracti128 [t3+r10*4+400*4+40], m0, 1 + mova [t3+r10*4+400*4+24], xm1 + vextracti128 [t3+r10*4+400*4+56], m1, 1 + add r10, 16 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-2] +.v1_loop: + mova m4, [t1+r10*2+400* 6] + mova m5, [t1+r10*2+400* 8] + mova m6, [t1+r10*2+400*10] + paddw m1, m4, [t2+r10*2+400* 6] + paddd m2, m5, [t2+r10*2+400* 8] + paddd m3, m6, [t2+r10*2+400*10] + mova [t2+r10*2+400* 6], m4 + mova [t2+r10*2+400* 8], m5 + mova [t2+r10*2+400*10], m6 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + pmaddwd m2, m0, m0 ; b3 * b3 + paddd m5, m3 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + vpbroadcastd m2, [base+pd_0xf00801c7] + psubd m5, m3 + pmulld m4, m14 ; p3 * s1 + pmulld m5, m14 + pmaddwd m0, m2 ; b3 * 455 + pmaddwd m1, m2 + paddusw m4, m2 + paddusw m5, m2 + psrad m3, m4, 20 ; min(z3, 255) - 256 + vpgatherdd m2, [r12+m3*4], m4 ; x3 + psrad m4, m5, 20 + vpgatherdd m3, [r12+m4*4], m5 + vpbroadcastd m4, [base+pd_34816] + pmulld m0, m2 + vpbroadcastd m8, [base+pd_m4096] + pmulld m1, m3 + paddd m0, m4 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m4 + pand m0, m8 + pand m8, m1 + por m0, m2 ; a3 | (b3 << 12) + por m8, m3 + mova m4, [t3+r10*4+400*8+ 8] + mova m5, [t3+r10*4+400*0+ 8] + mova m6, [t3+r10*4+400*0+40] + paddw m1, m4, [t2+r10*2+400*0] + paddd m2, m5, [t2+r10*2+400*2] + paddd m3, m6, [t2+r10*2+400*4] + paddw m1, [t1+r10*2+400*0] + paddd m2, [t1+r10*2+400*2] + paddd m3, [t1+r10*2+400*4] + mova [t2+r10*2+400*0], m4 + mova [t2+r10*2+400*2], m5 + mova [t2+r10*2+400*4], m6 + vpbroadcastd m4, [base+pd_25] + mova [t3+r10*4+400*8+ 8], xm0 + vextracti128 [t3+r10*4+400*8+40], m0, 1 + mova [t3+r10*4+400*8+24], xm8 + vextracti128 [t3+r10*4+400*8+56], m8, 1 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 + pmulld m2, m4 ; a5 * 25 + pmulld m3, m4 + pmaddwd m4, m0, m0 ; b5 * b5 + pmaddwd m5, m1, m1 + psubd m2, m4 ; p5 + vpbroadcastd m4, [base+pd_0xf00800a4] + psubd m3, m5 + pmulld m2, m13 ; p5 * s0 + pmulld m3, m13 + pmaddwd m0, m4 ; b5 * 164 + pmaddwd m1, m4 + paddusw m2, m4 + paddusw m3, m4 + psrad m5, m2, 20 ; min(z5, 255) - 256 + vpgatherdd m4, [r12+m5*4], m2 ; x5 + psrad m2, m3, 20 + vpgatherdd m5, [r12+m2*4], m3 + pmulld m0, m4 + vpbroadcastd m6, [base+pd_34816] + pmulld m1, m5 + paddd m0, m6 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m6 + vpbroadcastd m6, [base+pd_m4096] + pand m0, m6 + pand m1, m6 + por m0, m4 ; a5 | (b5 << 12) + por m1, m5 + mova [t3+r10*4+400*0+ 8], xm0 + vextracti128 [t3+r10*4+400*0+40], m0, 1 + mova [t3+r10*4+400*0+24], xm1 + vextracti128 [t3+r10*4+400*0+56], m1, 1 + add r10, 16 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t3+r10*4+400*0+4] + paddd m1, m0, [t3+r10*4+400*0+0] + mova m4, [t3+r10*4+400*4+0] + paddd m1, [t3+r10*4+400*0+8] + mova m5, [t3+r10*4+400*8+0] + paddd m4, [t3+r10*4+400*4+8] + paddd m5, [t3+r10*4+400*8+8] + paddd m2, m4, [t3+r10*4+400*4+4] + paddd m3, m5, [t3+r10*4+400*8+4] + paddd m0, m1 + pslld m1, 2 + pslld m2, 2 + paddd m1, m0 ; ab5 565 + paddd m3, m3 ; ab3[ 0] 222 + psubd m2, m4 ; ab3[-1] 343 + mova [t3+r10*4+400*20], m3 + pandn m0, m6, m1 ; a5 565 + mova [t3+r10*4+400*24], m2 + psrld m1, 12 ; b5 565 + mova [t3+r10*4+400*12], m0 + paddd m3, m3 + mova [t3+r10*4+400*16], m1 + psubd m3, m5 ; ab3[ 0] 343 + mova [t3+r10*4+400*28], m3 + add r10, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m0, [t3+r10*4+4] + paddd m4, m0, [t3+r10*4+0] + paddd m4, [t3+r10*4+8] + paddd m0, m4 + pslld m4, 2 + paddd m4, m0 + pandn m0, m6, m4 + psrld m4, 12 + paddd m2, m0, [t3+r10*4+400*12] ; a5 + mova [t3+r10*4+400*12], m0 + paddd m0, m4, [t3+r10*4+400*16] ; b5 + (1 << 8) + mova [t3+r10*4+400*16], m4 + mova m3, [t3+r10*4+400*4+0] + paddd m3, [t3+r10*4+400*4+8] + paddd m5, m3, [t3+r10*4+400*4+4] + paddd m5, m5 ; ab3[ 1] 222 + mova m4, [t3+r10*4+400*20] + paddd m1, m4, [t3+r10*4+400*24] ; ab3[ 0] 222 + ab3[-1] 343 + mova [t3+r10*4+400*20], m5 + paddd m5, m5 + psubd m5, m3 ; ab3[ 1] 343 + mova [t3+r10*4+400*24], m5 + paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343 + pandn m3, m6, m1 + psrld m1, 12 + pandn m5, m6, m4 + psrld m4, 12 + paddd m3, m5 ; a3 + paddd m1, m4 ; b3 + (1 << 8) + pmovzxbd m4, [dstq+r10] + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) + psrld m0, 9 + pslld m1, 7 + pblendw m0, m1, 0xaa + pmaddwd m0, m15 + psubd m0, m6 + psrad m0, 13 + paddd m0, m4 + vextracti128 xm1, m0, 1 + packssdw xm0, xm1 + packuswb xm0, xm0 + movq [dstq+r10], xm0 + add r10, 8 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m3, [t3+r10*4+400*8+0] + paddd m3, [t3+r10*4+400*8+8] + paddd m5, m3, [t3+r10*4+400*8+4] + paddd m5, m5 ; ab3[ 1] 222 + mova m4, [t3+r10*4+400*20] + paddd m1, m4, [t3+r10*4+400*28] ; ab3[ 0] 222 + ab3[-1] 343 + mova [t3+r10*4+400*20], m5 + paddd m5, m5 + psubd m5, m3 ; ab3[ 1] 343 + mova [t3+r10*4+400*28], m5 + paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343 + pandn m3, m6, m1 + psrld m1, 12 + pandn m5, m6, m4 + psrld m4, 12 + paddd m3, m5 ; -a3 + paddd m1, m4 ; b3 + (1 << 8) + pmovzxbd m4, [dstq+r10] + pmaddwd m2, m4, [t3+r10*4+400*12] ; -a5 * src + mova m0, [t3+r10*4+400*16] ; b5 + (1 << 7) + pmaddwd m3, m4 ; -a3 * src + psubd m0, m2 ; a5 * src + b5 + (1 << 7) + psubd m1, m3 ; a3 * src + b3 + (1 << 8) + psrld m0, 8 + pslld m1, 7 + pblendw m0, m1, 0xaa + pmaddwd m0, m15 + psubd m0, m6 + psrad m0, 13 + paddd m0, m4 + vextracti128 xm1, m0, 1 + packssdw xm0, xm1 + packuswb xm0, xm0 + movq [dstq+r10], xm0 + add r10, 8 + jl .n1_loop + add dstq, strideq + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/looprestoration_avx512.asm b/third_party/dav1d/src/x86/looprestoration_avx512.asm new file mode 100644 index 0000000000..1e571774ca --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration_avx512.asm @@ -0,0 +1,2122 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +wiener_shufA: db 1, 2, 7, 6, 3, 4, 9, 8, 5, 6, 11, 10, 7, 8, 13, 12 +wiener_shufB: db 2, 3, 8, 7, 4, 5, 10, 9, 6, 7, 12, 11, 8, 9, 14, 13 +wiener_shufC: db 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11 +wiener_shufD: db 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 +wiener_perm32: db 1, 9, 3, 11, 5, 13, 7, 15, 33, 41, 35, 43, 37, 45, 39, 47 + db 17, 25, 19, 27, 21, 29, 23, 31, 49, 57, 51, 59, 53, 61, 55, 63 +sgr_shuf: db 128, 1, -1, 2,132, 3, -1, 4,136, 5, -1, 6,140, 7, -1, 8 + db 129, 9, -1, 10,133, 11, -1, 12,137, -1, -1, -1,141, -1, 0,128 +sgr_mix_perm: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55 +r_ext_mask: times 68 db -1 + times 4 db 0 +wiener_x_shuf: db 0, 2, -1, 0 +wiener_x_add: db 0, 1,127, 0 + +pw_61448: times 2 dw 61448 +pw_164_455: dw 164, 455 +pd_m16380: dd -16380 +pd_m4096: dd -4096 +pd_m25 dd -25 +pd_m9: dd -9 +pd_34816: dd 34816 +pd_8421376: dd 8421376 + +cextern sgr_x_by_x + +SECTION .text + +DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers + +INIT_ZMM avx512icl +cglobal wiener_filter7_8bpc, 4, 15, 20, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt + mov fltq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + vbroadcasti32x4 m6, [wiener_shufA] + vbroadcasti32x4 m7, [wiener_shufB] + mov r10d, 0xfffe + vbroadcasti32x4 m8, [wiener_shufC] + vbroadcasti32x4 m9, [wiener_shufD] + kmovw k1, r10d + vpbroadcastd m0, [wiener_x_shuf] + vpbroadcastd m1, [wiener_x_add] + mov r10, 0xaaaaaaaaaaaaaaaa + vpbroadcastd m11, [fltq+ 0] + vpbroadcastd m12, [fltq+ 4] + kmovq k2, r10 + vpbroadcastd m10, [pd_m16380] + packsswb m11, m11 ; x0 x1 x0 x1 + vpbroadcastd m14, [fltq+16] + pshufb m12, m0 + vpbroadcastd m15, [fltq+20] + paddb m12, m1 ; x2 x3+1 x2 127 + vpbroadcastd m13, [pd_8421376] + psllw m14, 5 ; y0 y1 + psllw m15, 5 ; y2 y3 + cmp wd, 32 ; the minimum lr unit size for chroma in 4:2:0 is 32 + jle .w32 ; pixels, so we need a special case for small widths + lea t1, [rsp+wq*2+16] + add lpfq, wq + add dstq, wq + neg wq + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 384*2 + add r10, strideq + mov [rsp], r10 ; below + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call .v + RET +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call .v +.v2: + call .v + jmp .v1 +.h: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm16, [leftq] + vmovdqu32 m16{k1}, [lpfq+r10-4] + add leftq, 4 + jmp .h_main +.h_extend_left: + vpbroadcastb xm16, [lpfq+r10] ; the masked load ensures that no exception + vmovdqu32 m16{k1}, [lpfq+r10-4] ; gets raised from accessing invalid memory + jmp .h_main +.h_top: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m16, [lpfq+r10-4] +.h_main: + movu m17, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -66 + jl .h_have_right + push r0 + lea r0, [r_ext_mask+65] + vpbroadcastb m0, [lpfq-1] + vpternlogd m16, m0, [r0+r10+0], 0xe4 ; c ? a : b + vpternlogd m17, m0, [r0+r10+8], 0xe4 + pop r0 +.h_have_right: + pshufb m4, m16, m6 + mova m0, m10 + vpdpbusd m0, m4, m11 + pshufb m4, m16, m7 + mova m2, m10 + vpdpbusd m2, m4, m11 + pshufb m4, m17, m6 + mova m1, m10 + vpdpbusd m1, m4, m11 + pshufb m4, m17, m7 + mova m3, m10 + vpdpbusd m3, m4, m11 + pshufb m4, m16, m8 + vpdpbusd m0, m4, m12 + pshufb m16, m9 + vpdpbusd m2, m16, m12 + pshufb m4, m17, m8 + vpdpbusd m1, m4, m12 + pshufb m17, m9 + vpdpbusd m3, m17, m12 + packssdw m0, m2 + packssdw m1, m3 + psraw m0, 3 + psraw m1, 3 + mova [t1+r10*2+ 0], m0 + mova [t1+r10*2+64], m1 + add r10, 64 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm16, [leftq] + vmovdqu32 m16{k1}, [lpfq+r10-4] + add leftq, 4 + jmp .hv_main +.hv_extend_left: + vpbroadcastb xm16, [lpfq+r10] + vmovdqu32 m16{k1}, [lpfq+r10-4] + jmp .hv_main +.hv_bottom: + mov r10, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m16, [lpfq+r10-4] +.hv_main: + movu m17, [lpfq+r10+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -66 + jl .hv_have_right + push r0 + lea r0, [r_ext_mask+65] + vpbroadcastb m0, [lpfq-1] + vpternlogd m16, m0, [r0+r10+0], 0xe4 ; c ? a : b + vpternlogd m17, m0, [r0+r10+8], 0xe4 + pop r0 +.hv_have_right: + pshufb m4, m16, m6 + mova m0, m10 + vpdpbusd m0, m4, m11 + pshufb m4, m16, m7 + mova m2, m10 + vpdpbusd m2, m4, m11 + pshufb m4, m17, m6 + mova m1, m10 + vpdpbusd m1, m4, m11 + pshufb m4, m17, m7 + mova m3, m10 + vpdpbusd m3, m4, m11 + pshufb m4, m16, m8 + vpdpbusd m0, m4, m12 + pshufb m16, m9 + vpdpbusd m2, m16, m12 + pshufb m4, m17, m8 + vpdpbusd m1, m4, m12 + pshufb m17, m9 + vpdpbusd m3, m17, m12 + packssdw m0, m2 + packssdw m1, m3 + psraw m0, 3 + psraw m1, 3 + mova m16, [t4+r10*2] + paddw m16, [t2+r10*2] + mova m3, [t3+r10*2] + mova m17, [t4+r10*2+64] + paddw m17, [t2+r10*2+64] + mova m5, [t3+r10*2+64] + punpcklwd m4, m16, m3 + mova m2, m13 + vpdpwssd m2, m4, m15 + punpcklwd m18, m17, m5 + mova m4, m13 + vpdpwssd m4, m18, m15 + punpckhwd m16, m3 + mova m3, m13 + vpdpwssd m3, m16, m15 + punpckhwd m17, m5 + mova m5, m13 + vpdpwssd m5, m17, m15 + mova m17, [t5+r10*2] + paddw m17, [t1+r10*2] + paddw m16, m0, [t6+r10*2] + mova m19, [t5+r10*2+64] + paddw m19, [t1+r10*2+64] + paddw m18, m1, [t6+r10*2+64] + mova [t0+r10*2+ 0], m0 + mova [t0+r10*2+64], m1 + punpcklwd m0, m16, m17 + vpdpwssd m2, m0, m14 + punpcklwd m1, m18, m19 + vpdpwssd m4, m1, m14 + punpckhwd m16, m17 + vpdpwssd m3, m16, m14 + punpckhwd m18, m19 + vpdpwssd m5, m18, m14 + packuswb m2, m4 + psrlw m2, 8 + vpackuswb m2{k2}, m3, m5 + movu [dstq+r10], m2 ; We don't have a separate 5-tap version so the 7-tap + add r10, 64 ; function is used for chroma as well, and in some + jl .hv_loop ; esoteric edge cases chroma dst pointers may only + mov t6, t5 ; have a 32-byte alignment despite having a width + mov t5, t4 ; larger than 32, so use an unaligned store here. + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 + add dstq, strideq + ret +.v: + mov r10, wq +.v_loop: + mova m4, [t4+r10*2+ 0] + paddw m4, [t2+r10*2+ 0] + mova m1, [t3+r10*2+ 0] + mova m5, [t4+r10*2+64] + paddw m5, [t2+r10*2+64] + mova m3, [t3+r10*2+64] + punpcklwd m6, m4, m1 + mova m0, m13 + vpdpwssd m0, m6, m15 + punpcklwd m6, m5, m3 + mova m2, m13 + vpdpwssd m2, m6, m15 + punpckhwd m4, m1 + mova m1, m13 + vpdpwssd m1, m4, m15 + punpckhwd m5, m3 + mova m3, m13 + vpdpwssd m3, m5, m15 + mova m5, [t1+r10*2+ 0] + paddw m4, m5, [t6+r10*2+ 0] + paddw m5, [t5+r10*2+ 0] + mova m7, [t1+r10*2+64] + paddw m6, m7, [t6+r10*2+64] + paddw m7, [t5+r10*2+64] + punpcklwd m8, m4, m5 + vpdpwssd m0, m8, m14 + punpcklwd m8, m6, m7 + vpdpwssd m2, m8, m14 + punpckhwd m4, m5 + vpdpwssd m1, m4, m14 + punpckhwd m6, m7 + vpdpwssd m3, m6, m14 + packuswb m0, m2 + psrlw m0, 8 + vpackuswb m0{k2}, m1, m3 + movu [dstq+r10], m0 + add r10, 64 + jl .v_loop + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq + ret +.w32: + lea r10, [r_ext_mask+73] + mova ym18, [wiener_perm32] + lea t1, [rsp+16] + sub r10, wq + test edgeb, 4 ; LR_HAVE_TOP + jz .w32_no_top + call .w32_h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 32*2 + call .w32_h_top + lea r9, [lpfq+strideq*4] + mov lpfq, dstq + mov t4, t1 + add t1, 32*2 + add r9, strideq + mov [rsp], r9 ; below + call .w32_h + mov t3, t1 + mov t2, t1 + dec hd + jz .w32_v1 + add lpfq, strideq + add t1, 32*2 + call .w32_h + mov t2, t1 + dec hd + jz .w32_v2 + add lpfq, strideq + add t1, 32*2 + call .w32_h + dec hd + jz .w32_v3 +.w32_main: + lea t0, [t1+32*2] +.w32_main_loop: + call .w32_hv + dec hd + jnz .w32_main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .w32_v3 + mov lpfq, [rsp] + call .w32_hv_bottom + add lpfq, strideq + call .w32_hv_bottom +.w32_v1: + call .w32_v + RET +.w32_no_top: + lea r9, [lpfq+strideq*4] + mov lpfq, dstq + lea r9, [r9+strideq*2] + mov [rsp], r9 + call .w32_h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .w32_v1 + add lpfq, strideq + add t1, 32*2 + call .w32_h + mov t2, t1 + dec hd + jz .w32_v2 + add lpfq, strideq + add t1, 32*2 + call .w32_h + dec hd + jz .w32_v3 + lea t0, [t1+32*2] + call .w32_hv + dec hd + jz .w32_v3 + add t0, 32*8 + call .w32_hv + dec hd + jnz .w32_main +.w32_v3: + call .w32_v +.w32_v2: + call .w32_v + jmp .w32_v1 +.w32_h: + test edgeb, 1 ; LR_HAVE_LEFT + jz .w32_h_extend_left + movd xm16, [leftq] + vmovdqu32 ym16{k1}, [lpfq-4] + add leftq, 4 + jmp .w32_h_main +.w32_h_extend_left: + vpbroadcastb xm16, [lpfq] ; the masked load ensures that no exception + vmovdqu32 ym16{k1}, [lpfq-4] ; gets raised from accessing invalid memory + jmp .w32_h_main +.w32_h_top: + test edgeb, 1 ; LR_HAVE_LEFT + jz .w32_h_extend_left + movu ym16, [lpfq-4] +.w32_h_main: + vinserti32x8 m16, [lpfq+4], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .w32_h_have_right + vpbroadcastb m0, [lpfq+wq-1] + movu ym17, [r10-8] + vinserti32x8 m17, [r10+0], 1 + vpternlogd m16, m0, m17, 0xe4 ; c ? a : b +.w32_h_have_right: + pshufb m2, m16, m6 + mova m0, m10 + vpdpbusd m0, m2, m11 + pshufb m2, m16, m7 + mova m1, m10 + vpdpbusd m1, m2, m11 + pshufb m2, m16, m8 + vpdpbusd m0, m2, m12 + pshufb m16, m9 + vpdpbusd m1, m16, m12 + packssdw m0, m1 + psraw m0, 3 + mova [t1], m0 + ret +.w32_hv: + add lpfq, strideq + test edgeb, 1 ; LR_HAVE_LEFT + jz .w32_hv_extend_left + movd xm16, [leftq] + vmovdqu32 ym16{k1}, [lpfq-4] + add leftq, 4 + jmp .w32_hv_main +.w32_hv_extend_left: + vpbroadcastb xm16, [lpfq] + vmovdqu32 ym16{k1}, [lpfq-4] + jmp .w32_hv_main +.w32_hv_bottom: + test edgeb, 1 ; LR_HAVE_LEFT + jz .w32_hv_extend_left + movu ym16, [lpfq-4] +.w32_hv_main: + vinserti32x8 m16, [lpfq+4], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .w32_hv_have_right + vpbroadcastb m0, [lpfq+wq-1] + movu ym17, [r10-8] + vinserti32x8 m17, [r10+0], 1 + vpternlogd m16, m0, m17, 0xe4 +.w32_hv_have_right: + mova m3, [t4] + paddw m3, [t2] + mova m2, [t3] + pshufb m4, m16, m6 + mova m0, m10 + vpdpbusd m0, m4, m11 + pshufb m4, m16, m7 + mova m5, m10 + vpdpbusd m5, m4, m11 + punpcklwd m4, m3, m2 + mova m1, m13 + vpdpwssd m1, m4, m15 + punpckhwd m3, m2 + mova m2, m13 + vpdpwssd m2, m3, m15 + pshufb m4, m16, m8 + vpdpbusd m0, m4, m12 + pshufb m16, m9 + vpdpbusd m5, m16, m12 + packssdw m0, m5 + psraw m0, 3 + mova m4, [t5] + paddw m4, [t1] + paddw m3, m0, [t6] + mova [t0], m0 + punpcklwd m0, m3, m4 + vpdpwssd m1, m0, m14 + punpckhwd m3, m4 + vpdpwssd m2, m3, m14 + packuswb m1, m2 + vpermb m16, m18, m1 + mova [dstq], ym16 + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 + add dstq, strideq + ret +.w32_v: + mova m2, [t4] + paddw m2, [t2] + mova m1, [t3] + mova m4, [t1] + paddw m3, m4, [t6] + paddw m4, [t5] + punpcklwd m5, m2, m1 + mova m0, m13 + vpdpwssd m0, m5, m15 + punpckhwd m2, m1 + mova m1, m13 + vpdpwssd m1, m2, m15 + punpcklwd m2, m3, m4 + vpdpwssd m0, m2, m14 + punpckhwd m3, m4 + vpdpwssd m1, m3, m14 + packuswb m0, m1 + vpermb m16, m18, m0 + mova [dstq], ym16 + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + add dstq, strideq + ret + +cglobal sgr_filter_5x5_8bpc, 4, 13, 23, 416*24+16, dst, stride, left, lpf, \ + w, h, edge, params + mov paramsq, r6mp + mov wd, wm + mov hd, hm + mov edged, r7m + vbroadcasti32x4 m5, [sgr_shuf+1] + add lpfq, wq + vbroadcasti32x4 m6, [sgr_shuf+9] + add dstq, wq + vbroadcasti32x4 m7, [sgr_shuf+3] + lea t3, [rsp+wq*4+16+416*12] + vbroadcasti32x4 m8, [sgr_shuf+7] + pxor m4, m4 + vpbroadcastd m9, [pd_m25] + vpsubd m11, m4, [paramsq+0] {1to16} ; -s0 + vpbroadcastw m15, [paramsq+8] ; w0 + lea t1, [rsp+wq*2+20] + vpbroadcastd m10, [pw_164_455] + neg wq + vpbroadcastd m12, [pw_61448] ; (15 << 12) + (1 << 3) + mov r10d, 0xfe + vpbroadcastd m13, [pd_m4096] + kmovb k1, r10d + vpbroadcastd m14, [pd_34816] ; (1 << 11) + (1 << 15) + mov r10, 0x3333333333333333 + mova m18, [sgr_x_by_x+64*0] + kmovq k2, r10 + mova m19, [sgr_x_by_x+64*1] + lea r12, [r_ext_mask+75] + mova m20, [sgr_x_by_x+64*2] + psllw m15, 4 + mova m21, [sgr_x_by_x+64*3] + lea r10, [lpfq+strideq*4] + mova ym22, [sgr_shuf] + add r10, strideq + mov [rsp], r10 ; below + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call .top_fixup + add t1, 416*6 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, strideq + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + test hd, hd + jz .odd_height + call .h + add lpfq, strideq + call .hv + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .h_top + add lpfq, strideq + call .hv_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + call .n0 + call .n1 +.odd_height_end: + call .v + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+416*6] + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + jmp .main +.no_top_height1: + call .v + call .prep_n + jmp .odd_height_end +.h: ; horizontal boxsum + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .h_main +.h_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu ym17, [lpfq+r10-2] +.h_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -34 + jl .h_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r12+r10-8] + vinserti32x8 m16, [r12+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.h_have_right: + pshufb m3, m17, m5 + pmullw m2, m3, m3 + pshufb m1, m17, m6 + paddw m0, m3, m1 + shufps m3, m1, q2121 + paddw m0, m3 + punpcklwd m16, m3, m1 + punpckhwd m3, m1 + punpcklwd m1, m2, m4 + vpdpwssd m1, m16, m16 + punpckhwd m2, m4 + vpdpwssd m2, m3, m3 + pshufb m16, m17, m7 + paddw m0, m16 + pshufb m17, m8 + paddw m0, m17 ; sum + punpcklwd m3, m16, m17 + vpdpwssd m1, m3, m3 ; sumsq + punpckhwd m16, m17 + vpdpwssd m2, m16, m16 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+r10*2+416*0] + paddd m1, [t1+r10*2+416*2] + paddd m2, [t1+r10*2+416*4] +.h_loop_end: + mova [t1+r10*2+416*0], m0 + mova [t1+r10*2+416*2], m1 + mova [t1+r10*2+416*4], m2 + add r10, 32 + jl .h_loop + ret +.top_fixup: + lea r10, [wq-2] +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+r10*2+416*0] + mova m1, [t1+r10*2+416*2] + mova m2, [t1+r10*2+416*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+r10*2+416*0], m0 + mova [t2+r10*2+416*2], m1 + mova [t2+r10*2+416*4], m2 + add r10, 32 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .hv_main +.hv_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .hv_main +.hv_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu ym17, [lpfq+r10-2] +.hv_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -34 + jl .hv_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r12+r10-8] + vinserti32x8 m16, [r12+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.hv_have_right: + pshufb m1, m17, m5 + pmullw m3, m1, m1 + pshufb m2, m17, m6 + paddw m0, m1, m2 + shufps m1, m2, q2121 + paddw m0, m1 + punpcklwd m16, m1, m2 + punpckhwd m1, m2 + punpcklwd m2, m3, m4 + vpdpwssd m2, m16, m16 + punpckhwd m3, m4 + vpdpwssd m3, m1, m1 + pshufb m16, m17, m7 + paddw m0, m16 + pshufb m17, m8 + paddw m0, m17 ; h sum + punpcklwd m1, m16, m17 + vpdpwssd m2, m1, m1 ; h sumsq + punpckhwd m16, m17 + vpdpwssd m3, m16, m16 + paddw m1, m0, [t1+r10*2+416*0] + paddd m16, m2, [t1+r10*2+416*2] + paddd m17, m3, [t1+r10*2+416*4] + test hd, hd + jz .hv_last_row +.hv_main2: + paddd m16, [t2+r10*2+416*2] ; hv sumsq + paddd m17, [t2+r10*2+416*4] + paddw m1, [t2+r10*2+416*0] ; hv sum + mova [t0+r10*2+416*2], m2 + mova [t0+r10*2+416*4], m3 + mova [t0+r10*2+416*0], m0 + pmulld m16, m9 ; -a * 25 + pmulld m17, m9 + punpcklwd m0, m1, m4 ; b + vpdpwssd m16, m0, m0 ; -p + punpckhwd m1, m4 + vpdpwssd m17, m1, m1 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + pmulld m16, m11 ; p * s + pmulld m17, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m14 + vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) + vpternlogd m17, m1, m13, 0xd8 + mova [t3+r10*4+ 8], m16 ; The neighbor calculations requires + mova [t3+r10*4+ 24], xm17 ; 13 bits for a and 21 bits for b. + vextracti32x4 [t3+r10*4+ 56], m17, 2 ; Packing them allows for 12+20, but + mova [t3+r10*4+ 72], m17 ; that gets us most of the way. + vextracti128 [t3+r10*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+104], m16, 3 + add r10, 32 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+r10*2+416*0], m1 + paddw m1, m0 + mova [t1+r10*2+416*2], m16 + paddd m16, m2 + mova [t1+r10*2+416*4], m17 + paddd m17, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab + lea r10, [wq-2] +.v_loop: + mova m2, [t1+r10*2+416*2] + paddd m16, m2, [t2+r10*2+416*2] + mova m3, [t1+r10*2+416*4] + paddd m17, m3, [t2+r10*2+416*4] + paddd m2, m2 + paddd m3, m3 + paddd m16, m2 ; hv sumsq + paddd m17, m3 + pmulld m16, m9 ; -a * 25 + pmulld m17, m9 + mova m0, [t1+r10*2+416*0] + paddw m1, m0, [t2+r10*2+416*0] + paddw m0, m0 + paddw m1, m0 ; hv sum + punpcklwd m0, m1, m4 ; b + vpdpwssd m16, m0, m0 ; -p + punpckhwd m1, m4 + vpdpwssd m17, m1, m1 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + pmulld m16, m11 ; p * s + pmulld m17, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m14 + vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) + vpternlogd m17, m1, m13, 0xd8 + mova [t3+r10*4+ 8], m16 + mova [t3+r10*4+ 24], xm17 + vextracti32x4 [t3+r10*4+ 56], m17, 2 + mova [t3+r10*4+ 72], m17 + vextracti128 [t3+r10*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+104], m16, 3 + add r10, 32 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t3+r10*4+ 4] + movu m1, [t3+r10*4+68] + paddd m2, m0, [t3+r10*4+ 0] + paddd m3, m1, [t3+r10*4+64] + paddd m2, [t3+r10*4+ 8] + paddd m3, [t3+r10*4+72] + paddd m0, m2 + pslld m2, 2 + paddd m1, m3 + pslld m3, 2 + paddd m2, m0 ; ab 565 + paddd m3, m1 + pandn m0, m13, m2 ; a + psrld m2, 12 ; b + pandn m1, m13, m3 + psrld m3, 12 + mova [t3+r10*4+416*4+ 0], m0 + mova [t3+r10*4+416*8+ 0], m2 + mova [t3+r10*4+416*4+64], m1 + mova [t3+r10*4+416*8+64], m3 + add r10, 32 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m16, [t3+r10*4+ 4] + movu m17, [t3+r10*4+68] + paddd m0, m16, [t3+r10*4+ 0] + paddd m1, m17, [t3+r10*4+64] + paddd m0, [t3+r10*4+ 8] + paddd m1, [t3+r10*4+72] + paddd m16, m0 + pslld m0, 2 + paddd m17, m1 + pslld m1, 2 + paddd m0, m16 + paddd m1, m17 + pandn m16, m13, m0 + psrld m0, 12 + pandn m17, m13, m1 + psrld m1, 12 + paddd m2, m16, [t3+r10*4+416*4+ 0] ; a + paddd m3, m17, [t3+r10*4+416*4+64] + mova [t3+r10*4+416*4+ 0], m16 + mova [t3+r10*4+416*4+64], m17 + paddd m16, m0, [t3+r10*4+416*8+ 0] ; b + (1 << 8) + paddd m17, m1, [t3+r10*4+416*8+64] + mova [t3+r10*4+416*8+ 0], m0 + mova [t3+r10*4+416*8+64], m1 + pmovzxbd m0, [dstq+r10+ 0] + pmovzxbd m1, [dstq+r10+16] + pmaddwd m2, m0 ; a * src + pmaddwd m3, m1 + packssdw m0, m1 + psubd m16, m2 ; b - a * src + (1 << 8) + psubd m17, m3 + psrad m16, 9 + psrad m17, 9 + packssdw m16, m17 + pmulhrsw m16, m15 + paddw m16, m0 + packuswb m16, m16 + vpermd m16, m22, m16 + mova [dstq+r10], ym16 + add r10, 32 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + pmovzxbd m0, [dstq+r10+ 0] + pmovzxbd m1, [dstq+r10+16] + pmaddwd m2, m0, [t3+r10*4+416*4+ 0] ; a * src + pmaddwd m3, m1, [t3+r10*4+416*4+64] + mova m16, [t3+r10*4+416*8+ 0] ; b + (1 << 7) + mova m17, [t3+r10*4+416*8+64] + packssdw m0, m1 + psubd m16, m2 ; b - a * src + (1 << 7) + psubd m17, m3 + psrad m16, 8 + psrad m17, 8 + packssdw m16, m17 + pmulhrsw m16, m15 + paddw m16, m0 + packuswb m16, m16 + vpermd m16, m22, m16 + mova [dstq+r10], ym16 + add r10, 32 + jl .n1_loop + add dstq, strideq + ret + +cglobal sgr_filter_3x3_8bpc, 4, 15, 22, -416*28-16, dst, stride, left, lpf, \ + w, h, edge, params + mov paramsq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + vbroadcasti32x4 m5, [sgr_shuf+3] + add lpfq, wq + vbroadcasti32x4 m6, [sgr_shuf+5] + add dstq, wq + vbroadcasti32x4 m7, [sgr_shuf+7] + pxor m4, m4 + vpbroadcastd m8, [pd_m9] + vpsubd m11, m4, [paramsq+4] {1to16} ; -s1 + vpbroadcastw m15, [paramsq+10] ; w1 + lea t1, [rsp+wq*2+20] + vpbroadcastd m10, [pw_164_455] + lea t3, [rsp+wq*4+16+416*12] + vpbroadcastd m12, [pw_61448] ; (15 << 12) + (1 << 3) + neg wq + vpbroadcastd m13, [pd_m4096] + mov r10d, 0xfe + vpbroadcastd m14, [pd_34816] ; (1 << 11) + (1 << 15) + kmovb k1, r10d + mova m18, [sgr_x_by_x+64*0] + mov r10, 0x3333333333333333 + mova m19, [sgr_x_by_x+64*1] + kmovq k2, r10 + mova m20, [sgr_x_by_x+64*2] + psllw m15, 4 + mova m21, [sgr_x_by_x+64*3] + lea r14, [r_ext_mask+75] + mova ym9, [sgr_shuf] + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + add t1, 416*6 + call .h_top + lea t4, [lpfq+strideq*4] + mov lpfq, dstq + add t4, strideq + mov [rsp], t4 ; below + mov t0, t2 + call .hv +.main: + mov t5, t3 + add t3, 416*4 + dec hd + jz .height1 + add lpfq, strideq + call .hv + call .prep_n + dec hd + jz .extend_bottom +.main_loop: + add lpfq, strideq + call .hv + call .n + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv_bottom + call .n + add lpfq, strideq + call .hv_bottom +.end: + call .n + RET +.height1: + call .v + call .prep_n + mov t2, t1 + call .v + jmp .end +.extend_bottom: + call .v + call .n + mov t2, t1 + call .v + jmp .end +.no_top: + lea t4, [lpfq+strideq*4] + mov lpfq, dstq + lea t4, [t4+strideq*2] + mov [rsp], t4 + call .h + lea t0, [t1+416*6] + mov t2, t1 + call .v + jmp .main +.h: ; horizontal boxsum + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .h_main +.h_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu ym17, [lpfq+r10-2] +.h_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -33 + jl .h_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r14+r10-8] + vinserti32x8 m16, [r14+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.h_have_right: + pshufb m0, m17, m5 + pmullw m2, m0, m0 + pshufb m16, m17, m6 + paddw m0, m16 + pshufb m17, m7 + paddw m0, m17 ; sum + punpcklwd m3, m16, m17 + punpcklwd m1, m2, m4 + vpdpwssd m1, m3, m3 ; sumsq + punpckhwd m16, m17 + punpckhwd m2, m4 + vpdpwssd m2, m16, m16 + mova [t1+r10*2+416*0], m0 + mova [t1+r10*2+416*2], m1 + mova [t1+r10*2+416*4], m2 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .hv_main +.hv_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .hv_main +.hv_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu ym17, [lpfq+r10-2] +.hv_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp r10d, -33 + jl .hv_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r14+r10-8] + vinserti32x8 m16, [r14+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.hv_have_right: + pshufb m0, m17, m5 + pmullw m3, m0, m0 + pshufb m1, m17, m6 + paddw m0, m1 + pshufb m17, m7 + paddw m0, m17 ; h sum + punpcklwd m16, m17, m1 + punpcklwd m2, m3, m4 + vpdpwssd m2, m16, m16 ; h sumsq + punpckhwd m17, m1 + punpckhwd m3, m4 + vpdpwssd m3, m17, m17 + paddw m1, m0, [t2+r10*2+416*0] + paddw m1, [t1+r10*2+416*0] ; hv sum + paddd m16, m2, [t2+r10*2+416*2] + paddd m17, m3, [t2+r10*2+416*4] + paddd m16, [t1+r10*2+416*2] ; hv sumsq + paddd m17, [t1+r10*2+416*4] + mova [t0+r10*2+416*0], m0 + mova [t0+r10*2+416*2], m2 + mova [t0+r10*2+416*4], m3 + pmulld m16, m8 ; -a * 9 + pmulld m17, m8 + punpcklwd m0, m4, m1 ; b + vpdpwssd m16, m0, m0 ; -p + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + pmulld m16, m11 ; p * s + pmulld m17, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddd m0, m14 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m14 + vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) + vpternlogd m17, m1, m13, 0xd8 + mova [t3+r10*4+ 8], m16 + mova [t3+r10*4+ 24], xm17 + vextracti32x4 [t3+r10*4+ 56], m17, 2 + mova [t3+r10*4+ 72], m17 + vextracti128 [t3+r10*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+104], m16, 3 + add r10, 32 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + ret +.v: ; vertical boxsum + ab + lea r10, [wq-2] +.v_loop: + mova m16, [t1+r10*2+416*2] + mova m17, [t1+r10*2+416*4] + paddd m16, m16 + paddd m17, m17 + paddd m16, [t2+r10*2+416*2] ; hv sumsq + paddd m17, [t2+r10*2+416*4] + pmulld m16, m8 ; -a * 9 + pmulld m17, m8 + mova m1, [t1+r10*2+416*0] + paddw m1, m1 + paddw m1, [t2+r10*2+416*0] ; hv sum + punpcklwd m0, m4, m1 ; b + vpdpwssd m16, m0, m0 ; -p + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + pmulld m16, m11 ; p * s + pmulld m17, m11 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m20 + paddusw m17, m12 + psraw m17, 4 ; min(z, 255) - 256 + vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x + pandn m16, m13, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddd m0, m14 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m14 + vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) + vpternlogd m17, m1, m13, 0xd8 + mova [t3+r10*4+ 8], m16 + mova [t3+r10*4+ 24], xm17 + vextracti32x4 [t3+r10*4+ 56], m17, 2 + mova [t3+r10*4+ 72], m17 + vextracti128 [t3+r10*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+104], m16, 3 + add r10, 32 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + mov r10, wq + mov t4, t3 + add t3, 416*4 +.prep_n_loop: + mova m2, [t5+r10*4+0] + mova m3, [t4+r10*4+0] + paddd m2, [t5+r10*4+8] + paddd m3, [t4+r10*4+8] + paddd m0, m2, [t5+r10*4+4] + paddd m1, m3, [t4+r10*4+4] + pslld m0, 2 + paddd m1, m1 ; ab[ 0] 222 + psubd m0, m2 ; ab[-1] 343 + mova [t3+r10*4+416*4], m1 + paddd m1, m1 + mova [t5+r10*4], m0 + psubd m1, m3 ; ab[ 0] 343 + mova [t4+r10*4], m1 + add r10, 16 + jl .prep_n_loop + ret +; a+b are packed together in a single dword, but we can't do the +; full neighbor calculations before splitting them since we don't +; have sufficient precision. The solution is to do the calculations +; in two equal halves and split a and b before doing the final sum. +ALIGN function_align +.n: ; neighbor + output + mov r10, wq +.n_loop: + mova m16, [t3+r10*4+ 0] + paddd m16, [t3+r10*4+ 8] + paddd m17, m16, [t3+r10*4+ 4] + paddd m17, m17 ; ab[+1] 222 + mova m2, [t3+r10*4+416*4+ 0] + paddd m0, m2, [t5+r10*4+ 0] ; ab[ 0] 222 + ab[-1] 343 + mova m3, [t3+r10*4+416*4+64] + paddd m1, m3, [t5+r10*4+64] + mova [t3+r10*4+416*4+ 0], m17 + paddd m17, m17 + psubd m17, m16 ; ab[+1] 343 + mova [t5+r10*4+ 0], m17 + paddd m2, m17 ; ab[ 0] 222 + ab[+1] 343 + mova m16, [t3+r10*4+64] + paddd m16, [t3+r10*4+72] + paddd m17, m16, [t3+r10*4+68] + paddd m17, m17 + mova [t3+r10*4+416*4+64], m17 + paddd m17, m17 + psubd m17, m16 + mova [t5+r10*4+64], m17 + pandn m16, m13, m0 + psrld m0, 12 + paddd m3, m17 + pandn m17, m13, m2 + psrld m2, 12 + paddd m16, m17 ; a + pandn m17, m13, m1 + psrld m1, 12 + paddd m0, m2 ; b + (1 << 8) + pandn m2, m13, m3 + psrld m3, 12 + paddd m17, m2 + pmovzxbd m2, [dstq+r10+ 0] + paddd m1, m3 + pmovzxbd m3, [dstq+r10+16] + pmaddwd m16, m2 ; a * src + pmaddwd m17, m3 + packssdw m2, m3 + psubd m0, m16 ; b - a * src + (1 << 8) + psubd m1, m17 + psrad m0, 9 + psrad m1, 9 + packssdw m0, m1 + pmulhrsw m0, m15 + paddw m0, m2 + packuswb m0, m0 + vpermd m16, m9, m0 + mova [dstq+r10], ym16 + add r10, 32 + jl .n_loop + mov r10, t5 + mov t5, t4 + mov t4, r10 + add dstq, strideq + ret + +cglobal sgr_filter_mix_8bpc, 4, 13, 28, 416*56+8, dst, stride, left, lpf, \ + w, h, edge, params + mov paramsq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + vbroadcasti128 m5, [sgr_shuf+1] + add lpfq, wq + vbroadcasti128 m6, [sgr_shuf+9] + add dstq, wq + vbroadcasti128 m7, [sgr_shuf+3] + lea t3, [rsp+wq*4+416*24+8] + vbroadcasti128 m8, [sgr_shuf+7] + pxor m4, m4 + vpbroadcastd m9, [pd_m9] + vpsubd m11, m4, [paramsq+0] {1to16} ; -s0 + vpbroadcastd m14, [pw_61448] + vpsubd m12, m4, [paramsq+4] {1to16} ; -s1 + vpbroadcastd m26, [paramsq+8] ; w0 w1 + lea t1, [rsp+wq*2+12] + vpbroadcastd m10, [pd_m25] + neg wq + vpbroadcastd m13, [pw_164_455] + mov r10d, 0xfe + vpbroadcastd m15, [pd_34816] + kmovb k1, r10d + mova m20, [sgr_x_by_x+64*0] + mov r10, 0x3333333333333333 + mova m21, [sgr_x_by_x+64*1] + kmovq k2, r10 + mova m22, [sgr_x_by_x+64*2] + lea r12, [r_ext_mask+75] + mova m23, [sgr_x_by_x+64*3] + vpbroadcastd m24, [pd_m4096] + vpbroadcastd m25, [sgr_shuf+28] ; 0x8000____ + psllw m26, 5 + mova xm27, [sgr_mix_perm] + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t2, t1 + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx512icl).top_fixup + add t1, 416*12 + call .h_top + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov [rsp], r10 ; below + call .hv0 +.main: + dec hd + jz .height1 + add lpfq, strideq + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + add lpfq, strideq + call .hv0 + test hd, hd + jz .odd_height + add lpfq, strideq + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, [rsp] + call .hv0_bottom + add lpfq, strideq + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov [rsp], r10 + call .h + lea t2, [t1+416*12] + lea r10, [wq-2] +.top_fixup_loop: + mova m0, [t1+r10*2+416* 0] + mova m1, [t1+r10*2+416* 2] + mova m2, [t1+r10*2+416* 4] + paddw m0, m0 + mova m3, [t1+r10*2+416* 6] + paddd m1, m1 + mova m16, [t1+r10*2+416* 8] + paddd m2, m2 + mova m17, [t1+r10*2+416*10] + mova [t2+r10*2+416* 0], m0 + mova [t2+r10*2+416* 2], m1 + mova [t2+r10*2+416* 4], m2 + mova [t2+r10*2+416* 6], m3 + mova [t2+r10*2+416* 8], m16 + mova [t2+r10*2+416*10], m17 + add r10, 32 + jl .top_fixup_loop + call .v0 + jmp .main +.h: ; horizontal boxsums + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .h_main +.h_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .h_main +.h_top: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu ym17, [lpfq+r10-2] +.h_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp r10d, -34 + jl .h_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r12+r10-8] + vinserti32x8 m16, [r12+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.h_have_right: + pshufb m3, m17, m5 + pshufb m18, m17, m6 + shufps m0, m3, m18, q2121 + pmullw m2, m0, m0 + pshufb m19, m17, m7 + paddw m0, m19 + pshufb m17, m8 + paddw m0, m17 ; sum3 + punpcklwd m16, m19, m17 + punpcklwd m1, m2, m4 + vpdpwssd m1, m16, m16 ; sumsq3 + punpckhwd m19, m17 + punpckhwd m2, m4 + vpdpwssd m2, m19, m19 + mova [t1+r10*2+416* 6], m0 + mova [t1+r10*2+416* 8], m1 + mova [t1+r10*2+416*10], m2 + punpcklwd m19, m3, m18 + paddw m0, m3 + vpdpwssd m1, m19, m19 ; sumsq5 + punpckhwd m3, m18 + paddw m0, m18 ; sum5 + vpdpwssd m2, m3, m3 + mova [t1+r10*2+416* 0], m0 + mova [t1+r10*2+416* 2], m1 + mova [t1+r10*2+416* 4], m2 + add r10, 32 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsums + vertical boxsum3 + ab3 (even rows) + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .hv0_main +.hv0_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .hv0_main +.hv0_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left +.hv0_loop: + movu ym17, [lpfq+r10-2] +.hv0_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp r10d, -34 + jl .hv0_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r12+r10-8] + vinserti32x8 m16, [r12+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.hv0_have_right: + pshufb m18, m17, m5 + pshufb m19, m17, m6 + shufps m1, m18, m19, q2121 + pmullw m3, m1, m1 + pshufb m0, m17, m7 + paddw m1, m0 + pshufb m17, m8 + paddw m1, m17 ; sum3 + punpcklwd m16, m0, m17 + punpcklwd m2, m3, m4 + vpdpwssd m2, m16, m16 ; sumsq3 + punpckhwd m0, m17 + punpckhwd m3, m4 + vpdpwssd m3, m0, m0 + paddw m0, m1, [t1+r10*2+416* 6] + paddd m16, m2, [t1+r10*2+416* 8] + paddd m17, m3, [t1+r10*2+416*10] + mova [t1+r10*2+416* 6], m1 + mova [t1+r10*2+416* 8], m2 + mova [t1+r10*2+416*10], m3 + paddw m1, m18 + paddw m1, m19 ; sum5 + mova [t3+r10*4+416*8+ 8], m1 + paddw m1, [t1+r10*2+416* 0] + mova [t1+r10*2+416* 0], m1 + punpcklwd m1, m18, m19 + vpdpwssd m2, m1, m1 ; sumsq5 + punpckhwd m18, m19 + vpdpwssd m3, m18, m18 + mova [t3+r10*4+416*0+ 8], m2 ; we need a clean copy of the last row + mova [t3+r10*4+416*0+72], m3 ; in case height is odd + paddd m2, [t1+r10*2+416* 2] + paddd m3, [t1+r10*2+416* 4] + mova [t1+r10*2+416* 2], m2 + mova [t1+r10*2+416* 4], m3 + paddw m1, m0, [t2+r10*2+416* 6] + paddd m2, m16, [t2+r10*2+416* 8] + paddd m3, m17, [t2+r10*2+416*10] + mova [t2+r10*2+416* 6], m0 + mova [t2+r10*2+416* 8], m16 + mova [t2+r10*2+416*10], m17 + pmulld m16, m2, m9 ; -a3 * 9 + pmulld m17, m3, m9 + punpcklwd m0, m4, m1 ; b3 + vpdpwssd m16, m0, m0 ; -p3 + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + pmulld m16, m12 ; p3 * s1 + pmulld m17, m12 + pmaddwd m0, m13 ; b3 * 455 + pmaddwd m1, m13 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m22 + paddusw m17, m14 + psraw m17, 4 ; min(z3, 255) - 256 + vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x3 + pandn m16, m24, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m15 + vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) + vpternlogd m17, m1, m24, 0xd8 + mova [t3+r10*4+416*4+ 8], m16 + mova [t3+r10*4+416*4+ 24], xm17 + vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2 + mova [t3+r10*4+416*4+ 72], m17 + vextracti128 [t3+r10*4+416*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+416*4+104], m16, 3 + add r10, 32 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movd xm17, [leftq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + add leftq, 4 + jmp .hv1_main +.hv1_extend_left: + vpbroadcastb xm17, [lpfq+wq] + vmovdqu32 ym17{k1}, [lpfq+wq-4] + jmp .hv1_main +.hv1_bottom: + lea r10, [wq-2] + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left +.hv1_loop: + movu ym17, [lpfq+r10-2] +.hv1_main: + vinserti32x8 m17, [lpfq+r10+6], 1 + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp r10d, -34 + jl .hv1_have_right + vpbroadcastb m0, [lpfq-1] + movu ym16, [r12+r10-8] + vinserti32x8 m16, [r12+r10+0], 1 + vpternlogd m17, m0, m16, 0xe4 +.hv1_have_right: + pshufb m3, m17, m5 + pshufb m19, m17, m6 + shufps m2, m3, m19, q2121 + pmullw m1, m2, m2 + pshufb m18, m17, m7 + paddw m2, m18 + pshufb m17, m8 + paddw m2, m17 ; sum3 + punpcklwd m16, m17, m18 + punpcklwd m0, m1, m4 + vpdpwssd m0, m16, m16 ; sumsq3 + punpckhwd m17, m18 + punpckhwd m1, m4 + vpdpwssd m1, m17, m17 + paddd m16, m0, [t2+r10*2+416* 8] + paddd m17, m1, [t2+r10*2+416*10] + mova [t2+r10*2+416* 8], m0 + mova [t2+r10*2+416*10], m1 + punpcklwd m18, m3, m19 + vpdpwssd m0, m18, m18 ; sumsq5 + punpckhwd m18, m3, m19 + vpdpwssd m1, m18, m18 + paddw m3, m19 + pmulld m16, m9 ; -a3 * 9 + pmulld m17, m9 + paddd m18, m0, [t2+r10*2+416*2] + paddd m19, m1, [t2+r10*2+416*4] + paddd m18, [t1+r10*2+416*2] + paddd m19, [t1+r10*2+416*4] + mova [t2+r10*2+416*2], m0 + mova [t2+r10*2+416*4], m1 + pmulld m18, m10 ; -a5 * 25 + pmulld m19, m10 + paddw m1, m2, [t2+r10*2+416* 6] + mova [t2+r10*2+416* 6], m2 + paddw m2, m3 ; sum5 + paddw m3, m2, [t2+r10*2+416*0] + paddw m3, [t1+r10*2+416*0] + mova [t2+r10*2+416*0], m2 + punpcklwd m0, m4, m1 ; b3 + vpdpwssd m16, m0, m0 ; -p3 + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + punpcklwd m2, m3, m4 ; b5 + vpdpwssd m18, m2, m2 ; -p5 + punpckhwd m3, m4 + vpdpwssd m19, m3, m3 + pmulld m16, m12 ; p3 * s1 + pmulld m17, m12 + pmulld m18, m11 ; p5 * s0 + pmulld m19, m11 + pmaddwd m0, m13 ; b3 * 455 + pmaddwd m1, m13 + pmaddwd m2, m13 ; b5 * 164 + pmaddwd m3, m13 + vpalignr m17{k2}, m16, m16, 2 + vpalignr m19{k2}, m18, m18, 2 + paddusw m17, m14 + mova m16, m22 + psraw m17, 4 ; min(z3, 255) - 256 + vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] + paddusw m19, m14 + mova m18, m22 + psraw m19, 4 ; min(z5, 255) - 256 + vpermt2b m18, m19, m23 ; sgr_x_by_x[128..255] + vpmovb2m k4, m19 + vpermi2b m19, m20, m21 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x3 + vmovdqu8 m19{k4}, m18 ; x5 + pandn m16, m24, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + pandn m18, m24, m19 + psrld m19, 16 + pmulld m2, m18 + pmulld m3, m19 + paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m15 + vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) + vpternlogd m17, m1, m24, 0xd8 + mova [t3+r10*4+416*8+ 8], m16 + mova [t3+r10*4+416*8+ 24], xm17 + vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2 + paddd m2, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m3, m15 + mova [t3+r10*4+416*8+ 72], m17 + vextracti128 [t3+r10*4+416*8+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+416*8+104], m16, 3 + vpternlogd m18, m2, m24, 0xd8 ; a5 | (b5 << 12) + vpternlogd m19, m3, m24, 0xd8 + mova [t3+r10*4+416*0+ 8], m18 + mova [t3+r10*4+416*0+ 24], xm19 + vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2 + mova [t3+r10*4+416*0+ 72], m19 + vextracti128 [t3+r10*4+416*0+ 72], ym18, 1 + vextracti32x4 [t3+r10*4+416*0+104], m18, 3 + add r10, 32 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) + lea r10, [wq-2] +.v0_loop: + mova m2, [t1+r10*2+416* 8] + mova m3, [t1+r10*2+416*10] + paddd m2, m2 + paddd m3, m3 + paddd m16, m2, [t2+r10*2+416* 8] + paddd m17, m3, [t2+r10*2+416*10] + mova m0, [t1+r10*2+416* 6] + paddw m0, m0 + paddw m1, m0, [t2+r10*2+416* 6] + pmulld m16, m9 ; -a3 * 9 + pmulld m17, m9 + mova [t2+r10*2+416* 6], m0 + mova [t2+r10*2+416* 8], m2 + mova [t2+r10*2+416*10], m3 + mova m2, [t1+r10*2+416*0] + mova m3, [t1+r10*2+416*2] + mova m18, [t1+r10*2+416*4] + punpcklwd m0, m4, m1 ; b3 + vpdpwssd m16, m0, m0 ; -p3 + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + pmulld m16, m12 ; p3 * s1 + pmulld m17, m12 + pmaddwd m0, m13 ; b3 * 455 + pmaddwd m1, m13 + mova [t3+r10*4+416*8+ 8], m2 + mova [t3+r10*4+416*0+ 8], m3 + mova [t3+r10*4+416*0+72], m18 + vpalignr m17{k2}, m16, m16, 2 + mova m16, m22 + paddusw m17, m14 + psraw m17, 4 ; min(z3, 255) - 256 + vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x3 + pandn m16, m24, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + paddw m2, m2 ; cc5 + paddd m3, m3 + paddd m18, m18 + mova [t1+r10*2+416*0], m2 + mova [t1+r10*2+416*2], m3 + mova [t1+r10*2+416*4], m18 + paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m15 + vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) + vpternlogd m17, m1, m24, 0xd8 + mova [t3+r10*4+416*4+ 8], m16 + mova [t3+r10*4+416*4+ 24], xm17 + vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2 + mova [t3+r10*4+416*4+ 72], m17 + vextracti128 [t3+r10*4+416*4+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+416*4+104], m16, 3 + add r10, 32 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) + lea r10, [wq-2] +.v1_loop: + mova m0, [t1+r10*2+416* 8] + paddd m16, m0, [t2+r10*2+416* 8] + mova m1, [t1+r10*2+416*10] + paddd m17, m1, [t2+r10*2+416*10] + mova m2, [t3+r10*4+416*0+ 8] + paddd m18, m2, [t2+r10*2+416* 2] + mova m3, [t3+r10*4+416*0+72] + paddd m19, m3, [t2+r10*2+416* 4] + paddd m18, [t1+r10*2+416* 2] + paddd m19, [t1+r10*2+416* 4] + mova [t2+r10*2+416* 8], m0 + mova [t2+r10*2+416*10], m1 + mova [t2+r10*2+416* 2], m2 + mova [t2+r10*2+416* 4], m3 + pmulld m16, m9 ; -a3 * 9 + pmulld m17, m9 + pmulld m18, m10 ; -a5 * 25 + pmulld m19, m10 + mova m0, [t1+r10*2+416* 6] + paddw m1, m0, [t2+r10*2+416* 6] + mova m2, [t3+r10*4+416*8+ 8] + paddw m3, m2, [t2+r10*2+416*0] + paddw m3, [t1+r10*2+416*0] + mova [t2+r10*2+416* 6], m0 + mova [t2+r10*2+416*0], m2 + punpcklwd m0, m4, m1 ; b3 + vpdpwssd m16, m0, m0 ; -p3 + punpckhwd m1, m4, m1 + vpdpwssd m17, m1, m1 + punpcklwd m2, m3, m4 ; b5 + vpdpwssd m18, m2, m2 ; -p5 + punpckhwd m3, m4 + vpdpwssd m19, m3, m3 + pmulld m16, m12 ; p3 * s1 + pmulld m17, m12 + pmulld m18, m11 ; p5 * s0 + pmulld m19, m11 + pmaddwd m0, m13 ; b3 * 455 + pmaddwd m1, m13 + pmaddwd m2, m13 ; b5 * 164 + pmaddwd m3, m13 + vpalignr m17{k2}, m16, m16, 2 + vpalignr m19{k2}, m18, m18, 2 + paddusw m17, m14 + mova m16, m22 + psraw m17, 4 ; min(z3, 255) - 256 + vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] + vpmovb2m k3, m17 + vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] + paddusw m19, m14 + mova m18, m22 + psraw m19, 4 ; min(z5, 255) - 256 + vpermt2b m18, m19, m23 ; sgr_x_by_x[128..255] + vpmovb2m k4, m19 + vpermi2b m19, m20, m21 ; sgr_x_by_x[ 0..127] + vmovdqu8 m17{k3}, m16 ; x3 + vmovdqu8 m19{k4}, m18 ; x5 + pandn m16, m24, m17 + psrld m17, 16 + pmulld m0, m16 + pmulld m1, m17 + pandn m18, m24, m19 + psrld m19, m19, 16 + pmulld m2, m18 + pmulld m3, m19 + paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m15 + vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) + vpternlogd m17, m1, m24, 0xd8 + mova [t3+r10*4+416*8+ 8], m16 + mova [t3+r10*4+416*8+ 24], xm17 + vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2 + paddd m2, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m3, m15 + mova [t3+r10*4+416*8+ 72], m17 + vextracti128 [t3+r10*4+416*8+ 72], ym16, 1 + vextracti32x4 [t3+r10*4+416*8+104], m16, 3 + vpternlogd m18, m2, m24, 0xd8 ; a5 | (b5 << 12) + vpternlogd m19, m3, m24, 0xd8 + mova [t3+r10*4+416*0+ 8], m18 + mova [t3+r10*4+416*0+ 24], xm19 + vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2 + mova [t3+r10*4+416*0+ 72], m19 + vextracti128 [t3+r10*4+416*0+ 72], ym18, 1 + vextracti32x4 [t3+r10*4+416*0+104], m18, 3 + add r10, 32 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + mov r10, wq +.prep_n_loop: + movu m0, [t3+r10*4+416*0+4] + paddd m1, m0, [t3+r10*4+416*0+0] + mova m16, [t3+r10*4+416*4+0] + paddd m1, [t3+r10*4+416*0+8] + mova m17, [t3+r10*4+416*8+0] + paddd m16, [t3+r10*4+416*4+8] + paddd m17, [t3+r10*4+416*8+8] + paddd m2, m16, [t3+r10*4+416*4+4] + paddd m3, m17, [t3+r10*4+416*8+4] + paddd m0, m1 + pslld m1, 2 + pslld m2, 2 + paddd m1, m0 ; ab5 565 + paddd m3, m3 ; ab3[ 0] 222 + psubd m2, m16 ; ab3[-1] 343 + mova [t3+r10*4+416*20], m3 + pandn m0, m24, m1 ; a5 565 + mova [t3+r10*4+416*24], m2 + psrld m1, 12 ; b5 565 + mova [t3+r10*4+416*12], m0 + paddd m3, m3 + mova [t3+r10*4+416*16], m1 + psubd m3, m17 ; ab3[ 0] 343 + mova [t3+r10*4+416*28], m3 + add r10, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + mov r10, wq +.n0_loop: + movu m2, [t3+r10*4+4] + paddd m3, m2, [t3+r10*4+0] + paddd m3, [t3+r10*4+8] + mova m1, [t3+r10*4+416*4+0] + paddd m2, m3 + pslld m3, 2 + paddd m1, [t3+r10*4+416*4+8] + paddd m3, m2 + pandn m2, m24, m3 + psrld m3, 12 + paddd m0, m2, [t3+r10*4+416*12] ; a5 + paddd m16, m3, [t3+r10*4+416*16] ; b5 + (1 << 8) + mova [t3+r10*4+416*12], m2 + mova [t3+r10*4+416*16], m3 + paddd m2, m1, [t3+r10*4+416*4+4] + paddd m2, m2 ; ab3[ 1] 222 + mova m3, [t3+r10*4+416*20] + paddd m17, m3, [t3+r10*4+416*24] ; ab3[ 0] 222 + ab3[-1] 343 + mova [t3+r10*4+416*20], m2 + paddd m2, m2 + psubd m2, m1 ; ab3[ 1] 343 + mova [t3+r10*4+416*24], m2 + paddd m2, m3 ; ab3[ 0] 222 + ab3[ 1] 343 + pandn m1, m24, m17 + psrld m17, 12 + pandn m3, m24, m2 + psrld m2, 12 + paddd m1, m3 ; a3 + pmovzxbd m3, [dstq+r10] + paddd m17, m2 ; b3 + (1 << 8) + pmaddwd m0, m3 ; a5 * src + pmaddwd m1, m3 ; a3 * src + vpshldd m3, m25, 16 ; (dst << 16) + (1 << 15) + psubd m16, m0 ; b5 - a5 * src + (1 << 8) + psubd m17, m1 ; b3 - a3 * src + (1 << 8) + psrld m16, 9 + pslld m17, 7 + vmovdqu8 m17{k2}, m16 + vpdpwssd m3, m17, m26 + packuswb m3, m2 + vpermb m16, m27, m3 + mova [dstq+r10], xm16 + add r10, 16 + jl .n0_loop + add dstq, strideq + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + mov r10, wq +.n1_loop: + mova m1, [t3+r10*4+416*8+0] + paddd m1, [t3+r10*4+416*8+8] + paddd m2, m1, [t3+r10*4+416*8+4] + paddd m2, m2 ; ab3[ 1] 222 + mova m0, [t3+r10*4+416*20] + paddd m17, m0, [t3+r10*4+416*28] ; ab3[ 0] 222 + ab3[-1] 343 + pmovzxbd m3, [dstq+r10] + mova [t3+r10*4+416*20], m2 + paddd m2, m2 + psubd m2, m1 ; ab3[ 1] 343 + mova [t3+r10*4+416*28], m2 + paddd m0, m2 ; ab3[ 0] 222 + ab3[ 1] 343 + pandn m1, m24, m17 + psrld m17, 12 + pandn m2, m24, m0 + psrld m0, 12 + paddd m1, m2 ; a3 + paddd m17, m0 ; b3 + (1 << 8) + mova m16, [t3+r10*4+416*16] ; b5 + (1 << 7) + pmaddwd m1, m3 ; a3 * src + pmaddwd m0, m3, [t3+r10*4+416*12] ; a5 * src + vpshldd m3, m25, 16 ; (dst << 16) + (1 << 15) + psubd m17, m1 ; b3 - a3 * src + (1 << 8) + psubd m16, m0 ; b5 - a5 * src + (1 << 7) + pslld m17, 7 + palignr m17{k2}, m16, m16, 1 + vpdpwssd m3, m17, m26 + packuswb m3, m3 + vpermb m16, m27, m3 + mova [dstq+r10], xm16 + add r10, 16 + jl .n1_loop + add dstq, strideq + ret + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/looprestoration_sse.asm b/third_party/dav1d/src/x86/looprestoration_sse.asm new file mode 100644 index 0000000000..01eb6fa348 --- /dev/null +++ b/third_party/dav1d/src/x86/looprestoration_sse.asm @@ -0,0 +1,3681 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; Copyright © 2018, VideoLabs +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +wiener_init: db 6, 7, 6, 7, 6, 7, 6, 7, 0, 0, 0, 0, 2, 4, 2, 4 +wiener_shufA: db 1, 7, 2, 8, 3, 9, 4, 10, 5, 11, 6, 12, 7, 13, 8, 14 +wiener_shufB: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 +wiener_shufC: db 6, 5, 7, 6, 8, 7, 9, 8, 10, 9, 11, 10, 12, 11, 13, 12 +wiener_shufD: db 4, -1, 5, -1, 6, -1, 7, -1, 8, -1, 9, -1, 10, -1, 11, -1 +wiener_l_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +sgr_lshuf3: db 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 +sgr_lshuf5: db 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +pb_0to15: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + +pb_right_ext_mask: times 24 db 0xff + times 8 db 0 +pb_1: times 16 db 1 +pb_3: times 16 db 3 +pw_256: times 8 dw 256 +pw_2056: times 8 dw 2056 +pw_m16380: times 8 dw -16380 +pd_4096: times 4 dd 4096 +pd_34816: times 4 dd 34816 +pd_0xffff: times 4 dd 0xffff +pd_0xf00800a4: times 4 dd 0xf00800a4 +pd_0xf00801c7: times 4 dd 0xf00801c7 + +cextern sgr_x_by_x + +SECTION .text + +%macro movif64 2 ; dst, src + %if ARCH_X86_64 + mov %1, %2 + %endif +%endmacro + +%macro movif32 2 ; dst, src + %if ARCH_X86_32 + mov %1, %2 + %endif +%endmacro + +%if ARCH_X86_32 + %define PIC_base_offset $$ + + %macro SETUP_PIC 1-3 1,0 ; PIC_reg, save_PIC_reg, restore_PIC_reg + %assign pic_reg_stk_off 4 + %xdefine PIC_reg %1 + %if %2 == 1 + mov [esp], %1 + %endif + LEA PIC_reg, PIC_base_offset + %if %3 == 1 + XCHG_PIC_REG + %endif + %endmacro + + %macro XCHG_PIC_REG 0 + mov [esp+pic_reg_stk_off], PIC_reg + %assign pic_reg_stk_off (pic_reg_stk_off+4) % 8 + mov PIC_reg, [esp+pic_reg_stk_off] + %endmacro + + %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) + +%else + %macro XCHG_PIC_REG 0 + %endmacro + + %define PIC_sym(sym) (sym) +%endif + +%macro WIENER 0 +%if ARCH_X86_64 +DECLARE_REG_TMP 9, 7, 10, 11, 12, 13, 14 ; ring buffer pointers +cglobal wiener_filter7_8bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \ + w, h, edge, flt, x + %define tmpstrideq strideq + %define base 0 + mov fltq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + movq m14, [fltq] + add lpfq, wq + movq m7, [fltq+16] + add dstq, wq + lea t1, [rsp+wq*2+16] + mova m15, [pw_2056] + neg wq +%if cpuflag(ssse3) + pshufb m14, [wiener_init] + mova m8, [wiener_shufA] + pshufd m12, m14, q2222 ; x0 x0 + mova m9, [wiener_shufB] + pshufd m13, m14, q3333 ; x1 x2 + mova m10, [wiener_shufC] + punpcklqdq m14, m14 ; x3 + mova m11, [wiener_shufD] +%else + mova m10, [pw_m16380] + punpcklwd m14, m14 + pshufd m11, m14, q0000 ; x0 + pshufd m12, m14, q1111 ; x1 + pshufd m13, m14, q2222 ; x2 + pshufd m14, m14, q3333 ; x3 +%endif +%else +DECLARE_REG_TMP 4, 0, _, 5 +%if cpuflag(ssse3) + %define m10 [base+wiener_shufC] + %define m11 [base+wiener_shufD] + %define stk_off 96 +%else + %define m10 [base+pw_m16380] + %define m11 [stk+96] + %define stk_off 112 +%endif +cglobal wiener_filter7_8bpc, 0, 7, 8, -384*12-stk_off, _, x, left, lpf, tmpstride + %define base r6-pb_right_ext_mask-21 + %define stk esp + %define dstq leftq + %define edgeb byte edged + %define edged [stk+ 8] + %define dstmp [stk+12] + %define hd dword [stk+16] + %define wq [stk+20] + %define strideq [stk+24] + %define leftmp [stk+28] + %define t2 [stk+32] + %define t4 [stk+36] + %define t5 [stk+40] + %define t6 [stk+44] + %define m8 [base+wiener_shufA] + %define m9 [base+wiener_shufB] + %define m12 [stk+48] + %define m13 [stk+64] + %define m14 [stk+80] + %define m15 [base+pw_2056] + mov r1, r6m ; flt + mov r0, r0m ; dst + mov r4, r4m ; w + mov lpfq, lpfm + mov r2, r7m ; edge + mov r5, r5m ; h + movq m3, [r1+ 0] + movq m7, [r1+16] + add r0, r4 + mov r1, r1m ; stride + add lpfq, r4 + mov edged, r2 + mov r2, r2m ; left + mov dstmp, r0 + lea t1, [rsp+r4*2+stk_off] + mov hd, r5 + neg r4 + LEA r6, pb_right_ext_mask+21 + mov wq, r4 + mov strideq, r1 + mov leftmp, r2 + mov r4, r1 +%if cpuflag(ssse3) + pshufb m3, [base+wiener_init] + pshufd m1, m3, q2222 + pshufd m2, m3, q3333 + punpcklqdq m3, m3 +%else + punpcklwd m3, m3 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova m11, m0 +%endif + mova m12, m1 + mova m13, m2 + mova m14, m3 +%endif + psllw m7, 5 + pshufd m6, m7, q0000 ; y0 y1 + pshufd m7, m7, q1111 ; y2 y3 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t6, t1 + mov t5, t1 + add t1, 384*2 + call .h_top + lea t3, [lpfq+tmpstrideq*4] + mov lpfq, dstmp + add t3, tmpstrideq + mov [rsp], t3 ; below + mov t4, t1 + add t1, 384*2 + call .h + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 +.main: + lea t0, [t1+384*2] +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v3 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.v1: + call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v + RET +.no_top: + lea t3, [lpfq+tmpstrideq*4] + mov lpfq, dstmp + lea t3, [t3+tmpstrideq*2] + mov [rsp], t3 + call .h + mov t6, t1 + mov t5, t1 + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + mov t2, t1 + dec hd + jz .v2 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v3 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v3 + add t0, 384*8 + call .hv + dec hd + jnz .main +.v3: + call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v +.v2: + call mangle(private_prefix %+ _wiener_filter7_8bpc_ssse3).v + jmp .v1 +.extend_right: + movd m2, [lpfq-4] +%if ARCH_X86_64 + push r0 + lea r0, [pb_right_ext_mask+21] + movu m0, [r0+xq+0] + movu m1, [r0+xq+8] + pop r0 +%else + movu m0, [r6+xq+0] + movu m1, [r6+xq+8] +%endif +%if cpuflag(ssse3) + pshufb m2, [base+pb_3] +%else + punpcklbw m2, m2 + pshuflw m2, m2, q3333 + punpcklqdq m2, m2 +%endif + pand m4, m0 + pand m5, m1 + pandn m0, m2 + pandn m1, m2 + por m4, m0 + por m5, m1 + ret +.h: + %define stk esp+4 ; offset due to call + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movifnidn leftq, leftmp + mova m4, [lpfq+xq] + movd m5, [leftq] + add leftq, 4 + pslldq m4, 4 + por m4, m5 + movifnidn leftmp, leftq + jmp .h_main +.h_extend_left: +%if cpuflag(ssse3) + mova m4, [lpfq+xq] + pshufb m4, [base+wiener_l_shuf] +%else + mova m5, [lpfq+xq] + pshufd m4, m5, q2103 + punpcklbw m5, m5 + punpcklwd m5, m5 + movss m4, m5 +%endif + jmp .h_main +.h_top: + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+xq-4] +.h_main: + movu m5, [lpfq+xq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp xd, -18 + jl .h_have_right + call .extend_right +.h_have_right: +%macro %%h7 0 +%if cpuflag(ssse3) + pshufb m0, m4, m8 + pmaddubsw m0, m12 + pshufb m1, m5, m8 + pmaddubsw m1, m12 + pshufb m2, m4, m9 + pmaddubsw m2, m13 + pshufb m3, m5, m9 + pmaddubsw m3, m13 + paddw m0, m2 + pshufb m2, m4, m10 + pmaddubsw m2, m13 + paddw m1, m3 + pshufb m3, m5, m10 + pmaddubsw m3, m13 + pshufb m4, m11 + paddw m0, m2 + pmullw m2, m14, m4 + pshufb m5, m11 + paddw m1, m3 + pmullw m3, m14, m5 + psllw m4, 7 + psllw m5, 7 + paddw m0, m2 + mova m2, [base+pw_m16380] + paddw m1, m3 + paddw m4, m2 + paddw m5, m2 + paddsw m0, m4 + paddsw m1, m5 +%else + psrldq m0, m4, 1 + pslldq m1, m4, 1 + pxor m3, m3 + punpcklbw m0, m3 + punpckhbw m1, m3 + paddw m0, m1 + pmullw m0, m11 + psrldq m1, m4, 2 + pslldq m2, m4, 2 + punpcklbw m1, m3 + punpckhbw m2, m3 + paddw m1, m2 + pmullw m1, m12 + paddw m0, m1 + pshufd m2, m4, q0321 + punpcklbw m2, m3 + pmullw m1, m14, m2 + paddw m0, m1 + psrldq m1, m4, 3 + pslldq m4, 3 + punpcklbw m1, m3 + punpckhbw m4, m3 + paddw m1, m4 + pmullw m1, m13 + paddw m0, m1 + psllw m2, 7 + paddw m2, m10 + paddsw m0, m2 + psrldq m1, m5, 1 + pslldq m2, m5, 1 + punpcklbw m1, m3 + punpckhbw m2, m3 + paddw m1, m2 + pmullw m1, m11 + psrldq m2, m5, 2 + pslldq m4, m5, 2 + punpcklbw m2, m3 + punpckhbw m4, m3 + paddw m2, m4 + pmullw m2, m12 + paddw m1, m2 + pshufd m4, m5, q0321 + punpcklbw m4, m3 + pmullw m2, m14, m4 + paddw m1, m2 + psrldq m2, m5, 3 + pslldq m5, 3 + punpcklbw m2, m3 + punpckhbw m5, m3 + paddw m2, m5 + pmullw m2, m13 + paddw m1, m2 + psllw m4, 7 + paddw m4, m10 + paddsw m1, m4 +%endif +%endmacro + %%h7 + psraw m0, 3 + psraw m1, 3 + paddw m0, m15 + paddw m1, m15 + mova [t1+xq*2+ 0], m0 + mova [t1+xq*2+16], m1 + add xq, 16 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movifnidn leftq, leftmp + mova m4, [lpfq+xq] + movd m5, [leftq] + add leftq, 4 + pslldq m4, 4 + por m4, m5 + movifnidn leftmp, leftq + jmp .hv_main +.hv_extend_left: +%if cpuflag(ssse3) + mova m4, [lpfq+xq] + pshufb m4, [base+wiener_l_shuf] +%else + mova m5, [lpfq+xq] + pshufd m4, m5, q2103 + punpcklbw m5, m5 + punpcklwd m5, m5 + movss m4, m5 +%endif + jmp .hv_main +.hv_bottom: + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m4, [lpfq+xq-4] +.hv_main: + movu m5, [lpfq+xq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp xd, -18 + jl .hv_have_right + call .extend_right +.hv_have_right: + %%h7 +%if ARCH_X86_64 + mova m2, [t4+xq*2] + paddw m2, [t2+xq*2] +%else + mov r2, t4 + mova m2, [r2+xq*2] + mov r2, t2 + paddw m2, [r2+xq*2] + mov r2, t5 +%endif + mova m3, [t3+xq*2] +%if ARCH_X86_64 + mova m5, [t5+xq*2] +%else + mova m5, [r2+xq*2] + mov r2, t6 +%endif + paddw m5, [t1+xq*2] + psraw m0, 3 + psraw m1, 3 + paddw m0, m15 + paddw m1, m15 +%if ARCH_X86_64 + paddw m4, m0, [t6+xq*2] +%else + paddw m4, m0, [r2+xq*2] + mov r2, t4 +%endif + mova [t0+xq*2], m0 + punpcklwd m0, m2, m3 + pmaddwd m0, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4, m5 + pmaddwd m3, m6 + punpckhwd m4, m5 + pmaddwd m4, m6 + paddd m0, m3 + mova m3, [t3+xq*2+16] + paddd m4, m2 +%if ARCH_X86_64 + mova m2, [t4+xq*2+16] + paddw m2, [t2+xq*2+16] + mova m5, [t5+xq*2+16] +%else + mova m2, [r2+xq*2+16] + mov r2, t2 + paddw m2, [r2+xq*2+16] + mov r2, t5 + mova m5, [r2+xq*2+16] + mov r2, t6 +%endif + paddw m5, [t1+xq*2+16] + packuswb m0, m4 +%if ARCH_X86_64 + paddw m4, m1, [t6+xq*2+16] +%else + paddw m4, m1, [r2+xq*2+16] + mov dstq, dstmp +%endif + mova [t0+xq*2+16], m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4, m5 + pmaddwd m3, m6 + punpckhwd m4, m5 + pmaddwd m4, m6 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+xq], m0 + add xq, 16 + jl .hv_loop + add dstq, strideq +%if ARCH_X86_64 + mov t6, t5 + mov t5, t4 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t6 +%else + mov dstmp, dstq + mov r1, t5 + mov r2, t4 + mov t6, r1 + mov t5, r2 + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, r1 +%endif + ret +%if cpuflag(ssse3) ; identical in sse2 and ssse3, so share code +.v: + mov xq, wq +.v_loop: +%if ARCH_X86_64 + mova m1, [t4+xq*2] + paddw m1, [t2+xq*2] +%else + mov r2, t4 + mova m1, [r2+xq*2] + mov r2, t2 + paddw m1, [r2+xq*2] + mov r2, t6 +%endif + mova m2, [t3+xq*2] + mova m4, [t1+xq*2] +%if ARCH_X86_64 + paddw m3, m4, [t6+xq*2] + paddw m4, [t5+xq*2] +%else + paddw m3, m4, [r2+xq*2] + mov r2, t5 + paddw m4, [r2+xq*2] + mov r2, t4 +%endif + punpcklwd m0, m1, m2 + pmaddwd m0, m7 + punpckhwd m1, m2 + pmaddwd m1, m7 + punpcklwd m2, m3, m4 + pmaddwd m2, m6 + punpckhwd m3, m4 + pmaddwd m3, m6 + paddd m0, m2 + paddd m1, m3 +%if ARCH_X86_64 + mova m2, [t4+xq*2+16] + paddw m2, [t2+xq*2+16] +%else + mova m2, [r2+xq*2+16] + mov r2, t2 + paddw m2, [r2+xq*2+16] + mov r2, t6 +%endif + mova m3, [t3+xq*2+16] + mova m5, [t1+xq*2+16] +%if ARCH_X86_64 + paddw m4, m5, [t6+xq*2+16] + paddw m5, [t5+xq*2+16] +%else + paddw m4, m5, [r2+xq*2+16] + mov r2, t5 + paddw m5, [r2+xq*2+16] + movifnidn dstq, dstmp +%endif + packuswb m0, m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4, m5 + pmaddwd m3, m6 + punpckhwd m4, m5 + pmaddwd m4, m6 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+xq], m0 + add xq, 16 + jl .v_loop + add dstq, strideq +%if ARCH_X86_64 + mov t6, t5 + mov t5, t4 +%else + mov dstmp, dstq + mov r1, t5 + mov r2, t4 + mov t6, r1 + mov t5, r2 +%endif + mov t4, t3 + mov t3, t2 + mov t2, t1 + ret +%endif + +%if ARCH_X86_64 +cglobal wiener_filter5_8bpc, 4, 13, 16, 384*8+16, dst, stride, left, lpf, \ + w, h, edge, flt, x + mov fltq, r6mp + mov wd, wm + movifnidn hd, hm + mov edged, r7m + movq m14, [fltq] + add lpfq, wq + movq m7, [fltq+16] + add dstq, wq + mova m8, [pw_m16380] + lea t1, [rsp+wq*2+16] + mova m15, [pw_2056] + neg wq +%if cpuflag(ssse3) + pshufb m14, [wiener_init] + mova m9, [wiener_shufB] + pshufd m13, m14, q3333 ; x1 x2 + mova m10, [wiener_shufC] + punpcklqdq m14, m14 ; x3 + mova m11, [wiener_shufD] + mova m12, [wiener_l_shuf] +%else + punpcklwd m14, m14 + pshufd m11, m14, q1111 ; x1 + pshufd m13, m14, q2222 ; x2 + pshufd m14, m14, q3333 ; x3 +%endif +%else +%if cpuflag(ssse3) + %define stk_off 80 +%else + %define m11 [stk+80] + %define stk_off 96 +%endif +cglobal wiener_filter5_8bpc, 0, 7, 8, -384*8-stk_off, _, x, left, lpf, tmpstride + %define stk esp + %define leftmp [stk+28] + %define m8 [base+pw_m16380] + %define m12 [base+wiener_l_shuf] + %define m14 [stk+48] + mov r1, r6m ; flt + mov r0, r0m ; dst + mov r4, r4m ; w + mov lpfq, lpfm + mov r2, r7m ; edge + mov r5, r5m ; h + movq m2, [r1+ 0] + movq m7, [r1+16] + add r0, r4 + mov r1, r1m ; stride + add lpfq, r4 + mov edged, r2 + mov r2, r2m ; left + mov dstmp, r0 + lea t1, [rsp+r4*2+stk_off] + mov hd, r5 + neg r4 + LEA r6, pb_right_ext_mask+21 + mov wq, r4 + mov strideq, r1 + mov leftmp, r2 + mov r4, r1 +%if cpuflag(ssse3) + pshufb m2, [base+wiener_init] + pshufd m1, m2, q3333 + punpcklqdq m2, m2 +%else + punpcklwd m2, m2 + pshufd m0, m2, q1111 + pshufd m1, m2, q2222 + pshufd m2, m2, q3333 + mova m11, m0 +%endif + mova m13, m1 + mova m14, m2 +%endif + psllw m7, 5 + pshufd m6, m7, q0000 ; __ y1 + pshufd m7, m7, q1111 ; y2 y3 + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, strideq + mov t4, t1 + add t1, 384*2 + call .h_top + lea xq, [lpfq+tmpstrideq*4] + mov lpfq, dstmp + mov t3, t1 + add t1, 384*2 + add xq, tmpstrideq + mov [rsp], xq ; below + call .h + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 +.main: + mov t0, t4 +.main_loop: + call .hv + dec hd + jnz .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .v2 + mov lpfq, [rsp] + call .hv_bottom + add lpfq, strideq + call .hv_bottom +.end: + RET +.no_top: + lea t3, [lpfq+tmpstrideq*4] + mov lpfq, dstmp + lea t3, [t3+tmpstrideq*2] + mov [rsp], t3 + call .h + mov t4, t1 + mov t3, t1 + mov t2, t1 + dec hd + jz .v1 + add lpfq, strideq + add t1, 384*2 + call .h + dec hd + jz .v2 + lea t0, [t1+384*2] + call .hv + dec hd + jz .v2 + add t0, 384*6 + call .hv + dec hd + jnz .main +.v2: + call mangle(private_prefix %+ _wiener_filter5_8bpc_ssse3).v + add dstq, strideq + mov t4, t3 + mov t3, t2 + mov t2, t1 + movifnidn dstmp, dstq +.v1: + call mangle(private_prefix %+ _wiener_filter5_8bpc_ssse3).v + jmp .end +.h: + %define stk esp+4 + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movifnidn leftq, leftmp + mova m4, [lpfq+xq] + movd m5, [leftq] + add leftq, 4 + pslldq m4, 4 + por m4, m5 + movifnidn leftmp, leftq + jmp .h_main +.h_extend_left: +%if cpuflag(ssse3) + mova m4, [lpfq+xq] + pshufb m4, m12 +%else + mova m5, [lpfq+xq] + pshufd m4, m5, q2103 + punpcklbw m5, m5 + punpcklwd m5, m5 + movss m4, m5 +%endif + jmp .h_main +.h_top: + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left +.h_loop: + movu m4, [lpfq+xq-4] +.h_main: + movu m5, [lpfq+xq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp xd, -17 + jl .h_have_right + call mangle(private_prefix %+ _wiener_filter7_8bpc %+ SUFFIX).extend_right +.h_have_right: +%macro %%h5 0 +%if cpuflag(ssse3) + pshufb m0, m4, m9 + pmaddubsw m0, m13 + pshufb m1, m5, m9 + pmaddubsw m1, m13 + pshufb m2, m4, m10 + pmaddubsw m2, m13 + pshufb m3, m5, m10 + pmaddubsw m3, m13 + pshufb m4, m11 + paddw m0, m2 + pmullw m2, m14, m4 + pshufb m5, m11 + paddw m1, m3 + pmullw m3, m14, m5 + psllw m4, 7 + psllw m5, 7 + paddw m4, m8 + paddw m5, m8 + paddw m0, m2 + paddw m1, m3 + paddsw m0, m4 + paddsw m1, m5 +%else + psrldq m0, m4, 2 + pslldq m1, m4, 2 + pxor m3, m3 + punpcklbw m0, m3 + punpckhbw m1, m3 + paddw m0, m1 + pmullw m0, m11 + pshufd m2, m4, q0321 + punpcklbw m2, m3 + pmullw m1, m14, m2 + paddw m0, m1 + psrldq m1, m4, 3 + pslldq m4, 3 + punpcklbw m1, m3 + punpckhbw m4, m3 + paddw m1, m4 + pmullw m1, m13 + paddw m0, m1 + psllw m2, 7 + paddw m2, m8 + paddsw m0, m2 + psrldq m1, m5, 2 + pslldq m4, m5, 2 + punpcklbw m1, m3 + punpckhbw m4, m3 + paddw m1, m4 + pmullw m1, m11 + pshufd m4, m5, q0321 + punpcklbw m4, m3 + pmullw m2, m14, m4 + paddw m1, m2 + psrldq m2, m5, 3 + pslldq m5, 3 + punpcklbw m2, m3 + punpckhbw m5, m3 + paddw m2, m5 + pmullw m2, m13 + paddw m1, m2 + psllw m4, 7 + paddw m4, m8 + paddsw m1, m4 +%endif +%endmacro + %%h5 + psraw m0, 3 + psraw m1, 3 + paddw m0, m15 + paddw m1, m15 + mova [t1+xq*2+ 0], m0 + mova [t1+xq*2+16], m1 + add xq, 16 + jl .h_loop + ret +ALIGN function_align +.hv: + add lpfq, strideq + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movifnidn leftq, leftmp + mova m4, [lpfq+xq] + movd m5, [leftq] + add leftq, 4 + pslldq m4, 4 + por m4, m5 + movifnidn leftmp, leftq + jmp .hv_main +.hv_extend_left: +%if cpuflag(ssse3) + mova m4, [lpfq+xq] + pshufb m4, m12 +%else + mova m5, [lpfq+xq] + pshufd m4, m5, q2103 + punpcklbw m5, m5 + punpcklwd m5, m5 + movss m4, m5 +%endif + jmp .hv_main +.hv_bottom: + mov xq, wq + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left +.hv_loop: + movu m4, [lpfq+xq-4] +.hv_main: + movu m5, [lpfq+xq+4] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp xd, -17 + jl .hv_have_right + call mangle(private_prefix %+ _wiener_filter7_8bpc %+ SUFFIX).extend_right +.hv_have_right: + %%h5 + mova m2, [t3+xq*2] + paddw m2, [t1+xq*2] + psraw m0, 3 + psraw m1, 3 + paddw m0, m15 + paddw m1, m15 +%if ARCH_X86_64 + mova m3, [t2+xq*2] + paddw m4, m0, [t4+xq*2] +%else + mov r2, t2 + mova m3, [r2+xq*2] + mov r2, t4 + paddw m4, m0, [r2+xq*2] +%endif + mova [t0+xq*2], m0 + punpcklwd m0, m2, m3 + pmaddwd m0, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4, m4 + pmaddwd m3, m6 + punpckhwd m4, m4 + pmaddwd m4, m6 + paddd m0, m3 + paddd m4, m2 + mova m2, [t3+xq*2+16] + paddw m2, [t1+xq*2+16] + packuswb m0, m4 +%if ARCH_X86_64 + mova m3, [t2+xq*2+16] + paddw m4, m1, [t4+xq*2+16] +%else + paddw m4, m1, [r2+xq*2+16] + mov r2, t2 + mova m3, [r2+xq*2+16] + mov dstq, dstmp +%endif + mova [t0+xq*2+16], m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4, m4 + pmaddwd m3, m6 + punpckhwd m4, m4 + pmaddwd m4, m6 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+xq], m0 + add xq, 16 + jl .hv_loop + add dstq, strideq + mov t4, t3 + mov t3, t2 + mov t2, t1 + mov t1, t0 + mov t0, t4 + movifnidn dstmp, dstq + ret +%if cpuflag(ssse3) +.v: + mov xq, wq +.v_loop: + mova m3, [t1+xq*2] + paddw m1, m3, [t3+xq*2] +%if ARCH_X86_64 + mova m2, [t2+xq*2] + paddw m3, [t4+xq*2] +%else + mov r2, t2 + mova m2, [r2+xq*2] + mov r2, t4 + paddw m3, [r2+xq*2] +%endif + punpcklwd m0, m1, m2 + pmaddwd m0, m7 + punpckhwd m1, m2 + pmaddwd m1, m7 + punpcklwd m2, m3 + pmaddwd m2, m6 + punpckhwd m3, m3 + pmaddwd m3, m6 + paddd m0, m2 + paddd m1, m3 + mova m4, [t1+xq*2+16] + paddw m2, m4, [t3+xq*2+16] +%if ARCH_X86_64 + mova m3, [t2+xq*2+16] + paddw m4, [t4+xq*2+16] +%else + paddw m4, [r2+xq*2+16] + mov r2, t2 + mova m3, [r2+xq*2+16] + mov dstq, dstmp +%endif + packuswb m0, m1 + punpcklwd m1, m2, m3 + pmaddwd m1, m7 + punpckhwd m2, m3 + pmaddwd m2, m7 + punpcklwd m3, m4 + pmaddwd m3, m6 + punpckhwd m4, m4 + pmaddwd m4, m6 + paddd m1, m3 + paddd m2, m4 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 + packuswb m0, m1 + mova [dstq+xq], m0 + add xq, 16 + jl .v_loop + ret +%endif +%endmacro + +INIT_XMM sse2 +WIENER + +INIT_XMM ssse3 +WIENER + +;;;;;;;;;;;;;;;;;;;;;;;;;; +;; self-guided ;; +;;;;;;;;;;;;;;;;;;;;;;;;;; + +%macro GATHERDD 3 ; dst, src, tmp + movd %3d, %2 + %if ARCH_X86_64 + movd %1, [r13+%3] + pextrw %3d, %2, 2 + pinsrw %1, [r13+%3+2], 3 + pextrw %3d, %2, 4 + pinsrw %1, [r13+%3+2], 5 + pextrw %3d, %2, 6 + pinsrw %1, [r13+%3+2], 7 + %else + movd %1, [base+sgr_x_by_x-0xf03+%3] + pextrw %3, %2, 2 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 3 + pextrw %3, %2, 4 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 5 + pextrw %3, %2, 6 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 7 + %endif +%endmacro + +%macro GATHER_X_BY_X 5 ; dst, src0, src1, tmp32, tmp32_restore + %if ARCH_X86_64 + %define tmp r14 + %else + %define tmp %4 + %endif + GATHERDD %1, %2, tmp + GATHERDD %2, %3, tmp + movif32 %4, %5 + psrld %1, 24 + psrld %2, 24 + packssdw %1, %2 +%endmacro + +%macro MULLD 3 ; dst, src, tmp + pmulhuw %3, %1, %2 + pmullw %1, %2 + pslld %3, 16 + paddd %1, %3 +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 0, 1, 2, 3, 5 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 5*16 + %else + %assign extra_stack 3*16 + %endif +cglobal sgr_filter_5x5_8bpc, 1, 7, 8, -400*24-16-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*0+4*6] + %define stridemp dword [esp+calloff+16*0+4*7] + %define leftm dword [esp+calloff+16*3+4*0] + %define lpfm dword [esp+calloff+16*3+4*1] + %define w0m dword [esp+calloff+16*3+4*2] + %define hd dword [esp+calloff+16*3+4*3] + %define edgeb byte [esp+calloff+16*3+4*4] + %define edged dword [esp+calloff+16*3+4*4] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t0m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define m8 [base+pb_1] + %define m9 [esp+calloff+16*2] + %define m10 [base+pd_0xf00800a4] + %define m11 [base+sgr_lshuf5] + %define m12 [base+pd_34816] + %define m13 [base+pb_0to15] + %define r10 r4 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +DECLARE_REG_TMP 8, 7, 9, 11, 12 +cglobal sgr_filter_5x5_8bpc, 4, 15, 14, -400*24-16, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + movifnidn hd, hm + mov edged, r7m + movu m9, [paramsq] + add lpfq, wq + mova m8, [pb_1] + lea t1, [rsp+wq*2+20] + mova m10, [pd_0xf00800a4] + add dstq, wq + lea t3, [rsp+wq*4+400*12+16] + mova m12, [pd_34816] ; (1 << 11) + (1 << 15) + lea t4, [rsp+wq*2+400*20+16] + pshufhw m7, m9, q0000 + pshufb m9, [pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + neg wq + mova m13, [pb_0to15] + pxor m6, m6 + mova m11, [sgr_lshuf5] + psllw m7, 4 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + movu m1, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*20+16] + mov t3m, t3 + pshufhw m7, m1, q0000 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 2 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + movif32 t2m, t1 + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t0m, t2 + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, stridemp + movif32 t4, t4m + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp +%if ARCH_X86_64 + test hb, hb +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + call .h + add lpfq, stridemp + call .hv + movif32 dstq, dstm + call .n0 + call .n1 + sub hd, 2 + movif32 t0, t0m + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .h_top + add lpfq, stridemp + call .hv_bottom +.end: + movif32 dstq, dstm + call .n0 + call .n1 +.end2: + RET +.height1: + movif32 t4, t4m + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + movif32 dstq, dstm + call .n0 + call .n1 +.odd_height_end: + call .v + movif32 dstq, dstm + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h + lea t2, [t1+400*6] + movif32 t2m, t2 + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + movif32 t0m, t0 + jmp .main +.no_top_height1: + movif32 t3, t3m + movif32 t4, t4m + call .v + call .prep_n + jmp .odd_height_end +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movd m1, wd + movd m3, [lpfq-1] + pshufb m1, m6 + pshufb m3, m6 + psubb m2, m8, m1 + pcmpgtb m2, m13 + pand m5, m2 + pandn m2, m3 + por m5, m2 + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r4-2] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m11 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-2] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq-1] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -10 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m2, m5, m4, 2 + paddw m0, m4, m2 + palignr m3, m5, m4, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + paddd m1, m3 + punpckhwd m3, m4, m5 + pmaddwd m3, m3 + shufps m4, m5, q2121 + paddw m0, m4 ; sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m2, m3 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+wq*2+400*0] + paddd m1, [t1+wq*2+400*2] + paddd m2, [t1+wq*2+400*4] +.h_loop_end: + paddd m1, m5 ; sumsq + paddd m2, m4 + mova [t1+wq*2+400*0], m0 + mova [t1+wq*2+400*2], m1 + mova [t1+wq*2+400*4], m2 + add wq, 8 + jl .h_loop + ret +.top_fixup: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+wq*2+400*0] + mova m1, [t1+wq*2+400*2] + mova m2, [t1+wq*2+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m1 + mova [t2+wq*2+400*4], m2 + add wq, 8 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv_main +.hv_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m11 + jmp .hv_main +.hv_bottom: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv_loop_start +%endif +.hv_loop: + movif32 lpfq, hvsrcm +.hv_loop_start: + movu m5, [lpfq+wq-1] +.hv_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -10 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t3, hd + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m3, m5, m4, 2 + paddw m0, m4, m3 + palignr m1, m5, m4, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m1, m4, m5 + pmaddwd m1, m1 + paddd m2, m1 + punpckhwd m1, m4, m5 + pmaddwd m1, m1 + shufps m4, m5, q2121 + paddw m0, m4 ; h sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m3, m1 + paddd m2, m5 ; h sumsq + paddd m3, m4 + paddw m1, m0, [t1+wq*2+400*0] + paddd m4, m2, [t1+wq*2+400*2] + paddd m5, m3, [t1+wq*2+400*4] +%if ARCH_X86_64 + test hd, hd +%else + test t3, t3 +%endif + jz .hv_last_row +.hv_main2: + paddw m1, [t2+wq*2+400*0] ; hv sum + paddd m4, [t2+wq*2+400*2] ; hv sumsq + paddd m5, [t2+wq*2+400*4] + mova [t0+wq*2+400*0], m0 + pslld m0, m4, 4 + mova [t0+wq*2+400*2], m2 + mova [t0+wq*2+400*4], m3 + pslld m2, m4, 3 + paddd m4, m0 + pslld m0, m5, 4 + paddd m4, m2 ; a * 25 + pslld m2, m5, 3 + paddd m5, m0 + paddd m5, m2 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m2 ; p * s + MULLD m5, m9, m2 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m12 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m12 + mova [t4+wq*2+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + movif32 t2m, t2 + movif32 t0m, t0 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+wq*2+400*0], m1 + paddw m1, m0 + mova [t1+wq*2+400*2], m4 + paddd m4, m2 + mova [t1+wq*2+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.v_loop: + mova m0, [t1+wq*2+400*0] + mova m2, [t1+wq*2+400*2] + mova m3, [t1+wq*2+400*4] + paddw m1, m0, [t2+wq*2+400*0] + paddd m4, m2, [t2+wq*2+400*2] + paddd m5, m3, [t2+wq*2+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + pslld m0, m4, 4 + paddd m5, m3 + pslld m2, m4, 3 + paddd m4, m0 + pslld m0, m5, 4 + paddd m4, m2 ; a * 25 + pslld m2, m5, 3 + paddd m5, m0 + paddd m5, m2 + punpcklwd m0, m1, m6 + punpckhwd m1, m6 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m2 ; p * s + MULLD m5, m9, m2 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m12 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m12 + mova [t4+wq*2+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+ 2] + movu m3, [t4+wq*2+ 4] + movu m1, [t3+wq*4+ 4] + movu m4, [t3+wq*4+ 8] + movu m2, [t3+wq*4+20] + movu m5, [t3+wq*4+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*2+ 0] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + mova [t4+wq*2+400*2+ 0], m0 + mova [t3+wq*4+400*4+ 0], m1 + mova [t3+wq*4+400*4+16], m2 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*2+ 2] + movu m3, [t4+wq*2+ 4] + movu m1, [t3+wq*4+ 4] + movu m4, [t3+wq*4+ 8] + movu m2, [t3+wq*4+20] + movu m5, [t3+wq*4+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*2+ 0] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + paddw m3, m0, [t4+wq*2+400*2+ 0] + paddd m4, m1, [t3+wq*4+400*4+ 0] + paddd m5, m2, [t3+wq*4+400*4+16] + mova [t4+wq*2+400*2+ 0], m0 + mova [t3+wq*4+400*4+ 0], m1 + mova [t3+wq*4+400*4+16], m2 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + movq m0, [dstq+wq] + mova m3, [t4+wq*2+400*2+ 0] + mova m4, [t3+wq*4+400*4+ 0] + mova m5, [t3+wq*4+400*4+16] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 7) + psubd m5, m3 + psrad m4, 8 + psrad m5, 8 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 4*16 + %else + %assign extra_stack 2*16 + %endif +cglobal sgr_filter_3x3_8bpc, 1, 7, 8, -400*42-16-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*2+4*0] + %define stridemp dword [esp+calloff+16*2+4*1] + %define leftm dword [esp+calloff+16*2+4*2] + %define lpfm dword [esp+calloff+16*2+4*3] + %define w0m dword [esp+calloff+16*2+4*4] + %define hd dword [esp+calloff+16*2+4*5] + %define edgeb byte [esp+calloff+16*2+4*6] + %define edged dword [esp+calloff+16*2+4*6] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %define m8 [base+pb_0to15] + %define m9 [esp+calloff+16*1] + %define m10 [base+pd_0xf00801c7] + %define m11 [base+pd_34816] + %define m12 m6 + %define m13 [base+sgr_lshuf3] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_3x3_8bpc, 4, 15, 14, -400*42-8, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + mov hd, hm + mov edged, r7m + movq m9, [paramsq+4] + add lpfq, wq + lea t1, [rsp+wq*2+12] + mova m8, [pb_0to15] + add dstq, wq + lea t3, [rsp+wq*4+400*12+8] + mova m10, [pd_0xf00801c7] + lea t4, [rsp+wq*2+400*32+8] + mova m11, [pd_34816] + pshuflw m7, m9, q3333 + pshufb m9, [pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + neg wq + pxor m6, m6 + mova m13, [sgr_lshuf3] + psllw m7, 4 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + movq m1, [r1+4] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*32+16] + mov t3m, t3 + pshuflw m7, m1, q3333 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 2 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + mov t2, t1 + add t1, 400*6 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv0 +%if ARCH_X86_64 + test hb, hb +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*6] +.top_fixup_loop: + mova m0, [t1+wq*2+400*0] + mova m1, [t1+wq*2+400*2] + mova m2, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m1 + mova [t2+wq*2+400*4], m2 + add wq, 8 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movd m0, [lpfq-1] + movd m1, wd + mova m3, m8 + pshufb m0, m6 + pshufb m1, m6 + mova m2, m6 + psubb m2, m1 + pcmpgtb m2, m3 + pand m5, m2 + pandn m2, m0 + por m5, m2 + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r4-2] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-2] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -9 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + mova [t1+wq*2+400*0], m1 + mova [t1+wq*2+400*2], m2 + mova [t1+wq*2+400*4], m3 + add wq, 8 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m5, [lpfq+wq] +.hv0_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -9 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + paddw m0, m1, [t1+wq*2+400*0] + paddd m4, m2, [t1+wq*2+400*2] + paddd m5, m3, [t1+wq*2+400*4] + mova [t1+wq*2+400*0], m1 + mova [t1+wq*2+400*2], m2 + mova [t1+wq*2+400*4], m3 + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m5, [lpfq+wq] +.hv1_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -9 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m1, m5, m4, 2 + paddw m0, m4, m1 + punpcklwd m2, m4, m1 + pmaddwd m2, m2 + punpckhwd m3, m4, m1 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m0, m5 ; h sum + punpcklwd m1, m5, m6 + pmaddwd m1, m1 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m1 ; h sumsq + paddd m3, m5 + paddw m1, m0, [t2+wq*2+400*0] + paddd m4, m2, [t2+wq*2+400*2] + paddd m5, m3, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m2 + mova [t2+wq*2+400*4], m3 + pslld m2, m4, 3 + pslld m3, m5, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+400*2 +4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.v1_loop: + mova m0, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+400*0+ 4] + movu m1, [t3+wq*4+400*0+ 8] + movu m2, [t3+wq*4+400*0+24] + movu m3, [t4+wq*2+400*0+ 2] + movu m4, [t3+wq*4+400*0+ 4] + movu m5, [t3+wq*4+400*0+20] + paddw m0, [t4+wq*2+400*0+ 0] + paddd m1, [t3+wq*4+400*0+ 0] + paddd m2, [t3+wq*4+400*0+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[-1] 444 + pslld m4, 2 ; b[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a[-1] 343 + psubd m4, m1 ; b[-1] 343 + psubd m5, m2 + mova [t4+wq*2+400*4], m3 + mova [t3+wq*4+400*8+ 0], m4 + mova [t3+wq*4+400*8+16], m5 + movu m0, [t4+wq*2+400*2+ 4] + movu m1, [t3+wq*4+400*4+ 8] + movu m2, [t3+wq*4+400*4+24] + movu m3, [t4+wq*2+400*2+ 2] + movu m4, [t3+wq*4+400*4+ 4] + movu m5, [t3+wq*4+400*4+20] + paddw m0, [t4+wq*2+400*2+ 0] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m2, [t3+wq*4+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[ 0] 444 + pslld m4, 2 ; b[ 0] 444 + pslld m5, 2 + mova [t4+wq*2+400* 6], m3 + mova [t3+wq*4+400*12+ 0], m4 + mova [t3+wq*4+400*12+16], m5 + psubw m3, m0 ; a[ 0] 343 + psubd m4, m1 ; b[ 0] 343 + psubd m5, m2 + mova [t4+wq*2+400* 8], m3 + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m5 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m3, [t4+wq*2+400*0+4] + movu m1, [t4+wq*2+400*0+2] + paddw m3, [t4+wq*2+400*0+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*2+400*4] + paddw m3, [t4+wq*2+400*6] + mova [t4+wq*2+400*4], m2 + mova [t4+wq*2+400*6], m1 + movu m4, [t3+wq*4+400*0+8] + movu m1, [t3+wq*4+400*0+4] + paddd m4, [t3+wq*4+400*0+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*4+400* 8+ 0] + paddd m4, [t3+wq*4+400*12+ 0] + mova [t3+wq*4+400* 8+ 0], m2 + mova [t3+wq*4+400*12+ 0], m1 + movu m5, [t3+wq*4+400*0+24] + movu m1, [t3+wq*4+400*0+20] + paddd m5, [t3+wq*4+400*0+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*4+400* 8+16] + paddd m5, [t3+wq*4+400*12+16] + mova [t3+wq*4+400* 8+16], m2 + mova [t3+wq*4+400*12+16], m1 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*2+400*2+4] + movu m1, [t4+wq*2+400*2+2] + paddw m3, [t4+wq*2+400*2+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*2+400*6] + paddw m3, [t4+wq*2+400*8] + mova [t4+wq*2+400*6], m1 + mova [t4+wq*2+400*8], m2 + movu m4, [t3+wq*4+400*4+8] + movu m1, [t3+wq*4+400*4+4] + paddd m4, [t3+wq*4+400*4+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*4+400*12+ 0] + paddd m4, [t3+wq*4+400*16+ 0] + mova [t3+wq*4+400*12+ 0], m1 + mova [t3+wq*4+400*16+ 0], m2 + movu m5, [t3+wq*4+400*4+24] + movu m1, [t3+wq*4+400*4+20] + paddd m5, [t3+wq*4+400*4+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*4+400*12+16] + paddd m5, [t3+wq*4+400*16+16] + mova [t3+wq*4+400*12+16], m1 + mova [t3+wq*4+400*16+16], m2 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 10*16 + %else + %assign extra_stack 8*16 + %endif +cglobal sgr_filter_mix_8bpc, 1, 7, 8, -400*66-48-extra_stack, \ + dst, stride, left, lpf, w + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*8+4*0] + %define stridemp dword [esp+calloff+16*8+4*1] + %define leftm dword [esp+calloff+16*8+4*2] + %define lpfm dword [esp+calloff+16*8+4*3] + %define w0m dword [esp+calloff+16*8+4*4] + %define hd dword [esp+calloff+16*8+4*5] + %define edgeb byte [esp+calloff+16*8+4*6] + %define edged dword [esp+calloff+16*8+4*6] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r5m + %define edgeb byte r7m + %define edged dword r7m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %xdefine m8 m6 + %define m9 [base+pd_0xffff] + %define m10 [base+pd_34816] + %define m11 [base+pd_0xf00801c7] + %define m12 [base+pd_0xf00800a4] + %define m13 [esp+calloff+16*4] + %define m14 [esp+calloff+16*5] + %define m15 [esp+calloff+16*6] + %define m6 [esp+calloff+16*7] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov wd, [rstk+stack_offset+20] + mov dstm, dstq + mov stridemp, strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+24] + mov r2, [rstk+stack_offset+32] + mov lpfm, lpfq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_mix_8bpc, 4, 15, 16, -400*66-40, dst, stride, left, lpf, \ + w, h, edge, params +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, r6mp + lea r13, [sgr_x_by_x-0xf03] + movifnidn hd, hm + mov edged, r7m + mova m15, [paramsq] + add lpfq, wq + mova m9, [pd_0xffff] + lea t1, [rsp+wq*2+44] + mova m10, [pd_34816] + add dstq, wq + lea t3, [rsp+wq*4+400*24+40] + mova m11, [pd_0xf00801c7] + lea t4, [rsp+wq*2+400*52+40] + mova m12, [base+pd_0xf00800a4] + neg wq + pshuflw m13, m15, q0000 + pshuflw m14, m15, q2222 + pshufhw m15, m15, q1010 + punpcklqdq m13, m13 ; s0 + punpcklqdq m14, m14 ; s1 + punpckhqdq m15, m15 ; w0 w1 + pxor m6, m6 + psllw m15, 2 + DEFINE_ARGS dst, stride, left, lpf, _, h, edge, _, _, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+28] ; params + LEA r6, $$ + mova m2, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+52] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*24+48] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*52+48] + mov t3m, t3 + mov t4m, t4 + neg wq + pshuflw m0, m2, q0000 + pshuflw m1, m2, q2222 + pshufhw m2, m2, q1010 + punpcklqdq m0, m0 ; s0 + punpcklqdq m1, m1 ; s1 + punpckhqdq m2, m2 ; w0 w1 + mov w1m, wd + pxor m3, m3 + psllw m2, 2 + mova m13, m0 + mova m14, m1 + sub wd, 2 + mova m15, m2 + mova m6, m3 + mov lpfq, lpfm + mov w0m, wd + %define strideq r5 +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, stridemp + mov t2, t1 +%if ARCH_X86_64 + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup +%else + mov wq, w0m + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup_loop +%endif + add t1, 400*12 + call .h_top + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + add r10, strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv0 +%if ARCH_X86_64 + test hd, hd +%else + mov r4, hd + test r4, r4 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom + movif32 lpfq, hvsrcm + add lpfq, stridemp + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + movif32 strideq, stridemp + lea r10, [lpfq+strideq*4] + mov lpfq, dstq + lea r10, [r10+strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*12] +.top_fixup_loop: + mova m0, [t1+wq*2+400* 0] + mova m1, [t1+wq*2+400* 2] + mova m2, [t1+wq*2+400* 4] + paddw m0, m0 + mova m3, [t1+wq*2+400* 6] + paddd m1, m1 + mova m4, [t1+wq*2+400* 8] + paddd m2, m2 + mova m5, [t1+wq*2+400*10] + mova [t2+wq*2+400* 0], m0 + mova [t2+wq*2+400* 2], m1 + mova [t2+wq*2+400* 4], m2 + mova [t2+wq*2+400* 6], m3 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 + add wq, 8 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 +%if ARCH_X86_64 + SWAP m8, m6 +%endif + movd m1, wd + movd m3, [lpfq-1] + pshufb m1, m8 + pshufb m3, m8 + psubb m2, [base+pb_1], m1 + pcmpgtb m2, [base+pb_0to15] + pand m5, m2 + pandn m2, m3 + por m5, m2 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r4-2] +%else + %define leftq r4 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r4-2] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq-1] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT +%if ARCH_X86_32 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .h_have_right + cmp wd, -10 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; sum3 + punpcklwd m7, m0, m8 + pmaddwd m7, m7 + punpckhwd m0, m8 + pmaddwd m0, m0 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + paddd m2, m7 ; sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + mova [t1+wq*2+400* 6], m1 + mova [t1+wq*2+400* 8], m2 + mova [t1+wq*2+400*10], m3 + paddw m8, m1 ; sum5 + paddd m7, m2 ; sumsq5 + paddd m5, m3 + mova [t1+wq*2+400* 0], m8 + mova [t1+wq*2+400* 2], m7 + mova [t1+wq*2+400* 4], m5 + add wq, 8 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m5, [lpfq+wq-1] +.hv0_main: + test edgeb, 2 ; LR_HAVE_RIGHT +%if ARCH_X86_32 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .hv0_have_right + cmp wd, -10 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + movif32 t3, t3m + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; h sum3 + punpcklwd m7, m0, m8 + pmaddwd m7, m7 + punpckhwd m0, m8 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + pmaddwd m0, m0 + paddd m2, m7 ; h sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + paddw m8, m1 ; h sum5 + paddd m7, m2 ; h sumsq5 + paddd m5, m3 + mova [t3+wq*4+400*8+ 8], m8 + mova [t3+wq*4+400*0+ 8], m7 + mova [t3+wq*4+400*0+24], m5 + paddw m8, [t1+wq*2+400* 0] + paddd m7, [t1+wq*2+400* 2] + paddd m5, [t1+wq*2+400* 4] + mova [t1+wq*2+400* 0], m8 + mova [t1+wq*2+400* 2], m7 + mova [t1+wq*2+400* 4], m5 + paddw m0, m1, [t1+wq*2+400* 6] + paddd m4, m2, [t1+wq*2+400* 8] + paddd m5, m3, [t1+wq*2+400*10] + mova [t1+wq*2+400* 6], m1 + mova [t1+wq*2+400* 8], m2 + mova [t1+wq*2+400*10], m3 + paddw m1, m0, [t2+wq*2+400* 6] + paddd m2, m4, [t2+wq*2+400* 8] + paddd m3, m5, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m0 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m5, [lpfq+wq-1] +.hv1_main: + test edgeb, 2 ; LR_HAVE_RIGHT +%if ARCH_X86_32 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .hv1_have_right + cmp wd, -10 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m7, m5, m4, 2 + palignr m3, m5, m4, 4 + paddw m2, m7, m3 + punpcklwd m0, m7, m3 + pmaddwd m0, m0 + punpckhwd m7, m3 + pmaddwd m7, m7 + palignr m3, m5, m4, 6 + paddw m2, m3 ; h sum3 + punpcklwd m1, m3, m8 + pmaddwd m1, m1 + punpckhwd m3, m8 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + pmaddwd m3, m3 + paddd m0, m1 ; h sumsq3 + palignr m5, m4, 8 + punpckhwd m1, m4, m5 + paddw m8, m4, m5 + pmaddwd m1, m1 + punpcklwd m4, m5 + pmaddwd m4, m4 + paddd m7, m3 + paddw m5, m2, [t2+wq*2+400* 6] + mova [t2+wq*2+400* 6], m2 + paddw m8, m2 ; h sum5 + paddd m2, m0, [t2+wq*2+400* 8] + paddd m3, m7, [t2+wq*2+400*10] + mova [t2+wq*2+400* 8], m0 + mova [t2+wq*2+400*10], m7 + paddd m4, m0 ; h sumsq5 + paddd m1, m7 + pslld m0, m2, 3 + pslld m7, m3, 3 + paddd m2, m0 ; a3 * 9 + paddd m3, m7 +%if ARCH_X86_32 + mova [esp+20], m8 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + punpcklwd m0, m5, m8 ; b3 + pmaddwd m7, m0, m0 + punpckhwd m5, m8 + pmaddwd m8, m5, m5 + psubd m2, m7 ; p3 + psubd m3, m8 + MULLD m2, m14, m8 ; p3 * s1 + MULLD m3, m14, m8 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m5, m11 + paddusw m2, m11 + paddusw m3, m11 + psrld m2, 20 ; min(z3, 255) + movif32 t3, t3m + psrld m3, 20 + GATHER_X_BY_X m8, m2, m3, r0, dstm + punpcklwd m2, m8, m8 + punpckhwd m3, m8, m8 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m5, m10 + psrld m0, 12 + psrld m5, 12 + mova [t4+wq*2+400*4+ 4], m8 + mova [t3+wq*4+400*8+ 8], m0 + mova [t3+wq*4+400*8+24], m5 +%if ARCH_X86_32 + mova m8, [esp+20] +%else + SWAP m6, m8 + pxor m6, m6 +%endif + paddw m5, m8, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m1, [t2+wq*2+400*4] + paddw m5, [t1+wq*2+400*0] + paddd m2, [t1+wq*2+400*2] + paddd m3, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m8 + pslld m0, m2, 4 + mova [t2+wq*2+400*2], m4 + pslld m8, m3, 4 + mova [t2+wq*2+400*4], m1 + pslld m4, m2, 3 + paddd m2, m0 + pslld m7, m3, 3 + paddd m3, m8 + paddd m2, m4 ; a5 * 25 + paddd m3, m7 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + punpcklwd m0, m5, m7 ; b5 + pmaddwd m4, m0, m0 + punpckhwd m5, m7 + pmaddwd m1, m5, m5 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + psubd m2, m4 ; p5 + psubd m3, m1 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m5, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m1, m2, m3, r0, dstm + punpcklwd m2, m1, m1 + punpckhwd m3, m1, m1 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m5, m10 + mova [t4+wq*2+4], m1 + psrld m0, 12 + psrld m5, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m5 + add wq, 8 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq*2+400* 6] + mova m4, [t1+wq*2+400* 8] + mova m5, [t1+wq*2+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq*2+400* 6] + paddd m2, m4, [t2+wq*2+400* 8] + paddd m3, m5, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m0 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + psubd m5, m3 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova m3, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + mova [t3+wq*4+400*8+ 8], m3 + mova [t3+wq*4+400*0+ 8], m4 + mova [t3+wq*4+400*0+24], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+wq*2+400*0], m3 + mova [t1+wq*2+400*2], m4 + mova [t1+wq*2+400*4], m5 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r4-2] +%else + mov wd, w0m +%endif +.v1_loop: + mova m4, [t1+wq*2+400* 6] + mova m5, [t1+wq*2+400* 8] + mova m7, [t1+wq*2+400*10] + paddw m1, m4, [t2+wq*2+400* 6] + paddd m2, m5, [t2+wq*2+400* 8] + paddd m3, m7, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m4 + mova [t2+wq*2+400* 8], m5 + mova [t2+wq*2+400*10], m7 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + psubd m5, m3 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*4+4], m3 + psrld m0, 12 + psrld m8, m1, 12 + mova m4, [t3+wq*4+400*8+ 8] + mova m5, [t3+wq*4+400*0+ 8] + mova m7, [t3+wq*4+400*0+24] + paddw m1, m4, [t2+wq*2+400*0] + paddd m2, m5, [t2+wq*2+400*2] + paddd m3, m7, [t2+wq*2+400*4] + paddw m1, [t1+wq*2+400*0] + paddd m2, [t1+wq*2+400*2] + paddd m3, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m4 + mova [t2+wq*2+400*2], m5 + mova [t2+wq*2+400*4], m7 + pslld m4, m2, 4 + mova [t3+wq*4+400*8+ 8], m0 + pslld m5, m3, 4 + mova [t3+wq*4+400*8+24], m8 + pslld m7, m2, 3 + paddd m2, m4 + pslld m8, m3, 3 + paddd m3, m5 + paddd m2, m7 ; a5 * 25 + paddd m3, m8 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + punpcklwd m0, m1, m7 ; b5 + pmaddwd m4, m0, m0 + punpckhwd m1, m7 + pmaddwd m5, m1, m1 + psubd m2, m4 ; p5 + psubd m3, m5 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m1, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m4, m2, m3, r0, dstm + punpcklwd m2, m4, m4 + punpckhwd m3, m4, m4 + MULLD m0, m2, m7 + MULLD m1, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+4], m4 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r4 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+400*0+ 2] + movu m1, [t3+wq*4+400*0+ 4] + movu m2, [t3+wq*4+400*0+20] + movu m7, [t4+wq*2+400*0+ 4] + movu m8, [t3+wq*4+400*0+ 8] + paddw m3, m0, [t4+wq*2+400*0+ 0] + paddd m4, m1, [t3+wq*4+400*0+ 0] + paddd m5, m2, [t3+wq*4+400*0+16] + paddw m3, m7 + paddd m4, m8 + movu m7, [t3+wq*4+400*0+24] + paddw m0, m3 + paddd m1, m4 + psllw m3, 2 + pslld m4, 2 + paddd m5, m7 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a5 565 + paddd m1, m4 ; b5 565 + paddd m2, m5 + mova [t4+wq*2+400* 6+ 0], m0 + mova [t3+wq*4+400*12+ 0], m1 + mova [t3+wq*4+400*12+16], m2 + movu m0, [t4+wq*2+400*2+ 4] + movu m1, [t3+wq*4+400*4+ 8] + movu m2, [t3+wq*4+400*4+24] + movu m3, [t4+wq*2+400*2+ 2] + movu m4, [t3+wq*4+400*4+ 4] + movu m5, [t3+wq*4+400*4+20] + paddw m0, [t4+wq*2+400*2+ 0] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m2, [t3+wq*4+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[-1] 444 + pslld m4, 2 ; b3[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a3[-1] 343 + psubd m4, m1 ; b3[-1] 343 + psubd m5, m2 + mova [t4+wq*2+400* 8+ 0], m3 + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m5 + movu m0, [t4+wq*2+400*4+ 4] + movu m1, [t3+wq*4+400*8+ 8] + movu m2, [t3+wq*4+400*8+24] + movu m3, [t4+wq*2+400*4+ 2] + movu m4, [t3+wq*4+400*8+ 4] + movu m5, [t3+wq*4+400*8+20] + paddw m0, [t4+wq*2+400*4+ 0] + paddd m1, [t3+wq*4+400*8+ 0] + paddd m2, [t3+wq*4+400*8+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[ 0] 444 + pslld m4, 2 ; b3[ 0] 444 + pslld m5, 2 + mova [t4+wq*2+400*10+ 0], m3 + mova [t3+wq*4+400*20+ 0], m4 + mova [t3+wq*4+400*20+16], m5 + psubw m3, m0 ; a3[ 0] 343 + psubd m4, m1 ; b3[ 0] 343 + psubd m5, m2 + mova [t4+wq*2+400*12+ 0], m3 + mova [t3+wq*4+400*24+ 0], m4 + mova [t3+wq*4+400*24+16], m5 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r4 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*2+ 4] + movu m2, [t4+wq*2+ 2] + paddw m0, [t4+wq*2+ 0] + paddw m0, m2 + paddw m2, m0 + psllw m0, 2 + paddw m0, m2 ; a5 + movu m4, [t3+wq*4+ 8] + movu m5, [t3+wq*4+24] + movu m1, [t3+wq*4+ 4] + movu m3, [t3+wq*4+20] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddd m4, m1 + paddd m5, m3 + paddd m1, m4 + paddd m3, m5 + pslld m4, 2 + pslld m5, 2 + paddd m4, m1 ; b5 + paddd m5, m3 + movu m2, [t4+wq*2+400* 6] + paddw m2, m0 + mova [t4+wq*2+400* 6], m0 + paddd m0, m4, [t3+wq*4+400*12+ 0] + paddd m1, m5, [t3+wq*4+400*12+16] + mova [t3+wq*4+400*12+ 0], m4 + mova [t3+wq*4+400*12+16], m5 + mova [rsp+16+ARCH_X86_32*4], m1 + movu m3, [t4+wq*2+400*2+4] + movu m5, [t4+wq*2+400*2+2] + paddw m3, [t4+wq*2+400*2+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + movu m3, [t4+wq*2+400* 8] + paddw m3, [t4+wq*2+400*10] + paddw m3, m4 + mova [t4+wq*2+400* 8], m4 + mova [t4+wq*2+400*10], m5 + movu m1, [t3+wq*4+400*4+ 8] + movu m5, [t3+wq*4+400*4+ 4] + movu m7, [t3+wq*4+400*4+24] + movu m8, [t3+wq*4+400*4+20] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m7, [t3+wq*4+400*4+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 +%if ARCH_X86_32 + mova [esp+52], m8 + psubd m8, m7 +%else + psubd m6, m8, m7 + SWAP m8, m6 +%endif + paddd m1, m4, [t3+wq*4+400*16+ 0] + paddd m7, m8, [t3+wq*4+400*16+16] + paddd m1, [t3+wq*4+400*20+ 0] + paddd m7, [t3+wq*4+400*20+16] + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m8 + mova [t3+wq*4+400*20+ 0], m5 +%if ARCH_X86_32 + mova m8, [esp+52] +%else + SWAP m8, m6 + pxor m6, m6 +%endif + mova [t3+wq*4+400*20+16], m8 + mova [rsp+32+ARCH_X86_32*4], m7 + movq m4, [dstq+wq] + punpcklbw m4, m6 + punpcklwd m5, m4, m6 + punpcklwd m7, m2, m6 + pmaddwd m7, m5 ; a5 * src + punpcklwd m8, m3, m6 + pmaddwd m8, m5 ; a3 * src + punpckhwd m5, m4, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + psubd m0, m7 ; b5 - a5 * src + (1 << 8) - (src << 13) + psubd m1, m8 ; b3 - a3 * src + (1 << 8) - (src << 13) + psrld m0, 9 + pslld m1, 7 + pand m0, m9 + pandn m8, m9, m1 + por m0, m8 + mova m1, [rsp+16+ARCH_X86_32*4] + psubd m1, m2 + mova m2, [rsp+32+ARCH_X86_32*4] + psubd m2, m3 + mova m3, [base+pd_4096] + psrld m1, 9 + pslld m2, 7 + pand m1, m9 + pandn m5, m9, m2 + por m1, m5 + pmaddwd m0, m15 + pmaddwd m1, m15 + paddd m0, m3 + paddd m1, m3 + psrad m0, 13 + psrad m1, 13 + packssdw m0, m1 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r4 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*2+400*4+4] + movu m5, [t4+wq*2+400*4+2] + paddw m3, [t4+wq*2+400*4+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + paddw m3, m4, [t4+wq*2+400*12] + paddw m3, [t4+wq*2+400*10] + mova [t4+wq*2+400*10], m5 + mova [t4+wq*2+400*12], m4 + movu m1, [t3+wq*4+400*8+ 8] + movu m5, [t3+wq*4+400*8+ 4] + movu m7, [t3+wq*4+400*8+24] + movu m8, [t3+wq*4+400*8+20] + paddd m1, [t3+wq*4+400*8+ 0] + paddd m7, [t3+wq*4+400*8+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 + psubd m0, m8, m7 + paddd m1, m4, [t3+wq*4+400*24+ 0] + paddd m7, m0, [t3+wq*4+400*24+16] + paddd m1, [t3+wq*4+400*20+ 0] + paddd m7, [t3+wq*4+400*20+16] + mova [t3+wq*4+400*20+ 0], m5 + mova [t3+wq*4+400*20+16], m8 + mova [t3+wq*4+400*24+ 0], m4 + mova [t3+wq*4+400*24+16], m0 + movq m5, [dstq+wq] + mova m2, [t4+wq*2+400* 6] + punpcklbw m5, m6 + punpcklwd m4, m5, m6 + punpcklwd m8, m2, m6 + pmaddwd m8, m4 ; a5 * src + punpcklwd m0, m3, m6 + pmaddwd m0, m4 ; a3 * src + punpckhwd m4, m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m4 + punpckhwd m3, m6 + pmaddwd m3, m4 + psubd m1, m0 ; b3 - a3 * src + (1 << 8) - (src << 13) + mova m0, [t3+wq*4+400*12+ 0] + psubd m0, m8 ; b5 - a5 * src + (1 << 8) - (src << 13) + mova m4, [t3+wq*4+400*12+16] + psubd m4, m2 + psubd m7, m3 + pslld m1, 7 + psrld m0, 8 + psrld m4, 8 + pslld m7, 7 + pandn m3, m9, m1 + pand m0, m9 + por m0, m3 + pand m4, m9 + pandn m2, m9, m7 + por m2, m4 + mova m1, [base+pd_4096] + pmaddwd m0, m15 + pmaddwd m2, m15 + paddd m0, m1 + paddd m2, m1 + psrad m0, 13 + psrad m2, 13 + packssdw m0, m2 + paddw m0, m5 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, stridemp + movif32 dstm, dstq + ret diff --git a/third_party/dav1d/src/x86/mc.h b/third_party/dav1d/src/x86/mc.h new file mode 100644 index 0000000000..65c607e180 --- /dev/null +++ b/third_party/dav1d/src/x86/mc.h @@ -0,0 +1,299 @@ +/* + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018-2021, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/mc.h" + +#define decl_fn(type, name) \ + decl_##type##_fn(BF(name, sse2)); \ + decl_##type##_fn(BF(name, ssse3)); \ + decl_##type##_fn(BF(name, avx2)); \ + decl_##type##_fn(BF(name, avx512icl)); +#define init_mc_fn(type, name, suffix) \ + c->mc[type] = BF(dav1d_put_##name, suffix) +#define init_mct_fn(type, name, suffix) \ + c->mct[type] = BF(dav1d_prep_##name, suffix) +#define init_mc_scaled_fn(type, name, suffix) \ + c->mc_scaled[type] = BF(dav1d_put_##name, suffix) +#define init_mct_scaled_fn(type, name, suffix) \ + c->mct_scaled[type] = BF(dav1d_prep_##name, suffix) + +decl_fn(mc, dav1d_put_8tap_regular); +decl_fn(mc, dav1d_put_8tap_regular_smooth); +decl_fn(mc, dav1d_put_8tap_regular_sharp); +decl_fn(mc, dav1d_put_8tap_smooth); +decl_fn(mc, dav1d_put_8tap_smooth_regular); +decl_fn(mc, dav1d_put_8tap_smooth_sharp); +decl_fn(mc, dav1d_put_8tap_sharp); +decl_fn(mc, dav1d_put_8tap_sharp_regular); +decl_fn(mc, dav1d_put_8tap_sharp_smooth); +decl_fn(mc, dav1d_put_bilin); + +decl_fn(mct, dav1d_prep_8tap_regular); +decl_fn(mct, dav1d_prep_8tap_regular_smooth); +decl_fn(mct, dav1d_prep_8tap_regular_sharp); +decl_fn(mct, dav1d_prep_8tap_smooth); +decl_fn(mct, dav1d_prep_8tap_smooth_regular); +decl_fn(mct, dav1d_prep_8tap_smooth_sharp); +decl_fn(mct, dav1d_prep_8tap_sharp); +decl_fn(mct, dav1d_prep_8tap_sharp_regular); +decl_fn(mct, dav1d_prep_8tap_sharp_smooth); +decl_fn(mct, dav1d_prep_bilin); + +decl_fn(mc_scaled, dav1d_put_8tap_scaled_regular); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_regular_smooth); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_regular_sharp); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_smooth); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_smooth_regular); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_smooth_sharp); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_sharp); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_sharp_regular); +decl_fn(mc_scaled, dav1d_put_8tap_scaled_sharp_smooth); +decl_fn(mc_scaled, dav1d_put_bilin_scaled); + +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_regular); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_regular_smooth); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_regular_sharp); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_smooth); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_smooth_regular); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_smooth_sharp); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_sharp); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_sharp_regular); +decl_fn(mct_scaled, dav1d_prep_8tap_scaled_sharp_smooth); +decl_fn(mct_scaled, dav1d_prep_bilin_scaled); + +decl_fn(avg, dav1d_avg); +decl_fn(w_avg, dav1d_w_avg); +decl_fn(mask, dav1d_mask); +decl_fn(w_mask, dav1d_w_mask_420); +decl_fn(w_mask, dav1d_w_mask_422); +decl_fn(w_mask, dav1d_w_mask_444); +decl_fn(blend, dav1d_blend); +decl_fn(blend_dir, dav1d_blend_v); +decl_fn(blend_dir, dav1d_blend_h); + +decl_fn(warp8x8, dav1d_warp_affine_8x8); +decl_warp8x8_fn(BF(dav1d_warp_affine_8x8, sse4)); +decl_fn(warp8x8t, dav1d_warp_affine_8x8t); +decl_warp8x8t_fn(BF(dav1d_warp_affine_8x8t, sse4)); + +decl_fn(emu_edge, dav1d_emu_edge); + +decl_fn(resize, dav1d_resize); + +static ALWAYS_INLINE void mc_dsp_init_x86(Dav1dMCDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if(!(flags & DAV1D_X86_CPU_FLAG_SSE2)) + return; + +#if BITDEPTH == 8 + init_mct_fn(FILTER_2D_BILINEAR, bilin, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, sse2); + + c->warp8x8 = BF(dav1d_warp_affine_8x8, sse2); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, sse2); +#endif + + if(!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) + return; + + init_mc_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, ssse3); + init_mc_fn(FILTER_2D_BILINEAR, bilin, ssse3); + + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, ssse3); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, ssse3); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, ssse3); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, ssse3); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, ssse3); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, ssse3); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, ssse3); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, ssse3); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, ssse3); + init_mct_fn(FILTER_2D_BILINEAR, bilin, ssse3); + + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, ssse3); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, ssse3); + init_mc_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, ssse3); + + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, ssse3); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, ssse3); + init_mct_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, ssse3); + + c->avg = BF(dav1d_avg, ssse3); + c->w_avg = BF(dav1d_w_avg, ssse3); + c->mask = BF(dav1d_mask, ssse3); + c->w_mask[0] = BF(dav1d_w_mask_444, ssse3); + c->w_mask[1] = BF(dav1d_w_mask_422, ssse3); + c->w_mask[2] = BF(dav1d_w_mask_420, ssse3); + c->blend = BF(dav1d_blend, ssse3); + c->blend_v = BF(dav1d_blend_v, ssse3); + c->blend_h = BF(dav1d_blend_h, ssse3); + c->warp8x8 = BF(dav1d_warp_affine_8x8, ssse3); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, ssse3); + c->emu_edge = BF(dav1d_emu_edge, ssse3); + c->resize = BF(dav1d_resize, ssse3); + + if(!(flags & DAV1D_X86_CPU_FLAG_SSE41)) + return; + +#if BITDEPTH == 8 + c->warp8x8 = BF(dav1d_warp_affine_8x8, sse4); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, sse4); +#endif + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) + return; + + init_mc_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); + init_mc_fn(FILTER_2D_BILINEAR, bilin, avx2); + + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx2); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); + init_mct_fn(FILTER_2D_BILINEAR, bilin, avx2); + + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, avx2); + + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, avx2); + + c->avg = BF(dav1d_avg, avx2); + c->w_avg = BF(dav1d_w_avg, avx2); + c->mask = BF(dav1d_mask, avx2); + c->w_mask[0] = BF(dav1d_w_mask_444, avx2); + c->w_mask[1] = BF(dav1d_w_mask_422, avx2); + c->w_mask[2] = BF(dav1d_w_mask_420, avx2); + c->blend = BF(dav1d_blend, avx2); + c->blend_v = BF(dav1d_blend_v, avx2); + c->blend_h = BF(dav1d_blend_h, avx2); + c->warp8x8 = BF(dav1d_warp_affine_8x8, avx2); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, avx2); + c->emu_edge = BF(dav1d_emu_edge, avx2); + c->resize = BF(dav1d_resize, avx2); + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) + return; + + init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, avx512icl); + init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx512icl); + init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx512icl); + init_mc_fn (FILTER_2D_8TAP_SHARP, 8tap_sharp, avx512icl); + init_mc_fn (FILTER_2D_BILINEAR, bilin, avx512icl); + + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx512icl); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx512icl); + init_mct_fn(FILTER_2D_BILINEAR, bilin, avx512icl); + + c->avg = BF(dav1d_avg, avx512icl); + c->w_avg = BF(dav1d_w_avg, avx512icl); + c->mask = BF(dav1d_mask, avx512icl); + c->w_mask[0] = BF(dav1d_w_mask_444, avx512icl); + c->w_mask[1] = BF(dav1d_w_mask_422, avx512icl); + c->w_mask[2] = BF(dav1d_w_mask_420, avx512icl); + c->blend = BF(dav1d_blend, avx512icl); + c->blend_v = BF(dav1d_blend_v, avx512icl); + c->blend_h = BF(dav1d_blend_h, avx512icl); + c->warp8x8 = BF(dav1d_warp_affine_8x8, avx512icl); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, avx512icl); + c->resize = BF(dav1d_resize, avx512icl); +#endif +} diff --git a/third_party/dav1d/src/x86/mc16_avx2.asm b/third_party/dav1d/src/x86/mc16_avx2.asm new file mode 100644 index 0000000000..61eeaa1007 --- /dev/null +++ b/third_party/dav1d/src/x86/mc16_avx2.asm @@ -0,0 +1,5879 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +; dav1d_obmc_masks[] * -512 +const obmc_masks_avx2 + dw 0, 0, -9728, 0, -12800, -7168, -2560, 0 + dw -14336, -11264, -8192, -5632, -3584, -1536, 0, 0 + dw -15360, -13824, -12288, -10752, -9216, -7680, -6144, -5120 + dw -4096, -3072, -2048, -1536, 0, 0, 0, 0 + dw -15872, -14848, -14336, -13312, -12288, -11776, -10752, -10240 + dw -9728, -8704, -8192, -7168, -6656, -6144, -5632, -4608 + dw -4096, -3584, -3072, -2560, -2048, -2048, -1536, -1024 + dw 0, 0, 0, 0, 0, 0, 0, 0 + +deint_shuf: dd 0, 4, 1, 5, 2, 6, 3, 7 +subpel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 +subpel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 +subpel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 +subpel_s_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 +subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 +rescale_mul2: dd 0, 1, 4, 5, 2, 3, 6, 7 +resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +bdct_lb_q: times 8 db 0 + times 8 db 4 + times 8 db 8 + times 8 db 12 + +prep_mul: dw 16, 16, 4, 4 +put_bilin_h_rnd: dw 8, 8, 10, 10 +put_8tap_h_rnd: dd 34, 40 +s_8tap_h_rnd: dd 2, 8 +s_8tap_h_sh: dd 2, 4 +put_s_8tap_v_rnd: dd 512, 128 +put_s_8tap_v_sh: dd 10, 8 +prep_8tap_1d_rnd: dd 8 - (8192 << 4) +prep_8tap_2d_rnd: dd 32 - (8192 << 5) +warp8x8t_rnd: dd 16384 - (8192 << 15) +warp8x8_shift: dd 5, 3 +warp8x8_rnd: dw 4096, 4096, 16384, 16384 +bidir_rnd: dw -16400, -16400, -16388, -16388 +bidir_mul: dw 2048, 2048, 8192, 8192 + +%define pw_16 prep_mul +%define pd_512 put_s_8tap_v_rnd + +pw_2: times 2 dw 2 +pw_64: times 2 dw 64 +pw_2048: times 2 dw 2048 +pw_8192: times 2 dw 8192 +pw_27615: times 2 dw 27615 +pw_32766: times 2 dw 32766 +pw_m512: times 2 dw -512 +pd_32: dd 32 +pd_63: dd 63 +pd_64: dd 64 +pd_32768: dd 32768 +pd_65538: dd 65538 +pd_m524256: dd -524256 ; -8192 << 6 + 32 +pd_0x3ff: dd 0x3ff +pq_0x40000000: dq 0x40000000 + dd 0 + +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +BIDIR_JMP_TABLE avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, avx2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, avx2, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, avx2, 2, 4, 8, 16, 32, 64, 128 + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_16bpc_avx2.put) +%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_16bpc_avx2.prep) + +BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 + +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_16bpc_%2) +%%table: + %rep %0 - 2 + dw %%base %+ .w%3 - %%base + %rotate 1 + %endrep + %rotate 2 + %%dy_1024: + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base + %rotate 1 + %endrep + %rotate 2 + %%dy_2048: + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base + %rotate 1 + %endrep +%endmacro + +SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128 + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +cextern mc_warp_filter +cextern resize_filter + +SECTION .text + +INIT_XMM avx2 +cglobal put_bilin_16bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy + mov mxyd, r6m ; mx + lea r7, [put_avx2] +%if UNIX64 + DECLARE_REG_TMP 8 + %define org_w r8d + mov r8d, wd +%else + DECLARE_REG_TMP 7 + %define org_w wm +%endif + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [r7+wq*2+table_offset(put,)] + add wq, r7 + jmp wq +.put_w2: + mov r6d, [srcq+ssq*0] + mov r7d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6d + mov [dstq+dsq*1], r7d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r6, [srcq+ssq*0] + mov r7, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6 + mov [dstq+dsq*1], r7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +INIT_YMM avx2 +.put_w16: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu m0, [srcq+ssq*0+32*0] + movu m1, [srcq+ssq*0+32*1] + movu m2, [srcq+ssq*1+32*0] + movu m3, [srcq+ssq*1+32*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+32*0], m0 + mova [dstq+dsq*0+32*1], m1 + mova [dstq+dsq*1+32*0], m2 + mova [dstq+dsq*1+32*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] + movu m2, [srcq+32*2] + movu m3, [srcq+32*3] + add srcq, ssq + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + add dstq, dsq + dec hd + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] + movu m2, [srcq+32*2] + movu m3, [srcq+32*3] + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + movu m0, [srcq+32*4] + movu m1, [srcq+32*5] + movu m2, [srcq+32*6] + movu m3, [srcq+32*7] + add srcq, ssq + mova [dstq+32*4], m0 + mova [dstq+32*5], m1 + mova [dstq+32*6], m2 + mova [dstq+32*7], m3 + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + movd xm5, mxyd + mov mxyd, r7m ; my + vpbroadcastd m4, [pw_16] + vpbroadcastw m5, xm5 + psubw m4, m5 + test mxyd, mxyd + jnz .hv + ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v + movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)] + mov r6d, r8m ; bitdepth_max + add wq, r7 + shr r6d, 11 + vpbroadcastd m3, [r7-put_avx2+put_bilin_h_rnd+r6*4] + jmp wq +.h_w2: + movq xm1, [srcq+ssq*0] + movhps xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmullw xm0, xm4, xm1 + psrlq xm1, 16 + pmullw xm1, xm5 + paddw xm0, xm3 + paddw xm0, xm1 + psrlw xm0, 4 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + movq xm1, [srcq+ssq*0+2] + movhps xm1, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + pmullw xm0, xm4 + pmullw xm1, xm5 + paddw xm0, xm3 + paddw xm0, xm1 + psrlw xm0, 4 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 + movu xm1, [srcq+ssq*0+2] + vinserti128 m1, [srcq+ssq*1+2], 1 + lea srcq, [srcq+ssq*2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 4 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + pmullw m0, m4, [srcq+ssq*0] + pmullw m1, m5, [srcq+ssq*0+2] + paddw m0, m3 + paddw m0, m1 + pmullw m1, m4, [srcq+ssq*1] + pmullw m2, m5, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + paddw m1, m3 + paddw m1, m2 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + pmullw m0, m4, [srcq+32*0] + pmullw m1, m5, [srcq+32*0+2] + paddw m0, m3 + paddw m0, m1 + pmullw m1, m4, [srcq+32*1] + pmullw m2, m5, [srcq+32*1+2] + add srcq, ssq + paddw m1, m3 + paddw m1, m2 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + add dstq, dsq + dec hd + jg .h_w32 + RET +.h_w64: +.h_w128: + movifnidn t0d, org_w +.h_w64_loop0: + mov r6d, t0d +.h_w64_loop: + pmullw m0, m4, [srcq+r6*2-32*1] + pmullw m1, m5, [srcq+r6*2-32*1+2] + paddw m0, m3 + paddw m0, m1 + pmullw m1, m4, [srcq+r6*2-32*2] + pmullw m2, m5, [srcq+r6*2-32*2+2] + paddw m1, m3 + paddw m1, m2 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+r6*2-32*1], m0 + mova [dstq+r6*2-32*2], m1 + sub r6d, 32 + jg .h_w64_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w64_loop0 + RET +.v: + movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)] + shl mxyd, 11 + movd xm5, mxyd + add wq, r7 + vpbroadcastw m5, xm5 + jmp wq +.v_w2: + movd xm0, [srcq+ssq*0] +.v_w2_loop: + movd xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq xm2, xm0, xm1 + movd xm0, [srcq+ssq*0] + punpckldq xm1, xm0 + psubw xm1, xm2 + pmulhrsw xm1, xm5 + paddw xm1, xm2 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq xm0, [srcq+ssq*0] +.v_w4_loop: + movq xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklqdq xm2, xm0, xm1 + movq xm0, [srcq+ssq*0] + punpcklqdq xm1, xm0 + psubw xm1, xm2 + pmulhrsw xm1, xm5 + paddw xm1, xm2 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movu xm0, [srcq+ssq*0] +.v_w8_loop: + vbroadcasti128 m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m2, m0, m1, 0xf0 + vbroadcasti128 m0, [srcq+ssq*0] + vpblendd m1, m0, 0xf0 + psubw m1, m2 + pmulhrsw m1, m5 + paddw m1, m2 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w32: + movu m0, [srcq+ssq*0+32*0] + movu m1, [srcq+ssq*0+32*1] +.v_w32_loop: + movu m2, [srcq+ssq*1+32*0] + movu m3, [srcq+ssq*1+32*1] + lea srcq, [srcq+ssq*2] + psubw m4, m2, m0 + pmulhrsw m4, m5 + paddw m4, m0 + movu m0, [srcq+ssq*0+32*0] + mova [dstq+dsq*0+32*0], m4 + psubw m4, m3, m1 + pmulhrsw m4, m5 + paddw m4, m1 + movu m1, [srcq+ssq*0+32*1] + mova [dstq+dsq*0+32*1], m4 + psubw m4, m0, m2 + pmulhrsw m4, m5 + paddw m4, m2 + mova [dstq+dsq*1+32*0], m4 + psubw m4, m1, m3 + pmulhrsw m4, m5 + paddw m4, m3 + mova [dstq+dsq*1+32*1], m4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w32_loop + RET +.v_w16: +.v_w64: +.v_w128: + movifnidn t0d, org_w + add t0d, t0d + mov r4, srcq + lea r6d, [hq+t0*8-256] + mov r7, dstq +.v_w16_loop0: + movu m0, [srcq+ssq*0] +.v_w16_loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + psubw m1, m3, m0 + pmulhrsw m1, m5 + paddw m1, m0 + movu m0, [srcq+ssq*0] + psubw m2, m0, m3 + pmulhrsw m2, m5 + paddw m2, m3 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + add r4, 32 + add r7, 32 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 + vpbroadcastd m3, [pw_2] + movd xm6, mxyd + vpbroadcastd m7, [pw_8192] + add wq, r7 + vpbroadcastw m6, xm6 + test dword r8m, 0x800 + jnz .hv_12bpc + psllw m4, 2 + psllw m5, 2 + vpbroadcastd m7, [pw_2048] +.hv_12bpc: + jmp wq +.hv_w2: + vpbroadcastq xm1, [srcq+ssq*0] + pmullw xm0, xm4, xm1 + psrlq xm1, 16 + pmullw xm1, xm5 + paddw xm0, xm3 + paddw xm0, xm1 + psrlw xm0, 2 +.hv_w2_loop: + movq xm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xm2, [srcq+ssq*0] + pmullw xm1, xm4, xm2 + psrlq xm2, 16 + pmullw xm2, xm5 + paddw xm1, xm3 + paddw xm1, xm2 + psrlw xm1, 2 ; 1 _ 2 _ + shufpd xm2, xm0, xm1, 0x01 ; 0 _ 1 _ + mova xm0, xm1 + psubw xm1, xm2 + paddw xm1, xm1 + pmulhw xm1, xm6 + paddw xm1, xm2 + pmulhrsw xm1, xm7 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + pmullw xm0, xm4, [srcq+ssq*0-8] + pmullw xm1, xm5, [srcq+ssq*0-6] + paddw xm0, xm3 + paddw xm0, xm1 + psrlw xm0, 2 +.hv_w4_loop: + movq xm1, [srcq+ssq*1] + movq xm2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + movhps xm1, [srcq+ssq*0] + movhps xm2, [srcq+ssq*0+2] + pmullw xm1, xm4 + pmullw xm2, xm5 + paddw xm1, xm3 + paddw xm1, xm2 + psrlw xm1, 2 ; 1 2 + shufpd xm2, xm0, xm1, 0x01 ; 0 1 + mova xm0, xm1 + psubw xm1, xm2 + paddw xm1, xm1 + pmulhw xm1, xm6 + paddw xm1, xm2 + pmulhrsw xm1, xm7 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + pmullw xm0, xm4, [srcq+ssq*0] + pmullw xm1, xm5, [srcq+ssq*0+2] + paddw xm0, xm3 + paddw xm0, xm1 + psrlw xm0, 2 + vinserti128 m0, xm0, 1 +.hv_w8_loop: + movu xm1, [srcq+ssq*1] + movu xm2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + vinserti128 m1, [srcq+ssq*0], 1 + vinserti128 m2, [srcq+ssq*0+2], 1 + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m1, 2 ; 1 2 + vperm2i128 m2, m0, m1, 0x21 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: +.hv_w32: +.hv_w64: +.hv_w128: +%if UNIX64 + lea r6d, [r8*2-32] +%else + mov r6d, wm + lea r6d, [r6*2-32] +%endif + mov r4, srcq + lea r6d, [hq+r6*8] + mov r7, dstq +.hv_w16_loop0: + pmullw m0, m4, [srcq+ssq*0] + pmullw m1, m5, [srcq+ssq*0+2] + paddw m0, m3 + paddw m0, m1 + psrlw m0, 2 +.hv_w16_loop: + pmullw m1, m4, [srcq+ssq*1] + pmullw m2, m5, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + paddw m1, m3 + paddw m1, m2 + psrlw m1, 2 + psubw m2, m1, m0 + paddw m2, m2 + pmulhw m2, m6 + paddw m2, m0 + pmulhrsw m2, m7 + mova [dstq+dsq*0], m2 + pmullw m0, m4, [srcq+ssq*0] + pmullw m2, m5, [srcq+ssq*0+2] + paddw m0, m3 + paddw m0, m2 + psrlw m0, 2 + psubw m2, m0, m1 + paddw m2, m2 + pmulhw m2, m6 + paddw m2, m1 + pmulhrsw m2, m7 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + add r4, 32 + add r7, 32 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .hv_w16_loop0 + RET + +cglobal prep_bilin_16bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea r6, [prep_avx2] +%if UNIX64 + DECLARE_REG_TMP 7 + %define org_w r7d +%else + DECLARE_REG_TMP 6 + %define org_w r5m +%endif + mov org_w, wd + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [r6+wq*2+table_offset(prep,)] + mov r5d, r7m ; bitdepth_max + vpbroadcastd m5, [r6-prep_avx2+pw_8192] + add wq, r6 + shr r5d, 11 + vpbroadcastd m4, [r6-prep_avx2+prep_mul+r5*4] + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movq xm0, [srcq+strideq*0] + movhps xm0, [srcq+strideq*1] + vpbroadcastq m1, [srcq+strideq*2] + vpbroadcastq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m0, m1, 0x30 + vpblendd m0, m2, 0xc0 + pmullw m0, m4 + psubw m0, m5 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*2] + vinserti128 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pmullw m0, m4 + pmullw m1, m4 + psubw m0, m5 + psubw m1, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + pmullw m0, m4, [srcq+strideq*0] + pmullw m1, m4, [srcq+strideq*1] + pmullw m2, m4, [srcq+strideq*2] + pmullw m3, m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + psubw m0, m5 + psubw m1, m5 + psubw m2, m5 + psubw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmullw m0, m4, [srcq+strideq*0+32*0] + pmullw m1, m4, [srcq+strideq*0+32*1] + pmullw m2, m4, [srcq+strideq*1+32*0] + pmullw m3, m4, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] + psubw m0, m5 + psubw m1, m5 + psubw m2, m5 + psubw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 2 + jg .prep_w32 + RET +.prep_w64: + pmullw m0, m4, [srcq+32*0] + pmullw m1, m4, [srcq+32*1] + pmullw m2, m4, [srcq+32*2] + pmullw m3, m4, [srcq+32*3] + add srcq, strideq + psubw m0, m5 + psubw m1, m5 + psubw m2, m5 + psubw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + dec hd + jg .prep_w64 + RET +.prep_w128: + pmullw m0, m4, [srcq+32*0] + pmullw m1, m4, [srcq+32*1] + pmullw m2, m4, [srcq+32*2] + pmullw m3, m4, [srcq+32*3] + psubw m0, m5 + psubw m1, m5 + psubw m2, m5 + psubw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + pmullw m0, m4, [srcq+32*4] + pmullw m1, m4, [srcq+32*5] + pmullw m2, m4, [srcq+32*6] + pmullw m3, m4, [srcq+32*7] + add tmpq, 32*8 + add srcq, strideq + psubw m0, m5 + psubw m1, m5 + psubw m2, m5 + psubw m3, m5 + mova [tmpq-32*4], m0 + mova [tmpq-32*3], m1 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + dec hd + jg .prep_w128 + RET +.h: + movd xm5, mxyd + mov mxyd, r6m ; my + vpbroadcastd m4, [pw_16] + vpbroadcastw m5, xm5 + vpbroadcastd m3, [pw_32766] + psubw m4, m5 + test dword r7m, 0x800 + jnz .h_12bpc + psllw m4, 2 + psllw m5, 2 +.h_12bpc: + test mxyd, mxyd + jnz .hv + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)] + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + movu xm1, [srcq+strideq*0] + vinserti128 m1, [srcq+strideq*2], 1 + movu xm2, [srcq+strideq*1] + vinserti128 m2, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq m0, m1, m2 + psrldq m1, 2 + pslldq m2, 6 + pmullw m0, m4 + vpblendd m1, m2, 0xcc + pmullw m1, m5 + psubw m0, m3 + paddw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4 + RET +.h_w8: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*0+2] + vinserti128 m1, [srcq+strideq*1+2], 1 + lea srcq, [srcq+strideq*2] + pmullw m0, m4 + pmullw m1, m5 + psubw m0, m3 + paddw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + pmullw m0, m4, [srcq+strideq*0] + pmullw m1, m5, [srcq+strideq*0+2] + psubw m0, m3 + paddw m0, m1 + pmullw m1, m4, [srcq+strideq*1] + pmullw m2, m5, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + psubw m1, m3 + paddw m1, m2 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 2 + jg .h_w16 + RET +.h_w32: +.h_w64: +.h_w128: + movifnidn t0d, org_w +.h_w32_loop0: + mov r3d, t0d +.h_w32_loop: + pmullw m0, m4, [srcq+r3*2-32*1] + pmullw m1, m5, [srcq+r3*2-32*1+2] + psubw m0, m3 + paddw m0, m1 + pmullw m1, m4, [srcq+r3*2-32*2] + pmullw m2, m5, [srcq+r3*2-32*2+2] + psubw m1, m3 + paddw m1, m2 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+r3*2-32*1], m0 + mova [tmpq+r3*2-32*2], m1 + sub r3d, 32 + jg .h_w32_loop + add srcq, strideq + lea tmpq, [tmpq+t0*2] + dec hd + jg .h_w32_loop0 + RET +.v: + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)] + movd xm5, mxyd + vpbroadcastd m4, [pw_16] + vpbroadcastw m5, xm5 + vpbroadcastd m3, [pw_32766] + add wq, r6 + lea stride3q, [strideq*3] + psubw m4, m5 + test dword r7m, 0x800 + jnz .v_12bpc + psllw m4, 2 + psllw m5, 2 +.v_12bpc: + jmp wq +.v_w4: + movq xm0, [srcq+strideq*0] +.v_w4_loop: + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq xm1, [srcq+strideq*1] + vpblendd m2, m0, 0x03 ; 0 2 2 2 + vpbroadcastq m0, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0xf0 ; 1 1 3 3 + vpbroadcastq m0, [srcq+strideq*0] + vpblendd m1, m2, 0x33 ; 0 1 2 3 + vpblendd m0, m2, 0x0c ; 4 2 4 4 + punpckhqdq m2, m1, m0 ; 1 2 3 4 + pmullw m1, m4 + pmullw m2, m5 + psubw m1, m3 + paddw m1, m2 + psraw m1, 2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movu xm0, [srcq+strideq*0] +.v_w8_loop: + vbroadcasti128 m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpblendd m1, m0, m2, 0xf0 ; 0 1 + vbroadcasti128 m0, [srcq+strideq*0] + vpblendd m2, m0, 0xf0 ; 1 2 + pmullw m1, m4 + pmullw m2, m5 + psubw m1, m3 + paddw m1, m2 + psraw m1, 2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + movu m0, [srcq+strideq*0] +.v_w16_loop: + movu m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + pmullw m0, m4 + pmullw m1, m5, m2 + psubw m0, m3 + paddw m1, m0 + movu m0, [srcq+strideq*0] + psraw m1, 2 + pmullw m2, m4 + mova [tmpq+32*0], m1 + pmullw m1, m5, m0 + psubw m2, m3 + paddw m1, m2 + psraw m1, 2 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: +.v_w64: +.v_w128: +%if WIN64 + PUSH r7 +%endif + movifnidn r7d, org_w + add r7d, r7d + mov r3, srcq + lea r6d, [hq+r7*8-256] + mov r5, tmpq +.v_w32_loop0: + movu m0, [srcq+strideq*0] +.v_w32_loop: + movu m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + pmullw m0, m4 + pmullw m1, m5, m2 + psubw m0, m3 + paddw m1, m0 + movu m0, [srcq+strideq*0] + psraw m1, 2 + pmullw m2, m4 + mova [tmpq+r7*0], m1 + pmullw m1, m5, m0 + psubw m2, m3 + paddw m1, m2 + psraw m1, 2 + mova [tmpq+r7*1], m1 + lea tmpq, [tmpq+r7*2] + sub hd, 2 + jg .v_w32_loop + add r3, 32 + add r5, 32 + movzx hd, r6b + mov srcq, r3 + mov tmpq, r5 + sub r6d, 1<<8 + jg .v_w32_loop0 +%if WIN64 + POP r7 +%endif + RET +.hv: + WIN64_SPILL_XMM 7 + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + movd xm6, mxyd + add wq, r6 + lea stride3q, [strideq*3] + vpbroadcastw m6, xm6 + jmp wq +.hv_w4: + movu xm1, [srcq+strideq*0] +%if WIN64 + movaps [rsp+24], xmm7 +%endif + pmullw xm0, xm4, xm1 + psrldq xm1, 2 + pmullw xm1, xm5 + psubw xm0, xm3 + paddw xm0, xm1 + psraw xm0, 2 + vpbroadcastq m0, xm0 +.hv_w4_loop: + movu xm1, [srcq+strideq*1] + vinserti128 m1, [srcq+stride3q ], 1 + movu xm2, [srcq+strideq*2] + lea srcq, [srcq+strideq*4] + vinserti128 m2, [srcq+strideq*0], 1 + punpcklqdq m7, m1, m2 + psrldq m1, 2 + pslldq m2, 6 + pmullw m7, m4 + vpblendd m1, m2, 0xcc + pmullw m1, m5 + psubw m7, m3 + paddw m1, m7 + psraw m1, 2 ; 1 2 3 4 + vpblendd m0, m1, 0x3f + vpermq m2, m0, q2103 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop +%if WIN64 + movaps xmm7, [rsp+24] +%endif + RET +.hv_w8: + pmullw xm0, xm4, [srcq+strideq*0] + pmullw xm1, xm5, [srcq+strideq*0+2] + psubw xm0, xm3 + paddw xm0, xm1 + psraw xm0, 2 + vinserti128 m0, xm0, 1 +.hv_w8_loop: + movu xm1, [srcq+strideq*1] + movu xm2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + vinserti128 m1, [srcq+strideq*0], 1 + vinserti128 m2, [srcq+strideq*0+2], 1 + pmullw m1, m4 + pmullw m2, m5 + psubw m1, m3 + paddw m1, m2 + psraw m1, 2 ; 1 2 + vperm2i128 m2, m0, m1, 0x21 ; 0 1 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: +.hv_w32: +.hv_w64: +.hv_w128: +%if WIN64 + PUSH r7 +%endif + movifnidn r7d, org_w + add r7d, r7d + mov r3, srcq + lea r6d, [hq+r7*8-256] + mov r5, tmpq +.hv_w16_loop0: + pmullw m0, m4, [srcq] + pmullw m1, m5, [srcq+2] + psubw m0, m3 + paddw m0, m1 + psraw m0, 2 +.hv_w16_loop: + pmullw m1, m4, [srcq+strideq*1] + pmullw m2, m5, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + psubw m1, m3 + paddw m1, m2 + psraw m1, 2 + psubw m2, m1, m0 + pmulhrsw m2, m6 + paddw m2, m0 + mova [tmpq+r7*0], m2 + pmullw m0, m4, [srcq+strideq*0] + pmullw m2, m5, [srcq+strideq*0+2] + psubw m0, m3 + paddw m0, m2 + psraw m0, 2 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+r7*1], m2 + lea tmpq, [tmpq+r7*2] + sub hd, 2 + jg .hv_w16_loop + add r3, 32 + add r5, 32 + movzx hd, r6b + mov srcq, r3 + mov tmpq, r5 + sub r6d, 1<<8 + jg .hv_w16_loop0 +%if WIN64 + POP r7 +%endif + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; prefix, type, type_h, type_v +cglobal %1_%2_16bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX) +%endif +%endmacro + +%if WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +%define PUT_8TAP_FN FN put_8tap, +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR + +cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my +%define base r8-put_avx2 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx2] + movifnidn wd, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r8+wq*2+table_offset(put,)] + add wq, r8 +%if WIN64 + pop r8 +%endif + jmp wq +.h_w2: + movzx mxd, mxb + sub srcq, 2 + mova xm2, [subpel_h_shuf2] + vpbroadcastd xm3, [base+subpel_filters+mxq*8+2] + pmovsxbw xm3, xm3 +.h_w2_loop: + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm2 + pshufb xm1, xm2 + pmaddwd xm0, xm3 + pmaddwd xm1, xm3 + phaddd xm0, xm1 + paddd xm0, xm4 + psrad xm0, 6 + packusdw xm0, xm0 + pminsw xm0, xm5 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + sub srcq, 2 + pmovsxbw xm3, [base+subpel_filters+mxq*8] + WIN64_SPILL_XMM 8 + vbroadcasti128 m6, [subpel_h_shufA] + vbroadcasti128 m7, [subpel_h_shufB] + pshufd xm3, xm3, q2211 + vpbroadcastq m2, xm3 + vpermq m3, m3, q1111 +.h_w4_loop: + movu xm1, [srcq+ssq*0] + vinserti128 m1, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pshufb m0, m1, m6 ; 0 1 1 2 2 3 3 4 + pshufb m1, m7 ; 2 3 3 4 4 5 5 6 + pmaddwd m0, m2 + pmaddwd m1, m3 + paddd m0, m4 + paddd m0, m1 + psrad m0, 6 + vextracti128 xm1, m0, 1 + packusdw xm0, xm1 + pminsw xm0, xm5 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h: + test myd, 0xf00 + jnz .hv + mov r7d, r8m + vpbroadcastw m5, r8m + shr r7d, 11 + vpbroadcastd m4, [base+put_8tap_h_rnd+r7*4] + cmp wd, 4 + je .h_w4 + jl .h_w2 + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 13 + shr mxd, 16 + sub srcq, 6 + vpbroadcastq m0, [base+subpel_filters+mxq*8] + vbroadcasti128 m6, [subpel_h_shufA] + vbroadcasti128 m7, [subpel_h_shufB] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + cmp wd, 8 + jg .h_w16 +.h_w8: +%macro PUT_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 + pmaddwd m%5, m9, m%4 ; abcd1 + pmaddwd m%1, m8 ; abcd0 + pshufb m%2, m7 ; 6 7 7 8 8 9 9 a + shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m%5, m4 + paddd m%1, m%5 + pmaddwd m%5, m11, m%2 ; abcd3 + paddd m%1, m%5 + pmaddwd m%5, m10, m%4 ; abcd2 + pshufb m%3, m7 ; a b b c c d d e + pmaddwd m%4, m8 ; efgh0 + paddd m%1, m%5 + pmaddwd m%5, m9, m%2 ; efgh1 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m11 ; efgh3 + pmaddwd m%2, m10 ; efgh2 + paddd m%4, m4 + paddd m%4, m%5 + paddd m%3, m%4 + paddd m%2, m%3 + psrad m%1, 6 + psrad m%2, 6 + packusdw m%1, m%2 + pminsw m%1, m5 +%endmacro + movu xm0, [srcq+ssq*0+ 0] + vinserti128 m0, [srcq+ssq*1+ 0], 1 + movu xm2, [srcq+ssq*0+16] + vinserti128 m2, [srcq+ssq*1+16], 1 + lea srcq, [srcq+ssq*2] + shufpd m1, m0, m2, 0x05 + PUT_8TAP_H 0, 1, 2, 3, 12 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + mov r6d, wd +.h_w16_loop: + movu m0, [srcq+r6*2-32] + movu m1, [srcq+r6*2-24] + movu m2, [srcq+r6*2-16] + PUT_8TAP_H 0, 1, 2, 3, 12 + mova [dstq+r6*2-32], m0 + sub r6d, 16 + jg .h_w16_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w16 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + vpbroadcastq m0, [base+subpel_filters+myq*8] + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 15 + vpbroadcastd m6, [pd_32] + vpbroadcastw m7, r8m + lea r6, [ssq*3] + sub srcq, r6 + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + cmp wd, 4 + jg .v_w8 + je .v_w4 +.v_w2: + movd xm2, [srcq+ssq*0] + pinsrd xm2, [srcq+ssq*1], 1 + pinsrd xm2, [srcq+ssq*2], 2 + pinsrd xm2, [srcq+r6 ], 3 ; 0 1 2 3 + lea srcq, [srcq+ssq*4] + movd xm3, [srcq+ssq*0] + vpbroadcastd xm1, [srcq+ssq*1] + vpbroadcastd xm0, [srcq+ssq*2] + add srcq, r6 + vpblendd xm3, xm1, 0x02 ; 4 5 + vpblendd xm1, xm0, 0x02 ; 5 6 + palignr xm4, xm3, xm2, 4 ; 1 2 3 4 + punpcklwd xm3, xm1 ; 45 56 + punpcklwd xm1, xm2, xm4 ; 01 12 + punpckhwd xm2, xm4 ; 23 34 +.v_w2_loop: + vpbroadcastd xm4, [srcq+ssq*0] + pmaddwd xm5, xm8, xm1 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm9 ; a1 b1 + paddd xm5, xm6 + paddd xm5, xm2 + mova xm2, xm3 + pmaddwd xm3, xm10 ; a2 b2 + paddd xm5, xm3 + vpblendd xm3, xm0, xm4, 0x02 ; 6 7 + vpbroadcastd xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm4, xm0, 0x02 ; 7 8 + punpcklwd xm3, xm4 ; 67 78 + pmaddwd xm4, xm11, xm3 ; a3 b3 + paddd xm5, xm4 + psrad xm5, 6 + packusdw xm5, xm5 + pminsw xm5, xm7 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq xm1, [srcq+ssq*0] + vpbroadcastq m0, [srcq+ssq*1] + vpbroadcastq m2, [srcq+ssq*2] + vpbroadcastq m4, [srcq+r6 ] + lea srcq, [srcq+ssq*4] + vpbroadcastq m3, [srcq+ssq*0] + vpbroadcastq m5, [srcq+ssq*1] + vpblendd m1, m0, 0x30 + vpblendd m0, m2, 0x30 + punpcklwd m1, m0 ; 01 12 + vpbroadcastq m0, [srcq+ssq*2] + add srcq, r6 + vpblendd m2, m4, 0x30 + vpblendd m4, m3, 0x30 + punpcklwd m2, m4 ; 23 34 + vpblendd m3, m5, 0x30 + vpblendd m5, m0, 0x30 + punpcklwd m3, m5 ; 45 56 +.v_w4_loop: + vpbroadcastq m4, [srcq+ssq*0] + pmaddwd m5, m8, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m9 ; a1 b1 + paddd m5, m6 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m10 ; a2 b2 + paddd m5, m3 + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m4, m0, 0x30 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m11, m3 ; a3 b3 + paddd m5, m4 + psrad m5, 6 + vextracti128 xm4, m5, 1 + packusdw xm5, xm4 + pminsw xm5, xm7 + movq [dstq+dsq*0], xm5 + movhps [dstq+dsq*1], xm5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + shl wd, 5 + mov r7, srcq + mov r8, dstq + lea wd, [hq+wq-256] +.v_w8_loop0: + vbroadcasti128 m4, [srcq+ssq*0] + vbroadcasti128 m5, [srcq+ssq*1] + vbroadcasti128 m0, [srcq+r6 ] + vbroadcasti128 m6, [srcq+ssq*2] + lea srcq, [srcq+ssq*4] + vbroadcasti128 m1, [srcq+ssq*0] + vbroadcasti128 m2, [srcq+ssq*1] + vbroadcasti128 m3, [srcq+ssq*2] + add srcq, r6 + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + shufpd m6, m2, 0x0c + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + shufpd m0, m3, 0x0c + punpcklwd m3, m6, m0 ; 23 + punpckhwd m6, m0 ; 56 +.v_w8_loop: + vbroadcasti128 m14, [srcq+ssq*0] + pmaddwd m12, m8, m1 ; a0 + pmaddwd m13, m8, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m12, m3 + paddd m13, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m12, m5 + vbroadcasti128 m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + paddd m13, m6 + shufpd m6, m0, m14, 0x0d + shufpd m0, m14, m5, 0x0c + punpcklwd m5, m6, m0 ; 67 + punpckhwd m6, m0 ; 78 + pmaddwd m14, m11, m5 ; a3 + paddd m12, m14 + pmaddwd m14, m11, m6 ; b3 + paddd m13, m14 + psrad m12, 5 + psrad m13, 5 + packusdw m12, m13 + pxor m13, m13 + pavgw m12, m13 + pminsw m12, m7 + vpermq m12, m12, q3120 + mova [dstq+dsq*0], xm12 + vextracti128 [dstq+dsq*1], m12, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + add r7, 16 + add r8, 16 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 + jg .v_w8_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + vpbroadcastw m15, r8m + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + vpbroadcastd m0, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + vpbroadcastq m1, [base+subpel_filters+myq*8] + vpbroadcastd m6, [pd_512] + lea r6, [ssq*3] + sub srcq, 2 + sub srcq, r6 + pxor m7, m7 + punpcklbw m7, m0 + punpcklbw m1, m1 + psraw m1, 8 ; sign-extend + test dword r8m, 0x800 + jz .hv_10bit + psraw m7, 2 + psllw m1, 2 +.hv_10bit: + pshufd m11, m1, q0000 + pshufd m12, m1, q1111 + pshufd m13, m1, q2222 + pshufd m14, m1, q3333 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 m9, [subpel_h_shuf2] + vbroadcasti128 m1, [srcq+r6 ] ; 3 3 + movu xm3, [srcq+ssq*2] + movu xm0, [srcq+ssq*0] + movu xm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*4] + vinserti128 m3, [srcq+ssq*0], 1 ; 2 4 + vinserti128 m0, [srcq+ssq*1], 1 ; 0 5 + vinserti128 m2, [srcq+ssq*2], 1 ; 1 6 + add srcq, r6 + pshufb m1, m9 + pshufb m3, m9 + pshufb m0, m9 + pshufb m2, m9 + pmaddwd m1, m7 + pmaddwd m3, m7 + pmaddwd m0, m7 + pmaddwd m2, m7 + phaddd m1, m3 + phaddd m0, m2 + paddd m1, m6 + paddd m0, m6 + psrad m1, 10 + psrad m0, 10 + packssdw m1, m0 ; 3 2 0 1 + vextracti128 xm0, m1, 1 ; 3 4 5 6 + pshufd xm2, xm1, q1301 ; 2 3 1 2 + pshufd xm3, xm0, q2121 ; 4 5 4 5 + punpckhwd xm1, xm2 ; 01 12 + punpcklwd xm2, xm0 ; 23 34 + punpckhwd xm3, xm0 ; 45 56 +.hv_w2_loop: + movu xm4, [srcq+ssq*0] + movu xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm4, xm9 + pshufb xm5, xm9 + pmaddwd xm4, xm7 + pmaddwd xm5, xm7 + phaddd xm4, xm5 + pmaddwd xm5, xm11, xm1 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm12 ; a1 b1 + paddd xm5, xm2 + mova xm2, xm3 + pmaddwd xm3, xm13 ; a2 b2 + paddd xm5, xm3 + paddd xm4, xm6 + psrad xm4, 10 + packssdw xm4, xm4 + palignr xm3, xm4, xm0, 12 + mova xm0, xm4 + punpcklwd xm3, xm0 ; 67 78 + pmaddwd xm4, xm14, xm3 ; a3 b3 + paddd xm5, xm6 + paddd xm5, xm4 + psrad xm5, 10 + packusdw xm5, xm5 + pminsw xm5, xm15 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + vbroadcasti128 m9, [subpel_h_shufA] + vbroadcasti128 m10, [subpel_h_shufB] + pshufd m8, m7, q1111 + pshufd m7, m7, q0000 + movu xm1, [srcq+ssq*0] + vinserti128 m1, [srcq+ssq*1], 1 ; 0 1 + vbroadcasti128 m0, [srcq+r6 ] + vinserti128 m2, m0, [srcq+ssq*2], 0 ; 2 3 + lea srcq, [srcq+ssq*4] + vinserti128 m0, [srcq+ssq*0], 1 ; 3 4 + movu xm3, [srcq+ssq*1] + vinserti128 m3, [srcq+ssq*2], 1 ; 5 6 + add srcq, r6 + pshufb m4, m1, m9 + pshufb m1, m10 + pmaddwd m4, m7 + pmaddwd m1, m8 + pshufb m5, m2, m9 + pshufb m2, m10 + pmaddwd m5, m7 + pmaddwd m2, m8 + paddd m4, m6 + paddd m1, m4 + pshufb m4, m0, m9 + pshufb m0, m10 + pmaddwd m4, m7 + pmaddwd m0, m8 + paddd m5, m6 + paddd m2, m5 + pshufb m5, m3, m9 + pshufb m3, m10 + pmaddwd m5, m7 + pmaddwd m3, m8 + paddd m4, m6 + paddd m4, m0 + paddd m5, m6 + paddd m5, m3 + vperm2i128 m0, m1, m2, 0x21 + psrld m1, 10 + psrld m2, 10 + vperm2i128 m3, m4, m5, 0x21 + pslld m4, 6 + pslld m5, 6 + pblendw m2, m4, 0xaa ; 23 34 + pslld m0, 6 + pblendw m1, m0, 0xaa ; 01 12 + psrld m3, 10 + pblendw m3, m5, 0xaa ; 45 56 + psrad m0, m5, 16 +.hv_w4_loop: + movu xm4, [srcq+ssq*0] + vinserti128 m4, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pmaddwd m5, m11, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m12 ; a1 b1 + paddd m5, m6 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m13 ; a2 b2 + paddd m5, m3 + pshufb m3, m4, m9 + pshufb m4, m10 + pmaddwd m3, m7 + pmaddwd m4, m8 + paddd m3, m6 + paddd m4, m3 + psrad m4, 10 + packssdw m0, m4 ; _ 7 6 8 + vpermq m3, m0, q1122 ; _ 6 _ 7 + punpckhwd m3, m0 ; 67 78 + mova m0, m4 + pmaddwd m4, m14, m3 ; a3 b3 + paddd m4, m5 + psrad m4, 10 + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, xm15 + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + vpbroadcastq m2, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + pmovsxbw xm1, [base+subpel_filters+myq*8] + shl wd, 5 + lea r6, [ssq*3] + sub srcq, 6 + sub srcq, r6 + pxor m0, m0 + punpcklbw m0, m2 + mov r7, srcq + mov r8, dstq + lea wd, [hq+wq-256] + test dword r8m, 0x800 + jz .hv_w8_10bit + psraw m0, 2 + psllw xm1, 2 +.hv_w8_10bit: + pshufd m11, m0, q0000 + pshufd m12, m0, q1111 + pshufd m13, m0, q2222 + pshufd m14, m0, q3333 +%if WIN64 + %define v_mul (rsp+stack_offset+40) ; r4m +%else + %define v_mul (rsp-24) ; red zone +%endif + mova [v_mul], xm1 +.hv_w8_loop0: +%macro PUT_8TAP_HV_H 3 ; dst/src+0, src+8, src+16 + pshufb m2, m%1, m9 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m8 ; 0 1 1 2 2 3 3 4 + pmaddwd m3, m12, m2 + pmaddwd m%1, m11 + pshufb m%2, m9 ; 6 7 7 8 8 9 9 a + shufpd m2, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m3, m10 + paddd m%1, m3 + pmaddwd m3, m14, m%2 + paddd m%1, m3 + pmaddwd m3, m13, m2 + pshufb m%3, m9 ; a b b c c d d e + pmaddwd m2, m11 + paddd m%1, m3 + pmaddwd m3, m12, m%2 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m14 + pmaddwd m%2, m13 + paddd m2, m10 + paddd m2, m3 + paddd m%3, m2 + paddd m%2, m%3 + psrad m%1, 10 + psrad m%2, 10 + packssdw m%1, m%2 +%endmacro + movu xm4, [srcq+r6 *1+ 0] + vbroadcasti128 m8, [subpel_h_shufA] + movu xm6, [srcq+r6 *1+ 8] + vbroadcasti128 m9, [subpel_h_shufB] + movu xm0, [srcq+r6 *1+16] + vpbroadcastd m10, [pd_512] + movu xm5, [srcq+ssq*0+ 0] + vinserti128 m5, [srcq+ssq*4+ 0], 1 + movu xm1, [srcq+ssq*0+16] + vinserti128 m1, [srcq+ssq*4+16], 1 + shufpd m7, m5, m1, 0x05 + INIT_XMM avx2 + PUT_8TAP_HV_H 4, 6, 0 ; 3 + INIT_YMM avx2 + PUT_8TAP_HV_H 5, 7, 1 ; 0 4 + movu xm0, [srcq+ssq*2+ 0] + vinserti128 m0, [srcq+r6 *2+ 0], 1 + movu xm1, [srcq+ssq*2+16] + vinserti128 m1, [srcq+r6 *2+16], 1 + shufpd m7, m0, m1, 0x05 + PUT_8TAP_HV_H 0, 7, 1 ; 2 6 + movu xm6, [srcq+ssq*1+ 0] + movu xm1, [srcq+ssq*1+16] + lea srcq, [srcq+ssq*4] + vinserti128 m6, [srcq+ssq*1+ 0], 1 + vinserti128 m1, [srcq+ssq*1+16], 1 + add srcq, r6 + shufpd m7, m6, m1, 0x05 + PUT_8TAP_HV_H 6, 7, 1 ; 1 5 + vpermq m4, m4, q1100 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + vpermq m7, m0, q3120 + punpcklwd m3, m7, m4 ; 23 + punpckhwd m4, m5 ; 34 + punpcklwd m1, m5, m6 ; 01 + punpckhwd m5, m6 ; 45 + punpcklwd m2, m6, m7 ; 12 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vpbroadcastd m9, [v_mul+4*0] + vpbroadcastd m7, [v_mul+4*1] + vpbroadcastd m10, [v_mul+4*2] + pmaddwd m8, m9, m1 ; a0 + pmaddwd m9, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m7 ; a1 + pmaddwd m4, m7 ; b1 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m8, m5 + paddd m9, m6 + movu xm5, [srcq+ssq*0] + vinserti128 m5, [srcq+ssq*1], 1 + vbroadcasti128 m7, [subpel_h_shufA] + vbroadcasti128 m10, [subpel_h_shufB] + movu xm6, [srcq+ssq*0+16] + vinserti128 m6, [srcq+ssq*1+16], 1 + vextracti128 [dstq], m0, 1 + pshufb m0, m5, m7 ; 01 + pshufb m5, m10 ; 23 + pmaddwd m0, m11 + pmaddwd m5, m12 + paddd m0, m5 + pshufb m5, m6, m7 ; 89 + pshufb m6, m10 ; ab + pmaddwd m5, m13 + pmaddwd m6, m14 + paddd m6, m5 + movu xm5, [srcq+ssq*0+8] + vinserti128 m5, [srcq+ssq*1+8], 1 + lea srcq, [srcq+ssq*2] + pshufb m7, m5, m7 + pshufb m5, m10 + pmaddwd m10, m13, m7 + pmaddwd m7, m11 + paddd m0, m10 + vpbroadcastd m10, [pd_512] + paddd m6, m7 + pmaddwd m7, m14, m5 + pmaddwd m5, m12 + paddd m0, m7 + paddd m5, m6 + vbroadcasti128 m6, [dstq] + paddd m8, m10 + paddd m9, m10 + paddd m0, m10 + paddd m5, m10 + vpbroadcastd m10, [v_mul+4*3] + psrad m0, 10 + psrad m5, 10 + packssdw m0, m5 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m10, m5 ; a3 + pmaddwd m10, m6 ; b3 + paddd m7, m8 + paddd m9, m10 + psrad m7, 10 + psrad m9, 10 + packusdw m7, m9 + pminsw m7, m15 + vpermq m7, m7, q3120 + mova [dstq+dsq*0], xm7 + vextracti128 [dstq+dsq*1], m7, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + add r7, 16 + add r8, 16 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 + jg .hv_w8_loop0 + RET + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my +%define base r7-prep_avx2 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep_avx2] + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + mov r6d, r7m ; bitdepth_max + movzx wd, word [r7+wq*2+table_offset(prep,)] + vpbroadcastd m5, [r7-prep_avx2+pw_8192] + shr r6d, 11 + add wq, r7 + vpbroadcastd m4, [base+prep_mul+r6*4] + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h_w4: + movzx mxd, mxb + sub srcq, 2 + pmovsxbw xm0, [base+subpel_filters+mxq*8] + vbroadcasti128 m3, [subpel_h_shufA] + vbroadcasti128 m4, [subpel_h_shufB] + WIN64_SPILL_XMM 8 + pshufd xm0, xm0, q2211 + test dword r7m, 0x800 + jnz .h_w4_12bpc + psllw xm0, 2 +.h_w4_12bpc: + vpbroadcastq m6, xm0 + vpermq m7, m0, q1111 +.h_w4_loop: + movu xm1, [srcq+strideq*0] + vinserti128 m1, [srcq+strideq*2], 1 + movu xm2, [srcq+strideq*1] + vinserti128 m2, [srcq+r6 ], 1 + lea srcq, [srcq+strideq*4] + pshufb m0, m1, m3 ; 0 1 1 2 2 3 3 4 + pshufb m1, m4 ; 2 3 3 4 4 5 5 6 + pmaddwd m0, m6 + pmaddwd m1, m7 + paddd m0, m5 + paddd m0, m1 + pshufb m1, m2, m3 + pshufb m2, m4 + pmaddwd m1, m6 + pmaddwd m2, m7 + paddd m1, m5 + paddd m1, m2 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [prep_8tap_1d_rnd] ; 8 - (8192 << 4) + lea r6, [strideq*3] + cmp wd, 4 + je .h_w4 + shr mxd, 16 + sub srcq, 6 + vpbroadcastq m0, [base+subpel_filters+mxq*8] + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 12 + vbroadcasti128 m6, [subpel_h_shufA] + vbroadcasti128 m7, [subpel_h_shufB] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + test dword r7m, 0x800 + jnz .h_12bpc + psllw m0, 2 +.h_12bpc: + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + cmp wd, 8 + jg .h_w16 +.h_w8: +%macro PREP_8TAP_H 5 ; dst/src+0, src+8, src+16, tmp[1-2] + pshufb m%4, m%1, m7 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m6 ; 0 1 1 2 2 3 3 4 + pmaddwd m%5, m9, m%4 ; abcd1 + pmaddwd m%1, m8 ; abcd0 + pshufb m%2, m7 ; 6 7 7 8 8 9 9 a + shufpd m%4, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m%5, m5 + paddd m%1, m%5 + pmaddwd m%5, m11, m%2 ; abcd3 + paddd m%1, m%5 + pmaddwd m%5, m10, m%4 ; abcd2 + pshufb m%3, m7 ; a b b c c d d e + pmaddwd m%4, m8 ; efgh0 + paddd m%1, m%5 + pmaddwd m%5, m9, m%2 ; efgh1 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m11 ; efgh3 + pmaddwd m%2, m10 ; efgh2 + paddd m%4, m5 + paddd m%4, m%5 + paddd m%3, m%4 + paddd m%2, m%3 + psrad m%1, 4 + psrad m%2, 4 + packssdw m%1, m%2 +%endmacro + movu xm0, [srcq+strideq*0+ 0] + vinserti128 m0, [srcq+strideq*1+ 0], 1 + movu xm2, [srcq+strideq*0+16] + vinserti128 m2, [srcq+strideq*1+16], 1 + lea srcq, [srcq+strideq*2] + shufpd m1, m0, m2, 0x05 + PREP_8TAP_H 0, 1, 2, 3, 4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + add wd, wd +.h_w16_loop0: + mov r6d, wd +.h_w16_loop: + movu m0, [srcq+r6-32] + movu m1, [srcq+r6-24] + movu m2, [srcq+r6-16] + PREP_8TAP_H 0, 1, 2, 3, 4 + mova [tmpq+r6-32], m0 + sub r6d, 32 + jg .h_w16_loop + add srcq, strideq + add tmpq, wq + dec hd + jg .h_w16_loop0 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + vpbroadcastq m0, [base+subpel_filters+myq*8] + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 15 + vpbroadcastd m7, [prep_8tap_1d_rnd] + lea r6, [strideq*3] + sub srcq, r6 + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + test dword r7m, 0x800 + jnz .v_12bpc + psllw m0, 2 +.v_12bpc: + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + cmp wd, 4 + jg .v_w8 +.v_w4: + movq xm1, [srcq+strideq*0] + vpbroadcastq m0, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m4, [srcq+r6 ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpblendd m1, m0, 0x30 + vpblendd m0, m2, 0x30 + punpcklwd m1, m0 ; 01 12 + vpbroadcastq m0, [srcq+strideq*2] + add srcq, r6 + vpblendd m2, m4, 0x30 + vpblendd m4, m3, 0x30 + punpcklwd m2, m4 ; 23 34 + vpblendd m3, m5, 0x30 + vpblendd m5, m0, 0x30 + punpcklwd m3, m5 ; 45 56 +.v_w4_loop: + vpbroadcastq m4, [srcq+strideq*0] + pmaddwd m5, m8, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m9 ; a1 b1 + paddd m5, m7 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m10 ; a2 b2 + paddd m5, m3 + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpblendd m4, m0, 0x30 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m11, m3 ; a3 b3 + paddd m5, m4 + psrad m5, 4 + vextracti128 xm4, m5, 1 + packssdw xm5, xm4 + mova [tmpq], xm5 + add tmpq, 16 + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: +%if WIN64 + push r8 +%endif + mov r8d, wd + shl wd, 5 + mov r5, srcq + mov r7, tmpq + lea wd, [hq+wq-256] +.v_w8_loop0: + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + vbroadcasti128 m0, [srcq+r6 ] + vbroadcasti128 m6, [srcq+strideq*2] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m1, [srcq+strideq*0] + vbroadcasti128 m2, [srcq+strideq*1] + vbroadcasti128 m3, [srcq+strideq*2] + add srcq, r6 + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + shufpd m6, m2, 0x0c + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + shufpd m0, m3, 0x0c + punpcklwd m3, m6, m0 ; 23 + punpckhwd m6, m0 ; 56 +.v_w8_loop: + vbroadcasti128 m14, [srcq+strideq*0] + pmaddwd m12, m8, m1 ; a0 + pmaddwd m13, m8, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m12, m7 + paddd m13, m7 + paddd m12, m3 + paddd m13, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m12, m5 + vbroadcasti128 m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + paddd m13, m6 + shufpd m6, m0, m14, 0x0d + shufpd m0, m14, m5, 0x0c + punpcklwd m5, m6, m0 ; 67 + punpckhwd m6, m0 ; 78 + pmaddwd m14, m11, m5 ; a3 + paddd m12, m14 + pmaddwd m14, m11, m6 ; b3 + paddd m13, m14 + psrad m12, 4 + psrad m13, 4 + packssdw m12, m13 + vpermq m12, m12, q3120 + mova [tmpq+r8*0], xm12 + vextracti128 [tmpq+r8*2], m12, 1 + lea tmpq, [tmpq+r8*4] + sub hd, 2 + jg .v_w8_loop + add r5, 16 + add r7, 16 + movzx hd, wb + mov srcq, r5 + mov tmpq, r7 + sub wd, 1<<8 + jg .v_w8_loop0 +%if WIN64 + pop r8 +%endif + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + vpbroadcastd m15, [prep_8tap_2d_rnd] + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + vpbroadcastd m0, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + vpbroadcastq m1, [base+subpel_filters+myq*8] + lea r6, [strideq*3] + sub srcq, 2 + sub srcq, r6 + pxor m7, m7 + punpcklbw m7, m0 + punpcklbw m1, m1 + psraw m7, 4 + psraw m1, 8 + test dword r7m, 0x800 + jz .hv_w4_10bit + psraw m7, 2 +.hv_w4_10bit: + pshufd m11, m1, q0000 + pshufd m12, m1, q1111 + pshufd m13, m1, q2222 + pshufd m14, m1, q3333 +.hv_w4: + vbroadcasti128 m9, [subpel_h_shufA] + vbroadcasti128 m10, [subpel_h_shufB] + pshufd m8, m7, q1111 + pshufd m7, m7, q0000 + movu xm1, [srcq+strideq*0] + vinserti128 m1, [srcq+strideq*1], 1 ; 0 1 + vbroadcasti128 m0, [srcq+r6 ] + vinserti128 m2, m0, [srcq+strideq*2], 0 ; 2 3 + lea srcq, [srcq+strideq*4] + vinserti128 m0, [srcq+strideq*0], 1 ; 3 4 + movu xm3, [srcq+strideq*1] + vinserti128 m3, [srcq+strideq*2], 1 ; 5 6 + add srcq, r6 + pshufb m4, m1, m9 + pshufb m1, m10 + pmaddwd m4, m7 + pmaddwd m1, m8 + pshufb m5, m2, m9 + pshufb m2, m10 + pmaddwd m5, m7 + pmaddwd m2, m8 + paddd m4, m15 + paddd m1, m4 + pshufb m4, m0, m9 + pshufb m0, m10 + pmaddwd m4, m7 + pmaddwd m0, m8 + paddd m5, m15 + paddd m2, m5 + pshufb m5, m3, m9 + pshufb m3, m10 + pmaddwd m5, m7 + pmaddwd m3, m8 + paddd m4, m15 + paddd m4, m0 + paddd m5, m15 + paddd m5, m3 + vperm2i128 m0, m1, m2, 0x21 + psrld m1, 6 + psrld m2, 6 + vperm2i128 m3, m4, m5, 0x21 + pslld m4, 10 + pslld m5, 10 + pblendw m2, m4, 0xaa ; 23 34 + pslld m0, 10 + pblendw m1, m0, 0xaa ; 01 12 + psrld m3, 6 + pblendw m3, m5, 0xaa ; 45 56 + psrad m0, m5, 16 +.hv_w4_loop: + movu xm4, [srcq+strideq*0] + vinserti128 m4, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + pmaddwd m5, m11, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m12 ; a1 b1 + paddd m5, m15 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m13 ; a2 b2 + paddd m5, m3 + pshufb m3, m4, m9 + pshufb m4, m10 + pmaddwd m3, m7 + pmaddwd m4, m8 + paddd m3, m15 + paddd m4, m3 + psrad m4, 6 + packssdw m0, m4 ; _ 7 6 8 + vpermq m3, m0, q1122 ; _ 6 _ 7 + punpckhwd m3, m0 ; 67 78 + mova m0, m4 + pmaddwd m4, m14, m3 ; a3 b3 + paddd m4, m5 + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, 16 + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + vpbroadcastq m2, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmovle myd, mxd + pmovsxbw xm1, [base+subpel_filters+myq*8] +%if WIN64 + PUSH r8 +%endif + mov r8d, wd + shl wd, 5 + lea r6, [strideq*3] + sub srcq, 6 + sub srcq, r6 + mov r5, srcq + mov r7, tmpq + lea wd, [hq+wq-256] + pxor m0, m0 + punpcklbw m0, m2 + mova [v_mul], xm1 + psraw m0, 4 + test dword r7m, 0x800 + jz .hv_w8_10bit + psraw m0, 2 +.hv_w8_10bit: + pshufd m11, m0, q0000 + pshufd m12, m0, q1111 + pshufd m13, m0, q2222 + pshufd m14, m0, q3333 +.hv_w8_loop0: +%macro PREP_8TAP_HV_H 3 ; dst/src+0, src+8, src+16 + pshufb m2, m%1, m9 ; 2 3 3 4 4 5 5 6 + pshufb m%1, m8 ; 0 1 1 2 2 3 3 4 + pmaddwd m3, m12, m2 + pmaddwd m%1, m11 + pshufb m%2, m9 ; 6 7 7 8 8 9 9 a + shufpd m2, m%2, 0x05 ; 4 5 5 6 6 7 7 8 + paddd m3, m15 + paddd m%1, m3 + pmaddwd m3, m14, m%2 + paddd m%1, m3 + pmaddwd m3, m13, m2 + pshufb m%3, m9 ; a b b c c d d e + pmaddwd m2, m11 + paddd m%1, m3 + pmaddwd m3, m12, m%2 + shufpd m%2, m%3, 0x05 ; 8 9 9 a a b b c + pmaddwd m%3, m14 + pmaddwd m%2, m13 + paddd m2, m15 + paddd m2, m3 + paddd m2, m%3 + paddd m2, m%2 + psrad m%1, 6 + psrad m2, 6 + packssdw m%1, m2 +%endmacro + movu xm4, [srcq+r6 + 0] + vbroadcasti128 m8, [subpel_h_shufA] + movu xm6, [srcq+r6 + 8] + vbroadcasti128 m9, [subpel_h_shufB] + movu xm0, [srcq+r6 +16] + movu xm5, [srcq+strideq*0+ 0] + vinserti128 m5, [srcq+strideq*4+ 0], 1 + movu xm1, [srcq+strideq*0+16] + vinserti128 m1, [srcq+strideq*4+16], 1 + shufpd m7, m5, m1, 0x05 + INIT_XMM avx2 + PREP_8TAP_HV_H 4, 6, 0 ; 3 + INIT_YMM avx2 + PREP_8TAP_HV_H 5, 7, 1 ; 0 4 + movu xm0, [srcq+strideq*2+ 0] + vinserti128 m0, [srcq+r6 *2+ 0], 1 + movu xm1, [srcq+strideq*2+16] + vinserti128 m1, [srcq+r6 *2+16], 1 + shufpd m7, m0, m1, 0x05 + PREP_8TAP_HV_H 0, 7, 1 ; 2 6 + movu xm6, [srcq+strideq*1+ 0] + movu xm1, [srcq+strideq*1+16] + lea srcq, [srcq+strideq*4] + vinserti128 m6, [srcq+strideq*1+ 0], 1 + vinserti128 m1, [srcq+strideq*1+16], 1 + add srcq, r6 + shufpd m7, m6, m1, 0x05 + PREP_8TAP_HV_H 6, 7, 1 ; 1 5 + vpermq m4, m4, q1100 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + vpermq m7, m0, q3120 + punpcklwd m3, m7, m4 ; 23 + punpckhwd m4, m5 ; 34 + punpcklwd m1, m5, m6 ; 01 + punpckhwd m5, m6 ; 45 + punpcklwd m2, m6, m7 ; 12 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vpbroadcastd m9, [v_mul+4*0] + vpbroadcastd m7, [v_mul+4*1] + vpbroadcastd m10, [v_mul+4*2] + pmaddwd m8, m9, m1 ; a0 + pmaddwd m9, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m7 ; a1 + pmaddwd m4, m7 ; b1 + paddd m8, m15 + paddd m9, m15 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m8, m5 + paddd m9, m6 + movu xm5, [srcq+strideq*0] + vinserti128 m5, [srcq+strideq*1], 1 + vbroadcasti128 m7, [subpel_h_shufA] + vbroadcasti128 m10, [subpel_h_shufB] + movu xm6, [srcq+strideq*0+16] + vinserti128 m6, [srcq+strideq*1+16], 1 + vextracti128 [tmpq], m0, 1 + pshufb m0, m5, m7 ; 01 + pshufb m5, m10 ; 23 + pmaddwd m0, m11 + pmaddwd m5, m12 + paddd m0, m15 + paddd m0, m5 + pshufb m5, m6, m7 ; 89 + pshufb m6, m10 ; ab + pmaddwd m5, m13 + pmaddwd m6, m14 + paddd m5, m15 + paddd m6, m5 + movu xm5, [srcq+strideq*0+8] + vinserti128 m5, [srcq+strideq*1+8], 1 + lea srcq, [srcq+strideq*2] + pshufb m7, m5, m7 + pshufb m5, m10 + pmaddwd m10, m13, m7 + pmaddwd m7, m11 + paddd m0, m10 + paddd m6, m7 + pmaddwd m7, m14, m5 + pmaddwd m5, m12 + paddd m0, m7 + paddd m5, m6 + vbroadcasti128 m6, [tmpq] + vpbroadcastd m10, [v_mul+4*3] + psrad m0, 6 + psrad m5, 6 + packssdw m0, m5 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m10, m5 ; a3 + pmaddwd m10, m6 ; b3 + paddd m7, m8 + paddd m9, m10 + psrad m7, 6 + psrad m9, 6 + packssdw m7, m9 + vpermq m7, m7, q3120 + mova [tmpq+r8*0], xm7 + vextracti128 [tmpq+r8*2], m7, 1 + lea tmpq, [tmpq+r8*4] + sub hd, 2 + jg .hv_w8_loop + add r5, 16 + add r7, 16 + movzx hd, wb + mov srcq, r5 + mov tmpq, r7 + sub wd, 1<<8 + jg .hv_w8_loop0 +%if WIN64 + POP r8 +%endif + RET + +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro REMAP_REG 2 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %xdefine r14_save r14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + %xdefine r14 r14_save + %undef r14_save + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%macro MC_8TAP_SCALED_H 8-9 0 ; dst, tmp[0-6], load_hrnd + movu xm%1, [srcq+ r4*2] + movu xm%2, [srcq+ r6*2] + movu xm%3, [srcq+ r7*2] + movu xm%4, [srcq+ r9*2] + vinserti128 m%1, [srcq+r10*2], 1 + vinserti128 m%2, [srcq+r11*2], 1 + vinserti128 m%3, [srcq+r13*2], 1 + vinserti128 m%4, [srcq+ rX*2], 1 + add srcq, ssq + movu xm%5, [srcq+ r4*2] + movu xm%6, [srcq+ r6*2] + movu xm%7, [srcq+ r7*2] + movu xm%8, [srcq+ r9*2] + vinserti128 m%5, [srcq+r10*2], 1 + vinserti128 m%6, [srcq+r11*2], 1 + vinserti128 m%7, [srcq+r13*2], 1 + vinserti128 m%8, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m%1, m12 + pmaddwd m%2, m13 + pmaddwd m%3, m14 + pmaddwd m%4, m15 + pmaddwd m%5, m12 + pmaddwd m%6, m13 + pmaddwd m%7, m14 + pmaddwd m%8, m15 + phaddd m%1, m%2 + %if %9 + mova m10, [rsp+0x00] + %endif + phaddd m%3, m%4 + phaddd m%5, m%6 + phaddd m%7, m%8 + phaddd m%1, m%3 + phaddd m%5, m%7 + paddd m%1, m10 + paddd m%5, m10 + psrad m%1, xm11 + psrad m%5, xm11 + packssdw m%1, m%5 +%endmacro + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isput 1 + %assign isprep 0 +cglobal put_8tap_scaled_16bpc, 4, 14, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %xdefine base_reg r12 + mov r7d, pxmaxm +%else + %assign isput 0 + %assign isprep 1 +cglobal prep_8tap_scaled_16bpc, 4, 14, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %define tmp_stridem qword [rsp+0xd0] + %xdefine base_reg r11 +%endif + lea base_reg, [%1_8tap_scaled_16bpc_avx2] +%define base base_reg-%1_8tap_scaled_16bpc_avx2 + tzcnt wd, wm + vpbroadcastd m8, dxm +%if isprep && UNIX64 + movd xm10, mxd + vpbroadcastd m10, xm10 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 + mov r6d, pxmaxm +%else + vpbroadcastd m10, mxm + %if isput + vpbroadcastw m11, pxmaxm + %else + mov r6d, pxmaxm + %endif +%endif + mov dyd, dym +%if isput + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %else + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %endif + %define dsm [rsp+0x98] + %define rX r1 + %define rXd r1d +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %else + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %define hm [rsp+0x98] + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define rX r14 + %define rXd r14d +%endif + shr r7d, 11 + vpbroadcastd m6, [base+pd_0x3ff] + vpbroadcastd m12, [base+s_8tap_h_rnd+r7*4] + movd xm7, [base+s_8tap_h_sh+r7*4] +%if isput + vpbroadcastd m13, [base+put_s_8tap_v_rnd+r7*4] + pinsrd xm7, [base+put_s_8tap_v_sh+r7*4], 2 +%else + vpbroadcastd m13, [base+pd_m524256] +%endif + pxor m9, m9 + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0,1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd xm4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + movu xm2, [srcq+ssq*2] + movu xm3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*0], 1 ; 0 4 + vinserti128 m1, [srcq+ssq*1], 1 ; 1 5 + vinserti128 m2, [srcq+ssq*2], 1 ; 2 6 + vinserti128 m3, [srcq+ss3q ], 1 ; 3 7 + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m10}, m0, m1, m2, m3 + REPX {pmaddwd x, m15}, m0, m1, m2, m3 + phaddd m0, m1 + phaddd m2, m3 + paddd m0, m12 + paddd m2, m12 + psrad m0, xm7 + psrad m2, xm7 + packssdw m0, m2 ; 0 1 2 3 4 5 6 7 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 ; 1 2 3 4 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + pshufd xm4, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm4 ; 45 56 + punpckhwd xm4, xm1, xm4 ; 67 __ +.w2_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm14, r6q + pmovsxbw xm14, xm14 + pshufd xm8, xm14, q0000 + pshufd xm9, xm14, q1111 + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pshufd xm8, xm14, q2222 + pshufd xm14, xm14, q3333 + paddd xm5, xm6 + pmaddwd xm6, xm2, xm8 + pmaddwd xm8, xm4, xm14 + psrldq xm9, xm7, 8 + paddd xm5, xm6 + paddd xm5, xm13 + paddd xm5, xm8 + psrad xm5, xm9 + packusdw xm5, xm5 + pminsw xm5, xm11 + movd [dstq], xm5 + add dstq, dsq + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w2_loop + movu xm5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps xm3, xm0, q1032 ; 01 12 + shufps xm0, xm2, q1032 ; 23 34 + shufps xm2, xm4, q1032 ; 45 56 + pshufb xm5, xm10 + pmaddwd xm5, xm15 + phaddd xm5, xm5 + paddd xm5, xm12 + psrad xm5, xm7 + packssdw xm5, xm5 + palignr xm1, xm5, xm1, 12 + punpcklqdq xm1, xm1 ; 6 7 6 7 + punpcklwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +.w2_skip_line: + movu xm6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova xm3, xm0 ; 01 12 + mova xm0, xm2 ; 23 34 + pshufb xm5, xm10 + pshufb xm6, xm10 + pmaddwd xm5, xm15 + pmaddwd xm6, xm15 + phaddd xm5, xm6 + paddd xm5, xm12 + psrad xm5, xm7 + packssdw xm5, xm5 ; 6 7 6 7 + palignr xm1, xm5, xm1, 8 ; 4 5 6 7 + pshufd xm5, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm5 ; 45 56 + punpckhwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +%endif +.w4: + mov myd, mym + mova [rsp+0x00], m12 +%if isput + mova [rsp+0x20], xm13 +%else + SWAP m11, m13 +%endif + mova [rsp+0x30], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m0, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + pcmpeqd m6, m9 + punpckldq m10, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pblendvb m14, m2, m10 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu xm7, [srcq+ssq*0] + movu xm9, [srcq+ssq*1] + movu xm8, [srcq+ssq*2] + movu xm10, [srcq+ss3q ] + movu xm1, [srcq+r4 ] + movu xm3, [srcq+r6 ] + movu xm2, [srcq+r11 ] + movu xm4, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + vinserti128 m7, [srcq+ssq*0], 1 + vinserti128 m9, [srcq+ssq*1], 1 + vinserti128 m8, [srcq+ssq*2], 1 + vinserti128 m10, [srcq+ss3q ], 1 + vinserti128 m1, [srcq+r4 ], 1 + vinserti128 m3, [srcq+r6 ], 1 + vinserti128 m2, [srcq+r11 ], 1 + vinserti128 m4, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m0 + paddb m13, m0 + REPX {pshufb x, m12}, m7, m9, m8, m10 + REPX {pmaddwd x, m14}, m7, m9, m8, m10 + REPX {pshufb x, m13}, m1, m2, m3, m4 + REPX {pmaddwd x, m15}, m1, m2, m3, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x30] + phaddd m7, m1 + phaddd m9, m3 + phaddd m8, m2 + phaddd m10, m4 + REPX {paddd x, m5}, m7, m9, m8, m10 + REPX {psrad x, xm6}, m7, m9, m8, m10 + packssdw m7, m9 ; 0 1 4 5 + packssdw m8, m10 ; 2 3 6 7 + vextracti128 xm9, m7, 1 ; 4 5 + vextracti128 xm3, m8, 1 ; 6 7 + shufps xm4, xm7, xm8, q1032 ; 1 2 + shufps xm5, xm8, xm9, q1032 ; 3 4 + shufps xm6, xm9, xm3, q1032 ; 5 6 + psrldq xm10, xm3, 8 ; 7 _ + punpcklwd xm0, xm7, xm4 ; 01 + punpckhwd xm7, xm4 ; 12 + punpcklwd xm1, xm8, xm5 ; 23 + punpckhwd xm8, xm5 ; 34 + punpcklwd xm2, xm9, xm6 ; 45 + punpckhwd xm9, xm6 ; 56 + punpcklwd xm3, xm10 ; 67 + mova [rsp+0x40], xm7 + mova [rsp+0x50], xm8 + mova [rsp+0x60], xm9 +.w4_loop: + and myd, 0x3ff + mov r11d, 64 << 24 + mov r13d, myd + shr r13d, 6 + lea r13d, [t1+r13] + cmovnz r11q, [base+subpel_filters+r13*8] + movq xm9, r11q + pmovsxbw xm9, xm9 + pshufd xm7, xm9, q0000 + pshufd xm8, xm9, q1111 + pmaddwd xm4, xm0, xm7 + pmaddwd xm5, xm1, xm8 + pshufd xm7, xm9, q2222 + pshufd xm9, xm9, q3333 + pmaddwd xm6, xm2, xm7 + pmaddwd xm8, xm3, xm9 +%if isput + mova xm7, [rsp+0x20] + movd xm9, [rsp+0x38] +%else + SWAP m7, m11 +%endif + paddd xm4, xm5 + paddd xm6, xm8 + paddd xm4, xm6 + paddd xm4, xm7 +%if isput + psrad xm4, xm9 + packusdw xm4, xm4 + pminuw xm4, xm11 + movq [dstq], xm4 + add dstq, dsq +%else + SWAP m11, m7 + psrad xm4, 6 + packssdw xm4, xm4 + movq [tmpq], xm4 + add tmpq, 8 +%endif + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w4_loop + mova xm8, [rsp+0x00] + movd xm9, [rsp+0x30] + movu xm4, [srcq] + movu xm5, [srcq+r4] + test myd, 0x400 + jz .w4_skip_line + mova xm0, [rsp+0x40] + mova [rsp+0x40], xm1 + mova xm1, [rsp+0x50] + mova [rsp+0x50], xm2 + mova xm2, [rsp+0x60] + mova [rsp+0x60], xm3 + pshufb xm4, xm12 + pshufb xm5, xm13 + pmaddwd xm4, xm14 + pmaddwd xm5, xm15 + phaddd xm4, xm5 + paddd xm4, xm8 + psrad xm4, xm9 + packssdw xm4, xm4 + punpcklwd xm3, xm10, xm4 + mova xm10, xm4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu xm6, [srcq+ssq*1] + movu xm7, [srcq+r6] + movu m0, [rsp+0x50] + pshufb xm4, xm12 + pshufb xm6, xm12 + pshufb xm5, xm13 + pshufb xm7, xm13 + pmaddwd xm4, xm14 + pmaddwd xm6, xm14 + pmaddwd xm5, xm15 + pmaddwd xm7, xm15 + mova [rsp+0x40], m0 + phaddd xm4, xm5 + phaddd xm6, xm7 + paddd xm4, xm8 + paddd xm6, xm8 + psrad xm4, xm9 + psrad xm6, xm9 + packssdw xm4, xm6 + punpcklwd xm9, xm10, xm4 + mova [rsp+0x60], xm9 + psrldq xm10, xm4, 8 + mova xm0, xm1 + mova xm1, xm2 + mova xm2, xm3 + punpcklwd xm3, xm4, xm10 + lea srcq, [srcq+ssq*2] + jmp .w4_loop + SWAP m10, m13 +%if isprep + SWAP m13, m11 +%endif +.w8: + mov dword [rsp+0x80], 1 + movifprep tmp_stridem, 16 + jmp .w_start +.w16: + mov dword [rsp+0x80], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [rsp+0x80], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [rsp+0x80], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [rsp+0x80], 16 + movifprep tmp_stridem, 256 +.w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free +%if isput + movifnidn dsm, dsq + mova [rsp+0xb0], xm7 +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + shr t0d, 16 + sub srcq, 6 + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0x84], t0d + mov [rsp+0x88], srcq + mov [rsp+0x90], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + jmp .hloop +.hloop_prep: + dec dword [rsp+0x80] + jz .ret + add qword [rsp+0x90], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x40] + vpbroadcastd m15, [rsp+0x84] + pxor m9, m9 + mov srcq, [rsp+0x88] + mov r0q, [rsp+0x90] ; dstq / tmpq +.hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x40], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq [rsp+0xa0], xm1 + movq [rsp+0xa8], xm7 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x60], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x60] + vbroadcasti128 m9, [base+subpel_s_shuf8] + mov myd, mym + mov dyd, dym + pshufb m0, m9 ; 01a 01b + pshufb m1, m9 ; 23a 23b + pshufb m2, m9 ; 45a 45b + pshufb m3, m9 ; 67a 67b +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm9, r6q + punpcklqdq xm9, xm9 + pmovsxbw m9, xm9 + pshufd m8, m9, q0000 + pshufd m7, m9, q1111 + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m7 + pshufd m8, m9, q2222 + pshufd m9, m9, q3333 + pmaddwd m6, m2, m8 + pmaddwd m7, m3, m9 +%if isput + psrldq xm8, xm11, 8 +%endif + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, xm8 + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xb0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [rsp+0x60], myd + mov r4d, [rsp+0xa0] + mov r6d, [rsp+0xa4] + mov r7d, [rsp+0xa8] + mov r9d, [rsp+0xac] + jz .skip_line + vbroadcasti128 m9, [base+wswap] + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + mov myd, [rsp+0x60] + mov dyd, dym + pshufb m0, m9 + pshufb m1, m9 + pshufb m2, m9 + pshufb m3, m9 + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + phaddd m4, m6 + paddd m4, m10 + psrad m4, xm11 + pslld m4, 16 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .vloop +.skip_line: + mova m0, m1 + mova m1, m2 + mova m2, m3 + MC_8TAP_SCALED_H 3, 10, 4, 5, 6, 7, 8, 9, 1 + vbroadcasti128 m9, [base+subpel_s_shuf8] + mov myd, [rsp+0x60] + mov dyd, dym + pshufb m3, m9 + jmp .vloop + SWAP m1, m12, m10 + SWAP m7, m11 +.dy1: + movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy1_w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0-1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + movu xm2, [srcq+ssq*2] + movu xm3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*1], 1 + vinserti128 m2, [srcq+ssq*2], 1 + add srcq, ss3q + movq xm6, r4q + pmovsxbw xm6, xm6 + pshufd xm8, xm6, q0000 + pshufd xm9, xm6, q1111 + pshufd xm14, xm6, q2222 + pshufd xm6, xm6, q3333 + REPX {pshufb x, m10}, m0, m1, m2 + pshufb xm3, xm10 + REPX {pmaddwd x, m15}, m0, m1, m2 + pmaddwd xm3, xm15 + phaddd m0, m1 + phaddd m2, m3 + paddd m0, m12 + paddd m2, m12 + psrad m0, xm7 + psrad m2, xm7 + packssdw m0, m2 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 + pshufd xm4, xm1, q2121 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + punpcklwd xm2, xm1, xm4 ; 45 56 +.dy1_w2_loop: + movu xm1, [srcq+ssq*0] + movu xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm1, xm10 + pshufb xm5, xm10 + pmaddwd xm1, xm15 + pmaddwd xm5, xm15 + phaddd xm1, xm5 + pmaddwd xm5, xm3, xm8 + mova xm3, xm0 + pmaddwd xm0, xm9 + paddd xm1, xm12 + psrad xm1, xm7 + packssdw xm1, xm1 + paddd xm5, xm0 + mova xm0, xm2 + pmaddwd xm2, xm14 + paddd xm5, xm2 + palignr xm2, xm1, xm4, 12 + punpcklwd xm2, xm1 ; 67 78 + pmaddwd xm4, xm2, xm6 + paddd xm5, xm13 + paddd xm5, xm4 + mova xm4, xm1 + psrldq xm1, xm7, 8 + psrad xm5, xm1 + packusdw xm5, xm5 + pminsw xm5, xm11 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +.dy1_w4: + mov myd, mym +%if isput + mova [rsp+0x50], xm11 +%endif + mova [rsp+0x00], m12 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m4, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + pcmpeqd m6, m9 + punpckldq m10, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pblendvb m14, m2, m10 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*2] + lea r11, [r4+ssq*1] + lea r13, [r4+ss3q ] + movu xm0, [srcq+ssq*0] + movu xm7, [srcq+r4 ] + movu xm1, [srcq+ssq*2] + movu xm8, [srcq+r6 ] + vinserti128 m0, [srcq+ssq*1], 1 ; 0 1 + vinserti128 m7, [srcq+r11 ], 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 2 3 + vinserti128 m8, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + movu xm2, [srcq+ssq*0] + movu xm9, [srcq+r4 ] + movu xm3, [srcq+ssq*2] ; 6 _ + movu xm10, [srcq+r6 ] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + vinserti128 m9, [srcq+r11 ], 1 + lea srcq, [srcq+ss3q ] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m4 + paddb m13, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x40] + pshufb m0, m12 + pshufb m1, m12 + pmaddwd m0, m14 + pmaddwd m1, m14 + pshufb m7, m13 + pshufb m8, m13 + pmaddwd m7, m15 + pmaddwd m8, m15 + pshufb m2, m12 + pshufb xm3, xm12 + pmaddwd m2, m14 + pmaddwd xm3, xm14 + pshufb m9, m13 + pshufb xm10, xm13 + pmaddwd m9, m15 + pmaddwd xm10, xm15 + phaddd m0, m7 + phaddd m1, m8 + phaddd m2, m9 + phaddd xm3, xm10 + paddd m0, m5 + paddd m1, m5 + paddd m2, m5 + paddd xm3, xm5 + psrad m0, xm6 + psrad m1, xm6 + psrad m2, xm6 + psrad xm3, xm6 + vperm2i128 m4, m0, m1, 0x21 ; 1 2 + vperm2i128 m5, m1, m2, 0x21 ; 3 4 + vperm2i128 m6, m2, m3, 0x21 ; 5 6 + shr myd, 6 + mov r13d, 64 << 24 + lea myd, [t1+myq] + cmovnz r13q, [base+subpel_filters+myq*8] + pslld m4, 16 + pslld m5, 16 + pslld m6, 16 + pblendw m0, m4, 0xaa ; 01 12 + pblendw m1, m5, 0xaa ; 23 34 + pblendw m2, m6, 0xaa ; 45 56 + movq xm10, r13q + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 +.dy1_w4_loop: + movu xm11, [srcq+ssq*0] + movu xm6, [srcq+r4 ] + vinserti128 m11, [srcq+ssq*1], 1 + vinserti128 m6, [srcq+r11 ], 1 + lea srcq, [srcq+ssq*2] + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pshufb m11, m12 + pshufb m6, m13 + pmaddwd m11, m14 + pmaddwd m6, m15 + paddd m4, [rsp+0x20] + phaddd m11, m6 + pmaddwd m6, m2, m9 + paddd m11, [rsp+0x00] + psrad m11, [rsp+0x40] + mova m0, m1 + mova m1, m2 + paddd m5, m6 + paddd m4, m5 + vinserti128 m2, m3, xm11, 1 + pslld m3, m11, 16 + pblendw m2, m3, 0xaa ; 67 78 + pmaddwd m5, m2, m10 + vextracti128 xm3, m11, 1 + paddd m4, m5 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0x50] + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy1_w4_loop + MC_8TAP_SCALED_RET + SWAP m10, m13 +.dy1_w8: + mov dword [rsp+0xa0], 1 + movifprep tmp_stridem, 16 + jmp .dy1_w_start +.dy1_w16: + mov dword [rsp+0xa0], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [rsp+0xa0], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [rsp+0xa0], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [rsp+0xa0], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free + mov myd, mym +%if isput + %define dsm [rsp+0xb8] + movifnidn dsm, dsq + mova [rsp+0xc0], xm7 +%else + %if UNIX64 + %define hm [rsp+0xb8] + %endif +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm11 + shr t0d, 16 + sub srcq, 6 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0xa4], t0d + mov [rsp+0xa8], srcq + mov [rsp+0xb0], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+0x50], xm0 + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [rsp+0xa0] + jz .ret + add qword [rsp+0xb0], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x60] + vpbroadcastd m15, [rsp+0xa4] + pxor m9, m9 + mov srcq, [rsp+0xa8] + mov r0q, [rsp+0xb0] ; dstq / tmpq + mova m10, [rsp+0x00] + mova xm11, [rsp+0x40] +.dy1_hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x60], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x80], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x80] + vbroadcasti128 m7, [base+subpel_s_shuf8] + vpbroadcastd m8, [rsp+0x50] + vpbroadcastd m9, [rsp+0x54] + vpbroadcastd m10, [rsp+0x58] + vpbroadcastd m11, [rsp+0x5c] + pshufb m0, m7 ; 01a 01b + pshufb m1, m7 ; 23a 23b + pshufb m2, m7 ; 45a 45b + pshufb m3, m7 ; 67a 67b +.dy1_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xc0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy1_hloop_prep + vbroadcasti128 m7, [base+wswap] + pshufb m0, m7 + pshufb m1, m7 + pshufb m2, m7 + pshufb m3, m7 + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + phaddd m4, m6 + paddd m4, [rsp+0x00] + psrad m4, [rsp+0x40] + pslld m4, 16 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .dy1_vloop + SWAP m1, m12, m10 + SWAP m7, m11 +.dy2: + movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy2_w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0-1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd xm4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*2] + movu xm2, [srcq+ssq*4] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*1], 1 ; 0 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 2 3 + lea srcq, [srcq+ssq*4] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + lea srcq, [srcq+ssq*2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m0, m10 + pshufb m1, m10 + pshufb m2, m10 + pmaddwd m0, m15 + pmaddwd m1, m15 + pmaddwd m2, m15 + movq xm6, r4q + pmovsxbw xm6, xm6 + phaddd m0, m1 + phaddd m1, m2 + paddd m0, m12 + paddd m1, m12 + psrad m0, xm7 + psrad m1, xm7 + packssdw m0, m1 ; 0 2 2 4 1 3 3 5 + vextracti128 xm1, m0, 1 + pshufd xm8, xm6, q0000 + pshufd xm9, xm6, q1111 + pshufd xm14, xm6, q2222 + pshufd xm6, xm6, q3333 + punpcklwd xm2, xm0, xm1 ; 01 23 + punpckhwd xm1, xm0, xm1 ; 23 45 +.dy2_w2_loop: + movu xm3, [srcq+ssq*0] + movu xm5, [srcq+ssq*2] + vinserti128 m3, [srcq+ssq*1], 1 ; 6 7 + vinserti128 m5, [srcq+ss3q ], 1 ; 8 9 + lea srcq, [srcq+ssq*4] + pmaddwd xm4, xm2, xm8 + pmaddwd xm1, xm9 + pshufb m3, m10 + pshufb m5, m10 + pmaddwd m3, m15 + pmaddwd m5, m15 + phaddd m3, m5 + paddd xm4, xm1 + paddd m3, m12 + psrad m3, xm7 + packssdw m3, m3 + pshufd m3, m3, q2100 + palignr m0, m3, m0, 12 ; 4 6 6 8 5 7 7 9 + vextracti128 xm1, m0, 1 + punpcklwd xm2, xm0, xm1 ; 45 67 + punpckhwd xm1, xm0, xm1 ; 67 89 + pmaddwd xm3, xm2, xm14 + pmaddwd xm5, xm1, xm6 + paddd xm4, xm13 + paddd xm4, xm3 + psrldq xm3, xm7, 8 + paddd xm4, xm5 + psrad xm4, xm3 + packusdw xm4, xm4 + pminsw xm4, xm11 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +.dy2_w4: + mov myd, mym +%if isput + mova [rsp+0x50], xm11 +%endif + mova [rsp+0x00], m12 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m4, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + shr myd, 6 + mov r13d, 64 << 24 + lea myd, [t1+myq] + cmovnz r13q, [base+subpel_filters+myq*8] + pcmpeqd m6, m9 + punpckldq m11, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + movq xm10, r13q + pblendvb m14, m2, m11 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu xm0, [srcq+ssq*0] + movu xm7, [srcq+r4 ] + movu xm1, [srcq+ssq*1] + movu xm8, [srcq+r6 ] + vinserti128 m0, [srcq+ssq*2], 1 ; 0 2 + vinserti128 m7, [srcq+r11 ], 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 1 3 + vinserti128 m8, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + movu xm2, [srcq+ssq*0] + movu xm9, [srcq+r4 ] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + vinserti128 m9, [srcq+r6 ], 1 + lea srcq, [srcq+ssq*2] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m4 + paddb m13, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x40] + pshufb m0, m12 + pshufb m1, m12 + pshufb m2, m12 + pmaddwd m0, m14 + pmaddwd m1, m14 + pmaddwd m2, m14 + pshufb m7, m13 + pshufb m8, m13 + pshufb m9, m13 + pmaddwd m7, m15 + pmaddwd m8, m15 + pmaddwd m9, m15 + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 + phaddd m0, m7 + phaddd m1, m8 + phaddd m2, m9 + paddd m0, m5 + paddd m1, m5 + paddd m2, m5 + psrad m0, xm6 + psrad m1, xm6 + psrad m2, xm6 + vperm2i128 m3, m0, m2, 0x21 ; 2 4 + vperm2i128 m2, m1, 0x13 ; 3 5 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + packssdw m0, m3 ; 0 2 2 4 + packssdw m1, m2 ; 1 3 3 5 + punpckhwd m2, m0, m1 ; 23 45 + punpcklwd m0, m1 ; 01 23 +.dy2_w4_loop: + movu xm1, [srcq+ssq*0] + movu xm6, [srcq+r4 ] + movu xm3, [srcq+ssq*1] + movu xm11, [srcq+r6 ] + vinserti128 m1, [srcq+ssq*2], 1 ; 6 8 + vinserti128 m6, [srcq+r11 ], 1 + vinserti128 m3, [srcq+ss3q ], 1 ; 7 9 + vinserti128 m11, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + pmaddwd m4, m0, m7 + pmaddwd m5, m2, m8 + pshufb m1, m12 + pshufb m3, m12 + pmaddwd m1, m14 + pmaddwd m3, m14 + mova m0, [rsp+0x00] + pshufb m6, m13 + pshufb m11, m13 + pmaddwd m6, m15 + pmaddwd m11, m15 + paddd m4, m5 + movd xm5, [rsp+0x40] + phaddd m1, m6 + phaddd m3, m11 + paddd m1, m0 + paddd m3, m0 + psrad m1, xm5 + psrad m3, xm5 + pslld m3, 16 + pblendw m1, m3, 0xaa ; 67 89 + vperm2i128 m0, m2, m1, 0x21 ; 45 67 + paddd m4, [rsp+0x20] + mova m2, m1 + pmaddwd m5, m0, m9 + pmaddwd m6, m2, m10 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0x50] + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET + SWAP m10, m13 +.dy2_w8: + mov dword [rsp+0xa0], 1 + movifprep tmp_stridem, 16 + jmp .dy2_w_start +.dy2_w16: + mov dword [rsp+0xa0], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [rsp+0xa0], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [rsp+0xa0], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [rsp+0xa0], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free + mov myd, mym +%if isput + movifnidn dsm, dsq + mova [rsp+0xc0], xm7 +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm11 + shr t0d, 16 + sub srcq, 6 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0xa4], t0d + mov [rsp+0xa8], srcq + mov [rsp+0xb0], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+0x50], xm0 + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [rsp+0xa0] + jz .ret + add qword [rsp+0xb0], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x60] + vpbroadcastd m15, [rsp+0xa4] + pxor m9, m9 + mov srcq, [rsp+0xa8] + mov r0q, [rsp+0xb0] ; dstq / tmpq + mova m10, [rsp+0x00] + mova xm11, [rsp+0x40] +.dy2_hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x60], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x80], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x80] + vbroadcasti128 m7, [base+subpel_s_shuf8] + vpbroadcastd m8, [rsp+0x50] + vpbroadcastd m9, [rsp+0x54] + vpbroadcastd m10, [rsp+0x58] + vpbroadcastd m11, [rsp+0x5c] + pshufb m0, m7 ; 01a 01b + pshufb m1, m7 ; 23a 23b + pshufb m2, m7 ; 45a 45b + pshufb m3, m7 ; 67a 67b +.dy2_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xc0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy2_hloop_prep + mova m0, m1 + mova m1, m2 + mova m2, m3 + movu xm3, [srcq+ r4*2] + movu xm4, [srcq+ r6*2] + movu xm5, [srcq+ r7*2] + movu xm6, [srcq+ r9*2] + vinserti128 m3, [srcq+r10*2], 1 + vinserti128 m4, [srcq+r11*2], 1 + vinserti128 m5, [srcq+r13*2], 1 + vinserti128 m6, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m3, m12 + pmaddwd m4, m13 + pmaddwd m5, m14 + pmaddwd m6, m15 + phaddd m3, m4 + phaddd m5, m6 + phaddd m3, m5 + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + mova m5, [rsp+0x00] + movd xm7, [rsp+0x40] + phaddd m4, m6 + paddd m3, m5 + paddd m4, m5 + psrad m3, xm7 + psrad m4, xm7 + pslld m4, 16 + pblendw m3, m4, 0xaa + jmp .dy2_vloop +.ret: + MC_8TAP_SCALED_RET 0 +%undef isput +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled_16bpc + mov t0d, (5*15 << 16) | 5*15 + mov t1d, t0d + jmp mangle(private_prefix %+ _%1_8tap_scaled_16bpc %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%else +DECLARE_REG_TMP 6, 8 +%endif + +%define PUT_8TAP_SCALED_FN FN put_8tap_scaled, +BILIN_SCALED_FN put +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, +BILIN_SCALED_FN prep +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + +%macro WARP_V 5 ; dst, 01, 23, 45, 67 + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm8, [filterq+myq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 ; a e + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+deltaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; b f + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm9, [filterq+myq *8] + vinserti128 m9, [filterq+tmp1q*8], 1 ; c g + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+gammaq] ; my += gamma + punpcklwd m8, m0 + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; d h + punpcklwd m0, m9, m0 + punpckldq m9, m8, m0 + punpckhdq m0, m8, m0 + punpcklbw m8, m11, m9 ; a0 a1 b0 b1 c0 c1 d0 d1 << 8 + punpckhbw m9, m11, m9 ; a2 a3 b2 b3 c2 c3 d2 d3 << 8 + pmaddwd m%2, m8 + pmaddwd m9, m%3 + punpcklbw m8, m11, m0 ; a4 a5 b4 b5 c4 c5 d4 d5 << 8 + punpckhbw m0, m11, m0 ; a6 a7 b6 b7 c6 c7 d6 d7 << 8 + pmaddwd m8, m%4 + pmaddwd m0, m%5 + paddd m9, m%2 + mova m%2, m%3 + paddd m0, m8 + mova m%3, m%4 + mova m%4, m%5 + paddd m%1, m0, m9 +%endmacro + +cglobal warp_affine_8x8t_16bpc, 4, 14, 16, tmp, ts + mov r6d, r7m + lea r9, [$$] + shr r6d, 11 + vpbroadcastd m13, [r9-$$+warp8x8_shift+r6*4] + vpbroadcastd m14, [warp8x8t_rnd] + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx2).main + jmp .start +.loop: + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx2).main2 + lea tmpq, [tmpq+tsq*4] +.start: + paddd m7, m14 + paddd m0, m14 + psrad m7, 15 + psrad m0, 15 + packssdw m7, m0 + vpermq m7, m7, q3120 + mova [tmpq+tsq*0], xm7 + vextracti128 [tmpq+tsq*2], m7, 1 + dec r4d + jg .loop +.end: + RET + +cglobal warp_affine_8x8_16bpc, 4, 14, 16, dst, ds, src, ss, abcd, mx, tmp2, \ + alpha, beta, filter, tmp1, delta, \ + my, gamma + mov r6d, r7m + lea filterq, [$$] + shr r6d, 11 + vpbroadcastd m13, [filterq-$$+warp8x8_shift+r6*4] + vpbroadcastd m14, [filterq-$$+warp8x8_rnd +r6*4] + vpbroadcastw m15, r7m ; pixel_max + call .main + jmp .start +.loop: + call .main2 + lea dstq, [dstq+dsq*2] +.start: + psrad m7, 16 + psrad m0, 16 + packusdw m7, m0 + pmulhrsw m7, m14 + pminsw m7, m15 + vpermq m7, m7, q3120 + mova [dstq+dsq*0], xm7 + vextracti128 [dstq+dsq*1], m7, 1 + dec r4d + jg .loop +.end: + RET +ALIGN function_align +.main: + ; Stack args offset by one (r4m -> r5m etc.) due to call +%if WIN64 + mov abcdq, r5m + mov mxd, r6m +%endif + movsx alphad, word [abcdq+2*0] + movsx betad, word [abcdq+2*1] + vpbroadcastd m12, [pd_32768] + pxor m11, m11 + add filterq, mc_warp_filter-$$ + lea tmp1q, [ssq*3] + add mxd, 512+(64<<10) + lea tmp2d, [alphaq*3] + sub srcq, tmp1q ; src -= src_stride*3 + sub betad, tmp2d ; beta -= alpha*3 + mov myd, r7m + call .h + psrld m1, m0, 16 + call .h + pblendw m1, m0, 0xaa ; 01 + psrld m2, m0, 16 + call .h + pblendw m2, m0, 0xaa ; 12 + psrld m3, m0, 16 + call .h + pblendw m3, m0, 0xaa ; 23 + psrld m4, m0, 16 + call .h + pblendw m4, m0, 0xaa ; 34 + psrld m5, m0, 16 + call .h + pblendw m5, m0, 0xaa ; 45 + psrld m6, m0, 16 + call .h + pblendw m6, m0, 0xaa ; 56 + movsx deltad, word [abcdq+2*2] + movsx gammad, word [abcdq+2*3] + add myd, 512+(64<<10) + mov r4d, 4 + lea tmp1d, [deltaq*3] + sub gammad, tmp1d ; gamma -= delta*3 +.main2: + call .h + psrld m7, m6, 16 + pblendw m7, m0, 0xaa ; 67 + WARP_V 7, 1, 3, 5, 7 + call .h + psrld m10, m5, 16 + pblendw m10, m0, 0xaa ; 78 + WARP_V 0, 2, 4, 6, 10 + ret +ALIGN function_align +.h: + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + movu xm10, [srcq-6] + vinserti128 m10, [srcq+2], 1 + shr mxd, 10 ; 0 + shr tmp1d, 10 ; 4 + movq xm0, [filterq+mxq *8] + vinserti128 m0, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+alphaq*1] + movu xm8, [srcq-4] + vinserti128 m8, [srcq+4], 1 + shr tmp2d, 10 ; 1 + shr tmp1d, 10 ; 5 + movq xm9, [filterq+tmp2q*8] + vinserti128 m9, [filterq+tmp1q*8], 1 + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + shr mxd, 10 ; 2 + shr tmp1d, 10 ; 6 + punpcklbw m0, m11, m0 + pmaddwd m0, m10 + movu xm10, [srcq-2] + vinserti128 m10, [srcq+6], 1 + punpcklbw m9, m11, m9 + pmaddwd m9, m8 + movq xm8, [filterq+mxq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + phaddd m0, m9 ; 0 1 4 5 + movu xm9, [srcq+0] + vinserti128 m9, [srcq+8], 1 + shr tmp2d, 10 ; 3 + shr tmp1d, 10 ; 7 + punpcklbw m8, m11, m8 + pmaddwd m8, m10 + movq xm10, [filterq+tmp2q*8] + vinserti128 m10, [filterq+tmp1q*8], 1 + punpcklbw m10, m11, m10 + pmaddwd m9, m10 + add srcq, ssq + phaddd m8, m9 ; 2 3 6 7 + phaddd m0, m8 ; 0 1 2 3 4 5 6 7 + vpsllvd m0, m13 + paddd m0, m12 ; rounded 14-bit result in upper 16 bits of dword + ret + +%macro BIDIR_FN 0 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq ], xm0 + movhps [dstq+strideq*1], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + movq [dstq ], xm1 + movhps [dstq+strideq*1], xm1 + vextracti128 xm1, m1, 1 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 + cmp hd, 8 + je .ret + lea dstq, [dstq+strideq*4] + movq [dstq ], xm2 + movhps [dstq+strideq*1], xm2 + vextracti128 xm2, m2, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + lea dstq, [dstq+strideq*4] + movq [dstq ], xm3 + movhps [dstq+strideq*1], xm3 + vextracti128 xm3, m3, 1 + movq [dstq+strideq*2], xm3 + movhps [dstq+stride3q ], xm3 +.ret: + RET +.w8: + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + cmp hd, 4 + jne .w8_loop_start + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 +.w8_loop_start: + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm2 + vextracti128 [dstq+strideq*1], m2, 1 + mova [dstq+strideq*2], xm3 + vextracti128 [dstq+stride3q ], m3, 1 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] +.w16: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] +.w32: + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + call .main + mova [dstq+32*4], m0 + mova [dstq+32*5], m1 + mova [dstq+32*6], m2 + mova [dstq+32*7], m3 + dec hd + jg .w128_loop + RET +%endmacro + +%if WIN64 +DECLARE_REG_TMP 5 +%else +DECLARE_REG_TMP 7 +%endif + +cglobal avg_16bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg_avx2_table + lea r6, [avg_avx2_table] + tzcnt wd, wm + mov t0d, r6m ; pixel_max + movsxd wq, [r6+wq*4] + shr t0d, 11 + vpbroadcastd m4, [base+bidir_rnd+t0*4] + vpbroadcastd m5, [base+bidir_mul+t0*4] + movifnidn hd, hm + add wq, r6 + BIDIR_FN +ALIGN function_align +.main: + mova m0, [tmp1q+32*0] + paddsw m0, [tmp2q+32*0] + mova m1, [tmp1q+32*1] + paddsw m1, [tmp2q+32*1] + mova m2, [tmp1q+32*2] + paddsw m2, [tmp2q+32*2] + mova m3, [tmp1q+32*3] + paddsw m3, [tmp2q+32*3] + add tmp1q, 32*4 + add tmp2q, 32*4 + pmaxsw m0, m4 + pmaxsw m1, m4 + pmaxsw m2, m4 + pmaxsw m3, m4 + psubsw m0, m4 + psubsw m1, m4 + psubsw m2, m4 + psubsw m3, m4 + pmulhw m0, m5 + pmulhw m1, m5 + pmulhw m2, m5 + pmulhw m3, m5 + ret + +cglobal w_avg_16bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, stride3 + lea r6, [w_avg_avx2_table] + tzcnt wd, wm + mov t0d, r6m ; weight + vpbroadcastw m8, r7m ; pixel_max + vpbroadcastd m7, [r6-w_avg_avx2_table+pd_65538] + movsxd wq, [r6+wq*4] + paddw m7, m8 + add wq, r6 + lea r6d, [t0-16] + shl t0d, 16 + sub t0d, r6d ; 16-weight, weight + pslld m7, 7 + rorx r6d, t0d, 30 ; << 2 + test dword r7m, 0x800 + cmovz r6d, t0d + movifnidn hd, hm + movd xm6, r6d + vpbroadcastd m6, xm6 + BIDIR_FN +ALIGN function_align +.main: + mova m4, [tmp1q+32*0] + mova m0, [tmp2q+32*0] + punpckhwd m5, m0, m4 + punpcklwd m0, m4 + mova m4, [tmp1q+32*1] + mova m1, [tmp2q+32*1] + pmaddwd m5, m6 + pmaddwd m0, m6 + paddd m5, m7 + paddd m0, m7 + psrad m5, 8 + psrad m0, 8 + packusdw m0, m5 + punpckhwd m5, m1, m4 + punpcklwd m1, m4 + mova m4, [tmp1q+32*2] + mova m2, [tmp2q+32*2] + pmaddwd m5, m6 + pmaddwd m1, m6 + paddd m5, m7 + paddd m1, m7 + psrad m5, 8 + psrad m1, 8 + packusdw m1, m5 + punpckhwd m5, m2, m4 + punpcklwd m2, m4 + mova m4, [tmp1q+32*3] + mova m3, [tmp2q+32*3] + add tmp1q, 32*4 + add tmp2q, 32*4 + pmaddwd m5, m6 + pmaddwd m2, m6 + paddd m5, m7 + paddd m2, m7 + psrad m5, 8 + psrad m2, 8 + packusdw m2, m5 + punpckhwd m5, m3, m4 + punpcklwd m3, m4 + pmaddwd m5, m6 + pmaddwd m3, m6 + paddd m5, m7 + paddd m3, m7 + psrad m5, 8 + psrad m3, 8 + packusdw m3, m5 + pminsw m0, m8 + pminsw m1, m8 + pminsw m2, m8 + pminsw m3, m8 + ret + +cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask_avx2_table + lea r7, [mask_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m8, [base+pw_64] + vpbroadcastd m9, [base+bidir_rnd+r6*4] + vpbroadcastd m10, [base+bidir_mul+r6*4] + mov maskq, maskmp + add wq, r7 + BIDIR_FN +ALIGN function_align +.main: +%macro MASK 1 + pmovzxbw m5, [maskq+16*%1] + mova m%1, [tmp1q+32*%1] + mova m6, [tmp2q+32*%1] + punpckhwd m4, m%1, m6 + punpcklwd m%1, m6 + psubw m7, m8, m5 + punpckhwd m6, m5, m7 ; m, 64-m + punpcklwd m5, m7 + pmaddwd m4, m6 ; tmp1 * m + tmp2 * (64-m) + pmaddwd m%1, m5 + psrad m4, 5 + psrad m%1, 5 + packssdw m%1, m4 + pmaxsw m%1, m9 + psubsw m%1, m9 + pmulhw m%1, m10 +%endmacro + MASK 0 + MASK 1 + MASK 2 + MASK 3 + add maskq, 16*4 + add tmp1q, 32*4 + add tmp2q, 32*4 + ret + +cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx2_table + lea r7, [w_mask_420_avx2_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movd xm0, r7m ; sign + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + vpbroadcastd m11, [base+pw_64] + vpbroadcastd m12, [base+bidir_rnd+r6*4] + vpbroadcastd m13, [base+bidir_mul+r6*4] + movd xm14, [base+pw_2] + mov maskq, maskmp + psubw xm14, xm0 + vpbroadcastw m14, xm14 + add wq, r7 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + phaddd m4, m5 + paddw m4, m14 + psrlw m4, 2 + packuswb m4, m4 + vextracti128 xm5, m4, 1 + punpcklwd xm4, xm5 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + mova [maskq], xm4 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti128 xm1, m1, 1 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti128 xm2, m2, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm3 + movhps [dstq+strideq*1], xm3 + vextracti128 xm3, m3, 1 + movq [dstq+strideq*2], xm3 + movhps [dstq+stride3q ], xm3 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w8: + vperm2i128 m6, m4, m5, 0x21 + vpblendd m4, m5, 0xf0 + paddw m4, m14 + paddw m4, m6 + psrlw m4, 2 + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + mova [maskq], xm4 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm2 + vextracti128 [dstq+strideq*1], m2, 1 + mova [dstq+strideq*2], xm3 + vextracti128 [dstq+stride3q ], m3, 1 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w16: + punpcklqdq m6, m4, m5 + punpckhqdq m4, m5 + paddw m6, m14 + paddw m4, m6 + psrlw m4, 2 + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + pshufd xm4, xm4, q3120 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + mova [maskq], xm4 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w32: + paddw m4, m14 + paddw m4, m5 + psrlw m15, m4, 2 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + call .main + mova m6, [deint_shuf] + paddw m4, m14 + paddw m4, m5 + psrlw m4, 2 + packuswb m15, m4 + vpermd m4, m6, m15 + mova [dstq+strideq*2+32*0], m0 + mova [dstq+strideq*2+32*1], m1 + mova [dstq+stride3q +32*0], m2 + mova [dstq+stride3q +32*1], m3 + mova [maskq], m4 + sub hd, 4 + jg .w32_loop + RET +.w64_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w64: + paddw m4, m14 + paddw m15, m14, m5 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*0+32*2], m2 + mova [dstq+strideq*0+32*3], m3 + mova [maskq], m4 ; no available registers + call .main + paddw m4, [maskq] + mova m6, [deint_shuf] + paddw m5, m15 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 ; 0 2 4 6 1 3 5 7 + vpermd m4, m6, m4 + mova [dstq+strideq*1+32*0], m0 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*1+32*2], m2 + mova [dstq+strideq*1+32*3], m3 + mova [maskq], m4 + sub hd, 2 + jg .w64_loop + RET +.w128_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 64 +.w128: + paddw m4, m14 + paddw m5, m14 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*0+32*2], m2 + mova [dstq+strideq*0+32*3], m3 + mova [maskq+32*0], m4 + mova [dstq+strideq], m5 + call .main + paddw m4, m14 + paddw m15, m14, m5 + mova [dstq+strideq*0+32*4], m0 + mova [dstq+strideq*0+32*5], m1 + mova [dstq+strideq*0+32*6], m2 + mova [dstq+strideq*0+32*7], m3 + mova [maskq+32*1], m4 + call .main + paddw m4, [maskq+32*0] + paddw m5, [dstq+strideq] + mova m6, [deint_shuf] + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m6, m4 + mova [dstq+strideq*1+32*0], m0 + mova [dstq+strideq*1+32*1], m1 + mova [dstq+strideq*1+32*2], m2 + mova [dstq+strideq*1+32*3], m3 + mova [maskq+32*0], m4 + call .main + paddw m4, [maskq+32*1] + mova m6, [deint_shuf] + paddw m5, m15 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m6, m4 + mova [dstq+strideq*1+32*4], m0 + mova [dstq+strideq*1+32*5], m1 + mova [dstq+strideq*1+32*6], m2 + mova [dstq+strideq*1+32*7], m3 + mova [maskq+32*1], m4 + sub hd, 2 + jg .w128_loop + RET +ALIGN function_align +.main: +%macro W_MASK 2-6 11, 12, 13 ; dst/src1, mask/src2, pw_64, rnd, mul + mova m%1, [tmp1q+32*%1] + mova m%2, [tmp2q+32*%1] + punpcklwd m8, m%2, m%1 + punpckhwd m9, m%2, m%1 + psubsw m%1, m%2 + pabsw m%1, m%1 + psubusw m7, m10, m%1 + psrlw m7, 10 ; 64-m + psubw m%2, m%3, m7 ; m + punpcklwd m%1, m7, m%2 + punpckhwd m7, m%2 + pmaddwd m%1, m8 + pmaddwd m7, m9 + psrad m%1, 5 + psrad m7, 5 + packssdw m%1, m7 + pmaxsw m%1, m%4 + psubsw m%1, m%4 + pmulhw m%1, m%5 +%endmacro + W_MASK 0, 4 + W_MASK 1, 5 + phaddw m4, m5 + W_MASK 2, 5 + W_MASK 3, 6 + phaddw m5, m6 + add tmp1q, 32*4 + add tmp2q, 32*4 + ret + +cglobal w_mask_422_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx2_table + lea r7, [w_mask_422_avx2_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + vpbroadcastb m14, r7m ; sign + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m10, [base+pw_27615] + vpbroadcastd m11, [base+pw_64] + vpbroadcastd m12, [base+bidir_rnd+r6*4] + vpbroadcastd m13, [base+bidir_mul+r6*4] + mova m15, [base+deint_shuf] + mov maskq, maskmp + add wq, r7 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti128 xm1, m1, 1 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti128 xm2, m2, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm3 + movhps [dstq+strideq*1], xm3 + vextracti128 xm3, m3, 1 + movq [dstq+strideq*2], xm3 + movhps [dstq+stride3q ], xm3 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] +.w8: + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm2 + vextracti128 [dstq+strideq*1], m2, 1 + mova [dstq+strideq*2], xm3 + vextracti128 [dstq+stride3q ], m3, 1 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] +.w16: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] +.w32: + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + call .main + mova [dstq+32*4], m0 + mova [dstq+32*5], m1 + mova [dstq+32*6], m2 + mova [dstq+32*7], m3 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + W_MASK 0, 4 + W_MASK 1, 5 + phaddw m4, m5 + W_MASK 2, 5 + W_MASK 3, 6 + phaddw m5, m6 + add tmp1q, 32*4 + add tmp2q, 32*4 + packuswb m4, m5 + pxor m5, m5 + psubb m4, m14 + pavgb m4, m5 + vpermd m4, m15, m4 + mova [maskq], m4 + add maskq, 32 + ret + +cglobal w_mask_444_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx2_table + lea r7, [w_mask_444_avx2_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m10, [base+pw_27615] + vpbroadcastd m4, [base+pw_64] + vpbroadcastd m5, [base+bidir_rnd+r6*4] + vpbroadcastd m6, [base+bidir_mul+r6*4] + mov maskq, maskmp + add wq, r7 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti128 xm1, m1, 1 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 + je .w4_end + call .main + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti128 xm0, m0, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti128 xm1, m1, 1 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] +.w8: + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + sub hd, 4 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + call .main + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + call .main + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + call .main + mova [dstq+32*4], m0 + mova [dstq+32*5], m1 + call .main + mova [dstq+32*6], m0 + mova [dstq+32*7], m1 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + W_MASK 0, 2, 4, 5, 6 + W_MASK 1, 3, 4, 5, 6 + packuswb m2, m3 + vpermq m2, m2, q3120 + add tmp1q, 32*2 + add tmp2q, 32*2 + mova [maskq], m2 + add maskq, 32 + ret + +; (a * (64 - m) + b * m + 32) >> 6 +; = (((b - a) * m + 32) >> 6) + a +; = (((b - a) * (m << 9) + 16384) >> 15) + a +; except m << 9 overflows int16_t when m == 64 (which is possible), +; but if we negate m it works out (-64 << 9 == -32768). +; = (((a - b) * (m * -512) + 16384) >> 15) + a +cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_avx2_table + lea r6, [blend_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r6+wq*4] + movifnidn maskq, maskmp + vpbroadcastd m6, [base+pw_m512] + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + pmovzxbw m3, [maskq] + movq xm0, [dstq+dsq*0] + movhps xm0, [dstq+dsq*1] + vpbroadcastq m1, [dstq+dsq*2] + vpbroadcastq m2, [dstq+r6 ] + vpblendd m0, m1, 0x30 + vpblendd m0, m2, 0xc0 + psubw m1, m0, [tmpq] + add maskq, 16 + add tmpq, 32 + pmullw m3, m6 + pmulhrsw m1, m3 + paddw m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + movq [dstq+dsq*2], xm1 + movhps [dstq+r6 ], xm1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w4 + RET +.w8: + pmovzxbw m4, [maskq+16*0] + pmovzxbw m5, [maskq+16*1] + mova xm0, [dstq+dsq*0] + vinserti128 m0, [dstq+dsq*1], 1 + mova xm1, [dstq+dsq*2] + vinserti128 m1, [dstq+r6 ], 1 + psubw m2, m0, [tmpq+32*0] + psubw m3, m1, [tmpq+32*1] + add maskq, 16*2 + add tmpq, 32*2 + pmullw m4, m6 + pmullw m5, m6 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + mova [dstq+dsq*2], xm1 + vextracti128 [dstq+r6 ], m1, 1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w8 + RET +.w16: + pmovzxbw m4, [maskq+16*0] + pmovzxbw m5, [maskq+16*1] + mova m0, [dstq+dsq*0] + psubw m2, m0, [tmpq+ 32*0] + mova m1, [dstq+dsq*1] + psubw m3, m1, [tmpq+ 32*1] + add maskq, 16*2 + add tmpq, 32*2 + pmullw m4, m6 + pmullw m5, m6 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16 + RET +.w32: + pmovzxbw m4, [maskq+16*0] + pmovzxbw m5, [maskq+16*1] + mova m0, [dstq+32*0] + psubw m2, m0, [tmpq+32*0] + mova m1, [dstq+32*1] + psubw m3, m1, [tmpq+32*1] + add maskq, 16*2 + add tmpq, 32*2 + pmullw m4, m6 + pmullw m5, m6 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + add dstq, dsq + dec hd + jg .w32 + RET + +INIT_XMM avx2 +cglobal blend_v_16bpc, 3, 6, 6, dst, ds, tmp, w, h +%define base r5-blend_v_avx2_table + lea r5, [blend_v_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + jmp wq +.w2: + vpbroadcastd m2, [base+obmc_masks_avx2+2*2] +.w2_loop: + movd m0, [dstq+dsq*0] + pinsrd m0, [dstq+dsq*1], 1 + movq m1, [tmpq] + add tmpq, 4*2 + psubw m1, m0, m1 + pmulhrsw m1, m2 + paddw m0, m1 + movd [dstq+dsq*0], m0 + pextrd [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w2_loop + RET +.w4: + vpbroadcastq m2, [base+obmc_masks_avx2+4*2] +.w4_loop: + movq m0, [dstq+dsq*0] + movhps m0, [dstq+dsq*1] + psubw m1, m0, [tmpq] + add tmpq, 8*2 + pmulhrsw m1, m2 + paddw m0, m1 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +INIT_YMM avx2 +.w8: + vbroadcasti128 m2, [base+obmc_masks_avx2+8*2] +.w8_loop: + mova xm0, [dstq+dsq*0] + vinserti128 m0, [dstq+dsq*1], 1 + psubw m1, m0, [tmpq] + add tmpq, 16*2 + pmulhrsw m1, m2 + paddw m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + mova m4, [base+obmc_masks_avx2+16*2] +.w16_loop: + mova m0, [dstq+dsq*0] + psubw m2, m0, [tmpq+ 32*0] + mova m1, [dstq+dsq*1] + psubw m3, m1, [tmpq+ 32*1] + add tmpq, 32*2 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16_loop + RET +.w32: +%if WIN64 + movaps [rsp+ 8], xmm6 + movaps [rsp+24], xmm7 +%endif + mova m6, [base+obmc_masks_avx2+32*2] + vbroadcasti128 m7, [base+obmc_masks_avx2+32*3] +.w32_loop: + mova m0, [dstq+dsq*0+32*0] + psubw m3, m0, [tmpq +32*0] + mova xm2, [dstq+dsq*0+32*1] + mova xm5, [tmpq +32*1] + mova m1, [dstq+dsq*1+32*0] + psubw m4, m1, [tmpq +32*2] + vinserti128 m2, [dstq+dsq*1+32*1], 1 + vinserti128 m5, [tmpq +32*3], 1 + add tmpq, 32*4 + psubw m5, m2, m5 + pmulhrsw m3, m6 + pmulhrsw m4, m6 + pmulhrsw m5, m7 + paddw m0, m3 + paddw m1, m4 + paddw m2, m5 + mova [dstq+dsq*0+32*0], m0 + mova [dstq+dsq*1+32*0], m1 + mova [dstq+dsq*0+32*1], xm2 + vextracti128 [dstq+dsq*1+32*1], m2, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w32_loop +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + RET + +%macro BLEND_H_ROW 2-3 0; dst_off, tmp_off, inc_tmp + mova m0, [dstq+32*(%1+0)] + psubw m2, m0, [tmpq+32*(%2+0)] + mova m1, [dstq+32*(%1+1)] + psubw m3, m1, [tmpq+32*(%2+1)] +%if %3 + add tmpq, 32*%3 +%endif + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + mova [dstq+32*(%1+0)], m0 + mova [dstq+32*(%1+1)], m1 +%endmacro + +INIT_XMM avx2 +cglobal blend_h_16bpc, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_h_avx2_table + lea r5, [blend_h_avx2_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + lea maskq, [base+obmc_masks_avx2+hq*2] + lea hd, [hq*3] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd m0, [dstq+dsq*0] + pinsrd m0, [dstq+dsq*1], 1 + movd m2, [maskq+hq*2] + movq m1, [tmpq] + add tmpq, 4*2 + punpcklwd m2, m2 + psubw m1, m0, m1 + pmulhrsw m1, m2 + paddw m0, m1 + movd [dstq+dsq*0], m0 + pextrd [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +.w4: + mova m3, [blend_shuf] +.w4_loop: + movq m0, [dstq+dsq*0] + movhps m0, [dstq+dsq*1] + movd m2, [maskq+hq*2] + psubw m1, m0, [tmpq] + add tmpq, 8*2 + pshufb m2, m3 + pmulhrsw m1, m2 + paddw m0, m1 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +INIT_YMM avx2 +.w8: + vbroadcasti128 m3, [blend_shuf] + shufpd m3, m3, 0x0c +.w8_loop: + mova xm0, [dstq+dsq*0] + vinserti128 m0, [dstq+dsq*1], 1 + vpbroadcastd m2, [maskq+hq*2] + psubw m1, m0, [tmpq] + add tmpq, 16*2 + pshufb m2, m3 + pmulhrsw m1, m2 + paddw m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop + RET +.w16: + vpbroadcastw m4, [maskq+hq*2] + vpbroadcastw m5, [maskq+hq*2+2] + mova m0, [dstq+dsq*0] + psubw m2, m0, [tmpq+ 32*0] + mova m1, [dstq+dsq*1] + psubw m3, m1, [tmpq+ 32*1] + add tmpq, 32*2 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w16 + RET +.w32: + vpbroadcastw m4, [maskq+hq*2] + BLEND_H_ROW 0, 0, 2 + add dstq, dsq + inc hq + jl .w32 + RET +.w64: + vpbroadcastw m4, [maskq+hq*2] + BLEND_H_ROW 0, 0 + BLEND_H_ROW 2, 2, 4 + add dstq, dsq + inc hq + jl .w64 + RET +.w128: + vpbroadcastw m4, [maskq+hq*2] + BLEND_H_ROW 0, 0 + BLEND_H_ROW 2, 2, 8 + BLEND_H_ROW 4, -4 + BLEND_H_ROW 6, -2 + add dstq, dsq + inc hq + jl .w128 + RET + +cglobal emu_edge_16bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ + bottomext, rightext + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor r12d, r12d + lea r10, [ihq-1] + cmp yq, ihq + cmovs r10, yq + test yq, yq + cmovs r10, r12 + imul r10, sstrideq + add srcq, r10 + + ; ref += iclip(x, 0, iw - 1) + lea r10, [iwq-1] + cmp xq, iwq + cmovs r10, xq + test xq, xq + cmovs r10, r12 + lea srcq, [srcq+r10*2] + + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) + lea bottomextq, [yq+bhq] + sub bottomextq, ihq + lea r3, [bhq-1] + cmovs bottomextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, r12 + cmp bottomextq, bhq + cmovns bottomextq, r3 + cmp topextq, bhq + cmovg topextq, r3 + + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + lea rightextq, [xq+bwq] + sub rightextq, iwq + lea r2, [bwq-1] + cmovs rightextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, r12 + cmp rightextq, bwq + cmovns rightextq, r2 + cmp leftextq, bwq + cmovns leftextq, r2 + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \ + dst, dstride, src, sstride, bottomext, rightext + + ; center_h = bh - top_ext - bottom_ext + lea r3, [bottomextq+topextq] + sub centerhq, r3 + + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq + imul r2, dstrideq + add dstq, r2 + mov r9m, dstq + + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq + lea r3, [rightextq+leftextq] + sub centerwq, r3 + +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix +.v_loop_%3: +%if %1 + ; left extension + xor r3, r3 + vpbroadcastw m0, [srcq] +.left_loop_%3: + mova [dstq+r3*2], m0 + add r3, 16 + cmp r3, leftextq + jl .left_loop_%3 + + ; body + lea r12, [dstq+leftextq*2] +%endif + xor r3, r3 +.body_loop_%3: + movu m0, [srcq+r3*2] +%if %1 + movu [r12+r3*2], m0 +%else + movu [dstq+r3*2], m0 +%endif + add r3, 16 + cmp r3, centerwq + jl .body_loop_%3 + +%if %2 + ; right extension +%if %1 + lea r12, [r12+centerwq*2] +%else + lea r12, [dstq+centerwq*2] +%endif + xor r3, r3 + vpbroadcastw m0, [srcq+centerwq*2-2] +.right_loop_%3: + movu [r12+r3*2], m0 + add r3, 16 + cmp r3, rightextq + jl .right_loop_%3 + +%endif + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 +%endmacro + + test leftextq, leftextq + jnz .need_left_ext + test rightextq, rightextq + jnz .need_right_ext + v_loop 0, 0, 0 + jmp .body_done + +.need_left_ext: + test rightextq, rightextq + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: + ; bottom edge extension + test bottomextq, bottomextq + jz .top + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 +.bottom_x_loop: + mova m0, [srcq+r1*2] + lea r3, [dstq+r1*2] + mov r4, bottomextq +.bottom_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .bottom_y_loop + add r1, 16 + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end + mov srcq, r9m + mov dstq, dstm + xor r1, r1 +.top_x_loop: + mova m0, [srcq+r1*2] + lea r3, [dstq+r1*2] + mov r4, topextq +.top_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .top_y_loop + add r1, 16 + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +cglobal resize_16bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax + LEA r7, $$ +%define base r7-$$ + vpbroadcastd m3, [base+pd_64] + vpbroadcastw xm7, pxmaxm + pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7] + pslld m5, 3 ; dx*8 + pslld m6, 14 + paddd m8, m2 ; mx+[0..7]*dx +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx +.loop_x: + vpbroadcastd m10, [base+pd_63] + pxor m2, m2 + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + pand m9, m10 ; filter offset (masked) + ; load source pixels + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vextracti128 xm0, m0, 1 + movu xm10, [srcq+r8*2] + movu xm11, [srcq+r9*2] + movu xm12, [srcq+r10*2] + movu xm13, [srcq+r11*2] + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vinserti128 m10, [srcq+r8*2], 1 + vinserti128 m11, [srcq+r9*2], 1 + vinserti128 m12, [srcq+r10*2], 1 + vinserti128 m13, [srcq+r11*2], 1 + ptest m1, m1 + jz .filter + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vextracti128 xm1, m1, 1 + movu xm14, [base+resize_shuf+8+r8*2] + movu xm15, [base+resize_shuf+8+r9*2] + movu xm0, [base+resize_shuf+8+r10*2] + movu xm2, [base+resize_shuf+8+r11*2] + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vinserti128 m14, [base+resize_shuf+8+r8*2], 1 + vinserti128 m15, [base+resize_shuf+8+r9*2], 1 + vinserti128 m0, [base+resize_shuf+8+r10*2], 1 + vinserti128 m2, [base+resize_shuf+8+r11*2], 1 + pshufb m10, m14 + pshufb m11, m15 + pshufb m12, m0 + pshufb m13, m2 +.filter: + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vextracti128 xm9, m9, 1 + movq xm14, [base+resize_filter+r8*8] + movq xm15, [base+resize_filter+r9*8] + movq xm0, [base+resize_filter+r10*8] + movq xm2, [base+resize_filter+r11*8] + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + movhps xm14, [base+resize_filter+r8*8] + movhps xm15, [base+resize_filter+r9*8] + movhps xm0, [base+resize_filter+r10*8] + movhps xm2, [base+resize_filter+r11*8] + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pmovsxbw m0, xm0 + pmovsxbw m2, xm2 + pmaddwd m10, m14 + pmaddwd m11, m15 + pmaddwd m12, m0 + pmaddwd m13, m2 + phaddd m10, m11 + phaddd m12, m13 + phaddd m10, m12 + psubd m10, m3, m10 + psrad m10, 7 + vextracti128 xm0, m10, 1 + packusdw xm10, xm0 + pminsw xm10, xm7 + mova [dstq+xq*2], xm10 + paddd m4, m5 + add xd, 8 + cmp xd, dst_wd + jl .loop_x + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc16_avx512.asm b/third_party/dav1d/src/x86/mc16_avx512.asm new file mode 100644 index 0000000000..585ba53e08 --- /dev/null +++ b/third_party/dav1d/src/x86/mc16_avx512.asm @@ -0,0 +1,4858 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 + db 32, 33, 34, 35, 34, 35, 36, 37, 36, 37, 38, 39, 38, 39, 40, 41 +spel_h_shufC: db 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15, 16, 17 + db 40, 41, 42, 43, 42, 43, 44, 45, 44, 45, 46, 47, 46, 47, 48, 49 + db 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23, 24, 25 + db 48, 49, 50, 51, 50, 51, 52, 53, 52, 53, 54, 55, 54, 55, 56, 57 +spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 + db 36, 37, 38, 39, 38, 39, 40, 41, 40, 41, 42, 43, 42, 43, 44, 45 +spel_h_shufD: db 12, 13, 14, 15, 14, 15, 16, 17, 16, 17, 18, 19, 18, 19, 20, 21 + db 44, 45, 46, 47, 46, 47, 48, 49, 48, 49, 50, 51, 50, 51, 52, 53 + db 20, 21, 22, 23, 22, 23, 24, 25, 24, 25, 26, 27, 26, 27, 28, 29 + db 52, 53, 54, 55, 54, 55, 56, 57, 56, 57, 58, 59, 58, 59, 60, 61 +spel_v_shuf8: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 + db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39 + db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 + db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47 +spel_v_shuf16: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 + db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 + db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 + db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 +prep_endA: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30 + db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62 + db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94 + db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126 +prep_endB: db 1, 2, 5, 6, 9, 10, 13, 14, 33, 34, 37, 38, 41, 42, 45, 46 + db 17, 18, 21, 22, 25, 26, 29, 30, 49, 50, 53, 54, 57, 58, 61, 62 + db 65, 66, 69, 70, 73, 74, 77, 78, 97, 98,101,102,105,106,109,110 + db 81, 82, 85, 86, 89, 90, 93, 94,113,114,117,118,121,122,125,126 +prep_endC: db 1, 2, 5, 6, 9, 10, 13, 14, 65, 66, 69, 70, 73, 74, 77, 78 + db 17, 18, 21, 22, 25, 26, 29, 30, 81, 82, 85, 86, 89, 90, 93, 94 + db 33, 34, 37, 38, 41, 42, 45, 46, 97, 98,101,102,105,106,109,110 + db 49, 50, 53, 54, 57, 58, 61, 62,113,114,117,118,121,122,125,126 +spel_shuf4a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30 + db 17, 18, 33, 34, 21, 22, 37, 38, 25, 26, 41, 42, 29, 30, 45, 46 +spel_shuf4b: db 18, 19, 33, 34, 22, 23, 37, 38, 26, 27, 41, 42, 30, 31, 45, 46 + db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62 +spel_shuf8a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30 + db 17, 18, 65, 66, 21, 22, 69, 70, 25, 26, 73, 74, 29, 30, 77, 78 + db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62 + db 49, 50, 97, 98, 53, 54,101,102, 57, 58,105,106, 61, 62,109,110 +spel_shuf8b: db 18, 19, 65, 66, 22, 23, 69, 70, 26, 27, 73, 74, 30, 31, 77, 78 + db 65, 66, 81, 82, 69, 70, 85, 86, 73, 74, 89, 90, 77, 78, 93, 94 + db 50, 51, 97, 98, 54, 55,101,102, 58, 59,105,106, 62, 63,109,110 + db 97, 98,113,114,101,102,117,118,105,106,121,122,109,110,125,126 +spel_shuf16: db 1, 2, 33, 34, 5, 6, 37, 38, 9, 10, 41, 42, 13, 14, 45, 46 + db 17, 18, 49, 50, 21, 22, 53, 54, 25, 26, 57, 58, 29, 30, 61, 62 + db 65, 66, 97, 98, 69, 70,101,102, 73, 74,105,106, 77, 78,109,110 + db 81, 82,113,114, 85, 86,117,118, 89, 90,121,122, 93, 94,125,126 +spel_shuf32: db 1, 2, 65, 66, 5, 6, 69, 70, 9, 10, 73, 74, 13, 14, 77, 78 + db 17, 18, 81, 82, 21, 22, 85, 86, 25, 26, 89, 90, 29, 30, 93, 94 + db 33, 34, 97, 98, 37, 38,101,102, 41, 42,105,106, 45, 46,109,110 + db 49, 50,113,114, 53, 54,117,118, 57, 58,121,122, 61, 62,125,126 +spel_h_shuf2b: db 1, 2, 17, 18, 5, 6, 21, 22, 17, 18, 33, 34, 21, 22, 37, 38 + db 33, 34, 49, 50, 37, 38, 53, 54, 49, 50, 9, 10, 53, 54, 13, 14 + db 9, 10, 25, 26, 13, 14, 29, 30, 25, 26, 41, 42, 29, 30, 45, 46 +spel_shuf2: db 10, 11, 17, 18, 14, 15, 21, 22, 17, 18, 25, 26, 21, 22, 29, 30 +spel_h_shuf2a: db 0, 1, 2, 3, 2, 3, 4, 5, 16, 17, 18, 19, 18, 19, 20, 21 + db 4, 5, 6, 7, 6, 7, 8, 9, 20, 21, 22, 23, 22, 23, 24, 25 +w_mask_end42x: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +w_mask_end444: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 + db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94 + db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126 +w_mask_shuf4: db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 + db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 + db 64, 66, 72, 74, 68, 70, 76, 78, 80, 82, 88, 90, 84, 86, 92, 94 + db 96, 98,104,106,100,102,108,110,112,114,120,122,116,118,124,126 +w_mask_shuf8: db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 + db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 + db 64, 66, 80, 82, 68, 70, 84, 86, 72, 74, 88, 90, 76, 78, 92, 94 + db 96, 98,112,114,100,102,116,118,104,106,120,122,108,110,124,126 +w_mask_shuf16: db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 + db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 + db 64, 66, 96, 98, 68, 70,100,102, 72, 74,104,106, 76, 78,108,110 + db 80, 82,112,114, 84, 86,116,118, 88, 90,120,122, 92, 94,124,126 +warp8x8_permA: db 0, 1, 2, 3, 32, 33, 34, 35, 2, 3, 4, 5, 34, 35, 36, 37 + db 4, 5, 6, 7, 36, 37, 38, 39, 6, 7, 8, 9, 38, 39, 40, 41 + db 8, 9, 10, 11, 40, 41, 42, 43, 10, 11, 12, 13, 42, 43, 44, 45 + db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49 +warp8x8_permB: db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49 + db 16, 17, 18, 19, 48, 49, 50, 51, 18, 19, 20, 21, 50, 51, 52, 53 + db 20, 21, 22, 23, 52, 53, 54, 55, 22, 23, 24, 25, 54, 55, 56, 57 + db 24, 25, 26, 27, 56, 57, 58, 59, 26, 27, 28, 29, 58, 59, 60, 61 +warp8x8_end: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53 + db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55 + db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61 + db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63 +deint_q_shuf: ;dq 0, 2, 4, 6, 1, 3, 5, 7 +pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7 + dd 1 +pw_2048: times 2 dw 2048 + dd 3 +pw_8192: times 2 dw 8192 +avg_shift: dw 5, 5, 3, 3 +pw_27615: times 2 dw 27615 +pw_32766: times 2 dw 32766 +warp8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13 +warp8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15 +warp_shift_h: db 11, 19, 11, 19, 43, 51, 43, 51, 13, 21, 13, 21, 45, 53, 45, 53 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +resize_permA: dd 0, 4, 8, 12, 1, 5, 9, 13, 16, 20, 24, 28, 17, 21, 25, 29 +resize_permB: dd 2, 6, 10, 14, 3, 7, 11, 15, 18, 22, 26, 30, 19, 23, 27, 31 +resize_permC: dq 0, 1, 4, 5, 8, 9, 12, 13 +resize_permD: dq 2, 3, 6, 7, 10, 11, 14, 15 +resize_permE: dq 0, 2, 4, 6 +resize_shufA: db -1, 0, -1, 1, -1, 4, -1, 5, -1, 8, -1, 9, -1, 12, -1, 13 +resize_shufB: db -1, 2, -1, 3, -1, 6, -1, 7, -1, 10, -1, 11, -1, 14, -1, 15 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 + +prep_hv_shift: dq 6, 4 +put_bilin_h_rnd: dw 8, 8, 10, 10 +prep_mul: dw 16, 16, 4, 4 +put_8tap_h_rnd: dd 34, 40 +prep_8tap_rnd: dd 128 - (8192 << 8) +warp_8x8_rnd_h: dd 512, 2048 +warp_8x8_rnd_v: dd 262144, 65536 +warp_8x8t_rnd_v: dd 16384 - (8192 << 15) +avg_round: dw -16400, -16400, -16388, -16388 +w_avg_round: dd 128 + (8192 << 4), 32 + (8192 << 4) +mask_round: dd 512 + (8192 << 6), 128 + (8192 << 6) +w_mask_round: dd 128, 64 +bidir_shift: dw 6, 6, 4, 4 + +pb_64: times 4 db 64 +pw_m512: times 2 dw -512 +pw_2: times 2 dw 2 +pw_64: times 2 dw 64 +pd_32: dd 32 +pd_63: dd 63 +pd_128: dd 128 +pd_640: dd 640 +pd_2176: dd 2176 +pd_16384: dd 16384 +pd_0_4: dd 0, 4 + +%define pw_16 prep_mul +%define pd_512 warp_8x8_rnd_h + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_16bpc_avx512icl.put) +%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_16bpc_avx512icl.prep) + +BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx512icl, 2, 4, 8, 16, 32, 64, 128 + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +cextern mc_warp_filter +cextern obmc_masks_avx2 +cextern resize_filter + +SECTION .text + +%if WIN64 +DECLARE_REG_TMP 4 +%else +DECLARE_REG_TMP 8 +%endif + +INIT_ZMM avx512icl +cglobal put_bilin_16bpc, 4, 8, 13, dst, ds, src, ss, w, h, mxy + mov mxyd, r6m ; mx + lea r7, [put_avx512icl] + tzcnt t0d, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx t0d, word [r7+t0*2+table_offset(put,)] + add t0, r7 + jmp t0 +.put_w2: + mov r6d, [srcq+ssq*0] + mov r7d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6d + mov [dstq+dsq*1], r7d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r6, [srcq+ssq*0] + mov r7, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6 + mov [dstq+dsq*1], r7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + movu xmm0, [srcq+ssq*0] + movu xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], xmm0 + mova [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu ym0, [srcq+ssq*0] + movu ym1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], ym0 + mova [dstq+dsq*1], ym1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+ssq*0+64*0] + movu m1, [srcq+ssq*0+64*1] + movu m2, [srcq+ssq*1+64*0] + movu m3, [srcq+ssq*1+64*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+64*0], m0 + mova [dstq+dsq*0+64*1], m1 + mova [dstq+dsq*1+64*0], m2 + mova [dstq+dsq*1+64*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+64*0] + movu m1, [srcq+64*1] + movu m2, [srcq+64*2] + movu m3, [srcq+64*3] + add srcq, ssq + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + mova [dstq+64*2], m2 + mova [dstq+64*3], m3 + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + vpbroadcastw m5, mxyd + mov mxyd, r7m ; my + vpbroadcastd m4, [pw_16] + psubw m4, m5 + test mxyd, mxyd + jnz .hv + ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v + movzx t0d, word [r7+t0*2+table_offset(put, _bilin_h)] + mov r6d, r8m ; bitdepth_max + add t0, r7 + shr r6d, 11 + vpbroadcastd m6, [r7-put_avx512icl+put_bilin_h_rnd+r6*4] + jmp t0 +.h_w2: + movq xmm1, [srcq+ssq*0] + movhps xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmullw xmm0, xmm1, xm4 + psrlq xmm1, 16 + pmullw xmm1, xm5 + paddw xmm0, xm6 + paddw xmm0, xmm1 + psrlw xmm0, 4 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + movq xmm0, [srcq+ssq*0+0] + movhps xmm0, [srcq+ssq*1+0] + movq xmm1, [srcq+ssq*0+2] + movhps xmm1, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + pmullw xmm0, xm4 + pmullw xmm1, xm5 + paddw xmm0, xm6 + paddw xmm0, xmm1 + psrlw xmm0, 4 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu xm0, [srcq+ssq*0+0] + vinserti32x4 ym0, [srcq+ssq*1+0], 1 + movu xm1, [srcq+ssq*0+2] + vinserti32x4 ym1, [srcq+ssq*1+2], 1 + lea srcq, [srcq+ssq*2] + pmullw ym0, ym4 + pmullw ym1, ym5 + paddw ym0, ym6 + paddw ym0, ym1 + psrlw ym0, 4 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu ym0, [srcq+ssq*0+0] + vinserti32x8 m0, [srcq+ssq*1+0], 1 + movu ym1, [srcq+ssq*0+2] + vinserti32x8 m1, [srcq+ssq*1+2], 1 + lea srcq, [srcq+ssq*2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m6 + paddw m0, m1 + psrlw m0, 4 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + pmullw m0, m4, [srcq+ssq*0+0] + pmullw m2, m5, [srcq+ssq*0+2] + pmullw m1, m4, [srcq+ssq*1+0] + pmullw m3, m5, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + paddw m0, m6 + paddw m1, m6 + paddw m0, m2 + paddw m1, m3 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w32 + RET +.h_w64: + pmullw m0, m4, [srcq+64*0+0] + pmullw m2, m5, [srcq+64*0+2] + pmullw m1, m4, [srcq+64*1+0] + pmullw m3, m5, [srcq+64*1+2] + add srcq, ssq + paddw m0, m6 + paddw m1, m6 + paddw m0, m2 + paddw m1, m3 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + pmullw m0, m4, [srcq+64*0+0] + pmullw m7, m5, [srcq+64*0+2] + pmullw m1, m4, [srcq+64*1+0] + pmullw m8, m5, [srcq+64*1+2] + pmullw m2, m4, [srcq+64*2+0] + pmullw m9, m5, [srcq+64*2+2] + pmullw m3, m4, [srcq+64*3+0] + pmullw m10, m5, [srcq+64*3+2] + add srcq, ssq + REPX {paddw x, m6}, m0, m1, m2, m3 + paddw m0, m7 + paddw m1, m8 + paddw m2, m9 + paddw m3, m10 + REPX {psrlw x, 4}, m0, m1, m2, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + mova [dstq+64*2], m2 + mova [dstq+64*3], m3 + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx t0d, word [r7+t0*2+table_offset(put, _bilin_v)] + shl mxyd, 11 + vpbroadcastw m8, mxyd + add t0, r7 + jmp t0 +.v_w2: + movd xmm0, [srcq+ssq*0] +.v_w2_loop: + movd xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq xmm2, xmm0, xmm1 + movd xmm0, [srcq+ssq*0] + punpckldq xmm1, xmm0 + psubw xmm1, xmm2 + pmulhrsw xmm1, xm8 + paddw xmm1, xmm2 + movd [dstq+dsq*0], xmm1 + pextrd [dstq+dsq*1], xmm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq xmm0, [srcq+ssq*0] +.v_w4_loop: + movq xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklqdq xmm2, xmm0, xmm1 + movq xmm0, [srcq+ssq*0] + punpcklqdq xmm1, xmm0 + psubw xmm1, xmm2 + pmulhrsw xmm1, xm8 + paddw xmm1, xmm2 + movq [dstq+dsq*0], xmm1 + movhps [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movu xmm0, [srcq+ssq*0] +.v_w8_loop: + vbroadcasti128 ymm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd ymm2, ymm0, ymm1, 0xf0 + vbroadcasti128 ymm0, [srcq+ssq*0] + vpblendd ymm1, ymm0, 0xf0 + psubw ymm1, ymm2 + pmulhrsw ymm1, ym8 + paddw ymm1, ymm2 + mova [dstq+dsq*0], xmm1 + vextracti128 [dstq+dsq*1], ymm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + vzeroupper + RET +.v_w16: + movu ym0, [srcq+ssq*0] +.v_w16_loop: + movu ym3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + psubw ym1, ym3, ym0 + pmulhrsw ym1, ym8 + paddw ym1, ym0 + movu ym0, [srcq+ssq*0] + psubw ym2, ym0, ym3 + pmulhrsw ym2, ym8 + paddw ym2, ym3 + mova [dstq+dsq*0], ym1 + mova [dstq+dsq*1], ym2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: + movu m0, [srcq+ssq*0] +.v_w32_loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + psubw m1, m3, m0 + pmulhrsw m1, m8 + paddw m1, m0 + movu m0, [srcq+ssq*0] + psubw m2, m0, m3 + pmulhrsw m2, m8 + paddw m2, m3 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w32_loop + RET +.v_w64: + movu m0, [srcq+ssq*0+64*0] + movu m1, [srcq+ssq*0+64*1] +.v_w64_loop: + movu m2, [srcq+ssq*1+64*0] + movu m3, [srcq+ssq*1+64*1] + lea srcq, [srcq+ssq*2] + psubw m4, m2, m0 + pmulhrsw m4, m8 + paddw m4, m0 + movu m0, [srcq+ssq*0+64*0] + psubw m5, m3, m1 + pmulhrsw m5, m8 + paddw m5, m1 + movu m1, [srcq+ssq*0+64*1] + psubw m6, m0, m2 + pmulhrsw m6, m8 + psubw m7, m1, m3 + pmulhrsw m7, m8 + mova [dstq+dsq*0+64*0], m4 + mova [dstq+dsq*0+64*1], m5 + paddw m6, m2 + paddw m7, m3 + mova [dstq+dsq*1+64*0], m6 + mova [dstq+dsq*1+64*1], m7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + movu m0, [srcq+ssq*0+64*0] + movu m1, [srcq+ssq*0+64*1] + movu m2, [srcq+ssq*0+64*2] + movu m3, [srcq+ssq*0+64*3] +.v_w128_loop: + movu m4, [srcq+ssq*1+64*0] + movu m5, [srcq+ssq*1+64*1] + movu m6, [srcq+ssq*1+64*2] + movu m7, [srcq+ssq*1+64*3] + lea srcq, [srcq+ssq*2] + psubw m9, m4, m0 + pmulhrsw m9, m8 + paddw m9, m0 + movu m0, [srcq+ssq*0+64*0] + psubw m10, m5, m1 + pmulhrsw m10, m8 + paddw m10, m1 + movu m1, [srcq+ssq*0+64*1] + psubw m11, m6, m2 + pmulhrsw m11, m8 + paddw m11, m2 + movu m2, [srcq+ssq*0+64*2] + psubw m12, m7, m3 + pmulhrsw m12, m8 + paddw m12, m3 + movu m3, [srcq+ssq*0+64*3] + mova [dstq+dsq*0+64*0], m9 + psubw m9, m0, m4 + pmulhrsw m9, m8 + mova [dstq+dsq*0+64*1], m10 + psubw m10, m1, m5 + pmulhrsw m10, m8 + mova [dstq+dsq*0+64*2], m11 + psubw m11, m2, m6 + pmulhrsw m11, m8 + mova [dstq+dsq*0+64*3], m12 + psubw m12, m3, m7 + pmulhrsw m12, m8 + paddw m9, m4 + paddw m10, m5 + mova [dstq+dsq*1+64*0], m9 + mova [dstq+dsq*1+64*1], m10 + paddw m11, m6 + paddw m12, m7 + mova [dstq+dsq*1+64*2], m11 + mova [dstq+dsq*1+64*3], m12 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w128_loop + RET +.hv: + movzx t0d, word [r7+t0*2+table_offset(put, _bilin_hv)] + shl mxyd, 11 + vpbroadcastd m6, [pw_2] + vpbroadcastw m7, mxyd + vpbroadcastd m8, [pw_8192] + add t0, r7 + test dword r8m, 0x800 + jnz .hv_12bpc + psllw m4, 2 + psllw m5, 2 + vpbroadcastd m8, [pw_2048] +.hv_12bpc: + jmp t0 +.hv_w2: + vpbroadcastq xmm1, [srcq+ssq*0] + pmullw xmm0, xmm1, xm4 + psrlq xmm1, 16 + pmullw xmm1, xm5 + paddw xmm0, xm6 + paddw xmm0, xmm1 + psrlw xmm0, 2 +.hv_w2_loop: + movq xmm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xmm2, [srcq+ssq*0] + pmullw xmm1, xmm2, xm4 + psrlq xmm2, 16 + pmullw xmm2, xm5 + paddw xmm1, xm6 + paddw xmm1, xmm2 + psrlw xmm1, 2 ; 1 _ 2 _ + shufpd xmm2, xmm0, xmm1, 0x01 ; 0 _ 1 _ + mova xmm0, xmm1 + psubw xmm1, xmm2 + paddw xmm1, xmm1 + pmulhw xmm1, xm7 + paddw xmm1, xmm2 + pmulhrsw xmm1, xm8 + movd [dstq+dsq*0], xmm1 + pextrd [dstq+dsq*1], xmm1, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + pmullw xmm0, xm4, [srcq+ssq*0-8] + pmullw xmm1, xm5, [srcq+ssq*0-6] + paddw xmm0, xm6 + paddw xmm0, xmm1 + psrlw xmm0, 2 +.hv_w4_loop: + movq xmm1, [srcq+ssq*1+0] + movq xmm2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + movhps xmm1, [srcq+ssq*0+0] + movhps xmm2, [srcq+ssq*0+2] + pmullw xmm1, xm4 + pmullw xmm2, xm5 + paddw xmm1, xm6 + paddw xmm1, xmm2 + psrlw xmm1, 2 ; 1 2 + shufpd xmm2, xmm0, xmm1, 0x01 ; 0 1 + mova xmm0, xmm1 + psubw xmm1, xmm2 + paddw xmm1, xmm1 + pmulhw xmm1, xm7 + paddw xmm1, xmm2 + pmulhrsw xmm1, xm8 + movq [dstq+dsq*0], xmm1 + movhps [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + pmullw xmm0, xm4, [srcq+ssq*0+0] + pmullw xmm1, xm5, [srcq+ssq*0+2] + paddw xmm0, xm6 + paddw xmm0, xmm1 + psrlw xmm0, 2 + vinserti32x4 ym0, xmm0, 1 +.hv_w8_loop: + movu xm1, [srcq+ssq*1+0] + movu xm2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + vinserti32x4 ym1, [srcq+ssq*0+0], 1 + vinserti32x4 ym2, [srcq+ssq*0+2], 1 + pmullw ym1, ym4 + pmullw ym2, ym5 + paddw ym1, ym6 + paddw ym1, ym2 + psrlw ym1, 2 ; 1 2 + vshufi32x4 ym2, ym0, ym1, 0x01 ; 0 1 + mova ym0, ym1 + psubw ym1, ym2 + paddw ym1, ym1 + pmulhw ym1, ym7 + paddw ym1, ym2 + pmulhrsw ym1, ym8 + mova [dstq+dsq*0], xm1 + vextracti32x4 [dstq+dsq*1], ym1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + pmullw ym0, ym4, [srcq+ssq*0+0] + pmullw ym1, ym5, [srcq+ssq*0+2] + paddw ym0, ym6 + paddw ym0, ym1 + psrlw ym0, 2 + vinserti32x8 m0, ym0, 1 +.hv_w16_loop: + movu ym1, [srcq+ssq*1+0] + movu ym2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + vinserti32x8 m1, [srcq+ssq*0+0], 1 + vinserti32x8 m2, [srcq+ssq*0+2], 1 + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m6 + paddw m1, m2 + psrlw m1, 2 ; 1 2 + vshufi32x4 m2, m0, m1, q1032 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m7 + paddw m1, m2 + pmulhrsw m1, m8 + mova [dstq+dsq*0], ym1 + vextracti32x8 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: +.hv_w64: +.hv_w128: + movifnidn wd, wm + lea r6d, [hq+wq*8-256] + mov r4, srcq + mov r7, dstq +.hv_w32_loop0: + pmullw m0, m4, [srcq+ssq*0+0] + pmullw m1, m5, [srcq+ssq*0+2] + paddw m0, m6 + paddw m0, m1 + psrlw m0, 2 +.hv_w32_loop: + pmullw m3, m4, [srcq+ssq*1+0] + pmullw m1, m5, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + paddw m3, m6 + paddw m3, m1 + psrlw m3, 2 + psubw m1, m3, m0 + paddw m1, m1 + pmulhw m1, m7 + paddw m1, m0 + pmullw m0, m4, [srcq+ssq*0+0] + pmullw m2, m5, [srcq+ssq*0+2] + paddw m0, m6 + paddw m0, m2 + psrlw m0, 2 + psubw m2, m0, m3 + paddw m2, m2 + pmulhw m2, m7 + paddw m2, m3 + pmulhrsw m1, m8 + pmulhrsw m2, m8 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w32_loop + add r4, 64 + add r7, 64 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .hv_w32_loop0 + RET + +cglobal prep_bilin_16bpc, 3, 7, 16, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea r6, [prep_avx512icl] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [r6+wq*2+table_offset(prep,)] + mov r5d, r7m ; bitdepth_max + vpbroadcastd m5, [r6-prep_avx512icl+pw_8192] + add wq, r6 + shr r5d, 11 + vpbroadcastd m4, [r6-prep_avx512icl+prep_mul+r5*4] + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movq xmm0, [srcq+strideq*0] + movhps xmm0, [srcq+strideq*1] + vpbroadcastq ymm1, [srcq+strideq*2] + vpbroadcastq ymm2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd ymm0, ymm1, 0x30 + vpblendd ymm0, ymm2, 0xc0 + pmullw ymm0, ym4 + psubw ymm0, ym5 + mova [tmpq], ymm0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + vzeroupper + RET +.prep_w8: + movu xm0, [srcq+strideq*0] + vinserti32x4 ym0, [srcq+strideq*1], 1 + vinserti32x4 m0, [srcq+strideq*2], 2 + vinserti32x4 m0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pmullw m0, m4 + psubw m0, m5 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + movu ym1, [srcq+strideq*2] + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pmullw m0, m4 + pmullw m1, m4 + psubw m0, m5 + psubw m1, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 64*2 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmullw m0, m4, [srcq+strideq*0] + pmullw m1, m4, [srcq+strideq*1] + pmullw m2, m4, [srcq+strideq*2] + pmullw m3, m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 4 + jg .prep_w32 + RET +.prep_w64: + pmullw m0, m4, [srcq+strideq*0+64*0] + pmullw m1, m4, [srcq+strideq*0+64*1] + pmullw m2, m4, [srcq+strideq*1+64*0] + pmullw m3, m4, [srcq+strideq*1+64*1] + lea srcq, [srcq+strideq*2] + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 2 + jg .prep_w64 + RET +.prep_w128: + pmullw m0, m4, [srcq+64*0] + pmullw m1, m4, [srcq+64*1] + pmullw m2, m4, [srcq+64*2] + pmullw m3, m4, [srcq+64*3] + add srcq, strideq + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + dec hd + jg .prep_w128 + RET +.h: + vpbroadcastw m5, mxyd + mov mxyd, r6m ; my + vpbroadcastd m4, [pw_16] + vpbroadcastd m6, [pw_32766] + psubw m4, m5 + test dword r7m, 0x800 + jnz .h_12bpc + psllw m4, 2 + psllw m5, 2 +.h_12bpc: + test mxyd, mxyd + jnz .hv + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)] + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + movu xm1, [srcq+strideq*0] + vinserti32x4 ym1, [srcq+strideq*2], 1 + movu xm2, [srcq+strideq*1] + vinserti32x4 ym2, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1, ym2 + psrldq ym1, 2 + psrldq ym2, 2 + pmullw ym0, ym4 + punpcklqdq ym1, ym2 + pmullw ym1, ym5 + psubw ym0, ym6 + paddw ym0, ym1 + psraw ym0, 2 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4 + RET +.h_w8: + movu xm0, [srcq+strideq*0+0] + movu xm1, [srcq+strideq*0+2] + vinserti32x4 ym0, [srcq+strideq*1+0], 1 + vinserti32x4 ym1, [srcq+strideq*1+2], 1 + vinserti32x4 m0, [srcq+strideq*2+0], 2 + vinserti32x4 m1, [srcq+strideq*2+2], 2 + vinserti32x4 m0, [srcq+stride3q +0], 3 + vinserti32x4 m1, [srcq+stride3q +2], 3 + lea srcq, [srcq+strideq*4] + pmullw m0, m4 + pmullw m1, m5 + psubw m0, m6 + paddw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8 + RET +.h_w16: + movu ym0, [srcq+strideq*0+0] + vinserti32x8 m0, [srcq+strideq*1+0], 1 + movu ym1, [srcq+strideq*0+2] + vinserti32x8 m1, [srcq+strideq*1+2], 1 + lea srcq, [srcq+strideq*2] + pmullw m0, m4 + pmullw m1, m5 + psubw m0, m6 + paddw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 2 + jg .h_w16 + RET +.h_w32: + pmullw m0, m4, [srcq+strideq*0+0] + pmullw m2, m5, [srcq+strideq*0+2] + pmullw m1, m4, [srcq+strideq*1+0] + pmullw m3, m5, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + psubw m0, m6 + psubw m1, m6 + paddw m0, m2 + paddw m1, m3 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 64*2 + sub hd, 2 + jg .h_w32 + RET +.h_w64: + pmullw m0, m4, [srcq+ 0] + pmullw m2, m5, [srcq+ 2] + pmullw m1, m4, [srcq+64] + pmullw m3, m5, [srcq+66] + add srcq, strideq + psubw m0, m6 + psubw m1, m6 + paddw m0, m2 + paddw m1, m3 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 64*2 + dec hd + jg .h_w64 + RET +.h_w128: + pmullw m0, m4, [srcq+ 0] + pmullw m7, m5, [srcq+ 2] + pmullw m1, m4, [srcq+ 64] + pmullw m8, m5, [srcq+ 66] + pmullw m2, m4, [srcq+128] + pmullw m9, m5, [srcq+130] + pmullw m3, m4, [srcq+192] + pmullw m10, m5, [srcq+194] + add srcq, strideq + REPX {psubw x, m6}, m0, m1, m2, m3 + paddw m0, m7 + paddw m1, m8 + paddw m2, m9 + paddw m3, m10 + REPX {psraw x, 2}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)] + vpbroadcastw m9, mxyd + vpbroadcastd m8, [pw_16] + vpbroadcastd m10, [pw_32766] + add wq, r6 + lea stride3q, [strideq*3] + psubw m8, m9 + test dword r7m, 0x800 + jnz .v_12bpc + psllw m8, 2 + psllw m9, 2 +.v_12bpc: + jmp wq +.v_w4: + movq xmm0, [srcq+strideq*0] +.v_w4_loop: + vpbroadcastq xmm2, [srcq+strideq*1] + vpbroadcastq ymm1, [srcq+strideq*2] + vpbroadcastq ymm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd ymm2, ymm1, 0x30 + vpblendd ymm2, ymm3, 0xc0 + vpblendd ymm1, ymm2, ymm0, 0x03 ; 0 1 2 3 + movq xmm0, [srcq+strideq*0] + valignq ymm2, ymm0, ymm2, 1 ; 1 2 3 4 + pmullw ymm1, ym8 + pmullw ymm2, ym9 + psubw ymm1, ym10 + paddw ymm1, ymm2 + psraw ymm1, 2 + mova [tmpq], ymm1 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + movu xm0, [srcq+strideq*0] +.v_w8_loop: + vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 + vinserti32x4 m1, [srcq+strideq*2], 2 + vinserti32x4 m1, [srcq+stride3q ], 3 ; 0 1 2 3 + lea srcq, [srcq+strideq*4] + movu xm0, [srcq+strideq*0] + valignq m2, m0, m1, 2 ; 1 2 3 4 + pmullw m1, m8 + pmullw m2, m9 + psubw m1, m10 + paddw m1, m2 + psraw m1, 2 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + movu ym0, [srcq+strideq*0] +.v_w16_loop: + vinserti32x8 m1, m0, [srcq+strideq*1], 1 ; 0 1 + movu ym3, [srcq+strideq*2] + vinserti32x8 m2, m3, [srcq+stride3q ], 1 ; 2 3 + lea srcq, [srcq+strideq*4] + movu ym0, [srcq+strideq*0] + vshufi32x4 m3, m1, m3, q1032 ; 1 2 + vshufi32x4 m4, m2, m0, q1032 ; 3 4 + pmullw m1, m8 + pmullw m2, m8 + pmullw m3, m9 + pmullw m4, m9 + psubw m1, m10 + psubw m2, m10 + paddw m1, m3 + paddw m2, m4 + psraw m1, 2 + psraw m2, 2 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + movu m0, [srcq+strideq*0] +.v_w32_loop: + movu m3, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + pmullw m1, m8, m0 + movu m0, [srcq+strideq*0] + pmullw m2, m8, m3 + pmullw m3, m9 + pmullw m4, m9, m0 + psubw m1, m10 + psubw m2, m10 + paddw m1, m3 + paddw m2, m4 + psraw m1, 2 + psraw m2, 2 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 2 + jg .v_w32_loop + RET +.v_w64: + movu m0, [srcq+64*0] + movu m1, [srcq+64*1] +.v_w64_loop: + add srcq, strideq + pmullw m2, m8, m0 + movu m0, [srcq+64*0] + pmullw m3, m8, m1 + movu m1, [srcq+64*1] + pmullw m4, m9, m0 + pmullw m5, m9, m1 + psubw m2, m10 + psubw m3, m10 + paddw m2, m4 + paddw m3, m5 + psraw m2, 2 + psraw m3, 2 + mova [tmpq+64*0], m2 + mova [tmpq+64*1], m3 + add tmpq, 64*2 + dec hd + jg .v_w64_loop + RET +.v_w128: + movu m0, [srcq+64*0] + movu m1, [srcq+64*1] + movu m2, [srcq+64*2] + movu m3, [srcq+64*3] +.v_w128_loop: + add srcq, strideq + pmullw m4, m8, m0 + movu m0, [srcq+64*0] + pmullw m5, m8, m1 + movu m1, [srcq+64*1] + pmullw m6, m8, m2 + movu m2, [srcq+64*2] + pmullw m7, m8, m3 + movu m3, [srcq+64*3] + pmullw m11, m9, m0 + pmullw m12, m9, m1 + pmullw m13, m9, m2 + pmullw m14, m9, m3 + REPX {psubw x, m10}, m4, m5, m6, m7 + paddw m4, m11 + paddw m5, m12 + paddw m6, m13 + paddw m7, m14 + REPX {psraw x, 2}, m4, m5, m6, m7 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m5 + mova [tmpq+64*2], m6 + mova [tmpq+64*3], m7 + add tmpq, 64*4 + dec hd + jg .v_w128_loop + RET +.hv: + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + vpbroadcastw m7, mxyd + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.hv_w4: + movq xmm0, [srcq+strideq*0+0] + movq xmm1, [srcq+strideq*0+2] + pmullw xmm0, xm4 + pmullw xmm1, xm5 + psubw xmm0, xm6 + paddw xmm0, xmm1 + psraw xmm0, 2 + vpbroadcastq ym0, xmm0 +.hv_w4_loop: + movu xm1, [srcq+strideq*1] + vinserti128 ym1, [srcq+stride3q ], 1 + movu xm2, [srcq+strideq*2] + lea srcq, [srcq+strideq*4] + vinserti128 ym2, [srcq+strideq*0], 1 + punpcklqdq ym3, ym1, ym2 + psrldq ym1, 2 + psrldq ym2, 2 + pmullw ym3, ym4 + punpcklqdq ym1, ym2 + pmullw ym1, ym5 + psubw ym3, ym6 + paddw ym1, ym3 + psraw ym1, 2 ; 1 2 3 4 + valignq ym2, ym1, ym0, 3 ; 0 1 2 3 + mova ym0, ym1 + psubw ym1, ym2 + pmulhrsw ym1, ym7 + paddw ym1, ym2 + mova [tmpq], ym1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + pmullw xm0, xm4, [srcq+strideq*0+0] + pmullw xm1, xm5, [srcq+strideq*0+2] + psubw xm0, xm6 + paddw xm0, xm1 + psraw xm0, 2 + vinserti32x4 m0, xm0, 3 +.hv_w8_loop: + movu xm1, [srcq+strideq*1+0] + movu xm2, [srcq+strideq*1+2] + vinserti32x4 ym1, [srcq+strideq*2+0], 1 + vinserti32x4 ym2, [srcq+strideq*2+2], 1 + vinserti32x4 m1, [srcq+stride3q +0], 2 + vinserti32x4 m2, [srcq+stride3q +2], 2 + lea srcq, [srcq+strideq*4] + vinserti32x4 m1, [srcq+strideq*0+0], 3 + vinserti32x4 m2, [srcq+strideq*0+2], 3 + pmullw m1, m4 + pmullw m2, m5 + psubw m1, m6 + paddw m1, m2 + psraw m1, 2 ; 1 2 3 4 + valignq m2, m1, m0, 6 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m7 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + pmullw ym0, ym4, [srcq+strideq*0+0] + pmullw ym1, ym5, [srcq+strideq*0+2] + psubw ym0, ym6 + paddw ym0, ym1 + psraw ym0, 2 + vinserti32x8 m0, ym0, 1 +.hv_w16_loop: + movu ym1, [srcq+strideq*1+0] + movu ym2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + vinserti32x8 m1, [srcq+strideq*0+0], 1 + vinserti32x8 m2, [srcq+strideq*0+2], 1 + pmullw m1, m4 + pmullw m2, m5 + psubw m1, m6 + paddw m1, m2 + psraw m1, 2 ; 1 2 + vshufi32x4 m2, m0, m1, q1032 ; 0 1 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m7 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + pmullw m0, m4, [srcq+strideq*0+0] + pmullw m1, m5, [srcq+strideq*0+2] + psubw m0, m6 + paddw m0, m1 + psraw m0, 2 +.hv_w32_loop: + pmullw m3, m4, [srcq+strideq*1+0] + pmullw m1, m5, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + psubw m3, m6 + paddw m3, m1 + psraw m3, 2 + psubw m1, m3, m0 + pmulhrsw m1, m7 + paddw m1, m0 + pmullw m0, m4, [srcq+strideq*0+0] + pmullw m2, m5, [srcq+strideq*0+2] + psubw m0, m6 + paddw m0, m2 + psraw m0, 2 + psubw m2, m0, m3 + pmulhrsw m2, m7 + paddw m2, m3 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 2 + jg .hv_w32_loop + RET +.hv_w64: + pmullw m0, m4, [srcq+ 0] + pmullw m2, m5, [srcq+ 2] + pmullw m1, m4, [srcq+64] + pmullw m3, m5, [srcq+66] + psubw m0, m6 + psubw m1, m6 + paddw m0, m2 + paddw m1, m3 + psraw m0, 2 + psraw m1, 2 +.hv_w64_loop: + add srcq, strideq + pmullw m2, m4, [srcq+ 0] + pmullw m8, m5, [srcq+ 2] + pmullw m3, m4, [srcq+64] + pmullw m9, m5, [srcq+66] + psubw m2, m6 + psubw m3, m6 + paddw m2, m8 + paddw m3, m9 + psraw m2, 2 + psraw m3, 2 + psubw m8, m2, m0 + psubw m9, m3, m1 + pmulhrsw m8, m7 + pmulhrsw m9, m7 + paddw m8, m0 + mova m0, m2 + paddw m9, m1 + mova m1, m3 + mova [tmpq+64*0], m8 + mova [tmpq+64*1], m9 + add tmpq, 64*2 + dec hd + jg .hv_w64_loop + RET +.hv_w128: + pmullw m0, m4, [srcq+ 0] + pmullw m8, m5, [srcq+ 2] + pmullw m1, m4, [srcq+ 64] + pmullw m9, m5, [srcq+ 66] + pmullw m2, m4, [srcq+128] + pmullw m10, m5, [srcq+130] + pmullw m3, m4, [srcq+192] + pmullw m11, m5, [srcq+194] + REPX {psubw x, m6}, m0, m1, m2, m3 + paddw m0, m8 + paddw m1, m9 + paddw m2, m10 + paddw m3, m11 + REPX {psraw x, 2}, m0, m1, m2, m3 +.hv_w128_loop: + add srcq, strideq + pmullw m8, m4, [srcq+ 0] + pmullw m12, m5, [srcq+ 2] + pmullw m9, m4, [srcq+ 64] + pmullw m13, m5, [srcq+ 66] + pmullw m10, m4, [srcq+128] + pmullw m14, m5, [srcq+130] + pmullw m11, m4, [srcq+192] + pmullw m15, m5, [srcq+194] + REPX {psubw x, m6}, m8, m9, m10, m11 + paddw m8, m12 + paddw m9, m13 + paddw m10, m14 + paddw m11, m15 + REPX {psraw x, 2}, m8, m9, m10, m11 + psubw m12, m8, m0 + psubw m13, m9, m1 + psubw m14, m10, m2 + psubw m15, m11, m3 + REPX {pmulhrsw x, m7}, m12, m13, m14, m15 + paddw m12, m0 + mova m0, m8 + paddw m13, m1 + mova m1, m9 + mova [tmpq+64*0], m12 + mova [tmpq+64*1], m13 + paddw m14, m2 + mova m2, m10 + paddw m15, m3 + mova m3, m11 + mova [tmpq+64*2], m14 + mova [tmpq+64*3], m15 + add tmpq, 64*4 + dec hd + jg .hv_w128_loop + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v +cglobal %1_8tap_%2_16bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_8tap_16bpc %+ SUFFIX) +%endif +%endmacro + +%if WIN64 +DECLARE_REG_TMP 4, 5 +%define buf rsp+stack_offset+8 ; shadow space +%else +DECLARE_REG_TMP 7, 8 +%define buf rsp-40 ; red zone +%endif + +MC_8TAP_FN put, sharp, SHARP, SHARP +MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH +MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP +MC_8TAP_FN put, smooth, SMOOTH, SMOOTH +MC_8TAP_FN put, sharp_regular, SHARP, REGULAR +MC_8TAP_FN put, regular_sharp, REGULAR, SHARP +MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR +MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH +MC_8TAP_FN put, regular, REGULAR, REGULAR + +cglobal put_8tap_16bpc, 4, 9, 16, dst, ds, src, ss, w, h, mx, my +%define base r8-put_avx512icl + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx512icl] + movifnidn wd, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r8+wq*2+table_offset(put,)] + add wq, r8 +%if WIN64 + pop r8 +%endif + jmp wq +.h_w2: + movzx mxd, mxb + sub srcq, 2 + mova ym2, [spel_h_shuf2a] + pmovsxbw xmm4, [base+subpel_filters+mxq*8] + pshufd xmm3, xmm4, q1111 + pshufd xmm4, xmm4, q2222 +.h_w2_loop: + movu xm1, [srcq+ssq*0] + vinserti32x4 ym1, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + mova xmm0, xm8 + vpermb ym1, ym2, ym1 + vpdpwssd xmm0, xmm3, xm1 + vextracti32x4 xm1, ym1, 1 + vpdpwssd xmm0, xmm4, xm1 + psrad xmm0, 6 + packusdw xmm0, xmm0 + pminsw xmm0, xm9 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + sub srcq, 2 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + vbroadcasti32x4 ym4, [spel_h_shufA] + vbroadcasti32x4 ym5, [spel_h_shufB] + pshufd xmm0, xmm0, q2211 + vpbroadcastq ym6, xmm0 + vpermq ym7, ymm0, q1111 +.h_w4_loop: + movu xm2, [srcq+ssq*0] + vinserti32x4 ym2, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + mova ym0, ym8 + pshufb ym1, ym2, ym4 + vpdpwssd ym0, ym6, ym1 + pshufb ym2, ym5 + vpdpwssd ym0, ym7, ym2 + psrad ym0, 6 + vextracti32x4 xm1, ym0, 1 + packusdw xm0, xm1 + pminsw xmm0, xm0, xm9 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h: + test myd, 0xf00 + jnz .hv + mov r7d, r8m + vpbroadcastw m9, r8m + shr r7d, 11 + vpbroadcastd m8, [base+put_8tap_h_rnd+r7*4] + cmp wd, 4 + je .h_w4 + jl .h_w2 + shr mxd, 16 + sub srcq, 6 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + mova [buf], xmm0 + vpbroadcastd m10, xmm0 + vpbroadcastd m11, [buf+ 4] + vpbroadcastd m12, [buf+ 8] + vpbroadcastd m13, [buf+12] + sub wd, 16 + je .h_w16 + jg .h_w32 +.h_w8: + mova m4, [spel_h_shufA] + movu m5, [spel_h_shufB] + movu m6, [spel_h_shufC] + mova m7, [spel_h_shufD] +.h_w8_loop: + movu ym2, [srcq+ssq*0] + vinserti32x8 m2, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + mova m0, m8 + vpermb m1, m4, m2 + vpdpwssd m0, m10, m1 + vpermb m1, m5, m2 + vpdpwssd m0, m11, m1 + vpermb m1, m6, m2 + vpdpwssd m0, m12, m1 + vpermb m1, m7, m2 + vpdpwssd m0, m13, m1 + psrad m0, 6 + vextracti32x8 ym1, m0, 1 + packusdw ym0, ym1 + pminsw ym0, ym9 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8_loop + RET +.h_w16: + vbroadcasti32x4 m6, [spel_h_shufA] + vbroadcasti32x4 m7, [spel_h_shufB] +.h_w16_loop: + movu ym2, [srcq+ssq*0+ 0] + vinserti32x8 m2, [srcq+ssq*1+ 0], 1 + movu ym3, [srcq+ssq*0+16] + vinserti32x8 m3, [srcq+ssq*1+16], 1 + lea srcq, [srcq+ssq*2] + mova m0, m8 + mova m1, m8 + pshufb m4, m2, m6 + vpdpwssd m0, m10, m4 ; a0 + pshufb m4, m3, m6 + vpdpwssd m1, m12, m4 ; b2 + pshufb m4, m2, m7 + vpdpwssd m0, m11, m4 ; a1 + pshufb m4, m3, m7 + vpdpwssd m1, m13, m4 ; b3 + shufpd m2, m3, 0x55 + pshufb m4, m2, m6 + vpdpwssd m0, m12, m4 ; a2 + vpdpwssd m1, m10, m4 ; b0 + pshufb m2, m7 + vpdpwssd m0, m13, m2 ; a3 + vpdpwssd m1, m11, m2 ; b1 + psrad m0, 6 + psrad m1, 6 + packusdw m0, m1 + pminsw m0, m9 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16_loop + RET +.h_w32: + lea srcq, [srcq+wq*2] + vbroadcasti32x4 m6, [spel_h_shufA] + lea dstq, [dstq+wq*2] + vbroadcasti32x4 m7, [spel_h_shufB] + neg wq +.h_w32_loop0: + mov r6, wq +.h_w32_loop: + movu m2, [srcq+r6*2+ 0] + movu m3, [srcq+r6*2+ 8] + mova m0, m8 + mova m1, m8 + pshufb m4, m2, m6 + vpdpwssd m0, m10, m4 ; a0 + pshufb m4, m3, m6 + vpdpwssd m1, m10, m4 ; b0 + vpdpwssd m0, m12, m4 ; a2 + movu m4, [srcq+r6*2+16] + pshufb m3, m7 + vpdpwssd m1, m11, m3 ; b1 + vpdpwssd m0, m13, m3 ; a3 + pshufb m3, m4, m6 + vpdpwssd m1, m12, m3 ; b2 + pshufb m2, m7 + vpdpwssd m0, m11, m2 ; a1 + pshufb m4, m7 + vpdpwssd m1, m13, m4 ; b3 + psrad m0, 6 + psrad m1, 6 + packusdw m0, m1 + pminsw m0, m9 + mova [dstq+r6*2], m0 + add r6, 32 + jl .h_w32_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w32_loop0 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastd m10, [pd_32] + pmovsxbw xmm0, [base+subpel_filters+myq*8] + tzcnt r7d, wd + vpbroadcastw m11, r8m + lea r6, [ssq*3] + movzx r7d, word [r8+r7*2+table_offset(put, _8tap_v)] + sub srcq, r6 + mova [rsp+stack_offset+8], xmm0 + vpbroadcastd m12, xmm0 + add r7, r8 + vpbroadcastd m13, [rsp+stack_offset+12] + vpbroadcastd m14, [rsp+stack_offset+16] + vpbroadcastd m15, [rsp+stack_offset+20] + jmp r7 +.v_w2: + movd xmm2, [srcq+ssq*0] + pinsrd xmm2, [srcq+ssq*1], 1 + pinsrd xmm2, [srcq+ssq*2], 2 + add srcq, r6 + pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3 + movd xmm3, [srcq+ssq*1] + vpbroadcastd xmm1, [srcq+ssq*2] + add srcq, r6 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm3, xmm1, 0x02 ; 4 5 + vpblendd xmm1, xmm0, 0x02 ; 5 6 + palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4 + punpcklwd xmm3, xmm1 ; 45 56 + punpcklwd xmm1, xmm2, xmm4 ; 01 12 + punpckhwd xmm2, xmm4 ; 23 34 +.v_w2_loop: + vpbroadcastd xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova xmm5, xm10 + vpdpwssd xmm5, xm12, xmm1 ; a0 b0 + mova xmm1, xmm2 + vpdpwssd xmm5, xm13, xmm2 ; a1 b1 + mova xmm2, xmm3 + vpdpwssd xmm5, xm14, xmm3 ; a2 b2 + vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm4, xmm0, 0x02 ; 7 8 + punpcklwd xmm3, xmm4 ; 67 78 + vpdpwssd xmm5, xm15, xmm3 ; a3 b3 + psrad xmm5, 6 + packusdw xmm5, xmm5 + pminsw xmm5, xm11 + movd [dstq+dsq*0], xmm5 + pextrd [dstq+dsq*1], xmm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq xmm1, [srcq+ssq*0] + vpbroadcastq ymm0, [srcq+ssq*1] + vpbroadcastq ymm2, [srcq+ssq*2] + add srcq, r6 + vpbroadcastq ymm4, [srcq+ssq*0] + vpbroadcastq ymm3, [srcq+ssq*1] + vpbroadcastq ymm5, [srcq+ssq*2] + add srcq, r6 + vpblendd ymm1, ymm0, 0x30 + vpblendd ymm0, ymm2, 0x30 + punpcklwd ymm1, ymm0 ; 01 12 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm2, ymm4, 0x30 + vpblendd ymm4, ymm3, 0x30 + punpcklwd ymm2, ymm4 ; 23 34 + vpblendd ymm3, ymm5, 0x30 + vpblendd ymm5, ymm0, 0x30 + punpcklwd ymm3, ymm5 ; 45 56 +.v_w4_loop: + vpbroadcastq ymm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova ymm4, ym10 + vpdpwssd ymm4, ym12, ymm1 ; a0 b0 + mova ymm1, ymm2 + vpdpwssd ymm4, ym13, ymm2 ; a1 b1 + mova ymm2, ymm3 + vpdpwssd ymm4, ym14, ymm3 ; a2 b2 + vpblendd ymm3, ymm0, ymm5, 0x30 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm5, ymm0, 0x30 + punpcklwd ymm3, ymm5 ; 67 78 + vpdpwssd ymm4, ym15, ymm3 ; a3 b3 + psrad ymm4, 6 + vextracti128 xmm5, ymm4, 1 + packusdw xmm4, xmm5 + pminsw xmm4, xm11 + movq [dstq+dsq*0], xmm4 + movhps [dstq+dsq*1], xmm4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + vbroadcasti32x4 m2, [srcq+ssq*2] + vinserti32x4 m1, m2, [srcq+ssq*0], 0 + vinserti32x4 m1, [srcq+ssq*1], 1 ; 0 1 2 + add srcq, r6 + vinserti32x4 ym2, [srcq+ssq*0], 1 + vinserti32x4 m2, [srcq+ssq*1], 2 ; 2 3 4 + mova m6, [spel_v_shuf8] + movu xm0, [srcq+ssq*1] + vinserti32x4 ym0, [srcq+ssq*2], 1 + add srcq, r6 + vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6 + vpermb m1, m6, m1 ; 01 12 + vpermb m2, m6, m2 ; 23 34 + vpermb m3, m6, m0 ; 45 56 +.v_w8_loop: + vinserti32x4 m0, [srcq+ssq*1], 3 + lea srcq, [srcq+ssq*2] + movu xm5, [srcq+ssq*0] + mova m4, m10 + vpdpwssd m4, m12, m1 ; a0 b0 + mova m1, m2 + vshufi32x4 m0, m5, q1032 ; 6 7 8 + vpdpwssd m4, m13, m2 ; a1 b1 + mova m2, m3 + vpdpwssd m4, m14, m3 ; a2 b2 + vpermb m3, m6, m0 ; 67 78 + vpdpwssd m4, m15, m3 ; a3 b3 + psrad m4, 6 + vextracti32x8 ym5, m4, 1 + packusdw ym4, ym5 + pminsw ym4, ym11 + mova [dstq+dsq*0], xm4 + vextracti32x4 [dstq+dsq*1], ym4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + vbroadcasti32x8 m1, [srcq+ssq*1] + vinserti32x8 m0, m1, [srcq+ssq*0], 0 + vinserti32x8 m1, [srcq+ssq*2], 1 + mova m8, [spel_v_shuf16] + add srcq, r6 + movu ym3, [srcq+ssq*0] + vinserti32x8 m3, [srcq+ssq*1], 1 + movu ym5, [srcq+ssq*2] + add srcq, r6 + vinserti32x8 m5, [srcq+ssq*0], 1 + vpermb m0, m8, m0 ; 01 + vpermb m1, m8, m1 ; 12 + vpermb m3, m8, m3 ; 34 + vpermb m5, m8, m5 ; 56 + mova m9, [deint_q_shuf] + vpshrdd m2, m1, m3, 16 ; 23 + vpshrdd m4, m3, m5, 16 ; 45 +.v_w16_loop: + mova m6, m10 + mova m7, m10 + vpdpwssd m6, m12, m0 ; a0 + mova m0, m2 + vpdpwssd m7, m12, m1 ; b0 + mova m1, m3 + vpdpwssd m6, m13, m2 ; a1 + mova m2, m4 + vpdpwssd m7, m13, m3 ; b1 + mova m3, m5 + vpdpwssd m6, m14, m4 ; a2 + mova m4, m5 + vpdpwssd m7, m14, m5 ; b2 + movu ym5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x8 m5, [srcq+ssq*0], 1 + vpermb m5, m8, m5 ; 78 + vpshrdd m4, m5, 16 ; 67 + vpdpwssd m6, m15, m4 ; a3 + vpdpwssd m7, m15, m5 ; b3 + psrad m6, 6 + psrad m7, 6 + packusdw m6, m7 + pminsw m6, m11 + vpermq m6, m9, m6 + mova [dstq+dsq*0], ym6 + vextracti32x8 [dstq+dsq*1], m6, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: +.v_w64: +.v_w128: +%if WIN64 + movaps [rsp+stack_offset+8], xmm6 +%endif + lea wd, [hq+wq*8-256] + mov r7, srcq + mov r8, dstq +.v_w32_loop0: + movu m16, [srcq+ssq*0] + movu m17, [srcq+ssq*1] + movu m18, [srcq+ssq*2] + add srcq, r6 + movu m19, [srcq+ssq*0] + movu m20, [srcq+ssq*1] + movu m21, [srcq+ssq*2] + add srcq, r6 + movu m22, [srcq+ssq*0] + punpcklwd m0, m16, m17 ; 01l + punpckhwd m16, m17 ; 01h + punpcklwd m1, m17, m18 ; 12l + punpckhwd m17, m18 ; 12h + punpcklwd m2, m18, m19 ; 23l + punpckhwd m18, m19 ; 23h + punpcklwd m3, m19, m20 ; 34l + punpckhwd m19, m20 ; 34h + punpcklwd m4, m20, m21 ; 45l + punpckhwd m20, m21 ; 45h + punpcklwd m5, m21, m22 ; 56l + punpckhwd m21, m22 ; 56h +.v_w32_loop: + mova m6, m10 + vpdpwssd m6, m12, m0 ; a0l + mova m8, m10 + vpdpwssd m8, m12, m16 ; a0h + mova m7, m10 + vpdpwssd m7, m12, m1 ; b0l + mova m9, m10 + vpdpwssd m9, m12, m17 ; b0h + mova m0, m2 + vpdpwssd m6, m13, m2 ; a1l + mova m16, m18 + vpdpwssd m8, m13, m18 ; a1h + mova m1, m3 + vpdpwssd m7, m13, m3 ; b1l + mova m17, m19 + vpdpwssd m9, m13, m19 ; b1h + mova m2, m4 + vpdpwssd m6, m14, m4 ; a2l + mova m18, m20 + vpdpwssd m8, m14, m20 ; a2h + mova m3, m5 + vpdpwssd m7, m14, m5 ; b2l + mova m19, m21 + vpdpwssd m9, m14, m21 ; b2h + movu m21, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklwd m4, m22, m21 ; 67l + punpckhwd m20, m22, m21 ; 67h + movu m22, [srcq+ssq*0] + vpdpwssd m6, m15, m4 ; a3l + vpdpwssd m8, m15, m20 ; a3h + punpcklwd m5, m21, m22 ; 78l + punpckhwd m21, m22 ; 78h + vpdpwssd m7, m15, m5 ; b3l + vpdpwssd m9, m15, m21 ; b3h + REPX {psrad x, 6}, m6, m8, m7, m9 + packusdw m6, m8 + packusdw m7, m9 + pminsw m6, m11 + pminsw m7, m11 + mova [dstq+dsq*0], m6 + mova [dstq+dsq*1], m7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w32_loop + add r7, 64 + add r8, 64 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 + jg .v_w32_loop0 +%if WIN64 + movaps xmm6, [rsp+stack_offset+8] +%endif + vzeroupper + RET +.hv: + vpbroadcastw m11, r8m + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + pmovsxbw xmm1, [base+subpel_filters+myq*8] + lea r6, [ssq*3] + sub srcq, 2 + sub srcq, r6 + test dword r8m, 0x800 + jnz .hv_12bit + vpbroadcastd m10, [pd_2176] + psllw xmm0, 6 + jmp .hv_main +.hv_12bit: + vpbroadcastd m10, [pd_640] + psllw xmm0, 4 + psllw xmm1, 2 +.hv_main: + mova [buf+ 0], xmm0 + mova [buf+16], xmm1 + vpbroadcastd m8, [buf+ 4] + vpbroadcastd m9, [buf+ 8] + vpbroadcastd ym12, xmm1 + vpbroadcastd ym13, [buf+20] + vpbroadcastd ym14, [buf+24] + vpbroadcastd ym15, [buf+28] + movu xm4, [srcq+ssq*0] + vinserti32x4 ym4, [srcq+ssq*1], 1 + vinserti32x4 m4, [srcq+ssq*2], 2 + add srcq, r6 + vinserti32x4 m4, [srcq+ssq*0], 3 ; 0 1 2 3 + movu xm0, [srcq+ssq*1] + vinserti32x4 ym0, [srcq+ssq*2], 1 + add srcq, r6 + vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6 + cmp wd, 4 + je .hv_w4 + vbroadcasti32x4 m2, [spel_h_shufA] + mova m3, [spel_h_shuf2b] + mova ym6, [spel_h_shuf2a] + mova xm7, [spel_shuf2] + mova m1, m10 + pshufb m4, m2 + pshufb m0, m2 + punpcklqdq m2, m4, m0 + vpdpwssd m1, m8, m2 ; 04 15 26 3_ + punpckhqdq m4, m0 + vpdpwssd m1, m9, m4 + vpermb m1, m3, m1 ; 01 12 + vextracti32x4 xm2, ym1, 1 ; 23 34 + vextracti32x4 xm3, m1, 2 ; 45 56 +.hv_w2_loop: + movu xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x4 ym5, [srcq+ssq*0], 1 + mova xm4, xm10 + vpermb ym5, ym6, ym5 + pmaddwd xmm0, xm12, xm1 ; a0 b0 + vpdpwssd xm4, xm8, xm5 + vextracti32x4 xm5, ym5, 1 + mova xm1, xm2 + vpdpwssd xmm0, xm13, xm2 ; a1 b1 + vpdpwssd xm4, xm9, xm5 ; 7 8 + mova xm2, xm3 + vpdpwssd xmm0, xm14, xm3 ; a2 b2 + vpermt2b xm3, xm7, xm4 ; 67 78 + vpdpwssd xmm0, xm15, xm3 ; a3 b3 + psrad xmm0, 10 + packusdw xmm0, xmm0 + pminsw xmm0, xm11 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + vbroadcasti32x4 m19, [spel_h_shufA] + vbroadcasti32x4 m20, [spel_h_shufB] + mova ym6, [spel_shuf4a] + mova ym7, [spel_shuf4b] + mova m2, m10 + mova m3, m10 + pshufb m1, m4, m19 + vpdpwssd m2, m8, m1 + pshufb m1, m0, m19 + vpdpwssd m3, m8, m1 + pshufb m4, m20 + vpdpwssd m2, m9, m4 + pshufb m0, m20 + vpdpwssd m3, m9, m0 + vpermb m1, m6, m2 ; 01 12 + vshufi32x4 m2, m3, q1032 + vpermb m3, m6, m3 ; 45 56 + vpermb m2, m6, m2 ; 23 34 +.hv_w4_loop: + movu xm18, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 ym18, [srcq+ssq*0], 1 + mova ym4, ym10 + pshufb ym17, ym18, ym19 + pmaddwd ym16, ym12, ym1 ; a0 b0 + vpdpwssd ym4, ym8, ym17 + pshufb ym18, ym20 + mova ym1, ym2 + vpdpwssd ym16, ym13, ym2 ; a1 b1 + vpdpwssd ym4, ym9, ym18 ; 7 8 + mova ym2, ym3 + vpdpwssd ym16, ym14, ym3 ; a2 b2 + vpermt2b ym3, ym7, ym4 ; 67 78 + vpdpwssd ym16, ym15, ym3 ; a3 b3 + psrad ym16, 10 + vextracti128 xm17, ym16, 1 + packusdw xm16, xm17 + pminsw xm16, xm11 + movq [dstq+dsq*0], xm16 + movhps [dstq+dsq*1], xm16 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + vzeroupper + RET +.hv_w8: + shr mxd, 16 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + pmovsxbw xmm1, [base+subpel_filters+myq*8] + lea r6, [ssq*3] + sub srcq, 6 + sub srcq, r6 + test dword r8m, 0x800 + jnz .hv_w8_12bit + vpbroadcastd m10, [pd_2176] + psllw xmm0, 6 + jmp .hv_w8_main +.hv_w8_12bit: + vpbroadcastd m10, [pd_640] + psllw xmm0, 4 + psllw xmm1, 2 +.hv_w8_main: + mova [buf+ 0], xmm0 + mova [buf+16], xmm1 + vpbroadcastd m12, xmm0 + vpbroadcastd m13, [buf+ 4] + vpbroadcastd m14, [buf+ 8] + vpbroadcastd m15, [buf+12] + vpbroadcastd m16, xmm1 + vpbroadcastd m17, [buf+20] + vpbroadcastd m18, [buf+24] + vpbroadcastd m19, [buf+28] + cmp wd, 16 + je .hv_w16 + jg .hv_w32 + mova m5, [spel_h_shufA] + movu ym0, [srcq+ssq*0] + vinserti32x8 m0, [srcq+ssq*1], 1 ; 0 1 + movu ym9, [srcq+ssq*2] + add srcq, r6 + vinserti32x8 m9, [srcq+ssq*0], 1 ; 2 3 + movu ym20, [srcq+ssq*1] + vinserti32x8 m20, [srcq+ssq*2], 1 ; 4 5 + add srcq, r6 + movu ym21, [srcq+ssq*0] ; 6 + movu m6, [spel_h_shufB] + movu m7, [spel_h_shufC] + vpermb m8, m5, m0 + mova m1, m10 + vpdpwssd m1, m12, m8 ; a0 b0 + vpermb m8, m5, m9 + mova m2, m10 + vpdpwssd m2, m12, m8 ; c0 d0 + vpermb m8, m5, m20 + mova m3, m10 + vpdpwssd m3, m12, m8 ; e0 f0 + vpermb m8, m5, m21 + mova m4, m10 + vpdpwssd m4, m12, m8 ; g0 + vpermb m8, m6, m0 + vpdpwssd m1, m13, m8 ; a1 b1 + vpermb m8, m6, m9 + vpdpwssd m2, m13, m8 ; c1 d1 + vpermb m8, m6, m20 + vpdpwssd m3, m13, m8 ; e1 f1 + vpermb m8, m6, m21 + vpdpwssd m4, m13, m8 ; g1 + vpermb m8, m7, m0 + vpdpwssd m1, m14, m8 ; a2 b2 + vpermb m8, m7, m9 + vpdpwssd m2, m14, m8 ; c2 d2 + vpermb m8, m7, m20 + vpdpwssd m3, m14, m8 ; e2 f2 + vpermb m8, m7, m21 + vpdpwssd m4, m14, m8 ; g2 + mova m8, [spel_h_shufD] + vpermb m0, m8, m0 + vpdpwssd m1, m15, m0 ; a3 b3 + mova m0, [spel_shuf8a] + vpermb m9, m8, m9 + vpdpwssd m2, m15, m9 ; c3 d3 + mova m9, [spel_shuf8b] + vpermb m20, m8, m20 + vpdpwssd m3, m15, m20 ; e3 f3 + vpermb m21, m8, m21 + vpdpwssd m4, m15, m21 ; g3 + vpermt2b m1, m0, m2 ; 01 12 + vpermt2b m2, m0, m3 ; 23 34 + vpermt2b m3, m0, m4 ; 45 56 +.hv_w8_loop: + movu ym0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x8 m0, [srcq+ssq*0], 1 + mova m4, m10 + vpermb m21, m5, m0 + vpdpwssd m4, m12, m21 ; h0 i0 + vpermb m21, m6, m0 + pmaddwd m20, m16, m1 ; A0 B0 + vpdpwssd m4, m13, m21 ; h1 i1 + vpermb m21, m7, m0 + mova m1, m2 + vpdpwssd m20, m17, m2 ; A1 B1 + vpdpwssd m4, m14, m21 ; h2 i2 + vpermb m21, m8, m0 + mova m2, m3 + vpdpwssd m20, m18, m3 ; A2 B2 + vpdpwssd m4, m15, m21 ; h3 i3 + vpermt2b m3, m9, m4 ; 67 78 + vpdpwssd m20, m19, m3 ; A3 B3 + psrad m20, 10 + vextracti32x8 ym21, m20, 1 + packusdw ym20, ym21 + pminsw ym20, ym11 + mova [dstq+dsq*0], xm20 + vextracti128 [dstq+dsq*1], ym20, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + vzeroupper + RET +.hv_w16: + WIN64_SPILL_XMM 26 + vbroadcasti32x8 m5, [srcq+ssq*0+ 8] + vinserti32x8 m4, m5, [srcq+ssq*0+ 0], 0 + vinserti32x8 m5, [srcq+ssq*0+16], 1 ; 0 + movu ym6, [srcq+ssq*1+ 0] + movu ym7, [srcq+ssq*1+16] + vinserti32x8 m6, [srcq+ssq*2+ 0], 1 + vinserti32x8 m7, [srcq+ssq*2+16], 1 ; 1 2 + add srcq, r6 + movu ym22, [srcq+ssq*0+ 0] + movu ym23, [srcq+ssq*0+16] + vinserti32x8 m22, [srcq+ssq*1+ 0], 1 + vinserti32x8 m23, [srcq+ssq*1+16], 1 ; 3 4 + movu ym24, [srcq+ssq*2+ 0] + movu ym25, [srcq+ssq*2+16] + add srcq, r6 + vinserti32x8 m24, [srcq+ssq*0+ 0], 1 + vinserti32x8 m25, [srcq+ssq*0+16], 1 ; 5 6 + vbroadcasti32x4 m20, [spel_h_shufA] + vbroadcasti32x4 m21, [spel_h_shufB] + mova m9, [spel_shuf16] + pshufb m0, m4, m20 + mova m1, m10 + vpdpwssd m1, m12, m0 ; a0 + pshufb m0, m6, m20 + mova m2, m10 + vpdpwssd m2, m12, m0 ; b0 + pshufb m0, m7, m20 + mova m3, m10 + vpdpwssd m3, m14, m0 ; c2 + pshufb m0, m4, m21 + vpdpwssd m1, m13, m0 ; a1 + pshufb m0, m6, m21 + vpdpwssd m2, m13, m0 ; b1 + pshufb m0, m7, m21 + vpdpwssd m3, m15, m0 ; c3 + pshufb m0, m5, m20 + vpdpwssd m1, m14, m0 ; a2 + shufpd m6, m7, 0x55 + pshufb m7, m6, m20 + vpdpwssd m2, m14, m7 ; b2 + vpdpwssd m3, m12, m7 ; c0 + pshufb m5, m21 + vpdpwssd m1, m15, m5 ; a3 + pshufb m6, m21 + vpdpwssd m2, m15, m6 ; b3 + vpdpwssd m3, m13, m6 ; c1 + pshufb m0, m22, m20 + mova m4, m10 + vpdpwssd m4, m12, m0 ; d0 + pshufb m0, m23, m20 + mova m5, m10 + vpdpwssd m5, m14, m0 ; e2 + pshufb m0, m24, m20 + mova m6, m10 + vpdpwssd m6, m12, m0 ; f0 + pshufb m0, m25, m20 + mova m7, m10 + vpdpwssd m7, m14, m0 ; g2 + pshufb m0, m22, m21 + vpdpwssd m4, m13, m0 ; d1 + pshufb m0, m23, m21 + vpdpwssd m5, m15, m0 ; e3 + pshufb m0, m24, m21 + vpdpwssd m6, m13, m0 ; f1 + pshufb m0, m25, m21 + vpdpwssd m7, m15, m0 ; g3 + shufpd m22, m23, 0x55 + pshufb m23, m22, m20 + vpdpwssd m4, m14, m23 ; d2 + vpdpwssd m5, m12, m23 ; e0 + shufpd m24, m25, 0x55 + pshufb m25, m24, m20 + vpdpwssd m6, m14, m25 ; f2 + vpdpwssd m7, m12, m25 ; g0 + pshufb m22, m21 + vpdpwssd m4, m15, m22 ; d3 + vpdpwssd m5, m13, m22 ; e1 + pshufb m24, m21 + vpdpwssd m6, m15, m24 ; f3 + vpdpwssd m7, m13, m24 ; g1 + pslldq m1, 1 + vpermt2b m2, m9, m3 ; 12 + vpermt2b m4, m9, m5 ; 34 + vpermt2b m6, m9, m7 ; 56 + vpshrdd m1, m2, 16 ; 01 + vpshrdd m3, m2, m4, 16 ; 23 + vpshrdd m5, m4, m6, 16 ; 45 +.hv_w16_loop: + movu ym24, [srcq+ssq*1+ 0] + movu ym25, [srcq+ssq*1+16] + lea srcq, [srcq+ssq*2] + vinserti32x8 m24, [srcq+ssq*0+ 0], 1 + vinserti32x8 m25, [srcq+ssq*0+16], 1 + mova m7, m10 + mova m8, m10 + pshufb m0, m24, m20 + vpdpwssd m7, m12, m0 ; h0 + pshufb m0, m25, m20 + vpdpwssd m8, m14, m0 ; i2 + pmaddwd m22, m16, m1 ; A0 + mova m1, m3 + pmaddwd m23, m16, m2 ; B0 + mova m2, m4 + pshufb m0, m24, m21 + vpdpwssd m7, m13, m0 ; h1 + pshufb m0, m25, m21 + vpdpwssd m8, m15, m0 ; i3 + vpdpwssd m22, m17, m3 ; A1 + mova m3, m5 + vpdpwssd m23, m17, m4 ; B1 + mova m4, m6 + shufpd m24, m25, 0x55 + pshufb m25, m24, m20 + vpdpwssd m7, m14, m25 ; h2 + vpdpwssd m8, m12, m25 ; i0 + vpdpwssd m22, m18, m5 ; A2 + vpdpwssd m23, m18, m6 ; B2 + pshufb m24, m21 + vpdpwssd m7, m15, m24 ; h3 + vpdpwssd m8, m13, m24 ; i1 + vpermt2b m7, m9, m8 ; 78 + vpshrdd m5, m6, m7, 16 ; 67 + vpdpwssd m22, m19, m5 ; A3 + vpdpwssd m23, m19, m7 ; B3 + mova m6, m7 + psrad m22, 10 + psrad m23, 10 + vshufi32x4 m0, m22, m23, q3232 + vinserti32x8 m22, ym23, 1 + packusdw m22, m0 + pminsw m22, m11 + mova [dstq+dsq*0], ym22 + vextracti32x8 [dstq+dsq*1], m22, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 32 + vbroadcasti32x4 m20, [spel_h_shufA] + vbroadcasti32x4 m21, [spel_h_shufB] + mova m22, [spel_shuf32] + lea wd, [hq+wq*8-256] + mov r7, srcq + mov r8, dstq +.hv_w32_loop0: + movu m6, [srcq+ssq*0+ 0] + movu m7, [srcq+ssq*0+ 8] + movu m8, [srcq+ssq*0+16] + mova m0, m10 + mova m23, m10 + pshufb m9, m6, m20 + vpdpwssd m0, m12, m9 ; a0l + pshufb m9, m7, m20 + vpdpwssd m23, m12, m9 ; a0h + vpdpwssd m0, m14, m9 ; a2l + pshufb m7, m21 + vpdpwssd m23, m13, m7 ; a1h + vpdpwssd m0, m15, m7 ; a3l + pshufb m7, m8, m20 + vpdpwssd m23, m14, m7 ; a2h + pshufb m6, m21 + vpdpwssd m0, m13, m6 ; a1l + pshufb m8, m21 + vpdpwssd m23, m15, m8 ; a3h +%macro PUT_8TAP_HV_W32 5 ; dst_lo, dst_hi, stride_name, stride[1-2] + movu m6, [srcq+%3*%4+ 0] + movu m7, [srcq+%3*%4+ 8] + movu m8, [srcq+%3*%4+16] +%if %4 == 2 + add srcq, r6 +%endif + movu m29, [srcq+%3*%5+ 0] + movu m30, [srcq+%3*%5+ 8] + movu m31, [srcq+%3*%5+16] +%if %5 == 2 + add srcq, r6 +%endif + mova m%1, m10 + mova m9, m10 + pshufb m%2, m6, m20 + vpdpwssd m%1, m12, m%2 ; x0l + pshufb m%2, m29, m20 + vpdpwssd m9, m12, m%2 ; y0l + pshufb m6, m21 + vpdpwssd m%1, m13, m6 ; x1l + pshufb m29, m21 + vpdpwssd m9, m13, m29 ; y1l + pshufb m6, m7, m20 + mova m%2, m10 + vpdpwssd m%2, m12, m6 ; x0h + pshufb m29, m30, m20 + vpdpwssd m%1, m14, m6 ; y2l + mova m6, m10 + vpdpwssd m6, m12, m29 ; x0h + pshufb m7, m21 + vpdpwssd m9, m14, m29 ; y2l + pshufb m30, m21 + vpdpwssd m%2, m13, m7 ; x1h + vpdpwssd m%1, m15, m7 ; x3l + pshufb m7, m8, m20 + vpdpwssd m6, m13, m30 ; y1h + vpdpwssd m9, m15, m30 ; y3l + pshufb m30, m31, m20 + vpdpwssd m%2, m14, m7 ; x2h + pshufb m8, m21 + vpdpwssd m6, m14, m30 ; y2h + pshufb m31, m21 + vpdpwssd m%2, m15, m8 ; x3h + vpdpwssd m6, m15, m31 ; y3h +%if %1 == 1 + vpermt2b m0, m22, m%1 ; 01l + vpermt2b m23, m22, m%2 ; 01h +%endif + vpermt2b m%1, m22, m9 ; xyl + vpermt2b m%2, m22, m6 ; xyh +%endmacro + PUT_8TAP_HV_W32 1, 24, ssq, 1, 2 ; 12 + PUT_8TAP_HV_W32 3, 26, ssq, 0, 1 ; 34 + PUT_8TAP_HV_W32 5, 28, ssq, 2, 0 ; 56 + vpshrdd m2, m1, m3, 16 ; 23l + vpshrdd m25, m24, m26, 16 ; 23h + vpshrdd m4, m3, m5, 16 ; 45l + vpshrdd m27, m26, m28, 16 ; 45h +.hv_w32_loop: + movu m7, [srcq+ssq*1+ 0] + movu m9, [srcq+ssq*2+ 0] + movu m6, [srcq+ssq*1+ 8] + movu m8, [srcq+ssq*2+ 8] + mova m29, m10 + mova m31, m10 + pshufb m30, m7, m20 + vpdpwssd m29, m12, m30 ; h0l + pshufb m30, m9, m20 + vpdpwssd m31, m12, m30 ; i0l + pshufb m7, m21 + vpdpwssd m29, m13, m7 ; h1l + pshufb m9, m21 + vpdpwssd m31, m13, m9 ; i1l + pshufb m7, m6, m20 + vpdpwssd m29, m14, m7 ; h2l + pshufb m9, m8, m20 + vpdpwssd m31, m14, m9 ; i2l + pshufb m6, m21 + vpdpwssd m29, m15, m6 ; h3l + pshufb m8, m21 + vpdpwssd m31, m15, m8 ; i3l + mova m30, m10 + vpdpwssd m30, m12, m7 ; h0h + movu m7, [srcq+ssq*1+16] + lea srcq, [srcq+ssq*2] + vpermt2b m29, m22, m31 ; 78l + mova m31, m10 + vpdpwssd m31, m12, m9 ; i0h + movu m9, [srcq+ssq*0+16] + vpdpwssd m30, m13, m6 ; h1h + pshufb m6, m7, m20 + vpdpwssd m31, m13, m8 ; i1h + pshufb m8, m9, m20 + vpdpwssd m30, m14, m6 ; h2h + pmaddwd m6, m16, m0 ; A0l + pshufb m7, m21 + vpdpwssd m31, m14, m8 ; i2h + pmaddwd m8, m16, m23 ; A0h + pshufb m9, m21 + vpdpwssd m30, m15, m7 ; h3h + pmaddwd m7, m16, m1 ; B0l + vpdpwssd m31, m15, m9 ; i3h + pmaddwd m9, m16, m24 ; B0h + mova m0, m2 + vpdpwssd m6, m17, m2 ; A1l + mova m23, m25 + vpdpwssd m8, m17, m25 ; A1h + mova m1, m3 + vpdpwssd m7, m17, m3 ; B1l + mova m24, m26 + vpdpwssd m9, m17, m26 ; B1h + vpermt2b m30, m22, m31 ; 78h + vpdpwssd m6, m18, m4 ; A2l + mova m2, m4 + vpdpwssd m8, m18, m27 ; A2h + mova m25, m27 + vpdpwssd m7, m18, m5 ; B2l + mova m3, m5 + vpdpwssd m9, m18, m28 ; B2h + mova m26, m28 + vpshrdd m4, m5, m29, 16 ; 67l + vpdpwssd m6, m19, m4 ; A3l + vpshrdd m27, m28, m30, 16 ; 67h + vpdpwssd m8, m19, m27 ; A3h + mova m5, m29 + vpdpwssd m7, m19, m29 ; B3l + mova m28, m30 + vpdpwssd m9, m19, m30 ; B3h + REPX {psrad x, 10}, m6, m8, m7, m9 + packusdw m6, m8 + packusdw m7, m9 + pminsw m6, m11 + pminsw m7, m11 + mova [dstq+dsq*0], m6 + mova [dstq+dsq*1], m7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w32_loop + add r7, 64 + add r8, 64 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 + jg .hv_w32_loop0 + RET + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +MC_8TAP_FN prep, sharp, SHARP, SHARP +MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH +MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP +MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH +MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR +MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP +MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR +MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH +MC_8TAP_FN prep, regular, REGULAR, REGULAR + +cglobal prep_8tap_16bpc, 3, 8, 16, tmp, src, stride, w, h, mx, my, stride3 +%define base r7-prep_avx512icl + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep_avx512icl] + mov wd, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + mov r5d, r7m ; bitdepth_max + vpbroadcastd m5, [pw_8192] + movzx wd, word [r7+wq*2+table_offset(prep,)] + shr r5d, 11 + vpbroadcastd m4, [r7-prep_avx512icl+prep_mul+r5*4] + add wq, r7 + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h_w4: + movzx mxd, mxb + sub srcq, 2 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + mov r5d, r7m + vbroadcasti32x4 m4, [spel_h_shufA] + vbroadcasti32x4 m5, [spel_h_shufB] + shr r5d, 11 + mova ym9, [prep_endA] + psllw xmm0, [base+prep_hv_shift+r5*8] + mova [tmpq], xmm0 + vpbroadcastd m6, [tmpq+4] + vpbroadcastd m7, [tmpq+8] +.h_w4_loop: + movu xm2, [srcq+strideq*0] + vinserti32x4 ym2, [srcq+strideq*1], 1 + vinserti32x4 m2, [srcq+strideq*2], 2 + vinserti32x4 m2, [srcq+r6 ], 3 + lea srcq, [srcq+strideq*4] + mova m0, m10 + pshufb m1, m2, m4 + vpdpwssd m0, m6, m1 + pshufb m2, m5 + vpdpwssd m0, m7, m2 + vpermb m0, m9, m0 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m10, [prep_8tap_rnd] + lea r6, [strideq*3] + cmp wd, 4 + je .h_w4 + shr mxd, 16 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + mov r5d, r7m + sub srcq, 6 + shr r5d, 11 + psllw xmm0, [base+prep_hv_shift+r5*8] + mova [tmpq], xmm0 + vpbroadcastd m12, xmm0 + vpbroadcastd m13, [tmpq+ 4] + vpbroadcastd m14, [tmpq+ 8] + vpbroadcastd m15, [tmpq+12] + cmp wd, 16 + je .h_w16 + jg .h_w32 +.h_w8: + mova m6, [spel_h_shufA] + movu m7, [spel_h_shufB] + movu m8, [spel_h_shufC] + mova m9, [spel_h_shufD] + mova m11, [prep_endB] +.h_w8_loop: + movu ym4, [srcq+strideq*0] + vinserti32x8 m4, [srcq+strideq*1], 1 + movu ym5, [srcq+strideq*2] + vinserti32x8 m5, [srcq+r6 ], 1 + lea srcq, [srcq+strideq*4] + mova m0, m10 + mova m1, m10 + vpermb m2, m6, m4 + vpermb m3, m6, m5 + vpdpwssd m0, m12, m2 + vpdpwssd m1, m12, m3 + vpermb m2, m7, m4 + vpermb m3, m7, m5 + vpdpwssd m0, m13, m2 + vpdpwssd m1, m13, m3 + vpermb m2, m8, m4 + vpermb m3, m8, m5 + vpdpwssd m0, m14, m2 + vpdpwssd m1, m14, m3 + vpermb m2, m9, m4 + vpermb m3, m9, m5 + vpdpwssd m0, m15, m2 + vpdpwssd m1, m15, m3 + vpermt2b m0, m11, m1 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + vbroadcasti32x4 m6, [spel_h_shufA] + vbroadcasti32x4 m7, [spel_h_shufB] + mova m11, [prep_endC] +.h_w16_loop: + movu ym2, [srcq+strideq*0+ 0] + vinserti32x8 m2, [srcq+strideq*1+ 0], 1 + movu ym3, [srcq+strideq*0+16] + vinserti32x8 m3, [srcq+strideq*1+16], 1 + lea srcq, [srcq+strideq*2] + mova m0, m10 + mova m1, m10 + pshufb m4, m2, m6 + vpdpwssd m0, m12, m4 ; a0 + pshufb m4, m3, m6 + vpdpwssd m1, m14, m4 ; b2 + pshufb m4, m2, m7 + vpdpwssd m0, m13, m4 ; a1 + pshufb m4, m3, m7 + vpdpwssd m1, m15, m4 ; b3 + shufpd m2, m3, 0x55 + pshufb m4, m2, m6 + vpdpwssd m0, m14, m4 ; a2 + vpdpwssd m1, m12, m4 ; b0 + pshufb m2, m7 + vpdpwssd m0, m15, m2 ; a3 + vpdpwssd m1, m13, m2 ; b1 + vpermt2b m0, m11, m1 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 2 + jg .h_w16_loop + RET +.h_w32: + vbroadcasti32x4 m6, [spel_h_shufA] + lea srcq, [srcq+wq*2] + vbroadcasti32x4 m7, [spel_h_shufB] + neg wq + mova m11, [prep_endC] +.h_w32_loop0: + mov r6, wq +.h_w32_loop: + movu m2, [srcq+r6*2+ 0] + movu m3, [srcq+r6*2+ 8] + mova m0, m10 + mova m1, m10 + pshufb m4, m2, m6 + vpdpwssd m0, m12, m4 ; a0 + pshufb m4, m3, m6 + vpdpwssd m1, m12, m4 ; b0 + vpdpwssd m0, m14, m4 ; a2 + movu m4, [srcq+r6*2+16] + pshufb m3, m7 + vpdpwssd m1, m13, m3 ; b1 + vpdpwssd m0, m15, m3 ; a3 + pshufb m3, m4, m6 + vpdpwssd m1, m14, m3 ; b2 + pshufb m2, m7 + vpdpwssd m0, m13, m2 ; a1 + pshufb m4, m7 + vpdpwssd m1, m15, m4 ; b3 + vpermt2b m0, m11, m1 + mova [tmpq], m0 + add tmpq, 64 + add r6, 32 + jl .h_w32_loop + add srcq, strideq + dec hd + jg .h_w32_loop0 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + mov r5d, r7m + vpbroadcastd m10, [prep_8tap_rnd] + pmovsxbw xmm0, [base+subpel_filters+myq*8] + tzcnt r6d, wd + shr r5d, 11 + movzx r6d, word [r7+r6*2+table_offset(prep, _8tap_v)] + psllw xmm0, [base+prep_hv_shift+r5*8] + add r7, r6 + lea r6, [strideq*3] + sub srcq, r6 + mova [tmpq], xmm0 + vpbroadcastd m12, xmm0 + vpbroadcastd m13, [tmpq+ 4] + vpbroadcastd m14, [tmpq+ 8] + vpbroadcastd m15, [tmpq+12] + jmp r7 +.v_w4: + movq xmm1, [srcq+strideq*0] + vpbroadcastq ymm0, [srcq+strideq*1] + vpbroadcastq ymm2, [srcq+strideq*2] + add srcq, r6 + vpbroadcastq ymm4, [srcq+strideq*0] + vpbroadcastq ymm3, [srcq+strideq*1] + vpbroadcastq ymm5, [srcq+strideq*2] + mova xm11, [prep_endA] + add srcq, r6 + vpblendd ymm1, ymm0, 0x30 + vpblendd ymm0, ymm2, 0x30 + punpcklwd ymm1, ymm0 ; 01 12 + vpbroadcastq ymm0, [srcq+strideq*0] + vpblendd ymm2, ymm4, 0x30 + vpblendd ymm4, ymm3, 0x30 + punpcklwd ymm2, ymm4 ; 23 34 + vpblendd ymm3, ymm5, 0x30 + vpblendd ymm5, ymm0, 0x30 + punpcklwd ymm3, ymm5 ; 45 56 +.v_w4_loop: + vpbroadcastq ymm5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + mova ymm4, ym10 + vpdpwssd ymm4, ym12, ymm1 ; a0 b0 + mova ymm1, ymm2 + vpdpwssd ymm4, ym13, ymm2 ; a1 b1 + mova ymm2, ymm3 + vpdpwssd ymm4, ym14, ymm3 ; a2 b2 + vpblendd ymm3, ymm0, ymm5, 0x30 + vpbroadcastq ymm0, [srcq+strideq*0] + vpblendd ymm5, ymm0, 0x30 + punpcklwd ymm3, ymm5 ; 67 78 + vpdpwssd ymm4, ym15, ymm3 ; a3 b3 + vpermb ymm4, ym11, ymm4 + mova [tmpq], xmm4 + add tmpq, 16 + sub hd, 2 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + vbroadcasti32x4 m2, [srcq+strideq*2] + vinserti32x4 m1, m2, [srcq+strideq*0], 0 + vinserti32x4 m1, [srcq+strideq*1], 1 ; 0 1 2 + add srcq, r6 + vinserti32x4 ym2, [srcq+strideq*0], 1 + vinserti32x4 m2, [srcq+strideq*1], 2 ; 2 3 4 + mova m6, [spel_v_shuf8] + movu xm0, [srcq+strideq*1] + vinserti32x4 ym0, [srcq+strideq*2], 1 + add srcq, r6 + vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6 + mova ym11, [prep_endB] + vpermb m1, m6, m1 ; 01 12 + vpermb m2, m6, m2 ; 23 34 + vpermb m3, m6, m0 ; 45 56 +.v_w8_loop: + vinserti32x4 m0, [srcq+strideq*1], 3 + lea srcq, [srcq+strideq*2] + movu xm5, [srcq+strideq*0] + mova m4, m10 + vpdpwssd m4, m12, m1 ; a0 b0 + mova m1, m2 + vshufi32x4 m0, m5, q1032 ; 6 7 8 + vpdpwssd m4, m13, m2 ; a1 b1 + mova m2, m3 + vpdpwssd m4, m14, m3 ; a2 b2 + vpermb m3, m6, m0 ; 67 78 + vpdpwssd m4, m15, m3 ; a3 b3 + vpermb m4, m11, m4 + mova [tmpq], ym4 + add tmpq, 32 + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + vbroadcasti32x8 m1, [srcq+strideq*1] + vinserti32x8 m0, m1, [srcq+strideq*0], 0 + vinserti32x8 m1, [srcq+strideq*2], 1 + mova m8, [spel_v_shuf16] + add srcq, r6 + movu ym3, [srcq+strideq*0] + vinserti32x8 m3, [srcq+strideq*1], 1 + movu ym5, [srcq+strideq*2] + add srcq, r6 + vinserti32x8 m5, [srcq+strideq*0], 1 + mova m11, [prep_endA] + vpermb m0, m8, m0 ; 01 + vpermb m1, m8, m1 ; 12 + vpermb m3, m8, m3 ; 34 + vpermb m5, m8, m5 ; 56 + vpshrdd m2, m1, m3, 16 ; 23 + vpshrdd m4, m3, m5, 16 ; 45 +.v_w16_loop: + mova m6, m10 + mova m7, m10 + vpdpwssd m6, m12, m0 ; a0 + mova m0, m2 + vpdpwssd m7, m12, m1 ; b0 + mova m1, m3 + vpdpwssd m6, m13, m2 ; a1 + mova m2, m4 + vpdpwssd m7, m13, m3 ; b1 + mova m3, m5 + vpdpwssd m6, m14, m4 ; a2 + mova m4, m5 + vpdpwssd m7, m14, m5 ; b2 + movu ym5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti32x8 m5, [srcq+strideq*0], 1 + vpermb m5, m8, m5 ; 78 + vpshrdd m4, m5, 16 ; 67 + vpdpwssd m6, m15, m4 ; a3 + vpdpwssd m7, m15, m5 ; b3 + vpermt2b m6, m11, m7 + mova [tmpq], m6 + add tmpq, 64 + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: +.v_w64: +.v_w128: +%if WIN64 + PUSH r8 + movaps [rsp+stack_offset+8], xmm6 +%endif + lea r5, [hq+wq*8-256] + mov r7, srcq + mov r8, tmpq +.v_w32_loop0: + movu m16, [srcq+strideq*0] + movu m17, [srcq+strideq*1] + movu m18, [srcq+strideq*2] + add srcq, r6 + movu m19, [srcq+strideq*0] + movu m20, [srcq+strideq*1] + movu m21, [srcq+strideq*2] + add srcq, r6 + movu m22, [srcq+strideq*0] + mova m11, [prep_endC] + punpcklwd m0, m16, m17 ; 01l + punpckhwd m16, m17 ; 01h + punpcklwd m1, m17, m18 ; 12l + punpckhwd m17, m18 ; 12h + punpcklwd m2, m18, m19 ; 23l + punpckhwd m18, m19 ; 23h + punpcklwd m3, m19, m20 ; 34l + punpckhwd m19, m20 ; 34h + punpcklwd m4, m20, m21 ; 45l + punpckhwd m20, m21 ; 45h + punpcklwd m5, m21, m22 ; 56l + punpckhwd m21, m22 ; 56h +.v_w32_loop: + mova m6, m10 + vpdpwssd m6, m12, m0 ; a0l + mova m8, m10 + vpdpwssd m8, m12, m16 ; a0h + mova m7, m10 + vpdpwssd m7, m12, m1 ; b0l + mova m9, m10 + vpdpwssd m9, m12, m17 ; b0h + mova m0, m2 + vpdpwssd m6, m13, m2 ; a1l + mova m16, m18 + vpdpwssd m8, m13, m18 ; a1h + mova m1, m3 + vpdpwssd m7, m13, m3 ; b1l + mova m17, m19 + vpdpwssd m9, m13, m19 ; b1h + mova m2, m4 + vpdpwssd m6, m14, m4 ; a2l + mova m18, m20 + vpdpwssd m8, m14, m20 ; a2h + mova m3, m5 + vpdpwssd m7, m14, m5 ; b2l + mova m19, m21 + vpdpwssd m9, m14, m21 ; b2h + movu m21, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklwd m4, m22, m21 ; 67l + punpckhwd m20, m22, m21 ; 67h + movu m22, [srcq+strideq*0] + vpdpwssd m6, m15, m4 ; a3l + vpdpwssd m8, m15, m20 ; a3h + punpcklwd m5, m21, m22 ; 78l + punpckhwd m21, m22 ; 78h + vpdpwssd m7, m15, m5 ; b3l + vpdpwssd m9, m15, m21 ; b3h + vpermt2b m6, m11, m8 + vpermt2b m7, m11, m9 + mova [tmpq+wq*0], m6 + mova [tmpq+wq*2], m7 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w32_loop + add r7, 64 + add r8, 64 + movzx hd, r5b + mov srcq, r7 + mov tmpq, r8 + sub r5d, 1<<8 + jg .v_w32_loop0 +%if WIN64 + movaps xmm6, [rsp+stack_offset+8] + POP r8 +%endif + vzeroupper + RET +.hv: + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + mov r5d, r7m + pmovsxbw xmm1, [base+subpel_filters+myq*8] + lea r6, [strideq*3] + sub srcq, 2 + shr r5d, 11 + sub srcq, r6 + psllw xmm0, [base+prep_hv_shift+r5*8] + psllw xmm1, 2 + vpbroadcastd m10, [prep_8tap_rnd] + vpbroadcastd ym11, [pd_128] + mova xm21, [prep_endA] + mova [tmpq+ 0], xmm0 + mova [tmpq+16], xmm1 + vpbroadcastd m8, [tmpq+ 4] + vpbroadcastd m9, [tmpq+ 8] + vpbroadcastd ym12, xmm1 + vpbroadcastd ym13, [tmpq+20] + vpbroadcastd ym14, [tmpq+24] + vpbroadcastd ym15, [tmpq+28] + movu xm4, [srcq+strideq*0] + vinserti32x4 ym4, [srcq+strideq*1], 1 + vinserti32x4 m4, [srcq+strideq*2], 2 + add srcq, r6 + vinserti32x4 m4, [srcq+strideq*0], 3 ; 0 1 2 3 + movu xm0, [srcq+strideq*1] + vinserti32x4 ym0, [srcq+strideq*2], 1 + add srcq, r6 + vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6 + vbroadcasti32x4 m19, [spel_h_shufA] + vbroadcasti32x4 m20, [spel_h_shufB] + mova ym6, [spel_shuf4a] + mova ym7, [spel_shuf4b] + mova m2, m10 + mova m3, m10 + pshufb m1, m4, m19 + vpdpwssd m2, m8, m1 + pshufb m1, m0, m19 + vpdpwssd m3, m8, m1 + pshufb m4, m20 + vpdpwssd m2, m9, m4 + pshufb m0, m20 + vpdpwssd m3, m9, m0 + vpermb m1, m6, m2 ; 01 12 + vshufi32x4 m2, m3, q1032 + vpermb m3, m6, m3 ; 45 56 + vpermb m2, m6, m2 ; 23 34 +.hv_w4_loop: + movu xm18, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti128 ym18, [srcq+strideq*0], 1 + mova ym16, ym11 + mova ym4, ym10 + pshufb ym17, ym18, ym19 + vpdpwssd ym16, ym12, ym1 ; a0 b0 + vpdpwssd ym4, ym8, ym17 + pshufb ym18, ym20 + mova ym1, ym2 + vpdpwssd ym16, ym13, ym2 ; a1 b1 + vpdpwssd ym4, ym9, ym18 ; 7 8 + mova ym2, ym3 + vpdpwssd ym16, ym14, ym3 ; a2 b2 + vpermt2b ym3, ym7, ym4 ; 67 78 + vpdpwssd ym16, ym15, ym3 ; a3 b3 + vpermb ym16, ym21, ym16 + mova [tmpq], xm16 + add tmpq, 16 + sub hd, 2 + jg .hv_w4_loop + vzeroupper + RET +.hv_w8: + shr mxd, 16 + pmovsxbw xmm0, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + mov r5d, r7m + pmovsxbw xmm1, [base+subpel_filters+myq*8] + lea r6, [strideq*3] + sub srcq, 6 + shr r5d, 11 + sub srcq, r6 + vpbroadcastd m10, [prep_8tap_rnd] + vpbroadcastd m11, [pd_128] + psllw xmm0, [base+prep_hv_shift+r5*8] + psllw xmm1, 2 + mova [tmpq+ 0], xmm0 + mova [tmpq+16], xmm1 + vpbroadcastd m12, xmm0 + vpbroadcastd m13, [tmpq+ 4] + vpbroadcastd m14, [tmpq+ 8] + vpbroadcastd m15, [tmpq+12] + vpbroadcastd m16, xmm1 + vpbroadcastd m17, [tmpq+20] + vpbroadcastd m18, [tmpq+24] + vpbroadcastd m19, [tmpq+28] + cmp wd, 16 + je .hv_w16 + jg .hv_w32 + WIN64_SPILL_XMM 23 + mova m5, [spel_h_shufA] + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 ; 0 1 + movu ym9, [srcq+strideq*2] + add srcq, r6 + vinserti32x8 m9, [srcq+strideq*0], 1 ; 2 3 + movu ym20, [srcq+strideq*1] + vinserti32x8 m20, [srcq+strideq*2], 1 ; 4 5 + add srcq, r6 + movu ym21, [srcq+strideq*0] ; 6 + movu m6, [spel_h_shufB] + movu m7, [spel_h_shufC] + mova ym22, [prep_endB] + vpermb m8, m5, m0 + mova m1, m10 + vpdpwssd m1, m12, m8 ; a0 b0 + vpermb m8, m5, m9 + mova m2, m10 + vpdpwssd m2, m12, m8 ; c0 d0 + vpermb m8, m5, m20 + mova m3, m10 + vpdpwssd m3, m12, m8 ; e0 f0 + vpermb m8, m5, m21 + mova m4, m10 + vpdpwssd m4, m12, m8 ; g0 + vpermb m8, m6, m0 + vpdpwssd m1, m13, m8 ; a1 b1 + vpermb m8, m6, m9 + vpdpwssd m2, m13, m8 ; c1 d1 + vpermb m8, m6, m20 + vpdpwssd m3, m13, m8 ; e1 f1 + vpermb m8, m6, m21 + vpdpwssd m4, m13, m8 ; g1 + vpermb m8, m7, m0 + vpdpwssd m1, m14, m8 ; a2 b2 + vpermb m8, m7, m9 + vpdpwssd m2, m14, m8 ; c2 d2 + vpermb m8, m7, m20 + vpdpwssd m3, m14, m8 ; e2 f2 + vpermb m8, m7, m21 + vpdpwssd m4, m14, m8 ; g2 + mova m8, [spel_h_shufD] + vpermb m0, m8, m0 + vpdpwssd m1, m15, m0 ; a3 b3 + mova m0, [spel_shuf8a] + vpermb m9, m8, m9 + vpdpwssd m2, m15, m9 ; c3 d3 + mova m9, [spel_shuf8b] + vpermb m20, m8, m20 + vpdpwssd m3, m15, m20 ; e3 f3 + vpermb m21, m8, m21 + vpdpwssd m4, m15, m21 ; g3 + vpermt2b m1, m0, m2 ; 01 12 + vpermt2b m2, m0, m3 ; 23 34 + vpermt2b m3, m0, m4 ; 45 56 +.hv_w8_loop: + movu ym0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti32x8 m0, [srcq+strideq*0], 1 + mova m4, m10 + mova m20, m11 + vpermb m21, m5, m0 + vpdpwssd m4, m12, m21 ; h0 i0 + vpermb m21, m6, m0 + vpdpwssd m20, m16, m1 ; A0 B0 + vpdpwssd m4, m13, m21 ; h1 i1 + vpermb m21, m7, m0 + mova m1, m2 + vpdpwssd m20, m17, m2 ; A1 B1 + vpdpwssd m4, m14, m21 ; h2 i2 + vpermb m21, m8, m0 + mova m2, m3 + vpdpwssd m20, m18, m3 ; A2 B2 + vpdpwssd m4, m15, m21 ; h3 i3 + vpermt2b m3, m9, m4 ; 67 78 + vpdpwssd m20, m19, m3 ; A3 B3 + vpermb m20, m22, m20 + mova [tmpq], ym20 + add tmpq, 32 + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 27 + vbroadcasti32x8 m5, [srcq+strideq*0+ 8] + vinserti32x8 m4, m5, [srcq+strideq*0+ 0], 0 + vinserti32x8 m5, [srcq+strideq*0+16], 1 ; 0 + movu ym6, [srcq+strideq*1+ 0] + movu ym7, [srcq+strideq*1+16] + vinserti32x8 m6, [srcq+strideq*2+ 0], 1 + vinserti32x8 m7, [srcq+strideq*2+16], 1 ; 1 2 + add srcq, r6 + movu ym22, [srcq+strideq*0+ 0] + movu ym23, [srcq+strideq*0+16] + vinserti32x8 m22, [srcq+strideq*1+ 0], 1 + vinserti32x8 m23, [srcq+strideq*1+16], 1 ; 3 4 + movu ym24, [srcq+strideq*2+ 0] + movu ym25, [srcq+strideq*2+16] + add srcq, r6 + vinserti32x8 m24, [srcq+strideq*0+ 0], 1 + vinserti32x8 m25, [srcq+strideq*0+16], 1 ; 5 6 + vbroadcasti32x4 m20, [spel_h_shufA] + vbroadcasti32x4 m21, [spel_h_shufB] + mova m9, [spel_shuf16] + mova m26, [prep_endB] + pshufb m0, m4, m20 + mova m1, m10 + vpdpwssd m1, m12, m0 ; a0 + pshufb m0, m6, m20 + mova m2, m10 + vpdpwssd m2, m12, m0 ; b0 + pshufb m0, m7, m20 + mova m3, m10 + vpdpwssd m3, m14, m0 ; c2 + pshufb m0, m4, m21 + vpdpwssd m1, m13, m0 ; a1 + pshufb m0, m6, m21 + vpdpwssd m2, m13, m0 ; b1 + pshufb m0, m7, m21 + vpdpwssd m3, m15, m0 ; c3 + pshufb m0, m5, m20 + vpdpwssd m1, m14, m0 ; a2 + shufpd m6, m7, 0x55 + pshufb m7, m6, m20 + vpdpwssd m2, m14, m7 ; b2 + vpdpwssd m3, m12, m7 ; c0 + pshufb m5, m21 + vpdpwssd m1, m15, m5 ; a3 + pshufb m6, m21 + vpdpwssd m2, m15, m6 ; b3 + vpdpwssd m3, m13, m6 ; c1 + pshufb m0, m22, m20 + mova m4, m10 + vpdpwssd m4, m12, m0 ; d0 + pshufb m0, m23, m20 + mova m5, m10 + vpdpwssd m5, m14, m0 ; e2 + pshufb m0, m24, m20 + mova m6, m10 + vpdpwssd m6, m12, m0 ; f0 + pshufb m0, m25, m20 + mova m7, m10 + vpdpwssd m7, m14, m0 ; g2 + pshufb m0, m22, m21 + vpdpwssd m4, m13, m0 ; d1 + pshufb m0, m23, m21 + vpdpwssd m5, m15, m0 ; e3 + pshufb m0, m24, m21 + vpdpwssd m6, m13, m0 ; f1 + pshufb m0, m25, m21 + vpdpwssd m7, m15, m0 ; g3 + shufpd m22, m23, 0x55 + pshufb m23, m22, m20 + vpdpwssd m4, m14, m23 ; d2 + vpdpwssd m5, m12, m23 ; e0 + shufpd m24, m25, 0x55 + pshufb m25, m24, m20 + vpdpwssd m6, m14, m25 ; f2 + vpdpwssd m7, m12, m25 ; g0 + pshufb m22, m21 + vpdpwssd m4, m15, m22 ; d3 + vpdpwssd m5, m13, m22 ; e1 + pshufb m24, m21 + vpdpwssd m6, m15, m24 ; f3 + vpdpwssd m7, m13, m24 ; g1 + pslldq m1, 1 + vpermt2b m2, m9, m3 ; 12 + vpermt2b m4, m9, m5 ; 34 + vpermt2b m6, m9, m7 ; 56 + vpshrdd m1, m2, 16 ; 01 + vpshrdd m3, m2, m4, 16 ; 23 + vpshrdd m5, m4, m6, 16 ; 45 +.hv_w16_loop: + movu ym24, [srcq+strideq*1+ 0] + movu ym25, [srcq+strideq*1+16] + lea srcq, [srcq+strideq*2] + vinserti32x8 m24, [srcq+strideq*0+ 0], 1 + vinserti32x8 m25, [srcq+strideq*0+16], 1 + mova m7, m10 + mova m8, m10 + pshufb m0, m24, m20 + vpdpwssd m7, m12, m0 ; h0 + mova m22, m11 + pshufb m0, m25, m20 + vpdpwssd m8, m14, m0 ; i2 + mova m23, m11 + vpdpwssd m22, m16, m1 ; A0 + mova m1, m3 + vpdpwssd m23, m16, m2 ; B0 + mova m2, m4 + pshufb m0, m24, m21 + vpdpwssd m7, m13, m0 ; h1 + pshufb m0, m25, m21 + vpdpwssd m8, m15, m0 ; i3 + vpdpwssd m22, m17, m3 ; A1 + mova m3, m5 + vpdpwssd m23, m17, m4 ; B1 + mova m4, m6 + shufpd m24, m25, 0x55 + pshufb m25, m24, m20 + vpdpwssd m7, m14, m25 ; h2 + vpdpwssd m8, m12, m25 ; i0 + vpdpwssd m22, m18, m5 ; A2 + vpdpwssd m23, m18, m6 ; B2 + pshufb m24, m21 + vpdpwssd m7, m15, m24 ; h3 + vpdpwssd m8, m13, m24 ; i1 + vpermt2b m7, m9, m8 ; 78 + vpshrdd m5, m6, m7, 16 ; 67 + vpdpwssd m22, m19, m5 ; A3 + vpdpwssd m23, m19, m7 ; B3 + mova m6, m7 + vpermt2b m22, m26, m23 + mova [tmpq], m22 + add tmpq, 64 + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: +%if WIN64 + %assign stack_offset stack_offset - stack_size_padded + PUSH r8 + %assign regs_used regs_used + 1 + WIN64_SPILL_XMM 32 +%endif + vbroadcasti32x4 m20, [spel_h_shufA] + vbroadcasti32x4 m21, [spel_h_shufB] + mova m22, [spel_shuf32] + lea r5d, [hq+wq*8-256] + mov r7, srcq + mov r8, tmpq +.hv_w32_loop0: + movu m6, [srcq+strideq*0+ 0] + movu m7, [srcq+strideq*0+ 8] + movu m8, [srcq+strideq*0+16] + mova m0, m10 + mova m23, m10 + pshufb m9, m6, m20 + vpdpwssd m0, m12, m9 ; a0l + pshufb m9, m7, m20 + vpdpwssd m23, m12, m9 ; a0h + vpdpwssd m0, m14, m9 ; a2l + pshufb m7, m21 + vpdpwssd m23, m13, m7 ; a1h + vpdpwssd m0, m15, m7 ; a3l + pshufb m7, m8, m20 + vpdpwssd m23, m14, m7 ; a2h + pshufb m6, m21 + vpdpwssd m0, m13, m6 ; a1l + pshufb m8, m21 + vpdpwssd m23, m15, m8 ; a3h + PUT_8TAP_HV_W32 1, 24, strideq, 1, 2 ; 12 + PUT_8TAP_HV_W32 3, 26, strideq, 0, 1 ; 34 + PUT_8TAP_HV_W32 5, 28, strideq, 2, 0 ; 56 + vpshrdd m2, m1, m3, 16 ; 23l + vpshrdd m25, m24, m26, 16 ; 23h + vpshrdd m4, m3, m5, 16 ; 45l + vpshrdd m27, m26, m28, 16 ; 45h +.hv_w32_loop: + movu m7, [srcq+strideq*1+ 0] + movu m9, [srcq+strideq*2+ 0] + movu m6, [srcq+strideq*1+ 8] + movu m8, [srcq+strideq*2+ 8] + mova m29, m10 + mova m31, m10 + pshufb m30, m7, m20 + vpdpwssd m29, m12, m30 ; h0l + pshufb m30, m9, m20 + vpdpwssd m31, m12, m30 ; i0l + pshufb m7, m21 + vpdpwssd m29, m13, m7 ; h1l + pshufb m9, m21 + vpdpwssd m31, m13, m9 ; i1l + pshufb m7, m6, m20 + vpdpwssd m29, m14, m7 ; h2l + pshufb m9, m8, m20 + vpdpwssd m31, m14, m9 ; i2l + pshufb m6, m21 + vpdpwssd m29, m15, m6 ; h3l + pshufb m8, m21 + vpdpwssd m31, m15, m8 ; i3l + mova m30, m10 + vpdpwssd m30, m12, m7 ; h0h + movu m7, [srcq+strideq*1+16] + lea srcq, [srcq+strideq*2] + vpermt2b m29, m22, m31 ; 78l + mova m31, m10 + vpdpwssd m31, m12, m9 ; i0h + movu m9, [srcq+strideq*0+16] + vpdpwssd m30, m13, m6 ; h1h + pshufb m6, m7, m20 + vpdpwssd m31, m13, m8 ; i1h + pshufb m8, m9, m20 + vpdpwssd m30, m14, m6 ; h2h + mova m6, m11 + vpdpwssd m6, m16, m0 ; A0l + pshufb m7, m21 + vpdpwssd m31, m14, m8 ; i2h + mova m8, m11 + vpdpwssd m8, m16, m23 ; A0h + pshufb m9, m21 + vpdpwssd m30, m15, m7 ; h3h + mova m7, m11 + vpdpwssd m7, m16, m1 ; B0l + vpdpwssd m31, m15, m9 ; i3h + mova m9, m11 + vpdpwssd m9, m16, m24 ; B0h + mova m0, m2 + vpdpwssd m6, m17, m2 ; A1l + mova m23, m25 + vpdpwssd m8, m17, m25 ; A1h + mova m1, m3 + vpdpwssd m7, m17, m3 ; B1l + mova m24, m26 + vpdpwssd m9, m17, m26 ; B1h + vpermt2b m30, m22, m31 ; 78h + mova m31, [prep_endC] + vpdpwssd m6, m18, m4 ; A2l + mova m2, m4 + vpdpwssd m8, m18, m27 ; A2h + mova m25, m27 + vpdpwssd m7, m18, m5 ; B2l + mova m3, m5 + vpdpwssd m9, m18, m28 ; B2h + mova m26, m28 + vpshrdd m4, m5, m29, 16 ; 67l + vpdpwssd m6, m19, m4 ; A3l + vpshrdd m27, m28, m30, 16 ; 67h + vpdpwssd m8, m19, m27 ; A3h + mova m5, m29 + vpdpwssd m7, m19, m29 ; B3l + mova m28, m30 + vpdpwssd m9, m19, m30 ; B3h + vpermt2b m6, m31, m8 + vpermt2b m7, m31, m9 + mova [tmpq+wq*0], m6 + mova [tmpq+wq*2], m7 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .hv_w32_loop + add r7, 64 + add r8, 64 + movzx hd, r5b + mov srcq, r7 + mov tmpq, r8 + sub r5d, 1<<8 + jg .hv_w32_loop0 + RET + +%if WIN64 +DECLARE_REG_TMP 5 +%else +DECLARE_REG_TMP 7 +%endif + +cglobal warp_affine_8x8t_16bpc, 4, 7, 22, tmp, ts +%define base r6-pd_0to7 + mov t0d, r7m + lea r6, [pd_0to7] + shr t0d, 11 + vpbroadcastd m8, [base+warp_8x8t_rnd_v] + vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4] + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main + psrad m14, m16, 15 + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 + psrad m16, 15 + packssdw m14, m16 + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 + psrad m15, m16, 15 + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 + add tsq, tsq + psrad m16, 15 + packssdw m15, m16 + jmp mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).end + +cglobal warp_affine_8x8_16bpc, 4, 7, 22, dst, ds, src, ss, abcd + mov t0d, r7m ; pixel_max + lea r6, [pd_0to7] + shr t0d, 11 + vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4] + vpbroadcastd m8, [base+warp_8x8_rnd_v+t0*4] + call .main + psrad m14, m16, 13 + call .main2 + psrad m16, 13 + packusdw m14, m16 + call .main2 + psrad m15, m16, 13 + call .main2 + vpbroadcastd m0, [base+bidir_shift+t0*4] + vpsrlvw m14, m0 + psrad m16, 13 + packusdw m15, m16 + vpsrlvw m15, m0 +.end: + mova m0, [base+warp8x8_end] + vpermb m16, m0, m14 + lea r2, [dsq*3] + mova [dstq+dsq*0], xm16 + vextracti128 [dstq+dsq*1], ym16, 1 + vextracti32x4 [dstq+dsq*2], m16, 2 + vextracti32x4 [dstq+r2 ], m16, 3 + vpermb m16, m0, m15 + lea dstq, [dstq+dsq*4] + mova [dstq+dsq*0], xm16 + vextracti128 [dstq+dsq*1], ym16, 1 + vextracti32x4 [dstq+dsq*2], m16, 2 + vextracti32x4 [dstq+r2 ], m16, 3 + RET +.main: + vpbroadcastd ym3, [base+pd_512] +%if WIN64 + mov abcdq, r5mp + vpaddd ym18, ym3, r6m {1to8} ; mx +%else + add r5d, 512 + vpbroadcastd ym18, r5d +%endif + vpaddd ym20, ym3, r7m {1to8} ; my + mova ym16, [base+pd_0to7] + vpbroadcastd ym19, [abcdq+4*0] ; alpha + vpbroadcastd ym21, [abcdq+4*1] ; gamma + lea r4, [ssq*3+6] + vpdpwssd ym18, ym19, ym16 ; tmx + vpdpwssd ym20, ym21, ym16 ; tmy + sub srcq, r4 + mova m10, [base+warp8x8_permA] + lea r4, [mc_warp_filter+64*8] + vbroadcasti32x4 m12, [base+warp8x8_permC] + kxnorb k1, k1, k1 + vbroadcasti32x4 m13, [base+warp8x8_permD] + movu ym5, [srcq+0] + vinserti32x8 m5, [srcq+8], 1 + psrad ym17, ym18, 10 + mova m11, [base+warp8x8_permB] + kmovb k2, k1 + vpgatherdq m3{k1}, [r4+ym17*8] ; filter_x0 + psrad ym19, 16 ; beta + psrad ym21, 16 ; delta + paddd ym18, ym19 + vpermb m4, m10, m5 + vpbroadcastq m9, [base+warp_shift_h+t0*8] + pshufd m3, m3, q3120 + paddd m7, m1, m1 + pshufb m2, m3, m12 + vpdpwssd m1, m4, m2 + vpermb m5, m11, m5 + vshufi32x4 m4, m5, q1021 + pshufb m3, m13 + vpdpwssd m1, m4, m3 + call .h + psllq m2, m1, 32 + paddd m1, m2 + vpmultishiftqb m1, m9, m1 + vpshrdq m1, m0, 48 ; 01 12 + call .h + vpshrdq m2, m1, m0, 48 ; 23 34 + call .h + vpshrdq m3, m2, m0, 48 ; 45 56 +.main2: + call .h + psrad ym6, ym20, 10 + kmovb k1, k2 + paddd ym17, ym20, ym21 ; my += delta + vpgatherdq m20{k2}, [r4+ym6*8] ; filter_y0 + psrad ym16, ym17, 10 + kmovb k2, k1 + vpgatherdq m6{k1}, [r4+ym16*8] ; filter_y1 + shufps m5, m20, m6, q2020 + mova m16, m8 + pshufb m4, m5, m12 + vpdpwssd m16, m1, m4 ; a0 b0 + pshufb m5, m13 + mova m1, m2 + vpdpwssd m16, m2, m5 ; a1 b1 + shufps m6, m20, m6, q3131 + paddd ym20, ym17, ym21 + pshufb m4, m6, m12 + mova m2, m3 + vpdpwssd m16, m3, m4 ; a2 b2 + vpshrdq m3, m0, 48 ; 67 78 + pshufb m6, m13 + vpdpwssd m16, m3, m6 ; a3 b3 + ret +ALIGN function_align +.h: + movu ym16, [srcq+ssq*1] + psrad ym6, ym18, 10 + lea srcq, [srcq+ssq*2] + vinserti32x8 m5, m16, [srcq+ssq*0], 1 + kmovb k1, k2 + paddd ym17, ym18, ym19 ; mx += beta + vpgatherdq m18{k2}, [r4+ym6*8] ; filter_x1 + psrad ym16, ym17, 10 + kmovb k2, k1 + vpgatherdq m6{k1}, [r4+ym16*8] ; filter_x2 + vpermb m4, m10, m5 + shufps m16, m18, m6, q2020 + shufps m6, m18, m6, q3131 + mova m0, m7 + pshufb m18, m16, m12 + vpdpwssd m0, m4, m18 ; a0 b0 + vpermb m5, m11, m5 + pshufb m18, m6, m13 + vpdpwssd m0, m5, m18 ; a3 b3 + paddd ym18, ym17, ym19 + vshufi32x4 m17, m4, m5, q1021 + pshufb m16, m13 + vpdpwssd m0, m17, m16 ; a1 b1 + vshufi32x4 m4, m5, q2132 + pshufb m6, m12 + vpdpwssd m0, m4, m6 ; a2 b2 + vpmultishiftqb m0, m9, m0 ; a a b b + ret + +%macro BIDIR_FN 0 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq ], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm2, ym0, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + cmp hd, 8 + jl .w4_end + vextracti32x4 xm2, m0, 2 + lea dstq, [dstq+strideq*4] + movq [dstq ], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm0, m0, 3 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq ], xm1 + movhps [dstq+strideq*1], xm1 + vextracti32x4 xm0, ym1, 1 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + vextracti32x4 xm0, m1, 2 + lea dstq, [dstq+strideq*4] + movq [dstq ], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm1, m1, 3 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] +.w8: + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm1 + vextracti32x4 [dstq+strideq*1], ym1, 1 + vextracti32x4 [dstq+strideq*2], m1, 2 + vextracti32x4 [dstq+stride3q ], m1, 3 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] +.w16: + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] +.w32: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + call .main + mova [dstq+64*2], m0 + mova [dstq+64*3], m1 + dec hd + jg .w128_loop + RET +%endmacro + +%if WIN64 +DECLARE_REG_TMP 5 +%else +DECLARE_REG_TMP 7 +%endif + +cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg_avx512icl_table + lea r6, [avg_avx512icl_table] + tzcnt wd, wm + mov t0d, r6m ; pixel_max + movsxd wq, [r6+wq*4] + shr t0d, 11 + vpbroadcastd m2, [base+avg_round+t0*4] + vpbroadcastd m3, [base+avg_shift+t0*4] + movifnidn hd, hm + add wq, r6 + BIDIR_FN +ALIGN function_align +.main: + mova m0, [tmp1q+64*0] + paddsw m0, [tmp2q+64*0] + mova m1, [tmp1q+64*1] + paddsw m1, [tmp2q+64*1] + add tmp1q, 64*2 + add tmp2q, 64*2 + pmaxsw m0, m2 + pmaxsw m1, m2 + psubsw m0, m2 + psubsw m1, m2 + vpsrlvw m0, m3 + vpsrlvw m1, m3 + ret + +cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-w_avg_avx512icl_table + lea r6, [w_avg_avx512icl_table] + tzcnt wd, wm + mov t0d, r7m ; pixel_max + shr t0d, 11 + movsxd wq, [r6+wq*4] + vpbroadcastd m5, [base+w_avg_round+t0*4] + vpbroadcastd m7, [base+bidir_shift+t0*4] + add wq, r6 + mov r6d, r6m ; weight + lea t0d, [r6-16] + shl r6d, 16 + sub r6d, t0d ; 16-weight, weight + movifnidn hd, hm + vpbroadcastd m6, r6d + BIDIR_FN +ALIGN function_align +.main: + mova m3, [tmp1q+64*0] + mova m1, [tmp2q+64*0] + mova m0, [tmp1q+64*1] + mova m4, [tmp2q+64*1] + add tmp1q, 64*2 + add tmp2q, 64*2 + punpcklwd m2, m1, m3 + punpckhwd m1, m3 + punpcklwd m3, m4, m0 + punpckhwd m4, m0 + mova m0, m5 + vpdpwssd m0, m6, m2 + mova m2, m5 + vpdpwssd m2, m6, m1 + mova m1, m5 + vpdpwssd m1, m6, m3 + mova m3, m5 + vpdpwssd m3, m6, m4 + REPX {psrad x, 2}, m0, m2, m1, m3 + packusdw m0, m2 + packusdw m1, m3 + vpsrlvw m0, m7 + vpsrlvw m1, m7 + ret + +cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask_avx512icl_table + lea r7, [mask_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m8, [base+pw_64] + vpbroadcastd m9, [base+mask_round+r6*4] + vpbroadcastd m10, [base+bidir_shift+r6*4] + mov maskq, maskmp + add wq, r7 + BIDIR_FN +ALIGN function_align +.main: + pmovzxbw m1, [maskq+32*0] + mova m4, [tmp1q+64*0] + mova m2, [tmp2q+64*0] + pmovzxbw m6, [maskq+32*1] + mova m5, [tmp1q+64*1] + mova m3, [tmp2q+64*1] + add maskq, 32*2 + add tmp1q, 64*2 + add tmp2q, 64*2 + punpcklwd m7, m4, m2 + punpckhwd m4, m2 + psubw m0, m8, m1 + punpcklwd m2, m1, m0 ; m, 64-m + punpckhwd m1, m0 + mova m0, m9 + vpdpwssd m0, m7, m2 + mova m2, m9 + vpdpwssd m2, m4, m1 ; tmp1 * m + tmp2 * (64-m) + punpcklwd m7, m5, m3 + punpckhwd m5, m3 + psubw m1, m8, m6 + punpcklwd m3, m6, m1 + punpckhwd m6, m1 + mova m1, m9 + vpdpwssd m1, m7, m3 + mova m3, m9 + vpdpwssd m3, m5, m6 + REPX {psrad x, 4}, m0, m2, m1, m3 + packusdw m0, m2 + packusdw m1, m3 + vpsrlvw m0, m10 + vpsrlvw m1, m10 + ret + +cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx512icl_table + lea r7, [w_mask_420_avx512icl_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + vpbroadcastd m11, [base+pw_64] + vpbroadcastd m12, [base+mask_round+r6*4] + vpbroadcastd m13, [base+bidir_shift+r6*4] + mov r6d, r7m ; sign + vpbroadcastd m14, [base+w_mask_round+r6*4] + mova ym15, [w_mask_end42x] + mov maskq, maskmp + add wq, r7 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + mova m4, [w_mask_shuf4] + vpermt2b m2, m4, m3 + mova m3, m14 + vpdpbusd m3, m2, [pb_64] {1to16} + vpermb m3, m15, m3 + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm2, ym0, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + mova [maskq], xm3 + cmp hd, 8 + jl .w4_end + vextracti32x4 xm2, m0, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm0, m0, 3 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti32x4 xm2, ym1, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + vextracti32x4 xm2, m1, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm1, m1, 3 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 +.w4_end: + RET +.w8: + mova m8, [w_mask_shuf8] + vpbroadcastd m9, [pb_64] + jmp .w8_start +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w8_start: + vpermt2b m2, m8, m3 + mova m3, m14 + vpdpbusd m3, m2, m9 + vpermb m3, m15, m3 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + mova [maskq], xm3 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm1 + vextracti32x4 [dstq+strideq*1], ym1, 1 + vextracti32x4 [dstq+strideq*2], m1, 2 + vextracti32x4 [dstq+stride3q ], m1, 3 + jg .w8_loop +.w8_end: + RET +.w16: + mova m8, [w_mask_shuf16] + vpbroadcastd m9, [pb_64] + jmp .w16_start +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w16_start: + vpermt2b m2, m8, m3 + mova m3, m14 + vpdpbusd m3, m2, m9 + vpermb m3, m15, m3 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + mova [maskq], xm3 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w32: + paddw m2, m3 + mova m8, m14 + vpdpwssd m8, m11, m2 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + call .main + paddw m2, m3 + mova m3, m14 + vpdpwssd m3, m11, m2 + vpermt2b m8, m15, m3 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m1 + mova [maskq], ym8 + sub hd, 4 + jg .w32_loop + RET +.w64_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w64: + mova m8, m2 + mova m9, m3 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*0+64*1], m1 + call .main + paddw m8, m2 + paddw m9, m3 + mova m2, m14 + vpdpwssd m2, m11, m8 + mova m3, m14 + vpdpwssd m3, m11, m9 + vpermt2b m2, m15, m3 + mova [dstq+strideq*1+64*0], m0 + mova [dstq+strideq*1+64*1], m1 + mova [maskq], ym2 + sub hd, 2 + jg .w64_loop + RET +.w128_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 64 +.w128: + mova m16, m2 + mova m8, m3 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*0+64*1], m1 + call .main + mova m17, m2 + mova m9, m3 + mova [dstq+strideq*0+64*2], m0 + mova [dstq+strideq*0+64*3], m1 + call .main + paddw m2, m16 + paddw m3, m8 + mova m16, m14 + vpdpwssd m16, m11, m2 + mova m8, m14 + vpdpwssd m8, m11, m3 + mova [dstq+strideq*1+64*0], m0 + mova [dstq+strideq*1+64*1], m1 + call .main + paddw m2, m17 + paddw m3, m9 + mova m17, m14 + vpdpwssd m17, m11, m2 + mova m9, m14 + vpdpwssd m9, m11, m3 + vpermt2b m16, m15, m8 + vpermt2b m17, m15, m9 + mova [dstq+strideq*1+64*2], m0 + mova [dstq+strideq*1+64*3], m1 + mova [maskq+32*0], ym16 + mova [maskq+32*1], ym17 + sub hd, 2 + jg .w128_loop + vzeroupper + RET +ALIGN function_align +.main: + mova m1, [tmp1q+64*0] + mova m3, [tmp2q+64*0] + mova m4, [tmp1q+64*1] + mova m7, [tmp2q+64*1] + add tmp1q, 64*2 + add tmp2q, 64*2 + psubsw m6, m1, m3 + punpcklwd m5, m3, m1 + pabsw m6, m6 + punpckhwd m3, m1 + psubusw m6, m10, m6 + psrlw m6, 10 ; 64-m + psubw m2, m11, m6 ; m + punpcklwd m1, m6, m2 + punpckhwd m6, m2 + mova m0, m12 + vpdpwssd m0, m5, m1 + mova m1, m12 + vpdpwssd m1, m3, m6 + psubsw m5, m4, m7 + punpcklwd m6, m7, m4 + pabsw m5, m5 + punpckhwd m7, m4 + psubusw m5, m10, m5 + psrlw m5, 10 + psubw m3, m11, m5 + punpcklwd m4, m5, m3 + psrad m0, 4 + punpckhwd m5, m3 + psrad m1, 4 + packusdw m0, m1 + mova m1, m12 + vpdpwssd m1, m6, m4 + mova m4, m12 + vpdpwssd m4, m7, m5 + psrad m1, 4 + psrad m4, 4 + packusdw m1, m4 + vpsrlvw m0, m13 + vpsrlvw m1, m13 + ret + +cglobal w_mask_422_16bpc, 4, 8, 15, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx512icl_table + lea r7, [w_mask_422_avx512icl_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + vpbroadcastd m9, [base+pw_64] + vpbroadcastd m10, [base+mask_round+r6*4] + vpbroadcastd m11, [base+bidir_shift+r6*4] + mov r6d, r7m ; sign + vpbroadcastd m12, [base+w_mask_round+r6*4] + mova ym13, [w_mask_end42x] + mov maskq, maskmp + add wq, r7 + paddw m14, m9, m9 ; pw_128 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm2, ym0, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + cmp hd, 8 + jl .w4_end + vextracti32x4 xm2, m0, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm0, m0, 3 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti32x4 xm2, ym1, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + vextracti32x4 xm2, m1, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm1, m1, 3 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] +.w8: + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm1 + vextracti32x4 [dstq+strideq*1], ym1, 1 + vextracti32x4 [dstq+strideq*2], m1, 2 + vextracti32x4 [dstq+stride3q ], m1, 3 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] +.w16: + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] +.w32: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + call .main + mova [dstq+64*2], m0 + mova [dstq+64*3], m1 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + mova m1, [tmp1q+64*0] + mova m3, [tmp2q+64*0] + mova m4, [tmp1q+64*1] + mova m7, [tmp2q+64*1] + add tmp1q, 64*2 + add tmp2q, 64*2 + psubsw m6, m1, m3 + punpcklwd m5, m3, m1 + pabsw m6, m6 + punpckhwd m3, m1 + psubusw m6, m8, m6 + psrlw m6, 10 + psubw m2, m9, m6 + punpcklwd m1, m6, m2 + punpckhwd m6, m2 + mova m0, m10 + vpdpwssd m0, m5, m1 + mova m1, m10 + vpdpwssd m1, m3, m6 + psubsw m5, m4, m7 + punpcklwd m6, m7, m4 + pabsw m5, m5 + punpckhwd m7, m4 + psubusw m5, m8, m5 + psrlw m5, 10 + psubw m3, m9, m5 + punpcklwd m4, m5, m3 + psrad m0, 4 + punpckhwd m5, m3 + psrad m1, 4 + packusdw m0, m1 + mova m1, m10 + vpdpwssd m1, m6, m4 + mova m4, m10 + vpdpwssd m4, m7, m5 + mova m5, m12 + vpdpwssd m5, m14, m2 + mova m2, m12 + vpdpwssd m2, m14, m3 + psrad m1, 4 + psrad m4, 4 + packusdw m1, m4 + vpermt2b m5, m13, m2 + vpsrlvw m0, m11 + vpsrlvw m1, m11 + mova [maskq], ym5 + add maskq, 32 + ret + +cglobal w_mask_444_16bpc, 4, 8, 13, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx512icl_table + lea r7, [w_mask_444_avx512icl_table] + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movifnidn hd, hm + shr r6d, 11 + movsxd wq, [r7+wq*4] + vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + vpbroadcastd m9, [base+pw_64] + vpbroadcastd m10, [base+mask_round+r6*4] + mova m11, [w_mask_end444] + vpbroadcastd m12, [base+bidir_shift+r6*4] + mov maskq, maskmp + add wq, r7 + call .main + lea stride3q, [strideq*3] + jmp wq +.w4: + movq [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm0 + vextracti32x4 xm2, ym0, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + cmp hd, 8 + jl .w4_end + vextracti32x4 xm2, m0, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm0, m0, 3 + movq [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm0 + je .w4_end + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm1 + vextracti32x4 xm2, ym1, 1 + movq [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm2 + vextracti32x4 xm2, m1, 2 + lea dstq, [dstq+strideq*4] + movq [dstq+strideq*0], xm2 + movhps [dstq+strideq*1], xm2 + vextracti32x4 xm1, m1, 3 + movq [dstq+strideq*2], xm1 + movhps [dstq+stride3q ], xm1 +.w4_end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*4] +.w8: + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], ym0, 1 + vextracti32x4 [dstq+strideq*2], m0, 2 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 8 + jl .w8_end + lea dstq, [dstq+strideq*4] + mova [dstq+strideq*0], xm1 + vextracti32x4 [dstq+strideq*1], ym1, 1 + vextracti32x4 [dstq+strideq*2], m1, 2 + vextracti32x4 [dstq+stride3q ], m1, 3 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*4] +.w16: + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+stride3q ], m1, 1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] +.w32: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + call .main + mova [dstq+64*2], m0 + mova [dstq+64*3], m1 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + mova m1, [tmp1q+64*0] + mova m3, [tmp2q+64*0] + mova m4, [tmp1q+64*1] + mova m7, [tmp2q+64*1] + add tmp1q, 64*2 + add tmp2q, 64*2 + psubsw m6, m1, m3 + punpcklwd m5, m3, m1 + pabsw m6, m6 + punpckhwd m3, m1 + psubusw m6, m8, m6 + psrlw m6, 10 + psubw m2, m9, m6 + punpcklwd m1, m6, m2 + punpckhwd m6, m2 + mova m0, m10 + vpdpwssd m0, m5, m1 + mova m1, m10 + vpdpwssd m1, m3, m6 + psubsw m5, m4, m7 + punpcklwd m6, m7, m4 + pabsw m5, m5 + punpckhwd m7, m4 + psubusw m5, m8, m5 + psrlw m5, 10 + psubw m3, m9, m5 + punpcklwd m4, m5, m3 + psrad m0, 4 + punpckhwd m5, m3 + psrad m1, 4 + packusdw m0, m1 + mova m1, m10 + vpdpwssd m1, m6, m4 + mova m4, m10 + vpdpwssd m4, m7, m5 + vpermt2b m2, m11, m3 + psrad m1, 4 + psrad m4, 4 + packusdw m1, m4 + vpsrlvw m0, m12 + vpsrlvw m1, m12 + mova [maskq], m2 + add maskq, 64 + ret + +cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_avx512icl_table + lea r6, [blend_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r6+wq*4] + movifnidn maskq, maskmp + vpbroadcastd m6, [base+pw_m512] + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + pmovzxbw ym19, [maskq] + movq xm16, [dstq+dsq*0] + movhps xm16, [dstq+dsq*1] + vpbroadcastq ym17, [dstq+dsq*2] + vpbroadcastq ym18, [dstq+r6 ] + pmullw ym19, ym6 + vpblendd ym16, ym17, 0x30 + vpblendd ym16, ym18, 0xc0 + psubw ym17, ym16, [tmpq] + add maskq, 16 + add tmpq, 32 + pmulhrsw ym17, ym19 + paddw ym16, ym17 + vextracti128 xm17, ym16, 1 + movq [dstq+dsq*0], xm16 + movhps [dstq+dsq*1], xm16 + movq [dstq+dsq*2], xm17 + movhps [dstq+r6 ], xm17 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w4 + vzeroupper + RET +.w8: + pmovzxbw m2, [maskq] + mova xm0, [dstq+dsq*0] + vinserti32x4 ym0, [dstq+dsq*1], 1 + vinserti32x4 m0, [dstq+dsq*2], 2 + vinserti32x4 m0, [dstq+r6 ], 3 + pmullw m2, m6 + psubw m1, m0, [tmpq] + add maskq, 32 + add tmpq, 64 + pmulhrsw m1, m2 + paddw m0, m1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + vextracti32x4 [dstq+dsq*2], m0, 2 + vextracti32x4 [dstq+r6 ], m0, 3 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w8 + RET +.w16: + pmovzxbw m4, [maskq+32*0] + pmovzxbw m5, [maskq+32*1] + mova ym0, [dstq+dsq*0] + vinserti32x8 m0, [dstq+dsq*1], 1 + mova ym1, [dstq+dsq*2] + vinserti32x8 m1, [dstq+r6 ], 1 + pmullw m4, m6 + pmullw m5, m6 + psubw m2, m0, [tmpq+64*0] + psubw m3, m1, [tmpq+64*1] + add maskq, 32*2 + add tmpq, 64*2 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + mova [dstq+dsq*2], ym1 + vextracti32x8 [dstq+r6 ], m1, 1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w16 + RET +.w32: + pmovzxbw m4, [maskq+32*0] + pmovzxbw m5, [maskq+32*1] + mova m0, [dstq+dsq*0] + mova m1, [dstq+dsq*1] + pmullw m4, m6 + pmullw m5, m6 + psubw m2, m0, [tmpq+ 64*0] + psubw m3, m1, [tmpq+ 64*1] + add maskq, 32*2 + add tmpq, 64*2 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w32 + RET + +cglobal blend_v_16bpc, 3, 6, 5, dst, ds, tmp, w, h + lea r5, [blend_v_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + jmp wq +.w2: + vpbroadcastd xmm2, [obmc_masks_avx2+2*2] +.w2_loop: + movd xmm0, [dstq+dsq*0] + pinsrd xmm0, [dstq+dsq*1], 1 + movq xmm1, [tmpq] + add tmpq, 4*2 + psubw xmm1, xmm0, xmm1 + pmulhrsw xmm1, xmm2 + paddw xmm0, xmm1 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w2_loop + RET +.w4: + vpbroadcastq xmm2, [obmc_masks_avx2+4*2] +.w4_loop: + movq xmm0, [dstq+dsq*0] + movhps xmm0, [dstq+dsq*1] + psubw xmm1, xmm0, [tmpq] + add tmpq, 8*2 + pmulhrsw xmm1, xmm2 + paddw xmm0, xmm1 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +.w8: + vbroadcasti32x4 ym2, [obmc_masks_avx2+8*2] +.w8_loop: + mova xm0, [dstq+dsq*0] + vinserti32x4 ym0, [dstq+dsq*1], 1 + psubw ym1, ym0, [tmpq] + add tmpq, 16*2 + pmulhrsw ym1, ym2 + paddw ym0, ym1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + vbroadcasti32x8 m2, [obmc_masks_avx2+16*2] +.w16_loop: + mova ym0, [dstq+dsq*0] + vinserti32x8 m0, [dstq+dsq*1], 1 + psubw m1, m0, [tmpq] + add tmpq, 32*2 + pmulhrsw m1, m2 + paddw m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16_loop + RET +.w32: + mova m4, [obmc_masks_avx2+32*2] +.w32_loop: + mova m0, [dstq+dsq*0] + psubw m2, m0, [tmpq+ 64*0] + mova m1, [dstq+dsq*1] + psubw m3, m1, [tmpq+ 64*1] + add tmpq, 64*2 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w32_loop + RET + +cglobal blend_h_16bpc, 3, 7, 9, dst, ds, tmp, w, h, mask +%define base r6-$$ + lea r6, [$$] + tzcnt wd, wm + mov hd, hm + movsxd wq, [base+blend_h_avx512icl_table+wq*4] + lea maskq, [base+obmc_masks_avx2+hq*2] + lea hd, [hq*3] + lea wq, [base+blend_h_avx512icl_table+wq] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd xmm0, [dstq+dsq*0] + pinsrd xmm0, [dstq+dsq*1], 1 + movd xmm2, [maskq+hq*2] + movq xmm1, [tmpq] + add tmpq, 4*2 + punpcklwd xmm2, xmm2 + psubw xmm1, xmm0, xmm1 + pmulhrsw xmm1, xmm2 + paddw xmm0, xmm1 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +.w4: + mova xmm3, [blend_shuf] +.w4_loop: + movq xmm0, [dstq+dsq*0] + movhps xmm0, [dstq+dsq*1] + movd xmm2, [maskq+hq*2] + psubw xmm1, xmm0, [tmpq] + add tmpq, 8*2 + pshufb xmm2, xmm3 + pmulhrsw xmm1, xmm2 + paddw xmm0, xmm1 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +.w8: + vbroadcasti32x4 ym3, [blend_shuf] + shufpd ym3, ym3, 0x0c +.w8_loop: + mova xm0, [dstq+dsq*0] + vinserti32x4 ym0, [dstq+dsq*1], 1 + vpbroadcastd ym2, [maskq+hq*2] + psubw ym1, ym0, [tmpq] + add tmpq, 16*2 + pshufb ym2, ym3 + pmulhrsw ym1, ym2 + paddw ym0, ym1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop + RET +.w16: + vbroadcasti32x4 m3, [blend_shuf] + shufpd m3, m3, 0xf0 +.w16_loop: + mova ym0, [dstq+dsq*0] + vinserti32x8 m0, [dstq+dsq*1], 1 + vpbroadcastd m2, [maskq+hq*2] + psubw m1, m0, [tmpq] + add tmpq, 32*2 + pshufb m2, m3 + pmulhrsw m1, m2 + paddw m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w16_loop + RET +.w32: + vpbroadcastw m4, [maskq+hq*2] + vpbroadcastw m5, [maskq+hq*2+2] + mova m0, [dstq+dsq*0] + psubw m2, m0, [tmpq+ 64*0] + mova m1, [dstq+dsq*1] + psubw m3, m1, [tmpq+ 64*1] + add tmpq, 64*2 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w32 + RET +.w64: + vpbroadcastw m4, [maskq+hq*2] + mova m0, [dstq+64*0] + psubw m2, m0, [tmpq+64*0] + mova m1, [dstq+64*1] + psubw m3, m1, [tmpq+64*1] + add tmpq, 64*2 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + inc hq + jl .w64 + RET +.w128: + vpbroadcastw m8, [maskq+hq*2] + mova m0, [dstq+64*0] + psubw m4, m0, [tmpq+64*0] + mova m1, [dstq+64*1] + psubw m5, m1, [tmpq+64*1] + mova m2, [dstq+64*2] + psubw m6, m2, [tmpq+64*2] + mova m3, [dstq+64*3] + psubw m7, m3, [tmpq+64*3] + add tmpq, 64*4 + REPX {pmulhrsw x, m8}, m4, m5, m6, m7 + paddw m0, m4 + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + mova [dstq+64*2], m2 + mova [dstq+64*3], m3 + add dstq, dsq + inc hq + jl .w128 + RET + +cglobal resize_16bpc, 6, 12, 32, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + mov r6, ~0 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + kmovq k6, r6 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax + LEA r7, $$ +%define base r7-$$ + vpbroadcastd m3, [base+pd_16384] + vpbroadcastd m7, [base+pd_63] + mova m24, [base+resize_permA] + mova m25, [base+resize_permB] + mova m26, [base+resize_permC] + mova m27, [base+resize_permD] + vbroadcasti32x4 m28, [base+resize_shufA] + vbroadcasti32x4 m29, [base+resize_shufB] + mova m30, [base+resize_permE] + vpbroadcastw ym31, pxmaxm + vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15] + pslld m5, 4 ; dx*16 + pslld m6, 14 + pxor m2, m2 +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx +.loop_x: + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + vptestmd k5, m1, m1 + pand m9, m7 ; filter offset (masked) + ktestw k5, k5 + jz .load + vpbroadcastq m14, [base+pd_0_4] + vpermq m10, m0, q1100 + vpermq m11, m0, q3322 + vpermq m20, m1, q1100 + vpermq m21, m1, q3322 + punpckldq m10, m10 + punpckldq m11, m11 + punpckldq m20, m20 + punpckldq m21, m21 + paddd m10, m14 + paddd m11, m14 + paddd m20, m14 + paddd m21, m14 + vextracti32x8 ym12, m10, 1 + vextracti32x8 ym13, m11, 1 + vextracti32x8 ym22, m20, 1 + vextracti32x8 ym23, m21, 1 + kmovq k1, k6 + kmovq k2, k6 + kmovq k3, k6 + kmovq k4, k6 + vpgatherdq m16{k1}, [srcq+ym10*2] ; 0 1 2 3 + vpgatherdq m17{k2}, [srcq+ym11*2] ; 4 5 6 7 + vpgatherdq m18{k3}, [srcq+ym12*2] ; 8 9 A B + vpgatherdq m19{k4}, [srcq+ym13*2] ; C D E F + kmovq k1, k6 + kmovq k2, k6 + kmovq k3, k6 + kmovq k4, k6 + vpgatherdq m0{k1}, [base+resize_shuf+8+ym20*2] + vpgatherdq m1{k2}, [base+resize_shuf+8+ym21*2] + vpgatherdq m14{k3}, [base+resize_shuf+8+ym22*2] + vpgatherdq m15{k4}, [base+resize_shuf+8+ym23*2] + pshufb m16, m0 + pshufb m17, m1 + pshufb m18, m14 + pshufb m19, m15 + mova m20, m24 + mova m22, m24 + mova m21, m25 + mova m23, m25 + vpermi2d m20, m16, m17 ; 0-3a 0-3b 4-7a 4-7b + vpermi2d m21, m16, m17 ; 0-3c 0-3d 4-7c 4-7d + vpermi2d m22, m18, m19 ; 8-Ba 8-Bb C-Fa C-Fb + vpermi2d m23, m18, m19 ; 8-Bc 8-Bd C-Fc C-Fd + mova m15, m26 + mova m17, m26 + mova m16, m27 + mova m18, m27 + vpermi2q m15, m20, m22 ; 0-3a 4-7a 8-Ba C-Fa + vpermi2q m16, m20, m22 ; 0-3b 4-7b 8-Bb C-Fb + vpermi2q m17, m21, m23 ; 0-3c 4-7c 8-Bc C-Fc + vpermi2q m18, m21, m23 ; 0-3d 4-7d 8-Bd C-Fd + kmovq k1, k6 + kmovq k2, k6 + vpgatherdd m11{k1}, [base+resize_filter+m9*8+0] + vpgatherdd m13{k2}, [base+resize_filter+m9*8+4] + pshufb m10, m11, m28 + pshufb m11, m11, m29 + pshufb m12, m13, m28 + pshufb m13, m13, m29 + jmp .filter +.load: + kmovq k1, k6 + kmovq k2, k6 + kmovq k3, k6 + kmovq k4, k6 + vpgatherdd m11{k1}, [base+resize_filter+m9*8+0] + vpgatherdd m13{k2}, [base+resize_filter+m9*8+4] + pshufb m10, m11, m28 + pshufb m11, m11, m29 + pshufb m12, m13, m28 + pshufb m13, m13, m29 + vpgatherdd m15{k3}, [srcq+m0*2+ 0] + vpgatherdd m16{k4}, [srcq+m0*2+ 4] + kmovq k1, k6 + kmovq k2, k6 + vpgatherdd m17{k1}, [srcq+m0*2+ 8] + vpgatherdd m18{k2}, [srcq+m0*2+12] +.filter: + mova m14, m2 + vpdpwssd m14, m15, m10 + vpdpwssd m14, m16, m11 + vpdpwssd m14, m17, m12 + vpdpwssd m14, m18, m13 + psubd m14, m3, m14 + psrad m14, 15 + packusdw m14, m14 + vpermq m14, m30, m14 + pminsw ym14, ym31 + mova [dstq+xq*2], ym14 + paddd m4, m5 + add xd, 16 + cmp xd, dst_wd + jl .loop_x + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc16_sse.asm b/third_party/dav1d/src/x86/mc16_sse.asm new file mode 100644 index 0000000000..fde8e372a3 --- /dev/null +++ b/third_party/dav1d/src/x86/mc16_sse.asm @@ -0,0 +1,8731 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +; dav1d_obmc_masks[] << 9 +obmc_masks: dw 0, 0, 9728, 0, 12800, 7168, 2560, 0 + dw 14336, 11264, 8192, 5632, 3584, 1536, 0, 0 + dw 15360, 13824, 12288, 10752, 9216, 7680, 6144, 5120 + dw 4096, 3072, 2048, 1536, 0, 0, 0, 0 + dw 15872, 14848, 14336, 13312, 12288, 11776, 10752, 10240 + dw 9728, 8704, 8192, 7168, 6656, 6144, 5632, 4608 + dw 4096, 3584, 3072, 2560, 2048, 2048, 1536, 1024 + +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 +spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 +spel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 +spel_s_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 +spel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +unpckw: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +rescale_mul: dd 0, 1, 2, 3 +resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 +bdct_lb_q: times 8 db 0 + times 8 db 4 + times 8 db 8 + times 8 db 12 + +pw_2: times 8 dw 2 +pw_16: times 4 dw 16 +prep_mul: times 4 dw 16 + times 8 dw 4 +pw_64: times 8 dw 64 +pw_256: times 8 dw 256 +pw_2048: times 4 dw 2048 +bidir_mul: times 4 dw 2048 +pw_8192: times 8 dw 8192 +pw_27615: times 8 dw 27615 +pw_32766: times 8 dw 32766 +pw_m512: times 8 dw -512 +pd_63: times 4 dd 63 +pd_64: times 4 dd 64 +pd_512: times 4 dd 512 +pd_m524256: times 4 dd -524256 ; -8192 << 6 + 32 +pd_0x3ff: times 4 dd 0x3ff +pd_0x4000: times 4 dd 0x4000 +pq_0x400000: times 2 dq 0x400000 +pq_0x40000000: times 2 dq 0x40000000 +pd_65538: times 2 dd 65538 + +put_bilin_h_rnd: times 4 dw 8 + times 4 dw 10 +s_8tap_h_rnd: times 2 dd 2 + times 2 dd 8 +put_s_8tap_v_rnd: times 2 dd 512 + times 2 dd 128 +s_8tap_h_sh: dd 2, 4 +put_s_8tap_v_sh: dd 10, 8 +bidir_rnd: times 4 dw -16400 + times 4 dw -16388 +put_8tap_h_rnd: dd 34, 34, 40, 40 +prep_8tap_1d_rnd: times 2 dd 8 - (8192 << 4) +prep_8tap_2d_rnd: times 4 dd 32 - (8192 << 5) + +warp8x8_shift: dd 11, 13 +warp8x8_rnd1: dd 1024, 1024, 4096, 4096 +warp8x8_rnd2: times 4 dw 4096 + times 4 dw 16384 +warp8x8t_rnd: times 2 dd 16384 - (8192 << 15) + +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +BIDIR_JMP_TABLE avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, ssse3, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, ssse3, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, ssse3, 2, 4, 8, 16, 32, 64, 128 + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_16bpc_ssse3.put) +%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_16bpc_ssse3.prep) + +BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 + +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_16bpc_%2) +%%table: + %rep %0 - 2 + dw %%base %+ .w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_1024: + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_2048: + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base + %rotate 1 + %endrep +%endmacro + +SCALED_JMP_TABLE put_8tap_scaled, ssse3, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, ssse3, 4, 8, 16, 32, 64, 128 + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +cextern mc_warp_filter +cextern resize_filter + +SECTION .text + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +INIT_XMM ssse3 +cglobal put_bilin_16bpc, 4, 7, 0, dst, ds, src, ss, w, h, mxy +%define base t0-put_ssse3 + mov mxyd, r6m ; mx + LEA t0, put_ssse3 + movifnidn wd, wm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + tzcnt wd, wd + movzx wd, word [base+put_ssse3_table+wq*2] + add wq, t0 + movifnidn hd, hm + jmp wq +.put_w2: + mov r4d, [srcq+ssq*0] + mov r6d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r4d + mov [dstq+dsq*1], r6d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + movq m0, [srcq+ssq*0] + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq [dstq+dsq*0], m0 + movq [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu m0, [srcq+ssq*0+16*0] + movu m1, [srcq+ssq*0+16*1] + movu m2, [srcq+ssq*1+16*0] + movu m3, [srcq+ssq*1+16*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+16*0], m0 + mova [dstq+dsq*0+16*1], m1 + mova [dstq+dsq*1+16*0], m2 + mova [dstq+dsq*1+16*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + add srcq, ssq + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + add dstq, dsq + dec hd + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + movu m0, [srcq+16*4] + movu m1, [srcq+16*5] + movu m2, [srcq+16*6] + movu m3, [srcq+16*7] + add srcq, ssq + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + mova [dstq+16*6], m2 + mova [dstq+16*7], m3 + add dstq, dsq + dec hd + jg .put_w64 + RET +.put_w128: + add srcq, 16*8 + add dstq, 16*8 +.put_w128_loop: + movu m0, [srcq-16*8] + movu m1, [srcq-16*7] + movu m2, [srcq-16*6] + movu m3, [srcq-16*5] + mova [dstq-16*8], m0 + mova [dstq-16*7], m1 + mova [dstq-16*6], m2 + mova [dstq-16*5], m3 + movu m0, [srcq-16*4] + movu m1, [srcq-16*3] + movu m2, [srcq-16*2] + movu m3, [srcq-16*1] + mova [dstq-16*4], m0 + mova [dstq-16*3], m1 + mova [dstq-16*2], m2 + mova [dstq-16*1], m3 + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + movu m0, [srcq+16*4] + movu m1, [srcq+16*5] + movu m2, [srcq+16*6] + movu m3, [srcq+16*7] + add srcq, ssq + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + mova [dstq+16*6], m2 + mova [dstq+16*7], m3 + add dstq, dsq + dec hd + jg .put_w128_loop + RET +.h: + movd m5, mxyd + mov mxyd, r7m ; my + mova m4, [base+pw_16] + pshufb m5, [base+pw_256] + psubw m4, m5 + test mxyd, mxyd + jnz .hv + ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v + mov r6d, r8m ; bitdepth_max + shr r6d, 11 + movddup m3, [base+put_bilin_h_rnd+r6*8] + movifnidn hd, hm + sub wd, 8 + jg .h_w16 + je .h_w8 + cmp wd, -4 + je .h_w4 +.h_w2: + movq m1, [srcq+ssq*0] + movhps m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmullw m0, m4, m1 + psrlq m1, 16 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 4 + movd [dstq+dsq*0], m0 + punpckhqdq m0, m0 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + movq m0, [srcq+ssq*0] + movhps m0, [srcq+ssq*1] + movq m1, [srcq+ssq*0+2] + movhps m1, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 4 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*0+2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + movu m1, [srcq+ssq*1] + movu m2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + neg wq +.h_w16_loop0: + mov r6, wq +.h_w16_loop: + movu m0, [srcq+r6*2+ 0] + movu m1, [srcq+r6*2+ 2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + movu m1, [srcq+r6*2+16] + movu m2, [srcq+r6*2+18] + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m0, 4 + psrlw m1, 4 + mova [dstq+r6*2+16*0], m0 + mova [dstq+r6*2+16*1], m1 + add r6, 16 + jl .h_w16_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w16_loop0 + RET +.v: + shl mxyd, 11 + movd m5, mxyd + pshufb m5, [base+pw_256] + movifnidn hd, hm + cmp wd, 4 + jg .v_w8 + je .v_w4 +.v_w2: + movd m0, [srcq+ssq*0] +.v_w2_loop: + movd m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklqdq m2, m0, m1 + movd m0, [srcq+ssq*0] + punpcklqdq m1, m0 + psubw m1, m2 + pmulhrsw m1, m5 + paddw m1, m2 + movd [dstq+dsq*0], m1 + punpckhqdq m1, m1 + movd [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movq m0, [srcq+ssq*0] +.v_w4_loop: + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklqdq m2, m0, m1 + movq m0, [srcq+ssq*0] + punpcklqdq m1, m0 + psubw m1, m2 + pmulhrsw m1, m5 + paddw m1, m2 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: +%if ARCH_X86_64 +%if WIN64 + push r7 +%endif + shl wd, 5 + mov r7, srcq + lea r6d, [wq+hq-256] + mov r4, dstq +%else + mov r6, srcq +%endif +.v_w8_loop0: + movu m0, [srcq+ssq*0] +.v_w8_loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + psubw m1, m3, m0 + pmulhrsw m1, m5 + paddw m1, m0 + movu m0, [srcq+ssq*0] + psubw m2, m0, m3 + pmulhrsw m2, m5 + paddw m2, m3 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop +%if ARCH_X86_64 + add r7, 16 + add r4, 16 + movzx hd, r6b + mov srcq, r7 + mov dstq, r4 + sub r6d, 1<<8 +%else + mov dstq, dstmp + add r6, 16 + mov hd, hm + add dstq, 16 + mov srcq, r6 + mov dstmp, dstq + sub wd, 8 +%endif + jg .v_w8_loop0 +%if WIN64 + pop r7 +%endif + RET +.hv: + WIN64_SPILL_XMM 8 + shl mxyd, 11 + mova m3, [base+pw_2] + movd m6, mxyd + mova m7, [base+pw_8192] + pshufb m6, [base+pw_256] + test dword r8m, 0x800 + jnz .hv_12bpc + psllw m4, 2 + psllw m5, 2 + mova m7, [base+pw_2048] +.hv_12bpc: + movifnidn hd, hm + cmp wd, 4 + jg .hv_w8 + je .hv_w4 +.hv_w2: + movddup m0, [srcq+ssq*0] + pshufhw m1, m0, q0321 + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 2 +.hv_w2_loop: + movq m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps m2, [srcq+ssq*0] + pmullw m1, m4, m2 + psrlq m2, 16 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m1, 2 ; 1 _ 2 _ + shufpd m2, m0, m1, 0x01 ; 0 _ 1 _ + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + movd [dstq+dsq*0], m1 + punpckhqdq m1, m1 + movd [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + movddup m0, [srcq+ssq*0] + movddup m1, [srcq+ssq*0+2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 2 +.hv_w4_loop: + movq m1, [srcq+ssq*1] + movq m2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + movhps m1, [srcq+ssq*0] + movhps m2, [srcq+ssq*0+2] + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m1, 2 ; 1 2 + shufpd m2, m0, m1, 0x01 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: +%if ARCH_X86_64 +%if WIN64 + push r7 +%endif + shl wd, 5 + lea r6d, [wq+hq-256] + mov r4, srcq + mov r7, dstq +%else + mov r6, srcq +%endif +.hv_w8_loop0: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*0+2] + pmullw m0, m4 + pmullw m1, m5 + paddw m0, m3 + paddw m0, m1 + psrlw m0, 2 +.hv_w8_loop: + movu m1, [srcq+ssq*1] + movu m2, [srcq+ssq*1+2] + lea srcq, [srcq+ssq*2] + pmullw m1, m4 + pmullw m2, m5 + paddw m1, m3 + paddw m1, m2 + psrlw m1, 2 + psubw m2, m1, m0 + paddw m2, m2 + pmulhw m2, m6 + paddw m2, m0 + pmulhrsw m2, m7 + mova [dstq+dsq*0], m2 + movu m0, [srcq+ssq*0] + movu m2, [srcq+ssq*0+2] + pmullw m0, m4 + pmullw m2, m5 + paddw m0, m3 + paddw m0, m2 + psrlw m0, 2 + psubw m2, m0, m1 + paddw m2, m2 + pmulhw m2, m6 + paddw m2, m1 + pmulhrsw m2, m7 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop +%if ARCH_X86_64 + add r4, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 +%else + mov dstq, dstmp + add r6, 16 + mov hd, hm + add dstq, 16 + mov srcq, r6 + mov dstmp, dstq + sub wd, 8 +%endif + jg .hv_w8_loop0 +%if WIN64 + pop r7 +%endif + RET + +cglobal prep_bilin_16bpc, 4, 7, 0, tmp, src, stride, w, h, mxy, stride3 +%define base r6-prep_ssse3 + movifnidn mxyd, r5m ; mx + LEA r6, prep_ssse3 + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + tzcnt wd, wd + movzx wd, word [base+prep_ssse3_table+wq*2] + mov r5d, r7m ; bitdepth_max + mova m5, [base+pw_8192] + add wq, r6 + shr r5d, 11 + movddup m4, [base+prep_mul+r5*8] + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movq m0, [srcq+strideq*0] + movhps m0, [srcq+strideq*1] + movq m1, [srcq+strideq*2] + movhps m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmullw m0, m4 + pmullw m1, m4 + psubw m0, m5 + psubw m1, m5 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + add tmpq, 16*2 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + movu m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movu m0, [srcq+strideq*0+16*0] + movu m1, [srcq+strideq*0+16*1] + movu m2, [srcq+strideq*1+16*0] + movu m3, [srcq+strideq*1+16*1] + lea srcq, [srcq+strideq*2] + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 2 + jg .prep_w16 + RET +.prep_w32: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + add srcq, strideq + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + dec hd + jg .prep_w32 + RET +.prep_w64: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + movu m0, [srcq+16*4] + movu m1, [srcq+16*5] + movu m2, [srcq+16*6] + movu m3, [srcq+16*7] + add srcq, strideq + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*4], m0 + mova [tmpq+16*5], m1 + mova [tmpq+16*6], m2 + mova [tmpq+16*7], m3 + add tmpq, 16*8 + dec hd + jg .prep_w64 + RET +.prep_w128: + movu m0, [srcq+16* 0] + movu m1, [srcq+16* 1] + movu m2, [srcq+16* 2] + movu m3, [srcq+16* 3] + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + movu m0, [srcq+16* 4] + movu m1, [srcq+16* 5] + movu m2, [srcq+16* 6] + movu m3, [srcq+16* 7] + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq+16*4], m0 + mova [tmpq+16*5], m1 + mova [tmpq+16*6], m2 + mova [tmpq+16*7], m3 + movu m0, [srcq+16* 8] + movu m1, [srcq+16* 9] + movu m2, [srcq+16*10] + movu m3, [srcq+16*11] + add tmpq, 16*16 + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq-16*8], m0 + mova [tmpq-16*7], m1 + mova [tmpq-16*6], m2 + mova [tmpq-16*5], m3 + movu m0, [srcq+16*12] + movu m1, [srcq+16*13] + movu m2, [srcq+16*14] + movu m3, [srcq+16*15] + add srcq, strideq + REPX {pmullw x, m4}, m0, m1, m2, m3 + REPX {psubw x, m5}, m0, m1, m2, m3 + mova [tmpq-16*4], m0 + mova [tmpq-16*3], m1 + mova [tmpq-16*2], m2 + mova [tmpq-16*1], m3 + dec hd + jg .prep_w128 + RET +.h: + movd m4, mxyd + mov mxyd, r6m ; my + mova m3, [base+pw_16] + pshufb m4, [base+pw_256] + mova m5, [base+pw_32766] + psubw m3, m4 + test dword r7m, 0x800 + jnz .h_12bpc + psllw m3, 2 + psllw m4, 2 +.h_12bpc: + test mxyd, mxyd + jnz .hv + sub wd, 8 + je .h_w8 + jg .h_w16 +.h_w4: + movq m0, [srcq+strideq*0] + movhps m0, [srcq+strideq*1] + movq m1, [srcq+strideq*0+2] + movhps m1, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 16 + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + add tmpq, 16*2 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + lea srcq, [srcq+wq*2] + neg wq +.h_w16_loop0: + mov r6, wq +.h_w16_loop: + movu m0, [srcq+r6*2+ 0] + movu m1, [srcq+r6*2+ 2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + movu m1, [srcq+r6*2+16] + movu m2, [srcq+r6*2+18] + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + add tmpq, 16*2 + add r6, 16 + jl .h_w16_loop + add srcq, strideq + dec hd + jg .h_w16_loop0 + RET +.v: + movd m4, mxyd + mova m3, [base+pw_16] + pshufb m4, [base+pw_256] + mova m5, [base+pw_32766] + psubw m3, m4 + test dword r7m, 0x800 + jnz .v_12bpc + psllw m3, 2 + psllw m4, 2 +.v_12bpc: + cmp wd, 8 + je .v_w8 + jg .v_w16 +.v_w4: + movq m0, [srcq+strideq*0] +.v_w4_loop: + movq m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklqdq m1, m0, m2 ; 0 1 + movq m0, [srcq+strideq*0] + punpcklqdq m2, m0 ; 1 2 + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m1, 2 + mova [tmpq], m1 + add tmpq, 16 + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movu m0, [srcq+strideq*0] +.v_w8_loop: + movu m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + pmullw m0, m3 + pmullw m1, m4, m2 + psubw m0, m5 + paddw m1, m0 + movu m0, [srcq+strideq*0] + psraw m1, 2 + pmullw m2, m3 + mova [tmpq+16*0], m1 + pmullw m1, m4, m0 + psubw m2, m5 + paddw m1, m2 + psraw m1, 2 + mova [tmpq+16*1], m1 + add tmpq, 16*2 + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: +%if WIN64 + push r7 +%endif + mov r5, srcq +%if ARCH_X86_64 + lea r6d, [wq*4-32] + mov wd, wd + lea r6d, [hq+r6*8] + mov r7, tmpq +%else + mov r6d, wd +%endif +.v_w16_loop0: + movu m0, [srcq+strideq*0] +.v_w16_loop: + movu m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + pmullw m0, m3 + pmullw m1, m4, m2 + psubw m0, m5 + paddw m1, m0 + movu m0, [srcq+strideq*0] + psraw m1, 2 + pmullw m2, m3 + mova [tmpq+wq*0], m1 + pmullw m1, m4, m0 + psubw m2, m5 + paddw m1, m2 + psraw m1, 2 + mova [tmpq+wq*2], m1 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w16_loop +%if ARCH_X86_64 + add r5, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 +%else + mov tmpq, tmpmp + add r5, 16 + mov hd, hm + add tmpq, 16 + mov srcq, r5 + mov tmpmp, tmpq + sub r6d, 8 +%endif + jg .v_w16_loop0 +%if WIN64 + pop r7 +%endif + RET +.hv: + WIN64_SPILL_XMM 7 + shl mxyd, 11 + movd m6, mxyd + pshufb m6, [base+pw_256] + cmp wd, 8 + je .hv_w8 + jg .hv_w16 +.hv_w4: + movddup m0, [srcq+strideq*0] + movddup m1, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + psraw m0, 2 +.hv_w4_loop: + movq m1, [srcq+strideq*1] + movq m2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + movhps m1, [srcq+strideq*0] + movhps m2, [srcq+strideq*0+2] + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m1, 2 ; 1 2 + shufpd m2, m0, m1, 0x01 ; 0 1 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 16 + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + psraw m0, 2 +.hv_w8_loop: + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m1, 2 + psubw m2, m1, m0 + pmulhrsw m2, m6 + paddw m2, m0 + mova [tmpq+16*0], m2 + movu m0, [srcq+strideq*0] + movu m2, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m2, m4 + psubw m0, m5 + paddw m0, m2 + psraw m0, 2 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+16*1], m2 + add tmpq, 16*2 + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: +%if WIN64 + push r7 +%endif + mov r5, srcq +%if ARCH_X86_64 + lea r6d, [wq*4-32] + mov wd, wd + lea r6d, [hq+r6*8] + mov r7, tmpq +%else + mov r6d, wd +%endif +.hv_w16_loop0: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m1, m4 + psubw m0, m5 + paddw m0, m1 + psraw m0, 2 +.hv_w16_loop: + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*1+2] + lea srcq, [srcq+strideq*2] + pmullw m1, m3 + pmullw m2, m4 + psubw m1, m5 + paddw m1, m2 + psraw m1, 2 + psubw m2, m1, m0 + pmulhrsw m2, m6 + paddw m2, m0 + mova [tmpq+wq*0], m2 + movu m0, [srcq+strideq*0] + movu m2, [srcq+strideq*0+2] + pmullw m0, m3 + pmullw m2, m4 + psubw m0, m5 + paddw m0, m2 + psraw m0, 2 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+wq*2], m2 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .hv_w16_loop +%if ARCH_X86_64 + add r5, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 +%else + mov tmpq, tmpmp + add r5, 16 + mov hd, hm + add tmpq, 16 + mov srcq, r5 + mov tmpmp, tmpq + sub r6d, 8 +%endif + jg .hv_w16_loop0 +%if WIN64 + pop r7 +%endif + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; prefix, type, type_h, type_v +cglobal %1_%2_16bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX) +%endif +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 1, 2, 6 +%elif WIN64 +DECLARE_REG_TMP 4, 5, 8 +%else +DECLARE_REG_TMP 7, 8, 8 +%endif + +%define PUT_8TAP_FN FN put_8tap, +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR + +%if ARCH_X86_32 +cglobal put_8tap_16bpc, 0, 7, 8, dst, ds, src, ss, w, h, mx, my +%define mxb r0b +%define mxd r0 +%define mxq r0 +%define myb r1b +%define myd r1 +%define myq r1 +%define m8 [esp+16*0] +%define m9 [esp+16*1] +%define m10 [esp+16*2] +%define m11 [esp+16*3] +%define m12 [esp+16*4] +%define m13 [esp+16*5] +%define m14 [esp+16*6] +%define m15 [esp+16*7] +%else +cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my +%endif +%define base t2-put_ssse3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + LEA t2, put_ssse3 + movifnidn wd, wm + movifnidn srcq, srcmp + movifnidn ssq, ssmp + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [base+put_ssse3_table+wq*2] + movifnidn dstq, dstmp + movifnidn dsq, dsmp + add wq, t2 +%if WIN64 + pop r8 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + mov myd, r8m + movd m5, r8m + shr myd, 11 + movddup m4, [base+put_8tap_h_rnd+myq*8] + movifnidn dsq, dsmp + pshufb m5, [base+pw_256] + cmp wd, 4 + jg .h_w8 + movzx mxd, mxb + lea srcq, [srcq-2] + movq m3, [base+subpel_filters+mxq*8] + movifnidn dstq, dstmp + punpcklbw m3, m3 + psraw m3, 8 ; sign-extend + je .h_w4 +.h_w2: + mova m2, [base+spel_h_shuf2] + pshufd m3, m3, q2121 +.h_w2_loop: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m2 + pshufb m1, m2 + pmaddwd m0, m3 + pmaddwd m1, m3 + phaddd m0, m1 + paddd m0, m4 + psrad m0, 6 + packssdw m0, m0 + pxor m1, m1 + pminsw m0, m5 + pmaxsw m0, m1 + movd [dstq+dsq*0], m0 + pshuflw m0, m0, q3232 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + WIN64_SPILL_XMM 8 + mova m6, [base+spel_h_shufA] + mova m7, [base+spel_h_shufB] + pshufd m2, m3, q1111 + pshufd m3, m3, q2222 +.h_w4_loop: + movu m1, [srcq] + add srcq, ssq + pshufb m0, m1, m6 ; 0 1 1 2 2 3 3 4 + pshufb m1, m7 ; 2 3 3 4 4 5 5 6 + pmaddwd m0, m2 + pmaddwd m1, m3 + paddd m0, m4 + paddd m0, m1 + psrad m0, 6 + packssdw m0, m0 + pxor m1, m1 + pminsw m0, m5 + pmaxsw m0, m1 + movq [dstq], m0 + add dstq, dsq + dec hd + jg .h_w4_loop + RET +.h_w8: +%if WIN64 + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 12 +%endif + shr mxd, 16 + movq m3, [base+subpel_filters+mxq*8] + movifnidn dstq, dstmp + mova m6, [base+spel_h_shufA] + mova m7, [base+spel_h_shufB] +%if UNIX64 + mov wd, wd +%endif + lea srcq, [srcq+wq*2] + punpcklbw m3, m3 + lea dstq, [dstq+wq*2] + psraw m3, 8 + neg wq +%if ARCH_X86_32 + ALLOC_STACK -16*4 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 +%else + pshufd m8, m3, q0000 + pshufd m9, m3, q1111 + pshufd m10, m3, q2222 + pshufd m11, m3, q3333 +%endif +.h_w8_loop0: + mov r6, wq +.h_w8_loop: + movu m0, [srcq+r6*2- 6] + movu m1, [srcq+r6*2+ 2] + pshufb m2, m0, m6 ; 0 1 1 2 2 3 3 4 + pshufb m0, m7 ; 2 3 3 4 4 5 5 6 + pmaddwd m2, m8 ; abcd0 + pmaddwd m0, m9 ; abcd1 + pshufb m3, m1, m6 ; 4 5 5 6 6 7 7 8 + pshufb m1, m7 ; 6 7 7 8 8 9 9 a + paddd m2, m4 + paddd m0, m2 + pmaddwd m2, m10, m3 ; abcd2 + pmaddwd m3, m8 ; efgh0 + paddd m0, m2 + pmaddwd m2, m11, m1 ; abcd3 + pmaddwd m1, m9 ; efgh1 + paddd m0, m2 + movu m2, [srcq+r6*2+10] + paddd m3, m4 + paddd m1, m3 + pshufb m3, m2, m6 ; 8 9 9 a a b b c + pshufb m2, m7 ; a b b c c d d e + pmaddwd m3, m10 ; efgh2 + pmaddwd m2, m11 ; efgh3 + paddd m1, m3 + paddd m1, m2 + psrad m0, 6 + psrad m1, 6 + packssdw m0, m1 + pxor m1, m1 + pminsw m0, m5 + pmaxsw m0, m1 + mova [dstq+r6*2], m0 + add r6, 8 + jl .h_w8_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w8_loop0 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovb myd, mxd + movq m3, [base+subpel_filters+myq*8] +%if STACK_ALIGNMENT < 16 + %xdefine rstk rsp +%else + %assign stack_offset stack_offset - stack_size_padded +%endif +%if WIN64 + WIN64_SPILL_XMM 15 +%endif + movd m7, r8m + movifnidn dstq, dstmp + movifnidn dsq, dsmp + punpcklbw m3, m3 + pshufb m7, [base+pw_256] + psraw m3, 8 ; sign-extend +%if ARCH_X86_32 + ALLOC_STACK -16*7 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 +%else + pshufd m8, m3, q0000 + pshufd m9, m3, q1111 + pshufd m10, m3, q2222 + pshufd m11, m3, q3333 +%endif + lea r6, [ssq*3] + sub srcq, r6 + cmp wd, 2 + jne .v_w4 +.v_w2: + movd m1, [srcq+ssq*0] + movd m4, [srcq+ssq*1] + movd m2, [srcq+ssq*2] + add srcq, r6 + movd m5, [srcq+ssq*0] + movd m3, [srcq+ssq*1] + movd m6, [srcq+ssq*2] + add srcq, r6 + movd m0, [srcq+ssq*0] + punpckldq m1, m4 ; 0 1 + punpckldq m4, m2 ; 1 2 + punpckldq m2, m5 ; 2 3 + punpckldq m5, m3 ; 3 4 + punpckldq m3, m6 ; 4 5 + punpckldq m6, m0 ; 5 6 + punpcklwd m1, m4 ; 01 12 + punpcklwd m2, m5 ; 23 34 + punpcklwd m3, m6 ; 45 56 + pxor m6, m6 +.v_w2_loop: + movd m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m5, m8, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m9 ; a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m10 ; a2 b2 + paddd m5, m3 + punpckldq m3, m0, m4 ; 6 7 + movd m0, [srcq+ssq*0] + punpckldq m4, m0 ; 7 8 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m11, m3 ; a3 b3 + paddd m5, m4 + psrad m5, 5 + packssdw m5, m5 + pmaxsw m5, m6 + pavgw m5, m6 + pminsw m5, m7 + movd [dstq+dsq*0], m5 + pshuflw m5, m5, q3232 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: +%if ARCH_X86_32 + shl wd, 14 +%if STACK_ALIGNMENT < 16 + mov [esp+4*29], srcq + mov [esp+4*30], dstq +%else + mov srcmp, srcq +%endif + lea wd, [wq+hq-(1<<16)] +%else + shl wd, 6 + mov r7, srcq + mov r8, dstq + lea wd, [wq+hq-(1<<8)] +%endif +.v_w4_loop0: + movq m1, [srcq+ssq*0] + movq m2, [srcq+ssq*1] + movq m3, [srcq+ssq*2] + add srcq, r6 + movq m4, [srcq+ssq*0] + movq m5, [srcq+ssq*1] + movq m6, [srcq+ssq*2] + add srcq, r6 + movq m0, [srcq+ssq*0] + punpcklwd m1, m2 ; 01 + punpcklwd m2, m3 ; 12 + punpcklwd m3, m4 ; 23 + punpcklwd m4, m5 ; 34 + punpcklwd m5, m6 ; 45 + punpcklwd m6, m0 ; 56 +%if ARCH_X86_32 + jmp .v_w4_loop_start +.v_w4_loop: + mova m1, m12 + mova m2, m13 + mova m3, m14 +.v_w4_loop_start: + pmaddwd m1, m8 ; a0 + pmaddwd m2, m8 ; b0 + mova m12, m3 + mova m13, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m1, m3 + paddd m2, m4 + mova m14, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m1, m5 + paddd m2, m6 + movq m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklwd m5, m0, m6 ; 67 + movq m0, [srcq+ssq*0] + pmaddwd m3, m11, m5 ; a3 + punpcklwd m6, m0 ; 78 + paddd m1, m3 + pmaddwd m3, m11, m6 ; b3 + paddd m2, m3 + psrad m1, 5 + psrad m2, 5 + packssdw m1, m2 + pxor m2, m2 + pmaxsw m1, m2 + pavgw m1, m2 + pminsw m1, m7 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop +%if STACK_ALIGNMENT < 16 + mov srcq, [esp+4*29] + mov dstq, [esp+4*30] + movzx hd, ww + add srcq, 8 + add dstq, 8 + mov [esp+4*29], srcq + mov [esp+4*30], dstq +%else + mov srcq, srcmp + mov dstq, dstmp + movzx hd, ww + add srcq, 8 + add dstq, 8 + mov srcmp, srcq + mov dstmp, dstq +%endif + sub wd, 1<<16 +%else +.v_w4_loop: + pmaddwd m12, m8, m1 ; a0 + pmaddwd m13, m8, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m12, m3 + paddd m13, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m12, m5 + paddd m13, m6 + movq m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklwd m5, m0, m6 ; 67 + movq m0, [srcq+ssq*0] + pmaddwd m14, m11, m5 ; a3 + punpcklwd m6, m0 ; 78 + paddd m12, m14 + pmaddwd m14, m11, m6 ; b3 + paddd m13, m14 + psrad m12, 5 + psrad m13, 5 + packssdw m12, m13 + pxor m13, m13 + pmaxsw m12, m13 + pavgw m12, m13 + pminsw m12, m7 + movq [dstq+dsq*0], m12 + movhps [dstq+dsq*1], m12 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + add r7, 8 + add r8, 8 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 +%endif + jg .v_w4_loop0 + RET +.hv: +%if STACK_ALIGNMENT < 16 + %xdefine rstk rsp +%else + %assign stack_offset stack_offset - stack_size_padded +%endif +%if ARCH_X86_32 + movd m4, r8m + mova m6, [base+pd_512] + pshufb m4, [base+pw_256] +%else +%if WIN64 + ALLOC_STACK 16*6, 16 +%endif + movd m15, r8m + pshufb m15, [base+pw_256] +%endif + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + je .hv_w4 + movq m0, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovb myd, mxd + movq m3, [base+subpel_filters+myq*8] +%if ARCH_X86_32 + mov dstq, dstmp + mov dsq, dsmp + mova m5, [base+spel_h_shuf2] + ALLOC_STACK -16*8 +%else + mova m6, [base+pd_512] + mova m9, [base+spel_h_shuf2] +%endif + pshuflw m0, m0, q2121 + pxor m7, m7 + punpcklbw m7, m0 + punpcklbw m3, m3 + psraw m3, 8 ; sign-extend + test dword r8m, 0x800 + jz .hv_w2_10bpc + psraw m7, 2 + psllw m3, 2 +.hv_w2_10bpc: + lea r6, [ssq*3] + sub srcq, 2 + sub srcq, r6 +%if ARCH_X86_32 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova m9, m5 + mova m11, m0 + mova m12, m1 + mova m13, m2 + mova m14, m3 + mova m15, m4 +%else + pshufd m11, m3, q0000 + pshufd m12, m3, q1111 + pshufd m13, m3, q2222 + pshufd m14, m3, q3333 +%endif + movu m2, [srcq+ssq*0] + movu m3, [srcq+ssq*1] + movu m1, [srcq+ssq*2] + add srcq, r6 + movu m4, [srcq+ssq*0] +%if ARCH_X86_32 + REPX {pshufb x, m5}, m2, m3, m1, m4 +%else + REPX {pshufb x, m9}, m2, m3, m1, m4 +%endif + REPX {pmaddwd x, m7}, m2, m3, m1, m4 + phaddd m2, m3 ; 0 1 + phaddd m1, m4 ; 2 3 + movu m3, [srcq+ssq*1] + movu m4, [srcq+ssq*2] + add srcq, r6 + movu m0, [srcq+ssq*0] +%if ARCH_X86_32 + REPX {pshufb x, m5}, m3, m4, m0 +%else + REPX {pshufb x, m9}, m3, m4, m0 +%endif + REPX {pmaddwd x, m7}, m3, m4, m0 + phaddd m3, m4 ; 4 5 + phaddd m0, m0 ; 6 6 + REPX {paddd x, m6}, m2, m1, m3, m0 + REPX {psrad x, 10}, m2, m1, m3, m0 + packssdw m2, m1 ; 0 1 2 3 + packssdw m3, m0 ; 4 5 6 _ + palignr m4, m3, m2, 4 ; 1 2 3 4 + pshufd m5, m3, q0321 ; 5 6 _ _ + punpcklwd m1, m2, m4 ; 01 12 + punpckhwd m2, m4 ; 23 34 + punpcklwd m3, m5 ; 45 56 +.hv_w2_loop: + movu m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movu m5, [srcq+ssq*0] + pshufb m4, m9 + pshufb m5, m9 + pmaddwd m4, m7 + pmaddwd m5, m7 + phaddd m4, m5 + pmaddwd m5, m11, m1 ; a0 b0 + mova m1, m2 + pmaddwd m2, m12 ; a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m13 ; a2 b2 + paddd m5, m3 + paddd m4, m6 + psrad m4, 10 ; 7 8 + packssdw m0, m4 + pshufd m3, m0, q2103 + punpckhwd m3, m0 ; 67 78 + mova m0, m4 + pmaddwd m4, m14, m3 ; a3 b3 + paddd m5, m6 + paddd m5, m4 + psrad m5, 10 + packssdw m5, m5 + pxor m4, m4 + pminsw m5, m15 + pmaxsw m5, m4 + movd [dstq+dsq*0], m5 + pshuflw m5, m5, q3232 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w8: + shr mxd, 16 +.hv_w4: + movq m2, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovb myd, mxd + movq m3, [base+subpel_filters+myq*8] +%if ARCH_X86_32 +%if STACK_ALIGNMENT < 16 + %xdefine rstk rsp +%else + %assign stack_offset stack_offset - stack_size_padded +%endif + mov dstq, dstmp + mov dsq, dsmp + mova m0, [base+spel_h_shufA] + mova m1, [base+spel_h_shufB] + ALLOC_STACK -16*15 + mova m8, m0 + mova m9, m1 + mova m14, m6 +%else + mova m8, [base+spel_h_shufA] + mova m9, [base+spel_h_shufB] +%endif + pxor m0, m0 + punpcklbw m0, m2 + punpcklbw m3, m3 + psraw m3, 8 + test dword r8m, 0x800 + jz .hv_w4_10bpc + psraw m0, 2 + psllw m3, 2 +.hv_w4_10bpc: + lea r6, [ssq*3] + sub srcq, 6 + sub srcq, r6 +%if ARCH_X86_32 + %define tmp esp+16*8 + shl wd, 14 +%if STACK_ALIGNMENT < 16 + mov [esp+4*61], srcq + mov [esp+4*62], dstq +%else + mov srcmp, srcq +%endif + mova [tmp+16*5], m4 + lea wd, [wq+hq-(1<<16)] + pshufd m1, m0, q0000 + pshufd m2, m0, q1111 + pshufd m5, m0, q2222 + pshufd m0, m0, q3333 + mova m10, m1 + mova m11, m2 + mova m12, m5 + mova m13, m0 +%else +%if WIN64 + %define tmp rsp +%else + %define tmp rsp-104 ; red zone +%endif + shl wd, 6 + mov r7, srcq + mov r8, dstq + lea wd, [wq+hq-(1<<8)] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 + mova [tmp+16*5], m15 +%endif + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova [tmp+16*1], m0 + mova [tmp+16*2], m1 + mova [tmp+16*3], m2 + mova [tmp+16*4], m3 +%macro PUT_8TAP_HV_H 4-5 m14 ; dst/src+0, src+8, tmp, shift, [pd_512] + pshufb m%3, m%1, m8 ; 0 1 1 2 2 3 3 4 + pshufb m%1, m9 ; 2 3 3 4 4 5 5 6 + pmaddwd m%3, m10 + pmaddwd m%1, m11 + paddd m%3, %5 + paddd m%1, m%3 + pshufb m%3, m%2, m8 ; 4 5 5 6 6 7 7 8 + pshufb m%2, m9 ; 6 7 7 8 8 9 9 a + pmaddwd m%3, m12 + pmaddwd m%2, m13 + paddd m%1, m%3 + paddd m%1, m%2 + psrad m%1, %4 +%endmacro +.hv_w4_loop0: +%if ARCH_X86_64 + mova m14, [pd_512] +%endif + movu m4, [srcq+ssq*0+0] + movu m1, [srcq+ssq*0+8] + movu m5, [srcq+ssq*1+0] + movu m2, [srcq+ssq*1+8] + movu m6, [srcq+ssq*2+0] + movu m3, [srcq+ssq*2+8] + add srcq, r6 + PUT_8TAP_HV_H 4, 1, 0, 10 + PUT_8TAP_HV_H 5, 2, 0, 10 + PUT_8TAP_HV_H 6, 3, 0, 10 + movu m7, [srcq+ssq*0+0] + movu m2, [srcq+ssq*0+8] + movu m1, [srcq+ssq*1+0] + movu m3, [srcq+ssq*1+8] + PUT_8TAP_HV_H 7, 2, 0, 10 + PUT_8TAP_HV_H 1, 3, 0, 10 + movu m2, [srcq+ssq*2+0] + movu m3, [srcq+ssq*2+8] + add srcq, r6 + PUT_8TAP_HV_H 2, 3, 0, 10 + packssdw m4, m7 ; 0 3 + packssdw m5, m1 ; 1 4 + movu m0, [srcq+ssq*0+0] + movu m1, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 1, 3, 10 + packssdw m6, m2 ; 2 5 + packssdw m7, m0 ; 3 6 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +%if ARCH_X86_32 + jmp .hv_w4_loop_start +.hv_w4_loop: + mova m1, [tmp+16*6] + mova m2, m15 +.hv_w4_loop_start: + mova m7, [tmp+16*1] + pmaddwd m1, m7 ; a0 + pmaddwd m2, m7 ; b0 + mova m7, [tmp+16*2] + mova [tmp+16*6], m3 + pmaddwd m3, m7 ; a1 + mova m15, m4 + pmaddwd m4, m7 ; b1 + mova m7, [tmp+16*3] + paddd m1, m3 + paddd m2, m4 + mova m3, m5 + pmaddwd m5, m7 ; a2 + mova m4, m6 + pmaddwd m6, m7 ; b2 + paddd m1, m5 + paddd m2, m6 + movu m7, [srcq+ssq*1+0] + movu m5, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + PUT_8TAP_HV_H 7, 5, 6, 10 + packssdw m0, m7 ; 6 7 + mova [tmp+16*0], m0 + movu m0, [srcq+ssq*0+0] + movu m5, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 5, 6, 10 + mova m6, [tmp+16*0] + packssdw m7, m0 ; 7 8 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, [tmp+16*4] + paddd m1, m7 ; a3 + pmaddwd m7, m6, [tmp+16*4] + paddd m2, m7 ; b3 + psrad m1, 9 + psrad m2, 9 + packssdw m1, m2 + pxor m7, m7 + pmaxsw m1, m7 + pavgw m7, m1 + pminsw m7, [tmp+16*5] + movq [dstq+dsq*0], m7 + movhps [dstq+dsq*1], m7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop +%if STACK_ALIGNMENT < 16 + mov srcq, [esp+4*61] + mov dstq, [esp+4*62] + add srcq, 8 + add dstq, 8 + mov [esp+4*61], srcq + mov [esp+4*62], dstq +%else + mov srcq, srcmp + mov dstq, dstmp + add srcq, 8 + add dstq, 8 + mov srcmp, srcq + mov dstmp, dstq +%endif + movzx hd, ww + sub wd, 1<<16 +%else +.hv_w4_loop: + mova m15, [tmp+16*1] + pmaddwd m14, m15, m1 ; a0 + pmaddwd m15, m2 ; b0 + mova m7, [tmp+16*2] + mova m1, m3 + pmaddwd m3, m7 ; a1 + mova m2, m4 + pmaddwd m4, m7 ; b1 + mova m7, [tmp+16*3] + paddd m14, m3 + paddd m15, m4 + mova m3, m5 + pmaddwd m5, m7 ; a2 + mova m4, m6 + pmaddwd m6, m7 ; b2 + paddd m14, m5 + paddd m15, m6 + movu m7, [srcq+ssq*1+0] + movu m5, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + PUT_8TAP_HV_H 7, 5, 6, 10, [pd_512] + packssdw m0, m7 ; 6 7 + mova [tmp+16*0], m0 + movu m0, [srcq+ssq*0+0] + movu m5, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 5, 6, 10, [pd_512] + mova m6, [tmp+16*0] + packssdw m7, m0 ; 7 8 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, [tmp+16*4] + paddd m14, m7 ; a3 + pmaddwd m7, m6, [tmp+16*4] + paddd m15, m7 ; b3 + psrad m14, 9 + psrad m15, 9 + packssdw m14, m15 + pxor m7, m7 + pmaxsw m14, m7 + pavgw m7, m14 + pminsw m7, [tmp+16*5] + movq [dstq+dsq*0], m7 + movhps [dstq+dsq*1], m7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + add r7, 8 + add r8, 8 + movzx hd, wb + mov srcq, r7 + mov dstq, r8 + sub wd, 1<<8 +%endif + jg .hv_w4_loop0 + RET +%undef tmp + +%if ARCH_X86_32 +DECLARE_REG_TMP 2, 1, 6, 4 +%elif WIN64 +DECLARE_REG_TMP 6, 4, 7, 4 +%else +DECLARE_REG_TMP 6, 7, 7, 8 +%endif + +%define PREP_8TAP_FN FN prep_8tap, +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +%if ARCH_X86_32 +cglobal prep_8tap_16bpc, 0, 7, 8, tmp, src, ss, w, h, mx, my +%define mxb r0b +%define mxd r0 +%define mxq r0 +%define myb r2b +%define myd r2 +%define myq r2 +%else +cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, ss, w, h, mx, my +%endif +%define base t2-prep_ssse3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + LEA t2, prep_ssse3 + movifnidn wd, wm + movifnidn srcq, srcmp + test mxd, 0xf00 + jnz .h + movifnidn hd, hm + test myd, 0xf00 + jnz .v + tzcnt wd, wd + mov myd, r7m ; bitdepth_max + movzx wd, word [base+prep_ssse3_table+wq*2] + mova m5, [base+pw_8192] + shr myd, 11 + add wq, t2 + movddup m4, [base+prep_mul+myq*8] + movifnidn ssq, ssmp + movifnidn tmpq, tmpmp + lea r6, [ssq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + movifnidn ssq, r2mp + movifnidn hd, r4m + movddup m5, [base+prep_8tap_1d_rnd] + cmp wd, 4 + jne .h_w8 + movzx mxd, mxb + movq m0, [base+subpel_filters+mxq*8] + mova m3, [base+spel_h_shufA] + mova m4, [base+spel_h_shufB] + movifnidn tmpq, tmpmp + sub srcq, 2 + WIN64_SPILL_XMM 8 + punpcklbw m0, m0 + psraw m0, 8 + test dword r7m, 0x800 + jnz .h_w4_12bpc + psllw m0, 2 +.h_w4_12bpc: + pshufd m6, m0, q1111 + pshufd m7, m0, q2222 +.h_w4_loop: + movu m1, [srcq+ssq*0] + movu m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m1, m3 ; 0 1 1 2 2 3 3 4 + pshufb m1, m4 ; 2 3 3 4 4 5 5 6 + pmaddwd m0, m6 + pmaddwd m1, m7 + paddd m0, m5 + paddd m0, m1 + pshufb m1, m2, m3 + pshufb m2, m4 + pmaddwd m1, m6 + pmaddwd m2, m7 + paddd m1, m5 + paddd m1, m2 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + mova [tmpq], m0 + add tmpq, 16 + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + WIN64_SPILL_XMM 11 + shr mxd, 16 + movq m2, [base+subpel_filters+mxq*8] + mova m4, [base+spel_h_shufA] + mova m6, [base+spel_h_shufB] + movifnidn tmpq, r0mp + add wd, wd + punpcklbw m2, m2 + add srcq, wq + psraw m2, 8 + add tmpq, wq + neg wq + test dword r7m, 0x800 + jnz .h_w8_12bpc + psllw m2, 2 +.h_w8_12bpc: + pshufd m7, m2, q0000 +%if ARCH_X86_32 + ALLOC_STACK -16*3 + pshufd m0, m2, q1111 + pshufd m1, m2, q2222 + pshufd m2, m2, q3333 + mova m8, m0 + mova m9, m1 + mova m10, m2 +%else + pshufd m8, m2, q1111 + pshufd m9, m2, q2222 + pshufd m10, m2, q3333 +%endif +.h_w8_loop0: + mov r6, wq +.h_w8_loop: + movu m0, [srcq+r6- 6] + movu m1, [srcq+r6+ 2] + pshufb m2, m0, m4 ; 0 1 1 2 2 3 3 4 + pshufb m0, m6 ; 2 3 3 4 4 5 5 6 + pmaddwd m2, m7 ; abcd0 + pmaddwd m0, m8 ; abcd1 + pshufb m3, m1, m4 ; 4 5 5 6 6 7 7 8 + pshufb m1, m6 ; 6 7 7 8 8 9 9 a + paddd m2, m5 + paddd m0, m2 + pmaddwd m2, m9, m3 ; abcd2 + pmaddwd m3, m7 ; efgh0 + paddd m0, m2 + pmaddwd m2, m10, m1 ; abcd3 + pmaddwd m1, m8 ; efgh1 + paddd m0, m2 + movu m2, [srcq+r6+10] + paddd m3, m5 + paddd m1, m3 + pshufb m3, m2, m4 ; a b b c c d d e + pshufb m2, m6 ; 8 9 9 a a b b c + pmaddwd m3, m9 ; efgh2 + pmaddwd m2, m10 ; efgh3 + paddd m1, m3 + paddd m1, m2 + psrad m0, 4 + psrad m1, 4 + packssdw m0, m1 + mova [tmpq+r6], m0 + add r6, 16 + jl .h_w8_loop + add srcq, ssq + sub tmpq, wq + dec hd + jg .h_w8_loop0 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + movq m3, [base+subpel_filters+myq*8] +%if STACK_ALIGNMENT < 16 + %xdefine rstk rsp +%else + %assign stack_offset stack_offset - stack_size_padded +%endif + WIN64_SPILL_XMM 15 + movddup m7, [base+prep_8tap_1d_rnd] + movifnidn ssq, r2mp + movifnidn tmpq, r0mp + punpcklbw m3, m3 + psraw m3, 8 ; sign-extend + test dword r7m, 0x800 + jnz .v_12bpc + psllw m3, 2 +.v_12bpc: +%if ARCH_X86_32 + ALLOC_STACK -16*7 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 +%else + pshufd m8, m3, q0000 + pshufd m9, m3, q1111 + pshufd m10, m3, q2222 + pshufd m11, m3, q3333 +%endif + lea r6, [ssq*3] + sub srcq, r6 + mov r6d, wd + shl wd, 6 + mov r5, srcq +%if ARCH_X86_64 + mov r7, tmpq +%elif STACK_ALIGNMENT < 16 + mov [esp+4*29], tmpq +%endif + lea wd, [wq+hq-(1<<8)] +.v_loop0: + movq m1, [srcq+ssq*0] + movq m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq m3, [srcq+ssq*0] + movq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq m5, [srcq+ssq*0] + movq m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq m0, [srcq+ssq*0] + punpcklwd m1, m2 ; 01 + punpcklwd m2, m3 ; 12 + punpcklwd m3, m4 ; 23 + punpcklwd m4, m5 ; 34 + punpcklwd m5, m6 ; 45 + punpcklwd m6, m0 ; 56 +%if ARCH_X86_32 + jmp .v_loop_start +.v_loop: + mova m1, m12 + mova m2, m13 + mova m3, m14 +.v_loop_start: + pmaddwd m1, m8 ; a0 + pmaddwd m2, m8 ; b0 + mova m12, m3 + mova m13, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m1, m3 + paddd m2, m4 + mova m14, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m1, m5 + paddd m2, m6 + movq m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklwd m5, m0, m6 ; 67 + movq m0, [srcq+ssq*0] + pmaddwd m3, m11, m5 ; a3 + punpcklwd m6, m0 ; 78 + paddd m1, m7 + paddd m1, m3 + pmaddwd m3, m11, m6 ; b3 + paddd m2, m7 + paddd m2, m3 + psrad m1, 4 + psrad m2, 4 + packssdw m1, m2 + movq [tmpq+r6*0], m1 + movhps [tmpq+r6*2], m1 + lea tmpq, [tmpq+r6*4] + sub hd, 2 + jg .v_loop +%if STACK_ALIGNMENT < 16 + mov tmpq, [esp+4*29] + add r5, 8 + add tmpq, 8 + mov srcq, r5 + mov [esp+4*29], tmpq +%else + mov tmpq, tmpmp + add r5, 8 + add tmpq, 8 + mov srcq, r5 + mov tmpmp, tmpq +%endif +%else +.v_loop: + pmaddwd m12, m8, m1 ; a0 + pmaddwd m13, m8, m2 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m9 ; a1 + pmaddwd m4, m9 ; b1 + paddd m12, m3 + paddd m13, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m10 ; a2 + pmaddwd m6, m10 ; b2 + paddd m12, m5 + paddd m13, m6 + movq m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklwd m5, m0, m6 ; 67 + movq m0, [srcq+ssq*0] + pmaddwd m14, m11, m5 ; a3 + punpcklwd m6, m0 ; 78 + paddd m12, m7 + paddd m12, m14 + pmaddwd m14, m11, m6 ; b3 + paddd m13, m7 + paddd m13, m14 + psrad m12, 4 + psrad m13, 4 + packssdw m12, m13 + movq [tmpq+r6*0], m12 + movhps [tmpq+r6*2], m12 + lea tmpq, [tmpq+r6*4] + sub hd, 2 + jg .v_loop + add r5, 8 + add r7, 8 + mov srcq, r5 + mov tmpq, r7 +%endif + movzx hd, wb + sub wd, 1<<8 + jg .v_loop0 + RET +.hv: +%if STACK_ALIGNMENT < 16 + %xdefine rstk rsp +%else + %assign stack_offset stack_offset - stack_size_padded +%endif + movzx t3d, mxb + shr mxd, 16 + cmp wd, 4 + cmove mxd, t3d + movifnidn hd, r4m + movq m2, [base+subpel_filters+mxq*8] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + movq m3, [base+subpel_filters+myq*8] +%if ARCH_X86_32 + mov ssq, r2mp + mov tmpq, r0mp + mova m0, [base+spel_h_shufA] + mova m1, [base+spel_h_shufB] + mova m4, [base+prep_8tap_2d_rnd] + ALLOC_STACK -16*14 + mova m8, m0 + mova m9, m1 + mova m14, m4 +%else +%if WIN64 + ALLOC_STACK 16*6, 16 +%endif + mova m8, [base+spel_h_shufA] + mova m9, [base+spel_h_shufB] +%endif + pxor m0, m0 + punpcklbw m0, m2 + punpcklbw m3, m3 + psraw m0, 4 + psraw m3, 8 + test dword r7m, 0x800 + jz .hv_10bpc + psraw m0, 2 +.hv_10bpc: + lea r6, [ssq*3] + sub srcq, 6 + sub srcq, r6 + mov r6d, wd + shl wd, 6 + mov r5, srcq +%if ARCH_X86_32 + %define tmp esp+16*8 +%if STACK_ALIGNMENT < 16 + mov [esp+4*61], tmpq +%endif + pshufd m1, m0, q0000 + pshufd m2, m0, q1111 + pshufd m5, m0, q2222 + pshufd m0, m0, q3333 + mova m10, m1 + mova m11, m2 + mova m12, m5 + mova m13, m0 +%else +%if WIN64 + %define tmp rsp +%else + %define tmp rsp-88 ; red zone +%endif + mov r7, tmpq + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 +%endif + lea wd, [wq+hq-(1<<8)] + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova [tmp+16*1], m0 + mova [tmp+16*2], m1 + mova [tmp+16*3], m2 + mova [tmp+16*4], m3 +.hv_loop0: +%if ARCH_X86_64 + mova m14, [prep_8tap_2d_rnd] +%endif + movu m4, [srcq+ssq*0+0] + movu m1, [srcq+ssq*0+8] + movu m5, [srcq+ssq*1+0] + movu m2, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + movu m6, [srcq+ssq*0+0] + movu m3, [srcq+ssq*0+8] + PUT_8TAP_HV_H 4, 1, 0, 6 + PUT_8TAP_HV_H 5, 2, 0, 6 + PUT_8TAP_HV_H 6, 3, 0, 6 + movu m7, [srcq+ssq*1+0] + movu m2, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + movu m1, [srcq+ssq*0+0] + movu m3, [srcq+ssq*0+8] + PUT_8TAP_HV_H 7, 2, 0, 6 + PUT_8TAP_HV_H 1, 3, 0, 6 + movu m2, [srcq+ssq*1+0] + movu m3, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + PUT_8TAP_HV_H 2, 3, 0, 6 + packssdw m4, m7 ; 0 3 + packssdw m5, m1 ; 1 4 + movu m0, [srcq+ssq*0+0] + movu m1, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 1, 3, 6 + packssdw m6, m2 ; 2 5 + packssdw m7, m0 ; 3 6 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +%if ARCH_X86_32 + jmp .hv_loop_start +.hv_loop: + mova m1, [tmp+16*5] + mova m2, m15 +.hv_loop_start: + mova m7, [tmp+16*1] + pmaddwd m1, m7 ; a0 + pmaddwd m2, m7 ; b0 + mova m7, [tmp+16*2] + mova [tmp+16*5], m3 + pmaddwd m3, m7 ; a1 + mova m15, m4 + pmaddwd m4, m7 ; b1 + mova m7, [tmp+16*3] + paddd m1, m14 + paddd m2, m14 + paddd m1, m3 + paddd m2, m4 + mova m3, m5 + pmaddwd m5, m7 ; a2 + mova m4, m6 + pmaddwd m6, m7 ; b2 + paddd m1, m5 + paddd m2, m6 + movu m7, [srcq+ssq*1+0] + movu m5, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + PUT_8TAP_HV_H 7, 5, 6, 6 + packssdw m0, m7 ; 6 7 + mova [tmp+16*0], m0 + movu m0, [srcq+ssq*0+0] + movu m5, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 5, 6, 6 + mova m6, [tmp+16*0] + packssdw m7, m0 ; 7 8 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, [tmp+16*4] + paddd m1, m7 ; a3 + pmaddwd m7, m6, [tmp+16*4] + paddd m2, m7 ; b3 + psrad m1, 6 + psrad m2, 6 + packssdw m1, m2 + movq [tmpq+r6*0], m1 + movhps [tmpq+r6*2], m1 + lea tmpq, [tmpq+r6*4] + sub hd, 2 + jg .hv_loop +%if STACK_ALIGNMENT < 16 + mov tmpq, [esp+4*61] + add r5, 8 + add tmpq, 8 + mov srcq, r5 + mov [esp+4*61], tmpq +%else + mov tmpq, tmpmp + add r5, 8 + add tmpq, 8 + mov srcq, r5 + mov tmpmp, tmpq +%endif +%else +.hv_loop: + mova m15, [tmp+16*1] + mova m7, [prep_8tap_2d_rnd] + pmaddwd m14, m15, m1 ; a0 + pmaddwd m15, m2 ; b0 + paddd m14, m7 + paddd m15, m7 + mova m7, [tmp+16*2] + mova m1, m3 + pmaddwd m3, m7 ; a1 + mova m2, m4 + pmaddwd m4, m7 ; b1 + mova m7, [tmp+16*3] + paddd m14, m3 + paddd m15, m4 + mova m3, m5 + pmaddwd m5, m7 ; a2 + mova m4, m6 + pmaddwd m6, m7 ; b2 + paddd m14, m5 + paddd m15, m6 + movu m7, [srcq+ssq*1+0] + movu m5, [srcq+ssq*1+8] + lea srcq, [srcq+ssq*2] + PUT_8TAP_HV_H 7, 5, 6, 6, [prep_8tap_2d_rnd] + packssdw m0, m7 ; 6 7 + mova [tmp+16*0], m0 + movu m0, [srcq+ssq*0+0] + movu m5, [srcq+ssq*0+8] + PUT_8TAP_HV_H 0, 5, 6, 6, [prep_8tap_2d_rnd] + mova m6, [tmp+16*0] + packssdw m7, m0 ; 7 8 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, [tmp+16*4] + paddd m14, m7 ; a3 + pmaddwd m7, m6, [tmp+16*4] + paddd m15, m7 ; b3 + psrad m14, 6 + psrad m15, 6 + packssdw m14, m15 + movq [tmpq+r6*0], m14 + movhps [tmpq+r6*2], m14 + lea tmpq, [tmpq+r6*4] + sub hd, 2 + jg .hv_loop + add r5, 8 + add r7, 8 + mov srcq, r5 + mov tmpq, r7 +%endif + movzx hd, wb + sub wd, 1<<8 + jg .hv_loop0 + RET +%undef tmp + +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro SAVE_REG 1 + %xdefine r%1_save r%1 + %xdefine r%1q_save r%1q + %xdefine r%1d_save r%1d + %if ARCH_X86_32 + %define r%1m_save [rstk+stack_offset+(%1+1)*4] + %endif +%endmacro + +%macro LOAD_REG 1 + %xdefine r%1 r%1_save + %xdefine r%1q r%1q_save + %xdefine r%1d r%1d_save + %if ARCH_X86_32 + %define r%1m r%1m_save + %endif + %undef r%1d_save + %undef r%1q_save + %undef r%1_save +%endmacro + +%macro REMAP_REG 2-3 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d + %if ARCH_X86_32 + %if %3 == 0 + %xdefine r%1m r%2m + %else + %define r%1m [rstk+stack_offset+(%1+1)*4] + %endif + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %if ARCH_X86_64 + SAVE_REG 14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %else + SAVE_REG 5 + %assign %%i 5 + %rep 5 + %assign %%j %%i-1 + REMAP_REG %%i, %%j, 0 + %assign %%i %%i-1 + %endrep + %endif + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %if ARCH_X86_64 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + LOAD_REG 14 + %else + %rep 4 + %assign %%j %%i+1 + REMAP_REG %%i, %%j, 1 + %assign %%i %%i+1 + %endrep + LOAD_REG 5 + %endif + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%if ARCH_X86_32 + %macro MC_4TAP_SCALED_H 1 ; dst_mem + movu m7, [srcq+ssq*0] + movu m2, [srcq+ssq*1] + movu m5, [r4 +ssq*0] + movu m6, [r4 +ssq*1] + lea srcq, [srcq+ssq*2] + lea r4, [r4 +ssq*2] + REPX {pshufb x, m12}, m7, m2 + REPX {pmaddwd x, m13}, m7, m2 + REPX {pshufb x, m14}, m5, m6 + REPX {pmaddwd x, m15}, m5, m6 + phaddd m7, m5 + phaddd m2, m6 + mova m5, [esp+0x00] + movd m6, [esp+0x10] + paddd m7, m5 + paddd m2, m5 + psrad m7, m6 + psrad m2, m6 + packssdw m7, m2 + mova [stk+%1], m7 + %endmacro +%endif + +%if ARCH_X86_64 + %macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6] + movu m%1, [srcq+ r4*2] + movu m%2, [srcq+ r6*2] + movu m%3, [srcq+ r7*2] + movu m%4, [srcq+ r9*2] + movu m%5, [srcq+r10*2] + movu m%6, [srcq+r11*2] + movu m%7, [srcq+r13*2] + movu m%8, [srcq+ rX*2] + add srcq, ssq + pmaddwd m%1, [stk+0x10] + pmaddwd m%2, [stk+0x20] + pmaddwd m%3, [stk+0x30] + pmaddwd m%4, [stk+0x40] + pmaddwd m%5, [stk+0x50] + pmaddwd m%6, [stk+0x60] + pmaddwd m%7, [stk+0x70] + pmaddwd m%8, [stk+0x80] + phaddd m%1, m%2 + phaddd m%3, m%4 + phaddd m%5, m%6 + phaddd m%7, m%8 + phaddd m%1, m%3 + phaddd m%5, m%7 + paddd m%1, hround + paddd m%5, hround + psrad m%1, m12 + psrad m%5, m12 + packssdw m%1, m%5 + %endmacro +%else + %macro MC_8TAP_SCALED_H 2-3 1 ; weights_mem_start, h_mem, load_fh_offsets + %if %3 == 1 + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + %endif + movu m0, [srcq+r0*2] + movu m1, [srcq+rX*2] + movu m2, [srcq+r4*2] + movu m3, [srcq+r5*2] + mov r0, [stk+16] + mov rX, [stk+20] + mov r4, [stk+24] + mov r5, [stk+28] + pmaddwd m0, [stk+%1+0x00] + pmaddwd m1, [stk+%1+0x10] + pmaddwd m2, [stk+%1+0x20] + pmaddwd m3, [stk+%1+0x30] + phaddd m0, m1 + phaddd m2, m3 + movu m4, [srcq+r0*2] + movu m5, [srcq+rX*2] + movu m6, [srcq+r4*2] + movu m7, [srcq+r5*2] + add srcq, ssq + pmaddwd m4, [stk+%1+0xa0] + pmaddwd m5, [stk+%1+0xb0] + pmaddwd m6, [stk+%1+0xc0] + pmaddwd m7, [stk+%1+0xd0] + phaddd m4, m5 + phaddd m6, m7 + phaddd m0, m2 + phaddd m4, m6 + paddd m0, hround + paddd m4, hround + psrad m0, m12 + psrad m4, m12 + packssdw m0, m4 + %if %2 != 0 + mova [stk+%2], m0 + %endif + %endmacro +%endif + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isput 1 + %assign isprep 0 + %if ARCH_X86_64 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled_16bpc, 2, 15, 16, 0x1c0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %else +cglobal put_8tap_scaled_16bpc, 2, 14, 16, 0x1c0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %endif + %else ; ARCH_X86_32 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled_16bpc, 0, 7, 8, 0x200, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %else +cglobal put_8tap_scaled_16bpc, 0, 7, 8, -0x200-0x30, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %endif + %endif + %xdefine base_reg r12 +%else ; prep + %assign isput 0 + %assign isprep 1 + %if ARCH_X86_64 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled_16bpc, 2, 15, 16, 0x1c0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %xdefine tmp_stridem r14q + %else +cglobal prep_8tap_scaled_16bpc, 2, 14, 16, 0x1c0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %define tmp_stridem qword [stk+0x138] + %endif + %xdefine base_reg r11 + %else ; ARCH_X86_32 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled_16bpc, 0, 7, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %else +cglobal prep_8tap_scaled_16bpc, 0, 6, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %endif + %define tmp_stridem dword [stk+0x138] + %endif +%endif +%if ARCH_X86_32 + mov [esp+0x1f0], t0d + mov [esp+0x1f4], t1d + %if isput && required_stack_alignment > STACK_ALIGNMENT + mov dstd, dstm + mov dsd, dsm + mov srcd, srcm + mov ssd, ssm + mov hd, hm + mov r4, mxm + %define r0m [esp+0x200] + %define dsm [esp+0x204] + %define dsmp dsm + %define r1m dsm + %define r2m [esp+0x208] + %define ssm [esp+0x20c] + %define r3m ssm + %define hm [esp+0x210] + %define mxm [esp+0x214] + mov r0m, dstd + mov dsm, dsd + mov r2m, srcd + mov ssm, ssd + mov hm, hd + mov r0, mym + mov r1, dxm + mov r2, dym + %define mym [esp+0x218] + %define dxm [esp+0x21c] + %define dym [esp+0x220] + mov mxm, r4 + mov mym, r0 + mov dxm, r1 + mov dym, r2 + tzcnt wd, wm + %endif + %if isput + mov r3, pxmaxm + %define pxmaxm r3 + %else + mov r2, pxmaxm + %endif + %if isprep && required_stack_alignment > STACK_ALIGNMENT + %xdefine base_reg r5 + %else + %xdefine base_reg r6 + %endif +%endif + LEA base_reg, %1_8tap_scaled_16bpc_ssse3 +%xdefine base base_reg-%1_8tap_scaled_16bpc_ssse3 +%if ARCH_X86_64 || isprep || required_stack_alignment <= STACK_ALIGNMENT + tzcnt wd, wm +%endif +%if ARCH_X86_64 + %if isput + mov r7d, pxmaxm + %endif +%else + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 +%endif + movd m8, dxm + movd m14, mxm +%if isput + movd m15, pxmaxm +%endif + pshufd m8, m8, q0000 + pshufd m14, m14, q0000 +%if isput + pshuflw m15, m15, q0000 + punpcklqdq m15, m15 +%endif +%if isprep + %if UNIX64 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 + %endif + %if ARCH_X86_64 + mov r6d, pxmaxm + %endif +%endif +%if ARCH_X86_64 + mov dyd, dym +%endif +%if isput + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %elif ARCH_X86_64 + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %else + %endif + %if ARCH_X86_64 + %if required_stack_alignment > STACK_ALIGNMENT + %define dsm [rsp+0x138] + %define rX r1 + %define rXd r1d + %else + %define dsm dsq + %define rX r14 + %define rXd r14d + %endif + %else + %define rX r1 + %endif +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %elif ARCH_X86_64 + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %xdefine hm r7m + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %if ARCH_X86_64 + %define rX r14 + %define rXd r14d + %else + %define rX r3 + %endif +%endif +%if ARCH_X86_64 + shr r7d, 11 + mova m10, [base+pd_0x3ff] + movddup m11, [base+s_8tap_h_rnd+r7*8] + movd m12, [base+s_8tap_h_sh+r7*4] + %if isput + movddup m13, [base+put_s_8tap_v_rnd+r7*8] + movd m7, [base+put_s_8tap_v_sh+r7*4] + %define pxmaxm [rsp] + mova pxmaxm, m15 + punpcklqdq m12, m7 + %endif + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q +%else + %define m10 [base+pd_0x3ff] + %define m11 [esp+0x00] + %define m12 [esp+0x10] + shr r3, 11 + movddup m1, [base+s_8tap_h_rnd+r3*8] + movd m2, [base+s_8tap_h_sh+r3*4] + %if isput + %define m13 [esp+0x20] + %define pxmaxm [esp+0x30] + %define stk esp+0x40 + movddup m5, [base+put_s_8tap_v_rnd+r3*8] + movd m6, [base+put_s_8tap_v_sh+r3*4] + mova pxmaxm, m15 + punpcklqdq m2, m6 + mova m13, m5 + %else + %define m13 [base+pd_m524256] + %endif + mov ssd, ssm + mova m11, m1 + mova m12, m2 + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + mov r1, [esp+0x1f4] + lea r0, [ssd*3] + movzx r2, r1b + shr r1, 16 + cmp dword hm, 6 + cmovs r1, r2 + mov [esp+0x1f4], r1 + %if isprep + mov r1, r1m + %endif + mov r2, r2m + sub srcq, r0 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define ss3q r0 + %define myd r4 + %define dyd dword dym + %define hd dword hm +%endif + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_ssse3_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.w2: + %if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d + %else + movzx r4, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r4 + %endif + pxor m9, m9 + punpckldq m9, m8 + paddd m14, m9 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m9, [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + pshufd m15, m15, q0321 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_q] + mova m6, [base+spel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m2, m2 + pcmpeqd m8, m2 + psrld m14, 10 + paddd m14, m14 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [stk], m14 + SWAP m5, m0 + SWAP m6, m3 + %define m15 m6 + %endif + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + movu m2, [srcq+ssq*2] + movu m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + punpckldq m15, m7 + %if ARCH_X86_64 + pshufb m14, m5 + paddb m14, m6 + pand m9, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m9 + movu m4, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + movu m7, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + %else + pand m7, m5, [base+pd_0x4000] + pandn m5, m15 + por m5, m7 + %define m15 m5 + %endif + punpcklbw m15, m15 + psraw m15, 8 + REPX {pshufb x, m14}, m0, m1, m2, m3 + REPX {pmaddwd x, m15}, m0, m1, m2, m3 + %if ARCH_X86_64 + REPX {pshufb x, m14}, m4, m5, m6, m7 + REPX {pmaddwd x, m15}, m4, m5, m6, m7 + phaddd m0, m1 + phaddd m2, m3 + phaddd m4, m5 + phaddd m6, m7 + REPX {paddd x, m11}, m0, m2, m4, m6 + REPX {psrad x, m12}, m0, m2, m4, m6 + packssdw m0, m2 ; 0 1 2 3 + packssdw m4, m6 ; 4 5 6 7 + SWAP m1, m4 + %else + mova [stk+0x10], m15 + phaddd m0, m1 + phaddd m2, m3 + movu m1, [srcq+ssq*0] + movu m7, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + movu m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m14}, m1, m7, m6, m3 + REPX {pmaddwd x, m15}, m1, m7, m6, m3 + phaddd m1, m7 + phaddd m6, m3 + REPX {paddd x, m11}, m0, m2, m1, m6 + REPX {psrad x, m12}, m0, m2, m1, m6 + packssdw m0, m2 + packssdw m1, m6 + %define m14 [stk+0x00] + %define m15 [stk+0x10] + %endif + palignr m2, m1, m0, 4 ; 1 2 3 4 + punpcklwd m3, m0, m2 ; 01 12 + punpckhwd m0, m2 ; 23 34 + pshufd m5, m1, q0321 ; 5 6 7 _ + punpcklwd m2, m1, m5 ; 45 56 + punpckhwd m4, m1, m5 ; 67 __ + %if ARCH_X86_32 + mov myd, mym + mov r0, r0m + mova [stk+0x20], m3 + mova [stk+0x30], m0 + mova [stk+0x40], m2 + mova [stk+0x50], m4 + %endif +.w2_loop: + and myd, 0x3ff + %if ARCH_X86_64 + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq m10, r6q + punpcklbw m10, m10 + psraw m10, 8 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pmaddwd m5, m3, m7 + pmaddwd m6, m0, m8 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + pmaddwd m7, m2, m9 + pmaddwd m8, m4, m10 + paddd m5, m6 + paddd m7, m8 + %else + mov r1, [esp+0x1f4] + xor r3, r3 + mov r5, myd + shr r5, 6 + lea r1, [r1+r5] + mov r5, 64 << 24 + cmovnz r3, [base+subpel_filters+r1*8+4] + cmovnz r5, [base+subpel_filters+r1*8+0] + movd m6, r3 + movd m7, r5 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m5, m7, q0000 + pshufd m6, m7, q1111 + pmaddwd m3, m5 + pmaddwd m0, m6 + pshufd m5, m7, q2222 + pshufd m7, m7, q3333 + pmaddwd m2, m5 + pmaddwd m4, m7 + paddd m3, m0 + paddd m2, m4 + SWAP m5, m3 + SWAP m7, m2 + %define m8 m3 + %endif + paddd m5, m13 + pshufd m6, m12, q1032 + pxor m8, m8 + paddd m5, m7 + psrad m5, m6 + packssdw m5, m5 + pmaxsw m5, m8 + pminsw m5, pxmaxm + movd [dstq], m5 + add dstq, dsmp + dec hd + jz .ret + %if ARCH_X86_64 + add myd, dyd + %else + add myd, dym + %endif + test myd, ~0x3ff + %if ARCH_X86_32 + SWAP m3, m5 + SWAP m2, m7 + mova m3, [stk+0x20] + mova m0, [stk+0x30] + mova m2, [stk+0x40] + mova m4, [stk+0x50] + %endif + jz .w2_loop + %if ARCH_X86_32 + mov r3, r3m + %endif + movu m5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps m3, m0, q1032 ; 01 12 + shufps m0, m2, q1032 ; 23 34 + shufps m2, m4, q1032 ; 45 56 + pshufb m5, m14 + pmaddwd m5, m15 + phaddd m5, m5 + paddd m5, m11 + psrad m5, m12 + packssdw m5, m5 + palignr m4, m5, m1, 12 + punpcklqdq m1, m4, m4 ; 6 7 6 7 + punpcklwd m4, m1, m5 ; 67 __ + %if ARCH_X86_32 + mova [stk+0x20], m3 + mova [stk+0x30], m0 + mova [stk+0x40], m2 + mova [stk+0x50], m4 + %endif + jmp .w2_loop +.w2_skip_line: + movu m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m3, m0 ; 01 12 + mova m0, m2 ; 23 34 + pshufb m5, m14 + pshufb m6, m14 + pmaddwd m5, m15 + pmaddwd m6, m15 + phaddd m5, m6 + paddd m5, m11 + psrad m5, m12 + packssdw m5, m5 ; 6 7 6 7 + punpckhqdq m1, m5 ; 4 5 6 7 + pshufd m5, m1, q0321 ; 5 6 7 _ + punpcklwd m2, m1, m5 ; 45 56 + punpckhwd m4, m1, m5 ; 67 __ + %if ARCH_X86_32 + mova [stk+0x20], m3 + mova [stk+0x30], m0 + mova [stk+0x40], m2 + mova [stk+0x50], m4 + %endif + jmp .w2_loop +%endif +INIT_XMM ssse3 +.w4: +%if ARCH_X86_64 + mov myd, mym + mova [rsp+0x10], m11 + mova [rsp+0x20], m12 + %if isput + mova [rsp+0x30], m13 + %endif + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d +%else + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + movzx r4, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r4 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m9, [base+pd_0x4000] +%else + %define m9 [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd m15, m0 + pshufd m7, m15, q1032 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd r6d, m15 + movd r13d, m7 + mova m10, [base+bdct_lb_q+ 0] + mova m11, [base+bdct_lb_q+16] + movd m13, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+ r6*8+2] + movd m15, [base+subpel_filters+r11*8+2] + movd m4, [base+subpel_filters+r13*8+2] +%else + movd r0, m15 + movd r4, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd rX, m15 + movd r5, m7 + mova m5, [base+bdct_lb_q+ 0] + mova m6, [base+bdct_lb_q+16] + movd m1, [base+subpel_filters+r0*8+2] + movd m2, [base+subpel_filters+rX*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m7, [base+subpel_filters+r5*8+2] + movifprep r3, r3m + SWAP m4, m7 + %define m10 m5 + %define m11 m6 + %define m12 m1 + %define m13 m1 +%endif + psrld m14, 10 + paddd m14, m14 + punpckldq m13, m2 + punpckldq m15, m4 + punpcklqdq m13, m15 + pxor m2, m2 + pcmpeqd m0, m2 +%if ARCH_X86_64 + pand m9, m0 +%else + pand m2, m9, m0 + %define m9 m2 + SWAP m7, m4 +%endif + pandn m0, m13 +%if ARCH_X86_64 + SWAP m13, m0 +%else + %define m13 m0 +%endif + por m13, m9 + punpckhbw m15, m13, m13 + punpcklbw m13, m13 + psraw m15, 8 + psraw m13, 8 + pshufb m12, m14, m10 + pshufb m14, m11 + mova m10, [base+spel_s_shuf2] + movd r4d, m14 + shr r4d, 24 +%if ARCH_X86_32 + mova [stk+0x20], m13 + mova [stk+0x30], m15 + pxor m2, m2 +%endif + pshufb m7, m14, m2 + psubb m14, m7 + paddb m12, m10 + paddb m14, m10 +%if ARCH_X86_64 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu m7, [srcq+ssq*0] + movu m9, [srcq+ssq*1] + movu m8, [srcq+ssq*2] + movu m10, [srcq+ss3q ] + movu m1, [srcq+r4 ] + movu m3, [srcq+r6 ] + movu m2, [srcq+r11 ] + movu m4, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m12}, m7, m9, m8, m10 + REPX {pmaddwd x, m13}, m7, m9, m8, m10 + REPX {pshufb x, m14}, m1, m2, m3, m4 + REPX {pmaddwd x, m15}, m1, m2, m3, m4 + mova m5, [rsp+0x10] + movd xm6, [rsp+0x20] + phaddd m7, m1 + phaddd m9, m3 + phaddd m8, m2 + phaddd m10, m4 + movu m1, [srcq+ssq*0] + movu m2, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + movu m4, [srcq+ss3q ] + REPX {paddd x, m5}, m7, m9, m8, m10 + REPX {psrad x, xm6}, m7, m9, m8, m10 + packssdw m7, m9 ; 0 1 + packssdw m8, m10 ; 2 3 + movu m0, [srcq+r4 ] + movu m9, [srcq+r6 ] + movu m10, [srcq+r11 ] + movu m11, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m12}, m1, m2, m3, m4 + REPX {pmaddwd x, m13}, m1, m2, m3, m4 + REPX {pshufb x, m14}, m0, m9, m10, m11 + REPX {pmaddwd x, m15}, m0, m9, m10, m11 + phaddd m1, m0 + phaddd m2, m9 + phaddd m3, m10 + phaddd m4, m11 + REPX {paddd x, m5}, m1, m2, m3, m4 + REPX {psrad x, xm6}, m1, m2, m3, m4 + packssdw m1, m2 ; 4 5 + packssdw m3, m4 ; 6 7 + SWAP m9, m1 + shufps m4, m7, m8, q1032 ; 1 2 + shufps m5, m8, m9, q1032 ; 3 4 + shufps m6, m9, m3, q1032 ; 5 6 + pshufd m10, m3, q1032 ; 7 _ + punpcklwd m0, m7, m4 ; 01 + punpckhwd m7, m4 ; 12 + punpcklwd m1, m8, m5 ; 23 + punpckhwd m8, m5 ; 34 + punpcklwd m2, m9, m6 ; 45 + punpckhwd m9, m6 ; 56 + punpcklwd m3, m10 ; 67 + mova [rsp+0x40], m7 + mova [rsp+0x50], m8 + mova [rsp+0x60], m9 +%else + mova [stk+0x00], m12 + mova [stk+0x10], m14 + add r4, srcq + MC_4TAP_SCALED_H 0x40 ; 0 1 + MC_4TAP_SCALED_H 0x50 ; 2 3 + MC_4TAP_SCALED_H 0x60 ; 4 5 + MC_4TAP_SCALED_H 0x70 ; 6 7 + mova m4, [stk+0x40] + mova m5, [stk+0x50] + mova m6, [stk+0x60] + mova m7, [stk+0x70] + mov [stk+0xc0], r4 + shufps m1, m4, m5, q1032 ; 1 2 + shufps m2, m5, m6, q1032 ; 3 4 + shufps m3, m6, m7, q1032 ; 5 6 + pshufd m0, m7, q1032 ; 7 _ + mova [stk+0xb0], m0 + punpcklwd m0, m4, m1 ; 01 + punpckhwd m4, m1 ; 12 + punpcklwd m1, m5, m2 ; 23 + punpckhwd m5, m2 ; 34 + punpcklwd m2, m6, m3 ; 45 + punpckhwd m6, m3 ; 56 + punpcklwd m3, m7, [stk+0xb0] ; 67 + mov myd, mym + mov r0, r0m + mova [stk+0x40], m0 ; 01 + mova [stk+0x50], m1 ; 23 + mova [stk+0x60], m2 ; 45 + mova [stk+0x70], m3 ; 67 + mova [stk+0x80], m4 ; 12 + mova [stk+0x90], m5 ; 34 + mova [stk+0xa0], m6 ; 56 + %define m12 [stk+0x00] + %define m14 [stk+0x10] + %define m13 [stk+0x20] + %define m15 [stk+0x30] + %define hrnd_mem [esp+0x00] + %define hsh_mem [esp+0x10] + %if isput + %define vrnd_mem [esp+0x20] + %else + %define vrnd_mem [base+pd_m524256] + %endif +%endif +.w4_loop: + and myd, 0x3ff +%if ARCH_X86_64 + mov r11d, 64 << 24 + mov r13d, myd + shr r13d, 6 + lea r13d, [t1+r13] + cmovnz r11q, [base+subpel_filters+r13*8] + movq m9, r11q + punpcklbw m9, m9 + psraw m9, 8 + pshufd m7, m9, q0000 + pshufd m8, m9, q1111 + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pshufd m7, m9, q2222 + pshufd m9, m9, q3333 + pmaddwd m6, m2, m7 + pmaddwd m8, m3, m9 + %if isput + movd m9, [rsp+0x28] + %define vrnd_mem [rsp+0x30] + %else + %define vrnd_mem [base+pd_m524256] + %endif + paddd m4, m5 + paddd m6, m8 + paddd m4, m6 + paddd m4, vrnd_mem +%else + mov mym, myd + mov r5, [esp+0x1f4] + xor r3, r3 + shr r4, 6 + lea r5, [r5+r4] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + movd m7, r4 + movd m6, r3 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m4, m7, q0000 + pshufd m5, m7, q1111 + pshufd m6, m7, q2222 + pshufd m7, m7, q3333 + pmaddwd m0, m4 + pmaddwd m1, m5 + pmaddwd m2, m6 + pmaddwd m3, m7 + %if isput + movd m4, [esp+0x18] + %endif + paddd m0, m1 + paddd m2, m3 + paddd m0, vrnd_mem + paddd m0, m2 + SWAP m4, m0 + %define m9 m0 +%endif +%if isput + pxor m5, m5 + psrad m4, m9 + packssdw m4, m4 + pmaxsw m4, m5 + pminsw m4, pxmaxm + movq [dstq], m4 + add dstq, dsmp +%else + psrad m4, 6 + packssdw m4, m4 + movq [tmpq], m4 + add tmpq, 8 +%endif + dec hd + jz .ret +%if ARCH_X86_64 + add myd, dyd + test myd, ~0x3ff + jz .w4_loop + mova m8, [rsp+0x10] + movd m9, [rsp+0x20] + movu m4, [srcq] + movu m5, [srcq+r4] + test myd, 0x400 + jz .w4_skip_line + mova m0, [rsp+0x40] + mova [rsp+0x40], m1 + mova m1, [rsp+0x50] + mova [rsp+0x50], m2 + mova m2, [rsp+0x60] + mova [rsp+0x60], m3 + pshufb m4, m12 + pshufb m5, m14 + pmaddwd m4, m13 + pmaddwd m5, m15 + phaddd m4, m5 + paddd m4, m8 + psrad m4, m9 + packssdw m4, m4 + punpcklwd m3, m10, m4 + mova m10, m4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu m6, [srcq+ssq*1] + movu m7, [srcq+r6] + mova m0, [rsp+0x50] + mova m11, [rsp+0x60] + pshufb m4, m12 + pshufb m6, m12 + pshufb m5, m14 + pshufb m7, m14 + pmaddwd m4, m13 + pmaddwd m6, m13 + pmaddwd m5, m15 + pmaddwd m7, m15 + mova [rsp+0x40], m0 + mova [rsp+0x50], m11 + phaddd m4, m5 + phaddd m6, m7 + paddd m4, m8 + paddd m6, m8 + psrad m4, m9 + psrad m6, m9 + packssdw m4, m6 + punpcklwd m9, m10, m4 + mova [rsp+0x60], m9 + pshufd m10, m4, q1032 + mova m0, m1 + mova m1, m2 + mova m2, m3 + punpcklwd m3, m4, m10 + lea srcq, [srcq+ssq*2] + jmp .w4_loop +%else + SWAP m0, m4 + mov myd, mym + mov r3, r3m + add myd, dym + test myd, ~0x3ff + jnz .w4_next_line + mova m0, [stk+0x40] + mova m1, [stk+0x50] + mova m2, [stk+0x60] + mova m3, [stk+0x70] + jmp .w4_loop +.w4_next_line: + mov r5, [stk+0xc0] + movu m4, [srcq] + movu m5, [r5] + test myd, 0x400 + jz .w4_skip_line + add [stk+0xc0], ssq + mova m0, [stk+0x80] + mova m3, [stk+0x50] + mova [stk+0x40], m0 + mova [stk+0x80], m3 + mova m1, [stk+0x90] + mova m6, [stk+0x60] + mova [stk+0x50], m1 + mova [stk+0x90], m6 + mova m2, [stk+0xa0] + mova m7, [stk+0x70] + mova [stk+0x60], m2 + mova [stk+0xa0], m7 + pshufb m4, m12 + pshufb m5, m14 + pmaddwd m4, m13 + pmaddwd m5, m15 + phaddd m4, m5 + paddd m4, hrnd_mem + psrad m4, hsh_mem + packssdw m4, m4 + punpcklwd m3, [stk+0xb0], m4 + mova [stk+0xb0], m4 + mova [stk+0x70], m3 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu m6, [srcq+ssq*1] + movu m7, [r5 +ssq*1] + lea r5, [r5 +ssq*2] + mov [stk+0xc0], r5 + mova m0, [stk+0x50] + mova m1, [stk+0x60] + mova m2, [stk+0x70] + mova m3, [stk+0x90] + pshufb m4, m12 + pshufb m6, m12 + pshufb m5, m14 + pshufb m7, m14 + pmaddwd m4, m13 + pmaddwd m6, m13 + pmaddwd m5, m15 + pmaddwd m7, m15 + mova [stk+0x40], m0 + mova [stk+0x50], m1 + mova [stk+0x60], m2 + mova [stk+0x80], m3 + phaddd m4, m5 + phaddd m6, m7 + mova m5, [stk+0xa0] + mova m7, [stk+0xb0] + paddd m4, hrnd_mem + paddd m6, hrnd_mem + psrad m4, hsh_mem + psrad m6, hsh_mem + packssdw m4, m6 + punpcklwd m7, m4 + pshufd m6, m4, q1032 + mova [stk+0x90], m5 + mova [stk+0xa0], m7 + mova [stk+0xb0], m6 + punpcklwd m3, m4, m6 + mova [stk+0x70], m3 + lea srcq, [srcq+ssq*2] + jmp .w4_loop +%endif +INIT_XMM ssse3 +%if ARCH_X86_64 + %define stk rsp+0x20 +%endif +.w8: + mov dword [stk+0xf0], 1 + movifprep tmp_stridem, 16 + jmp .w_start +.w16: + mov dword [stk+0xf0], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [stk+0xf0], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [stk+0xf0], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [stk+0xf0], 16 + movifprep tmp_stridem, 256 +.w_start: +%if ARCH_X86_64 + %ifidn %1, put + movifnidn dsm, dsq + %endif + mova [rsp+0x10], m11 + %define hround m11 + shr t0d, 16 + movd m15, t0d + %if isprep + mova m13, [base+pd_m524256] + %endif +%else + %define hround [esp+0x00] + %define m12 [esp+0x10] + %define m10 [base+pd_0x3ff] + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %if isprep + %define ssq ssm + %endif + mov r4, [esp+0x1f0] + shr r4, 16 + movd m15, r4 + mov r0, r0m + mov myd, mym +%endif + sub srcq, 6 + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + mova [stk+0x100], m7 + mova [stk+0x120], m15 + mov [stk+0x0f8], srcq + mov [stk+0x130], r0q ; dstq / tmpq +%if ARCH_X86_64 && UNIX64 + mov hm, hd +%elif ARCH_X86_32 + mov r5, hm + mov [stk+0x0f4], myd + mov [stk+0x134], r5 +%endif + jmp .hloop +.hloop_prep: + dec dword [stk+0x0f0] + jz .ret +%if ARCH_X86_64 + add qword [stk+0x130], 16 + mov hd, hm +%else + add dword [stk+0x130], 16 + mov myd, [stk+0x0f4] + mov r5, [stk+0x134] + mov r0, [stk+0x130] +%endif + mova m7, [stk+0x100] + mova m14, [stk+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] + mova m11, [rsp+0x10] +%endif + mova m15, [stk+0x120] + mov srcq, [stk+0x0f8] +%if ARCH_X86_64 + mov r0q, [stk+0x130] ; dstq / tmpq +%else + mov mym, myd + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.hloop: +%if ARCH_X86_64 + mova m9, [base+pq_0x40000000] +%else + %define m9 [base+pq_0x40000000] +%endif + pxor m1, m1 + psrld m2, m14, 10 + mova [stk], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m1 + pshufd m2, m5, q1032 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pxor m2, m2 + pcmpeqd m5, m2 + mova [stk+0x110], m14 + pshufd m4, m15, q1032 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + movq r11, m14 + punpckhqdq m14, m14 + movq rX, m14 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mov r4d, [stk+ 0] + mov r6d, [stk+ 4] + mov r7d, [stk+ 8] + mov r9d, [stk+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m14, m5, q1100 + pshufd m5, m5, q3322 + pand m7, m9, m4 + pand m8, m9, m6 + pand m15, m9, m14 + pand m9, m9, m5 + pandn m4, m0 + pandn m6, m1 + pandn m14, m2 + pandn m5, m3 + por m7, m4 + por m8, m6 + por m15, m14 + por m9, m5 + punpcklbw m0, m7, m7 + punpckhbw m7, m7 + punpcklbw m1, m8, m8 + punpckhbw m8, m8 + psraw m0, 8 + psraw m7, 8 + psraw m1, 8 + psraw m8, 8 + punpcklbw m2, m15, m15 + punpckhbw m15, m15 + punpcklbw m3, m9, m9 + punpckhbw m9, m9 + psraw m2, 8 + psraw m15, 8 + psraw m3, 8 + psraw m9, 8 + mova [stk+0x10], m0 + mova [stk+0x20], m7 + mova [stk+0x30], m1 + mova [stk+0x40], m8 + mova [stk+0x50], m2 + mova [stk+0x60], m15 + mova [stk+0x70], m3 + mova [stk+0x80], m9 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0 + mova [stk+0x90], m1 + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1 + mova [stk+0xa0], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2 + mova [stk+0xb0], m3 + MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3 + mova [stk+0xc0], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4 + mova [stk+0xd0], m5 + MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5 + MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6 + MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7 + mova m5, [stk+0xd0] + mova m1, [stk+0x90] + mova m2, [stk+0xa0] + mova m3, [stk+0xb0] + mova m9, [stk+0xc0] + mov myd, mym + mov dyd, dym + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m9 ; 23a + punpckhwd m3, m9 ; 23b + mova [stk+0x90], m4 + mova [stk+0xa0], m5 + mova [stk+0xb0], m6 + mova [stk+0xc0], m7 + %define hround [rsp+0x10] +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq m11, r6q + punpcklbw m11, m11 + psraw m11, 8 + pshufd m5, m11, q0000 + pshufd m7, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m4, m5, m0 + pmaddwd m5, m5, m1 + pmaddwd m6, m7, m2 + pmaddwd m7, m7, m3 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 + pmaddwd m6, [stk+0x90], m10 + pmaddwd m7, [stk+0xa0], m10 + pmaddwd m8, [stk+0xb0], m11 + pmaddwd m9, [stk+0xc0], m11 + paddd m4, m6 + paddd m5, m7 + %if isput + pshufd m6, m12, q1032 + %endif + paddd m4, m8 + paddd m5, m9 +%else + movd r0, m15 + movd rX, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r4, m15 + movd r5, m4 + mova m14, [stk+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [stk+16], m14 + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m9, m4 + pand m1, m9, m6 + pand m2, m9, m7 + pand m3, m9, m5 + pandn m4, [stk+0x20] + pandn m6, [stk+0x30] + pandn m7, [stk+0x40] + pandn m5, [stk+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + punpcklbw m4, m0, m0 + punpckhbw m0, m0 + punpcklbw m5, m1, m1 + punpckhbw m1, m1 + psraw m4, 8 + psraw m0, 8 + psraw m5, 8 + psraw m1, 8 + punpcklbw m6, m2, m2 + punpckhbw m2, m2 + punpcklbw m7, m3, m3 + punpckhbw m3, m3 + psraw m6, 8 + psraw m2, 8 + psraw m7, 8 + psraw m3, 8 + mova [stk+0x0a0], m4 + mova [stk+0x0b0], m0 + mova [stk+0x0c0], m5 + mova [stk+0x0d0], m1 + mova [stk+0x140], m6 + mova [stk+0x150], m2 + mova [stk+0x160], m7 + mova [stk+0x170], m3 + MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0 + MC_8TAP_SCALED_H 0xa0, 0x30 ; 1 + MC_8TAP_SCALED_H 0xa0, 0x40 ; 2 + MC_8TAP_SCALED_H 0xa0, 0x50 ; 3 + MC_8TAP_SCALED_H 0xa0, 0x60 ; 4 + MC_8TAP_SCALED_H 0xa0, 0x70 ; 5 + MC_8TAP_SCALED_H 0xa0, 0x80 ; 6 + MC_8TAP_SCALED_H 0xa0, 0x90 ; 7 + mova m5, [stk+0x60] + mova m6, [stk+0x70] + mova m7, [stk+0x80] + mova m0, [stk+0x90] + mov myd, mym + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [stk+0x60], m4 + mova [stk+0x70], m5 + mova [stk+0x80], m6 + mova [stk+0x90], m7 + mova m1, [stk+0x20] + mova m2, [stk+0x30] + mova m3, [stk+0x40] + mova m4, [stk+0x50] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 +.vloop: + mov r0, r0m + mov r5, [esp+0x1f4] + and myd, 0x3ff + mov mym, myd + xor r3, r3 + shr r4, 6 + lea r5, [r5+r4] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + movd m7, r4 + movd m6, r3 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m4, m7, q0000 + pshufd m5, m7, q1111 + pmaddwd m0, m4 + pmaddwd m1, m4 + pmaddwd m2, m5 + pmaddwd m3, m5 + pshufd m6, m7, q2222 + pshufd m7, m7, q3333 + paddd m0, m2 + paddd m1, m3 + pmaddwd m2, [stk+0x60], m6 + pmaddwd m3, [stk+0x70], m6 + pmaddwd m4, [stk+0x80], m7 + pmaddwd m5, [stk+0x90], m7 + %if isput + movd m6, [esp+0x18] + %endif + paddd m0, m2 + paddd m1, m3 + paddd m0, vrnd_mem + paddd m1, vrnd_mem + paddd m4, m0 + paddd m5, m1 +%endif +%ifidn %1, put + psrad m4, m6 + psrad m5, m6 + packssdw m4, m5 + pxor m7, m7 + pmaxsw m4, m7 + pminsw m4, pxmaxm + mova [dstq], m4 + add dstq, dsm +%else + psrad m4, 6 + psrad m5, 6 + packssdw m4, m5 + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep +%if ARCH_X86_64 + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [stk+0x140], myd + mov r4d, [stk+ 0] + mov r6d, [stk+ 4] + mov r7d, [stk+ 8] + mov r9d, [stk+12] + jz .skip_line + mova m14, [base+unpckw] + movu m8, [srcq+r10*2] + movu m9, [srcq+r11*2] + movu m10, [srcq+r13*2] + movu m11, [srcq+ rX*2] + movu m4, [srcq+ r4*2] + movu m5, [srcq+ r6*2] + movu m6, [srcq+ r7*2] + movu m7, [srcq+ r9*2] + add srcq, ssq + mov myd, [stk+0x140] + mov dyd, dym + pshufd m15, m14, q1032 + pshufb m0, m14 ; 0a 1a + pshufb m1, m14 ; 0b 1b + pshufb m2, m15 ; 3a 2a + pshufb m3, m15 ; 3b 2b + pmaddwd m8, [stk+0x50] + pmaddwd m9, [stk+0x60] + pmaddwd m10, [stk+0x70] + pmaddwd m11, [stk+0x80] + pmaddwd m4, [stk+0x10] + pmaddwd m5, [stk+0x20] + pmaddwd m6, [stk+0x30] + pmaddwd m7, [stk+0x40] + phaddd m8, m9 + phaddd m10, m11 + mova m11, hround + phaddd m4, m5 + phaddd m6, m7 + phaddd m8, m10 + phaddd m4, m6 + paddd m4, m11 + paddd m8, m11 + psrad m4, m12 + psrad m8, m12 + packssdw m4, m8 + pshufb m5, [stk+0x90], m14 ; 4a 5a + pshufb m6, [stk+0xa0], m14 ; 4b 5b + pshufb m7, [stk+0xb0], m15 ; 7a 6a + pshufb m8, [stk+0xc0], m15 ; 7b 6b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m5 ; 34a + punpcklwd m3, m6 ; 34b + punpckhwd m5, m7 ; 56a + punpckhwd m6, m8 ; 56b + punpcklwd m7, m4 ; 78a + punpckhqdq m4, m4 + punpcklwd m8, m4 ; 78b + mova [stk+0x90], m5 + mova [stk+0xa0], m6 + mova [stk+0xb0], m7 + mova [stk+0xc0], m8 + jmp .vloop +.skip_line: + MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 10, 11 + MC_8TAP_SCALED_H 8, 5, 6, 7, 9, 0, 10, 11 + mov myd, [stk+0x140] + mov dyd, dym + mova m0, m2 ; 01a + mova m1, m3 ; 01b + mova m2, [stk+0x90] ; 23a + mova m3, [stk+0xa0] ; 23b + mova m5, [stk+0xb0] ; 45a + mova m6, [stk+0xc0] ; 45b + punpcklwd m7, m4, m8 ; 67a + punpckhwd m4, m8 ; 67b + mova [stk+0x90], m5 + mova [stk+0xa0], m6 + mova [stk+0xb0], m7 + mova [stk+0xc0], m4 +%else + mov r0m, r0 + mov myd, mym + mov r3, r3m + add myd, dym + test myd, ~0x3ff + mov mym, myd + jnz .next_line + mova m0, [stk+0x20] + mova m1, [stk+0x30] + mova m2, [stk+0x40] + mova m3, [stk+0x50] + jmp .vloop +.next_line: + test myd, 0x400 + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + jz .skip_line + MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8 + mova m7, [base+unpckw] + pshufd m4, m7, q1032 + pshufb m0, [stk+0x20], m7 ; 0a 1a + pshufb m1, [stk+0x30], m7 ; 0b 1b + pshufb m2, [stk+0x40], m4 ; 3a 2a + pshufb m3, [stk+0x50], m4 ; 3b 2b + pshufb m5, [stk+0x60], m7 ; 4a 5a + pshufb m6, [stk+0x70], m7 ; 4b 5b + pshufb m7, [stk+0x80], m4 ; 7a 6a + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m5 ; 34a + punpcklwd m3, m6 ; 34b + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 + punpckhwd m5, m7 ; 56a + mova [stk+0x60], m5 + pshufb m5, [stk+0x90], m4 ; 7b 6b + punpcklwd m7, [stk+0xe0] ; 78a + punpckhwd m6, m5 ; 56b + mova [stk+0x70], m6 + movq m6, [stk+0xe8] + mova [stk+0x80], m7 + punpcklwd m5, m6 + mov myd, mym + mova [stk+0x90], m5 + jmp .vloop +.skip_line: + MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8 + MC_8TAP_SCALED_H 0xa0, 0 ; 9 + mova m7, [stk+0xe0] + mova m2, [stk+0x60] ; 23a + mova m3, [stk+0x70] ; 23b + mova m4, [stk+0x80] ; 45a + mova m5, [stk+0x90] ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova m0, [stk+0x40] ; 01a + mova m1, [stk+0x50] ; 01b + mov myd, mym + mova [stk+0x40], m2 + mova [stk+0x50], m3 + mova [stk+0x60], m4 + mova [stk+0x70], m5 + mova [stk+0x80], m6 + mova [stk+0x90], m7 + mova [stk+0x20], m0 + mova [stk+0x30], m1 +%endif + jmp .vloop +INIT_XMM ssse3 +.dy1: + movzx wd, word [base+%1_8tap_scaled_ssse3_dy1_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy1_w2: + %if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d + %else + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 + %define m11 [esp+0x00] + %define m12 [esp+0x10] + %define m13 [esp+0x20] + movzx r5, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r5 + mov r1, r1m + %endif + pxor m9, m9 + punpckldq m9, m8 + paddd m14, m9 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m9, [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + pshufd m15, m15, q0321 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_q] + mova m6, [base+spel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m2, m2 + pcmpeqd m8, m2 + psrld m14, 10 + paddd m14, m14 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [stk], m14 + SWAP m5, m0 + SWAP m6, m3 + %define m15 m6 + %endif + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + movu m2, [srcq+ssq*2] + movu m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + punpckldq m15, m7 + %if ARCH_X86_64 + pshufb m14, m5 + paddb m14, m6 + pand m9, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m9 + movu m4, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + add srcq, ss3q + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + %else + pand m7, m5, [base+pd_0x4000] + pandn m5, m15 + por m5, m7 + %define m15 m5 + mov myd, mym + mov r5, [esp+0x1f4] + xor r3, r3 + shr myd, 6 + lea r5, [r5+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + mov [stk+0x20], r3 + mov r3, r3m + %endif + punpcklbw m15, m15 + psraw m15, 8 + REPX {pshufb x, m14}, m0, m1, m2, m3 + REPX {pmaddwd x, m15}, m0, m1, m2, m3 + %if ARCH_X86_64 + REPX {pshufb x, m14}, m4, m5, m6 + REPX {pmaddwd x, m15}, m4, m5, m6 + phaddd m0, m1 + phaddd m2, m3 + phaddd m4, m5 + phaddd m6, m6 + REPX {paddd x, m11}, m0, m2, m4, m6 + REPX {psrad x, m12}, m0, m2, m4, m6 + packssdw m0, m2 ; 0 1 2 3 + packssdw m4, m6 ; 4 5 6 + SWAP m1, m4 + movq m10, r4 + %else + mova [stk+0x10], m15 + phaddd m0, m1 + phaddd m2, m3 + movu m1, [srcq+ssq*0] + movu m7, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + add srcq, ss3q + REPX {pshufb x, m14}, m1, m7, m6 + REPX {pmaddwd x, m15}, m1, m7, m6 + %define m14 [stk+0x00] + %define m15 [stk+0x10] + phaddd m1, m7 + phaddd m6, m6 + REPX {paddd x, m11}, m0, m2, m1, m6 + REPX {psrad x, m12}, m0, m2, m1, m6 + packssdw m0, m2 + packssdw m1, m6 + %define m8 m6 + %define m9 m4 + %define m10 m5 + movd m10, r4 + movd m9, [stk+0x20] + punpckldq m10, m9 + %endif + punpcklbw m10, m10 + psraw m10, 8 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + %if ARCH_X86_32 + mova [stk+0x50], m7 + mova [stk+0x60], m8 + mova [stk+0x70], m9 + mova [stk+0x80], m10 + %define m7 [stk+0x50] + %define m8 [stk+0x60] + %define m9 [stk+0x70] + %define m10 [stk+0x80] + %endif + palignr m2, m1, m0, 4 ; 1 2 3 4 + punpcklwd m3, m0, m2 ; 01 12 + punpckhwd m0, m2 ; 23 34 + pshufd m4, m1, q2121 ; 5 6 5 6 + punpcklwd m2, m1, m4 ; 45 56 + %if ARCH_X86_32 + mov r0, r0m + %endif +.dy1_w2_loop: + movu m1, [srcq+ssq*0] + movu m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m5, m3, m7 + mova m3, m0 + pmaddwd m0, m8 + pshufb m1, m14 + pshufb m6, m14 + pmaddwd m1, m15 + pmaddwd m6, m15 + phaddd m1, m6 + paddd m1, m11 + psrad m1, m12 + packssdw m1, m1 + paddd m5, m0 + mova m0, m2 + pmaddwd m2, m9 + paddd m5, m2 + palignr m2, m1, m4, 12 + punpcklwd m2, m1 ; 67 78 + pmaddwd m4, m2, m10 + paddd m5, m13 + paddd m5, m4 + pxor m6, m6 + mova m4, m1 + pshufd m1, m12, q1032 + psrad m5, m1 + packssdw m5, m5 + pmaxsw m5, m6 + pminsw m5, pxmaxm + movd [dstq+dsq*0], m5 + pshuflw m5, m5, q1032 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +INIT_XMM ssse3 +.dy1_w4: +%if ARCH_X86_64 + mov myd, mym + mova [rsp+0x10], m11 + mova [rsp+0x20], m12 + %if isput + mova [rsp+0x30], m13 + %define vrnd_mem [rsp+0x30] + %define stk rsp+0x40 + %else + %define vrnd_mem [base+pd_m524256] + %define stk rsp+0x30 + %endif + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d +%else + %define m10 [base+pd_0x3ff] + %define m9 [base+pd_0x4000] + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %if isprep + %define ssq r3 + %endif + movzx r5, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r5 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m9, [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd m15, m0 + pshufd m7, m15, q1032 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd r6d, m15 + movd r13d, m7 + mova m10, [base+bdct_lb_q+ 0] + mova m11, [base+bdct_lb_q+16] + movd m13, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+ r6*8+2] + movd m15, [base+subpel_filters+r11*8+2] + movd m4, [base+subpel_filters+r13*8+2] +%else + movd r0, m15 + movd r4, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd rX, m15 + movd r5, m7 + mova m5, [base+bdct_lb_q+ 0] + mova m6, [base+bdct_lb_q+16] + movd m1, [base+subpel_filters+r0*8+2] + movd m2, [base+subpel_filters+rX*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m7, [base+subpel_filters+r5*8+2] + SWAP m4, m7 + %if isprep + mov r3, r3m + %endif + %define m10 m5 + %define m11 m6 + %define m12 m1 + %define m13 m1 +%endif + psrld m14, 10 + paddd m14, m14 + punpckldq m13, m2 + punpckldq m15, m4 + punpcklqdq m13, m15 + pxor m2, m2 + pcmpeqd m0, m2 +%if ARCH_X86_64 + pand m9, m0 +%else + pand m2, m9, m0 + %define m9 m2 + SWAP m7, m4 +%endif + pandn m0, m13 +%if ARCH_X86_64 + SWAP m13, m0 +%else + %define m13 m0 +%endif + por m13, m9 + punpckhbw m15, m13, m13 + punpcklbw m13, m13 + psraw m15, 8 + psraw m13, 8 + pshufb m12, m14, m10 + pshufb m14, m11 + mova m10, [base+spel_s_shuf2] + movd r4d, m14 + shr r4d, 24 +%if ARCH_X86_32 + mova [stk+0x40], m13 + mova [stk+0x50], m15 + pxor m2, m2 +%endif + pshufb m7, m14, m2 + psubb m14, m7 + paddb m12, m10 + paddb m14, m10 +%if ARCH_X86_64 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu m7, [srcq+ssq*0] + movu m9, [srcq+ssq*1] + movu m8, [srcq+ssq*2] + movu m10, [srcq+ss3q ] + movu m1, [srcq+r4 ] + movu m3, [srcq+r6 ] + movu m2, [srcq+r11 ] + movu m4, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m12}, m7, m9, m8, m10 + REPX {pmaddwd x, m13}, m7, m9, m8, m10 + REPX {pshufb x, m14}, m1, m3, m2, m4 + REPX {pmaddwd x, m15}, m1, m3, m2, m4 + mova m5, [rsp+0x10] + movd xm6, [rsp+0x20] + phaddd m7, m1 + phaddd m9, m3 + phaddd m8, m2 + phaddd m10, m4 + movu m1, [srcq+ssq*0] + movu m2, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + REPX {paddd x, m5}, m7, m9, m8, m10 + REPX {psrad x, xm6}, m7, m9, m8, m10 + packssdw m7, m9 ; 0 1 + packssdw m8, m10 ; 2 3 + movu m0, [srcq+r4 ] + movu m9, [srcq+r6 ] + movu m10, [srcq+r11 ] + add srcq, ss3q + REPX {pshufb x, m12}, m1, m2, m3 + REPX {pmaddwd x, m13}, m1, m2, m3 + REPX {pshufb x, m14}, m0, m9, m10 + REPX {pmaddwd x, m15}, m0, m9, m10 + phaddd m1, m0 + phaddd m2, m9 + phaddd m3, m10 + shr myd, 6 + mov r13d, 64 << 24 + lea myd, [t1+myq] + cmovnz r13q, [base+subpel_filters+myq*8] + REPX {paddd x, m5}, m1, m2, m3 + REPX {psrad x, xm6}, m1, m2, m3 + packssdw m1, m2 ; 4 5 + packssdw m3, m3 ; 6 6 + SWAP m9, m1 + shufps m4, m7, m8, q1032 ; 1 2 + shufps m5, m8, m9, q1032 ; 3 4 + shufps m6, m9, m3, q1032 ; 5 6 + punpcklwd m0, m7, m4 ; 01 + punpckhwd m7, m4 ; 12 + punpcklwd m1, m8, m5 ; 23 + punpckhwd m8, m5 ; 34 + punpcklwd m2, m9, m6 ; 45 + punpckhwd m9, m6 ; 56 + movq m10, r13 + mova [stk+0x00], m1 + mova [stk+0x10], m8 + mova [stk+0x20], m2 + mova [stk+0x30], m9 + mova [stk+0x40], m3 + %define hrnd_mem [rsp+0x10] + %define hsh_mem [rsp+0x20] + %define vsh_mem [rsp+0x28] + %if isput + %define vrnd_mem [rsp+0x30] + %else + %define vrnd_mem [base+pd_m524256] + %endif +%else + mova [stk+0x20], m12 + mova [stk+0x30], m14 + add r4, srcq + MC_4TAP_SCALED_H 0x60 ; 0 1 + MC_4TAP_SCALED_H 0x70 ; 2 3 + MC_4TAP_SCALED_H 0x80 ; 4 5 + movu m7, [srcq] + movu m2, [r4] + add srcq, ssq + add r4, ssq + mov [stk+0xb0], r4 + pshufb m7, m12 + pshufb m2, m14 + pmaddwd m7, m13 + pmaddwd m2, m15 + phaddd m7, m2 + paddd m7, [esp+0x00] + psrad m7, [esp+0x10] + packssdw m7, m7 ; 6 6 + mova m4, [stk+0x60] + mova m5, [stk+0x70] + mova m6, [stk+0x80] + mov myd, mym + mov rX, [esp+0x1f4] + xor r5, r5 + shr myd, 6 + lea rX, [rX+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+rX*8+0] + cmovnz r5, [base+subpel_filters+rX*8+4] + mov r3, r3m + shufps m1, m4, m5, q1032 ; 1 2 + shufps m2, m5, m6, q1032 ; 3 4 + shufps m3, m6, m7, q1032 ; 5 6 + mova [stk+0xa0], m7 + punpcklwd m0, m4, m1 ; 01 + punpckhwd m4, m1 ; 12 + punpcklwd m1, m5, m2 ; 23 + punpckhwd m5, m2 ; 34 + punpcklwd m2, m6, m3 ; 45 + punpckhwd m6, m3 ; 56 + movd m7, r4 + movd m3, r5 + mov r0, r0m + %if isput + mov r1, r1m + %endif + mov r4, [stk+0xb0] + mova [stk+0xc0], m4 ; 12 + mova [stk+0x60], m1 ; 23 + mova [stk+0x70], m2 ; 45 + mova [stk+0x80], m5 ; 34 + mova [stk+0x90], m6 ; 56 + %define m12 [stk+0x20] + %define m14 [stk+0x30] + %define m13 [stk+0x40] + %define m15 [stk+0x50] + %define hrnd_mem [esp+0x00] + %define hsh_mem [esp+0x10] + %define vsh_mem [esp+0x18] + %if isput + %define vrnd_mem [esp+0x20] + %else + %define vrnd_mem [base+pd_m524256] + %endif + %define m10 m7 + punpckldq m10, m3 +%endif + punpcklbw m10, m10 + psraw m10, 8 + pshufd m3, m10, q0000 + pshufd m4, m10, q1111 + pshufd m5, m10, q2222 + pshufd m10, m10, q3333 +%if ARCH_X86_32 + %xdefine m8 m3 + %xdefine m9 m6 + %xdefine m11 m5 + %xdefine m6 m4 + mova [stk+0x100], m3 + mova [stk+0x110], m4 + mova [stk+0x120], m5 + mova [stk+0x130], m10 + %define m3 [stk+0x100] + %define m4 [stk+0x110] + %define m5 [stk+0x120] + %define m10 [stk+0x130] + mova m7, [stk+0xc0] + mova m8, [stk+0x80] +%endif +.dy1_w4_loop: + movu m11, [srcq+ssq*0] + movu m6, [srcq+ssq*1] + pmaddwd m0, m3 + pmaddwd m7, m3 + pmaddwd m1, m4 + pmaddwd m8, m4 + pmaddwd m2, m5 + pmaddwd m9, m5 + paddd m1, m0 + paddd m8, m7 +%if ARCH_X86_64 + movu m0, [srcq+r4] + movu m7, [srcq+r6] +%else + movu m0, [r4+ssq*0] + movu m7, [r4+ssq*1] + lea r4, [r4+ssq*2] +%endif + lea srcq, [srcq+ssq*2] + paddd m1, m2 + paddd m8, m9 + pshufb m11, m12 + pshufb m6, m12 + pmaddwd m11, m13 + pmaddwd m6, m13 + pshufb m0, m14 + pshufb m7, m14 + pmaddwd m0, m15 + pmaddwd m7, m15 + phaddd m11, m0 + phaddd m6, m7 + paddd m11, hrnd_mem + paddd m6, hrnd_mem + psrad m11, hsh_mem + psrad m6, hsh_mem + packssdw m11, m6 ; 7 8 +%if ARCH_X86_64 + shufps m9, [stk+0x40], m11, q1032 ; 6 7 + mova m0, [stk+0x00] + mova [stk+0x40], m11 +%else + shufps m9, [stk+0xa0], m11, q1032 ; 6 7 + mova m0, [stk+0x60] + mova [stk+0xa0], m11 +%endif + punpcklwd m2, m9, m11 ; 67 + punpckhwd m9, m11 ; 78 + pmaddwd m6, m2, m10 + pmaddwd m7, m9, m10 +%if isput + movd m11, vsh_mem +%endif + paddd m1, vrnd_mem + paddd m8, vrnd_mem + paddd m1, m6 + paddd m8, m7 +%if ARCH_X86_64 + mova m7, [stk+0x10] +%else + mova m7, [stk+0x80] +%endif +%if isput + psrad m1, m11 + psrad m8, m11 +%else + psrad m1, 6 + psrad m8, 6 +%endif + packssdw m1, m8 +%if ARCH_X86_64 + mova m8, [stk+0x30] +%else + mova m8, [stk+0x90] +%endif +%if isput + pxor m6, m6 + pmaxsw m1, m6 + pminsw m1, pxmaxm + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] +%else + mova [tmpq], m1 + add tmpq, 16 +%endif +%if ARCH_X86_64 + mova m1, [stk+0x20] + mova [stk+0x10], m8 + mova [stk+0x00], m1 + mova [stk+0x20], m2 + mova [stk+0x30], m9 +%else + mova m1, [stk+0x70] + mova [stk+0x80], m8 + mova [stk+0x60], m1 + mova [stk+0x70], m2 + mova [stk+0x90], m9 +%endif + sub hd, 2 + jg .dy1_w4_loop + MC_8TAP_SCALED_RET ; why not jz .ret? +INIT_XMM ssse3 +.dy1_w8: + mov dword [stk+0xf0], 1 + movifprep tmp_stridem, 16 + jmp .dy1_w_start +.dy1_w16: + mov dword [stk+0xf0], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [stk+0xf0], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [stk+0xf0], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [stk+0xf0], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: + mov myd, mym +%if ARCH_X86_64 + %ifidn %1, put + movifnidn dsm, dsq + %endif + mova [rsp+0x10], m11 + mova [rsp+0x20], m12 + %define hround m11 + %if isput + mova [rsp+0x30], m13 + %else + mova m13, [base+pd_m524256] + %endif + shr t0d, 16 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + movd m15, t0d +%else + %define hround [esp+0x00] + %define m12 [esp+0x10] + %define m10 [base+pd_0x3ff] + %define m8 m0 + %xdefine m14 m4 + %xdefine m15 m3 + %if isprep + %define ssq ssm + %endif + mov r5, [esp+0x1f0] + mov r3, [esp+0x1f4] + shr r5, 16 + movd m15, r5 + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r0, r0m + mov r3, r3m +%endif + sub srcq, 6 + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] +%if ARCH_X86_64 + movq m3, r4q +%else + movd m5, r4 + movd m6, r5 + punpckldq m5, m6 + SWAP m3, m5 +%endif + punpcklbw m3, m3 + psraw m3, 8 + mova [stk+0x100], m7 + mova [stk+0x120], m15 + mov [stk+0x0f8], srcq + mov [stk+0x130], r0q ; dstq / tmpq + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 +%if ARCH_X86_64 + mova [stk+0x140], m0 + mova [stk+0x150], m1 + mova [stk+0x160], m2 + mova [stk+0x170], m3 + %if UNIX64 + mov hm, hd + %endif +%else + mova [stk+0x180], m0 + mova [stk+0x190], m1 + mova [stk+0x1a0], m2 + mova [stk+0x1b0], m3 + SWAP m5, m3 + mov r5, hm + mov [stk+0x134], r5 +%endif + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [stk+0x0f0] + jz .ret +%if ARCH_X86_64 + add qword [stk+0x130], 16 + mov hd, hm +%else + add dword [stk+0x130], 16 + mov r5, [stk+0x134] + mov r0, [stk+0x130] +%endif + mova m7, [stk+0x100] + mova m14, [stk+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] + mova m11, [rsp+0x10] +%endif + mova m15, [stk+0x120] + mov srcq, [stk+0x0f8] +%if ARCH_X86_64 + mov r0q, [stk+0x130] ; dstq / tmpq +%else + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.dy1_hloop: +%if ARCH_X86_64 + mova m9, [base+pq_0x40000000] +%else + %define m9 [base+pq_0x40000000] +%endif + pxor m1, m1 + psrld m2, m14, 10 + mova [stk], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m1 + pshufd m2, m5, q1032 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pxor m2, m2 + pcmpeqd m5, m2 + mova [stk+0x110], m14 + pshufd m4, m15, q1032 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + movq r11, m14 + punpckhqdq m14, m14 + movq rX, m14 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mov r4d, [stk+ 0] + mov r6d, [stk+ 4] + mov r7d, [stk+ 8] + mov r9d, [stk+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m14, m5, q1100 + pshufd m5, m5, q3322 + pand m7, m9, m4 + pand m8, m9, m6 + pand m15, m9, m14 + pand m9, m9, m5 + pandn m4, m0 + pandn m6, m1 + pandn m14, m2 + pandn m5, m3 + por m7, m4 + por m8, m6 + por m15, m14 + por m9, m5 + punpcklbw m0, m7, m7 + punpckhbw m7, m7 + punpcklbw m1, m8, m8 + punpckhbw m8, m8 + psraw m0, 8 + psraw m7, 8 + psraw m1, 8 + psraw m8, 8 + punpcklbw m2, m15, m15 + punpckhbw m15, m15 + punpcklbw m3, m9, m9 + punpckhbw m9, m9 + psraw m2, 8 + psraw m15, 8 + psraw m3, 8 + psraw m9, 8 + mova [stk+0x10], m0 + mova [stk+0x20], m7 + mova [stk+0x30], m1 + mova [stk+0x40], m8 + mova [stk+0x50], m2 + mova [stk+0x60], m15 + mova [stk+0x70], m3 + mova [stk+0x80], m9 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0 + mova [stk+0x90], m1 + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1 + mova [stk+0xa0], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2 + mova [stk+0xb0], m3 + MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3 + mova [stk+0xc0], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4 + mova [stk+0xd0], m5 + MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5 + MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6 + MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7 + mova m5, [stk+0xd0] + mova m1, [stk+0x90] + mova m2, [stk+0xa0] + mova m3, [stk+0xb0] + mova m9, [stk+0xc0] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m9 ; 23a + punpckhwd m3, m9 ; 23b + mova m10, [stk+0x140] + mova m11, [stk+0x150] + mova m14, [stk+0x160] + mova m15, [stk+0x170] + mova [stk+0x90], m4 + mova [stk+0xa0], m5 + mova [stk+0xb0], m6 + mova [stk+0xc0], m7 + %define hround [rsp+0x10] + %define shift [rsp+0x20] + %if isput + %define vround [rsp+0x30] + %else + %define vround [base+pd_m524256] + %endif +.dy1_vloop: + pmaddwd m4, m0, m10 + pmaddwd m5, m1, m10 + pmaddwd m6, m2, m11 + pmaddwd m7, m3, m11 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 + pmaddwd m6, [stk+0x90], m14 + pmaddwd m7, [stk+0xa0], m14 + pmaddwd m8, [stk+0xb0], m15 + pmaddwd m9, [stk+0xc0], m15 + paddd m4, m6 + paddd m5, m7 + %if isput + pshufd m6, m12, q1032 + %endif + paddd m4, m8 + paddd m5, m9 +%else + movd r0, m15 + movd rX, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r4, m15 + movd r5, m4 + mova m14, [stk+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [stk+16], m14 + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m9, m4 + pand m1, m9, m6 + pand m2, m9, m7 + pand m3, m9, m5 + pandn m4, [stk+0x20] + pandn m6, [stk+0x30] + pandn m7, [stk+0x40] + pandn m5, [stk+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + punpcklbw m4, m0, m0 + punpckhbw m0, m0 + punpcklbw m5, m1, m1 + punpckhbw m1, m1 + psraw m4, 8 + psraw m0, 8 + psraw m5, 8 + psraw m1, 8 + punpcklbw m6, m2, m2 + punpckhbw m2, m2 + punpcklbw m7, m3, m3 + punpckhbw m3, m3 + psraw m6, 8 + psraw m2, 8 + psraw m7, 8 + psraw m3, 8 + mova [stk+0x0a0], m4 + mova [stk+0x0b0], m0 + mova [stk+0x0c0], m5 + mova [stk+0x0d0], m1 + mova [stk+0x140], m6 + mova [stk+0x150], m2 + mova [stk+0x160], m7 + mova [stk+0x170], m3 + MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0 + MC_8TAP_SCALED_H 0xa0, 0x30 ; 1 + MC_8TAP_SCALED_H 0xa0, 0x40 ; 2 + MC_8TAP_SCALED_H 0xa0, 0x50 ; 3 + MC_8TAP_SCALED_H 0xa0, 0x60 ; 4 + MC_8TAP_SCALED_H 0xa0, 0x70 ; 5 + MC_8TAP_SCALED_H 0xa0, 0x80 ; 6 + MC_8TAP_SCALED_H 0xa0, 0x90 ; 7 + mova m5, [stk+0x60] + mova m6, [stk+0x70] + mova m7, [stk+0x80] + mova m0, [stk+0x90] + mov r0, r0m + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [stk+0x60], m4 + mova [stk+0x70], m5 + mova [stk+0x80], m6 + mova [stk+0x90], m7 + mova m1, [stk+0x20] + mova m2, [stk+0x30] + mova m3, [stk+0x40] + mova m4, [stk+0x50] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova m4, [stk+0x180] + mova m5, [stk+0x190] + mova m6, [stk+0x1a0] + mova m7, [stk+0x1b0] + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 +.dy1_vloop: + pmaddwd m0, m4 + pmaddwd m1, m4 + pmaddwd m2, m5 + pmaddwd m3, m5 + paddd m0, m2 + paddd m1, m3 + pmaddwd m2, [stk+0x60], m6 + pmaddwd m3, [stk+0x70], m6 + pmaddwd m4, [stk+0x80], m7 + pmaddwd m5, [stk+0x90], m7 + %if isput + movd m6, [esp+0x18] + %endif + paddd m0, m2 + paddd m1, m3 + paddd m0, vrnd_mem + paddd m1, vrnd_mem + paddd m4, m0 + paddd m5, m1 +%endif +%ifidn %1, put + psrad m4, m6 + psrad m5, m6 + packssdw m4, m5 + pxor m7, m7 + pmaxsw m4, m7 + pminsw m4, pxmaxm + mova [dstq], m4 + add dstq, dsm +%else + psrad m4, 6 + psrad m5, 6 + packssdw m4, m5 + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy1_hloop_prep +%if ARCH_X86_64 + movu m8, [srcq+r10*2] + movu m9, [srcq+r11*2] + movu m12, [srcq+r13*2] + movu m13, [srcq+ rX*2] + movu m4, [srcq+ r4*2] + movu m5, [srcq+ r6*2] + movu m6, [srcq+ r7*2] + movu m7, [srcq+ r9*2] + add srcq, ssq + pmaddwd m8, [stk+0x50] + pmaddwd m9, [stk+0x60] + pmaddwd m12, [stk+0x70] + pmaddwd m13, [stk+0x80] + pmaddwd m4, [stk+0x10] + pmaddwd m5, [stk+0x20] + pmaddwd m6, [stk+0x30] + pmaddwd m7, [stk+0x40] + phaddd m8, m9 + phaddd m12, m13 + mova m9, [base+unpckw] + mova m13, hround + phaddd m4, m5 + phaddd m6, m7 + phaddd m8, m12 + phaddd m4, m6 + pshufd m5, m9, q1032 + pshufb m0, m9 ; 0a 1a + pshufb m1, m9 ; 0b 1b + pshufb m2, m5 ; 3a 2a + pshufb m3, m5 ; 3b 2b + mova m12, shift + paddd m4, m13 + paddd m8, m13 + psrad m4, m12 + psrad m8, m12 + packssdw m4, m8 + pshufb m6, [stk+0x90], m9 ; 4a 5a + pshufb m7, [stk+0xa0], m9 ; 4b 5b + pshufb m8, [stk+0xb0], m5 ; 7a 6a + pshufb m13, [stk+0xc0], m5 ; 7b 6b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m6 ; 34a + punpcklwd m3, m7 ; 34b + punpckhwd m6, m8 ; 56a + punpckhwd m7, m13 ; 56b + punpcklwd m8, m4 ; 78a + punpckhqdq m4, m4 + punpcklwd m13, m4 ; 78b + mova [stk+0x90], m6 + mova [stk+0xa0], m7 + mova [stk+0xb0], m8 + mova [stk+0xc0], m13 + mova m13, vround +%else + mov r0m, r0 + mov r3, r3m + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + MC_8TAP_SCALED_H 0xa0, 0xe0, 0 ; 8 + mova m7, [base+unpckw] + pshufd m4, m7, q1032 + pshufb m0, [stk+0x20], m7 ; 0a 1a + pshufb m1, [stk+0x30], m7 ; 0b 1b + pshufb m2, [stk+0x40], m4 ; 3a 2a + pshufb m3, [stk+0x50], m4 ; 3b 2b + pshufb m5, [stk+0x60], m7 ; 4a 5a + pshufb m6, [stk+0x70], m7 ; 4b 5b + pshufb m7, [stk+0x80], m4 ; 7a 6a + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m5 ; 34a + punpcklwd m3, m6 ; 34b + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 + punpckhwd m5, m7 ; 56a + mova [stk+0x60], m5 + pshufb m5, [stk+0x90], m4 ; 7b 6b + punpcklwd m7, [stk+0xe0] ; 78a + mova m4, [stk+0x180] + punpckhwd m6, m5 ; 56b + mova [stk+0x70], m6 + movq m6, [stk+0xe8] + mova [stk+0x80], m7 + mova m7, [stk+0x1b0] + punpcklwd m5, m6 + mova m6, [stk+0x1a0] + mova [stk+0x90], m5 + mova m5, [stk+0x190] + mov r0, r0m +%endif + jmp .dy1_vloop +INIT_XMM ssse3 +%if ARCH_X86_64 + %define stk rsp+0x20 +%endif +.dy2: + movzx wd, word [base+%1_8tap_scaled_ssse3_dy2_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy2_w2: + %if ARCH_X86_64 + mov myd, mym + mova [rsp+0x10], m13 + %define vrnd_mem [rsp+0x10] + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d + %else + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 + %define m11 [esp+0x00] + %define m12 [esp+0x10] + %define vrnd_mem [esp+0x20] + mov r1, r1m + movzx r5, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r5 + %endif + pxor m9, m9 + punpckldq m9, m8 + paddd m14, m9 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m9, [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + pshufd m15, m15, q0321 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_q] + mova m6, [base+spel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m2, m2 + pcmpeqd m8, m2 + psrld m14, 10 + paddd m14, m14 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [stk], m14 + SWAP m5, m0 + SWAP m6, m3 + %define m15 m6 + %endif + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*2] + movu m2, [srcq+ssq*4] + punpckldq m15, m7 + %if ARCH_X86_64 + pshufb m14, m5 + paddb m14, m6 + pand m9, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m9 + movu m4, [srcq+ssq*1] + movu m5, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + movu m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + %else + pand m7, m5, [base+pd_0x4000] + pandn m5, m15 + por m5, m7 + %define m15 m5 + mov myd, mym + mov r5, [esp+0x1f4] + xor r3, r3 + shr myd, 6 + lea r5, [r5+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + mov [stk+0x20], r3 + mov r3, r3m + %endif + punpcklbw m15, m15 + psraw m15, 8 + REPX {pshufb x, m14}, m0, m1, m2 + REPX {pmaddwd x, m15}, m0, m1, m2 + %if ARCH_X86_64 + REPX {pshufb x, m14}, m4, m5, m6 + REPX {pmaddwd x, m15}, m4, m5, m6 + phaddd m0, m1 + phaddd m1, m2 + phaddd m4, m5 + phaddd m5, m6 + REPX {paddd x, m11}, m0, m1, m4, m5 + REPX {psrad x, m12}, m0, m1, m4, m5 + packssdw m0, m1 ; 0 2 2 4 + packssdw m4, m5 ; 1 3 3 5 + SWAP m2, m4 + movq m10, r4 + %else + mova [stk+0x10], m15 + phaddd m0, m1 + phaddd m1, m2 + movu m2, [srcq+ssq*1] + movu m7, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + movu m6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + REPX {pshufb x, m14}, m2, m7, m6 + REPX {pmaddwd x, m15}, m2, m7, m6 + %define m14 [stk+0x00] + %define m15 [stk+0x10] + phaddd m2, m7 + phaddd m7, m6 + REPX {paddd x, m11}, m0, m1, m2, m7 + REPX {psrad x, m12}, m0, m1, m2, m7 + packssdw m0, m1 + packssdw m2, m7 + %define m8 m6 + %define m9 m4 + %define m10 m5 + movd m10, r4 + movd m9, [stk+0x20] + punpckldq m10, m9 + %endif + punpcklbw m10, m10 + psraw m10, 8 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + %if ARCH_X86_32 + mova [stk+0x50], m7 + mova [stk+0x60], m8 + mova [stk+0x70], m9 + mova [stk+0x80], m10 + %xdefine m13 m7 + %define m7 [stk+0x50] + %define m8 [stk+0x60] + %define m9 [stk+0x70] + %define m10 [stk+0x80] + %endif + punpcklwd m1, m0, m2 ; 01 23 + punpckhwd m3, m0, m2 ; 23 45 + %if ARCH_X86_32 + mov r4, r0m + %define dstq r4 + mova [stk+0x20], m3 + mova [stk+0x30], m0 + %endif +.dy2_w2_loop: + movu m4, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + movu m13, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pmaddwd m3, m8 + REPX {pshufb x, m14}, m4, m5, m6, m13 + REPX {pmaddwd x, m15}, m4, m5, m6, m13 + phaddd m4, m5 + phaddd m6, m13 + pmaddwd m5, m1, m7 + paddd m4, m11 + paddd m6, m11 + psrad m4, m12 + psrad m6, m12 + packssdw m4, m6 ; 6 7 8 9 + paddd m5, m3 + pshufd m3, m4, q2200 + pshufd m4, m4, q3311 + palignr m3, m0, 12 ; 4 6 6 8 + palignr m4, m2, 12 ; 5 7 7 9 + mova m0, m3 + mova m2, m4 + punpcklwd m1, m3, m4 + punpckhwd m3, m4 + pmaddwd m6, m1, m9 + pmaddwd m4, m3, m10 + paddd m5, vrnd_mem + paddd m6, m4 + paddd m5, m6 + pshufd m4, m12, q1032 + pxor m6, m6 + psrad m5, m4 + packssdw m5, m5 + pmaxsw m5, m6 + pminsw m5, pxmaxm + movd [dstq+dsq*0], m5 + pshuflw m5, m5, q1032 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +INIT_XMM ssse3 +.dy2_w4: +%if ARCH_X86_64 + mov myd, mym + mova [rsp+0x10], m11 + mova [rsp+0x20], m12 + %if isput + mova [rsp+0x30], m13 + %define vrnd_mem [rsp+0x30] + %define stk rsp+0x40 + %else + %define vrnd_mem [base+pd_m524256] + %define stk rsp+0x30 + %endif + movzx t0d, t0b + sub srcq, 2 + movd m15, t0d +%else + %define m10 [base+pd_0x3ff] + %define m9 [base+pd_0x4000] + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %if isprep + %define ssq r3 + %endif + movzx r5, byte [esp+0x1f0] + sub srcq, 2 + movd m15, r5 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m9, [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd m15, m0 + pshufd m7, m15, q1032 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd r6d, m15 + movd r13d, m7 + mova m10, [base+bdct_lb_q+ 0] + mova m11, [base+bdct_lb_q+16] + movd m13, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+ r6*8+2] + movd m15, [base+subpel_filters+r11*8+2] + movd m4, [base+subpel_filters+r13*8+2] +%else + movd r1, m15 + movd r4, m7 + pshufd m15, m15, q0321 + pshufd m7, m7, q0321 + movd r3, m15 + movd r5, m7 + mova m5, [base+bdct_lb_q+ 0] + mova m6, [base+bdct_lb_q+16] + movd m1, [base+subpel_filters+r1*8+2] + movd m2, [base+subpel_filters+r3*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m7, [base+subpel_filters+r5*8+2] + SWAP m4, m7 + mov r3, r3m + %if isprep + lea ss3q, [ssq*3] + %endif + %define m10 m5 + %define m11 m6 + %define m12 m1 + %define m13 m1 +%endif + psrld m14, 10 + paddd m14, m14 + punpckldq m13, m2 + punpckldq m15, m4 + punpcklqdq m13, m15 + pxor m2, m2 + pcmpeqd m0, m2 +%if ARCH_X86_64 + pand m9, m0 +%else + pand m2, m9, m0 + %define m9 m2 + SWAP m7, m4 +%endif + pandn m0, m13 +%if ARCH_X86_64 + SWAP m13, m0 +%else + %define m13 m0 +%endif + por m13, m9 + punpckhbw m15, m13, m13 + punpcklbw m13, m13 + psraw m15, 8 + psraw m13, 8 + pshufb m12, m14, m10 + pshufb m14, m11 + mova m10, [base+spel_s_shuf2] + movd r4d, m14 + shr r4d, 24 +%if ARCH_X86_32 + mova [stk+0x40], m13 + mova [stk+0x50], m15 + pxor m2, m2 +%endif + pshufb m7, m14, m2 + psubb m14, m7 + paddb m12, m10 + paddb m14, m10 +%if ARCH_X86_64 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu m1, [srcq+ssq*0] + movu m8, [srcq+ssq*2] + movu m9, [srcq+ssq*1] + movu m10, [srcq+ss3q ] + movu m7, [srcq+r4 ] + movu m2, [srcq+r11 ] + movu m3, [srcq+r6 ] + movu m4, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m12}, m1, m9, m8, m10 + REPX {pmaddwd x, m13}, m1, m9, m8, m10 + REPX {pshufb x, m14}, m7, m3, m2, m4 + REPX {pmaddwd x, m15}, m7, m3, m2, m4 + mova m5, [rsp+0x10] + movd xm6, [rsp+0x20] + phaddd m1, m7 + phaddd m8, m2 + phaddd m9, m3 + phaddd m10, m4 + movu m2, [srcq+ssq*0] + movu m3, [srcq+ssq*1] + REPX {paddd x, m5}, m1, m9, m8, m10 + REPX {psrad x, xm6}, m1, m9, m8, m10 + packssdw m1, m8 ; 0 2 + packssdw m9, m10 ; 1 3 + movu m0, [srcq+r4 ] + movu m8, [srcq+r6 ] + lea srcq, [srcq+ssq*2] + REPX {pshufb x, m12}, m2, m3 + REPX {pmaddwd x, m13}, m2, m3 + REPX {pshufb x, m14}, m0, m8 + REPX {pmaddwd x, m15}, m0, m8 + phaddd m2, m0 + phaddd m3, m8 + shr myd, 6 + mov r9d, 64 << 24 + lea myd, [t1+myq] + cmovnz r9q, [base+subpel_filters+myq*8] + REPX {paddd x, m5}, m2, m3 + REPX {psrad x, xm6}, m2, m3 + packssdw m2, m3 ; 4 5 + pshufd m3, m2, q1032 ; 5 _ + punpcklwd m0, m1, m9 ; 01 + punpckhwd m1, m9 ; 23 + punpcklwd m2, m3 ; 45 + movq m10, r9 + %define hrnd_mem [rsp+0x10] + %define hsh_mem [rsp+0x20] + %define vsh_mem [rsp+0x28] + %if isput + %define vrnd_mem [rsp+0x30] + %else + %define vrnd_mem [base+pd_m524256] + %endif +%else + mova [stk+0x20], m12 + mova [stk+0x30], m14 + add r4, srcq + MC_4TAP_SCALED_H 0x60 ; 0 1 + MC_4TAP_SCALED_H 0x70 ; 2 3 + MC_4TAP_SCALED_H 0x80 ; 4 5 + mov [stk+0xe0], r4 + mova m3, [base+spel_s_shuf8] + mova m0, [stk+0x60] + mova m1, [stk+0x70] + mova m2, [stk+0x80] + mov myd, mym + mov rX, [esp+0x1f4] + xor r5, r5 + shr myd, 6 + lea rX, [rX+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+rX*8+0] + cmovnz r5, [base+subpel_filters+rX*8+4] + mov r3, r3m + pshufb m0, m3 ; 01 + pshufb m1, m3 ; 23 + pshufb m2, m3 ; 45 + movd m7, r4 + movd m4, r5 + mov r5, r0m + %if isput + mov r1, r1m + %endif + mov r4, [stk+0xe0] + %define dstq r5 + %define tmpq r5 + %define m12 [stk+0x20] + %define m14 [stk+0x30] + %define m13 [stk+0x40] + %define m15 [stk+0x50] + %define hrnd_mem [esp+0x00] + %define hsh_mem [esp+0x10] + %define vsh_mem [esp+0x18] + %if isput + %define vrnd_mem [esp+0x20] + %else + %define vrnd_mem [base+pd_m524256] + %endif + %define m10 m7 + punpckldq m10, m4 +%endif + punpcklbw m10, m10 + psraw m10, 8 + pshufd m3, m10, q0000 + pshufd m4, m10, q1111 + pshufd m5, m10, q2222 + pshufd m10, m10, q3333 +%if ARCH_X86_32 + %xdefine m8 m3 + %xdefine m9 m6 + %xdefine m11 m5 + %xdefine m6 m4 + mova [stk+0x100], m3 + mova [stk+0x110], m4 + mova [stk+0x120], m5 + mova [stk+0x130], m10 + %define m3 [stk+0x100] + %define m4 [stk+0x110] + %define m5 [stk+0x120] + %define m10 [stk+0x130] +%endif +.dy2_w4_loop: + pmaddwd m8, m0, m3 + pmaddwd m9, m1, m3 + mova m0, m2 + pmaddwd m1, m4 + pmaddwd m11, m2, m4 + paddd m8, vrnd_mem + paddd m9, vrnd_mem + pmaddwd m2, m5 + paddd m8, m1 + paddd m9, m11 + paddd m8, m2 + movu m6, [srcq+ssq*0] + movu m1, [srcq+ssq*2] +%if ARCH_X86_64 + movu m11, [srcq+r4 ] + movu m2, [srcq+r11] +%else + movu m11, [r4+ssq*0] + movu m2, [r4+ssq*2] +%endif + pshufb m6, m12 + pshufb m1, m12 + pmaddwd m6, m13 + pmaddwd m1, m13 + pshufb m11, m14 + pshufb m2, m14 + pmaddwd m11, m15 + pmaddwd m2, m15 + phaddd m6, m11 + phaddd m1, m2 + paddd m6, hrnd_mem + paddd m1, hrnd_mem + psrad m6, hsh_mem + psrad m1, hsh_mem + movu m7, [srcq+ssq*1] + movu m11, [srcq+ss3q ] + packssdw m6, m1 ; 6 8 +%if ARCH_X86_64 + movu m2, [srcq+r6 ] + movu m1, [srcq+r13] +%else + movu m2, [r4+ssq*1] + movu m1, [r4+ss3q ] +%endif + pshufb m7, m12 + pshufb m11, m12 + pmaddwd m7, m13 + pmaddwd m11, m13 + pshufb m2, m14 + pshufb m1, m14 + pmaddwd m2, m15 + pmaddwd m1, m15 + phaddd m7, m2 + phaddd m11, m1 + paddd m7, hrnd_mem + paddd m11, hrnd_mem + psrad m7, hsh_mem + psrad m11, hsh_mem + packssdw m7, m11 ; 7 9 +%if ARCH_X86_32 + lea r4, [r4+ssq*4] +%endif + lea srcq, [srcq+ssq*4] + punpcklwd m1, m6, m7 ; 67 + punpckhwd m6, m7 ; 89 + mova m2, m6 + pmaddwd m11, m1, m5 + pmaddwd m7, m1, m10 + pmaddwd m6, m10 + paddd m9, m11 +%if isput + movd m11, vsh_mem +%endif + paddd m8, m7 + paddd m9, m6 +%if isput + psrad m8, m11 + psrad m9, m11 + packssdw m8, m9 + pxor m7, m7 + pmaxsw m8, m7 + pminsw m8, pxmaxm + movq [dstq+dsq*0], m8 + movhps [dstq+dsq*1], m8 + lea dstq, [dstq+dsq*2] +%else + psrad m8, 6 + psrad m9, 6 + packssdw m8, m9 + mova [tmpq], m8 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET ; why not jz .ret? +INIT_XMM ssse3 +.dy2_w8: + mov dword [stk+0xf0], 1 + movifprep tmp_stridem, 16 + jmp .dy2_w_start +.dy2_w16: + mov dword [stk+0xf0], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [stk+0xf0], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [stk+0xf0], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [stk+0xf0], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: + mov myd, mym +%if ARCH_X86_64 + %ifidn %1, put + movifnidn dsm, dsq + %endif + mova [rsp+0x10], m11 + mova [rsp+0x20], m12 + %define hround m11 + %if isput + mova [rsp+0x30], m13 + %else + mova m13, [base+pd_m524256] + %endif + shr t0d, 16 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + movd m15, t0d +%else + %define hround [esp+0x00] + %define m12 [esp+0x10] + %define m10 [base+pd_0x3ff] + %define m8 m0 + %xdefine m14 m4 + %xdefine m15 m3 + %if isput + %define dstq r0 + %else + %define tmpq r0 + %define ssq ssm + %endif + mov r5, [esp+0x1f0] + mov r3, [esp+0x1f4] + shr r5, 16 + movd m15, r5 + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r0, r0m + mov r3, r3m +%endif + sub srcq, 6 + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] +%if ARCH_X86_64 + movq m3, r4q +%else + movd m5, r4 + movd m6, r5 + punpckldq m5, m6 + SWAP m3, m5 +%endif + punpcklbw m3, m3 + psraw m3, 8 + mova [stk+0x100], m7 + mova [stk+0x120], m15 + mov [stk+0x0f8], srcq + mov [stk+0x130], r0q ; dstq / tmpq + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 +%if ARCH_X86_64 + mova [stk+0x140], m0 + mova [stk+0x150], m1 + mova [stk+0x160], m2 + mova [stk+0x170], m3 + %if UNIX64 + mov hm, hd + %endif +%else + mova [stk+0x180], m0 + mova [stk+0x190], m1 + mova [stk+0x1a0], m2 + mova [stk+0x1b0], m3 + SWAP m5, m3 + mov r5, hm + mov [stk+0x134], r5 +%endif + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [stk+0x0f0] + jz .ret +%if ARCH_X86_64 + add qword [stk+0x130], 16 + mov hd, hm +%else + add dword [stk+0x130], 16 + mov r5, [stk+0x134] + mov r0, [stk+0x130] +%endif + mova m7, [stk+0x100] + mova m14, [stk+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] + mova m11, [rsp+0x10] +%endif + mova m15, [stk+0x120] + mov srcq, [stk+0x0f8] +%if ARCH_X86_64 + mov r0q, [stk+0x130] ; dstq / tmpq +%else + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.dy2_hloop: +%if ARCH_X86_64 + mova m9, [base+pq_0x40000000] +%else + %define m9 [base+pq_0x40000000] +%endif + pxor m1, m1 + psrld m2, m14, 10 + mova [stk], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m1 + pshufd m2, m5, q1032 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + pshufd m5, m5, q0321 + pshufd m2, m2, q0321 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pxor m2, m2 + pcmpeqd m5, m2 + mova [stk+0x110], m14 + pshufd m4, m15, q1032 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + movq r11, m14 + punpckhqdq m14, m14 + movq rX, m14 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mov r4d, [stk+ 0] + mov r6d, [stk+ 4] + mov r7d, [stk+ 8] + mov r9d, [stk+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m14, m5, q1100 + pshufd m5, m5, q3322 + pand m7, m9, m4 + pand m8, m9, m6 + pand m15, m9, m14 + pand m9, m9, m5 + pandn m4, m0 + pandn m6, m1 + pandn m14, m2 + pandn m5, m3 + por m7, m4 + por m8, m6 + por m15, m14 + por m9, m5 + punpcklbw m0, m7, m7 + punpckhbw m7, m7 + punpcklbw m1, m8, m8 + punpckhbw m8, m8 + psraw m0, 8 + psraw m7, 8 + psraw m1, 8 + psraw m8, 8 + punpcklbw m2, m15, m15 + punpckhbw m15, m15 + punpcklbw m3, m9, m9 + punpckhbw m9, m9 + psraw m2, 8 + psraw m15, 8 + psraw m3, 8 + psraw m9, 8 + mova [stk+0x10], m0 + mova [stk+0x20], m7 + mova [stk+0x30], m1 + mova [stk+0x40], m8 + mova [stk+0x50], m2 + mova [stk+0x60], m15 + mova [stk+0x70], m3 + mova [stk+0x80], m9 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10 ; 0 + mova [stk+0x90], m1 + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 1, 9, 10 ; 1 + mova [stk+0xa0], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10 ; 2 + mova [stk+0xb0], m3 + MC_8TAP_SCALED_H 4, 5, 6, 1, 2, 3, 9, 10 ; 3 + mova [stk+0xc0], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10 ; 4 + mova [stk+0xd0], m5 + MC_8TAP_SCALED_H 6, 1, 2, 3, 4, 5, 9, 10 ; 5 + MC_8TAP_SCALED_H 7, 1, 2, 3, 4, 5, 9, 10 ; 6 + MC_8TAP_SCALED_H 8, 1, 2, 3, 4, 5, 9, 10 ; 7 + mova m5, [stk+0xd0] + mova m1, [stk+0x90] + mova m2, [stk+0xa0] + mova m3, [stk+0xb0] + mova m9, [stk+0xc0] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m9 ; 23a + punpckhwd m3, m9 ; 23b + mova m10, [stk+0x140] + mova m11, [stk+0x150] + mova m14, [stk+0x160] + mova m15, [stk+0x170] + mova [stk+0x90], m4 + mova [stk+0xa0], m5 + mova [stk+0xb0], m6 + mova [stk+0xc0], m7 + %define hround [rsp+0x10] + %define shift [rsp+0x20] + %if isput + %define vround [rsp+0x30] + %else + %define vround [base+pd_m524256] + %endif +.dy2_vloop: + pmaddwd m4, m0, m10 + pmaddwd m5, m1, m10 + pmaddwd m6, m2, m11 + pmaddwd m7, m3, m11 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 + pmaddwd m6, [stk+0x90], m14 + pmaddwd m7, [stk+0xa0], m14 + pmaddwd m8, [stk+0xb0], m15 + pmaddwd m9, [stk+0xc0], m15 + paddd m4, m6 + paddd m5, m7 + %if isput + pshufd m6, m12, q1032 + %endif + paddd m4, m8 + paddd m5, m9 +%else + movd r0, m15 + movd rX, m4 + pshufd m15, m15, q0321 + pshufd m4, m4, q0321 + movd r4, m15 + movd r5, m4 + mova m14, [stk+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [stk+16], m14 + mov r0, [stk+ 0] + mov rX, [stk+ 4] + mov r4, [stk+ 8] + mov r5, [stk+12] + mova [stk+0x20], m0 + mova [stk+0x30], m1 + mova [stk+0x40], m2 + mova [stk+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m9, m4 + pand m1, m9, m6 + pand m2, m9, m7 + pand m3, m9, m5 + pandn m4, [stk+0x20] + pandn m6, [stk+0x30] + pandn m7, [stk+0x40] + pandn m5, [stk+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + punpcklbw m4, m0, m0 + punpckhbw m0, m0 + punpcklbw m5, m1, m1 + punpckhbw m1, m1 + psraw m4, 8 + psraw m0, 8 + psraw m5, 8 + psraw m1, 8 + punpcklbw m6, m2, m2 + punpckhbw m2, m2 + punpcklbw m7, m3, m3 + punpckhbw m3, m3 + psraw m6, 8 + psraw m2, 8 + psraw m7, 8 + psraw m3, 8 + mova [stk+0x0a0], m4 + mova [stk+0x0b0], m0 + mova [stk+0x0c0], m5 + mova [stk+0x0d0], m1 + mova [stk+0x140], m6 + mova [stk+0x150], m2 + mova [stk+0x160], m7 + mova [stk+0x170], m3 + MC_8TAP_SCALED_H 0xa0, 0x20, 0 ; 0 + MC_8TAP_SCALED_H 0xa0, 0x30 ; 1 + MC_8TAP_SCALED_H 0xa0, 0x40 ; 2 + MC_8TAP_SCALED_H 0xa0, 0x50 ; 3 + MC_8TAP_SCALED_H 0xa0, 0x60 ; 4 + MC_8TAP_SCALED_H 0xa0, 0x70 ; 5 + MC_8TAP_SCALED_H 0xa0, 0x80 ; 6 + MC_8TAP_SCALED_H 0xa0, 0x90 ; 7 + mova m5, [stk+0x60] + mova m6, [stk+0x70] + mova m7, [stk+0x80] + mova m0, [stk+0x90] + mov r0, r0m + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [stk+0x60], m4 + mova [stk+0x70], m5 + mova [stk+0x80], m6 + mova [stk+0x90], m7 + mova m1, [stk+0x20] + mova m2, [stk+0x30] + mova m3, [stk+0x40] + mova m4, [stk+0x50] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova m4, [stk+0x180] + mova m5, [stk+0x190] + mova m6, [stk+0x1a0] + mova m7, [stk+0x1b0] + mova [stk+0x40], m2 + mova [stk+0x50], m3 +.dy2_vloop: + pmaddwd m0, m4 + pmaddwd m1, m4 + pmaddwd m2, m5 + pmaddwd m3, m5 + paddd m0, m2 + paddd m1, m3 + pmaddwd m2, [stk+0x60], m6 + pmaddwd m3, [stk+0x70], m6 + pmaddwd m4, [stk+0x80], m7 + pmaddwd m5, [stk+0x90], m7 + %if isput + movd m6, [esp+0x18] + %endif + paddd m0, m2 + paddd m1, m3 + paddd m0, vrnd_mem + paddd m1, vrnd_mem + paddd m4, m0 + paddd m5, m1 +%endif +%ifidn %1, put + psrad m4, m6 + psrad m5, m6 + packssdw m4, m5 + pxor m7, m7 + pmaxsw m4, m7 + pminsw m4, pxmaxm + mova [dstq], m4 + add dstq, dsm +%else + psrad m4, 6 + psrad m5, 6 + packssdw m4, m5 + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy2_hloop_prep +%if ARCH_X86_64 + MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 0, 1 + mova [stk+0xd0], m4 + MC_8TAP_SCALED_H 8, 5, 6, 7, 9, 4, 0, 1 + mova m4, [stk+0xd0] + mova m0, m2 ; 01a + mova m1, m3 ; 01b + mova m2, [stk+0x90] ; 23a + mova m3, [stk+0xa0] ; 23b + mova m5, [stk+0xb0] ; 45a + mova m6, [stk+0xc0] ; 45b + punpcklwd m7, m4, m8 ; 67a + punpckhwd m4, m8 ; 67b + mova [stk+0x90], m5 + mova [stk+0xa0], m6 + mova [stk+0xb0], m7 + mova [stk+0xc0], m4 +%else + mov r0m, r0 + mov r3, r3m + MC_8TAP_SCALED_H 0xa0, 0xe0 ; 8 + MC_8TAP_SCALED_H 0xa0, 0 ; 9 + mova m7, [stk+0xe0] + mova m2, [stk+0x60] ; 23a + mova m3, [stk+0x70] ; 23b + mova m4, [stk+0x80] ; 45a + mova m5, [stk+0x90] ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova m0, [stk+0x40] ; 01a + mova m1, [stk+0x50] ; 01b + mova [stk+0x40], m2 + mova [stk+0x50], m3 + mova [stk+0x60], m4 + mova [stk+0x70], m5 + mova m4, [stk+0x180] + mova m5, [stk+0x190] + mova [stk+0x80], m6 + mova [stk+0x90], m7 + mova m6, [stk+0x1a0] + mova m7, [stk+0x1b0] + mov r0, r0m +%endif + jmp .dy2_vloop +INIT_XMM ssse3 +.ret: + MC_8TAP_SCALED_RET 0 +%if ARCH_X86_32 && !isprep && required_stack_alignment > STACK_ALIGNMENT + %define r0m [rstk+stack_offset+ 4] + %define r1m [rstk+stack_offset+ 8] + %define r2m [rstk+stack_offset+12] + %define r3m [rstk+stack_offset+16] +%endif +%undef isput +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled_16bpc + mov t0d, (5*15 << 16) | 5*15 + mov t1d, (5*15 << 16) | 5*15 + jmp mangle(private_prefix %+ _%1_8tap_scaled_16bpc %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%elif ARCH_X86_64 +DECLARE_REG_TMP 6, 8 +%else +DECLARE_REG_TMP 1, 2 +%endif +BILIN_SCALED_FN put +FN put_8tap_scaled, sharp, SHARP, SHARP +FN put_8tap_scaled, sharp_smooth, SHARP, SMOOTH +FN put_8tap_scaled, smooth_sharp, SMOOTH, SHARP +FN put_8tap_scaled, smooth, SMOOTH, SMOOTH +FN put_8tap_scaled, sharp_regular, SHARP, REGULAR +FN put_8tap_scaled, regular_sharp, REGULAR, SHARP +FN put_8tap_scaled, smooth_regular, SMOOTH, REGULAR +FN put_8tap_scaled, regular_smooth, REGULAR, SMOOTH +FN put_8tap_scaled, regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%elif ARCH_X86_64 +DECLARE_REG_TMP 6, 7 +%else +DECLARE_REG_TMP 1, 2 +%endif +BILIN_SCALED_FN prep +FN prep_8tap_scaled, sharp, SHARP, SHARP +FN prep_8tap_scaled, sharp_smooth, SHARP, SMOOTH +FN prep_8tap_scaled, smooth_sharp, SMOOTH, SHARP +FN prep_8tap_scaled, smooth, SMOOTH, SMOOTH +FN prep_8tap_scaled, sharp_regular, SHARP, REGULAR +FN prep_8tap_scaled, regular_sharp, REGULAR, SHARP +FN prep_8tap_scaled, smooth_regular, SMOOTH, REGULAR +FN prep_8tap_scaled, regular_smooth, REGULAR, SMOOTH +FN prep_8tap_scaled, regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + +%if ARCH_X86_64 +DECLARE_REG_TMP 6 +%else +DECLARE_REG_TMP 2 +%endif + +%if ARCH_X86_64 +; warp8x8t spills one less xmm register than warp8x8 on WIN64, compensate that +; by allocating 16 bytes more stack space so that stack offsets match up. +%if WIN64 && STACK_ALIGNMENT == 16 +%assign stksz 16*14 +%else +%assign stksz 16*13 +%endif +cglobal warp_affine_8x8t_16bpc, 4, 13, 9, stksz, dst, ds, src, ss, delta, \ + mx, tmp, alpha, beta, \ + filter, my, gamma, cnt +%assign stack_size_padded_8x8t stack_size_padded +%else +cglobal warp_affine_8x8t_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \ + filter, mx, my +%define m8 [esp+16*13] +%define m9 [esp+16*14] +%define cntd dword [esp+4*63] +%define dstq tmpq +%define dsq 0 +%if STACK_ALIGNMENT < 16 +%define dstm [esp+4*65] +%define dsm [esp+4*66] +%else +%define dstm r0m +%define dsm r1m +%endif +%endif +%define base filterq-$$ + mov t0d, r7m + LEA filterq, $$ + shr t0d, 11 +%if ARCH_X86_64 + movddup m8, [base+warp8x8t_rnd] +%else + movddup m1, [base+warp8x8t_rnd] + mov r1, r1m + add r1, r1 + mova m8, m1 + mov r1m, r1 ; ds *= 2 +%endif + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main + jmp .start +.loop: +%if ARCH_X86_64 + lea dstq, [dstq+dsq*4] +%else + add dstq, dsm + mov dstm, dstq +%endif + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main2 +.start: +%if ARCH_X86_32 + mov dstq, dstm +%endif + paddd m1, m8 + paddd m2, m8 + psrad m1, 15 + psrad m2, 15 + packssdw m1, m2 + mova [dstq+dsq*0], m1 + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main3 +%if ARCH_X86_32 + mov dstq, dstm + add dstq, dsm +%endif + paddd m1, m8 + paddd m2, m8 + psrad m1, 15 + psrad m2, 15 + packssdw m1, m2 + mova [dstq+dsq*2], m1 + dec cntd + jg .loop + RET + +%if ARCH_X86_64 +cglobal warp_affine_8x8_16bpc, 4, 13, 10, 16*13, dst, ds, src, ss, delta, \ + mx, tmp, alpha, beta, \ + filter, my, gamma, cnt +ASSERT stack_size_padded == stack_size_padded_8x8t +%else +cglobal warp_affine_8x8_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \ + filter, mx, my +%endif + mov t0d, r7m + LEA filterq, $$ + shr t0d, 11 +%if ARCH_X86_64 + movddup m8, [base+warp8x8_rnd2+t0*8] + movd m9, r7m ; pixel_max + pshufb m9, [base+pw_256] +%else + movddup m1, [base+warp8x8_rnd2+t0*8] + movd m2, r7m ; pixel_max + pshufb m2, [base+pw_256] + mova m8, m1 + mova m9, m2 +%endif + call .main + jmp .start +.loop: +%if ARCH_X86_64 + lea dstq, [dstq+dsq*2] +%else + add dstq, dsm + mov dstm, dstq +%endif + call .main2 +.start: +%if ARCH_X86_32 + mov dstq, dstm +%endif + psrad m1, 16 + psrad m2, 16 + packssdw m1, m2 + pmaxsw m1, m6 + pmulhrsw m1, m8 + pminsw m1, m9 + mova [dstq+dsq*0], m1 + call .main3 +%if ARCH_X86_32 + mov dstq, dstm + add dstq, dsm +%endif + psrad m1, 16 + psrad m2, 16 + packssdw m1, m2 + pmaxsw m1, m6 + pmulhrsw m1, m8 + pminsw m1, m9 + mova [dstq+dsq*1], m1 + dec cntd + jg .loop + RET +ALIGN function_align +.main: + ; Stack args offset by one (r4m -> r5m etc.) due to call +%if WIN64 + mov deltaq, r5m + mov mxd, r6m +%endif + movd m0, [base+warp8x8_shift+t0*4] + movddup m7, [base+warp8x8_rnd1+t0*8] + add filterq, mc_warp_filter-$$ +%if ARCH_X86_64 + movsx alphad, word [deltaq+2*0] + movsx betad, word [deltaq+2*1] + movsx gammad, word [deltaq+2*2] + movsx deltad, word [deltaq+2*3] + lea tmpq, [ssq*3] + add mxd, 512+(64<<10) + sub srcq, tmpq ; src -= ss*3 + imul tmpd, alphad, -7 + mov myd, r7m + add betad, tmpd ; beta -= alpha*7 + imul tmpd, gammad, -7 + add myd, 512+(64<<10) + mov cntd, 4 + add deltad, tmpd ; delta -= gamma*7 +%else +%if STACK_ALIGNMENT < 16 + %assign stack_offset stack_offset - gprsize +%endif + mov r3d, r5m ; abcd +%if STACK_ALIGNMENT < 16 + mov r0, r1m ; dst + mov r1, r2m ; ds + mov [esp+gprsize+4*65], r0 + mov [esp+gprsize+4*66], r1 +%endif + movsx alphad, word [r3+2*0] + movsx r2d, word [r3+2*1] + movsx gammad, word [r3+2*2] + movsx r3d, word [r3+2*3] + imul r5d, alphad, -7 + add r2d, r5d ; beta -= alpha*7 + imul r5d, gammad, -7 + mov [esp+gprsize+4*60], r2d + add r3d, r5d ; delta -= gamma*7 + mov [esp+gprsize+4*61], r3d + mov r3d, r4m ; ss + mov srcq, r3m + mov mxd, r6m + mov myd, r7m + mov dword [esp+gprsize+4*63], 4 ; cnt + mov [esp+gprsize+4*62], r3 + lea r3, [r3*3] + add mxd, 512+(64<<10) + add myd, 512+(64<<10) + sub srcq, r3 ; src -= ss*3 +%if STACK_ALIGNMENT < 16 + %assign stack_offset stack_offset + gprsize +%endif +%endif + mova [rsp+gprsize], m0 + pxor m6, m6 + call .h + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 01 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 1], m1 + mova [rsp+gprsize+16* 4], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 12 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 7], m1 + mova [rsp+gprsize+16*10], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 23 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 2], m1 + mova [rsp+gprsize+16* 5], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 34 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 8], m1 + mova [rsp+gprsize+16*11], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 45 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 3], m1 + mova [rsp+gprsize+16* 6], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 56 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 9], m1 + mova [rsp+gprsize+16*12], m5 + mova m5, m0 +.main2: + call .h +%macro WARP_V 6 ; 01l, 23l, 45l, 01h, 23h, 45h + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m4, [filterq+myq*8] ; a + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m2, [filterq+tmpq*8] ; b + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m3, [filterq+myq*8] ; c + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m1, [filterq+tmpq*8] ; d + lea tmpd, [myq+gammaq] + shr myd, 10 + punpcklwd m4, m2 + punpcklwd m3, m1 + punpckldq m2, m4, m3 + punpckhdq m4, m3 + punpcklbw m1, m6, m2 ; a0 a1 b0 b1 c0 c1 d0 d1 << 8 + pmaddwd m1, [rsp+gprsize+16*%1] + punpckhbw m3, m6, m2 ; a2 a3 b2 b3 c2 c3 d2 d3 << 8 + mova m2, [rsp+gprsize+16*%2] + pmaddwd m3, m2 + mova [rsp+gprsize+16*%1], m2 + paddd m1, m3 + punpcklbw m3, m6, m4 ; a4 a5 b4 b5 c4 c5 d4 d5 << 8 + mova m2, [rsp+gprsize+16*%3] + pmaddwd m3, m2 + mova [rsp+gprsize+16*%2], m2 + paddd m1, m3 + punpcklwd m3, m5, m0 ; 67 + punpckhbw m2, m6, m4 ; a6 a7 b6 b7 c6 c7 d6 d7 << 8 + pmaddwd m2, m3 + mova [rsp+gprsize+16*%3], m3 + paddd m1, m2 + movq m4, [filterq+myq*8] ; e + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] ; f + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m2, [filterq+myq*8] ; g +%if ARCH_X86_64 + lea myd, [tmpq+deltaq] ; my += delta +%else + mov myd, [esp+gprsize+4*61] + add myd, tmpd +%endif + shr tmpd, 10 + punpcklwd m4, m3 + movq m3, [filterq+tmpq*8] ; h + punpcklwd m2, m3 + punpckldq m3, m4, m2 + punpckhdq m4, m2 + punpcklbw m2, m6, m3 ; e0 e1 f0 f1 g0 g1 h0 h1 << 8 + pmaddwd m2, [rsp+gprsize+16*%4] + punpckhbw m6, m3 ; e2 e3 f2 f3 g2 g3 h2 h3 << 8 + mova m3, [rsp+gprsize+16*%5] + pmaddwd m6, m3 + mova [rsp+gprsize+16*%4], m3 + pxor m3, m3 + paddd m2, m6 + punpcklbw m3, m4 ; e4 e5 f4 f5 g4 g5 h4 h5 << 8 + mova m6, [rsp+gprsize+16*%6] + pmaddwd m3, m6 + mova [rsp+gprsize+16*%5], m6 + punpckhwd m5, m0 + pxor m6, m6 + paddd m2, m3 + punpckhbw m3, m6, m4 ; e6 e7 f6 f7 g6 g7 h6 h7 << 8 + pmaddwd m3, m5 + mova [rsp+gprsize+16*%6], m5 + mova m5, m0 + paddd m2, m3 +%endmacro + WARP_V 1, 2, 3, 4, 5, 6 + ret +.main3: + call .h + WARP_V 7, 8, 9, 10, 11, 12 + ret +ALIGN function_align +.h: + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + punpcklbw m0, m6, m3 + movu m3, [srcq-6] + pmaddwd m0, m3 ; 0 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m2, m6, m3 + movu m3, [srcq-4] + pmaddwd m2, m3 ; 1 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m0, m2 ; 0 1 + punpcklbw m2, m6, m3 + movu m3, [srcq-2] + pmaddwd m2, m3 ; 2 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m1, m6, m3 + movu m3, [srcq+0] + pmaddwd m1, m3 ; 3 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m2, m1 ; 2 3 + punpcklbw m1, m6, m3 + movu m3, [srcq+2] + pmaddwd m1, m3 ; 4 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + phaddd m0, m2 ; 0 1 2 3 + punpcklbw m2, m6, m3 + movu m3, [srcq+4] + pmaddwd m2, m3 ; 5 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m1, m2 ; 4 5 + punpcklbw m2, m6, m3 + movu m3, [srcq+6] + pmaddwd m2, m3 ; 6 +%if ARCH_X86_64 + lea mxd, [tmpq+betaq] ; mx += beta +%else + mov mxd, [esp+gprsize*2+4*60] + add mxd, tmpd +%endif + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m4, m6, m3 + movu m3, [srcq+8] +%if ARCH_X86_64 + add srcq, ssq +%else + add srcq, [esp+gprsize*2+4*62] +%endif + pmaddwd m3, m4 ; 7 + phaddd m2, m3 ; 6 7 + phaddd m1, m2 ; 4 5 6 7 + paddd m0, m7 + paddd m1, m7 + psrad m0, [rsp+gprsize*2] + psrad m1, [rsp+gprsize*2] + packssdw m0, m1 + ret + +%macro BIDIR_FN 0 + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] +.w4: + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop +.ret: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] +.w8: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jne .w8_loop + RET +.w16_loop: + call .main + add dstq, strideq +.w16: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + dec hd + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + call .main + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + call .main + mova [dstq+16*6], m0 + mova [dstq+16*7], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+16* 0], m0 + mova [dstq+16* 1], m1 + call .main + mova [dstq+16* 2], m0 + mova [dstq+16* 3], m1 + call .main + mova [dstq+16* 4], m0 + mova [dstq+16* 5], m1 + call .main + mova [dstq+16* 6], m0 + mova [dstq+16* 7], m1 + call .main + mova [dstq+16* 8], m0 + mova [dstq+16* 9], m1 + call .main + mova [dstq+16*10], m0 + mova [dstq+16*11], m1 + call .main + mova [dstq+16*12], m0 + mova [dstq+16*13], m1 + call .main + mova [dstq+16*14], m0 + mova [dstq+16*15], m1 + dec hd + jg .w128_loop + RET +%endmacro + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h +%define base r6-avg_ssse3_table + LEA r6, avg_ssse3_table + tzcnt wd, wm + mov t0d, r6m ; pixel_max + movsxd wq, [r6+wq*4] + shr t0d, 11 + movddup m2, [base+bidir_rnd+t0*8] + movddup m3, [base+bidir_mul+t0*8] + movifnidn hd, hm + add wq, r6 + BIDIR_FN +ALIGN function_align +.main: + mova m0, [tmp1q+16*0] + paddsw m0, [tmp2q+16*0] + mova m1, [tmp1q+16*1] + paddsw m1, [tmp2q+16*1] + add tmp1q, 16*2 + add tmp2q, 16*2 + pmaxsw m0, m2 + pmaxsw m1, m2 + psubsw m0, m2 + psubsw m1, m2 + pmulhw m0, m3 + pmulhw m1, m3 + ret + +cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h +%define base r6-w_avg_ssse3_table + LEA r6, w_avg_ssse3_table + tzcnt wd, wm + mov t0d, r6m ; weight + movd m6, r7m ; pixel_max + movddup m5, [base+pd_65538] + movsxd wq, [r6+wq*4] + pshufb m6, [base+pw_256] + add wq, r6 + lea r6d, [t0-16] + shl t0d, 16 + sub t0d, r6d ; 16-weight, weight + paddw m5, m6 + mov r6d, t0d + shl t0d, 2 + test dword r7m, 0x800 + cmovnz r6d, t0d + movifnidn hd, hm + movd m4, r6d + pslld m5, 7 + pxor m7, m7 + pshufd m4, m4, q0000 + BIDIR_FN +ALIGN function_align +.main: + mova m2, [tmp1q+16*0] + mova m0, [tmp2q+16*0] + punpckhwd m3, m0, m2 + punpcklwd m0, m2 + mova m2, [tmp1q+16*1] + mova m1, [tmp2q+16*1] + add tmp1q, 16*2 + add tmp2q, 16*2 + pmaddwd m3, m4 + pmaddwd m0, m4 + paddd m3, m5 + paddd m0, m5 + psrad m3, 8 + psrad m0, 8 + packssdw m0, m3 + punpckhwd m3, m1, m2 + punpcklwd m1, m2 + pmaddwd m3, m4 + pmaddwd m1, m4 + paddd m3, m5 + paddd m1, m5 + psrad m3, 8 + psrad m1, 8 + packssdw m1, m3 + pminsw m0, m6 + pminsw m1, m6 + pmaxsw m0, m7 + pmaxsw m1, m7 + ret + +%if ARCH_X86_64 +cglobal mask_16bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask +%else +cglobal mask_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, mask +%define hd dword r5m +%define m8 [base+pw_64] +%endif +%define base r6-mask_ssse3_table + LEA r6, mask_ssse3_table + tzcnt wd, wm + mov t0d, r7m ; pixel_max + shr t0d, 11 + movsxd wq, [r6+wq*4] + movddup m6, [base+bidir_rnd+t0*8] + movddup m7, [base+bidir_mul+t0*8] +%if ARCH_X86_64 + mova m8, [base+pw_64] + movifnidn hd, hm +%endif + add wq, r6 + mov maskq, r6mp + BIDIR_FN +ALIGN function_align +.main: + movq m3, [maskq+8*0] + mova m0, [tmp1q+16*0] + mova m4, [tmp2q+16*0] + pxor m5, m5 + punpcklbw m3, m5 + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + psubw m1, m8, m3 + punpckhwd m4, m3, m1 ; m, 64-m + punpcklwd m3, m1 + pmaddwd m2, m4 ; tmp1 * m + tmp2 * (64-m) + pmaddwd m0, m3 + movq m3, [maskq+8*1] + mova m1, [tmp1q+16*1] + mova m4, [tmp2q+16*1] + add maskq, 8*2 + add tmp1q, 16*2 + add tmp2q, 16*2 + psrad m2, 5 + psrad m0, 5 + packssdw m0, m2 + punpcklbw m3, m5 + punpckhwd m2, m1, m4 + punpcklwd m1, m4 + psubw m5, m8, m3 + punpckhwd m4, m3, m5 ; m, 64-m + punpcklwd m3, m5 + pmaddwd m2, m4 ; tmp1 * m + tmp2 * (64-m) + pmaddwd m1, m3 + psrad m2, 5 + psrad m1, 5 + packssdw m1, m2 + pmaxsw m0, m6 + pmaxsw m1, m6 + psubsw m0, m6 + psubsw m1, m6 + pmulhw m0, m7 + pmulhw m1, m7 + ret + +cglobal w_mask_420_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_420_ssse3_table + LEA t0, w_mask_420_ssse3_table + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movd m0, r7m ; sign + shr r6d, 11 + movsxd wq, [t0+wq*4] +%if ARCH_X86_64 + mova m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + mova m9, [base+pw_64] + movddup m10, [base+bidir_rnd+r6*8] + movddup m11, [base+bidir_mul+r6*8] +%else + mova m1, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 + mova m2, [base+pw_64] + movddup m3, [base+bidir_rnd+r6*8] + movddup m4, [base+bidir_mul+r6*8] + ALLOC_STACK -16*4 + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + mova [rsp+16*2], m3 + mova [rsp+16*3], m4 + %define m8 [rsp+gprsize+16*0] + %define m9 [rsp+gprsize+16*1] + %define m10 [rsp+gprsize+16*2] + %define m11 [rsp+gprsize+16*3] +%endif + movd m7, [base+pw_2] + psubw m7, m0 + pshufb m7, [base+pw_256] + add wq, t0 + movifnidn hd, r5m + mov maskq, r6mp + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 4 +.w4: + movq [dstq+strideq*0], m0 + phaddw m2, m3 + movhps [dstq+strideq*1], m0 + phaddd m2, m2 + lea dstq, [dstq+strideq*2] + paddw m2, m7 + movq [dstq+strideq*0], m1 + psrlw m2, 2 + movhps [dstq+strideq*1], m1 + packuswb m2, m2 + movd [maskq], m2 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 4 +.w8: + mova [dstq+strideq*0], m0 + paddw m2, m3 + phaddw m2, m2 + mova [dstq+strideq*1], m1 + paddw m2, m7 + psrlw m2, 2 + packuswb m2, m2 + movd [maskq], m2 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 8 +.w16: + mova [dstq+strideq*1+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*1+16*1], m3 + mova [dstq+strideq*0+16*1], m1 + call .main + paddw m2, [dstq+strideq*1+16*0] + paddw m3, [dstq+strideq*1+16*1] + mova [dstq+strideq*1+16*0], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*1], m1 + paddw m2, m7 + psrlw m2, 2 + packuswb m2, m2 + movq [maskq], m2 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 16 +.w32: + mova [dstq+strideq*1+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*1+16*1], m3 + mova [dstq+strideq*0+16*1], m1 + call .main + mova [dstq+strideq*0+16*2], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*3], m1 + call .main + paddw m2, [dstq+strideq*1+16*0] + paddw m3, [dstq+strideq*1+16*1] + mova [dstq+strideq*1+16*0], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*2], m2 + mova [dstq+strideq*1+16*1], m1 + call .main + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16*2] + paddw m2, [dstq+strideq*1+16*3] + mova [dstq+strideq*1+16*2], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16*3], m1 + packuswb m3, m2 + mova [maskq], m3 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 16*2 +.w64: + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*1+16*2], m3 + mova [dstq+strideq*0+16*1], m1 + call .main + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*2], m0 + mova [dstq+strideq*1+16*4], m3 + mova [dstq+strideq*0+16*3], m1 + call .main + mova [dstq+strideq*1+16*5], m2 + mova [dstq+strideq*0+16*4], m0 + mova [dstq+strideq*1+16*6], m3 + mova [dstq+strideq*0+16*5], m1 + call .main + mova [dstq+strideq*0+16*6], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*7], m2 + mova [dstq+strideq*0+16*7], m1 + call .main + paddw m2, [dstq+strideq*1+16*1] + paddw m3, [dstq+strideq*1+16*2] + mova [dstq+strideq*1+16*0], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*2], m2 + mova [dstq+strideq*1+16*1], m1 + call .main + paddw m2, [dstq+strideq*1+16*3] + paddw m3, [dstq+strideq*1+16*4] + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16*2] + mova [dstq+strideq*1+16*2], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16*3], m1 + packuswb m3, m2 + mova [maskq+16*0], m3 + call .main + paddw m2, [dstq+strideq*1+16*5] + paddw m3, [dstq+strideq*1+16*6] + mova [dstq+strideq*1+16*4], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*6], m2 + mova [dstq+strideq*1+16*5], m1 + call .main + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16*6] + paddw m2, [dstq+strideq*1+16*7] + mova [dstq+strideq*1+16*6], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16*7], m1 + packuswb m3, m2 + mova [maskq+16*1], m3 + sub hd, 2 + jg .w64_loop + RET +.w128_loop: + call .main + lea dstq, [dstq+strideq*2] + add maskq, 16*4 +.w128: + mova [dstq+strideq*1+16* 1], m2 + mova [dstq+strideq*0+16* 0], m0 + mova [dstq+strideq*1+16* 2], m3 + mova [dstq+strideq*0+16* 1], m1 + call .main + mova [dstq+strideq*1+16* 3], m2 + mova [dstq+strideq*0+16* 2], m0 + mova [dstq+strideq*1+16* 4], m3 + mova [dstq+strideq*0+16* 3], m1 + call .main + mova [dstq+strideq*1+16* 5], m2 + mova [dstq+strideq*0+16* 4], m0 + mova [dstq+strideq*1+16* 6], m3 + mova [dstq+strideq*0+16* 5], m1 + call .main + mova [dstq+strideq*1+16* 7], m2 + mova [dstq+strideq*0+16* 6], m0 + mova [dstq+strideq*1+16* 8], m3 + mova [dstq+strideq*0+16* 7], m1 + call .main + mova [dstq+strideq*1+16* 9], m2 + mova [dstq+strideq*0+16* 8], m0 + mova [dstq+strideq*1+16*10], m3 + mova [dstq+strideq*0+16* 9], m1 + call .main + mova [dstq+strideq*1+16*11], m2 + mova [dstq+strideq*0+16*10], m0 + mova [dstq+strideq*1+16*12], m3 + mova [dstq+strideq*0+16*11], m1 + call .main + mova [dstq+strideq*1+16*13], m2 + mova [dstq+strideq*0+16*12], m0 + mova [dstq+strideq*1+16*14], m3 + mova [dstq+strideq*0+16*13], m1 + call .main + mova [dstq+strideq*0+16*14], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*15], m2 + mova [dstq+strideq*0+16*15], m1 + call .main + paddw m2, [dstq+strideq*1+16* 1] + paddw m3, [dstq+strideq*1+16* 2] + mova [dstq+strideq*1+16* 0], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16* 2], m2 + mova [dstq+strideq*1+16* 1], m1 + call .main + paddw m2, [dstq+strideq*1+16* 3] + paddw m3, [dstq+strideq*1+16* 4] + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16* 2] + mova [dstq+strideq*1+16* 2], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16* 3], m1 + packuswb m3, m2 + mova [maskq+16*0], m3 + call .main + paddw m2, [dstq+strideq*1+16* 5] + paddw m3, [dstq+strideq*1+16* 6] + mova [dstq+strideq*1+16* 4], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16* 6], m2 + mova [dstq+strideq*1+16* 5], m1 + call .main + paddw m2, [dstq+strideq*1+16* 7] + paddw m3, [dstq+strideq*1+16* 8] + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16* 6] + mova [dstq+strideq*1+16* 6], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16* 7], m1 + packuswb m3, m2 + mova [maskq+16*1], m3 + call .main + paddw m2, [dstq+strideq*1+16* 9] + paddw m3, [dstq+strideq*1+16*10] + mova [dstq+strideq*1+16* 8], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*10], m2 + mova [dstq+strideq*1+16* 9], m1 + call .main + paddw m2, [dstq+strideq*1+16*11] + paddw m3, [dstq+strideq*1+16*12] + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16*10] + mova [dstq+strideq*1+16*10], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16*11], m1 + packuswb m3, m2 + mova [maskq+16*2], m3 + call .main + paddw m2, [dstq+strideq*1+16*13] + paddw m3, [dstq+strideq*1+16*14] + mova [dstq+strideq*1+16*12], m0 + phaddw m2, m3 + mova [dstq+strideq*1+16*14], m2 + mova [dstq+strideq*1+16*13], m1 + call .main + phaddw m2, m3 + paddw m3, m7, [dstq+strideq*1+16*14] + paddw m2, [dstq+strideq*1+16*15] + mova [dstq+strideq*1+16*14], m0 + paddw m2, m7 + psrlw m3, 2 + psrlw m2, 2 + mova [dstq+strideq*1+16*15], m1 + packuswb m3, m2 + mova [maskq+16*3], m3 + sub hd, 2 + jg .w128_loop + RET +ALIGN function_align +.main: +%macro W_MASK 2 ; dst/tmp_offset, mask + mova m%1, [tmp1q+16*%1] + mova m%2, [tmp2q+16*%1] + punpcklwd m4, m%2, m%1 + punpckhwd m5, m%2, m%1 + psubsw m%1, m%2 + pabsw m%1, m%1 + psubusw m6, m8, m%1 + psrlw m6, 10 ; 64-m + psubw m%2, m9, m6 ; m + punpcklwd m%1, m6, m%2 + punpckhwd m6, m%2 + pmaddwd m%1, m4 + pmaddwd m6, m5 + psrad m%1, 5 + psrad m6, 5 + packssdw m%1, m6 + pmaxsw m%1, m10 + psubsw m%1, m10 + pmulhw m%1, m11 +%endmacro + W_MASK 0, 2 + W_MASK 1, 3 + add tmp1q, 16*2 + add tmp2q, 16*2 + ret + +cglobal w_mask_422_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_422_ssse3_table + LEA t0, w_mask_422_ssse3_table + tzcnt wd, wm + mov r6d, r8m ; pixel_max + movd m7, r7m ; sign + shr r6d, 11 + movsxd wq, [t0+wq*4] +%if ARCH_X86_64 + mova m8, [base+pw_27615] + mova m9, [base+pw_64] + movddup m10, [base+bidir_rnd+r6*8] + movddup m11, [base+bidir_mul+r6*8] +%else + mova m1, [base+pw_27615] + mova m2, [base+pw_64] + movddup m3, [base+bidir_rnd+r6*8] + movddup m4, [base+bidir_mul+r6*8] + ALLOC_STACK -16*4 + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + mova [rsp+16*2], m3 + mova [rsp+16*3], m4 +%endif + pxor m0, m0 + add wq, t0 + pshufb m7, m0 + movifnidn hd, r5m + mov maskq, r6mp + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] +.w4: + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop +.end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] +.w8: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + call .main + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + call .main + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + call .main + mova [dstq+16*6], m0 + mova [dstq+16*7], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+16* 0], m0 + mova [dstq+16* 1], m1 + call .main + mova [dstq+16* 2], m0 + mova [dstq+16* 3], m1 + call .main + mova [dstq+16* 4], m0 + mova [dstq+16* 5], m1 + call .main + mova [dstq+16* 6], m0 + mova [dstq+16* 7], m1 + call .main + mova [dstq+16* 8], m0 + mova [dstq+16* 9], m1 + call .main + mova [dstq+16*10], m0 + mova [dstq+16*11], m1 + call .main + mova [dstq+16*12], m0 + mova [dstq+16*13], m1 + call .main + mova [dstq+16*14], m0 + mova [dstq+16*15], m1 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + W_MASK 0, 2 + W_MASK 1, 3 + phaddw m2, m3 + add tmp1q, 16*2 + add tmp2q, 16*2 + packuswb m2, m2 + pxor m3, m3 + psubb m2, m7 + pavgb m2, m3 + movq [maskq], m2 + add maskq, 8 + ret + +cglobal w_mask_444_16bpc, 4, 7, 12, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_444_ssse3_table + LEA t0, w_mask_444_ssse3_table + tzcnt wd, wm + mov r6d, r8m ; pixel_max + shr r6d, 11 + movsxd wq, [t0+wq*4] +%if ARCH_X86_64 + mova m8, [base+pw_27615] + mova m9, [base+pw_64] + movddup m10, [base+bidir_rnd+r6*8] + movddup m11, [base+bidir_mul+r6*8] +%else + mova m1, [base+pw_27615] + mova m2, [base+pw_64] + movddup m3, [base+bidir_rnd+r6*8] + movddup m7, [base+bidir_mul+r6*8] + ALLOC_STACK -16*3 + mova [rsp+16*0], m1 + mova [rsp+16*1], m2 + mova [rsp+16*2], m3 + %define m11 m7 +%endif + add wq, t0 + movifnidn hd, r5m + mov maskq, r6mp + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] +.w4: + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop +.end: + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] +.w8: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + sub hd, 2 + jg .w8_loop +.w8_end: + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + call .main + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + call .main + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + call .main + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + call .main + mova [dstq+16*6], m0 + mova [dstq+16*7], m1 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+16* 0], m0 + mova [dstq+16* 1], m1 + call .main + mova [dstq+16* 2], m0 + mova [dstq+16* 3], m1 + call .main + mova [dstq+16* 4], m0 + mova [dstq+16* 5], m1 + call .main + mova [dstq+16* 6], m0 + mova [dstq+16* 7], m1 + call .main + mova [dstq+16* 8], m0 + mova [dstq+16* 9], m1 + call .main + mova [dstq+16*10], m0 + mova [dstq+16*11], m1 + call .main + mova [dstq+16*12], m0 + mova [dstq+16*13], m1 + call .main + mova [dstq+16*14], m0 + mova [dstq+16*15], m1 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + W_MASK 0, 2 + W_MASK 1, 3 + packuswb m2, m3 + add tmp1q, 16*2 + add tmp2q, 16*2 + mova [maskq], m2 + add maskq, 16 + ret + +; (a * (64 - m) + b * m + 32) >> 6 +; = (((b - a) * m + 32) >> 6) + a +; = (((b - a) * (m << 9) + 16384) >> 15) + a +; except m << 9 overflows int16_t when m == 64 (which is possible), +; but if we negate m it works out (-64 << 9 == -32768). +; = (((a - b) * (m * -512) + 16384) >> 15) + a +cglobal blend_16bpc, 3, 7, 8, dst, stride, tmp, w, h, mask, stride3 +%define base r6-blend_ssse3_table + LEA r6, blend_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r6+wq*4] + movifnidn maskq, maskmp + mova m7, [base+pw_m512] + add wq, r6 + lea stride3q, [strideq*3] + pxor m6, m6 + jmp wq +.w4: + mova m5, [maskq] + movq m0, [dstq+strideq*0] + movhps m0, [dstq+strideq*1] + movq m1, [dstq+strideq*2] + movhps m1, [dstq+stride3q ] + psubw m2, m0, [tmpq+16*0] + psubw m3, m1, [tmpq+16*1] + add maskq, 16 + add tmpq, 32 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + pmullw m4, m7 + pmullw m5, m7 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+stride3q ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + mova m5, [maskq] + mova m0, [dstq+strideq*0] + mova m1, [dstq+strideq*1] + psubw m2, m0, [tmpq+16*0] + psubw m3, m1, [tmpq+16*1] + add maskq, 16 + add tmpq, 32 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + pmullw m4, m7 + pmullw m5, m7 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8 + RET +.w16: + mova m5, [maskq] + mova m0, [dstq+16*0] + mova m1, [dstq+16*1] + psubw m2, m0, [tmpq+16*0] + psubw m3, m1, [tmpq+16*1] + add maskq, 16 + add tmpq, 32 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + pmullw m4, m7 + pmullw m5, m7 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, strideq + dec hd + jg .w16 + RET +.w32: + mova m5, [maskq+16*0] + mova m0, [dstq+16*0] + mova m1, [dstq+16*1] + psubw m2, m0, [tmpq+16*0] + psubw m3, m1, [tmpq+16*1] + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + pmullw m4, m7 + pmullw m5, m7 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova m5, [maskq+16*1] + mova m0, [dstq+16*2] + mova m1, [dstq+16*3] + psubw m2, m0, [tmpq+16*2] + psubw m3, m1, [tmpq+16*3] + add maskq, 32 + add tmpq, 64 + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + pmullw m4, m7 + pmullw m5, m7 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+16*2], m0 + mova [dstq+16*3], m1 + add dstq, strideq + dec hd + jg .w32 + RET + +cglobal blend_v_16bpc, 3, 6, 6, dst, stride, tmp, w, h +%define base r5-blend_v_ssse3_table + LEA r5, blend_v_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + add wq, r5 + jmp wq +.w2: + movd m4, [base+obmc_masks+2*2] +.w2_loop: + movd m0, [dstq+strideq*0] + movd m2, [tmpq+4*0] + movd m1, [dstq+strideq*1] + movd m3, [tmpq+4*1] + add tmpq, 4*2 + psubw m2, m0 + psubw m3, m1 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + movd [dstq+strideq*0], m0 + movd [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w2_loop + RET +.w4: + movddup m2, [base+obmc_masks+4*2] +.w4_loop: + movq m0, [dstq+strideq*0] + movhps m0, [dstq+strideq*1] + mova m1, [tmpq] + add tmpq, 8*2 + psubw m1, m0 + pmulhrsw m1, m2 + paddw m0, m1 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w4_loop + RET +.w8: + mova m4, [base+obmc_masks+8*2] +.w8_loop: + mova m0, [dstq+strideq*0] + mova m2, [tmpq+16*0] + mova m1, [dstq+strideq*1] + mova m3, [tmpq+16*1] + add tmpq, 16*2 + psubw m2, m0 + psubw m3, m1 + pmulhrsw m2, m4 + pmulhrsw m3, m4 + paddw m0, m2 + paddw m1, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + mova m4, [base+obmc_masks+16*2] + movq m5, [base+obmc_masks+16*3] +.w16_loop: + mova m0, [dstq+16*0] + mova m2, [tmpq+16*0] + mova m1, [dstq+16*1] + mova m3, [tmpq+16*1] + add tmpq, 16*2 + psubw m2, m0 + psubw m3, m1 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, strideq + dec hd + jg .w16_loop + RET +.w32: +%if WIN64 + movaps [rsp+8], m6 +%endif + mova m4, [base+obmc_masks+16*4] + mova m5, [base+obmc_masks+16*5] + mova m6, [base+obmc_masks+16*6] +.w32_loop: + mova m0, [dstq+16*0] + mova m2, [tmpq+16*0] + mova m1, [dstq+16*1] + mova m3, [tmpq+16*1] + psubw m2, m0 + psubw m3, m1 + pmulhrsw m2, m4 + pmulhrsw m3, m5 + paddw m0, m2 + mova m2, [dstq+16*2] + paddw m1, m3 + mova m3, [tmpq+16*2] + add tmpq, 16*4 + psubw m3, m2 + pmulhrsw m3, m6 + paddw m2, m3 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + add dstq, strideq + dec hd + jg .w32_loop +%if WIN64 + movaps m6, [rsp+8] +%endif + RET + +%macro BLEND_H_ROW 2-3 0; dst_off, tmp_off, inc_tmp + mova m0, [dstq+16*(%1+0)] + mova m2, [tmpq+16*(%2+0)] + mova m1, [dstq+16*(%1+1)] + mova m3, [tmpq+16*(%2+1)] +%if %3 + add tmpq, 16*%3 +%endif + psubw m2, m0 + psubw m3, m1 + pmulhrsw m2, m5 + pmulhrsw m3, m5 + paddw m0, m2 + paddw m1, m3 + mova [dstq+16*(%1+0)], m0 + mova [dstq+16*(%1+1)], m1 +%endmacro + +cglobal blend_h_16bpc, 3, 7, 6, dst, ds, tmp, w, h, mask +%define base r6-blend_h_ssse3_table + LEA r6, blend_h_ssse3_table + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + movddup m4, [base+blend_shuf] + lea maskq, [base+obmc_masks+hq*2] + lea hd, [hq*3] + add wq, r6 + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd m0, [dstq+dsq*0] + movd m2, [dstq+dsq*1] + movd m3, [maskq+hq*2] + movq m1, [tmpq] + add tmpq, 4*2 + punpckldq m0, m2 + punpcklwd m3, m3 + psubw m1, m0 + pmulhrsw m1, m3 + paddw m0, m1 + movd [dstq+dsq*0], m0 + psrlq m0, 32 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +.w4: + mova m3, [base+blend_shuf] +.w4_loop: + movq m0, [dstq+dsq*0] + movhps m0, [dstq+dsq*1] + movd m2, [maskq+hq*2] + mova m1, [tmpq] + add tmpq, 8*2 + psubw m1, m0 + pshufb m2, m3 + pmulhrsw m1, m2 + paddw m0, m1 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +.w8: + movddup m5, [base+blend_shuf+8] +%if WIN64 + movaps [rsp+ 8], m6 + movaps [rsp+24], m7 +%endif +.w8_loop: + movd m7, [maskq+hq*2] + mova m0, [dstq+dsq*0] + mova m2, [tmpq+16*0] + mova m1, [dstq+dsq*1] + mova m3, [tmpq+16*1] + add tmpq, 16*2 + pshufb m6, m7, m4 + psubw m2, m0 + pshufb m7, m5 + psubw m3, m1 + pmulhrsw m2, m6 + pmulhrsw m3, m7 + paddw m0, m2 + paddw m1, m3 + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop +%if WIN64 + movaps m6, [rsp+ 8] + movaps m7, [rsp+24] +%endif + RET +.w16: + movd m5, [maskq+hq*2] + pshufb m5, m4 + BLEND_H_ROW 0, 0, 2 + add dstq, dsq + inc hq + jl .w16 + RET +.w32: + movd m5, [maskq+hq*2] + pshufb m5, m4 + BLEND_H_ROW 0, 0 + BLEND_H_ROW 2, 2, 4 + add dstq, dsq + inc hq + jl .w32 + RET +.w64: + movd m5, [maskq+hq*2] + pshufb m5, m4 + BLEND_H_ROW 0, 0 + BLEND_H_ROW 2, 2 + BLEND_H_ROW 4, 4 + BLEND_H_ROW 6, 6, 8 + add dstq, dsq + inc hq + jl .w64 + RET +.w128: + movd m5, [maskq+hq*2] + pshufb m5, m4 + BLEND_H_ROW 0, 0 + BLEND_H_ROW 2, 2 + BLEND_H_ROW 4, 4 + BLEND_H_ROW 6, 6, 16 + BLEND_H_ROW 8, -8 + BLEND_H_ROW 10, -6 + BLEND_H_ROW 12, -4 + BLEND_H_ROW 14, -2 + add dstq, dsq + inc hq + jl .w128 + RET + +; emu_edge args: +; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih, +; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride, +; const pixel *ref, const ptrdiff_t ref_stride +; +; bw, bh total filled size +; iw, ih, copied block -> fill bottom, right +; x, y, offset in bw/bh -> fill top, left +cglobal emu_edge_16bpc, 10, 13, 1, bw, bh, iw, ih, x, \ + y, dst, dstride, src, sstride, \ + bottomext, rightext, blk + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + +%if ARCH_X86_64 + %define reg_zero r12q + %define reg_tmp r10 + %define reg_src srcq + %define reg_bottomext bottomextq + %define reg_rightext rightextq + %define reg_blkm r9m +%else + %define reg_zero r6 + %define reg_tmp r0 + %define reg_src r1 + %define reg_bottomext r0 + %define reg_rightext r1 + %define reg_blkm r2m +%endif + ; + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor reg_zero, reg_zero + lea reg_tmp, [ihq-1] + cmp yq, ihq + cmovs reg_tmp, yq + test yq, yq + cmovs reg_tmp, reg_zero +%if ARCH_X86_64 + imul reg_tmp, sstrideq + add srcq, reg_tmp +%else + imul reg_tmp, sstridem + mov reg_src, srcm + add reg_src, reg_tmp +%endif + ; + ; ref += iclip(x, 0, iw - 1) + lea reg_tmp, [iwq-1] + cmp xq, iwq + cmovs reg_tmp, xq + test xq, xq + cmovs reg_tmp, reg_zero + lea reg_src, [reg_src+reg_tmp*2] +%if ARCH_X86_32 + mov srcm, reg_src +%endif + ; + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) +%if ARCH_X86_32 + mov r1, r1m ; restore bh +%endif + lea reg_bottomext, [yq+bhq] + sub reg_bottomext, ihq + lea r3, [bhq-1] + cmovs reg_bottomext, reg_zero + ; + + DEFINE_ARGS bw, bh, iw, ih, x, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, reg_zero + cmp reg_bottomext, bhq + cmovns reg_bottomext, r3 + cmp topextq, bhq + cmovg topextq, r3 + %if ARCH_X86_32 + mov r4m, reg_bottomext + ; + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + mov r0, r0m ; restore bw + %endif + lea reg_rightext, [xq+bwq] + sub reg_rightext, iwq + lea r2, [bwq-1] + cmovs reg_rightext, reg_zero + + DEFINE_ARGS bw, bh, iw, ih, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, reg_zero + cmp reg_rightext, bwq + cmovns reg_rightext, r2 + %if ARCH_X86_32 + mov r3m, r1 + %endif + cmp leftextq, bwq + cmovns leftextq, r2 + +%undef reg_zero +%undef reg_tmp +%undef reg_src +%undef reg_bottomext +%undef reg_rightext + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; center_h = bh - top_ext - bottom_ext +%if ARCH_X86_64 + lea r3, [bottomextq+topextq] + sub centerhq, r3 +%else + mov r1, centerhm ; restore r1 + sub centerhq, topextq + sub centerhq, r4m + mov r1m, centerhq +%endif + ; + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq +%if ARCH_X86_64 + imul r2, dstrideq +%else + mov r6, r6m ; restore dstq + imul r2, dstridem +%endif + add dstq, r2 + mov reg_blkm, dstq ; save pointer for ext + ; + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq +%if ARCH_X86_64 + lea r3, [rightextq+leftextq] + sub centerwq, r3 +%else + sub centerwq, r3m + sub centerwq, leftextq +%endif + +; vloop Macro +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix + %if ARCH_X86_64 + %define reg_tmp r12 + %else + %define reg_tmp r0 + %endif +.v_loop_%3: + %if ARCH_X86_32 + mov r0, r0m + mov r1, r1m + %endif +%if %1 + ; left extension + %if ARCH_X86_64 + movd m0, [srcq] + %else + mov r3, srcm + movd m0, [r3] + %endif + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + xor r3, r3 +.left_loop_%3: + mova [dstq+r3*2], m0 + add r3, mmsize/2 + cmp r3, leftextq + jl .left_loop_%3 + ; body + lea reg_tmp, [dstq+leftextq*2] +%endif + xor r3, r3 +.body_loop_%3: + %if ARCH_X86_64 + movu m0, [srcq+r3*2] + %else + mov r1, srcm + movu m0, [r1+r3*2] + %endif +%if %1 + movu [reg_tmp+r3*2], m0 +%else + movu [dstq+r3*2], m0 +%endif + add r3, mmsize/2 + cmp r3, centerwq + jl .body_loop_%3 +%if %2 + ; right extension +%if %1 + lea reg_tmp, [reg_tmp+centerwq*2] +%else + lea reg_tmp, [dstq+centerwq*2] +%endif + %if ARCH_X86_64 + movd m0, [srcq+centerwq*2-2] + %else + mov r3, srcm + movd m0, [r3+centerwq*2-2] + %endif + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + xor r3, r3 +.right_loop_%3: + movu [reg_tmp+r3*2], m0 + add r3, mmsize/2 + %if ARCH_X86_64 + cmp r3, rightextq + %else + cmp r3, r3m + %endif + jl .right_loop_%3 +%endif + %if ARCH_X86_64 + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 + %else + add dstq, dstridem + mov r0, sstridem + add srcm, r0 + sub dword centerhm, 1 + jg .v_loop_%3 + mov r0, r0m ; restore r0 + %endif +%endmacro ; vloop MACRO + + test leftextq, leftextq + jnz .need_left_ext + %if ARCH_X86_64 + test rightextq, rightextq + jnz .need_right_ext + %else + cmp leftextq, r3m ; leftextq == 0 + jne .need_right_ext + %endif + v_loop 0, 0, 0 + jmp .body_done + + ;left right extensions +.need_left_ext: + %if ARCH_X86_64 + test rightextq, rightextq + %else + mov r3, r3m + test r3, r3 + %endif + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: +; r0 ; bw +; r1 ;; x loop +; r4 ;; y loop +; r5 ; topextq +; r6 ;dstq +; r7 ;dstrideq +; r8 ; srcq +%if ARCH_X86_64 + %define reg_dstride dstrideq +%else + %define reg_dstride r2 +%endif + ; + ; bottom edge extension + %if ARCH_X86_64 + test bottomextq, bottomextq + jz .top + %else + xor r1, r1 + cmp r1, r4m + je .top + %endif + ; + %if ARCH_X86_64 + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 + %else + mov r3, dstq + mov reg_dstride, dstridem + sub r3, reg_dstride + mov srcm, r3 + %endif + ; +.bottom_x_loop: + %if ARCH_X86_64 + mova m0, [srcq+r1*2] + lea r3, [dstq+r1*2] + mov r4, bottomextq + %else + mov r3, srcm + mova m0, [r3+r1*2] + lea r3, [dstq+r1*2] + mov r4, r4m + %endif + ; +.bottom_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .bottom_y_loop + add r1, mmsize/2 + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end +%if ARCH_X86_64 + mov srcq, reg_blkm +%else + mov r3, reg_blkm + mov reg_dstride, dstridem +%endif + mov dstq, dstm + xor r1, r1 + ; +.top_x_loop: +%if ARCH_X86_64 + mova m0, [srcq+r1*2] +%else + mov r3, reg_blkm + mova m0, [r3+r1*2] +%endif + lea r3, [dstq+r1*2] + mov r4, topextq + ; +.top_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .top_y_loop + add r1, mmsize/2 + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +%undef reg_dstride +%undef reg_blkm +%undef reg_tmp + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +%if ARCH_X86_64 +cglobal resize_16bpc, 0, 12, 16, 1*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%elif STACK_ALIGNMENT >= 16 +cglobal resize_16bpc, 0, 7, 8, 6*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%else +cglobal resize_16bpc, 0, 6, 8, 6*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%endif + movifnidn dstq, dstmp + movifnidn srcq, srcmp +%if STACK_ALIGNMENT >= 16 + movifnidn dst_wd, dst_wm +%endif +%if ARCH_X86_64 + movifnidn hd, hm +%endif + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + movd m4, pxmaxm + movd m7, dxm + movd m6, mx0m + movd m5, src_wm + punpcklwd m4, m4 + pshufd m4, m4, q0000 + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + pshufd m5, m5, q0000 + mova [rsp+16*3*ARCH_X86_32], m4 +%if ARCH_X86_64 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x + LEA r7, $$ + %define base r7-$$ +%else + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x + %define hd dword r5m + %if STACK_ALIGNMENT >= 16 + LEA r6, $$ + %define base r6-$$ + %else + LEA r4, $$ + %define base r4-$$ + %endif +%endif +%if ARCH_X86_64 + mova m12, [base+pd_64] + mova m11, [base+pd_63] +%else + %define m12 [base+pd_64] + %define m11 [base+pd_63] +%endif + pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3] + pslld m7, 2 ; dx*4 + pslld m5, 14 + paddd m6, m4 ; mx+[0..3]*dx + SCRATCH 7, 15, 0 + SCRATCH 6, 14, 1 + SCRATCH 5, 13, 2 + pxor m1, m1 +.loop_y: + xor xd, xd + mova m0, m14 ; per-line working version of mx +.loop_x: + pcmpgtd m1, m0 + pandn m1, m0 + psrad m2, m0, 8 ; filter offset (unmasked) + pcmpgtd m3, m13, m1 + pand m1, m3 + pandn m3, m13 + por m1, m3 + psubd m3, m0, m1 ; pshufb offset + psrad m1, 14 ; clipped src_x offset + psrad m3, 14 ; pshufb edge_emu offset + pand m2, m11 ; filter offset (masked) + ; load source pixels +%if ARCH_X86_64 + movd r8d, m1 + pshuflw m1, m1, q3232 + movd r9d, m1 + punpckhqdq m1, m1 + movd r10d, m1 + psrlq m1, 32 + movd r11d, m1 + movu m4, [srcq+r8*2] + movu m5, [srcq+r9*2] + movu m6, [srcq+r10*2] + movu m7, [srcq+r11*2] + ; if no emulation is required, we don't need to shuffle or emulate edges + packssdw m3, m3 + movq r11, m3 + test r11, r11 + jz .filter + movsx r8, r11w + sar r11, 16 + movsx r9, r11w + sar r11, 16 + movsx r10, r11w + sar r11, 16 + movu m1, [base+resize_shuf+8+r8*2] + movu m3, [base+resize_shuf+8+r9*2] + movu m8, [base+resize_shuf+8+r10*2] + movu m9, [base+resize_shuf+8+r11*2] + pshufb m4, m1 + pshufb m5, m3 + pshufb m6, m8 + pshufb m7, m9 +.filter: + movd r8d, m2 + pshuflw m2, m2, q3232 + movd r9d, m2 + punpckhqdq m2, m2 + movd r10d, m2 + psrlq m2, 32 + movd r11d, m2 + movq m8, [base+resize_filter+r8*8] + movq m2, [base+resize_filter+r9*8] + pxor m9, m9 + punpcklbw m1, m9, m8 + punpcklbw m3, m9, m2 + psraw m1, 8 + psraw m3, 8 + movq m10, [base+resize_filter+r10*8] + movq m2, [base+resize_filter+r11*8] + punpcklbw m8, m9, m10 + punpcklbw m9, m2 + psraw m8, 8 + psraw m9, 8 + pmaddwd m4, m1 + pmaddwd m5, m3 + pmaddwd m6, m8 + pmaddwd m7, m9 + phaddd m4, m5 +%else + movd r3, m1 + pshuflw m1, m1, q3232 + movd r1, m1 + punpckhqdq m1, m1 + movu m4, [srcq+r3*2] + movu m5, [srcq+r1*2] + movd r3, m1 + psrlq m1, 32 + movd r1, m1 + movu m6, [srcq+r3*2] + movu m7, [srcq+r1*2] + ; if no emulation is required, we don't need to shuffle or emulate edges + pxor m1, m1 + pcmpeqb m1, m3 + pmovmskb r3d, m1 + cmp r3d, 0xffff + je .filter + movd r3, m3 + movu m1, [base+resize_shuf+8+r3*2] + pshuflw m3, m3, q3232 + movd r1, m3 + pshufb m4, m1 + movu m1, [base+resize_shuf+8+r1*2] + punpckhqdq m3, m3 + movd r3, m3 + pshufb m5, m1 + movu m1, [base+resize_shuf+8+r3*2] + psrlq m3, 32 + movd r1, m3 + pshufb m6, m1 + movu m1, [base+resize_shuf+8+r1*2] + pshufb m7, m1 +.filter: + mova [esp+4*16], m6 + mova [esp+5*16], m7 + movd r3, m2 + pshuflw m2, m2, q3232 + movd r1, m2 + movq m6, [base+resize_filter+r3*8] + movq m7, [base+resize_filter+r1*8] + pxor m3, m3 + punpcklbw m1, m3, m6 + punpcklbw m3, m7 + psraw m1, 8 + psraw m3, 8 + pmaddwd m4, m1 + pmaddwd m5, m3 + punpckhqdq m2, m2 + movd r3, m2 + psrlq m2, 32 + movd r1, m2 + phaddd m4, m5 + movq m2, [base+resize_filter+r3*8] + movq m5, [base+resize_filter+r1*8] + mova m6, [esp+4*16] + mova m7, [esp+5*16] + pxor m3, m3 + punpcklbw m1, m3, m2 + punpcklbw m3, m5 + psraw m1, 8 + psraw m3, 8 + pmaddwd m6, m1 + pmaddwd m7, m3 +%endif + phaddd m6, m7 + phaddd m4, m6 + pxor m1, m1 + psubd m2, m12, m4 + psrad m2, 7 + packssdw m2, m2 + pmaxsw m2, m1 + pminsw m2, [rsp+16*3*ARCH_X86_32] + movq [dstq+xq*2], m2 + paddd m0, m15 + add xd, 4 +%if STACK_ALIGNMENT >= 16 + cmp xd, dst_wd +%else + cmp xd, dst_wm +%endif + jl .loop_x + add dstq, dst_stridemp + add srcq, src_stridemp + dec hd + jg .loop_y + RET diff --git a/third_party/dav1d/src/x86/mc_avx2.asm b/third_party/dav1d/src/x86/mc_avx2.asm new file mode 100644 index 0000000000..3b208033bd --- /dev/null +++ b/third_party/dav1d/src/x86/mc_avx2.asm @@ -0,0 +1,5669 @@ +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018-2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +; dav1d_obmc_masks[] with 64-x interleaved +obmc_masks: db 0, 0, 0, 0 + ; 2 + db 45, 19, 64, 0 + ; 4 + db 39, 25, 50, 14, 59, 5, 64, 0 + ; 8 + db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 + ; 16 + db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 + db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 + ; 32 + db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 + db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 + db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 + db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0 + +warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 + db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 +warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 + db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 +subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11 +subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 +bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 +bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +pb_8x0_8x8: db 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8 +bdct_lb_dw: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 +wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 + +wm_420_sign: dd 0x01020102, 0x01010101 +wm_422_sign: dd 0x80808080, 0x7f7f7f7f + +pb_64: times 4 db 64 +pw_m256: times 2 dw -256 +pw_15: times 2 dw 15 +pw_32: times 2 dw 32 +pw_34: times 2 dw 34 +pw_258: times 2 dw 258 +pw_512: times 2 dw 512 +pw_1024: times 2 dw 1024 +pw_2048: times 2 dw 2048 +pw_6903: times 2 dw 6903 +pw_8192: times 2 dw 8192 +pd_32: dd 32 +pd_63: dd 63 +pd_512: dd 512 +pd_32768: dd 32768 +pd_0x3ff: dd 0x3ff +pd_0x4000: dd 0x4000 +pq_0x40000000: dq 0x40000000 + +cextern mc_subpel_filters +cextern mc_warp_filter2 +cextern resize_filter + +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) +%%table: + %rep %0 - 2 + dw %%base %+ .w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_1024: + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_2048: + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_8bpc_avx2.put) +%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_8bpc_avx2.prep) + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, avx2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, avx2, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, avx2, 2, 4, 8, 16, 32, 32, 32 + +SECTION .text + +INIT_XMM avx2 +cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy + movifnidn mxyd, r6m ; mx + lea r7, [put_avx2] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [r7+wq*2+table_offset(put,)] + add wq, r7 + jmp wq +.put_w2: + movzx r6d, word [srcq+ssq*0] + movzx r7d, word [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6w + mov [dstq+dsq*1], r7w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r6d, [srcq+ssq*0] + mov r7d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6d + mov [dstq+dsq*1], r7d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + mov r6, [srcq+ssq*0] + mov r7, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6 + mov [dstq+dsq*1], r7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +INIT_YMM avx2 +.put_w32: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+ssq*0+32*0] + movu m1, [srcq+ssq*0+32*1] + movu m2, [srcq+ssq*1+32*0] + movu m3, [srcq+ssq*1+32*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+32*0], m0 + mova [dstq+dsq*0+32*1], m1 + mova [dstq+dsq*1+32*0], m2 + mova [dstq+dsq*1+32*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] + movu m2, [srcq+32*2] + movu m3, [srcq+32*3] + add srcq, ssq + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 + ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 + imul mxyd, 255 + vbroadcasti128 m4, [bilin_h_shuf8] + add mxyd, 16 + movd xm5, mxyd + mov mxyd, r7m ; my + vpbroadcastw m5, xm5 + test mxyd, mxyd + jnz .hv + movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)] + vpbroadcastd m3, [pw_2048] + add wq, r7 + jmp wq +.h_w2: + movd xm0, [srcq+ssq*0] + pinsrd xm0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 + pmulhrsw xm0, xm3 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + mova xm4, [bilin_h_shuf4] +.h_w4_loop: + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 + pmulhrsw xm0, xm3 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pshufb xm1, xm4 + pmaddubsw xm0, xm5 + pmaddubsw xm1, xm5 + pmulhrsw xm0, xm3 + pmulhrsw xm1, xm3 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*1+8*0], 1 + movu xm1, [srcq+ssq*0+8*1] + vinserti128 m1, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w32 + RET +.h_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movu m1, [srcq+8*4] + movu m2, [srcq+8*5] + add srcq, ssq + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + packuswb m1, m2 + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + mov r6, -32*3 +.h_w128_loop: + movu m0, [srcq+r6+32*3+8*0] + movu m1, [srcq+r6+32*3+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+r6+32*3], m0 + add r6, 32 + jle .h_w128_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)] + imul mxyd, 255 + vpbroadcastd m5, [pw_2048] + add mxyd, 16 + add wq, r7 + movd xm4, mxyd + vpbroadcastw m4, xm4 + jmp wq +.v_w2: + movd xm0, [srcq+ssq*0] +.v_w2_loop: + pinsrw xm1, xm0, [srcq+ssq*1], 1 ; 0 1 + lea srcq, [srcq+ssq*2] + pinsrw xm0, xm1, [srcq+ssq*0], 0 ; 2 1 + pshuflw xm1, xm1, q2301 ; 1 0 + punpcklbw xm1, xm0 + pmaddubsw xm1, xm4 + pmulhrsw xm1, xm5 + packuswb xm1, xm1 + pextrw [dstq+dsq*0], xm1, 1 + pextrw [dstq+dsq*1], xm1, 0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xm0, [srcq+ssq*0] +.v_w4_loop: + vpbroadcastd xm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm1, xm2, xm0, 0x01 ; 0 1 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm2, xm0, 0x02 ; 1 2 + punpcklbw xm1, xm2 + pmaddubsw xm1, xm4 + pmulhrsw xm1, xm5 + packuswb xm1, xm1 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xm0, [srcq+ssq*0] +.v_w8_loop: + movq xm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw xm1, xm0, xm2 + movq xm0, [srcq+ssq*0] + punpcklbw xm2, xm0 + pmaddubsw xm1, xm4 + pmaddubsw xm2, xm4 + pmulhrsw xm1, xm5 + pmulhrsw xm2, xm5 + packuswb xm1, xm2 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + movu xm0, [srcq+ssq*0] +.v_w16_loop: + vbroadcasti128 m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m2, m3, m0, 0x0f ; 0 1 + vbroadcasti128 m0, [srcq+ssq*0] + vpblendd m3, m0, 0xf0 ; 1 2 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: +%macro PUT_BILIN_V_W32 0 + movu m0, [srcq+ssq*0] +%%loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m1, m0, m3 + punpckhbw m2, m0, m3 + movu m0, [srcq+ssq*0] + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + pmaddubsw m2, m4 + pmaddubsw m3, m4 + pmulhrsw m2, m5 + pmulhrsw m3, m5 + packuswb m2, m3 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg %%loop +%endmacro + PUT_BILIN_V_W32 + RET +.v_w64: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] +.v_w64_loop: + add srcq, ssq + movu m3, [srcq+32*0] + punpcklbw m2, m0, m3 + punpckhbw m0, m3 + pmaddubsw m2, m4 + pmaddubsw m0, m4 + pmulhrsw m2, m5 + pmulhrsw m0, m5 + packuswb m2, m0 + mova m0, m3 + movu m3, [srcq+32*1] + mova [dstq+32*0], m2 + punpcklbw m2, m1, m3 + punpckhbw m1, m3 + pmaddubsw m2, m4 + pmaddubsw m1, m4 + pmulhrsw m2, m5 + pmulhrsw m1, m5 + packuswb m2, m1 + mova m1, m3 + mova [dstq+32*1], m2 + add dstq, dsq + dec hd + jg .v_w64_loop + RET +.v_w128: + lea r6d, [hq+(3<<8)] + mov r4, srcq + mov r7, dstq +.v_w128_loop: + PUT_BILIN_V_W32 + add r4, 32 + add r7, 32 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .v_w128_loop + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 + ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 + movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 ; can't shift by 12 due to signed overflow + vpbroadcastd m7, [pw_15] + movd xm6, mxyd + add wq, r7 + paddb m5, m5 + vpbroadcastw m6, xm6 + jmp wq +.hv_w2: + vpbroadcastd xm0, [srcq+ssq*0] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 +.hv_w2_loop: + movd xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pinsrd xm1, [srcq+ssq*0], 1 + pshufb xm1, xm4 + pmaddubsw xm1, xm5 ; 1 _ 2 _ + shufps xm2, xm0, xm1, q1032 ; 0 _ 1 _ + mova xm0, xm1 + psubw xm1, xm2 + pmulhw xm1, xm6 + pavgw xm2, xm7 + paddw xm1, xm2 + psrlw xm1, 4 + packuswb xm1, xm1 + pextrw [dstq+dsq*0], xm1, 0 + pextrw [dstq+dsq*1], xm1, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova xm4, [bilin_h_shuf4] + movddup xm0, [srcq+ssq*0] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 +.hv_w4_loop: + movq xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xm1, [srcq+ssq*0] + pshufb xm1, xm4 + pmaddubsw xm1, xm5 ; 1 2 + shufps xm2, xm0, xm1, q1032 ; 0 1 + mova xm0, xm1 + psubw xm1, xm2 + pmulhw xm1, xm6 + pavgw xm2, xm7 + paddw xm1, xm2 + psrlw xm1, 4 + packuswb xm1, xm1 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti128 m0, [srcq+ssq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m1, [srcq+ssq*0], 1 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 + vperm2i128 m2, m0, m1, 0x21 ; 0 1 + mova m0, m1 + psubw m1, m2 + pmulhw m1, m6 + pavgw m2, m7 + paddw m1, m2 + psrlw m1, 4 + vextracti128 xm2, m1, 1 + packuswb xm1, xm2 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + movu m0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w16_loop: + movu xm2, [srcq+ssq*1+8*0] + vinserti128 m2, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + movu xm3, [srcq+ssq*0+8*0] + vinserti128 m3, [srcq+ssq*0+8*1], 1 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + psubw m1, m2, m0 + pmulhw m1, m6 + pavgw m0, m7 + paddw m1, m0 + pmaddubsw m0, m3, m5 + psubw m3, m0, m2 + pmulhw m3, m6 + pavgw m2, m7 + paddw m3, m2 + psrlw m1, 4 + psrlw m3, 4 + packuswb m1, m3 + vpermq m1, m1, q3120 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w128: + lea r6d, [hq+(3<<16)] + jmp .hv_w32_start +.hv_w64: + lea r6d, [hq+(1<<16)] +.hv_w32_start: + mov r4, srcq + mov r7, dstq +.hv_w32: +%if WIN64 + movaps r4m, xmm8 +%endif +.hv_w32_loop0: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w32_loop: + add srcq, ssq + movu m2, [srcq+8*0] + movu m3, [srcq+8*1] + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + psubw m8, m2, m0 + pmulhw m8, m6 + pavgw m0, m7 + paddw m8, m0 + mova m0, m2 + psubw m2, m3, m1 + pmulhw m2, m6 + pavgw m1, m7 + paddw m2, m1 + mova m1, m3 + psrlw m8, 4 + psrlw m2, 4 + packuswb m8, m2 + mova [dstq], m8 + add dstq, dsq + dec hd + jg .hv_w32_loop + add r4, 32 + add r7, 32 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<16 + jg .hv_w32_loop0 +%if WIN64 + movaps xmm8, r4m +%endif + RET + +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea r6, [prep%+SUFFIX] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [r6+wq*2+table_offset(prep,)] + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd xm0, [srcq+strideq*0] + pinsrd xm0, [srcq+strideq*1], 1 + pinsrd xm0, [srcq+strideq*2], 2 + pinsrd xm0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pmovzxbw m0, xm0 + psllw m0, 4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq xm0, [srcq+strideq*0] + movhps xm0, [srcq+strideq*1] + movq xm1, [srcq+strideq*2] + movhps xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmovzxbw m0, xm0 + pmovzxbw m1, xm1 + psllw m0, 4 + psllw m1, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + pmovzxbw m0, [srcq+strideq*0] + pmovzxbw m1, [srcq+strideq*1] + pmovzxbw m2, [srcq+strideq*2] + pmovzxbw m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmovzxbw m0, [srcq+strideq*0+16*0] + pmovzxbw m1, [srcq+strideq*0+16*1] + pmovzxbw m2, [srcq+strideq*1+16*0] + pmovzxbw m3, [srcq+strideq*1+16*1] + lea srcq, [srcq+strideq*2] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 2 + jg .prep_w32 + RET +.prep_w64: + pmovzxbw m0, [srcq+16*0] + pmovzxbw m1, [srcq+16*1] + pmovzxbw m2, [srcq+16*2] + pmovzxbw m3, [srcq+16*3] + add srcq, strideq + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + dec hd + jg .prep_w64 + RET +.prep_w128: + pmovzxbw m0, [srcq+16*0] + pmovzxbw m1, [srcq+16*1] + pmovzxbw m2, [srcq+16*2] + pmovzxbw m3, [srcq+16*3] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + pmovzxbw m0, [srcq+16*4] + pmovzxbw m1, [srcq+16*5] + pmovzxbw m2, [srcq+16*6] + pmovzxbw m3, [srcq+16*7] + add tmpq, 32*8 + add srcq, strideq + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq-32*4], m0 + mova [tmpq-32*3], m1 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + dec hd + jg .prep_w128 + RET +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] + imul mxyd, 255 + vbroadcasti128 m4, [bilin_h_shuf8] + add mxyd, 16 + movd xm5, mxyd + mov mxyd, r6m ; my + vpbroadcastw m5, xm5 + test mxyd, mxyd + jnz .hv + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)] + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + vbroadcasti128 m4, [bilin_h_shuf4] +.h_w4_loop: + movq xm0, [srcq+strideq*0] + movhps xm0, [srcq+strideq*1] + movq xm1, [srcq+strideq*2] + movhps xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 m0, xm1, 1 + pshufb m0, m4 + pmaddubsw m0, m5 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: +.h_w8_loop: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*2] + vinserti128 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: +.h_w16_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + movu xm2, [srcq+strideq*2+8*0] + vinserti128 m2, [srcq+strideq*2+8*1], 1 + movu xm3, [srcq+stride3q +8*0] + vinserti128 m3, [srcq+stride3q +8*1], 1 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: +.h_w32_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*0+8*2] + vinserti128 m1, [srcq+strideq*0+8*3], 1 + movu xm2, [srcq+strideq*1+8*0] + vinserti128 m2, [srcq+strideq*1+8*1], 1 + movu xm3, [srcq+strideq*1+8*2] + vinserti128 m3, [srcq+strideq*1+8*3], 1 + lea srcq, [srcq+strideq*2] + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 2 + jg .h_w32_loop + RET +.h_w64: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 + add srcq, strideq + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + dec hd + jg .h_w64 + RET +.h_w128: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + movu xm0, [srcq+8* 8] + vinserti128 m0, [srcq+8* 9], 1 + movu xm1, [srcq+8*10] + vinserti128 m1, [srcq+8*11], 1 + movu xm2, [srcq+8*12] + vinserti128 m2, [srcq+8*13], 1 + movu xm3, [srcq+8*14] + vinserti128 m3, [srcq+8*15], 1 + add tmpq, 32*8 + add srcq, strideq + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq-32*4], m0 + mova [tmpq-32*3], m1 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + dec hd + jg .h_w128 + RET +.v: + WIN64_SPILL_XMM 7 + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)] + imul mxyd, 255 + add mxyd, 16 + add wq, r6 + lea stride3q, [strideq*3] + movd xm6, mxyd + vpbroadcastw m6, xm6 + jmp wq +.v_w4: + movd xm0, [srcq+strideq*0] +.v_w4_loop: + vpbroadcastd m1, [srcq+strideq*2] + vpbroadcastd xm2, [srcq+strideq*1] + vpbroadcastd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0x05 ; 0 2 2 2 + vpbroadcastd m0, [srcq+strideq*0] + vpblendd m3, m2, 0x0f ; 1 1 3 3 + vpblendd m2, m1, m0, 0xa0 ; 0 2 2 4 + vpblendd m1, m3, 0xaa ; 0 1 2 3 + vpblendd m2, m3, 0x55 ; 1 2 3 4 + punpcklbw m1, m2 + pmaddubsw m1, m6 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq xm0, [srcq+strideq*0] +.v_w8_loop: + vpbroadcastq m1, [srcq+strideq*2] + vpbroadcastq m2, [srcq+strideq*1] + vpbroadcastq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0x03 ; 0 2 2 2 + vpbroadcastq m0, [srcq+strideq*0] + vpblendd m2, m3, 0xcc ; 1 3 1 3 + vpblendd m3, m2, m1, 0xf0 ; 1 3 2 2 + vpblendd m2, m1, 0x0f ; 0 2 1 3 + vpblendd m3, m0, 0xc0 ; 1 3 2 4 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + pmaddubsw m1, m6 + pmaddubsw m2, m6 + mova [tmpq+32*0], m1 + mova [tmpq+32*1], m2 + add tmpq, 32*2 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + vbroadcasti128 m0, [srcq+strideq*0] +.v_w16_loop: + vbroadcasti128 m1, [srcq+strideq*1] + vbroadcasti128 m2, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + shufpd m4, m0, m2, 0x0c ; 0 2 + vbroadcasti128 m0, [srcq+strideq*0] + shufpd m1, m3, 0x0c ; 1 3 + shufpd m2, m0, 0x0c ; 2 4 + punpcklbw m3, m4, m1 + punpcklbw m5, m1, m2 + punpckhbw m4, m1 + punpckhbw m1, m2 + pmaddubsw m3, m6 + pmaddubsw m5, m6 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + mova [tmpq+32*0], m3 + mova [tmpq+32*1], m5 + mova [tmpq+32*2], m4 + mova [tmpq+32*3], m1 + add tmpq, 32*4 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + vpermq m0, [srcq+strideq*0], q3120 +.v_w32_loop: + vpermq m1, [srcq+strideq*1], q3120 + vpermq m2, [srcq+strideq*2], q3120 + vpermq m3, [srcq+stride3q ], q3120 + lea srcq, [srcq+strideq*4] + punpcklbw m4, m0, m1 + punpckhbw m5, m0, m1 + vpermq m0, [srcq+strideq*0], q3120 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*0], m4 + mova [tmpq+32*1], m5 + punpcklbw m4, m1, m2 + punpckhbw m1, m2 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + punpcklbw m5, m2, m3 + punpckhbw m2, m3 + pmaddubsw m5, m6 + pmaddubsw m2, m6 + mova [tmpq+32*2], m4 + mova [tmpq+32*3], m1 + add tmpq, 32*8 + punpcklbw m1, m3, m0 + punpckhbw m3, m0 + pmaddubsw m1, m6 + pmaddubsw m3, m6 + mova [tmpq-32*4], m5 + mova [tmpq-32*3], m2 + mova [tmpq-32*2], m1 + mova [tmpq-32*1], m3 + sub hd, 4 + jg .v_w32_loop + RET +.v_w64: + vpermq m0, [srcq+strideq*0+32*0], q3120 + vpermq m1, [srcq+strideq*0+32*1], q3120 +.v_w64_loop: + vpermq m2, [srcq+strideq*1+32*0], q3120 + vpermq m3, [srcq+strideq*1+32*1], q3120 + lea srcq, [srcq+strideq*2] + punpcklbw m4, m0, m2 + punpckhbw m0, m2 + pmaddubsw m4, m6 + pmaddubsw m0, m6 + mova [tmpq+32*0], m4 + mova [tmpq+32*1], m0 + punpcklbw m4, m1, m3 + punpckhbw m5, m1, m3 + vpermq m0, [srcq+strideq*0+32*0], q3120 + vpermq m1, [srcq+strideq*0+32*1], q3120 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*2], m4 + mova [tmpq+32*3], m5 + add tmpq, 32*8 + punpcklbw m4, m2, m0 + punpckhbw m2, m0 + punpcklbw m5, m3, m1 + punpckhbw m3, m1 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + pmaddubsw m5, m6 + pmaddubsw m3, m6 + mova [tmpq-32*4], m4 + mova [tmpq-32*3], m2 + mova [tmpq-32*2], m5 + mova [tmpq-32*1], m3 + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + lea r6d, [hq+(3<<8)] + mov r3, srcq + mov r5, tmpq +.v_w128_loop0: + vpermq m0, [srcq+strideq*0], q3120 +.v_w128_loop: + vpermq m1, [srcq+strideq*1], q3120 + lea srcq, [srcq+strideq*2] + punpcklbw m2, m0, m1 + punpckhbw m3, m0, m1 + vpermq m0, [srcq+strideq*0], q3120 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + punpcklbw m4, m1, m0 + punpckhbw m1, m0 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + mova [tmpq+32*0], m2 + mova [tmpq+32*1], m3 + mova [tmpq+32*8], m4 + mova [tmpq+32*9], m1 + add tmpq, 32*16 + sub hd, 2 + jg .v_w128_loop + add r3, 32 + add r5, 64 + movzx hd, r6b + mov srcq, r3 + mov tmpq, r5 + sub r6d, 1<<8 + jg .v_w128_loop0 + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 7 + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + movd xm6, mxyd + vpbroadcastw m6, xm6 + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.hv_w4: + vbroadcasti128 m4, [bilin_h_shuf4] + vpbroadcastq m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w4_loop: + movq xm1, [srcq+strideq*1] + movhps xm1, [srcq+strideq*2] + movq xm2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movhps xm2, [srcq+strideq*0] + vinserti128 m1, xm2, 1 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 3 4 + vpblendd m2, m1, m0, 0xc0 + vpermq m2, m2, q2103 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti128 m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xm1, [srcq+strideq*1] + vinserti128 m1, [srcq+strideq*2], 1 + movu xm2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 m2, [srcq+strideq*0], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 ; 1 2 + vperm2i128 m3, m0, m1, 0x21 ; 0 1 + pmaddubsw m0, m2, m5 ; 3 4 + vperm2i128 m2, m1, m0, 0x21 ; 2 3 + psubw m1, m3 + pmulhrsw m1, m6 + paddw m1, m3 + psubw m3, m0, m2 + pmulhrsw m3, m6 + paddw m3, m2 + mova [tmpq+32*0], m1 + mova [tmpq+32*1], m3 + add tmpq, 32*2 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w16_loop: + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+32*0], m3 + mova [tmpq+32*1], m2 + add tmpq, 32*2 + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w32_loop: + add srcq, strideq + movu xm2, [srcq+8*0] + vinserti128 m2, [srcq+8*1], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m3, m2, m0 + pmulhrsw m3, m6 + paddw m3, m0 + mova m0, m2 + movu xm2, [srcq+8*2] + vinserti128 m2, [srcq+8*3], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + mova [tmpq+32*0], m3 + psubw m3, m2, m1 + pmulhrsw m3, m6 + paddw m3, m1 + mova m1, m2 + mova [tmpq+32*1], m3 + add tmpq, 32*2 + dec hd + jg .hv_w32_loop + RET +.hv_w128: + lea r3d, [hq+(7<<8)] + mov r6d, 256 + jmp .hv_w64_start +.hv_w64: + lea r3d, [hq+(3<<8)] + mov r6d, 128 +.hv_w64_start: +%if WIN64 + PUSH r7 +%endif + mov r5, srcq + mov r7, tmpq +.hv_w64_loop0: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w64_loop: + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+r6*0], m3 + mova [tmpq+r6*1], m2 + lea tmpq, [tmpq+r6*2] + sub hd, 2 + jg .hv_w64_loop + add r5, 16 + add r7, 32 + movzx hd, r3b + mov srcq, r5 + mov tmpq, r7 + sub r3d, 1<<8 + jg .hv_w64_loop0 +%if WIN64 + POP r7 +%endif + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; fn, type, type_h, type_v +cglobal %1_%2_8bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) +%endif +%endmacro + +%if WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +%define PUT_8TAP_FN FN put_8tap, +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR + +cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx2] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r8+wq*2+table_offset(put,)] + add wq, r8 + lea r6, [ssq*3] + lea r7, [dsq*3] +%if WIN64 + pop r8 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) + WIN64_SPILL_XMM 11 + cmp wd, 4 + jl .h_w2 + vbroadcasti128 m6, [subpel_h_shufA] + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m7, [subpel_h_shufB] + vbroadcasti128 m8, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] + vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0] + vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4] + add wq, r8 + jmp wq +.h_w2: + movzx mxd, mxb + dec srcq + mova xm4, [subpel_h_shuf4] + vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] +.h_w2_loop: + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm3 + phaddw xm0, xm0 + paddw xm0, xm5 + psraw xm0, 6 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] +.h_w4_loop: + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm6 + pshufb xm1, xm6 + pmaddubsw xm0, xm3 + pmaddubsw xm1, xm3 + phaddw xm0, xm1 + paddw xm0, xm5 + psraw xm0, 6 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: +%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] + pshufb m%2, m%1, m7 + pshufb m%3, m%1, m8 + pshufb m%1, m6 + pmaddubsw m%4, m%2, m9 + pmaddubsw m%2, m10 + pmaddubsw m%3, m10 + pmaddubsw m%1, m9 + paddw m%3, m%4 + paddw m%1, m%2 + phaddw m%1, m%3 + paddw m%1, m5 + psraw m%1, 6 +%endmacro + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 1, 2, 3 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*1+8*0], 1 + movu xm1, [srcq+ssq*0+8*1] + vinserti128 m1, [srcq+ssq*1+8*1], 1 + PUT_8TAP_H 0, 2, 3, 4 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + sub dstq, r6 + mov r4, r6 +.h_loop: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+r6], m0 + add r6, 32 + jle .h_loop + add srcq, ssq + add dstq, dsq + mov r6, r4 + dec hd + jg .h_loop + RET +.v: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + tzcnt r6d, wd + movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] + vpbroadcastd m7, [pw_512] + lea myq, [r8+myq*8+subpel_filters-put_avx2] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + add r6, r8 + lea ss3q, [ssq*3] + sub srcq, ss3q + jmp r6 +.v_w2: + movd xm2, [srcq+ssq*0] + pinsrw xm2, [srcq+ssq*1], 2 + pinsrw xm2, [srcq+ssq*2], 4 + add srcq, ss3q + pinsrw xm2, [srcq+ssq*0], 6 ; 0 1 2 3 + movd xm3, [srcq+ssq*1] + vpbroadcastd xm1, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm3, xm1, 0x02 ; 4 5 + vpblendd xm1, xm0, 0x02 ; 5 6 + palignr xm4, xm3, xm2, 4 ; 1 2 3 4 + punpcklbw xm3, xm1 ; 45 56 + punpcklbw xm1, xm2, xm4 ; 01 12 + punpckhbw xm2, xm4 ; 23 34 +.v_w2_loop: + pmaddubsw xm5, xm1, xm8 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm9 ; a1 b1 + paddw xm5, xm2 + mova xm2, xm3 + pmaddubsw xm3, xm10 ; a2 b2 + paddw xm5, xm3 + vpbroadcastd xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm3, xm0, xm4, 0x02 ; 6 7 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm4, xm0, 0x02 ; 7 8 + punpcklbw xm3, xm4 ; 67 78 + pmaddubsw xm4, xm3, xm11 ; a3 b3 + paddw xm5, xm4 + pmulhrsw xm5, xm7 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xm2, [srcq+ssq*0] + pinsrd xm2, [srcq+ssq*1], 1 + pinsrd xm2, [srcq+ssq*2], 2 + add srcq, ss3q + pinsrd xm2, [srcq+ssq*0], 3 ; 0 1 2 3 + movd xm3, [srcq+ssq*1] + vpbroadcastd xm1, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm3, xm1, 0x02 ; 4 5 + vpblendd xm1, xm0, 0x02 ; 5 6 + palignr xm4, xm3, xm2, 4 ; 1 2 3 4 + punpcklbw xm3, xm1 ; 45 56 + punpcklbw xm1, xm2, xm4 ; 01 12 + punpckhbw xm2, xm4 ; 23 34 +.v_w4_loop: + pmaddubsw xm5, xm1, xm8 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm9 ; a1 b1 + paddw xm5, xm2 + mova xm2, xm3 + pmaddubsw xm3, xm10 ; a2 b2 + paddw xm5, xm3 + vpbroadcastd xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm3, xm0, xm4, 0x02 ; 6 7 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm4, xm0, 0x02 ; 7 8 + punpcklbw xm3, xm4 ; 67 78 + pmaddubsw xm4, xm3, xm11 ; a3 b3 + paddw xm5, xm4 + pmulhrsw xm5, xm7 + packuswb xm5, xm5 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m2, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m5, [srcq+ssq*0] + vpbroadcastq m3, [srcq+ssq*1] + vpbroadcastq m6, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m1, m4, 0x30 + vpblendd m4, m2, 0x30 + punpcklbw m1, m4 ; 01 12 + vpblendd m2, m5, 0x30 + vpblendd m5, m3, 0x30 + punpcklbw m2, m5 ; 23 34 + vpblendd m3, m6, 0x30 + vpblendd m6, m0, 0x30 + punpcklbw m3, m6 ; 45 56 +.v_w8_loop: + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m5, m1, m8 ; a0 b0 + mova m1, m2 + pmaddubsw m2, m9 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, m10 ; a2 b2 + paddw m5, m3 + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+ssq*0] + vpblendd m4, m0, 0x30 + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, m11 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + vextracti128 xm4, m5, 1 + packuswb xm5, xm4 + movq [dstq+dsq*0], xm5 + movhps [dstq+dsq*1], xm5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: +.v_w32: +.v_w64: +.v_w128: + lea r6d, [wq*8-128] + mov r4, srcq + mov r7, dstq + lea r6d, [hq+r6*2] +.v_w16_loop0: + vbroadcasti128 m4, [srcq+ssq*0] + vbroadcasti128 m5, [srcq+ssq*1] + vbroadcasti128 m6, [srcq+ssq*2] + add srcq, ss3q + vbroadcasti128 m0, [srcq+ssq*0] + vbroadcasti128 m1, [srcq+ssq*1] + vbroadcasti128 m2, [srcq+ssq*2] + add srcq, ss3q + vbroadcasti128 m3, [srcq+ssq*0] + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w16_loop: + vbroadcasti128 m12, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti128 m13, [srcq+ssq*0] + pmaddubsw m14, m1, m8 ; a0 + pmaddubsw m15, m2, m8 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, m9 ; a1 + pmaddubsw m4, m9 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, m10 ; a2 + pmaddubsw m6, m10 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, m11 ; a3 + pmaddubsw m13, m6, m11 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + packuswb m14, m15 + vpermq m14, m14, q3120 + mova [dstq+dsq*0], xm14 + vextracti128 [dstq+dsq*1], m14, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + add r4, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + dec srcq + vpbroadcastd m7, [r8+mxq*8+subpel_filters-put_avx2+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] + lea ss3q, [ssq*3] + sub srcq, ss3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m8, [pw_8192] + vpbroadcastd m9, [pd_512] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 m6, [subpel_h_shuf4] + movq xm2, [srcq+ssq*0] + movhps xm2, [srcq+ssq*1] + movq xm0, [srcq+ssq*2] + add srcq, ss3q + movhps xm0, [srcq+ssq*0] + vpbroadcastq m3, [srcq+ssq*1] + vpbroadcastq m4, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m1, [srcq+ssq*0] + vpblendd m2, m3, 0x30 + vpblendd m0, m1, 0x30 + vpblendd m2, m4, 0xc0 + pshufb m2, m6 + pshufb m0, m6 + pmaddubsw m2, m7 + pmaddubsw m0, m7 + phaddw m2, m0 + pmulhrsw m2, m8 + vextracti128 xm3, m2, 1 + palignr xm4, xm3, xm2, 4 + punpcklwd xm1, xm2, xm4 ; 01 12 + punpckhwd xm2, xm4 ; 23 34 + pshufd xm0, xm3, q2121 + punpcklwd xm3, xm0 ; 45 56 +.hv_w2_loop: + movq xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xm4, [srcq+ssq*0] + pshufb xm4, xm6 + pmaddubsw xm4, xm7 + pmaddwd xm5, xm1, xm10 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm11 ; a1 b1 + paddd xm5, xm2 + mova xm2, xm3 + pmaddwd xm3, xm12 ; a2 b2 + phaddw xm4, xm4 + pmulhrsw xm4, xm8 + paddd xm5, xm3 + palignr xm3, xm4, xm0, 12 + mova xm0, xm4 + punpcklwd xm3, xm0 ; 67 78 + pmaddwd xm4, xm3, xm13 ; a3 b3 + paddd xm5, xm9 + paddd xm5, xm4 + psrad xm5, 10 + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova m6, [subpel_h_shuf4] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m0, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m5, [srcq+ssq*0] + vpbroadcastq m3, [srcq+ssq*1] + vpblendd m2, m4, 0xcc ; 0 1 + vpbroadcastq m4, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq m1, [srcq+ssq*0] + vpblendd m0, m5, 0xcc ; 2 3 + vpblendd m3, m4, 0xcc ; 4 5 + pshufb m2, m6 + pshufb m0, m6 + pshufb m3, m6 + pshufb m1, m6 + pmaddubsw m2, m7 + pmaddubsw m0, m7 + pmaddubsw m3, m7 + pmaddubsw m1, m7 + phaddw m2, m0 + phaddw m3, m1 + pmulhrsw m2, m8 + pmulhrsw m3, m8 + palignr m4, m3, m2, 4 + punpcklwd m1, m2, m4 ; 01 12 + punpckhwd m2, m4 ; 23 34 + pshufd m0, m3, q2121 + punpcklwd m3, m0 ; 45 56 +.hv_w4_loop: + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m5, m1, m10 ; a0 b0 + mova m1, m2 + pmaddwd m2, m11 ; a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m12 ; a2 b2 + paddd m5, m3 + vpbroadcastq m3, [srcq+ssq*0] + vpblendd m4, m3, 0xcc ; 7 8 + pshufb m4, m6 + pmaddubsw m4, m7 + phaddw m4, m4 + pmulhrsw m4, m8 + palignr m3, m4, m0, 12 + mova m0, m4 + punpcklwd m3, m0 ; 67 78 + pmaddwd m4, m3, m13 ; a3 b3 + paddd m5, m9 + paddd m5, m4 + psrad m5, 10 + vextracti128 xm4, m5, 1 + packssdw xm5, xm4 + packuswb xm5, xm5 + pshuflw xm5, xm5, q3120 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0] + vpbroadcastd m11, [r8+mxq*8+subpel_filters-put_avx2+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] + lea ss3q, [ssq*3] + sub srcq, ss3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + lea r6d, [wq*8-64] + mov r4, srcq + mov r7, dstq + lea r6d, [hq+r6*4] +.hv_w8_loop0: + vbroadcasti128 m7, [subpel_h_shufA] + movu xm4, [srcq+ssq*0] + vbroadcasti128 m8, [subpel_h_shufB] + movu xm5, [srcq+ssq*1] + vbroadcasti128 m9, [subpel_h_shufC] + movu xm6, [srcq+ssq*2] + add srcq, ss3q + vbroadcasti128 m0, [srcq+ssq*0] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [srcq+ssq*1], 1 ; 1 4 + vinserti128 m6, [srcq+ssq*2], 1 ; 2 5 + add srcq, ss3q + vinserti128 m0, [srcq+ssq*0], 1 ; 3 6 +%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] + pshufb %3, %1, %6 + pshufb %4, %1, %7 + pshufb %1, %5 + pmaddubsw %2, %3, m10 + pmaddubsw %4, m11 + pmaddubsw %3, m11 + pmaddubsw %1, m10 + paddw %2, %4 + paddw %1, %3 + phaddw %1, %2 +%endmacro + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + vpbroadcastd m7, [pw_8192] + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + pmulhrsw m0, m7 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + vpermq m7, m0, q3120 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vextracti128 r6m, m0, 1 ; not enough registers + movu xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m0, [srcq+ssq*0], 1 ; 7 8 + pmaddwd m8, m1, m12 ; a0 + pmaddwd m9, m2, m12 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m13 ; a1 + pmaddwd m4, m13 ; b1 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m14 ; a2 + pmaddwd m6, m14 ; b2 + paddd m8, m5 + paddd m9, m6 + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + vbroadcasti128 m5, [subpel_h_shufA] + HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + vpbroadcastd m5, [pw_8192] + vpbroadcastd m7, [pd_512] + vbroadcasti128 m6, r6m + pmulhrsw m0, m5 + paddd m8, m7 + paddd m9, m7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, m15 ; a3 + paddd m8, m7 + pmaddwd m7, m6, m15 ; b3 + paddd m7, m9 + psrad m8, 10 + psrad m7, 10 + packssdw m8, m7 + vextracti128 xm7, m8, 1 + packuswb xm8, xm7 + pshufd xm7, xm8, q3120 + movq [dstq+dsq*0], xm7 + movhps [dstq+dsq*1], xm7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + add r4, 8 + add r7, 8 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +%macro PREP_8TAP_H 0 + pshufb m1, m0, m5 + pshufb m2, m0, m6 + pshufb m3, m0, m7 + pmaddubsw m1, m8 + pmaddubsw m0, m2, m8 + pmaddubsw m2, m9 + pmaddubsw m3, m9 + paddw m1, m2 + paddw m0, m3 + phaddw m0, m1, m0 + pmulhrsw m0, m4 +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep%+SUFFIX] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep,)] + add wq, r7 + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pw_8192] + vbroadcasti128 m5, [subpel_h_shufA] + WIN64_SPILL_XMM 10 + cmp wd, 4 + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] + add wq, r7 + jmp wq +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + lea stride3q, [strideq*3] +.h_w4_loop: + movq xm0, [srcq+strideq*0] + vpbroadcastq m2, [srcq+strideq*2] + movq xm1, [srcq+strideq*1] + vpblendd m0, m2, 0xf0 + vpbroadcastq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m2, 0xf0 + pshufb m0, m5 + pshufb m1, m5 + pmaddubsw m0, m6 + pmaddubsw m1, m6 + phaddw m0, m1 + pmulhrsw m0, m4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+strideq*1+8*0] + vinserti128 m0, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + mov r5, r6 +.h_loop: + movu xm0, [srcq+r6+8*0] + vinserti128 m0, [srcq+r6+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+r6+8*2] + vinserti128 m0, [srcq+r6+8*3], 1 + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + add r6, 32 + jle .h_loop + add srcq, strideq + mov r6, r5 + dec hd + jg .h_loop + RET +.v: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. + shr myd, 16 ; Note that the code is 8-tap only, having + cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 + cmove myd, mxd ; had a negligible effect on performance. + ; TODO: Would a 6-tap code path be worth it? + lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + vpbroadcastd m7, [pw_8192] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + cmp wd, 8 + jg .v_w16 + je .v_w8 +.v_w4: + movd xm0, [srcq+strideq*0] + vpbroadcastd m1, [srcq+strideq*2] + vpbroadcastd xm2, [srcq+strideq*1] + add srcq, stride3q + vpbroadcastd m3, [srcq+strideq*0] + vpblendd m1, m0, 0x01 ; 0 2 2 _ 2 _ _ _ + vpblendd m3, m2, 0x03 ; 1 1 3 3 3 3 _ _ + vpbroadcastd m0, [srcq+strideq*1] + vpbroadcastd m2, [srcq+strideq*2] + vpblendd m1, m0, 0x68 ; 0 2 2 4 2 4 4 _ + vpbroadcastd m0, [srcq+stride3q ] + vbroadcasti128 m5, [deint_shuf4] + vpblendd m3, m2, 0xc0 ; 1 1 3 3 3 3 5 5 + vpblendd m2, m3, m1, 0x55 ; 0 1 2 3 2 3 4 5 + vpblendd m3, m1, 0xaa ; 1 2 3 4 3 4 5 _ + punpcklbw m1, m2, m3 ; 01 12 23 34 + vpblendd m3, m0, 0x80 ; 1 2 3 4 3 4 5 6 + punpckhbw m2, m3 ; 23 34 45 56 +.v_w4_loop: + lea srcq, [srcq+strideq*4] + pinsrd xm0, [srcq+strideq*0], 1 + vpbroadcastd m3, [srcq+strideq*1] + vpbroadcastd m4, [srcq+strideq*2] + vpblendd m3, m0, 0x03 ; 6 7 8 _ 8 _ _ _ + vpbroadcastd m0, [srcq+stride3q ] + vpblendd m3, m4, 0x20 ; 6 7 8 _ 8 9 _ _ + vpblendd m3, m0, 0x40 ; 6 7 8 _ 8 9 a _ + pshufb m3, m5 ; 67 78 89 9a + pmaddubsw m4, m1, m8 + vperm2i128 m1, m2, m3, 0x21 ; 45 56 67 78 + pmaddubsw m2, m9 + paddw m4, m2 + mova m2, m3 + pmaddubsw m3, m11 + paddw m3, m4 + pmaddubsw m4, m1, m10 + paddw m3, m4 + pmulhrsw m3, m7 + mova [tmpq], m3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+strideq*0] + vpbroadcastq m4, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m5, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m6, [srcq+strideq*1] + vpbroadcastq m0, [srcq+strideq*2] + vpblendd m1, m4, 0x30 + vpblendd m4, m2, 0x30 + punpcklbw m1, m4 ; 01 12 + vpblendd m2, m5, 0x30 + vpblendd m5, m3, 0x30 + punpcklbw m2, m5 ; 23 34 + vpblendd m3, m6, 0x30 + vpblendd m6, m0, 0x30 + punpcklbw m3, m6 ; 45 56 +.v_w8_loop: + vpbroadcastq m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmaddubsw m5, m2, m9 ; a1 + pmaddubsw m6, m2, m8 ; b0 + vpblendd m2, m0, m4, 0x30 + vpbroadcastq m0, [srcq+strideq*0] + vpblendd m4, m0, 0x30 + punpcklbw m2, m4 ; 67 78 + pmaddubsw m1, m8 ; a0 + pmaddubsw m4, m3, m9 ; b1 + paddw m5, m1 + mova m1, m3 + pmaddubsw m3, m10 ; a2 + paddw m6, m4 + paddw m5, m3 + vpbroadcastq m4, [srcq+strideq*1] + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+strideq*2] + vpblendd m4, m0, 0x30 + punpcklbw m3, m4 ; 89 9a + pmaddubsw m4, m2, m11 ; a3 + paddw m5, m4 + pmaddubsw m4, m2, m10 ; b2 + paddw m6, m4 + pmaddubsw m4, m3, m11 ; b3 + paddw m6, m4 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + mova [tmpq+32*0], m5 + mova [tmpq+32*1], m6 + add tmpq, 32*2 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + add wd, wd + mov r5, srcq + mov r7, tmpq + lea r6d, [hq+wq*8-256] +.v_w16_loop0: + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m0, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*0] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m1, [srcq+strideq*0] + vbroadcasti128 m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+strideq*0] + shufpd m4, m4, m0, 0x0c + shufpd m5, m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w16_loop: + vbroadcasti128 m12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m13, [srcq+strideq*0] + pmaddubsw m14, m1, m8 ; a0 + pmaddubsw m15, m2, m8 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, m9 ; a1 + pmaddubsw m4, m9 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, m10 ; a2 + pmaddubsw m6, m10 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, m11 ; a3 + pmaddubsw m13, m6, m11 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + mova [tmpq+wq*0], m14 + mova [tmpq+wq*1], m15 + lea tmpq, [tmpq+wq*2] + sub hd, 2 + jg .v_w16_loop + add r5, 16 + add r7, 32 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + %assign stack_size_padded 0 + WIN64_SPILL_XMM 16 + cmp wd, 4 + je .hv_w4 + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + jmp .hv_w8 +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + mova m7, [subpel_h_shuf4] + pmovzxbd m9, [deint_shuf4] + vpbroadcastd m10, [pw_8192] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m11, [pd_32] + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + vpbroadcastq m2, [srcq+strideq*0] + vpbroadcastq m4, [srcq+strideq*1] + vpbroadcastq m0, [srcq+strideq*2] + vpbroadcastq m5, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m6, [srcq+strideq*1] + vpbroadcastq m1, [srcq+strideq*2] + vpblendd m2, m4, 0xcc ; 0 1 + vpblendd m0, m5, 0xcc ; 2 3 + vpblendd m3, m6, 0xcc ; 4 5 + pshufb m2, m7 ; 00 01 10 11 02 03 12 13 + pshufb m0, m7 ; 20 21 30 31 22 23 32 33 + pshufb m3, m7 ; 40 41 50 51 42 43 52 53 + pshufb m1, m7 ; 60 61 60 61 62 63 62 63 + pmaddubsw m2, m8 + pmaddubsw m0, m8 + pmaddubsw m3, m8 + pmaddubsw m1, m8 + phaddw m2, m0 ; 0a 1a 2a 3a 0b 1b 2b 3b + phaddw m3, m1 ; 4a 5a 6a __ 4b 5b 6b __ + pmulhrsw m2, m10 + pmulhrsw m3, m10 + palignr m4, m3, m2, 4 ; 1a 2a 3a 4a 1b 2b 3b 4b + punpcklwd m1, m2, m4 ; 01 12 + punpckhwd m2, m4 ; 23 34 + pshufd m0, m3, q2121 + punpcklwd m3, m0 ; 45 56 +.hv_w4_loop: + pmaddwd m5, m1, m12 ; a0 b0 + pmaddwd m6, m2, m12 ; c0 d0 + pmaddwd m2, m13 ; a1 b1 + pmaddwd m4, m3, m13 ; c1 d1 + mova m1, m3 + pmaddwd m3, m14 ; a2 b2 + paddd m5, m2 + vpbroadcastq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + paddd m6, m4 + vpbroadcastq m4, [srcq+strideq*0] + paddd m5, m3 + vpbroadcastq m3, [srcq+strideq*1] + vpblendd m2, m4, 0xcc + vpbroadcastq m4, [srcq+strideq*2] + vpblendd m3, m4, 0xcc + pshufb m2, m7 + pshufb m3, m7 + pmaddubsw m2, m8 + pmaddubsw m3, m8 + phaddw m2, m3 + pmulhrsw m2, m10 + palignr m3, m2, m0, 12 + mova m0, m2 + punpcklwd m2, m3, m0 ; 67 78 + punpckhwd m3, m0 ; 89 9a + pmaddwd m4, m2, m14 ; c2 d2 + paddd m6, m11 + paddd m5, m11 + paddd m6, m4 + pmaddwd m4, m2, m15 ; a3 b3 + paddd m5, m4 + pmaddwd m4, m3, m15 ; c3 d3 + paddd m6, m4 + psrad m5, 6 + psrad m6, 6 + packssdw m5, m6 + vpermd m5, m9, m5 + mova [tmpq], m5 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + lea r6d, [wq*8-64] + mov r5, srcq + mov r7, tmpq + lea r6d, [hq+r6*4] +.hv_w8_loop0: + vbroadcasti128 m7, [subpel_h_shufA] + movu xm4, [srcq+strideq*0] + vbroadcasti128 m8, [subpel_h_shufB] + movu xm5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m9, [subpel_h_shufC] + movu xm6, [srcq+strideq*0] + vbroadcasti128 m0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [srcq+strideq*0], 1 ; 1 4 + vinserti128 m6, [srcq+strideq*1], 1 ; 2 5 + lea srcq, [srcq+strideq*2] + vinserti128 m0, [srcq+strideq*0], 1 ; 3 6 + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + vpbroadcastd m7, [pw_8192] + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + pmulhrsw m0, m7 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + vpermq m7, m0, q3120 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vextracti128 [tmpq], m0, 1 ; not enough registers + movu xm0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti128 m0, [srcq+strideq*0], 1 ; 7 8 + pmaddwd m8, m1, m12 ; a0 + pmaddwd m9, m2, m12 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m13 ; a1 + pmaddwd m4, m13 ; b1 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m14 ; a2 + pmaddwd m6, m14 ; b2 + paddd m8, m5 + paddd m9, m6 + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + vbroadcasti128 m5, [subpel_h_shufA] + HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + vpbroadcastd m5, [pw_8192] + vpbroadcastd m7, [pd_32] + vbroadcasti128 m6, [tmpq] + pmulhrsw m0, m5 + paddd m8, m7 + paddd m9, m7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, m15 ; a3 + paddd m8, m7 + pmaddwd m7, m6, m15 ; b3 + paddd m7, m9 + psrad m8, 6 + psrad m7, 6 + packssdw m8, m7 + vpermq m7, m8, q3120 + mova [tmpq+wq*0], xm7 + vextracti128 [tmpq+wq*2], m7, 1 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .hv_w8_loop + add r5, 8 + add r7, 16 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro REMAP_REG 2 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %xdefine r14_save r14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + %xdefine r14 r14_save + %undef r14_save + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6] + movq xm%1, [srcq+ r4] + movq xm%2, [srcq+ r6] + movhps xm%1, [srcq+ r7] + movhps xm%2, [srcq+ r9] + vinserti128 m%1, [srcq+r10], 1 + vinserti128 m%2, [srcq+r11], 1 + vpbroadcastq m%5, [srcq+r13] + vpbroadcastq m%6, [srcq+ rX] + add srcq, ssq + movq xm%3, [srcq+ r4] + movq xm%4, [srcq+ r6] + movhps xm%3, [srcq+ r7] + movhps xm%4, [srcq+ r9] + vinserti128 m%3, [srcq+r10], 1 + vinserti128 m%4, [srcq+r11], 1 + vpbroadcastq m%7, [srcq+r13] + vpbroadcastq m%8, [srcq+ rX] + add srcq, ssq + vpblendd m%1, m%5, 0xc0 + vpblendd m%2, m%6, 0xc0 + vpblendd m%3, m%7, 0xc0 + vpblendd m%4, m%8, 0xc0 + pmaddubsw m%1, m15 + pmaddubsw m%2, m10 + pmaddubsw m%3, m15 + pmaddubsw m%4, m10 + phaddw m%1, m%2 + phaddw m%3, m%4 + phaddw m%1, m%3 + pmulhrsw m%1, m12 +%endmacro + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isprep 0 +cglobal put_8tap_scaled_8bpc, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy + %xdefine base_reg r12 + %define rndshift 10 +%else + %assign isprep 1 +cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy + %define tmp_stridem qword [rsp+120] + %xdefine base_reg r11 + %define rndshift 6 +%endif + lea base_reg, [%1_8tap_scaled_8bpc_avx2] +%define base base_reg-%1_8tap_scaled_8bpc_avx2 + tzcnt wd, wm + vpbroadcastd m8, dxm +%if isprep && UNIX64 + movd xm14, mxd + vpbroadcastd m14, xm14 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 +%else + vpbroadcastd m14, mxm +%endif + mov dyd, dym +%ifidn %1, put + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %else + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %endif + %define dsm [rsp+112] + %define rX r1 + %define rXd r1d +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %else + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %define hm [rsp+112] + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define rX r14 + %define rXd r14d +%endif + vpbroadcastd m10, [base+pd_0x3ff] + vpbroadcastd m12, [base+pw_8192] +%ifidn %1, put + vpbroadcastd m13, [base+pd_512] +%else + vpbroadcastd m13, [base+pd_32] +%endif + pxor m9, m9 + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0,1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*2] + movhps xm0, [srcq+ssq*1] + movhps xm1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*2], 1 + vpbroadcastq m2, [srcq+ssq*1] + vpbroadcastq m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpblendd m15, m7, 0xaa + vpblendd m0, m2, 0xc0 ; 0 1 4 5 + vpblendd m1, m3, 0xc0 ; 2 3 6 7 + pblendvb m15, m11, m8 + pshufb m0, m14 + pshufb m1, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + phaddw m0, m1 + pmulhrsw m0, m12 ; 0 1 2 3 4 5 6 7 + vextracti128 xm1, m0, 1 ; 4 5 6 7 + palignr xm2, xm1, xm0, 4 ; 1 2 3 4 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + pshufd xm4, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm4 ; 45 56 + punpckhwd xm4, xm1, xm4 ; 67 __ +.w2_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm11, r6q + pmovsxbw xm11, xm11 + pshufd xm8, xm11, q0000 + pshufd xm9, xm11, q1111 + pshufd xm10, xm11, q2222 + pshufd xm11, xm11, q3333 + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pmaddwd xm7, xm2, xm10 + pmaddwd xm8, xm4, xm11 + paddd xm5, xm6 + paddd xm7, xm8 + paddd xm5, xm13 + paddd xm5, xm7 + psrad xm5, 10 + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq], xm5, 0 + add dstq, dsq + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w2_loop + movq xm5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps xm3, xm0, q1032 ; 01 12 + shufps xm0, xm2, q1032 ; 23 34 + shufps xm2, xm4, q1032 ; 45 56 + pshufb xm5, xm14 + pmaddubsw xm5, xm15 + phaddw xm5, xm5 + pmulhrsw xm5, xm12 + palignr xm1, xm5, xm1, 12 + punpcklqdq xm1, xm1 ; 6 7 6 7 + punpcklwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +.w2_skip_line: + movhps xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova xm3, xm0 ; 01 12 + mova xm0, xm2 ; 23 34 + pshufb xm5, xm14 + pmaddubsw xm5, xm15 + phaddw xm5, xm5 + pmulhrsw xm5, xm12 ; 6 7 6 7 + palignr xm1, xm5, xm1, 8 ; 4 5 6 7 + pshufd xm5, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm5 ; 45 56 + punpckhwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +%endif +.w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd xm15, xm0 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pinsrd xm15, [base+subpel_filters+r6*8+2], 1 + pcmpeqd m0, m9 + psrld m14, 10 + movu xm7, [srcq+ssq*0] + movu xm9, [srcq+ssq*1] + pinsrd xm15, [base+subpel_filters+r11*8+2], 2 + movu xm8, [srcq+ssq*2] + movu xm10, [srcq+ss3q ] + pinsrd xm15, [base+subpel_filters+r13*8+2], 3 + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m7, [srcq+ssq*0], 1 + vinserti128 m9, [srcq+ssq*1], 1 + vinserti128 m15, xm15, 1 + vinserti128 m8, [srcq+ssq*2], 1 + vinserti128 m10, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + pblendvb m15, m11, m0 + pshufb m7, m14 + pshufb m9, m14 + pshufb m8, m14 + pshufb m10, m14 + pmaddubsw m7, m15 + pmaddubsw m9, m15 + pmaddubsw m8, m15 + pmaddubsw m10, m15 + phaddw m7, m9 + phaddw m8, m10 + pmulhrsw m7, m12 ; 0 1 4 5 + pmulhrsw m8, m12 ; 2 3 6 7 + vextracti128 xm9, m7, 1 ; 4 5 + vextracti128 xm3, m8, 1 ; 6 7 + shufps xm4, xm7, xm8, q1032 ; 1 2 + shufps xm5, xm8, xm9, q1032 ; 3 4 + shufps xm6, xm9, xm3, q1032 ; 5 6 + psrldq xm11, xm3, 8 ; 7 _ + punpcklwd xm0, xm7, xm4 ; 01 + punpckhwd xm7, xm4 ; 12 + punpcklwd xm1, xm8, xm5 ; 23 + punpckhwd xm8, xm5 ; 34 + punpcklwd xm2, xm9, xm6 ; 45 + punpckhwd xm9, xm6 ; 56 + punpcklwd xm3, xm11 ; 67 + mova [rsp+0x00], xm7 + mova [rsp+0x10], xm8 + mova [rsp+0x20], xm9 +.w4_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm10, r6q + pmovsxbw xm10, xm10 + pshufd xm7, xm10, q0000 + pshufd xm8, xm10, q1111 + pshufd xm9, xm10, q2222 + pshufd xm10, xm10, q3333 + pmaddwd xm4, xm0, xm7 + pmaddwd xm5, xm1, xm8 + pmaddwd xm6, xm2, xm9 + pmaddwd xm7, xm3, xm10 + paddd xm4, xm5 + paddd xm6, xm7 + paddd xm4, xm13 + paddd xm4, xm6 + psrad xm4, rndshift + packssdw xm4, xm4 +%ifidn %1, put + packuswb xm4, xm4 + movd [dstq], xm4 + add dstq, dsq +%else + movq [tmpq], xm4 + add tmpq, 8 +%endif + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w4_loop + movu xm4, [srcq] + test myd, 0x400 + jz .w4_skip_line + mova xm0, [rsp+0x00] + mova [rsp+0x00], xm1 + mova xm1, [rsp+0x10] + mova [rsp+0x10], xm2 + mova xm2, [rsp+0x20] + mova [rsp+0x20], xm3 + pshufb xm4, xm14 + pmaddubsw xm4, xm15 + phaddw xm4, xm4 + pmulhrsw xm4, xm12 + punpcklwd xm3, xm11, xm4 + mova xm11, xm4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu xm5, [srcq+ssq*1] + movu m6, [rsp+0x10] + pshufb xm4, xm14 + pshufb xm5, xm14 + pmaddubsw xm4, xm15 + pmaddubsw xm5, xm15 + movu [rsp+0x00], m6 + phaddw xm4, xm5 + pmulhrsw xm4, xm12 + punpcklwd xm9, xm11, xm4 + mova [rsp+0x20], xm9 + psrldq xm11, xm4, 8 + mova xm0, xm1 + mova xm1, xm2 + mova xm2, xm3 + punpcklwd xm3, xm4, xm11 + lea srcq, [srcq+ssq*2] + jmp .w4_loop +.w8: + mov dword [rsp+48], 1 + movifprep tmp_stridem, 16 + jmp .w_start +.w16: + mov dword [rsp+48], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [rsp+48], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [rsp+48], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [rsp+48], 16 + movifprep tmp_stridem, 256 +.w_start: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+72], t0d + mov [rsp+56], srcq + mov [rsp+64], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + jmp .hloop +.hloop_prep: + dec dword [rsp+48] + jz .ret + add qword [rsp+64], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp+16] + vpbroadcastd m15, [rsp+72] + pxor m9, m9 + mov srcq, [rsp+56] + mov r0q, [rsp+64] ; dstq / tmpq +.hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp+16], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + mova [rsp], xm14 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + mov dyd, dym + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + vbroadcasti128 m14, [base+wswap] +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm11, r6q + punpcklqdq xm11, xm11 + pmovsxbw m11, xm11 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pshufd m8, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m6, m2, m8 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [rsp+52], myd + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + jz .skip_line + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + add srcq, ssq + mov myd, [rsp+52] + mov dyd, dym + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .vloop +.skip_line: + mova m0, m1 + mova m1, m2 + mova m2, m3 + vpbroadcastq m7, [srcq+r13] + vpbroadcastq m8, [srcq+ rX] + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + add srcq, ssq + movq xm5, [srcq+ r4] + movq xm6, [srcq+ r6] + movhps xm5, [srcq+ r7] + movhps xm6, [srcq+ r9] + vinserti128 m5, [srcq+r10], 1 + vinserti128 m6, [srcq+r11], 1 + vpbroadcastq m9, [srcq+r13] + vpbroadcastq m11, [srcq+ rX] + add srcq, ssq + mov myd, [rsp+52] + mov dyd, dym + vpblendd m3, m7, 0xc0 + vpblendd m4, m8, 0xc0 + vpblendd m5, m9, 0xc0 + vpblendd m6, m11, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + pmaddubsw m5, m15 + pmaddubsw m6, m10 + phaddw m3, m4 + phaddw m5, m6 + psrld m4, m3, 16 + pslld m6, m5, 16 + paddw m3, m4 + paddw m5, m6 + pblendw m3, m5, 0xaa + pmulhrsw m3, m12 + jmp .vloop +.dy1: + movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy1_w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0-1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*2] + movhps xm0, [srcq+ssq*1] + movhps xm1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*2], 1 + vpbroadcastq m2, [srcq+ssq*1] + add srcq, ss3q + movq xm10, r4q + pmovsxbw xm10, xm10 + vpblendd m15, m7, 0xaa + pblendvb m15, m11, m8 + pshufd xm8, xm10, q0000 + pshufd xm9, xm10, q1111 + pshufd xm11, xm10, q3333 + pshufd xm10, xm10, q2222 + vpblendd m0, m2, 0xc0 + pshufb m1, m14 + pshufb m0, m14 + pmaddubsw m1, m15 + pmaddubsw m0, m15 + phaddw m0, m1 + pmulhrsw m0, m12 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 + pshufd xm4, xm1, q2121 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + punpcklwd xm2, xm1, xm4 ; 45 56 +.dy1_w2_loop: + movq xm1, [srcq+ssq*0] + movhps xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pmaddwd xm7, xm2, xm10 + mova xm3, xm0 + mova xm0, xm2 + paddd xm5, xm13 + paddd xm6, xm7 + pshufb xm1, xm14 + pmaddubsw xm1, xm15 + phaddw xm1, xm1 + pmulhrsw xm1, xm12 + palignr xm7, xm1, xm4, 12 + punpcklwd xm2, xm7, xm1 ; 67 78 + pmaddwd xm7, xm2, xm11 + mova xm4, xm1 + paddd xm5, xm6 + paddd xm5, xm7 + psrad xm5, rndshift + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +.dy1_w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + vpermq m8, m8, q3120 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r11d, xm15, 1 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + movu xm2, [srcq+ssq*0] + movu xm3, [srcq+ssq*2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pcmpeqd m8, m9 + psrld m14, 10 + pinsrd xm15, [base+subpel_filters+r11*8+2], 1 + vpblendd m7, [base+subpel_filters+r13*8+2-20], 0x20 + vinserti128 m2, [srcq+ssq*1], 1 + vinserti128 m3, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + movu xm4, [srcq+ssq*0] + movu xm5, [srcq+ssq*2] + vinserti128 m4, [srcq+ssq*1], 1 + add srcq, ss3q + vpblendd m15, m7, 0x30 + punpcklqdq m15, m15 + pblendvb m15, m11, m8 + movq xm10, r4q + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 + pshufb m2, m14 + pshufb m3, m14 + pshufb m4, m14 + pshufb xm5, xm14 + vpermq m2, m2, q3120 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + pmaddubsw m4, m15 + pmaddubsw m5, m15 + phaddw m2, m3 + phaddw m4, m5 + pmulhrsw m2, m12 + pmulhrsw m4, m12 + palignr m5, m4, m2, 4 + pshufd m3, m4, q2121 + punpcklwd m0, m2, m5 ; 01 12 + punpckhwd m1, m2, m5 ; 23 34 + punpcklwd m2, m4, m3 ; 45 56 +.dy1_w4_loop: + movu xm11, [srcq+ssq*0] + vinserti128 m11, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pmaddwd m6, m2, m9 + mova m0, m1 + mova m1, m2 + paddd m4, m13 + paddd m5, m6 + pshufb m11, m14 + vpermq m11, m11, q3120 + pmaddubsw m11, m15 + phaddw m11, m11 + pmulhrsw m11, m12 + palignr m6, m11, m3, 12 + punpcklwd m2, m6, m11 ; 67 78 + mova m3, m11 + pmaddwd m6, m2, m10 + paddd m4, m5 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + pshuflw xm4, xm4, q3120 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] +%else + pshufd xm4, xm4, q3120 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy1_w4_loop + MC_8TAP_SCALED_RET +.dy1_w8: + mov dword [rsp+72], 1 + movifprep tmp_stridem, 16 + jmp .dy1_w_start +.dy1_w16: + mov dword [rsp+72], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [rsp+72], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [rsp+72], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [rsp+72], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: + mov myd, mym +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+76], t0d + mov [rsp+80], srcq + mov [rsp+88], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+96], xm0 + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [rsp+72] + jz .ret + add qword [rsp+88], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp+32] + vpbroadcastd m15, [rsp+76] + pxor m9, m9 + mov srcq, [rsp+80] + mov r0q, [rsp+88] ; dstq / tmpq +.dy1_hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp+32], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movq [rsp+64], xm14 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + movu [rsp], m10 + vpbroadcastd m8, [rsp+0x60] + vpbroadcastd m9, [rsp+0x64] + vpbroadcastd m10, [rsp+0x68] + vpbroadcastd m11, [rsp+0x6c] + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + vbroadcasti128 m14, [base+wswap] +.dy1_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy1_hloop_prep + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, [rsp] + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .dy1_vloop +.dy2: + movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy2_w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0-1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + vpbroadcastq m2, [srcq+ssq*1] + movhps xm0, [srcq+ssq*2] + vpbroadcastq m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vpblendd m15, m7, 0xaa + pblendvb m15, m11, m8 + movhps xm1, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + vpblendd m0, m2, 0x30 + vpblendd m1, m4, 0xc0 + vpblendd m0, m3, 0xc0 + pshufb m0, m14 + pshufb m1, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + movq xm11, r4q + pmovsxbw xm11, xm11 + phaddw m0, m1 + pmulhrsw m0, m12 ; 0 2 _ 4 1 3 _ 5 + pshufd xm8, xm11, q0000 + pshufd xm9, xm11, q1111 + pshufd xm10, xm11, q2222 + pshufd xm11, xm11, q3333 + pshufd m2, m0, q3110 ; 0 2 2 4 1 3 3 5 + vextracti128 xm1, m2, 1 + punpcklwd xm3, xm2, xm1 ; 01 23 + punpckhwd xm2, xm1 ; 23 45 +.dy2_w2_loop: + movq xm6, [srcq+ssq*0] + vpbroadcastq m7, [srcq+ssq*1] + movhps xm6, [srcq+ssq*2] + vpbroadcastq m1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pmaddwd xm4, xm3, xm8 + pmaddwd xm5, xm2, xm9 + vpblendd m6, m7, 0x30 + vpblendd m6, m1, 0xc0 + pshufb m6, m14 + pmaddubsw m6, m15 + phaddw m6, m6 + pmulhrsw m6, m12 + palignr m0, m6, m0, 8 + pshufd m2, m0, q3221 + vextracti128 xm1, m2, 1 + punpcklwd xm3, xm2, xm1 ; 45 67 + punpckhwd xm2, xm1 ; 67 89 + pmaddwd xm6, xm3, xm10 + pmaddwd xm7, xm2, xm11 + paddd xm4, xm5 + paddd xm4, xm13 + paddd xm6, xm7 + paddd xm4, xm6 + psrad xm4, rndshift + packssdw xm4, xm4 + packuswb xm4, xm4 + pextrw [dstq+dsq*0], xm4, 0 + pextrw [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +.dy2_w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pinsrd xm15, [base+subpel_filters+r6*8+2], 1 + pcmpeqd m8, m9 + psrld m14, 10 + movu xm0, [srcq+ssq*0] + movu xm2, [srcq+ssq*2] + pinsrd xm15, [base+subpel_filters+r11*8+2], 2 + movu xm1, [srcq+ssq*1] + movu xm3, [srcq+ss3q ] + pinsrd xm15, [base+subpel_filters+r13*8+2], 3 + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + vinserti128 m15, xm15, 1 + pshufb m14, m5 + paddb m14, m6 + vinserti128 m2, [srcq+ssq*0], 1 + vinserti128 m3, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pblendvb m15, m11, m8 + pshufb xm0, xm14 + pshufb m2, m14 + pshufb xm1, xm14 + pshufb m3, m14 + pmaddubsw xm0, xm15 + pmaddubsw m2, m15 + pmaddubsw xm1, xm15 + pmaddubsw m3, m15 + movq xm11, r4q + punpcklqdq xm11, xm11 + pmovsxbw m11, xm11 + phaddw m0, m2 + phaddw m1, m3 + pmulhrsw m0, m12 ; 0 2 _ 4 + pmulhrsw m1, m12 ; 1 3 _ 5 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 + punpcklwd xm2, xm0, xm1 + punpckhwd m1, m0, m1 ; 23 45 + vinserti128 m0, m2, xm1, 1 ; 01 23 +.dy2_w4_loop: + movu xm6, [srcq+ssq*0] + movu xm7, [srcq+ssq*1] + vinserti128 m6, [srcq+ssq*2], 1 + vinserti128 m7, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pshufb m6, m14 + pshufb m7, m14 + pmaddubsw m6, m15 + pmaddubsw m7, m15 + psrld m2, m6, 16 + pslld m3, m7, 16 + paddw m6, m2 + paddw m7, m3 + pblendw m6, m7, 0xaa ; 67 89 + pmulhrsw m6, m12 + paddd m4, m5 + vperm2i128 m0, m1, m6, 0x21 ; 45 67 + mova m1, m6 + pmaddwd m6, m0, m10 + pmaddwd m7, m1, m11 + paddd m4, m13 + paddd m6, m7 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] +%else + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET +.dy2_w8: + mov dword [rsp+40], 1 + movifprep tmp_stridem, 16 + jmp .dy2_w_start +.dy2_w16: + mov dword [rsp+40], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [rsp+40], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [rsp+40], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [rsp+40], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: + mov myd, mym +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+64], t0d + mov [rsp+48], srcq + mov [rsp+56], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+0x50], xm0 + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [rsp+40] + jz .ret + add qword [rsp+56], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp] + vpbroadcastd m15, [rsp+64] + pxor m9, m9 + mov srcq, [rsp+48] + mov r0q, [rsp+56] ; dstq / tmpq +.dy2_hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + vpbroadcastd m8, [rsp+0x50] + vpbroadcastd m9, [rsp+0x54] + vpbroadcastd m11, [rsp+0x58] + vpbroadcastd m4, [rsp+0x5c] + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + SWAP m14, m4 +.dy2_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m11 + pmaddwd m7, m3, m14 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy2_hloop_prep + mova m0, m1 + mova m1, m2 + mova m2, m3 + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + vpbroadcastq m5, [srcq+r13] + vpbroadcastq m6, [srcq+ rX] + add srcq, ssq + vpblendd m3, m5, 0xc0 + vpblendd m4, m6, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + phaddw m3, m4 + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + psrld m5, m3, 16 + pslld m6, m4, 16 + paddw m3, m5 + paddw m4, m6 + pblendw m3, m4, 0xaa + pmulhrsw m3, m12 + jmp .dy2_vloop +.ret: + MC_8TAP_SCALED_RET 0 +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled_8bpc + mov t0d, (5*15 << 16) | 5*15 + mov t1d, t0d + jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%else +DECLARE_REG_TMP 6, 8 +%endif + +%define PUT_8TAP_SCALED_FN FN put_8tap_scaled, +%define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, + +BILIN_SCALED_FN put +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +BILIN_SCALED_FN prep +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + +%macro WARP_V 5 ; dst, 02, 46, 13, 57 + ; Can be done using gathers, but that's terribly slow on many CPU:s + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm8, [filterq+myq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 ; a e + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+deltaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; b f + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm9, [filterq+myq *8] + vinserti128 m9, [filterq+tmp1q*8], 1 ; c g + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+gammaq] ; my += gamma + shr tmp2d, 10 + shr tmp1d, 10 + punpcklwd m8, m0 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; d h + punpcklwd m0, m9, m0 + punpckldq m9, m8, m0 + punpckhdq m0, m8, m0 + punpcklbw m8, m11, m9 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 + punpckhbw m9, m11, m9 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 + pmaddwd m%2, m8 + pmaddwd m9, m%3 + punpcklbw m8, m11, m0 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 + punpckhbw m0, m11, m0 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 + pmaddwd m8, m%4 + pmaddwd m0, m%5 + paddd m%2, m9 + paddd m0, m8 + paddd m%1, m0, m%2 +%endmacro + +cglobal warp_affine_8x8t_8bpc, 0, 14, 0, tmp, ts +%if WIN64 + sub rsp, 0xa0 +%endif + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main +.loop: + psrad m7, 13 + psrad m0, 13 + packssdw m7, m0 + pmulhrsw m7, m14 ; (x + (1 << 6)) >> 7 + vpermq m7, m7, q3120 + mova [tmpq+tsq*0], xm7 + vextracti128 [tmpq+tsq*2], m7, 1 + dec r4d + jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).end + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main2 + lea tmpq, [tmpq+tsq*4] + jmp .loop + +cglobal warp_affine_8x8_8bpc, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \ + beta, filter, tmp1, delta, my, gamma +%if WIN64 + sub rsp, 0xa0 + %assign xmm_regs_used 16 + %assign stack_size_padded 0xa0 + %assign stack_offset stack_offset+stack_size_padded +%endif + call .main + jmp .start +.loop: + call .main2 + lea dstq, [dstq+dsq*2] +.start: + psrad m7, 18 + psrad m0, 18 + packusdw m7, m0 + pavgw m7, m11 ; (x + (1 << 10)) >> 11 + vextracti128 xm0, m7, 1 + packuswb xm7, xm0 + pshufd xm7, xm7, q3120 + movq [dstq+dsq*0], xm7 + movhps [dstq+dsq*1], xm7 + dec r4d + jg .loop +.end: + RET +ALIGN function_align +.main: + ; Stack args offset by one (r4m -> r5m etc.) due to call +%if WIN64 + mov abcdq, r5m + mov mxd, r6m + movaps [rsp+stack_offset+0x10], xmm6 + movaps [rsp+stack_offset+0x20], xmm7 + movaps [rsp+0x28], xmm8 + movaps [rsp+0x38], xmm9 + movaps [rsp+0x48], xmm10 + movaps [rsp+0x58], xmm11 + movaps [rsp+0x68], xmm12 + movaps [rsp+0x78], xmm13 + movaps [rsp+0x88], xmm14 + movaps [rsp+0x98], xmm15 +%endif + movsx alphad, word [abcdq+2*0] + movsx betad, word [abcdq+2*1] + mova m12, [warp_8x8_shufA] + mova m13, [warp_8x8_shufB] + vpbroadcastd m14, [pw_8192] + vpbroadcastd m15, [pd_32768] + pxor m11, m11 + lea filterq, [mc_warp_filter2] + lea tmp1q, [ssq*3+3] + add mxd, 512+(64<<10) + lea tmp2d, [alphaq*3] + sub srcq, tmp1q ; src -= src_stride*3 + 3 + sub betad, tmp2d ; beta -= alpha*3 + mov myd, r7m + call .h + psrld m1, m0, 16 + call .h + psrld m4, m0, 16 + call .h + pblendw m1, m0, 0xaa ; 02 + call .h + pblendw m4, m0, 0xaa ; 13 + call .h + psrld m2, m1, 16 + pblendw m2, m0, 0xaa ; 24 + call .h + psrld m5, m4, 16 + pblendw m5, m0, 0xaa ; 35 + call .h + psrld m3, m2, 16 + pblendw m3, m0, 0xaa ; 46 + movsx deltad, word [abcdq+2*2] + movsx gammad, word [abcdq+2*3] + add myd, 512+(64<<10) + mov r4d, 4 + lea tmp1d, [deltaq*3] + sub gammad, tmp1d ; gamma -= delta*3 +.main2: + call .h + psrld m6, m5, 16 + pblendw m6, m0, 0xaa ; 57 + WARP_V 7, 1, 3, 4, 6 + call .h + mova m1, m2 + mova m2, m3 + psrld m3, 16 + pblendw m3, m0, 0xaa ; 68 + WARP_V 0, 4, 6, 1, 3 + mova m4, m5 + mova m5, m6 + ret +ALIGN function_align +.h: + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + vbroadcasti128 m10, [srcq] + shr mxd, 10 + shr tmp1d, 10 + movq xm8, [filterq+mxq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+alphaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + shr mxd, 10 + shr tmp1d, 10 + movq xm9, [filterq+mxq *8] + vinserti128 m9, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + punpcklqdq m8, m0 ; 0 1 4 5 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 + punpcklqdq m9, m0 ; 2 3 6 7 + pshufb m0, m10, m12 + pmaddubsw m0, m8 + pshufb m10, m13 + pmaddubsw m10, m9 + add srcq, ssq + phaddw m0, m10 + pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 + paddd m0, m15 ; rounded 14-bit result in upper 16 bits of dword + ret + +%macro BIDIR_FN 1 ; op + %1 0 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + cmp hd, 8 + je .ret + %1 2 + lea dstq, [dstq+strideq*4] + vextracti128 xm1, m0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.ret: + RET +.w8_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*4] +.w8: + vextracti128 xm1, m0, 1 + movq [dstq ], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq*4] +.w16: + vpermq m0, m0, q3120 + mova [dstq ], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq*2] +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + %1_INC_PTR 4 + %1 0 + add dstq, strideq +.w64: + vpermq m0, m0, q3120 + mova [dstq], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+32], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + %1 0 + add dstq, strideq +.w128: + vpermq m0, m0, q3120 + mova [dstq+0*32], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+1*32], m0 + %1_INC_PTR 8 + %1 -4 + vpermq m0, m0, q3120 + mova [dstq+2*32], m0 + %1 -2 + vpermq m0, m0, q3120 + mova [dstq+3*32], m0 + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + mova m0, [tmp1q+(%1+0)*32] + paddw m0, [tmp2q+(%1+0)*32] + mova m1, [tmp1q+(%1+1)*32] + paddw m1, [tmp2q+(%1+1)*32] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + packuswb m0, m1 +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*32 + add tmp2q, %1*32 +%endmacro + +cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg %+ SUFFIX %+ _table + lea r6, [avg %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r6+wq*4] + vpbroadcastd m2, [base+pw_1024] + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m0, [tmp1q+(%1+0)*32] + psubw m2, m0, [tmp2q+(%1+0)*32] + mova m1, [tmp1q+(%1+1)*32] + psubw m3, m1, [tmp2q+(%1+1)*32] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-w_avg %+ SUFFIX %+ _table + lea r6, [w_avg %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + vpbroadcastw m4, r6m ; weight + movsxd wq, dword [r6+wq*4] + vpbroadcastd m5, [base+pw_2048] + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + pxor m0, m0 + mov tmp1q, tmp2q + psubw m4, m0, m4 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 + vpermq m3, [maskq+%1*16], q3120 + mova m0, [tmp2q+(%1+0)*32] + psubw m1, m0, [tmp1q+(%1+0)*32] + psubb m3, m4, m3 + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 + punpcklbw m2, m4, m3 ; -m << 9 + pmulhw m1, m2 + paddw m0, m1 + mova m1, [tmp2q+(%1+1)*32] + psubw m2, m1, [tmp1q+(%1+1)*32] + paddw m2, m2 + punpckhbw m3, m4, m3 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*16 + add tmp2q, %1*32 + add tmp1q, %1*32 +%endmacro + +cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask %+ SUFFIX %+ _table + lea r7, [mask %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + vpbroadcastd m5, [base+pw_2048] + pxor m4, m4 + add wq, r7 + BIDIR_FN MASK + +%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 + mova m%1, [tmp1q+32*%3] + mova m1, [tmp2q+32*%3] + psubw m1, m%1 + pabsw m%2, m1 + psubusw m%2, m6, m%2 + psrlw m%2, 8 ; 64 - m + psllw m2, m%2, 10 + pmulhw m1, m2 + paddw m%1, m1 + mova m1, [tmp1q+32*%4] + mova m2, [tmp2q+32*%4] + psubw m2, m1 + pabsw m3, m2 + psubusw m3, m6, m3 + psrlw m3, 8 +%if %5 + packuswb m%2, m3 + psubb m%2, m5, m%2 + vpermq m%2, m%2, q3120 +%else + phaddw m%2, m3 +%endif + psllw m3, 10 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m%1, m7 + pmulhrsw m1, m7 + packuswb m%1, m1 +%endmacro + +cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_avx2_table + lea r6, [blend_avx2_table] + tzcnt wd, wm + movifnidn maskq, maskmp + movifnidn hd, hm + movsxd wq, dword [r6+wq*4] + vpbroadcastd m4, [base+pb_64] + vpbroadcastd m5, [base+pw_512] + sub tmpq, maskq + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + vpbroadcastd xm1, [dstq+dsq*2] + pinsrd xm1, [dstq+r6 ], 3 + mova xm6, [maskq] + psubb xm3, xm4, xm6 + punpcklbw xm2, xm3, xm6 + punpckhbw xm3, xm6 + mova xm6, [maskq+tmpq] + add maskq, 4*4 + punpcklbw xm0, xm6 + punpckhbw xm1, xm6 + pmaddubsw xm0, xm2 + pmaddubsw xm1, xm3 + pmulhrsw xm0, xm5 + pmulhrsw xm1, xm5 + packuswb xm0, xm1 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + pextrd [dstq+dsq*2], xm0, 2 + pextrd [dstq+r6 ], xm0, 3 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w4 + RET +ALIGN function_align +.w8: + movq xm1, [dstq+dsq*0] + movhps xm1, [dstq+dsq*1] + vpbroadcastq m2, [dstq+dsq*2] + vpbroadcastq m3, [dstq+r6 ] + mova m0, [maskq] + mova m6, [maskq+tmpq] + add maskq, 8*4 + vpblendd m1, m2, 0x30 + vpblendd m1, m3, 0xc0 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + movq [dstq+dsq*2], xm1 + movhps [dstq+r6 ], xm1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w8 + RET +ALIGN function_align +.w16: + mova m0, [maskq] + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + mova m6, [maskq+tmpq] + add maskq, 16*2 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16 + RET +ALIGN function_align +.w32: + mova m0, [maskq] + mova m1, [dstq] + mova m6, [maskq+tmpq] + add maskq, 32 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .w32 + RET + +cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_v_avx2_table + lea r5, [blend_v_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r5+wq*4] + vpbroadcastd m5, [base+pw_512] + add wq, r5 + add maskq, obmc_masks-blend_v_avx2_table + jmp wq +.w2: + vpbroadcastd xm2, [maskq+2*2] +.w2_s0_loop: + movd xm0, [dstq+dsq*0] + pinsrw xm0, [dstq+dsq*1], 1 + movd xm1, [tmpq] + add tmpq, 2*2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w2_s0_loop + RET +ALIGN function_align +.w4: + vpbroadcastq xm2, [maskq+4*2] +.w4_loop: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + movq xm1, [tmpq] + add tmpq, 4*2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +ALIGN function_align +.w8: + mova xm3, [maskq+8*2] +.w8_loop: + movq xm0, [dstq+dsq*0] + vpbroadcastq xm1, [dstq+dsq*1] + mova xm2, [tmpq] + add tmpq, 8*2 + punpcklbw xm0, xm2 + punpckhbw xm1, xm2 + pmaddubsw xm0, xm3 + pmaddubsw xm1, xm3 + pmulhrsw xm0, xm5 + pmulhrsw xm1, xm5 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + vbroadcasti128 m3, [maskq+16*2] + vbroadcasti128 m4, [maskq+16*3] +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + mova m2, [tmpq] + add tmpq, 16*2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16_loop + RET +ALIGN function_align +.w32: + mova xm3, [maskq+16*4] + vinserti128 m3, [maskq+16*6], 1 + mova xm4, [maskq+16*5] + vinserti128 m4, [maskq+16*7], 1 +.w32_loop: + mova m1, [dstq] + mova m2, [tmpq] + add tmpq, 32 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .w32_loop + RET + +cglobal blend_h_8bpc, 4, 7, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_h_avx2_table + lea r5, [blend_h_avx2_table] + mov r6d, wd + tzcnt wd, wd + mov hd, hm + movsxd wq, dword [r5+wq*4] + vpbroadcastd m5, [base+pw_512] + add wq, r5 + lea maskq, [base+obmc_masks+hq*2] + lea hd, [hq*3] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd xm0, [dstq+dsq*0] + pinsrw xm0, [dstq+dsq*1], 1 + movd xm2, [maskq+hq*2] + movd xm1, [tmpq] + add tmpq, 2*2 + punpcklwd xm2, xm2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +ALIGN function_align +.w4: + mova xm3, [blend_shuf] +.w4_loop: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + movd xm2, [maskq+hq*2] + movq xm1, [tmpq] + add tmpq, 4*2 + pshufb xm2, xm3 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +ALIGN function_align +.w8: + vbroadcasti128 m4, [blend_shuf] + shufpd m4, m4, 0x03 +.w8_loop: + vpbroadcastq m1, [dstq+dsq*0] + movq xm0, [dstq+dsq*1] + vpblendd m0, m1, 0x30 + vpbroadcastd m3, [maskq+hq*2] + movq xm1, [tmpq+8*1] + vinserti128 m1, [tmpq+8*0], 1 + add tmpq, 8*2 + pshufb m3, m4 + punpcklbw m0, m1 + pmaddubsw m0, m3 + pmulhrsw m0, m5 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movhps [dstq+dsq*0], xm0 + movq [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop + RET +ALIGN function_align +.w16: + vbroadcasti128 m4, [blend_shuf] + shufpd m4, m4, 0x0c +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + vpbroadcastd m3, [maskq+hq*2] + mova m2, [tmpq] + add tmpq, 16*2 + pshufb m3, m4 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w16_loop + RET +ALIGN function_align +.w32: ; w32/w64/w128 + sub dsq, r6 +.w32_loop0: + vpbroadcastw m3, [maskq+hq*2] + mov wd, r6d +.w32_loop: + mova m1, [dstq] + mova m2, [tmpq] + add tmpq, 32 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, 32 + sub wd, 32 + jg .w32_loop + add dstq, dsq + inc hq + jl .w32_loop0 + RET + +cglobal emu_edge_8bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ + bottomext, rightext + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor r12d, r12d + lea r10, [ihq-1] + cmp yq, ihq + cmovs r10, yq + test yq, yq + cmovs r10, r12 + imul r10, sstrideq + add srcq, r10 + + ; ref += iclip(x, 0, iw - 1) + lea r10, [iwq-1] + cmp xq, iwq + cmovs r10, xq + test xq, xq + cmovs r10, r12 + add srcq, r10 + + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) + lea bottomextq, [yq+bhq] + sub bottomextq, ihq + lea r3, [bhq-1] + cmovs bottomextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, r12 + cmp bottomextq, bhq + cmovns bottomextq, r3 + cmp topextq, bhq + cmovg topextq, r3 + + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + lea rightextq, [xq+bwq] + sub rightextq, iwq + lea r2, [bwq-1] + cmovs rightextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, r12 + cmp rightextq, bwq + cmovns rightextq, r2 + cmp leftextq, bwq + cmovns leftextq, r2 + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \ + dst, dstride, src, sstride, bottomext, rightext + + ; center_h = bh - top_ext - bottom_ext + lea r3, [bottomextq+topextq] + sub centerhq, r3 + + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq + imul r2, dstrideq + add dstq, r2 + mov r9m, dstq + + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq + lea r3, [rightextq+leftextq] + sub centerwq, r3 + +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix +.v_loop_%3: +%if %1 + ; left extension + xor r3, r3 + vpbroadcastb m0, [srcq] +.left_loop_%3: + mova [dstq+r3], m0 + add r3, 32 + cmp r3, leftextq + jl .left_loop_%3 + + ; body + lea r12, [dstq+leftextq] +%endif + xor r3, r3 +.body_loop_%3: + movu m0, [srcq+r3] +%if %1 + movu [r12+r3], m0 +%else + movu [dstq+r3], m0 +%endif + add r3, 32 + cmp r3, centerwq + jl .body_loop_%3 + +%if %2 + ; right extension +%if %1 + add r12, centerwq +%else + lea r12, [dstq+centerwq] +%endif + xor r3, r3 + vpbroadcastb m0, [srcq+centerwq-1] +.right_loop_%3: + movu [r12+r3], m0 + add r3, 32 + cmp r3, rightextq + jl .right_loop_%3 + +%endif + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 +%endmacro + + test leftextq, leftextq + jnz .need_left_ext + test rightextq, rightextq + jnz .need_right_ext + v_loop 0, 0, 0 + jmp .body_done + +.need_left_ext: + test rightextq, rightextq + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: + ; bottom edge extension + test bottomextq, bottomextq + jz .top + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 +.bottom_x_loop: + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, bottomextq +.bottom_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .bottom_y_loop + add r1, 32 + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end + mov srcq, r9m + mov dstq, dstm + xor r1, r1 +.top_x_loop: + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, topextq +.top_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .top_y_loop + add r1, 32 + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +cglobal resize_8bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x + LEA r7, $$ +%define base r7-$$ + + vpbroadcastd xm3, [base+pw_m256] + vpbroadcastd m7, [base+pd_63] + vbroadcasti128 m15, [base+pb_8x0_8x8] + pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7] + pslld m5, 3 ; dx*8 + pslld m6, 14 + paddd m8, m2 ; mx+[0..7]*dx + pxor m2, m2 + + ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 + ; m8 = mx+[0..7]*dx, m5 = dx*8, m6 = src_w, m7 = 0x3f, m15=0,8 + +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx + +.loop_x: + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + pand m9, m7 ; filter offset (masked) + + ; load source pixels - this ugly code is vpgatherdq emulation since + ; directly using vpgatherdq on Haswell is quite a bit slower :( + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vextracti128 xm0, m0, 1 + movq xm12, [srcq+r8] + movq xm13, [srcq+r10] + movhps xm12, [srcq+r9] + movhps xm13, [srcq+r11] + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vinserti128 m12, [srcq+r8], 1 + vinserti128 m13, [srcq+r10], 1 + vpbroadcastq m10, [srcq+r9] + vpbroadcastq m11, [srcq+r11] + vpblendd m12, m10, 11000000b + vpblendd m13, m11, 11000000b + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + vptest m1, m1 + jz .filter + + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vextracti128 xm1, m1, 1 + movq xm14, [base+resize_shuf+4+r8] + movq xm0, [base+resize_shuf+4+r10] + movhps xm14, [base+resize_shuf+4+r9] + movhps xm0, [base+resize_shuf+4+r11] + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vinserti128 m14, [base+resize_shuf+4+r8], 1 + vinserti128 m0, [base+resize_shuf+4+r10], 1 + vpbroadcastq m10, [base+resize_shuf+4+r9] + vpbroadcastq m11, [base+resize_shuf+4+r11] + vpblendd m14, m10, 11000000b + vpblendd m0, m11, 11000000b + + paddb m14, m15 + paddb m0, m15 + pshufb m12, m14 + pshufb m13, m0 + +.filter: + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vextracti128 xm9, m9, 1 + movq xm10, [base+resize_filter+r8*8] + movq xm11, [base+resize_filter+r10*8] + movhps xm10, [base+resize_filter+r9*8] + movhps xm11, [base+resize_filter+r11*8] + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vinserti128 m10, [base+resize_filter+r8*8], 1 + vinserti128 m11, [base+resize_filter+r10*8], 1 + vpbroadcastq m14, [base+resize_filter+r9*8] + vpbroadcastq m1, [base+resize_filter+r11*8] + vpblendd m10, m14, 11000000b + vpblendd m11, m1, 11000000b + + pmaddubsw m12, m10 + pmaddubsw m13, m11 + phaddw m12, m13 + vextracti128 xm13, m12, 1 + phaddsw xm12, xm13 + pmulhrsw xm12, xm3 ; x=(x+64)>>7 + packuswb xm12, xm12 + movq [dstq+xq], xm12 + + paddd m4, m5 + add xd, 8 + cmp xd, dst_wd + jl .loop_x + + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +cglobal w_mask_420_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx2_table + lea r7, [w_mask_420_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m9, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_420_sign+r6*4] ; 258 - sign + add wq, r7 + W_MASK 0, 4, 0, 1 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm0, m4, 1 + vpblendd xm1, xm4, xm0, 0x05 + vpblendd xm4, xm0, 0x0a + pshufd xm1, xm1, q2301 + psubw xm4, xm8, xm4 + psubw xm4, xm1 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [maskq], xm4 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + phaddd m4, m5 + vextracti128 xm1, m0, 1 + psubw m4, m8, m4 + psrlw m4, 2 + vpermd m4, m9, m4 + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], xm4 + RET +.w8_loop: + add tmp1q, 2*32 + add tmp2q, 2*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 8 +.w8: + vextracti128 xm2, m4, 1 + vextracti128 xm1, m0, 1 + psubw xm4, xm8, xm4 + psubw xm4, xm2 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + movq [maskq], xm4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + punpckhqdq m1, m4, m5 + punpcklqdq m4, m5 + psubw m1, m8, m1 + psubw m1, m4 + psrlw m1, 2 + vpermq m0, m0, q3120 + packuswb m1, m1 + vpermd m1, m9, m1 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], xm1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 16 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + psubw m4, m8, m4 + psubw m4, m5 + psrlw m4, 2 + vpermq m0, m0, q3120 + packuswb m4, m4 + vpermd m4, m9, m4 + mova [dstq+strideq*1], m0 + mova [maskq], xm4 + sub hd, 2 + jg .w32_loop + RET +.w64_loop_even: + psubw m10, m8, m4 + psubw m11, m8, m5 + dec hd +.w64_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + test hd, 1 + jz .w64_loop_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq], m4 + add maskq, 32 + dec hd + jg .w64_loop + RET +.w128_loop_even: + psubw m12, m8, m4 + psubw m13, m8, m5 + dec hd +.w128_loop: + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + add tmp1q, 8*32 + add tmp2q, 8*32 + test hd, 1 + jz .w128_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*0], m4 + jmp .w128_odd +.w128_even: + psubw m10, m8, m4 + psubw m11, m8, m5 +.w128_odd: + W_MASK 0, 4, -4, -3 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, -2, -1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + test hd, 1 + jz .w128_loop_even + psubw m4, m12, m4 + psubw m5, m13, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*1], m4 + add maskq, 64 + dec hd + jg .w128_loop + RET + +cglobal w_mask_422_8bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx2_table + lea r7, [w_mask_422_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + pxor m9, m9 + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m10, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_422_sign+r6*4] ; 128 - sign + add wq, r7 + mov maskq, maskmp + W_MASK 0, 4, 0, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + mova [maskq], xm5 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermd m5, m10, m5 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], m5 + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w8: + vextracti128 xm5, m4, 1 + vextracti128 xm1, m0, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], xm5 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], m5 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*1], m0 + mova [maskq], m5 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq], m5 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32*2 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq+32*0], m5 + W_MASK 0, 4, 4, 5 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, 6, 7 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*3], m0 + mova [maskq+32*1], m5 + dec hd + jg .w128_loop + RET + +cglobal w_mask_444_8bpc, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx2_table + lea r7, [w_mask_444_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + add wq, r7 + W_MASK 0, 4, 0, 1, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + mova [maskq+32*0], m4 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + je .w4_end + W_MASK 0, 4, 2, 3, 1 + lea dstq, [dstq+strideq*4] + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq+32*1], m4 +.w4_end: + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w8: + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], m4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [maskq], m4 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq], m0 + mova [maskq], m4 + dec hd + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*2 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*4 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + W_MASK 0, 4, 4, 5, 1 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + mova [maskq+32*2], m4 + W_MASK 0, 4, 6, 7, 1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + mova [maskq+32*3], m4 + dec hd + jg .w128_loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_avx512.asm b/third_party/dav1d/src/x86/mc_avx512.asm new file mode 100644 index 0000000000..7897f1decc --- /dev/null +++ b/third_party/dav1d/src/x86/mc_avx512.asm @@ -0,0 +1,4538 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 + +obmc_masks: +pw_512: times 2 dw 512 + ; 2 + db 45, 19, 64, 0 + ; 4 + db 39, 25, 50, 14, 59, 5, 64, 0 + ; 8 + db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 + ; 16 + db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 + db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 + ; 32 + db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 + db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 + db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 + db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0 + +warp_8x8_permA: db 4, 5, 6, 7, 16, 17, 18, 19, 5, 6, 7, 8, 17, 18, 19, 20 + db 6, 7, 8, 9, 18, 19, 20, 21, 7, 8, 9, 10, 19, 20, 21, 22 + db 8, 9, 10, 11, 20, 21, 22, 23, 9, 10, 11, 12, 21, 22, 23, 24 + db 10, 11, 12, 13, 22, 23, 24, 25, 11, 12, 13, 14, 23, 24, 25, 26 +warp_8x8_permB: db 0, 1, 2, 3, 20, 21, 22, 23, 1, 2, 3, 4, 21, 22, 23, 24 + db 2, 3, 4, 5, 22, 23, 24, 25, 3, 4, 5, 6, 23, 24, 25, 26 + db 4, 5, 6, 7, 24, 25, 26, 27, 5, 6, 7, 8, 25, 26, 27, 28 + db 6, 7, 8, 9, 26, 27, 28, 29, 7, 8, 9, 10, 27, 28, 29, 30 +warp_8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13 +warp_8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15 +pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7 +warp_8x8_hpack: db 3, 11, 3, 11, 35, 43, 35, 43 +pd_16384: dd 16384 +pd_262144: dd 262144 +warp_8x8_end: db 0, 4, 16, 20, 32, 36, 48, 52, 2, 6, 18, 22, 34, 38, 50, 54 +warp_8x8t_end: db 2, 3, 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, 59 + db 6, 7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 63 +bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31 + db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63 + db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 + db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 +wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31 + db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63 + db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 + db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 +wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47 + db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63 + db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 + db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 +wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 + db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 +bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39 + db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47 +bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23 + db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31 +bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87 + db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39 +bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 + db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23 + db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31 +bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7 + db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31 +bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7 +spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 +spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42 + db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50 +spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 + db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54 +spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 +spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26 + db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34 +spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 +spel_v_perm16: db 32, 0, 33, 1, 34, 2, 35, 3, 36, 4, 37, 5, 38, 6, 39, 7 + db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 + db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23 + db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 +spel_v_perm32: db 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39 + db 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47 + db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55 + db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63 +spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23 + db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31 +spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39 + db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47 +spel_hv_perm4c: db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55 + db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63 +spel_hv_perm4d: db 18, 19, 0, 1, 22, 23, 4, 5, 26, 27, 8, 9, 30, 31, 12, 13 + db 0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29 +spel_hv_perm8a: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 + db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 + db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39 + db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47 +spel_hv_perm8b: db 32, 33, 48, 49, 34, 35, 50, 51, 36, 37, 52, 53, 38, 39, 54, 55 + db 40, 41, 56, 57, 42, 43, 58, 59, 44, 45, 60, 61, 46, 47, 62, 63 + db 48, 49, 64, 65, 50, 51, 66, 67, 52, 53, 68, 69, 54, 55, 70, 71 + db 56, 57, 72, 73, 58, 59, 74, 75, 60, 61, 76, 77, 62, 63, 78, 79 +spel_hv_perm8c: db 34, 35, 0, 1, 38, 39, 4, 5, 42, 43, 8, 9, 46, 47, 12, 13 + db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29 + db 0, 1, 32, 33, 4, 5, 36, 37, 8, 9, 40, 41, 12, 13, 44, 45 + db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61 +spel_hv_end16: db 1, 3, 17, 19, 5, 7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55 + db 9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63 +spel_hv_perm16a:db 0, 1, 2, 3, 32, 33, 34, 35, 1, 2, 3, 4, 33, 34, 35, 36 + db 2, 3, 4, 5, 34, 35, 36, 37, 3, 4, 5, 6, 35, 36, 37, 38 +spel_hv_perm16c:db 8, 9, 10, 11, 40, 41, 42, 43, 9, 10, 11, 12, 41, 42, 43, 44 + db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46 + db 16, 17, 18, 19, 48, 49, 50, 51, 17, 18, 19, 20, 49, 50, 51, 52 + db 18, 19, 20, 21, 50, 51, 52, 53, 19, 20, 21, 22, 51, 52, 53, 54 +spel_hv_perm16b:db 4, 5, 6, 7, 36, 37, 38, 39, 5, 6, 7, 8, 37, 38, 39, 40 + db 6, 7, 8, 9, 38, 39, 40, 41, 7, 8, 9, 10, 39, 40, 41, 42 + db 12, 13, 14, 15, 44, 45, 46, 47, 13, 14, 15, 16, 45, 46, 47, 48 + db 14, 15, 16, 17, 46, 47, 48, 49, 15, 16, 17, 18, 47, 48, 49, 50 +spel_hv_perm16d:db 0, 1, 2, 3, 1, 2, 3, 4, 4, 5, 6, 7, 5, 6, 7, 8 + db 2, 3, 4, 5, 3, 4, 5, 6, 6, 7, 8, 9, 7, 8, 9, 10 + db 8, 9, 10, 11, 9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16 + db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18 +spel_hv_perm16e:db 4, 5, 6, 7, 5, 6, 7, 8, 8, 9, 10, 11, 9, 10, 11, 12 + db 6, 7, 8, 9, 7, 8, 9, 10, 10, 11, 12, 13, 11, 12, 13, 14 + db 12, 13, 14, 15, 13, 14, 15, 16, 16, 17, 18, 19, 17, 18, 19, 20 + db 14, 15, 16, 17, 15, 16, 17, 18, 18, 19, 20, 21, 19, 20, 21, 22 +spel_hv_end: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55 +deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 +bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 +resize_permA: dd 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +resize_permB: dd 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 +resize_permC: dd 0, 4, 8, 12 +pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7 + +wm_420_perm64: dq 0xfedcba9876543210 +wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040 + +pb_8x0_8x8: times 8 db 0 + times 8 db 8 +pb_127: times 4 db 127 +pw_m128 times 2 dw -128 +pw_m256: times 2 dw -256 +pw_1024: times 2 dw 1024 +pw_2048: times 2 dw 2048 +pw_6903: times 2 dw 6903 +pw_8192: times 2 dw 8192 +pd_32: dd 32 +pd_34: dd 34 +pd_63: dd 63 +pd_512: dd 512 +pd_32768: dd 32768 + +%define pb_m64 (wm_sign+4) +%define pb_64 (wm_sign+8) +%define pd_2 (pd_0to7+8) + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) +cextern mc_warp_filter +cextern resize_filter + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + %%table: + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_8bpc_avx512icl.put) +%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_8bpc_avx512icl.prep) + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx512icl, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128 + +SECTION .text + +%macro WRAP_YMM 1+ +INIT_YMM cpuname + %1 +INIT_ZMM cpuname +%endmacro + +INIT_ZMM avx512icl +cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy + movifnidn mxyd, r6m ; mx + lea r7, [put_avx512icl] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [r7+wq*2+table_offset(put,)] + add wq, r7 + jmp wq +.put_w2: + movzx r6d, word [srcq+ssq*0] + movzx r7d, word [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6w + mov [dstq+dsq*1], r7w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r6d, [srcq+ssq*0] + mov r7d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6d + mov [dstq+dsq*1], r7d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + mov r6, [srcq+ssq*0] + mov r7, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r6 + mov [dstq+dsq*1], r7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu xmm0, [srcq+ssq*0] + movu xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], xmm0 + mova [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu ym0, [srcq+ssq*0] + movu ym1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], ym0 + mova [dstq+dsq*1], ym1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+ssq*0+64*0] + movu m1, [srcq+ssq*0+64*1] + movu m2, [srcq+ssq*1+64*0] + movu m3, [srcq+ssq*1+64*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+64*0], m0 + mova [dstq+dsq*0+64*1], m1 + mova [dstq+dsq*1+64*0], m2 + mova [dstq+dsq*1+64*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w128 + RET +.h: + ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 + ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 + imul mxyd, 0xff01 + vbroadcasti128 m4, [bilin_h_shuf8] + add mxyd, 16 << 8 + vpbroadcastw m5, mxyd + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .hv + movzx wd, word [r7+wq*2+table_offset(put, _bilin_h)] + vpbroadcastd m3, [pw_2048] + add wq, r7 + jmp wq +.h_w2: + movd xmm0, [srcq+ssq*0] + pinsrd xmm0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pshufb xmm0, xm4 + pmaddubsw xmm0, xm5 + pmulhrsw xmm0, xm3 + packuswb xmm0, xmm0 + pextrw [dstq+dsq*0], xmm0, 0 + pextrw [dstq+dsq*1], xmm0, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + mova xmm4, [bilin_h_shuf4] +.h_w4_loop: + movq xmm0, [srcq+ssq*0] + movhps xmm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xmm0, xmm4 + pmaddubsw xmm0, xm5 + pmulhrsw xmm0, xm3 + packuswb xmm0, xmm0 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+ssq*0] + vinserti32x4 ym0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pshufb ym0, ym4 + pmaddubsw ym0, ym5 + pmulhrsw ym0, ym3 + vpmovuswb xm0, ym0 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + mova m4, [bilin_h_perm16] +.h_w16_loop: + movu ym0, [srcq+ssq*0] + vinserti32x8 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + vpermb m0, m4, m0 + pmaddubsw m0, m5 + pmulhrsw m0, m3 + vpmovuswb ym0, m0 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16_loop + RET +.h_w32: + movu ym0, [srcq+ssq*0+8*0] + vinserti32x8 m0, [srcq+ssq*1+8*0], 1 + movu ym1, [srcq+ssq*0+8*1] + vinserti32x8 m1, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w32 + RET +.h_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + add srcq, ssq + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + movu m0, [srcq+8*0] + movu m2, [srcq+8*1] + movu m1, [srcq+8*8] + movu m6, [srcq+8*9] + add srcq, ssq + REPX {pshufb x, m4}, m0, m2, m1, m6 + REPX {pmaddubsw x, m5}, m0, m2, m1, m6 + REPX {pmulhrsw x, m3}, m0, m2, m1, m6 + packuswb m0, m2 + packuswb m1, m6 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [r7+wq*2+table_offset(put, _bilin_v)] + imul mxyd, 0xff01 + vpbroadcastd m5, [pw_2048] + add mxyd, 16 << 8 + add wq, r7 + vpbroadcastw m4, mxyd + jmp wq +.v_w2: + movd xmm0, [srcq+ssq*0] +.v_w2_loop: + pinsrw xmm1, xmm0, [srcq+ssq*1], 1 ; 0 1 + lea srcq, [srcq+ssq*2] + pinsrw xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1 + pshuflw xmm1, xmm1, q2301 ; 1 0 + punpcklbw xmm1, xmm0, xmm1 + pmaddubsw xmm1, xm4 + pmulhrsw xmm1, xm5 + packuswb xmm1, xmm1 + pextrw [dstq+dsq*0], xmm1, 1 + pextrw [dstq+dsq*1], xmm1, 0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xmm0, [srcq+ssq*0] +.v_w4_loop: + vpbroadcastd xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xmm2, xmm1, xmm0, 0x01 ; 0 1 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm1, xmm0, 0x02 ; 1 2 + punpcklbw xmm1, xmm2 + pmaddubsw xmm1, xm4 + pmulhrsw xmm1, xm5 + packuswb xmm1, xmm1 + movd [dstq+dsq*0], xmm1 + pextrd [dstq+dsq*1], xmm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xmm0, [srcq+ssq*0] +.v_w8_loop: + movq xmm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw xmm1, xmm3, xmm0 + movq xmm0, [srcq+ssq*0] + punpcklbw xmm2, xmm0, xmm3 + pmaddubsw xmm1, xm4 + pmaddubsw xmm2, xm4 + pmulhrsw xmm1, xm5 + pmulhrsw xmm2, xm5 + packuswb xmm1, xmm2 + movq [dstq+dsq*0], xmm1 + movhps [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + movu xmm0, [srcq+ssq*0] +.v_w16_loop: + vbroadcasti128 ymm2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd ymm3, ymm2, ymm0, 0x0f ; 0 1 + vbroadcasti128 ymm0, [srcq+ssq*0] + vpblendd ymm2, ymm2, ymm0, 0xf0 ; 1 2 + punpcklbw ymm1, ymm2, ymm3 + punpckhbw ymm2, ymm3 + pmaddubsw ymm1, ym4 + pmaddubsw ymm2, ym4 + pmulhrsw ymm1, ym5 + pmulhrsw ymm2, ym5 + packuswb ymm1, ymm2 + mova [dstq+dsq*0], xmm1 + vextracti128 [dstq+dsq*1], ymm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + vzeroupper + RET +.v_w32: + movu ym0, [srcq+ssq*0] + kxnorb k1, k1, k1 +.v_w32_loop: + vbroadcasti32x8 m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendmd m3{k1}, m2, m0 ; 0 1 + vbroadcasti32x8 m0, [srcq+ssq*0] + vpblendmd m2{k1}, m0, m2 ; 1 2 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*0], ym1 + vextracti32x8 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w32_loop + RET +.v_w64: + movu m0, [srcq+ssq*0] +.v_w64_loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m1, m3, m0 + punpckhbw m6, m3, m0 + movu m0, [srcq+ssq*0] + pmaddubsw m1, m4 + pmaddubsw m6, m4 + punpcklbw m2, m0, m3 + punpckhbw m7, m0, m3 + pmaddubsw m2, m4 + pmaddubsw m7, m4 + REPX {pmulhrsw x, m5}, m1, m6, m2, m7 + packuswb m1, m6 + packuswb m2, m7 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + movu m0, [srcq+64*0] + movu m1, [srcq+64*1] +.v_w128_loop: + add srcq, ssq + movu m2, [srcq+64*0] + movu m3, [srcq+64*1] + punpcklbw m6, m2, m0 + pmaddubsw m6, m4 + punpckhbw m0, m2, m0 + pmaddubsw m0, m4 + punpcklbw m7, m3, m1 + pmaddubsw m7, m4 + punpckhbw m1, m3, m1 + pmaddubsw m1, m4 + REPX {pmulhrsw x, m5}, m6, m0, m7, m1 + packuswb m6, m0 + mova m0, m2 + packuswb m7, m1 + mova m1, m3 + mova [dstq+64*0], m6 + mova [dstq+64*1], m7 + add dstq, dsq + dec hd + jg .v_w128_loop + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 + ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 + movzx wd, word [r7+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 ; can't shift by 12 due to signed overflow + vpbroadcastd m7, [pw_2048] + add wq, r7 + vpbroadcastw m6, mxyd + jmp wq +.hv_w2: + vpbroadcastd xmm0, [srcq+ssq*0] + pshufb xmm0, xm4 + pmaddubsw xmm0, xm5 +.hv_w2_loop: + movd xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pinsrd xmm1, [srcq+ssq*0], 1 + pshufb xmm1, xm4 + pmaddubsw xmm1, xm5 ; 1 _ 2 _ + shufps xmm2, xmm0, xmm1, q1032 ; 0 _ 1 _ + mova xmm0, xmm1 + psubw xmm1, xmm2 + paddw xmm1, xmm1 + pmulhw xmm1, xm6 + paddw xmm1, xmm2 + pmulhrsw xmm1, xm7 + packuswb xmm1, xmm1 + pextrw [dstq+dsq*0], xmm1, 0 + pextrw [dstq+dsq*1], xmm1, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova xmm4, [bilin_h_shuf4] + movddup xmm0, [srcq+ssq*0] + pshufb xmm0, xmm4 + pmaddubsw xmm0, xm5 +.hv_w4_loop: + movq xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xmm1, [srcq+ssq*0] + pshufb xmm1, xmm4 + pmaddubsw xmm1, xm5 ; 1 2 + shufps xmm2, xmm0, xmm1, q1032 ; 0 1 + mova xmm0, xmm1 + psubw xmm1, xmm2 + paddw xmm1, xmm1 + pmulhw xmm1, xm6 + paddw xmm1, xmm2 + pmulhrsw xmm1, xm7 + packuswb xmm1, xmm1 + movd [dstq+dsq*0], xmm1 + pextrd [dstq+dsq*1], xmm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti128 ym0, [srcq+ssq*0] + pshufb ym0, ym4 + pmaddubsw ym0, ym5 +.hv_w8_loop: + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 ym1, [srcq+ssq*0], 1 + pshufb ym1, ym4 + pmaddubsw ym1, ym5 ; 1 2 + valignq ym2, ym1, ym0, 2 + mova ym0, ym1 + psubw ym1, ym2 + paddw ym1, ym1 + pmulhw ym1, ym6 + paddw ym1, ym2 + pmulhrsw ym1, ym7 + vpmovuswb xm1, ym1 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + vbroadcasti32x8 m0, [srcq+ssq*0] + mova m4, [bilin_h_perm16] + vpermb m0, m4, m0 + pmaddubsw m0, m5 +.hv_w16_loop: + movu ym1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x8 m1, [srcq+ssq*0], 1 + vpermb m1, m4, m1 + pmaddubsw m1, m5 ; 1 2 + valignq m2, m1, m0, 4 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + vpmovuswb ym1, m1 + mova [dstq+dsq*0], xm1 + vextracti32x4 [dstq+dsq*1], ym1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+ssq*0] + pmovzxbq m8, [pb_02461357] + pmaddubsw m0, m5 +.hv_w32_loop: + vpermb m2, m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpermb m3, m4, [srcq+ssq*0] + pmaddubsw m2, m5 + psubw m1, m2, m0 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m0 + pmaddubsw m0, m3, m5 + psubw m3, m0, m2 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m2 + pmulhrsw m1, m7 + pmulhrsw m3, m7 + packuswb m1, m3 + vpermq m1, m8, m1 + mova [dstq+dsq*0], ym1 + vextracti32x8 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w32_loop + RET +.hv_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w64_loop: + add srcq, ssq + movu m2, [srcq+8*0] + movu m3, [srcq+8*1] + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + psubw m8, m2, m0 + psubw m9, m3, m1 + paddw m8, m8 + pmulhw m8, m6 + paddw m9, m9 + pmulhw m9, m6 + paddw m8, m0 + pmulhrsw m8, m7 + paddw m9, m1 + pmulhrsw m9, m7 + mova m0, m2 + mova m1, m3 + packuswb m8, m9 + mova [dstq], m8 + add dstq, dsq + dec hd + jg .hv_w64_loop + RET +.hv_w128: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + movu m2, [srcq+8*8] + movu m3, [srcq+8*9] + REPX {pshufb x, m4}, m0, m1, m2, m3 + REPX {pmaddubsw x, m5}, m0, m1, m2, m3 +.hv_w128_loop: + add srcq, ssq + movu m8, [srcq+8*0] + movu m9, [srcq+8*1] + movu m10, [srcq+8*8] + movu m11, [srcq+8*9] + REPX {pshufb x, m4}, m8, m9, m10, m11 + REPX {pmaddubsw x, m5}, m8, m9, m10, m11 + psubw m12, m8, m0 + psubw m13, m9, m1 + psubw m14, m10, m2 + psubw m15, m11, m3 + paddw m12, m12 + pmulhw m12, m6 + paddw m13, m13 + pmulhw m13, m6 + paddw m14, m14 + pmulhw m14, m6 + paddw m15, m15 + pmulhw m15, m6 + paddw m12, m0 + pmulhrsw m12, m7 + paddw m13, m1 + pmulhrsw m13, m7 + paddw m14, m2 + pmulhrsw m14, m7 + paddw m15, m3 + pmulhrsw m15, m7 + mova m0, m8 + mova m1, m9 + mova m2, m10 + mova m3, m11 + packuswb m12, m13 + packuswb m14, m15 + mova [dstq+64*0], m12 + mova [dstq+64*1], m14 + add dstq, dsq + dec hd + jg .hv_w128_loop + RET + +DECLARE_REG_TMP 3, 5, 6 + +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea t2, [prep_avx512icl] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [t2+wq*2+table_offset(prep,)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd xmm0, [srcq+strideq*0] + pinsrd xmm0, [srcq+strideq*1], 1 + pinsrd xmm0, [srcq+strideq*2], 2 + pinsrd xmm0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pmovzxbw ym0, xmm0 + psllw ym0, 4 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq xmm0, [srcq+strideq*0] + movq xmm1, [srcq+strideq*1] + vinserti128 ym0, ymm0, [srcq+strideq*2], 1 + vinserti128 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1 + pmovzxbw m0, ym0 + psllw m0, 4 + mova [tmpq], m0 + add tmpq, 32*2 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movu xmm0, [srcq+strideq*0] + vinserti128 ym0, ymm0, [srcq+strideq*1], 1 + movu xmm1, [srcq+strideq*2] + vinserti128 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pmovzxbw m0, ym0 + pmovzxbw m1, ym1 + psllw m0, 4 + psllw m1, 4 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmovzxbw m0, [srcq+strideq*0] + pmovzxbw m1, [srcq+strideq*1] + pmovzxbw m2, [srcq+strideq*2] + pmovzxbw m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 4 + jg .prep_w32 + RET +.prep_w64: + pmovzxbw m0, [srcq+strideq*0+32*0] + pmovzxbw m1, [srcq+strideq*0+32*1] + pmovzxbw m2, [srcq+strideq*1+32*0] + pmovzxbw m3, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 2 + jg .prep_w64 + RET +.prep_w128: + pmovzxbw m0, [srcq+32*0] + pmovzxbw m1, [srcq+32*1] + pmovzxbw m2, [srcq+32*2] + pmovzxbw m3, [srcq+32*3] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + add srcq, strideq + dec hd + jg .prep_w128 + RET +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + vpbroadcastw m5, mxyd + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .hv + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + vbroadcasti32x4 ym4, [bilin_h_shuf4] +.h_w4_loop: + movq xmm0, [srcq+strideq*0] + movq xmm1, [srcq+strideq*1] + vinserti32x4 ym0, ymm0, [srcq+strideq*2], 1 + vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1 + pshufb ym0, ym4 + pmaddubsw ym0, ym5 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + vbroadcasti32x4 m4, [bilin_h_shuf8] +.h_w8_loop: + movu xmm0, [srcq+strideq*0] + vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1 + vinserti32x4 m0, [srcq+strideq*2], 2 + vinserti32x4 m0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pmaddubsw m0, m5 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + mova m4, [bilin_h_perm16] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + movu ym1, [srcq+strideq*2] + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpermb m0, m4, m0 + vpermb m1, m4, m1 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 64*2 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: + mova m4, [bilin_h_perm32] +.h_w32_loop: + vpermb m0, m4, [srcq+strideq*0] + vpermb m1, m4, [srcq+strideq*1] + vpermb m2, m4, [srcq+strideq*2] + vpermb m3, m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 4 + jg .h_w32_loop + RET +.h_w64: + mova m4, [bilin_h_perm32] +.h_w64_loop: + vpermb m0, m4, [srcq+strideq*0+32*0] + vpermb m1, m4, [srcq+strideq*0+32*1] + vpermb m2, m4, [srcq+strideq*1+32*0] + vpermb m3, m4, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 2 + jg .h_w64_loop + RET +.h_w128: + mova m4, [bilin_h_perm32] +.h_w128_loop: + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + add srcq, strideq + dec hd + jg .h_w128_loop + RET +.v: + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + add wq, t2 + lea stride3q, [strideq*3] + vpbroadcastw m6, mxyd + jmp wq +.v_w4: + vpbroadcastd xm0, [srcq+strideq*0] + mov r3d, 0x29 + vbroadcasti32x4 ym3, [bilin_v_shuf4] + kmovb k1, r3d +.v_w4_loop: + vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____ + vpbroadcastd ym2, [srcq+strideq*2] + vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__ + lea srcq, [srcq+strideq*4] + vpbroadcastd ym0, [srcq+strideq*0] + punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_ + pshufb ym2, ym3 + pmaddubsw ym2, ym6 + mova [tmpq], ym2 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + mova m5, [bilin_v_perm8] + vbroadcasti32x4 ym0, [srcq+strideq*0] +.v_w8_loop: + vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 + vpbroadcastq ym0, [srcq+strideq*2] + vinserti32x4 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti32x4 ym0, [srcq+strideq*0], 0 + vpermt2b m1, m5, m0 + pmaddubsw m1, m6 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + mova m5, [bilin_v_perm16] + movu xm0, [srcq+strideq*0] +.v_w16_loop: + movu xm2, [srcq+strideq*2] + vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 + vpermt2b m1, m5, m2 + vinserti32x4 ym2, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + movu xm0, [srcq+strideq*0] + vpermt2b m2, m5, m0 + pmaddubsw m1, m6 + pmaddubsw m2, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + mova m5, [bilin_v_perm32] + movu ym0, [srcq+strideq*0] +.v_w32_loop: + movu ym2, [srcq+strideq*1] + movu ym3, [srcq+strideq*2] + movu ym4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpermt2b m0, m5, m2 + vpermt2b m2, m5, m3 + vpermt2b m3, m5, m4 + pmaddubsw m1, m0, m6 + movu ym0, [srcq+strideq*0] + vpermt2b m4, m5, m0 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m4, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m4 + add tmpq, 64*4 + sub hd, 4 + jg .v_w32_loop + RET +.v_w64: + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0] +.v_w64_loop: + vpermq m1, m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m1, m0 + punpckhbw m2, m1, m0 + vpermq m0, m5, [srcq+strideq*0] + punpcklbw m3, m0, m1 + punpckhbw m1, m0, m1 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m1, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m1 + add tmpq, 64*4 + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] +.v_w128_loop: + vpermq m2, m5, [srcq+strideq*1+ 0] + vpermq m3, m5, [srcq+strideq*1+64] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m2, m0 + punpckhbw m0, m2, m0 + pmaddubsw m4, m6 + pmaddubsw m0, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m0 + punpcklbw m4, m3, m1 + punpckhbw m1, m3, m1 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + mova [tmpq+64*2], m4 + mova [tmpq+64*3], m1 + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] + punpcklbw m4, m0, m2 + punpckhbw m2, m0, m2 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + mova [tmpq+64*4], m4 + mova [tmpq+64*5], m2 + punpcklbw m4, m1, m3 + punpckhbw m3, m1, m3 + pmaddubsw m4, m6 + pmaddubsw m3, m6 + mova [tmpq+64*6], m4 + mova [tmpq+64*7], m3 + add tmpq, 64*8 + sub hd, 2 + jg .v_w128_loop + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + vpbroadcastw m6, mxyd + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.hv_w4: + vbroadcasti32x4 ym4, [bilin_h_shuf4] + vpbroadcastq ym0, [srcq+strideq*0] + pshufb ym0, ym4 + pmaddubsw ym0, ym5 +.hv_w4_loop: + movq xmm1, [srcq+strideq*1] + movq xmm2, [srcq+strideq*2] + vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vinserti32x4 ym2, ymm2, [srcq+strideq*0], 1 + punpcklqdq ym1, ym2 + pshufb ym1, ym4 + pmaddubsw ym1, ym5 ; 1 2 3 4 + valignq ym2, ym1, ym0, 3 ; 0 1 2 3 + mova ym0, ym1 + psubw ym1, ym2 + pmulhrsw ym1, ym6 + paddw ym1, ym2 + mova [tmpq], ym1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti32x4 m4, [bilin_h_shuf8] + vbroadcasti32x4 m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xmm1, [srcq+strideq*1] + vinserti128 ym1, ymm1, [srcq+strideq*2], 1 + vinserti128 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti128 m1, [srcq+strideq*0], 3 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 3 4 + valignq m2, m1, m0, 6 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + mova m4, [bilin_h_perm16] + vbroadcasti32x8 m0, [srcq+strideq*0] + vpermb m0, m4, m0 + pmaddubsw m0, m5 +.hv_w16_loop: + movu ym1, [srcq+strideq*1] + vinserti32x8 m1, [srcq+strideq*2], 1 + movu ym2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti32x8 m2, [srcq+strideq*0], 1 + vpermb m1, m4, m1 + vpermb m2, m4, m2 + pmaddubsw m1, m5 ; 1 2 + vshufi32x4 m3, m0, m1, q1032 ; 0 1 + pmaddubsw m0, m2, m5 ; 3 4 + vshufi32x4 m2, m1, m0, q1032 ; 2 3 + psubw m1, m3 + pmulhrsw m1, m6 + paddw m1, m3 + psubw m3, m0, m2 + pmulhrsw m3, m6 + paddw m3, m2 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m3 + add tmpq, 64*2 + sub hd, 4 + jg .hv_w16_loop + RET +.hv_w32: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+strideq*0] + pmaddubsw m0, m5 +.hv_w32_loop: + vpermb m1, m4, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermb m2, m4, [srcq+strideq*0] + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+64*0], m3 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 2 + jg .hv_w32_loop + RET +.hv_w64: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w64_loop: + add srcq, strideq + vpermb m2, m4, [srcq+32*0] + vpermb m3, m4, [srcq+32*1] + pmaddubsw m2, m5 + pmaddubsw m3, m5 + psubw m7, m2, m0 + psubw m8, m3, m1 + pmulhrsw m7, m6 + pmulhrsw m8, m6 + paddw m7, m0 + mova m0, m2 + paddw m8, m1 + mova m1, m3 + mova [tmpq+64*0], m7 + mova [tmpq+64*1], m8 + add tmpq, 64*2 + dec hd + jg .hv_w64_loop + RET +.hv_w128: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] + REPX {pmaddubsw x, m5}, m0, m1, m2, m3 +.hv_w128_loop: + add srcq, strideq + vpermb m7, m4, [srcq+32*0] + vpermb m8, m4, [srcq+32*1] + vpermb m9, m4, [srcq+32*2] + vpermb m10, m4, [srcq+32*3] + REPX {pmaddubsw x, m5}, m7, m8, m9, m10 + psubw m11, m7, m0 + psubw m12, m8, m1 + psubw m13, m9, m2 + psubw m14, m10, m3 + REPX {pmulhrsw x, m6}, m11, m12, m13, m14 + paddw m11, m0 + mova m0, m7 + paddw m12, m1 + mova m1, m8 + paddw m13, m2 + mova m2, m9 + paddw m14, m3 + mova m3, m10 + mova [tmpq+64*0], m11 + mova [tmpq+64*1], m12 + mova [tmpq+64*2], m13 + mova [tmpq+64*3], m14 + add tmpq, 64*4 + dec hd + jg .hv_w128_loop + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; fn, type, type_h, type_v +cglobal %1_%2_8bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) +%endif +%endmacro + +%macro PUT_8TAP_H 4-5 0 ; dst/src, tmp[1-3], vpermb +%if %5 + vpermb m%2, m6, m%1 + vpermb m%3, m7, m%1 + vpermb m%4, m8, m%1 +%else +%if %2 < %4 ; reuse a previous value if possible + pshufb m%2, m%1, m6 +%endif + pshufb m%3, m%1, m7 + pshufb m%4, m%1, m8 +%endif + mova m%1, m5 + vpdpbusd m%1, m%2, m9 + mova m%2, m5 + vpdpbusd m%2, m%3, m9 + vpdpbusd m%1, m%3, m10 + vpdpbusd m%2, m%4, m10 + packusdw m%1, m%2 + psrlw m%1, 6 +%endmacro + +%if WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +%define PUT_8TAP_FN FN put_8tap, + +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR + +cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +%define base r8-put_avx512icl + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx512icl] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r8+wq*2+table_offset(put,)] + add wq, r8 + lea r6, [ssq*3] + lea r7, [dsq*3] +%if WIN64 + pop r8 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [pd_34] ; 2 + (8 << 2) + WIN64_SPILL_XMM 11 + cmp wd, 4 + jl .h_w2 + vbroadcasti128 m6, [subpel_h_shufA] + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m7, [subpel_h_shufB] + vbroadcasti128 m8, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] + vpbroadcastd m9, [base+mxq*8+subpel_filters+0] + vpbroadcastd m10, [base+mxq*8+subpel_filters+4] + add wq, r8 + jmp wq +.h_w2: + movzx mxd, mxb + dec srcq + mova xmm4, [subpel_h_shuf4] + vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] +.h_w2_loop: + movq xmm0, [srcq+ssq*0] + movhps xmm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xmm0, xmm4 + mova xmm1, xm5 + vpdpbusd xmm1, xmm0, xmm3 + packssdw xmm0, xmm1, xmm1 + psraw xmm0, 6 + packuswb xmm0, xm0 + pextrw [dstq+dsq*0], xmm0, 0 + pextrw [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd xmm3, [base+mxq*8+subpel_filters+2] +.h_w4_loop: + movq xmm0, [srcq+ssq*0] + movq xmm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xmm0, xm6 + pshufb xmm1, xm6 + mova xmm2, xm5 + vpdpbusd xmm2, xmm0, xmm3 + mova xmm0, xm5 + vpdpbusd xmm0, xmm1, xmm3 + packssdw xmm0, xmm2, xmm0 + psraw xmm0, 6 + packuswb xmm0, xmm0 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+ssq*0] + vinserti32x4 ym0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + WRAP_YMM PUT_8TAP_H 0, 1, 2, 3 + vpmovuswb xm0, ym0 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + mova m6, [spel_h_perm16a] + mova m7, [spel_h_perm16b] + mova m8, [spel_h_perm16c] +.h_w16_loop: + movu ym0, [srcq+ssq*0] + vinserti32x8 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 1, 2, 3, 1 + vpmovuswb ym0, m0 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], ym0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16_loop + RET +.h_w32: + movu ym0, [srcq+ssq*0+8*0] + vinserti32x8 m0, [srcq+ssq*1+8*0], 1 + movu ym1, [srcq+ssq*0+8*1] + vinserti32x8 m1, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 4, 3, 2 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w32 + RET +.h_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 4, 3, 2 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + movu m0, [srcq+8*0] + movu m2, [srcq+8*1] + movu m1, [srcq+8*8] + movu m3, [srcq+8*9] + add srcq, ssq + PUT_8TAP_H 0, 4, 11, 12 + PUT_8TAP_H 2, 12, 11, 4 + PUT_8TAP_H 1, 4, 11, 12 + PUT_8TAP_H 3, 12, 11, 4 + packuswb m0, m2 + packuswb m1, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + tzcnt r6d, wd + movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] + vpbroadcastd m7, [pw_512] + lea myq, [base+subpel_filters+myq*8] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + add r6, r8 + lea ss3q, [ssq*3] + sub srcq, ss3q + jmp r6 +.v_w2: + movd xmm2, [srcq+ssq*0] + pinsrw xmm2, [srcq+ssq*1], 2 + pinsrw xmm2, [srcq+ssq*2], 4 + add srcq, ss3q + pinsrw xmm2, [srcq+ssq*0], 6 ; 0 1 2 3 + movd xmm3, [srcq+ssq*1] + vpbroadcastd xmm1, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5 + vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6 + palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4 + punpcklbw xmm3, xmm1 ; 45 56 + punpcklbw xmm1, xmm2, xmm4 ; 01 12 + punpckhbw xmm2, xmm4 ; 23 34 +.v_w2_loop: + pmaddubsw xmm5, xmm1, xm8 ; a0 b0 + mova xmm1, xmm2 + pmaddubsw xmm2, xm9 ; a1 b1 + paddw xmm5, xmm2 + mova xmm2, xmm3 + pmaddubsw xmm3, xm10 ; a2 b2 + paddw xmm5, xmm3 + vpbroadcastd xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8 + punpcklbw xmm3, xmm4 ; 67 78 + pmaddubsw xmm4, xmm3, xm11 ; a3 b3 + paddw xmm5, xmm4 + pmulhrsw xmm5, xm7 + packuswb xmm5, xmm5 + pextrw [dstq+dsq*0], xmm5, 0 + pextrw [dstq+dsq*1], xmm5, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xmm2, [srcq+ssq*0] + pinsrd xmm2, [srcq+ssq*1], 1 + pinsrd xmm2, [srcq+ssq*2], 2 + add srcq, ss3q + pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3 + movd xmm3, [srcq+ssq*1] + vpbroadcastd xmm1, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm3, xmm3, xmm1, 0x02 ; 4 5 + vpblendd xmm1, xmm1, xmm0, 0x02 ; 5 6 + palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4 + punpcklbw xmm3, xmm1 ; 45 56 + punpcklbw xmm1, xmm2, xmm4 ; 01 12 + punpckhbw xmm2, xmm4 ; 23 34 +.v_w4_loop: + vpbroadcastd xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw xmm5, xmm1, xm8 ; a0 b0 + mova xmm1, xmm2 + pmaddubsw xmm2, xm9 ; a1 b1 + paddw xmm5, xmm2 + mova xmm2, xmm3 + pmaddubsw xmm3, xm10 ; a2 b2 + paddw xmm5, xmm3 + vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7 + vpbroadcastd xmm0, [srcq+ssq*0] + vpblendd xmm4, xmm4, xmm0, 0x02 ; 7 8 + punpcklbw xmm3, xmm4 ; 67 78 + pmaddubsw xmm4, xmm3, xm11 ; a3 b3 + paddw xmm5, xmm4 + pmulhrsw xmm5, xm7 + packuswb xmm5, xmm5 + movd [dstq+dsq*0], xmm5 + pextrd [dstq+dsq*1], xmm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xmm1, [srcq+ssq*0] + vpbroadcastq ymm0, [srcq+ssq*1] + vpbroadcastq ymm2, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq ymm5, [srcq+ssq*0] + vpbroadcastq ymm3, [srcq+ssq*1] + vpbroadcastq ymm4, [srcq+ssq*2] + add srcq, ss3q + vpblendd ymm1, ymm0, 0x30 + vpblendd ymm0, ymm2, 0x30 + punpcklbw ymm1, ymm0 ; 01 12 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm2, ymm5, 0x30 + vpblendd ymm5, ymm3, 0x30 + punpcklbw ymm2, ymm5 ; 23 34 + vpblendd ymm3, ymm4, 0x30 + vpblendd ymm4, ymm0, 0x30 + punpcklbw ymm3, ymm4 ; 45 56 +.v_w8_loop: + vpbroadcastq ymm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw ymm5, ymm1, ym8 ; a0 b0 + mova ymm1, ymm2 + pmaddubsw ymm2, ym9 ; a1 b1 + paddw ymm5, ymm2 + mova ymm2, ymm3 + pmaddubsw ymm3, ym10 ; a2 b2 + paddw ymm5, ymm3 + vpblendd ymm3, ymm0, ymm4, 0x30 + vpbroadcastq ymm0, [srcq+ssq*0] + vpblendd ymm4, ymm4, ymm0, 0x30 + punpcklbw ymm3, ymm4 ; 67 78 + pmaddubsw ymm4, ymm3, ym11 ; a3 b3 + paddw ymm5, ymm4 + pmulhrsw ymm5, ym7 + vextracti128 xmm4, ymm5, 1 + packuswb xmm5, xmm4 + movq [dstq+dsq*0], xmm5 + movhps [dstq+dsq*1], xmm5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + vzeroupper + RET +.v_w16: + mova m12, [spel_v_perm16] + vbroadcasti32x4 m1, [srcq+ssq*0] + vbroadcasti32x4 ym4, [srcq+ssq*1] + mov r6d, 0x0f + vbroadcasti32x4 m2, [srcq+ssq*2] + add srcq, ss3q + vbroadcasti32x4 ym5, [srcq+ssq*0] + kmovb k1, r6d + vbroadcasti32x4 m3, [srcq+ssq*1] + vbroadcasti32x4 ym6, [srcq+ssq*2] + add srcq, ss3q + vbroadcasti32x4 m0, [srcq+ssq*0] + vshufpd m1{k1}, m4, m2, 0xcc + vshufpd m2{k1}, m5, m3, 0xcc + vshufpd m3{k1}, m6, m0, 0xcc + vpermb m1, m12, m1 ; 01 12 + vpermb m2, m12, m2 ; 23 34 + vpermb m3, m12, m3 ; 45 56 +.v_w16_loop: + pmaddubsw m4, m1, m8 ; a0 b0 + mova m1, m2 + pmaddubsw m5, m2, m9 ; a1 b1 + mova m2, m3 + pmaddubsw m6, m3, m10 ; a2 b2 + mova m3, m0 + paddw m4, m5 + vbroadcasti32x4 ym5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m0, [srcq+ssq*0] + vshufpd m3{k1}, m5, m0, 0xcc + vpermb m3, m12, m3 ; 67 78 + pmaddubsw m5, m3, m11 ; a3 b3 + paddw m4, m6 + paddw m4, m5 + pmulhrsw m4, m7 + vextracti32x8 ym5, m4, 1 + packuswb ym4, ym5 + mova [dstq+dsq*0], xm4 + vextracti32x4 [dstq+dsq*1], ym4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: + mova m12, [spel_v_perm32] + pmovzxbq m14, [pb_02461357] + vpshrdw m13, m12, m12, 8 + movu ym0, [srcq+ssq*0] + vinserti32x8 m0, [srcq+ssq*1], 1 + vpermb m1, m12, m0 ; 01 + vinserti32x8 m0, [srcq+ssq*2], 0 + add srcq, ss3q + vpermb m2, m13, m0 ; 12 + vinserti32x8 m0, [srcq+ssq*0], 1 + vpermb m3, m12, m0 ; 23 + vinserti32x8 m0, [srcq+ssq*1], 0 + vpermb m4, m13, m0 ; 34 + vinserti32x8 m0, [srcq+ssq*2], 1 + add srcq, ss3q + vpermb m5, m12, m0 ; 45 + vinserti32x8 m0, [srcq+ssq*0], 0 + vpermb m6, m13, m0 ; 56 +.v_w32_loop: + vinserti32x8 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pmaddubsw m15, m1, m8 + mova m1, m3 + pmaddubsw m16, m2, m8 + mova m2, m4 + pmaddubsw m17, m3, m9 + mova m3, m5 + pmaddubsw m18, m4, m9 + mova m4, m6 + pmaddubsw m19, m5, m10 + vpermb m5, m12, m0 ; 67 + vinserti32x8 m0, [srcq+ssq*0], 0 + pmaddubsw m20, m6, m10 + vpermb m6, m13, m0 ; 78 + paddw m15, m17 + pmaddubsw m17, m5, m11 + paddw m16, m18 + pmaddubsw m18, m6, m11 + paddw m15, m19 + paddw m16, m20 + paddw m15, m17 + paddw m16, m18 + pmulhrsw m15, m7 + pmulhrsw m16, m7 + packuswb m15, m16 + vpermq m15, m14, m15 + mova [dstq+dsq*0], ym15 + vextracti32x8 [dstq+dsq*1], m15, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w32_loop + vzeroupper + RET +.v_w64: +.v_w128: + lea r6d, [hq+wq*4-256] + mov r4, srcq + mov r7, dstq +.v_loop0: + movu m2, [srcq+ssq*0] + movu m4, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + add srcq, ss3q + movu m13, [srcq+ssq*0] + movu m15, [srcq+ssq*1] + movu m17, [srcq+ssq*2] + add srcq, ss3q + movu m0, [srcq+ssq*0] + punpcklbw m1, m2, m4 ; 01l + punpckhbw m2, m4 ; 01h + punpcklbw m3, m4, m6 ; 12l + punpckhbw m4, m6 ; 12h + punpcklbw m5, m6, m13 ; 23l + punpckhbw m6, m13 ; 23h + punpcklbw m12, m13, m15 ; 34l + punpckhbw m13, m15 ; 34h + punpcklbw m14, m15, m17 ; 45l + punpckhbw m15, m17 ; 45h + punpcklbw m16, m17, m0 ; 56l + punpckhbw m17, m0 ; 56h +.v_loop: + pmaddubsw m18, m1, m8 ; a0l + mova m1, m5 + pmaddubsw m19, m2, m8 ; a0h + mova m2, m6 + pmaddubsw m20, m3, m8 ; b0l + mova m3, m12 + pmaddubsw m21, m4, m8 ; b0h + mova m4, m13 + pmaddubsw m5, m9 ; a1l + pmaddubsw m6, m9 ; a1h + pmaddubsw m12, m9 ; b1l + pmaddubsw m13, m9 ; b1h + paddw m18, m5 + mova m5, m14 + pmaddubsw m14, m10 ; a2l + paddw m19, m6 + mova m6, m15 + pmaddubsw m15, m10 ; a2h + paddw m20, m12 + mova m12, m16 + pmaddubsw m16, m10 ; b2l + paddw m21, m13 + mova m13, m17 + pmaddubsw m17, m10 ; b2h + paddw m18, m14 + paddw m19, m15 + paddw m20, m16 + paddw m21, m17 + movu m17, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m14, m0, m17 ; 67l + punpckhbw m15, m0, m17 ; 67h + pmaddubsw m16, m14, m11 ; a3l + pmaddubsw m0, m15, m11 ; a3h + paddw m18, m16 + paddw m19, m0 + movu m0, [srcq+ssq*0] + punpcklbw m16, m17, m0 ; 78l + punpckhbw m17, m0 ; 78h + pmulhrsw m18, m7 + pmulhrsw m19, m7 + packuswb m18, m19 + mova [dstq+dsq*0], m18 + pmaddubsw m18, m16, m11 ; b3l + pmaddubsw m19, m17, m11 ; b3h + paddw m18, m20 + paddw m19, m21 + pmulhrsw m18, m7 + pmulhrsw m19, m7 + packuswb m18, m19 + mova [dstq+dsq*1], m18 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_loop + add r4, 64 + add r7, 64 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 256 + jg .v_loop0 + vzeroupper + RET +.hv: + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + dec srcq + vpbroadcastd m7, [base+subpel_filters+mxq*8+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastd m8, [pd_2] + vpbroadcastq ym0, [base+subpel_filters+myq*8] + lea ss3q, [ssq*3] + vpbroadcastd ym9, [pd_32768] + mov r6, srcq + punpcklbw ym0, ym8, ym0 + sub r6, ss3q + psraw ym0, 2 ; << 6 + mova xm14, [spel_hv_end] + pshufd ym10, ym0, q0000 + pshufd ym11, ym0, q1111 + pshufd ym12, ym0, q2222 + pshufd ym13, ym0, q3333 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 ym6, [subpel_h_shuf4] + movq xmm2, [r6+ssq*0] + movhps xmm2, [r6+ssq*1] + movq xmm0, [r6+ssq*2] + movhps xmm0, [srcq+ssq*0] + vpbroadcastq ymm3, [srcq+ssq*1] + vpbroadcastq ymm4, [srcq+ssq*2] + add srcq, ss3q + vpbroadcastq ymm1, [srcq+ssq*0] + vpblendd ymm2, ymm3, 0x30 + vpblendd ymm0, ymm1, 0x30 ; 2 3 6 _ + vpblendd ymm2, ymm4, 0xc0 ; 0 1 4 5 + pshufb ymm2, ym6 + pshufb ymm0, ym6 + mova ymm1, ym8 + vpdpbusd ymm1, ymm2, ym7 + mova ymm2, ym8 + vpdpbusd ymm2, ymm0, ym7 + packssdw ymm2, ymm1, ymm2 + psraw ymm2, 2 + vextracti128 xmm3, ymm2, 1 + palignr xmm4, xmm3, xmm2, 4 + punpcklwd xmm1, xmm2, xmm4 ; 01 12 + punpckhwd xmm2, xmm4 ; 23 34 + pshufd xmm0, xmm3, q2121 + punpcklwd xmm3, xmm0 ; 45 56 +.hv_w2_loop: + movq xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xmm4, [srcq+ssq*0] + mova xmm5, xm9 + vpdpwssd xmm5, xmm1, xm10 ; a0 b0 + mova xmm1, xmm2 + vpdpwssd xmm5, xmm2, xm11 ; a1 b1 + pshufb xmm4, xm6 + mova xmm2, xmm3 + vpdpwssd xmm5, xmm3, xm12 ; a2 b2 + mova xmm3, xm8 + vpdpbusd xmm3, xmm4, xm7 + packssdw xmm4, xmm3, xmm3 + psraw xmm4, 2 + palignr xmm3, xmm4, xmm0, 12 + mova xmm0, xmm4 + punpcklwd xmm3, xmm4 ; 67 78 + vpdpwssd xmm5, xmm3, xm13 ; a3 b3 + packuswb xmm5, xmm5 + pshufb xmm5, xm14 + pextrw [dstq+dsq*0], xmm5, 0 + pextrw [dstq+dsq*1], xmm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + vzeroupper + RET +.hv_w4: + movq xmm1, [r6+ssq*0] + vpbroadcastq ym2, [r6+ssq*1] + vinserti32x4 ym1, ymm1, [r6+ssq*2], 1 + vinserti32x4 m2, [srcq+ssq*0], 2 + vinserti32x4 m1, [srcq+ssq*1], 2 + vinserti32x4 m2, [srcq+ssq*2], 3 ; _ 1 3 5 + vbroadcasti32x4 m6, [subpel_h_shufA] + add srcq, ss3q + vinserti32x4 m1, [srcq+ssq*0], 3 ; 0 2 4 6 + pshufb m2, m6 + pshufb m1, m6 + mova m0, m8 + vpdpbusd m0, m2, m7 + mova m4, m8 + vpdpbusd m4, m1, m7 + mova ym1, [spel_hv_perm4a] + mova ym2, [spel_hv_perm4b] + mova ym3, [spel_hv_perm4c] + packssdw m0, m4 + psraw m0, 2 ; _ 0 1 2 3 4 5 6 + mov r6d, 0x5555 + vpermb ym1, ym1, ym0 ; 01 12 + vpermb m2, m2, m0 ; 23 34 + vpermb m3, m3, m0 ; 45 56 + kmovw k1, r6d + mova ym15, [spel_hv_perm4d] +.hv_w4_loop: + movq xmm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x4 ym4, ymm4, [srcq+ssq*0], 1 + mova ym5, ym9 + vpdpwssd ym5, ym1, ym10 ; a0 b0 + mova ym1, ym2 + pshufb ym4, ym6 + mova ym0, ym8 + vpdpbusd ym0, ym4, ym7 + vpdpwssd ym5, ym2, ym11 ; a1 b1 + mova ym2, ym3 + vpdpwssd ym5, ym3, ym12 ; a2 b2 + vpsraw ym3{k1}, ym0, 2 ; 7 8 + vpermb ym3, ym15, ym3 ; 67 78 + vpdpwssd ym5, ym3, ym13 ; a3 b3 + packuswb ym5, ym5 + vpermb ym5, ym14, ym5 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [base+subpel_filters+mxq*8+0] + vpbroadcastd m11, [base+subpel_filters+mxq*8+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastd m8, [pd_2] + vpbroadcastq m0, [base+subpel_filters+myq*8] + vpbroadcastd m9, [pd_32768] + punpcklbw m0, m8, m0 + lea ss3q, [ssq*3] + psraw m0, 2 ; << 6 + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + cmp wd, 8 + jne .hv_w16 + mov r6, srcq + sub r6, ss3q + movu xmm1, [r6+ssq*0] + vinserti128 ymm1, [r6+ssq*1], 1 + movu xmm2, [srcq+ssq*1] + vinserti32x4 m6, zmm1, [r6+ssq*2], 2 + vinserti128 ymm2, [srcq+ssq*2], 1 + vinserti32x4 m6, [srcq+ssq*0], 3 ; 0 1 2 3 + add srcq, ss3q + vbroadcasti32x4 m4, [subpel_h_shufA] + vinserti32x4 m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _ + vbroadcasti32x4 m7, [subpel_h_shufB] + vbroadcasti32x4 m17, [subpel_h_shufC] + pshufb m1, m6, m4 ; 0 1 2 3 0123 + mova m2, m8 + vpdpbusd m2, m1, m10 + pshufb m5, m6, m7 ; 0 1 2 3 4567 + mova m1, m8 + vpdpbusd m1, m5, m10 + pshufb m4, m0, m4 ; 4 5 6 _ 0123 + mova m3, m8 + vpdpbusd m3, m4, m10 + pshufb m7, m0, m7 ; 4 5 6 _ 4567 + mova m4, m8 + vpdpbusd m4, m7, m10 + pshufb m6, m17 + vpdpbusd m2, m5, m11 + vpdpbusd m1, m6, m11 + pshufb m6, m0, m17 + vpdpbusd m3, m7, m11 + vpdpbusd m4, m6, m11 + mova m5, [spel_hv_perm8a] + mova m0, [spel_hv_perm8b] + mov r6, 0x55555555ff00 + packssdw m2, m1 + packssdw m3, m4 + mova m18, [spel_hv_perm8c] + psraw m2, 2 ; 0 1 2 3 + psraw m3, 2 ; 4 5 6 _ + vpermb m1, m5, m2 ; 01 12 + vbroadcasti32x8 m6, [subpel_h_shufA] + kmovq k1, r6 + vpermt2b m2, m0, m3 ; 23 34 + vbroadcasti32x8 m7, [subpel_h_shufB] + kshiftrq k2, k1, 16 + mova xm16, [spel_hv_end] + vpermb m3, m5, m3 ; 45 56 +.hv_w8_loop: + vbroadcasti32x4 ym4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti32x4 m4{k1}, [srcq+ssq*0] + mova m0, m9 + vpdpwssd m0, m1, m12 ; a0 b0 + pshufb m1, m4, m6 ; 7 8 0123 4567 + mova m5, m8 + vpdpbusd m5, m1, m10 + pshufb m4, m7 ; 7 8 4567 89ab + vpdpwssd m0, m2, m13 ; a1 b1 + mova m1, m2 + vpdpbusd m5, m4, m11 + mova m2, m3 + vpdpwssd m0, m3, m14 ; a2 b2 + psraw m3{k2}, m5, 2 ; 75 86 + vpermb m3, m18, m3 ; 67 78 + vpdpwssd m0, m3, m15 ; a3 b3 + packuswb m0, m0 + vpermb zmm1, m16, m0 + movq [dstq+dsq*0], xmm1 + movhps [dstq+dsq*1], xmm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + vzeroupper + RET +.hv_w16: + movu m7, [spel_hv_perm16a] + sub srcq, ss3q + mova m20, [spel_hv_perm16b] + lea r6d, [wq*2-32] + mova m21, [spel_hv_perm16c] + mov r4, srcq + mov r7, dstq + mova ym16, [spel_hv_end16] + lea r6d, [hq+r6*8] +.hv_w16_loop0: + movu ym17, [srcq+ssq*0] + vinserti32x8 m17, [srcq+ssq*1], 1 ; 0 1 + movu ym18, [srcq+ssq*2] + add srcq, ss3q + vinserti32x8 m18, [srcq+ssq*0], 1 ; 2 3 + movu ym19, [srcq+ssq*1] + vinserti32x8 m19, [srcq+ssq*2], 1 ; 4 5 + add srcq, ss3q + vpermb m2, m7, m17 ; 0 1 0123 89ab + vpermb m0, m20, m17 ; 0 1 4567 cdef + vpermb m4, m7, m18 ; 2 3 0123 89ab + mova m1, m8 + vpdpbusd m1, m2, m10 + vpermb m5, m20, m18 ; 2 3 4567 cdef + mova m2, m8 + vpdpbusd m2, m0, m10 + vpermb m17, m21, m17 ; 0 1 89ab ghij + mova m3, m8 + vpdpbusd m3, m4, m10 + vpermb m6, m7, m19 ; 4 5 0123 89ab + mova m4, m8 + vpdpbusd m4, m5, m10 + vpermb m18, m21, m18 ; 2 3 89ab ghij + vpdpbusd m1, m0, m11 + movu ym0, [srcq+ssq*0] ; 6 + vpdpbusd m2, m17, m11 + vpermb m17, m20, m19 ; 4 5 4567 cdef + vpdpbusd m3, m5, m11 + mova m5, m8 + vpdpbusd m5, m6, m10 + mova m6, m8 + vpdpbusd m6, m17, m10 + vpdpbusd m4, m18, m11 + mova m18, [spel_hv_perm16d] + vpermb m18, m18, m0 ; 6 0145 2367 89cd abef + vpdpbusd m5, m17, m11 + vpermb m19, m21, m19 ; 4 5 89ab ghij + mova m17, m8 + vpdpbusd m17, m18, m10 + mova m18, [spel_hv_perm16e] + vpermb m0, m18, m0 ; 6 4589 67ab cdgh efij + packssdw m1, m2 ; 01 + vpdpbusd m6, m19, m11 + packssdw m3, m4 ; 23 + vpdpbusd m17, m0, m11 + psraw m1, 2 + packssdw m5, m6 ; 45 + psraw m3, 2 + vpshrdd m2, m1, m3, 16 ; 12 + psraw m5, 2 + vpshrdd m4, m3, m5, 16 ; 34 + psraw m17, 2 + vpshrdd m6, m5, m17, 16 ; 56 +.hv_w16_loop: + movu ym18, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti32x8 m18, [srcq+ssq*0], 1 + mova m0, m9 + vpdpwssd m0, m1, m12 ; a0 + vpermb m1, m7, m18 ; 7 8 0123 89ab + mova m17, m9 + vpdpwssd m17, m2, m12 ; b0 + vpermb m2, m20, m18 ; 7 8 4567 cdef + mova m19, m8 + vpdpbusd m19, m1, m10 + vpermb m18, m21, m18 + mova m1, m8 + vpdpbusd m1, m2, m10 + vpdpwssd m0, m3, m13 ; a1 + vpdpwssd m17, m4, m13 ; b1 + vpdpbusd m19, m2, m11 + mova m2, m4 + vpdpbusd m1, m18, m11 + mova m4, m6 + vpdpwssd m0, m5, m14 ; a2 + vpdpwssd m17, m6, m14 ; b2 + packssdw m19, m1 + mova m1, m3 + mova m3, m5 + psraw m6, m19, 2 ; 7 8 + vpshrdd m5, m4, m6, 16 ; 6 7 + vpdpwssd m17, m6, m15 ; b3 + vpdpwssd m0, m5, m15 ; a3 + packuswb m0, m17 + vpermb zmm1, m16, m0 + mova [dstq+dsq*0], xmm1 + vextracti128 [dstq+dsq*1], ymm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + add r4, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .hv_w16_loop0 + vzeroupper + RET + +%macro PREP_8TAP_H 0 + vpermb m10, m5, m0 + vpermb m11, m5, m1 + vpermb m12, m6, m0 + vpermb m13, m6, m1 + vpermb m14, m7, m0 + vpermb m15, m7, m1 + mova m0, m4 + vpdpbusd m0, m10, m8 + mova m2, m4 + vpdpbusd m2, m12, m8 + mova m1, m4 + vpdpbusd m1, m11, m8 + mova m3, m4 + vpdpbusd m3, m13, m8 + vpdpbusd m0, m12, m9 + vpdpbusd m2, m14, m9 + vpdpbusd m1, m13, m9 + vpdpbusd m3, m15, m9 + packssdw m0, m2 + packssdw m1, m3 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, + +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep_avx512icl] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep,)] + add wq, r7 + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pd_2] + WIN64_SPILL_XMM 10 + cmp wd, 4 + je .h_w4 + tzcnt wd, wd + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4] + add wq, r7 + jmp wq +.h_w4: + movzx mxd, mxb + vbroadcasti128 ym5, [subpel_h_shufA] + mov r3d, 0x4 + dec srcq + vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2] + kmovb k1, r3d + lea stride3q, [strideq*3] +.h_w4_loop: + movq xm2, [srcq+strideq*0] + movq xm3, [srcq+strideq*1] + vpbroadcastq ym2{k1}, [srcq+strideq*2] + vpbroadcastq ym3{k1}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pshufb ym2, ym5 + pshufb ym3, ym5 + mova ym0, ym4 + vpdpbusd ym0, ym2, ym6 + mova ym1, ym4 + vpdpbusd ym1, ym3, ym6 + packssdw ym0, ym1 + psraw ym0, 2 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + vbroadcasti128 m5, [subpel_h_shufA] + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + lea stride3q, [strideq*3] +.h_w8_loop: + movu xmm3, [srcq+strideq*0] + vinserti128 ym3, ymm3, [srcq+strideq*1], 1 + vinserti128 m3, [srcq+strideq*2], 2 + vinserti128 m3, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m1, m3, m5 + pshufb m2, m3, m6 + mova m0, m4 + vpdpbusd m0, m1, m8 + mova m1, m4 + vpdpbusd m1, m2, m8 + pshufb m3, m7 + vpdpbusd m0, m2, m9 + vpdpbusd m1, m3, m9 + packssdw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + mova m5, [spel_h_perm16a] + mova m6, [spel_h_perm16b] + mova m7, [spel_h_perm16c] + lea stride3q, [strideq*3] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*2] + vinserti32x8 m0, [srcq+strideq*1], 1 + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] +.h_w32_loop: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 2 + jg .h_w32_loop + RET +.h_w64: + xor r6d, r6d + jmp .h_start +.h_w128: + mov r6, -64*1 +.h_start: + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] + sub srcq, r6 + mov r5, r6 +.h_loop: + movu m0, [srcq+r6+32*0] + movu m1, [srcq+r6+32*1] + PREP_8TAP_H + add tmpq, 64*2 + add r6, 64 + jle .h_loop + add srcq, strideq + mov r6, r5 + dec hd + jg .h_loop + RET +.v: + movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. + shr myd, 16 ; Note that the code is 8-tap only, having + tzcnt wd, wd + cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 + cmove myd, mxd ; had a negligible effect on performance. + ; TODO: Would a 6-tap code path be worth it? + lea myq, [r7+myq*8+subpel_filters-prep_avx512icl] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)] + add wq, r7 + lea stride3q, [strideq*3] + sub srcq, stride3q + vpbroadcastd m7, [pw_8192] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + jmp wq +.v_w4: + movd xmm0, [srcq+strideq*0] + vpbroadcastd ymm1, [srcq+strideq*2] + vpbroadcastd xmm2, [srcq+strideq*1] + vpbroadcastd ymm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd ymm1, ymm0, 0x01 ; 0 2 2 _ 2 _ _ _ + vpblendd ymm3, ymm2, 0x03 ; 1 1 3 3 3 3 _ _ + vpbroadcastd ymm0, [srcq+strideq*0] + vpbroadcastd ymm2, [srcq+strideq*1] + vpblendd ymm1, ymm0, 0x68 ; 0 2 2 4 2 4 4 _ + vpbroadcastd ymm0, [srcq+strideq*2] + vbroadcasti128 ymm5, [deint_shuf4] + vpblendd ymm3, ymm2, 0xc0 ; 1 1 3 3 3 3 5 5 + vpblendd ymm2, ymm3, ymm1, 0x55 ; 0 1 2 3 2 3 4 5 + vpblendd ymm3, ymm1, 0xaa ; 1 2 3 4 3 4 5 _ + punpcklbw ymm1, ymm2, ymm3 ; 01 12 23 34 + vpblendd ymm3, ymm0, 0x80 ; 1 2 3 4 3 4 5 6 + punpckhbw ymm2, ymm3 ; 23 34 45 56 +.v_w4_loop: + pinsrd xmm0, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpbroadcastd ymm3, [srcq+strideq*0] + vpbroadcastd ymm4, [srcq+strideq*1] + vpblendd ymm3, ymm4, 0x20 ; _ _ 8 _ 8 9 _ _ + vpblendd ymm3, ymm0, 0x03 ; 6 7 8 _ 8 9 _ _ + vpbroadcastd ymm0, [srcq+strideq*2] + vpblendd ymm3, ymm0, 0x40 ; 6 7 8 _ 8 9 a _ + pshufb ymm3, ymm5 ; 67 78 89 9a + pmaddubsw ymm4, ymm1, ym8 + vperm2i128 ymm1, ymm2, ymm3, 0x21 ; 45 56 67 78 + pmaddubsw ymm2, ym9 + paddw ymm4, ymm2 + mova ymm2, ymm3 + pmaddubsw ymm3, ym11 + paddw ymm3, ymm4 + pmaddubsw ymm4, ymm1, ym10 + paddw ymm3, ymm4 + pmulhrsw ymm3, ym7 + mova [tmpq], ymm3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + mov r3d, 0xf044 + kmovw k1, r3d + kshiftrw k2, k1, 8 + movq xm0, [srcq+strideq*0] + vpbroadcastq ym1, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpbroadcastq m6, [srcq+strideq*2] + vmovdqa64 ym0{k1}, ym1 + vmovdqa64 ym1{k1}, ym2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + punpcklbw ym0, ym1 ; 01 12 __ __ + punpcklbw m2, m3 ; 23 34 23 34 + punpcklbw m4, m5 ; 45 56 45 56 + vmovdqa64 m0{k2}, m2 ; 01 12 23 34 + vmovdqa64 m2{k2}, m4 ; 23 34 45 56 +.v_w8_loop: + vpbroadcastq m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + pmaddubsw m14, m0, m8 + pmaddubsw m15, m2, m9 + vpblendmq m0{k1}, m6, m1 + vpblendmq m2{k1}, m1, m3 + vpbroadcastq m6, [srcq+strideq*2] + paddw m14, m15 + punpcklbw m2, m0, m2 ; 67 78 67 78 + vpblendmq m12{k1}, m3, m5 + vpblendmq m13{k1}, m5, m6 + vpblendmq m0{k2}, m4, m2 ; 45 56 67 78 + punpcklbw m4, m12, m13 ; 89 9a 89 9a + vmovdqa64 m2{k2}, m4 ; 67 78 89 9a + pmaddubsw m12, m0, m10 + pmaddubsw m13, m2, m11 + paddw m14, m12 + paddw m14, m13 + pmulhrsw m14, m7 + mova [tmpq], m14 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + mov r3d, 0xf0 + kmovb k1, r3d + vbroadcasti128 m0, [srcq+strideq*0] + vbroadcasti128 m1, [srcq+strideq*1] + vbroadcasti128 m2, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + vmovdqa64 m0{k1}, m1 + vmovdqa64 m1{k1}, m2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b + shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b + shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_-- + shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_-- + punpckhbw m2, m0, m1 ; 23a 23b 34a 34b + punpcklbw m0, m1 ; 01a 01b 12a 12b + punpcklbw m4, m5 ; 45a 45b 56a 56b +.v_w16_loop: + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m5, [srcq+strideq*0] + vpblendmq m1{k1}, m6, m3 + vmovdqa64 m3{k1}, m5 + pmaddubsw m12, m0, m8 + pmaddubsw m13, m2, m8 + pmaddubsw m14, m2, m9 + pmaddubsw m15, m4, m9 + pmaddubsw m0, m4, m10 + vbroadcasti128 m2, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + paddw m12, m14 + paddw m13, m15 + paddw m12, m0 + vmovdqa64 m5{k1}, m2 + vmovdqa64 m2{k1}, m6 + mova m0, m4 + shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b + shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab + punpcklbw m2, m1, m3 ; 67a 67b 78a 78b + punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab + pmaddubsw m14, m2, m10 + pmaddubsw m15, m2, m11 + paddw m13, m14 + paddw m12, m15 + pmaddubsw m14, m4, m11 + paddw m13, m14 + pmulhrsw m12, m7 + pmulhrsw m13, m7 + mova [tmpq+ 0], m12 + mova [tmpq+64], m13 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + mova m18, [bilin_v_perm64] + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + movu ym3, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym4, [srcq+strideq*0] + movu ym5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym6, [srcq+strideq*0] + vpermq m0, m18, m0 + vpermq m1, m18, m1 + vpermq m2, m18, m2 + vpermq m3, m18, m3 + vpermq m4, m18, m4 + vpermq m5, m18, m5 + vpermq m6, m18, m6 + punpcklbw m0, m1 + punpcklbw m1, m2 + punpcklbw m2, m3 + punpcklbw m3, m4 + punpcklbw m4, m5 + punpcklbw m5, m6 +.v_w32_loop: + movu ym12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym13, [srcq+strideq*0] + pmaddubsw m14, m0, m8 + pmaddubsw m16, m2, m9 + pmaddubsw m15, m1, m8 + pmaddubsw m17, m3, m9 + mova m0, m2 + mova m1, m3 + vpermq m12, m18, m12 + vpermq m13, m18, m13 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m4, m10 + pmaddubsw m17, m5, m10 + punpcklbw m6, m12 + punpcklbw m12, m13 + mova m2, m4 + mova m3, m5 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m6, m11 + pmaddubsw m17, m12, m11 + mova m4, m6 + mova m5, m12 + paddw m14, m16 + paddw m15, m17 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + mova m6, m13 + mova [tmpq+ 0], m14 + mova [tmpq+64], m15 + add tmpq, 64*2 + sub hd, 2 + jg .v_w32_loop + vzeroupper + RET +.v_w64: + mov wd, 64 + jmp .v_start +.v_w128: + mov wd, 128 +.v_start: + WIN64_SPILL_XMM 27 + mova m26, [bilin_v_perm64] + lea r6d, [hq+wq*2] + mov r5, srcq + mov r7, tmpq +.v_loop0: + vpermq m0, m26, [srcq+strideq*0] + vpermq m1, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m2, m26, [srcq+strideq*0] + vpermq m3, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m4, m26, [srcq+strideq*0] + vpermq m5, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m6, m26, [srcq+strideq*0] + punpckhbw m12, m0, m1 + punpcklbw m0, m1 + punpckhbw m13, m1, m2 + punpcklbw m1, m2 + punpckhbw m14, m2, m3 + punpcklbw m2, m3 + punpckhbw m15, m3, m4 + punpcklbw m3, m4 + punpckhbw m16, m4, m5 + punpcklbw m4, m5 + punpckhbw m17, m5, m6 + punpcklbw m5, m6 +.v_loop: + vpermq m18, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m19, m26, [srcq+strideq*0] + pmaddubsw m20, m0, m8 + pmaddubsw m21, m12, m8 + pmaddubsw m22, m1, m8 + pmaddubsw m23, m13, m8 + mova m0, m2 + mova m12, m14 + mova m1, m3 + mova m13, m15 + pmaddubsw m2, m9 + pmaddubsw m14, m9 + pmaddubsw m3, m9 + pmaddubsw m15, m9 + punpckhbw m24, m6, m18 + punpcklbw m6, m18 + paddw m20, m2 + paddw m21, m14 + paddw m22, m3 + paddw m23, m15 + mova m2, m4 + mova m14, m16 + mova m3, m5 + mova m15, m17 + pmaddubsw m4, m10 + pmaddubsw m16, m10 + pmaddubsw m5, m10 + pmaddubsw m17, m10 + punpckhbw m25, m18, m19 + punpcklbw m18, m19 + paddw m20, m4 + paddw m21, m16 + paddw m22, m5 + paddw m23, m17 + mova m4, m6 + mova m16, m24 + mova m5, m18 + mova m17, m25 + pmaddubsw m6, m11 + pmaddubsw m24, m11 + pmaddubsw m18, m11 + pmaddubsw m25, m11 + paddw m20, m6 + paddw m21, m24 + paddw m22, m18 + paddw m23, m25 + pmulhrsw m20, m7 + pmulhrsw m21, m7 + pmulhrsw m22, m7 + pmulhrsw m23, m7 + mova m6, m19 + mova [tmpq+wq*0+ 0], m20 + mova [tmpq+wq*0+64], m21 + mova [tmpq+wq*2+ 0], m22 + mova [tmpq+wq*2+64], m23 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_loop + add r5, 64 + add r7, 128 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .v_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + %assign stack_size_padded 0 + WIN64_SPILL_XMM 16 + cmp wd, 4 + je .hv_w4 + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0] + vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + tzcnt wd, wd + vpbroadcastd m8, [pd_2] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)] + vpbroadcastd m9, [pd_32] + add wq, r7 + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] + lea stride3q, [strideq*3] + sub srcq, stride3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + jmp wq +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] + lea stride3q, [strideq*3] + sub srcq, stride3q + mov r3d, 0x04 + kmovb k1, r3d + kshiftlb k2, k1, 2 + kshiftlb k3, k1, 4 + vpbroadcastd m10, [pd_2] + vbroadcasti128 m16, [subpel_h_shufA] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m11, [pd_32] + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + movq xm3, [srcq+strideq*0] + vpbroadcastq ym2, [srcq+strideq*1] + vpbroadcastq ym3{k1}, [srcq+strideq*2] + vpbroadcastq m2{k2}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3{k2}, [srcq+strideq*0] + vpbroadcastq m2{k3}, [srcq+strideq*1] + vpbroadcastq m3{k3}, [srcq+strideq*2] + mova m17, [spel_hv_perm4a] + movu m18, [spel_hv_perm4b] + mova m0, m10 + mova m1, m10 + pshufb m2, m16 + pshufb m3, m16 + vpdpbusd m0, m2, m8 + vpdpbusd m1, m3, m8 + packssdw m0, m1 ; _ 0 1 2 3 4 5 6 + psraw m0, 2 + vpermb m1, m17, m0 ; 01 12 23 34 + vpermb m2, m18, m0 ; 23 34 45 56 +.hv_w4_loop: + movq xm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movq xm4, [srcq+strideq*0] + vpbroadcastq ym3{k1}, [srcq+strideq*1] + vpbroadcastq ym4{k1}, [srcq+strideq*2] + mova ym5, ym10 + mova ym6, ym10 + pshufb ym3, ym16 + pshufb ym4, ym16 + vpdpbusd ym5, ym3, ym8 + vpdpbusd ym6, ym4, ym8 + mova m7, m11 + packssdw ym5, ym6 ; 7 8 9 a _ _ _ _ + psraw ym5, 2 + valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a + vpdpwssd m7, m1, m12 + vpdpwssd m7, m2, m13 + vpermb m1, m17, m0 ; 45 56 67 78 + vpermb m2, m18, m0 ; 67 78 89 9a + vpdpwssd m7, m1, m14 + vpdpwssd m7, m2, m15 + psrad m7, 6 + vpmovdw [tmpq], m7 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + vzeroupper + RET +.hv_w8: + WIN64_SPILL_XMM 24 + vbroadcasti128 m16, [subpel_h_shufA] + vbroadcasti128 m17, [subpel_h_shufB] + vbroadcasti128 m18, [subpel_h_shufC] + vinserti128 ym0, [srcq+strideq*0], 1 + vinserti128 m0, [srcq+strideq*1], 2 + vinserti128 m0, [srcq+strideq*2], 3 + movu xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym1, [srcq+strideq*0], 1 + vinserti128 m1, [srcq+strideq*1], 2 + vinserti128 m1, [srcq+strideq*2], 3 + mova m2, m8 + mova m4, m8 + mova m3, m8 + mova m5, m8 + pshufb m20, m0, m16 + pshufb m21, m0, m17 + pshufb m22, m0, m18 + pshufb m23, m1, m16 + pshufb m6, m1, m17 + pshufb m7, m1, m18 + vpdpbusd m2, m20, m10 + vpdpbusd m4, m21, m10 + vpdpbusd m2, m21, m11 + vpdpbusd m4, m22, m11 + vpdpbusd m3, m23, m10 + vpdpbusd m5, m6, m10 + vpdpbusd m3, m6, m11 + vpdpbusd m5, m7, m11 + packssdw m2, m4 + packssdw m3, m5 + psraw m2, 2 ; _ 0 1 2 + psraw m3, 2 ; 3 4 5 6 + valignq m0, m3, m2, 2 ; 0 1 2 3 + valignq m1, m3, m2, 4 ; 1 2 3 4 + valignq m2, m3, m2, 6 ; 2 3 4 5 + punpcklwd m4, m0, m1 ; 01a 12a 23a 34a + punpckhwd m5, m0, m1 ; 01b 12b 23b 34b + punpcklwd m6, m2, m3 ; 23a 34a 45a 56a + punpckhwd m7, m2, m3 ; 23b 34b 45b 56b +.hv_w8_loop: + movu xm19, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym19, [srcq+strideq*0], 1 + vinserti128 m19, [srcq+strideq*1], 2 + vinserti128 m19, [srcq+strideq*2], 3 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m4, m12 + vpdpwssd m21, m5, m12 + vpdpwssd m20, m6, m13 + vpdpwssd m21, m7, m13 + pshufb m0, m19, m16 + pshufb m1, m19, m17 + pshufb m2, m19, m18 + vpdpbusd m22, m0, m10 + vpdpbusd m23, m1, m10 + vpdpbusd m22, m1, m11 + vpdpbusd m23, m2, m11 + packssdw m22, m23 + psraw m22, 2 ; 7 8 9 A + valignq m0, m22, m3, 2 ; 4 5 6 7 + valignq m1, m22, m3, 4 ; 5 6 7 8 + valignq m2, m22, m3, 6 ; 6 7 8 9 + mova m3, m22 + punpcklwd m4, m0, m1 ; 45a 56a 67a 78a + punpckhwd m5, m0, m1 ; 45b 56b 67b 78b + punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa + punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq], m20 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + mov wd, 16*2 + jmp .hv_start +.hv_w32: + mov wd, 32*2 + jmp .hv_start +.hv_w64: + mov wd, 64*2 + jmp .hv_start +.hv_w128: + mov wd, 128*2 +.hv_start: + WIN64_SPILL_XMM 31 + mova m16, [spel_h_perm16a] + mova m17, [spel_h_perm16b] + mova m18, [spel_h_perm16c] + lea r6d, [hq+wq*8-256] + mov r5, srcq + mov r7, tmpq +.hv_loop0: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym1, [srcq+strideq*0] + vinserti32x8 m1, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + vinserti32x8 m2, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym3, [srcq+strideq*0] + mova m4, m8 + mova m5, m8 + mova m6, m8 + mova m7, m8 + vpermb m19, m16, m0 + vpermb m20, m17, m0 + vpermb m21, m18, m0 + vpermb m22, m16, m1 + vpermb m23, m17, m1 + vpermb m24, m18, m1 + vpermb m25, m16, m2 + vpermb m26, m17, m2 + vpermb m27, m18, m2 + vpermb ym28, ym16, ym3 + vpermb ym29, ym17, ym3 + vpermb ym30, ym18, ym3 + mova m0, m8 + mova m1, m8 + mova ym2, ym8 + mova ym3, ym8 + vpdpbusd m4, m19, m10 + vpdpbusd m5, m20, m10 + vpdpbusd m6, m22, m10 + vpdpbusd m7, m23, m10 + vpdpbusd m0, m25, m10 + vpdpbusd m1, m26, m10 + vpdpbusd ym2, ym28, ym10 + vpdpbusd ym3, ym29, ym10 + vpdpbusd m4, m20, m11 + vpdpbusd m5, m21, m11 + vpdpbusd m6, m23, m11 + vpdpbusd m7, m24, m11 + vpdpbusd m0, m26, m11 + vpdpbusd m1, m27, m11 + vpdpbusd ym2, ym29, ym11 + vpdpbusd ym3, ym30, ym11 + packssdw m4, m5 + packssdw m6, m7 + packssdw m0, m1 + packssdw ym2, ym3 + psraw m4, 2 ; 0a 0b 1a 1b + psraw m6, 2 ; 2a 2b 3a 3b + psraw m0, 2 ; 4a 4b 5a 5b + psraw ym2, 2 ; 6a 6b __ __ + vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b + vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b + vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b + punpcklwd m2, m4, m5 ; 01a 01c 12a 12c + punpckhwd m3, m4, m5 ; 01b 01d 12b 12d + punpcklwd m4, m6, m7 ; 23a 23c 34a 34c + punpckhwd m5, m6, m7 ; 23b 23d 34b 34d + punpcklwd m6, m0, m1 ; 45a 45c 56a 56c + punpckhwd m7, m0, m1 ; 45b 45d 56b 56d +.hv_loop: + movu ym19, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti32x8 m19, [srcq+strideq*0], 1 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m2, m12 + vpdpwssd m21, m3, m12 + vpdpwssd m20, m4, m13 + vpdpwssd m21, m5, m13 + vpermb m24, m16, m19 + vpermb m25, m17, m19 + vpermb m26, m18, m19 + vpdpbusd m22, m24, m10 + vpdpbusd m23, m25, m10 + vpdpbusd m22, m25, m11 + vpdpbusd m23, m26, m11 + packssdw m22, m23 + psraw m22, 2 ; 7a 7b 8a 8b + vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b + mova m2, m4 + mova m3, m5 + mova m1, m22 + mova m4, m6 + mova m5, m7 + punpcklwd m6, m0, m1 ; 67a 67c 78a 78c + punpckhwd m7, m0, m1 ; 67b 67d 78b 78d + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq+wq*0], ym20 + vextracti32x8 [tmpq+wq*1], m20, 1 + lea tmpq, [tmpq+wq*2] + sub hd, 2 + jg .hv_loop + add r5, 16 + add r7, 32 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .hv_loop0 + RET + +cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts + vpbroadcastd m9, [pd_16384] + mova ym15, [warp_8x8t_end] + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main + jmp .start +.loop: + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main2 + lea tmpq, [tmpq+tsq*4] +.start: + paddd m16, m16 + vpermb m16, m15, m16 + mova [tmpq+tsq*0], xm16 + vextracti128 [tmpq+tsq*2], ym16, 1 + sub r6d, 0x1800 + jg .loop + RET + +cglobal warp_affine_8x8_8bpc, 4, 7, 22, dst, ds, src, ss, abcd, filter + vpbroadcastd m9, [pd_262144] + mova xm15, [warp_8x8_end] + call .main + jmp .start +.loop: + call .main2 + lea dstq, [dstq+dsq*2] +.start: + psrad m16, 19 + packuswb m16, m16 + vpermb m16, m15, m16 + movq [dstq+dsq*0], xm16 + movhps [dstq+dsq*1], xm16 + sub r6d, 0x1800 + jg .loop + RET +ALIGN function_align +.main: + vpbroadcastd m1, [pd_512] +%if WIN64 + mov abcdq, r5mp + vpaddd ym18, ym1, r6m {1to8} ; mx +%else + add r5d, 512 + vpbroadcastd ym18, r5d +%endif + vpaddd ym20, ym1, r7m {1to8} ; my + mova ym16, [pd_0to7] + vpbroadcastd ym19, [abcdq+4*0] + vpbroadcastd ym21, [abcdq+4*1] + lea r4, [ssq*3+3] + mova m10, [warp_8x8_permA] + mov r6d, 0x5555 + mova m11, [warp_8x8_permB] + lea filterq, [mc_warp_filter+64*8] + vpbroadcastq m12, [warp_8x8_hpack] + sub srcq, r4 ; src -= src_stride*3 + 3 + vbroadcasti32x4 m13, [warp_8x8_permC] + kxnorb k2, k2, k2 + vbroadcasti32x4 m14, [warp_8x8_permD] + vpdpwssd ym18, ym19, ym16 ; alpha + vpdpwssd ym20, ym21, ym16 ; gamma + vbroadcasti32x4 m0, [srcq] + psrad ym19, 16 ; beta + psrad ym21, 16 ; delta + kmovw k1, r6d + psrad ym16, ym18, 10 + kmovb k3, k2 + paddd ym18, ym19 + vpgatherdq m2{k2}, [filterq+ym16*8] ; filter_x0 + psrld m1, 8 ; pd_2 + pshufb m0, m11 + paddd m8, m1, m1 ; pd_4 + vpdpbusd m1, m0, m2 + call .h + psllq m2, m1, 45 + pslld m1, 13 + paddd m1, m2 + vpshrdq m1, m0, 48 ; 01 12 + call .h + vpshrdq m2, m1, m0, 48 ; 23 34 + call .h + vpshrdq m3, m2, m0, 48 ; 45 56 +.main2: + call .h + psrad ym17, ym20, 10 + kmovb k2, k3 + paddd ym20, ym21 + vpgatherdq m7{k3}, [filterq+ym17*8] ; filter_y0 + psrad ym16, ym20, 10 + kmovb k3, k2 + paddd ym20, ym21 + vpgatherdq m17{k2}, [filterq+ym16*8] ; filter_y1 + shufps m5, m7, m17, q2020 ; a0 a1 a2 a3 b0 b1 b2 b3 A0 A1 A2 A3 B0 B1 B2 B3 + mova m16, m9 + pshufb m4, m5, m13 ; a0 a1 A0 A1 b0 b1 B0 B1 + vpdpwssd m16, m1, m4 + pshufb m5, m14 ; a2 a3 A2 A3 b2 b3 B2 B3 + mova m1, m2 + vpdpwssd m16, m2, m5 + shufps m5, m7, m17, q3131 ; a4 a5 a6 a7 b4 b5 b6 b7 A4 A5 A6 A7 B4 B5 B6 B7 + mova m2, m3 + pshufb m4, m5, m13 ; a4 a5 A4 A5 b4 b5 B4 B5 + vpdpwssd m16, m3, m4 + vpshrdq m3, m0, 48 ; 67 78 + pshufb m5, m14 ; a6 a7 A6 A7 b6 b7 B6 B7 + vpdpwssd m16, m3, m5 + ret +ALIGN function_align +.h: + movu xm5, [srcq+ssq*1] + psrad ym16, ym18, 10 + lea srcq, [srcq+ssq*2] + vinserti32x4 ym5, [srcq+ssq*0], 1 + kmovb k2, k3 + paddd ym18, ym19 + vpgatherdq m6{k3}, [filterq+ym16*8] ; filter_x1 + psrad ym17, ym18, 10 + kmovb k3, k2 + paddd ym18, ym19 + vpgatherdq m16{k2}, [filterq+ym17*8] ; filter_x2 + mova m0, m8 + vpermb m4, m10, m5 ; a4 b0 a5 b1 a6 b2 a7 b3 a8 b4 a9 b5 aa b6 ab b7 + vpshldq m17, m16, m6, 32 ; a4 a5 a6 a7 b0 b1 b2 b3 + vpdpbusd m0, m4, m17 + vpermb m5, m11, m5 ; a0 b4 a1 b5 a2 b6 a3 b7 a4 b8 a5 b9 a6 ba a7 bb + vmovdqa32 m16{k1}, m6 ; a0 a1 a2 a3 b4 b5 b6 b7 + vpdpbusd m0, m5, m16 + vpmultishiftqb m0, m12, m0 ; 1 1 2 2 (>> 3) + ret + +%macro BIDIR_FN 1 ; op + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM %1 0 + vextracti32x4 xm1, ym0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + jl .w4_ret + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.w4_ret: + RET +.w4_h16: + vpbroadcastd m7, strided + pmulld m7, [bidir_sctr_w4] + %1 0 + kxnorw k1, k1, k1 + vpscatterdd [dstq+m7]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM %1 0 + vextracti32x4 xm1, ym0, 1 + movq [dstq ], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + RET +.w8_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w8_h8: + %1 0 + vextracti32x4 xm1, ym0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq ], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + movhps [dstq ], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w16: + %1 0 + vpermq m0, m0, q3120 + mova [dstq ], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m7, [pb_02461357] +.w32_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m7, [pb_02461357] +.w64_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m7, [pb_02461357] +.w128_loop: + %1 0 + vpermq m6, m7, m0 + %1 2 + mova [dstq+64*0], m6 + %1_INC_PTR 4 + vpermq m6, m7, m0 + mova [dstq+64*1], m6 + add dstq, strideq + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + mova m0, [tmp1q+(%1+0)*mmsize] + paddw m0, [tmp2q+(%1+0)*mmsize] + mova m1, [tmp1q+(%1+1)*mmsize] + paddw m1, [tmp2q+(%1+1)*mmsize] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + packuswb m0, m1 +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +cglobal avg_8bpc, 4, 7, 5, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg_avx512icl_table + lea r6, [avg_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r6+wq*4] + vpbroadcastd m4, [base+pw_1024] + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m0, [tmp1q+(%1+0)*mmsize] + psubw m2, m0, [tmp2q+(%1+0)*mmsize] + mova m1, [tmp1q+(%1+1)*mmsize] + psubw m3, m1, [tmp2q+(%1+1)*mmsize] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-w_avg_avx512icl_table + lea r6, [w_avg_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + vpbroadcastw m4, r6m ; weight + movsxd wq, dword [r6+wq*4] + vpbroadcastd m5, [base+pw_2048] + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + pxor m0, m0 + mov tmp1q, tmp2q + psubw m4, m0, m4 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 +%if mmsize == 64 + vpermq m3, m8, [maskq+%1*32] +%else + vpermq m3, [maskq+%1*16], q3120 +%endif + mova m0, [tmp2q+(%1+0)*mmsize] + psubw m1, m0, [tmp1q+(%1+0)*mmsize] + psubb m3, m4, m3 + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 + punpcklbw m2, m4, m3 ; -m << 9 + pmulhw m1, m2 + paddw m0, m1 + mova m1, [tmp2q+(%1+1)*mmsize] + psubw m2, m1, [tmp1q+(%1+1)*mmsize] + paddw m2, m2 + punpckhbw m3, m4, m3 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*32 + add tmp2q, %1*64 + add tmp1q, %1*64 +%endmacro + +cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask_avx512icl_table + lea r7, [mask_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + pxor m4, m4 + mova m8, [base+bilin_v_perm64] + vpbroadcastd m5, [base+pw_2048] + add wq, r7 + BIDIR_FN MASK + +%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 + mova m%1, [tmp1q+mmsize*%3] + mova m1, [tmp2q+mmsize*%3] + psubw m1, m%1 + pabsw m%2, m1 + psubusw m%2, m6, m%2 + psrlw m%2, 8 ; 64 - m + psllw m2, m%2, 10 + pmulhw m1, m2 + paddw m%1, m1 + mova m1, [tmp1q+mmsize*%4] + mova m2, [tmp2q+mmsize*%4] + psubw m2, m1 + pabsw m3, m2 + psubusw m3, m6, m3 + vpshldw m%2, m3, 8 + psllw m3, m%2, 10 +%if %5 + psubb m%2, m5, m%2 +%endif + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m%1, m7 + pmulhrsw m1, m7 + packuswb m%1, m1 +%endmacro + +cglobal w_mask_420_8bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx512icl_table + lea r7, [w_mask_420_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pb_m64] ; -1 << 6 + mova ym10, [base+wm_420_mask+32] + vpbroadcastd m8, [base+wm_sign+r6*8] ; (258 - sign) << 6 + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + mova m5, [wm_420_perm4] + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm4+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vextracti32x4 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.w4_end: + vpermb ym8, ym10, ym8 + movq [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m11, strided + pmulld m11, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + vpdpbusd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + mova [maskq], xm8 + vpscatterdd [dstq+m11]{k1}, m0 + RET +.w8: + mova m5, [wm_420_perm8] + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm8+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vpermb m8, m10, m8 + mova [maskq], xm8 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 16 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + vpermb m1, m10, m1 + mova [maskq], xm1 + vextracti32x4 xm1, ym0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + sub hd, 8 + jg .w8_loop + RET +.w16: + mova m5, [wm_420_perm16] +.w16_loop: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14 + psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15 +.w64_loop: + W_MASK 0, 4, 0, 2 + W_MASK 11, 5, 1, 3 + mova m2, m8 + vpdpbusd m2, m4, m9 + mova m3, m8 + vpdpbusd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + mova m1, m0 + vpermt2q m0, m12, m11 + vpermt2q m1, m13, m11 + mova [maskq], ym2 + add maskq, 32 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w64_loop + RET +.w128: + pmovzxbq m14, [wm_420_perm64] + mova m10, [wm_420_mask] + psrlq m15, m14, 4 +.w128_loop: + W_MASK 0, 12, 0, 4 + W_MASK 11, 13, 1, 5 + mova m4, m8 + vpdpbusd m4, m12, m9 + mova m5, m8 + vpdpbusd m5, m13, m9 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*1+64*0], m1 + W_MASK 0, 12, 2, 6 + W_MASK 11, 13, 3, 7 + vprold m4, 16 + vprold m5, 16 + vpdpbusd m4, m12, m9 + vpdpbusd m5, m13, m9 + add tmp1q, 512 + add tmp2q, 512 + vpermt2b m4, m10, m5 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0+64*1], m0 + mova [dstq+strideq*1+64*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w128_loop + RET + +cglobal w_mask_422_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx512icl_table + lea r7, [w_mask_422_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pw_m128] + mova m10, [base+wm_422_mask] + vpbroadcastd m11, [base+pb_127] + add wq, r7 + vpbroadcastd m8, [base+wm_sign+4+r6*4] + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + vextracti32x4 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.w4_end: + pand xm8, xm11 + mova [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m5, strided + pmulld m5, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpdpwssd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + pand ym8, ym11 + mova [maskq], ym8 + vpscatterdd [dstq+m5]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + pand xm8, xm11 + mova [maskq], xm8 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + pand ym1, ym11 + mova [maskq], ym1 + vextracti32x4 xm1, ym0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + pand ym1, ym11 + mova [maskq], ym1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m5, [pb_02461357] +.w64_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m13, [pb_02461357] +.w128_loop: + W_MASK 0, 4, 0, 1 + W_MASK 12, 5, 2, 3 + mova m2, m8 + vpdpwssd m2, m4, m9 + mova m3, m8 + vpdpwssd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + vpermq m0, m13, m0 + vpermq m1, m13, m12 + pand m2, m11 + mova [maskq], m2 + add maskq, 64 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, strideq + dec hd + jg .w128_loop + RET + +cglobal w_mask_444_8bpc, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx512icl_table + lea r7, [w_mask_444_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + mova m8, [base+wm_444_mask] + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti32x4 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.w4_end: + RET +.w4_h16: + vpbroadcastd m9, strided + pmulld m9, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + kxnorw k1, k1, k1 + mova [maskq], m4 + vpscatterdd [dstq+m9]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti32x4 xm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + mova [maskq], m4 + vextracti32x4 xm1, ym0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movq [dstq+strideq*2], xm2 + movq [dstq+stride3q ], xm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm2 + movhps [dstq+stride3q ], xm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + vpermq m0, m0, q3120 + mova [maskq], m4 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m9, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m9, [pb_02461357] +.w64_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m11, [pb_02461357] +.w128_loop: + W_MASK 0, 4, 0, 1, 1 + W_MASK 10, 9, 2, 3, 1 + vpermb m4, m8, m4 + vpermb m9, m8, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermq m0, m11, m0 + vpermq m10, m11, m10 + mova [maskq+64*0], m4 + mova [maskq+64*1], m9 + add maskq, 128 + mova [dstq+64*0], m0 + mova [dstq+64*1], m10 + add dstq, strideq + dec hd + jg .w128_loop + RET + +cglobal blend_8bpc, 3, 7, 8, dst, ds, tmp, w, h, mask +%define base r6-blend_avx512icl_table + lea r6, [blend_avx512icl_table] + tzcnt wd, wm + movifnidn maskq, maskmp + movifnidn hd, hm + movsxd wq, [r6+wq*4] + vpbroadcastd m6, [base+pb_64] + vpbroadcastd m7, [base+pw_512] + sub tmpq, maskq + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + movd xmm0, [dstq+dsq*0] + pinsrd xmm0, [dstq+dsq*1], 1 + vpbroadcastd xmm1, [dstq+dsq*2] + pinsrd xmm1, [dstq+r6 ], 3 + mova xmm4, [maskq] + mova xmm5, [maskq+tmpq] + add maskq, 4*4 + psubb xmm3, xm6, xmm4 + punpcklbw xmm0, xmm5 + punpcklbw xmm2, xmm3, xmm4 + punpckhbw xmm1, xmm5 + punpckhbw xmm3, xmm4 + pmaddubsw xmm0, xmm2 + pmaddubsw xmm1, xmm3 + pmulhrsw xmm0, xm7 + pmulhrsw xmm1, xm7 + packuswb xmm0, xmm1 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + pextrd [dstq+dsq*2], xmm0, 2 + pextrd [dstq+r6 ], xmm0, 3 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w4 + RET +.w8: + movq xmm0, [dstq+dsq*0] + vpbroadcastq xmm1, [dstq+dsq*1] + vpbroadcastq ymm2, [dstq+dsq*2] + vpbroadcastq ymm3, [dstq+r6 ] + mova ymm4, [maskq] + mova ymm5, [maskq+tmpq] + add maskq, 8*4 + vpblendd ymm0, ymm2, 0x30 + vpblendd ymm1, ymm3, 0xc0 + psubb ymm3, ym6, ymm4 + punpcklbw ymm0, ymm5 + punpcklbw ymm2, ymm3, ymm4 + punpckhbw ymm1, ymm5 + punpckhbw ymm3, ymm4 + pmaddubsw ymm0, ymm2 + pmaddubsw ymm1, ymm3 + pmulhrsw ymm0, ym7 + pmulhrsw ymm1, ym7 + packuswb ymm0, ymm1 + vextracti128 xmm1, ymm0, 1 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + movq [dstq+dsq*2], xmm1 + movhps [dstq+r6 ], xmm1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w8 + vzeroupper + RET +.w16: + mova xm1, [dstq+dsq*0] + vinserti32x4 ym1, [dstq+dsq*1], 1 + vinserti32x4 m1, [dstq+dsq*2], 2 + mova m4, [maskq] + vinserti32x4 m1, [dstq+r6 ], 3 + mova m5, [maskq+tmpq] + add maskq, 16*4 + psubb m3, m6, m4 + punpcklbw m0, m1, m5 + punpcklbw m2, m3, m4 + punpckhbw m1, m5 + punpckhbw m3, m4 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m7 + pmulhrsw m1, m7 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], ym0, 1 + vextracti32x4 [dstq+dsq*2], m0, 2 + vextracti32x4 [dstq+r6 ], m0, 3 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w16 + RET +.w32: + mova ym1, [dstq+dsq*0] + vinserti32x8 m1, [dstq+dsq*1], 1 + mova m4, [maskq] + mova m5, [maskq+tmpq] + add maskq, 32*2 + psubb m3, m6, m4 + punpcklbw m0, m1, m5 + punpcklbw m2, m3, m4 + punpckhbw m1, m5 + punpckhbw m3, m4 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m7 + pmulhrsw m1, m7 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w32 + RET + +cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_v_avx512icl_table + lea r5, [blend_v_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, [r5+wq*4] + vpbroadcastd m5, [base+pw_512] + add wq, r5 + add maskq, obmc_masks-blend_v_avx512icl_table + jmp wq +.w2: + vpbroadcastd xmm2, [maskq+2*2] +.w2_s0_loop: + movd xmm0, [dstq+dsq*0] + pinsrw xmm0, [dstq+dsq*1], 1 + movd xmm1, [tmpq] + add tmpq, 2*2 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm2 + pmulhrsw xmm0, xm5 + packuswb xmm0, xmm0 + pextrw [dstq+dsq*0], xmm0, 0 + pextrw [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w2_s0_loop + RET +.w4: + vpbroadcastq xmm2, [maskq+4*2] +.w4_loop: + movd xmm0, [dstq+dsq*0] + pinsrd xmm0, [dstq+dsq*1], 1 + movq xmm1, [tmpq] + add tmpq, 4*2 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm2 + pmulhrsw xmm0, xm5 + packuswb xmm0, xmm0 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +.w8: + mova xmm3, [maskq+8*2] +.w8_loop: + movq xmm0, [dstq+dsq*0] + vpbroadcastq xmm1, [dstq+dsq*1] + mova xmm2, [tmpq] + add tmpq, 8*2 + punpcklbw xmm0, xmm2 + punpckhbw xmm1, xmm2 + pmaddubsw xmm0, xmm3 + pmaddubsw xmm1, xmm3 + pmulhrsw xmm0, xm5 + pmulhrsw xmm1, xm5 + packuswb xmm0, xmm1 + movq [dstq+dsq*0], xmm0 + movhps [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + vbroadcasti32x4 ym3, [maskq+16*2] + vbroadcasti32x4 ym4, [maskq+16*3] +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti32x4 ym1, [dstq+dsq*1], 1 + mova ym2, [tmpq] + add tmpq, 16*2 + punpcklbw ym0, ym1, ym2 + punpckhbw ym1, ym2 + pmaddubsw ym0, ym3 + pmaddubsw ym1, ym4 + pmulhrsw ym0, ym5 + pmulhrsw ym1, ym5 + packuswb ym0, ym1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16_loop + RET +.w32: + mova m4, [maskq+32*2] + vshufi32x4 m3, m4, m4, q2020 + vshufi32x4 m4, m4, q3131 +.w32_loop: + mova ym1, [dstq+dsq*0] + vinserti32x8 m1, [dstq+dsq*1], 1 + mova m2, [tmpq] + add tmpq, 32*2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w32_loop + RET + +cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask +%define base r6-blend_h_avx512icl_table + lea r6, [blend_h_avx512icl_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + lea maskq, [base+obmc_masks+hq*2] + vpbroadcastd m5, [base+pw_512] + lea hd, [hq*3] + add wq, r6 + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd xmm0, [dstq+dsq*0] + pinsrw xmm0, [dstq+dsq*1], 1 + movd xmm2, [maskq+hq*2] + movd xmm1, [tmpq] + add tmpq, 2*2 + punpcklwd xmm2, xmm2 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm2 + pmulhrsw xmm0, xm5 + packuswb xmm0, xmm0 + pextrw [dstq+dsq*0], xmm0, 0 + pextrw [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +.w4: + mova xmm3, [blend_shuf] +.w4_loop: + movd xmm0, [dstq+dsq*0] + pinsrd xmm0, [dstq+dsq*1], 1 + movd xmm2, [maskq+hq*2] + movq xmm1, [tmpq] + add tmpq, 4*2 + pshufb xmm2, xmm3 + punpcklbw xmm0, xmm1 + pmaddubsw xmm0, xmm2 + pmulhrsw xmm0, xm5 + packuswb xmm0, xmm0 + movd [dstq+dsq*0], xmm0 + pextrd [dstq+dsq*1], xmm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +.w8: + vbroadcasti128 ymm4, [blend_shuf] + shufpd ymm4, ymm4, 0x03 +.w8_loop: + vpbroadcastq ymm1, [dstq+dsq*0] + movq xmm0, [dstq+dsq*1] + vpblendd ymm0, ymm1, 0x30 + vpbroadcastd ymm3, [maskq+hq*2] + movq xmm1, [tmpq+8*1] + vinserti128 ymm1, [tmpq+8*0], 1 + add tmpq, 8*2 + pshufb ymm3, ymm4 + punpcklbw ymm0, ymm1 + pmaddubsw ymm0, ymm3 + pmulhrsw ymm0, ym5 + vextracti128 xmm1, ymm0, 1 + packuswb xmm0, xmm1 + movhps [dstq+dsq*0], xmm0 + movq [dstq+dsq*1], xmm0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop + vzeroupper + RET +.w16: + vbroadcasti32x4 ym4, [blend_shuf] + shufpd ym4, ym4, 0x0c +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti32x4 ym1, [dstq+dsq*1], 1 + vpbroadcastd ym3, [maskq+hq*2] + mova ym2, [tmpq] + add tmpq, 16*2 + pshufb ym3, ym4 + punpcklbw ym0, ym1, ym2 + punpckhbw ym1, ym2 + pmaddubsw ym0, ym3 + pmaddubsw ym1, ym3 + pmulhrsw ym0, ym5 + pmulhrsw ym1, ym5 + packuswb ym0, ym1 + mova [dstq+dsq*0], xm0 + vextracti32x4 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w16_loop + RET +.w32: + vbroadcasti32x4 m4, [blend_shuf] + shufpd m4, m4, 0xf0 +.w32_loop: + mova ym1, [dstq+dsq*0] + vinserti32x8 m1, [dstq+dsq*1], 1 + vpbroadcastd m3, [maskq+hq*2] + mova m2, [tmpq] + add tmpq, 32*2 + pshufb m3, m4 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], ym0 + vextracti32x8 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w32_loop + RET +.w64: + vpbroadcastw m3, [maskq+hq*2] + mova m1, [dstq] + mova m2, [tmpq] + add tmpq, 32*2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + inc hq + jl .w64 + RET +.w128: + vpbroadcastw m6, [maskq+hq*2] + mova m2, [dstq+64*0] + mova m1, [tmpq+64*0] + mova m3, [dstq+64*1] + mova m4, [tmpq+64*1] + add tmpq, 64*2 + punpcklbw m0, m2, m1 + punpckhbw m2, m1 + pmaddubsw m0, m6 + pmaddubsw m2, m6 + punpcklbw m1, m3, m4 + punpckhbw m3, m4 + pmaddubsw m1, m6 + pmaddubsw m3, m6 + REPX {pmulhrsw x, m5}, m0, m2, m1, m3 + packuswb m0, m2 + packuswb m1, m3 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, dsq + inc hq + jl .w128 + RET + +cglobal resize_8bpc, 6, 12, 19, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + mov r6, ~0 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + kmovq k3, r6 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x + LEA r7, $$ +%define base r7-$$ + vpbroadcastd m3, [base+pw_m256] + vpbroadcastd m7, [base+pd_63] + vbroadcasti32x4 m15, [base+pb_8x0_8x8] + vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15] + pslld m5, 4 ; dx*16 + pslld m6, 14 + pxor m2, m2 + mova m16, [base+resize_permA] + mova m17, [base+resize_permB] + mova xm18, [base+resize_permC] +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx +.loop_x: + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + vptestmd k4, m1, m1 + pand m9, m7 ; filter offset (masked) + ktestw k4, k4 + jz .load + vextracti32x8 ym12, m0, 1 + vextracti32x8 ym13, m1, 1 + kmovq k1, k3 + kmovq k2, k3 + vpgatherdq m10{k1}, [srcq+ym0] + vpgatherdq m11{k2}, [srcq+ym12] + kmovq k1, k3 + kmovq k2, k3 + vpgatherdq m14{k1}, [base+resize_shuf+4+ym1] + vpgatherdq m0{k2}, [base+resize_shuf+4+ym13] + mova m12, m16 + mova m13, m17 + paddb m14, m15 + paddb m0, m15 + pshufb m10, m14 + pshufb m11, m0 + vpermi2d m12, m10, m11 + vpermi2d m13, m10, m11 + jmp .filter +.load: + kmovq k1, k3 + kmovq k2, k3 + vpgatherdd m12{k1}, [srcq+m0+0] + vpgatherdd m13{k2}, [srcq+m0+4] +.filter: + kmovq k1, k3 + kmovq k2, k3 + vpgatherdd m10{k1}, [base+resize_filter+m9*8+0] + vpgatherdd m11{k2}, [base+resize_filter+m9*8+4] + mova m14, m2 + vpdpbusd m14, m12, m10 + vpdpbusd m14, m13, m11 + packssdw m14, m14 + pmulhrsw m14, m3 + packuswb m14, m14 + vpermd m14, m18, m14 + mova [dstq+xq], xm14 + paddd m4, m5 + add xd, 16 + cmp xd, dst_wd + jl .loop_x + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_sse.asm b/third_party/dav1d/src/x86/mc_sse.asm new file mode 100644 index 0000000000..54939c647a --- /dev/null +++ b/third_party/dav1d/src/x86/mc_sse.asm @@ -0,0 +1,9599 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; Copyright © 2018, VideoLabs +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +; dav1d_obmc_masks[] with 64-x interleaved +obmc_masks: db 0, 0, 0, 0 + ; 2 @4 + db 45, 19, 64, 0 + ; 4 @8 + db 39, 25, 50, 14, 59, 5, 64, 0 + ; 8 @16 + db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 + ; 16 @32 + db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 + db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 + ; 32 @64 + db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 + db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 + db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 + +warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 +warp_8x8_shufB: db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 +warp_8x8_shufC: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 +warp_8x8_shufD: db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11 +subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 +bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 +unpckw: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +rescale_mul: dd 0, 1, 2, 3 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 + +wm_420_sign: times 4 dw 258 + times 4 dw 257 +wm_422_sign: times 8 db 128 + times 8 db 127 + +pb_8x0_8x8: times 8 db 0 + times 8 db 8 +bdct_lb_dw: times 4 db 0 + times 4 db 4 + times 4 db 8 + times 4 db 12 + +pb_64: times 16 db 64 +pw_m256: times 8 dw -256 +pw_1: times 8 dw 1 +pw_2: times 8 dw 2 +pw_8: times 8 dw 8 +pw_15: times 8 dw 15 +pw_26: times 8 dw 26 +pw_34: times 8 dw 34 +pw_512: times 8 dw 512 +pw_1024: times 8 dw 1024 +pw_2048: times 8 dw 2048 +pw_6903: times 8 dw 6903 +pw_8192: times 8 dw 8192 +pd_32: times 4 dd 32 +pd_63: times 4 dd 63 +pd_512: times 4 dd 512 +pd_16384: times 4 dd 16484 +pd_32768: times 4 dd 32768 +pd_262144:times 4 dd 262144 +pd_0x3ff: times 4 dd 0x3ff +pd_0x4000:times 4 dd 0x4000 +pq_0x40000000: times 2 dq 0x40000000 + +const mc_warp_filter2 ; dav1d_mc_warp_filter[] reordered for pmaddubsw usage + ; [-1, 0) + db 0, 127, 0, 0, 0, 1, 0, 0, 0, 127, 0, 0, -1, 2, 0, 0 + db 1, 127, -1, 0, -3, 4, 0, 0, 1, 126, -2, 0, -4, 6, 1, 0 + db 1, 126, -3, 0, -5, 8, 1, 0, 1, 125, -4, 0, -6, 11, 1, 0 + db 1, 124, -4, 0, -7, 13, 1, 0, 2, 123, -5, 0, -8, 15, 1, 0 + db 2, 122, -6, 0, -9, 18, 1, 0, 2, 121, -6, 0, -10, 20, 1, 0 + db 2, 120, -7, 0, -11, 22, 2, 0, 2, 119, -8, 0, -12, 25, 2, 0 + db 3, 117, -8, 0, -13, 27, 2, 0, 3, 116, -9, 0, -13, 29, 2, 0 + db 3, 114, -10, 0, -14, 32, 3, 0, 3, 113, -10, 0, -15, 35, 2, 0 + db 3, 111, -11, 0, -15, 37, 3, 0, 3, 109, -11, 0, -16, 40, 3, 0 + db 3, 108, -12, 0, -16, 42, 3, 0, 4, 106, -13, 0, -17, 45, 3, 0 + db 4, 104, -13, 0, -17, 47, 3, 0, 4, 102, -14, 0, -17, 50, 3, 0 + db 4, 100, -14, 0, -17, 52, 3, 0, 4, 98, -15, 0, -18, 55, 4, 0 + db 4, 96, -15, 0, -18, 58, 3, 0, 4, 94, -16, 0, -18, 60, 4, 0 + db 4, 91, -16, 0, -18, 63, 4, 0, 4, 89, -16, 0, -18, 65, 4, 0 + db 4, 87, -17, 0, -18, 68, 4, 0, 4, 85, -17, 0, -18, 70, 4, 0 + db 4, 82, -17, 0, -18, 73, 4, 0, 4, 80, -17, 0, -18, 75, 4, 0 + db 4, 78, -18, 0, -18, 78, 4, 0, 4, 75, -18, 0, -17, 80, 4, 0 + db 4, 73, -18, 0, -17, 82, 4, 0, 4, 70, -18, 0, -17, 85, 4, 0 + db 4, 68, -18, 0, -17, 87, 4, 0, 4, 65, -18, 0, -16, 89, 4, 0 + db 4, 63, -18, 0, -16, 91, 4, 0, 4, 60, -18, 0, -16, 94, 4, 0 + db 3, 58, -18, 0, -15, 96, 4, 0, 4, 55, -18, 0, -15, 98, 4, 0 + db 3, 52, -17, 0, -14, 100, 4, 0, 3, 50, -17, 0, -14, 102, 4, 0 + db 3, 47, -17, 0, -13, 104, 4, 0, 3, 45, -17, 0, -13, 106, 4, 0 + db 3, 42, -16, 0, -12, 108, 3, 0, 3, 40, -16, 0, -11, 109, 3, 0 + db 3, 37, -15, 0, -11, 111, 3, 0, 2, 35, -15, 0, -10, 113, 3, 0 + db 3, 32, -14, 0, -10, 114, 3, 0, 2, 29, -13, 0, -9, 116, 3, 0 + db 2, 27, -13, 0, -8, 117, 3, 0, 2, 25, -12, 0, -8, 119, 2, 0 + db 2, 22, -11, 0, -7, 120, 2, 0, 1, 20, -10, 0, -6, 121, 2, 0 + db 1, 18, -9, 0, -6, 122, 2, 0, 1, 15, -8, 0, -5, 123, 2, 0 + db 1, 13, -7, 0, -4, 124, 1, 0, 1, 11, -6, 0, -4, 125, 1, 0 + db 1, 8, -5, 0, -3, 126, 1, 0, 1, 6, -4, 0, -2, 126, 1, 0 + db 0, 4, -3, 0, -1, 127, 1, 0, 0, 2, -1, 0, 0, 127, 0, 0 + ; [0, 1) + db 0, 0, 1, 0, 0, 127, 0, 0, 0, -1, 2, 0, 0, 127, 0, 0 + db 0, -3, 4, 1, 1, 127, -2, 0, 0, -5, 6, 1, 1, 127, -2, 0 + db 0, -6, 8, 1, 2, 126, -3, 0, -1, -7, 11, 2, 2, 126, -4, -1 + db -1, -8, 13, 2, 3, 125, -5, -1, -1, -10, 16, 3, 3, 124, -6, -1 + db -1, -11, 18, 3, 4, 123, -7, -1, -1, -12, 20, 3, 4, 122, -7, -1 + db -1, -13, 23, 3, 4, 121, -8, -1, -2, -14, 25, 4, 5, 120, -9, -1 + db -1, -15, 27, 4, 5, 119, -10, -1, -1, -16, 30, 4, 5, 118, -11, -1 + db -2, -17, 33, 5, 6, 116, -12, -1, -2, -17, 35, 5, 6, 114, -12, -1 + db -2, -18, 38, 5, 6, 113, -13, -1, -2, -19, 41, 6, 7, 111, -14, -2 + db -2, -19, 43, 6, 7, 110, -15, -2, -2, -20, 46, 6, 7, 108, -15, -2 + db -2, -20, 49, 6, 7, 106, -16, -2, -2, -21, 51, 7, 7, 104, -16, -2 + db -2, -21, 54, 7, 7, 102, -17, -2, -2, -21, 56, 7, 8, 100, -18, -2 + db -2, -22, 59, 7, 8, 98, -18, -2, -2, -22, 62, 7, 8, 96, -19, -2 + db -2, -22, 64, 7, 8, 94, -19, -2, -2, -22, 67, 8, 8, 91, -20, -2 + db -2, -22, 69, 8, 8, 89, -20, -2, -2, -22, 72, 8, 8, 87, -21, -2 + db -2, -21, 74, 8, 8, 84, -21, -2, -2, -22, 77, 8, 8, 82, -21, -2 + db -2, -21, 79, 8, 8, 79, -21, -2, -2, -21, 82, 8, 8, 77, -22, -2 + db -2, -21, 84, 8, 8, 74, -21, -2, -2, -21, 87, 8, 8, 72, -22, -2 + db -2, -20, 89, 8, 8, 69, -22, -2, -2, -20, 91, 8, 8, 67, -22, -2 + db -2, -19, 94, 8, 7, 64, -22, -2, -2, -19, 96, 8, 7, 62, -22, -2 + db -2, -18, 98, 8, 7, 59, -22, -2, -2, -18, 100, 8, 7, 56, -21, -2 + db -2, -17, 102, 7, 7, 54, -21, -2, -2, -16, 104, 7, 7, 51, -21, -2 + db -2, -16, 106, 7, 6, 49, -20, -2, -2, -15, 108, 7, 6, 46, -20, -2 + db -2, -15, 110, 7, 6, 43, -19, -2, -2, -14, 111, 7, 6, 41, -19, -2 + db -1, -13, 113, 6, 5, 38, -18, -2, -1, -12, 114, 6, 5, 35, -17, -2 + db -1, -12, 116, 6, 5, 33, -17, -2, -1, -11, 118, 5, 4, 30, -16, -1 + db -1, -10, 119, 5, 4, 27, -15, -1, -1, -9, 120, 5, 4, 25, -14, -2 + db -1, -8, 121, 4, 3, 23, -13, -1, -1, -7, 122, 4, 3, 20, -12, -1 + db -1, -7, 123, 4, 3, 18, -11, -1, -1, -6, 124, 3, 3, 16, -10, -1 + db -1, -5, 125, 3, 2, 13, -8, -1, -1, -4, 126, 2, 2, 11, -7, -1 + db 0, -3, 126, 2, 1, 8, -6, 0, 0, -2, 127, 1, 1, 6, -5, 0 + db 0, -2, 127, 1, 1, 4, -3, 0, 0, 0, 127, 0, 0, 2, -1, 0 + ; [1, 2) + db 0, 0, 127, 0, 0, 1, 0, 0, 0, 0, 127, 0, 0, -1, 2, 0 + db 0, 1, 127, -1, 0, -3, 4, 0, 0, 1, 126, -2, 0, -4, 6, 1 + db 0, 1, 126, -3, 0, -5, 8, 1, 0, 1, 125, -4, 0, -6, 11, 1 + db 0, 1, 124, -4, 0, -7, 13, 1, 0, 2, 123, -5, 0, -8, 15, 1 + db 0, 2, 122, -6, 0, -9, 18, 1, 0, 2, 121, -6, 0, -10, 20, 1 + db 0, 2, 120, -7, 0, -11, 22, 2, 0, 2, 119, -8, 0, -12, 25, 2 + db 0, 3, 117, -8, 0, -13, 27, 2, 0, 3, 116, -9, 0, -13, 29, 2 + db 0, 3, 114, -10, 0, -14, 32, 3, 0, 3, 113, -10, 0, -15, 35, 2 + db 0, 3, 111, -11, 0, -15, 37, 3, 0, 3, 109, -11, 0, -16, 40, 3 + db 0, 3, 108, -12, 0, -16, 42, 3, 0, 4, 106, -13, 0, -17, 45, 3 + db 0, 4, 104, -13, 0, -17, 47, 3, 0, 4, 102, -14, 0, -17, 50, 3 + db 0, 4, 100, -14, 0, -17, 52, 3, 0, 4, 98, -15, 0, -18, 55, 4 + db 0, 4, 96, -15, 0, -18, 58, 3, 0, 4, 94, -16, 0, -18, 60, 4 + db 0, 4, 91, -16, 0, -18, 63, 4, 0, 4, 89, -16, 0, -18, 65, 4 + db 0, 4, 87, -17, 0, -18, 68, 4, 0, 4, 85, -17, 0, -18, 70, 4 + db 0, 4, 82, -17, 0, -18, 73, 4, 0, 4, 80, -17, 0, -18, 75, 4 + db 0, 4, 78, -18, 0, -18, 78, 4, 0, 4, 75, -18, 0, -17, 80, 4 + db 0, 4, 73, -18, 0, -17, 82, 4, 0, 4, 70, -18, 0, -17, 85, 4 + db 0, 4, 68, -18, 0, -17, 87, 4, 0, 4, 65, -18, 0, -16, 89, 4 + db 0, 4, 63, -18, 0, -16, 91, 4, 0, 4, 60, -18, 0, -16, 94, 4 + db 0, 3, 58, -18, 0, -15, 96, 4, 0, 4, 55, -18, 0, -15, 98, 4 + db 0, 3, 52, -17, 0, -14, 100, 4, 0, 3, 50, -17, 0, -14, 102, 4 + db 0, 3, 47, -17, 0, -13, 104, 4, 0, 3, 45, -17, 0, -13, 106, 4 + db 0, 3, 42, -16, 0, -12, 108, 3, 0, 3, 40, -16, 0, -11, 109, 3 + db 0, 3, 37, -15, 0, -11, 111, 3, 0, 2, 35, -15, 0, -10, 113, 3 + db 0, 3, 32, -14, 0, -10, 114, 3, 0, 2, 29, -13, 0, -9, 116, 3 + db 0, 2, 27, -13, 0, -8, 117, 3, 0, 2, 25, -12, 0, -8, 119, 2 + db 0, 2, 22, -11, 0, -7, 120, 2, 0, 1, 20, -10, 0, -6, 121, 2 + db 0, 1, 18, -9, 0, -6, 122, 2, 0, 1, 15, -8, 0, -5, 123, 2 + db 0, 1, 13, -7, 0, -4, 124, 1, 0, 1, 11, -6, 0, -4, 125, 1 + db 0, 1, 8, -5, 0, -3, 126, 1, 0, 1, 6, -4, 0, -2, 126, 1 + db 0, 0, 4, -3, 0, -1, 127, 1, 0, 0, 2, -1, 0, 0, 127, 0 + db 0, 0, 2, -1, 0, 0, 127, 0 + +pw_258: times 2 dw 258 + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +%macro BIDIR_JMP_TABLE 2-* + ;evaluated at definition time (in loop below) + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) + ; dynamically generated label + %%table: + %rep %0 - 2 ; repeat for num args + dd %%prefix %+ .w%3 - %%base + %rotate 1 + %endrep +%endmacro + +BIDIR_JMP_TABLE avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, ssse3, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, ssse3, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, ssse3, 2, 4, 8, 16, 16, 16, 16 + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine prep_sse2 mangle(private_prefix %+ _prep_bilin_8bpc_sse2.prep) +%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_8bpc_ssse3.put) +%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_8bpc_ssse3.prep) + +BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +HV_JMP_TABLE prep, 8tap, sse2, 1, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, sse2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, ssse3, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, ssse3, 1, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, ssse3, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, ssse3, 7, 4, 8, 16, 32, 64, 128 + +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) +%%table: + %rep %0 - 2 + dw %%base %+ .w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_1024: + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base + %rotate 1 + %endrep + %rotate 2 +%%dy_2048: + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base + %rotate 1 + %endrep +%endmacro + +SCALED_JMP_TABLE put_8tap_scaled, ssse3, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, ssse3, 4, 8, 16, 32, 64, 128 + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +SECTION .text + +INIT_XMM ssse3 + +%if ARCH_X86_32 + DECLARE_REG_TMP 1 + %define base t0-put_ssse3 +%else + DECLARE_REG_TMP 7 + %define base 0 +%endif + +%macro RESTORE_DSQ_32 1 + %if ARCH_X86_32 + mov %1, dsm ; restore dsq + %endif +%endmacro + +cglobal put_bilin_8bpc, 1, 8, 0, dst, ds, src, ss, w, h, mxy + movifnidn mxyd, r6m ; mx + LEA t0, put_ssse3 + movifnidn srcq, srcmp + movifnidn ssq, ssmp + tzcnt wd, wm + mov hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [t0+wq*2+table_offset(put,)] + add wq, t0 + RESTORE_DSQ_32 t0 + jmp wq +.put_w2: + movzx r4d, word [srcq+ssq*0] + movzx r6d, word [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r4w + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r4d, [srcq+ssq*0] + mov r6d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r4d + mov [dstq+dsq*1], r6d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + movq m0, [srcq+ssq*0] + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq [dstq+dsq*0], m0 + movq [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu m0, [srcq+ssq*0+16*0] + movu m1, [srcq+ssq*0+16*1] + movu m2, [srcq+ssq*1+16*0] + movu m3, [srcq+ssq*1+16*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+16*0], m0 + mova [dstq+dsq*0+16*1], m1 + mova [dstq+dsq*1+16*0], m2 + mova [dstq+dsq*1+16*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + add srcq, ssq + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + add dstq, dsq + dec hd + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + movu m0, [srcq+16*4] + movu m1, [srcq+16*5] + movu m2, [srcq+16*6] + movu m3, [srcq+16*7] + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + mova [dstq+16*6], m2 + mova [dstq+16*7], m3 + add srcq, ssq + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 + ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 + imul mxyd, 0x00ff00ff + mova m4, [base+bilin_h_shuf8] + mova m0, [base+bilin_h_shuf4] + add mxyd, 0x00100010 + movd m5, mxyd + mov mxyd, r7m ; my + pshufd m5, m5, q0000 + test mxyd, mxyd + jnz .hv + movzx wd, word [t0+wq*2+table_offset(put, _bilin_h)] + mova m3, [base+pw_2048] + add wq, t0 + movifnidn dsq, dsmp + jmp wq +.h_w2: + pshufd m4, m4, q3120 ; m4 = {1, 0, 2, 1, 5, 4, 6, 5} +.h_w2_loop: + movd m0, [srcq+ssq*0] + movd m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq m0, m1 + pshufb m0, m4 + pmaddubsw m0, m5 + pmulhrsw m0, m3 + packuswb m0, m0 + movd r6d, m0 + mov [dstq+dsq*0], r6w + shr r6d, 16 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movq m4, [srcq+ssq*0] + movhps m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m4, m0 + pmaddubsw m4, m5 + pmulhrsw m4, m3 + packuswb m4, m4 + movd [dstq+dsq*0], m4 + psrlq m4, 32 + movd [dstq+dsq*1], m4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w16 + RET +.h_w32: + movu m0, [srcq+mmsize*0+8*0] + movu m1, [srcq+mmsize*0+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movu m1, [srcq+mmsize*1+8*0] + movu m2, [srcq+mmsize*1+8*1] + add srcq, ssq + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + packuswb m1, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, dsq + dec hd + jg .h_w32 + RET +.h_w64: + mov r6, -16*3 +.h_w64_loop: + movu m0, [srcq+r6+16*3+8*0] + movu m1, [srcq+r6+16*3+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+r6+16*3], m0 + add r6, 16 + jle .h_w64_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + mov r6, -16*7 +.h_w128_loop: + movu m0, [srcq+r6+16*7+8*0] + movu m1, [srcq+r6+16*7+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+r6+16*7], m0 + add r6, 16 + jle .h_w128_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [t0+wq*2+table_offset(put, _bilin_v)] + imul mxyd, 0x00ff00ff + mova m5, [base+pw_2048] + add mxyd, 0x00100010 + add wq, t0 + movd m4, mxyd + pshufd m4, m4, q0000 + movifnidn dsq, dsmp + jmp wq +.v_w2: + movd m0, [srcq+ssq*0] +.v_w2_loop: + pinsrw m0, [srcq+ssq*1], 1 ; 0 1 + lea srcq, [srcq+ssq*2] + pshuflw m1, m0, q2301 + pinsrw m0, [srcq+ssq*0], 0 ; 2 1 + punpcklbw m1, m0 + pmaddubsw m1, m4 + pmulhrsw m1, m5 + packuswb m1, m1 + movd r6d, m1 + mov [dstq+dsq*1], r6w + shr r6d, 16 + mov [dstq+dsq*0], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd m0, [srcq+ssq*0] +.v_w4_loop: + movd m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m1, m0 + movd m0, [srcq+ssq*0] + punpckldq m1, m2 ; 0 1 + punpckldq m2, m0 ; 1 2 + punpcklbw m1, m2 + pmaddubsw m1, m4 + pmulhrsw m1, m5 + packuswb m1, m1 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + ; + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq m0, [srcq+ssq*0] +.v_w8_loop: + movq m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m1, m0 + movq m0, [srcq+ssq*0] + punpcklbw m1, m2 + punpcklbw m2, m0 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +%macro PUT_BILIN_V_W16 0 + movu m0, [srcq+ssq*0] +%%loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m1, m0 + mova m2, m0 + movu m0, [srcq+ssq*0] + punpcklbw m1, m3 + punpckhbw m2, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + pmaddubsw m2, m4 + pmaddubsw m3, m4 + pmulhrsw m2, m5 + pmulhrsw m3, m5 + packuswb m2, m3 + mova [dstq+dsq*0], m1 + mova [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg %%loop +%endmacro +.v_w16: + PUT_BILIN_V_W16 + RET +.v_w128: + lea r6d, [hq+(7<<16)] + jmp .v_w16gt +.v_w64: + lea r6d, [hq+(3<<16)] + jmp .v_w16gt +.v_w32: + lea r6d, [hq+(1<<16)] +.v_w16gt: + mov r4, srcq +%if ARCH_X86_64 + mov r7, dstq +%endif +.v_w16gt_loop: + PUT_BILIN_V_W16 +%if ARCH_X86_64 + add r4, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 +%else + mov dstq, dstmp + add r4, 16 + movzx hd, r6w + add dstq, 16 + mov srcq, r4 + mov dstmp, dstq +%endif + sub r6d, 1<<16 + jg .v_w16gt + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 + ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 + movzx wd, word [t0+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 ; can't shift by 12 due to signed overflow + mova m7, [base+pw_15] + movd m6, mxyd + add wq, t0 + pshuflw m6, m6, q0000 + paddb m5, m5 + punpcklqdq m6, m6 + jmp wq +.hv_w2: + RESTORE_DSQ_32 t0 + movd m0, [srcq+ssq*0] + punpckldq m0, m0 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w2_loop: + movd m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movd m2, [srcq+ssq*0] + punpckldq m1, m2 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 _ 2 _ + shufps m2, m0, m1, q1032 ; 0 _ 1 _ + mova m0, m1 + psubw m1, m2 ; 2 * (src[x + src_stride] - src[x]) + pmulhw m1, m6 ; (my * (src[x + src_stride] - src[x]) >> 4 + pavgw m2, m7 ; src[x] + 8 + paddw m1, m2 ; src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8 + psrlw m1, 4 + packuswb m1, m1 +%if ARCH_X86_64 + movq r6, m1 +%else + pshuflw m1, m1, q2020 + movd r6d, m1 +%endif + mov [dstq+dsq*0], r6w + shr r6, gprsize*4 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova m4, [base+bilin_h_shuf4] + movddup m0, [srcq+ssq*0] + movifnidn dsq, dsmp + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w4_loop: + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps m1, [srcq+ssq*0] + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 + shufps m2, m0, m1, q1032 ; 0 1 + mova m0, m1 + psubw m1, m2 + pmulhw m1, m6 + pavgw m2, m7 + paddw m1, m2 + psrlw m1, 4 + packuswb m1, m1 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + movu m0, [srcq+ssq*0] + movifnidn dsq, dsmp + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m1, m2, m0 + pmulhw m1, m6 + pavgw m0, m7 + paddw m1, m0 + movu m0, [srcq+ssq*0] + pshufb m0, m4 + pmaddubsw m0, m5 + psubw m3, m0, m2 + pmulhw m3, m6 + pavgw m2, m7 + paddw m3, m2 + psrlw m1, 4 + psrlw m3, 4 + packuswb m1, m3 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w128: + lea r6d, [hq+(7<<16)] + jmp .hv_w16_start +.hv_w64: + lea r6d, [hq+(3<<16)] + jmp .hv_w16_start +.hv_w32: + lea r6d, [hq+(1<<16)] +.hv_w16_start: + mov r4, srcq +%if ARCH_X86_32 + %define m8 [dstq] +%else + mov r7, dstq +%endif +.hv_w16: + movifnidn dsq, dsmp +%if WIN64 + movaps r4m, m8 +%endif +.hv_w16_loop0: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w16_loop: + add srcq, ssq + movu m2, [srcq+8*0] + movu m3, [srcq+8*1] + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova m8, m2 + psubw m2, m0 + pmulhw m2, m6 + pavgw m0, m7 + paddw m2, m0 + mova m0, m3 + psubw m3, m1 + pmulhw m3, m6 + pavgw m1, m7 + paddw m3, m1 + mova m1, m0 + mova m0, m8 + psrlw m2, 4 + psrlw m3, 4 + packuswb m2, m3 + mova [dstq], m2 + add dstq, dsmp + dec hd + jg .hv_w16_loop +%if ARCH_X86_32 + mov dstq, dstm + add r4, 16 + movzx hd, r6w + add dstq, 16 + mov srcq, r4 + mov dstm, dstq +%else + add r4, 16 + add r7, 16 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 +%endif + sub r6d, 1<<16 + jg .hv_w16_loop0 +%if WIN64 + movaps m8, r4m +%endif + RET + +%macro PSHUFB_BILIN_H8 2 ; dst, src + %if cpuflag(ssse3) + pshufb %1, %2 + %else + psrldq %2, %1, 1 + punpcklbw %1, %2 + %endif +%endmacro + +%macro PSHUFB_BILIN_H4 3 ; dst, src, tmp + %if cpuflag(ssse3) + pshufb %1, %2 + %else + psrldq %2, %1, 1 + punpckhbw %3, %1, %2 + punpcklbw %1, %2 + punpcklqdq %1, %3 + %endif +%endmacro + +%macro PMADDUBSW 5 ; dst/src1, src2, zero, tmp, reset_zero + %if cpuflag(ssse3) + pmaddubsw %1, %2 + %else + %if %5 == 1 + pxor %3, %3 + %endif + punpckhbw %4, %1, %3 + punpcklbw %1, %1, %3 + pmaddwd %4, %2 + pmaddwd %1, %2 + packssdw %1, %4 + %endif +%endmacro + +%macro PMULHRSW 5 ; dst, src, tmp, rndval, shift + %if cpuflag(ssse3) + pmulhrsw %1, %2 + %else + punpckhwd %3, %1, %4 + punpcklwd %1, %4 + pmaddwd %3, %2 + pmaddwd %1, %2 + psrad %3, %5 + psrad %1, %5 + packssdw %1, %3 + %endif +%endmacro + +%macro PREP_BILIN 0 +%if ARCH_X86_32 + %define base r6-prep%+SUFFIX +%else + %define base 0 +%endif + +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + LEA r6, prep%+SUFFIX + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: +%if notcpuflag(ssse3) + add r6, prep_ssse3 - prep_sse2 + jmp prep_ssse3 +%else + movzx wd, word [r6+wq*2+table_offset(prep,)] + pxor m4, m4 + add wq, r6 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd m0, [srcq+strideq*0] + movd m1, [srcq+strideq*1] + movd m2, [srcq+strideq*2] + movd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpckldq m0, m1 + punpckldq m2, m3 + punpcklbw m0, m4 + punpcklbw m2, m4 + psllw m0, 4 + psllw m2, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m2 + add tmpq, 16*2 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq m0, [srcq+strideq*0] + movq m1, [srcq+strideq*1] + movq m2, [srcq+strideq*2] + movq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpcklbw m0, m4 + punpcklbw m1, m4 + punpcklbw m2, m4 + punpcklbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movu m1, [srcq+strideq*0] + movu m3, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 2 + jg .prep_w16 + RET +.prep_w128: + mov r3, -128 + jmp .prep_w32_start +.prep_w64: + mov r3, -64 + jmp .prep_w32_start +.prep_w32: + mov r3, -32 +.prep_w32_start: + sub srcq, r3 +.prep_w32_vloop: + mov r6, r3 +.prep_w32_hloop: + movu m1, [srcq+r6+16*0] + movu m3, [srcq+r6+16*1] + punpcklbw m0, m1, m4 + punpckhbw m1, m4 + punpcklbw m2, m3, m4 + punpckhbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + add r6, 32 + jl .prep_w32_hloop + add srcq, strideq + dec hd + jg .prep_w32_vloop + RET +%endif +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] +%if cpuflag(ssse3) + imul mxyd, 0x00ff00ff + mova m4, [base+bilin_h_shuf8] + add mxyd, 0x00100010 +%else + imul mxyd, 0xffff + add mxyd, 16 +%endif + movd m5, mxyd + mov mxyd, r6m ; my + pshufd m5, m5, q0000 + test mxyd, mxyd + jnz .hv + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)] +%if notcpuflag(ssse3) + WIN64_SPILL_XMM 8 + pxor m6, m6 +%endif + add wq, r6 + jmp wq +.h_w4: +%if cpuflag(ssse3) + mova m4, [base+bilin_h_shuf4] +%endif + lea stride3q, [strideq*3] +.h_w4_loop: + movq m0, [srcq+strideq*0] + movhps m0, [srcq+strideq*1] + movq m1, [srcq+strideq*2] + movhps m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + PSHUFB_BILIN_H4 m0, m4, m2 + PMADDUBSW m0, m5, m6, m2, 0 + PSHUFB_BILIN_H4 m1, m4, m2 + PMADDUBSW m1, m5, m6, m2, 0 + mova [tmpq+0 ], m0 + mova [tmpq+16], m1 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + lea stride3q, [strideq*3] +.h_w8_loop: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + movu m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + movu m0, [srcq+strideq*0+8*0] + movu m1, [srcq+strideq*0+8*1] + movu m2, [srcq+strideq*1+8*0] + movu m3, [srcq+strideq*1+8*1] + lea srcq, [srcq+strideq*2] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 2 + jg .h_w16 + RET +.h_w128: + mov r3, -128 + jmp .h_w32_start +.h_w64: + mov r3, -64 + jmp .h_w32_start +.h_w32: + mov r3, -32 +.h_w32_start: + sub srcq, r3 +.h_w32_vloop: + mov r6, r3 +.h_w32_hloop: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + movu m2, [srcq+r6+8*2] + movu m3, [srcq+r6+8*3] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + add r6, 32 + jl .h_w32_hloop + add srcq, strideq + dec hd + jg .h_w32_vloop + RET +.v: +%if notcpuflag(ssse3) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 8 +%endif + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)] +%if cpuflag(ssse3) + imul mxyd, 0x00ff00ff + add mxyd, 0x00100010 +%else + imul mxyd, 0xffff + pxor m6, m6 + add mxyd, 16 +%endif + add wq, r6 + lea stride3q, [strideq*3] + movd m5, mxyd + pshufd m5, m5, q0000 + jmp wq +.v_w4: + movd m0, [srcq+strideq*0] +.v_w4_loop: + movd m1, [srcq+strideq*1] + movd m2, [srcq+strideq*2] + movd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpckldq m0, m1 + punpckldq m1, m2 + punpcklbw m0, m1 ; 01 12 + PMADDUBSW m0, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + movd m0, [srcq+strideq*0] + punpckldq m2, m3 + punpckldq m3, m0 + punpcklbw m2, m3 ; 23 34 + PMADDUBSW m2, m5, m6, m7, 0 + mova [tmpq+16*1], m2 + add tmpq, 16*2 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq m0, [srcq+strideq*0] +.v_w8_loop: + movq m1, [srcq+strideq*1] + movq m2, [srcq+strideq*2] + movq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpcklbw m0, m1 ; 01 + punpcklbw m1, m2 ; 12 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + movq m0, [srcq+strideq*0] + punpcklbw m2, m3 ; 23 + punpcklbw m3, m0 ; 34 + PMADDUBSW m2, m5, m6, m7, 0 + mova [tmpq+16*1], m1 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + movu m0, [srcq+strideq*0] +.v_w16_loop: + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + movu m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpcklbw m4, m0, m1 + punpckhbw m0, m1 + PMADDUBSW m4, m5, m6, m7, 0 + PMADDUBSW m0, m5, m6, m7, 0 + mova [tmpq+16*0], m4 + punpcklbw m4, m1, m2 + punpckhbw m1, m2 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*1], m0 + movu m0, [srcq+strideq*0] + PMADDUBSW m1, m5, m6, m7, 0 + mova [tmpq+16*2], m4 + punpcklbw m4, m2, m3 + punpckhbw m2, m3 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*3], m1 + PMADDUBSW m2, m5, m6, m7, 0 + mova [tmpq+16*4], m4 + punpcklbw m4, m3, m0 + punpckhbw m3, m0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*5], m2 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*6], m4 + mova [tmpq+16*7], m3 + add tmpq, 16*8 + sub hd, 4 + jg .v_w16_loop + RET +.v_w128: + lea r3d, [hq+(3<<8)] + mov r6d, 256 + jmp .v_w32_start +.v_w64: + lea r3d, [hq+(1<<8)] + mov r6d, 128 + jmp .v_w32_start +.v_w32: + xor r3d, r3d + mov r6d, 64 +.v_w32_start: +%if ARCH_X86_64 + %if WIN64 + PUSH r7 + %endif + mov r7, tmpq +%endif + mov r5, srcq +.v_w32_hloop: + movu m0, [srcq+strideq*0+16*0] + movu m1, [srcq+strideq*0+16*1] +.v_w32_vloop: + movu m2, [srcq+strideq*1+16*0] + movu m3, [srcq+strideq*1+16*1] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m0, m2 + punpckhbw m0, m2 + PMADDUBSW m4, m5, m6, m7, 0 + PMADDUBSW m0, m5, m6, m7, 0 + mova [tmpq+16*0], m4 + mova [tmpq+16*1], m0 + movu m0, [srcq+strideq*0+16*0] + punpcklbw m4, m1, m3 + punpckhbw m1, m3 + PMADDUBSW m4, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + mova [tmpq+16*2], m4 + mova [tmpq+16*3], m1 + movu m1, [srcq+strideq*0+16*1] + add tmpq, r6 + punpcklbw m4, m2, m0 + punpckhbw m2, m0 + PMADDUBSW m4, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + mova [tmpq+16*0], m4 + mova [tmpq+16*1], m2 + punpcklbw m4, m3, m1 + punpckhbw m3, m1 + PMADDUBSW m4, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*2], m4 + mova [tmpq+16*3], m3 + add tmpq, r6 + sub hd, 2 + jg .v_w32_vloop + add r5, 32 + movzx hd, r3b + mov srcq, r5 +%if ARCH_X86_64 + add r7, 16*4 + mov tmpq, r7 +%else + mov tmpq, tmpmp + add tmpq, 16*4 + mov tmpmp, tmpq +%endif + sub r3d, 1<<8 + jg .v_w32_hloop +%if WIN64 + POP r7 +%endif + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) + movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)] +%assign stack_offset stack_offset - stack_size_padded +%if cpuflag(ssse3) + imul mxyd, 0x08000800 + WIN64_SPILL_XMM 8 +%else + or mxyd, 1<<16 + WIN64_SPILL_XMM 9 + %if ARCH_X86_64 + mova m8, [base+pw_8] + %else + %define m8 [base+pw_8] + %endif + pxor m7, m7 +%endif + movd m6, mxyd + add wq, r6 + pshufd m6, m6, q0000 + jmp wq +.hv_w4: +%if cpuflag(ssse3) + mova m4, [base+bilin_h_shuf4] + movddup m0, [srcq+strideq*0] +%else + movhps m0, [srcq+strideq*0] +%endif + lea r3, [strideq*3] + PSHUFB_BILIN_H4 m0, m4, m3 + PMADDUBSW m0, m5, m7, m4, 0 ; _ 0 +.hv_w4_loop: + movq m1, [srcq+strideq*1] + movhps m1, [srcq+strideq*2] + movq m2, [srcq+r3 ] + lea srcq, [srcq+strideq*4] + movhps m2, [srcq+strideq*0] + PSHUFB_BILIN_H4 m1, m4, m3 + PSHUFB_BILIN_H4 m2, m4, m3 + PMADDUBSW m1, m5, m7, m4, 0 ; 1 2 + PMADDUBSW m2, m5, m7, m4, 0 ; 3 4 + shufpd m0, m1, 0x01 ; 0 1 + shufpd m3, m1, m2, 0x01 ; 2 3 + psubw m1, m0 + PMULHRSW m1, m6, m4, m8, 4 + paddw m1, m0 + mova m0, m2 + psubw m2, m3 + PMULHRSW m2, m6, m4, m8, 4 + paddw m2, m3 + mova [tmpq+16*0], m1 + mova [tmpq+16*1], m2 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + movu m0, [srcq+strideq*0] + PSHUFB_BILIN_H8 m0, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 0 +.hv_w8_loop: + movu m1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu m2, [srcq+strideq*0] + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PMADDUBSW m1, m5, m7, m4, 0 ; 1 + PMADDUBSW m2, m5, m7, m4, 0 ; 2 + psubw m3, m1, m0 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m0 + mova m0, m2 + psubw m2, m1 + PMULHRSW m2, m6, m4, m8, 4 + paddw m2, m1 + mova [tmpq+16*0], m3 + mova [tmpq+16*1], m2 + add tmpq, 16*2 + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w128: + lea r3d, [hq+(7<<8)] + mov r5d, 256 + jmp .hv_w16_start +.hv_w64: + lea r3d, [hq+(3<<8)] + mov r5d, 128 + jmp .hv_w16_start +.hv_w32: + lea r3d, [hq+(1<<8)] + mov r5d, 64 + jmp .hv_w16_start +.hv_w16: + xor r3d, r3d + mov r5d, 32 +.hv_w16_start: +%if ARCH_X86_64 || cpuflag(ssse3) + mov r6, srcq +%endif +%if ARCH_X86_64 + %if WIN64 + PUSH r7 + %endif + mov r7, tmpq +%endif +.hv_w16_hloop: + movu m0, [srcq+strideq*0+8*0] + movu m1, [srcq+strideq*0+8*1] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 0a + PMADDUBSW m1, m5, m7, m4, 0 ; 0b +.hv_w16_vloop: + movu m2, [srcq+strideq*1+8*0] + PSHUFB_BILIN_H8 m2, m4 + PMADDUBSW m2, m5, m7, m4, 0 ; 1a + psubw m3, m2, m0 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m0 + mova [tmpq+16*0], m3 + movu m3, [srcq+strideq*1+8*1] + lea srcq, [srcq+strideq*2] + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m3, m5, m7, m4, 0 ; 1b + psubw m0, m3, m1 + PMULHRSW m0, m6, m4, m8, 4 + paddw m0, m1 + mova [tmpq+16*1], m0 + add tmpq, r5 + movu m0, [srcq+strideq*0+8*0] + PSHUFB_BILIN_H8 m0, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 2a + psubw m1, m0, m2 + PMULHRSW m1, m6, m4, m8, 4 + paddw m1, m2 + mova [tmpq+16*0], m1 + movu m1, [srcq+strideq*0+8*1] + PSHUFB_BILIN_H8 m1, m4 + PMADDUBSW m1, m5, m7, m4, 0 ; 2b + psubw m2, m1, m3 + PMULHRSW m2, m6, m4, m8, 4 + paddw m2, m3 + mova [tmpq+16*1], m2 + add tmpq, r5 + sub hd, 2 + jg .hv_w16_vloop + movzx hd, r3b +%if ARCH_X86_64 + add r6, 16 + add r7, 2*16 + mov srcq, r6 + mov tmpq, r7 +%elif cpuflag(ssse3) + mov tmpq, tmpm + add r6, 16 + add tmpq, 2*16 + mov srcq, r6 + mov tmpm, tmpq +%else + mov srcq, srcm + mov tmpq, tmpm + add srcq, 16 + add tmpq, 2*16 + mov srcm, srcq + mov tmpm, tmpq +%endif + sub r3d, 1<<8 + jg .hv_w16_hloop +%if WIN64 + POP r7 +%endif + RET +%endmacro + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; prefix, type, type_h, type_v +cglobal %1_%2_8bpc + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) +%endif +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 1, 2 +%elif WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +FN put_8tap, sharp, SHARP, SHARP +FN put_8tap, sharp_smooth, SHARP, SMOOTH +FN put_8tap, smooth_sharp, SMOOTH, SHARP +FN put_8tap, smooth, SMOOTH, SMOOTH +FN put_8tap, sharp_regular, SHARP, REGULAR +FN put_8tap, regular_sharp, REGULAR, SHARP +FN put_8tap, smooth_regular, SMOOTH, REGULAR +FN put_8tap, regular_smooth, REGULAR, SMOOTH +FN put_8tap, regular, REGULAR, REGULAR + +%if ARCH_X86_32 + %define base_reg r1 + %define base base_reg-put_ssse3 +%else + %define base_reg r8 + %define base 0 +%endif + +cglobal put_8tap_8bpc, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +%assign org_stack_offset stack_offset + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h +%if ARCH_X86_64 + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v +%else + imul ssd, mym, 0x010101 + add ssd, t1d ; 8tap_v, my, 4tap_v + mov srcq, srcm +%endif + mov wd, wm + movifnidn hd, hm + LEA base_reg, put_ssse3 + test mxd, 0xf00 + jnz .h +%if ARCH_X86_32 + test ssd, 0xf00 +%else + test myd, 0xf00 +%endif + jnz .v + tzcnt wd, wd + movzx wd, word [base_reg+wq*2+table_offset(put,)] + add wq, base_reg +; put_bilin mangling jump +%assign stack_offset org_stack_offset + movifnidn dsq, dsmp + movifnidn ssq, ssmp +%if WIN64 + pop r8 +%endif + lea r6, [ssq*3] + jmp wq +.h: +%if ARCH_X86_32 + test ssd, 0xf00 +%else + test myd, 0xf00 +%endif + jnz .hv + movifnidn ssq, ssmp + WIN64_SPILL_XMM 12 + cmp wd, 4 + jl .h_w2 + je .h_w4 + tzcnt wd, wd +%if ARCH_X86_64 + mova m10, [base+subpel_h_shufA] + mova m11, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] +%endif + shr mxd, 16 + sub srcq, 3 + movzx wd, word [base_reg+wq*2+table_offset(put, _8tap_h)] + movq m6, [base_reg+mxq*8+subpel_filters-put_ssse3] + mova m7, [base+pw_34] ; 2 + (8 << 2) + pshufd m5, m6, q0000 + pshufd m6, m6, q1111 + add wq, base_reg + jmp wq +.h_w2: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + mova m4, [base+subpel_h_shuf4] + movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] + mova m5, [base+pw_34] ; 2 + (8 << 2) + pshufd m3, m3, q0000 + movifnidn dsq, dsmp +.h_w2_loop: + movq m0, [srcq+ssq*0] + movhps m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pmaddubsw m0, m3 + phaddw m0, m0 + paddw m0, m5 ; pw34 + psraw m0, 6 + packuswb m0, m0 + movd r6d, m0 + mov [dstq+dsq*0], r6w + shr r6d, 16 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] + mova m6, [base+subpel_h_shufA] + mova m5, [base+pw_34] ; 2 + (8 << 2) + pshufd m3, m3, q0000 + movifnidn dsq, dsmp +.h_w4_loop: + movq m0, [srcq+ssq*0] ; 1 + movq m1, [srcq+ssq*1] ; 2 + lea srcq, [srcq+ssq*2] + pshufb m0, m6 ; subpel_h_shufA + pshufb m1, m6 ; subpel_h_shufA + pmaddubsw m0, m3 ; subpel_filters + pmaddubsw m1, m3 ; subpel_filters + phaddw m0, m1 + paddw m0, m5 ; pw34 + psraw m0, 6 + packuswb m0, m0 + movd [dstq+dsq*0], m0 + psrlq m0, 32 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] + %if ARCH_X86_32 + pshufb %2, %1, [base+subpel_h_shufB] + pshufb %3, %1, [base+subpel_h_shufC] + pshufb %1, [base+subpel_h_shufA] + %else + pshufb %2, %1, m11; subpel_h_shufB + pshufb %3, %1, m9 ; subpel_h_shufC + pshufb %1, m10 ; subpel_h_shufA + %endif + pmaddubsw %4, %2, m5 ; subpel +0 B0 + pmaddubsw %2, m6 ; subpel +4 B4 + pmaddubsw %3, m6 ; C4 + pmaddubsw %1, m5 ; A0 + paddw %3, %4 ; C4+B0 + paddw %1, %2 ; A0+B4 + phaddw %1, %3 + paddw %1, m7 ; pw34 + psraw %1, 6 +%endmacro +.h_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + PUT_8TAP_H m0, m2, m3, m4 + PUT_8TAP_H m1, m2, m3, m4 + packuswb m0, m1 +%if ARCH_X86_32 + movq [dstq], m0 + add dstq, dsm + movhps [dstq], m0 + add dstq, dsm +%else + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] +%endif + sub hd, 2 + jg .h_w8 + RET +.h_w128: + mov r4, -16*7 + jmp .h_w16_start +.h_w64: + mov r4, -16*3 + jmp .h_w16_start +.h_w32: + mov r4, -16*1 + jmp .h_w16_start +.h_w16: + xor r4d, r4d +.h_w16_start: + sub srcq, r4 + sub dstq, r4 +.h_w16_loop_v: + mov r6, r4 +.h_w16_loop_h: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + PUT_8TAP_H m0, m2, m3, m4 + PUT_8TAP_H m1, m2, m3, m4 + packuswb m0, m1 + mova [dstq+r6], m0 + add r6, 16 + jle .h_w16_loop_h + add srcq, ssq + add dstq, dsmp + dec hd + jg .h_w16_loop_v + RET +.v: +%if ARCH_X86_32 + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3] +%else + %assign stack_offset org_stack_offset + WIN64_SPILL_XMM 16 + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-put_ssse3] +%endif + tzcnt r6d, wd + movzx r6d, word [base_reg+r6*2+table_offset(put, _8tap_v)] + punpcklwd m0, m0 + mova m7, [base+pw_512] + add r6, base_reg +%if ARCH_X86_32 + %define subpel0 [rsp+mmsize*0] + %define subpel1 [rsp+mmsize*1] + %define subpel2 [rsp+mmsize*2] + %define subpel3 [rsp+mmsize*3] +%assign regs_used 2 ; use r1 (ds) as tmp for stack alignment if needed + ALLOC_STACK -16*4 +%assign regs_used 7 + pshufd m1, m0, q0000 + mova subpel0, m1 + pshufd m1, m0, q1111 + mova subpel1, m1 + pshufd m1, m0, q2222 + mova subpel2, m1 + pshufd m1, m0, q3333 + mova subpel3, m1 + mov ssq, [rstk+stack_offset+gprsize*4] + lea ssq, [ssq*3] + sub srcq, ssq + mov ssq, [rstk+stack_offset+gprsize*4] + mov dsq, [rstk+stack_offset+gprsize*2] +%else + %define subpel0 m8 + %define subpel1 m9 + %define subpel2 m10 + %define subpel3 m11 + lea ss3q, [ssq*3] + pshufd m8, m0, q0000 + sub srcq, ss3q + pshufd m9, m0, q1111 + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 +%endif + jmp r6 +.v_w2: + movd m1, [srcq+ssq*0] + movd m0, [srcq+ssq*1] +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + movd m2, [srcq+ssq*0] + movd m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movd m3, [srcq+ssq*0] + movd m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] +%else + movd m2, [srcq+ssq*2] + add srcq, ss3q + movd m5, [srcq+ssq*0] + movd m3, [srcq+ssq*1] + movd m4, [srcq+ssq*2] + add srcq, ss3q +%endif + punpcklwd m1, m0 ; 0 1 + punpcklwd m0, m2 ; 1 2 + punpcklbw m1, m0 ; 01 12 + movd m0, [srcq+ssq*0] + punpcklwd m2, m5 ; 2 3 + punpcklwd m5, m3 ; 3 4 + punpcklwd m3, m4 ; 4 5 + punpcklwd m4, m0 ; 5 6 + punpcklbw m2, m5 ; 23 34 + punpcklbw m3, m4 ; 45 56 +.v_w2_loop: + movd m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m5, m1, subpel0 ; a0 b0 + mova m1, m2 + pmaddubsw m2, subpel1 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, subpel2 ; a2 b2 + paddw m5, m3 + punpcklwd m3, m0, m4 ; 6 7 + movd m0, [srcq+ssq*0] + punpcklwd m4, m0 ; 7 8 + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, subpel3 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + packuswb m5, m5 + movd r6d, m5 + mov [dstq+dsq*0], r6w + shr r6d, 16 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: +%if ARCH_X86_32 +.v_w8: +.v_w16: +.v_w32: +.v_w64: +.v_w128: + shl wd, 14 +%if STACK_ALIGNMENT < 16 + %define dstm [rsp+mmsize*4+gprsize] + mov dstm, dstq +%endif + lea r6d, [hq+wq-(1<<16)] + mov r4, srcq +.v_w4_loop0: +%endif + movd m1, [srcq+ssq*0] + movd m0, [srcq+ssq*1] +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + movd m2, [srcq+ssq*0] + movd m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movd m3, [srcq+ssq*0] + movd m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] +%else + movd m2, [srcq+ssq*2] + add srcq, ss3q + movd m5, [srcq+ssq*0] + movd m3, [srcq+ssq*1] + movd m4, [srcq+ssq*2] + add srcq, ss3q +%endif + punpckldq m1, m0 ; 0 1 + punpckldq m0, m2 ; 1 2 + punpcklbw m1, m0 ; 01 12 + movd m0, [srcq+ssq*0] + punpckldq m2, m5 ; 2 3 + punpckldq m5, m3 ; 3 4 + punpckldq m3, m4 ; 4 5 + punpckldq m4, m0 ; 5 6 + punpcklbw m2, m5 ; 23 34 + punpcklbw m3, m4 ; 45 56 +.v_w4_loop: + movd m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m5, m1, subpel0 ; a0 b0 + mova m1, m2 + pmaddubsw m2, subpel1 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, subpel2 ; a2 b2 + paddw m5, m3 + punpckldq m3, m0, m4 ; 6 7 _ _ + movd m0, [srcq+ssq*0] + punpckldq m4, m0 ; 7 8 _ _ + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, subpel3 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + packuswb m5, m5 + movd [dstq+dsq*0], m5 + psrlq m5, 32 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop +%if ARCH_X86_32 + mov dstq, dstm + add r4, 4 + movzx hd, r6w + add dstq, 4 + mov srcq, r4 + mov dstm, dstq + sub r6d, 1<<16 + jg .v_w4_loop0 +%endif + RET +%if ARCH_X86_64 +.v_w8: +.v_w16: +.v_w32: +.v_w64: +.v_w128: + lea r6d, [wq*8-64] + mov r4, srcq + mov r7, dstq + lea r6d, [hq+r6*4] +.v_w8_loop0: + movq m1, [srcq+ssq*0] + movq m2, [srcq+ssq*1] + movq m3, [srcq+ssq*2] + add srcq, ss3q + movq m4, [srcq+ssq*0] + movq m5, [srcq+ssq*1] + movq m6, [srcq+ssq*2] + add srcq, ss3q + movq m0, [srcq+ssq*0] + punpcklbw m1, m2 ; 01 + punpcklbw m2, m3 ; 12 + punpcklbw m3, m4 ; 23 + punpcklbw m4, m5 ; 34 + punpcklbw m5, m6 ; 45 + punpcklbw m6, m0 ; 56 +.v_w8_loop: + movq m13, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddubsw m14, m1, subpel0 ; a0 + mova m1, m3 + pmaddubsw m15, m2, subpel0 ; b0 + mova m2, m4 + pmaddubsw m3, subpel1 ; a1 + mova m12, m0 + pmaddubsw m4, subpel1 ; b1 + movq m0, [srcq+ssq*0] + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + pmaddubsw m5, subpel2 ; a2 + mova m4, m6 + pmaddubsw m6, subpel2 ; b2 + punpcklbw m12, m13 ; 67 + punpcklbw m13, m0 ; 78 + paddw m14, m5 + mova m5, m12 + pmaddubsw m12, subpel3 ; a3 + paddw m15, m6 + mova m6, m13 + pmaddubsw m13, subpel3 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + packuswb m14, m15 + movq [dstq+dsq*0], m14 + movhps [dstq+dsq*1], m14 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + add r4, 8 + add r7, 8 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 + sub r6d, 1<<8 + jg .v_w8_loop0 + RET +%endif ;ARCH_X86_64 +%undef subpel0 +%undef subpel1 +%undef subpel2 +%undef subpel3 +.hv: + %assign stack_offset org_stack_offset + cmp wd, 4 + jg .hv_w8 +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m1, [base_reg+mxq*8+subpel_filters-put_ssse3+2] +%if ARCH_X86_32 + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3] + mov ssq, ssmp + lea r6, [ssq*3] + sub srcq, r6 + %define base_reg r6 + mov r6, r1; use as new base + %assign regs_used 2 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + mov dsq, [rstk+stack_offset+gprsize*2] + %define subpelv0 [rsp+mmsize*0] + %define subpelv1 [rsp+mmsize*1] + %define subpelv2 [rsp+mmsize*2] + %define subpelv3 [rsp+mmsize*3] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m6, m0, q0000 + mova subpelv0, m6 + pshufd m6, m0, q1111 + mova subpelv1, m6 + pshufd m6, m0, q2222 + mova subpelv2, m6 + pshufd m6, m0, q3333 + mova subpelv3, m6 +%else + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-put_ssse3] + ALLOC_STACK mmsize*14, 14 + lea ss3q, [ssq*3] + sub srcq, ss3q + %define subpelv0 m10 + %define subpelv1 m11 + %define subpelv2 m12 + %define subpelv3 m13 + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + mova m8, [base+pw_8192] + mova m9, [base+pd_512] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 +%endif + pshufd m7, m1, q0000 + cmp wd, 4 + je .hv_w4 +.hv_w2: + mova m6, [base+subpel_h_shuf4] + movq m2, [srcq+ssq*0] ; 0 + movhps m2, [srcq+ssq*1] ; 0 _ 1 +%if ARCH_X86_32 + %define w8192reg [base+pw_8192] + %define d512reg [base+pd_512] + lea srcq, [srcq+ssq*2] + movq m0, [srcq+ssq*0] ; 2 + movhps m0, [srcq+ssq*1] ; 2 _ 3 + lea srcq, [srcq+ssq*2] +%else + %define w8192reg m8 + %define d512reg m9 + movq m0, [srcq+ssq*2] ; 2 + add srcq, ss3q + movhps m0, [srcq+ssq*0] ; 2 _ 3 +%endif + pshufb m2, m6 ; 0 ~ 1 ~ + pshufb m0, m6 ; 2 ~ 3 ~ + pmaddubsw m2, m7 ; subpel_filters + pmaddubsw m0, m7 ; subpel_filters + phaddw m2, m0 ; 0 1 2 3 + pmulhrsw m2, w8192reg +%if ARCH_X86_32 + movq m3, [srcq+ssq*0] ; 4 + movhps m3, [srcq+ssq*1] ; 4 _ 5 + lea srcq, [srcq+ssq*2] +%else + movq m3, [srcq+ssq*1] ; 4 + movhps m3, [srcq+ssq*2] ; 4 _ 5 + add srcq, ss3q +%endif + movq m0, [srcq+ssq*0] ; 6 + pshufb m3, m6 ; 4 ~ 5 ~ + pshufb m0, m6 ; 6 ~ + pmaddubsw m3, m7 ; subpel_filters + pmaddubsw m0, m7 ; subpel_filters + phaddw m3, m0 ; 4 5 6 _ + pmulhrsw m3, w8192reg + palignr m4, m3, m2, 4; V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 0 1 1 2 + punpckhwd m2, m4 ; V 23 34 2 3 3 4 + pshufd m0, m3, q2121; V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 4 5 5 6 +.hv_w2_loop: + movq m4, [srcq+ssq*1] ; V 7 + lea srcq, [srcq+ssq*2] ; V + movhps m4, [srcq+ssq*0] ; V 7 8 + pshufb m4, m6 + pmaddubsw m4, m7 + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 ; V + pmaddwd m2, subpelv1 ; V a1 b1 + paddd m5, m2 ; V + mova m2, m3 ; V + pmaddwd m3, subpelv2 ; a2 b2 + phaddw m4, m4 + pmulhrsw m4, w8192reg + paddd m5, m3 ; V + palignr m3, m4, m0, 12 + mova m0, m4 + punpcklwd m3, m0 ; V 67 78 + pmaddwd m4, m3, subpelv3 ; V a3 b3 + paddd m5, d512reg + paddd m5, m4 + psrad m5, 10 + packssdw m5, m5 + packuswb m5, m5 + movd r4d, m5 + mov [dstq+dsq*0], r4w + shr r4d, 16 + mov [dstq+dsq*1], r4w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +%undef w8192reg +%undef d512reg +.hv_w4: +%define hv4_line_0_0 4 +%define hv4_line_0_1 5 +%define hv4_line_0_2 6 +%define hv4_line_0_3 7 +%define hv4_line_0_4 8 +%define hv4_line_0_5 9 +%define hv4_line_1_0 10 +%define hv4_line_1_1 11 +%define hv4_line_1_2 12 +%define hv4_line_1_3 13 +%macro SAVELINE_W4 3 + mova [rsp+mmsize*hv4_line_%3_%2], %1 +%endmacro +%macro RESTORELINE_W4 3 + mova %1, [rsp+mmsize*hv4_line_%3_%2] +%endmacro +%if ARCH_X86_32 + %define w8192reg [base+pw_8192] + %define d512reg [base+pd_512] +%else + %define w8192reg m8 + %define d512reg m9 +%endif + ; lower shuffle 0 1 2 3 4 + mova m6, [base+subpel_h_shuf4] + movq m5, [srcq+ssq*0] ; 0 _ _ _ + movhps m5, [srcq+ssq*1] ; 0 _ 1 _ +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + movq m4, [srcq+ssq*0] ; 2 _ _ _ + movhps m4, [srcq+ssq*1] ; 2 _ 3 _ + lea srcq, [srcq+ssq*2] +%else + movq m4, [srcq+ssq*2] ; 2 _ _ _ + movhps m4, [srcq+ss3q ] ; 2 _ 3 _ + lea srcq, [srcq+ssq*4] +%endif + pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ + pmaddubsw m2, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m2, m0 ;H 0 1 2 3 + pmulhrsw m2, w8192reg ;H pw_8192 + SAVELINE_W4 m2, 2, 0 + ; upper shuffle 2 3 4 5 6 + mova m6, [base+subpel_h_shuf4+16] + pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ + pmaddubsw m2, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m2, m0 ;H 0 1 2 3 + pmulhrsw m2, w8192reg ;H pw_8192 + ; + ; lower shuffle + mova m6, [base+subpel_h_shuf4] + movq m5, [srcq+ssq*0] ; 4 _ _ _ + movhps m5, [srcq+ssq*1] ; 4 _ 5 _ +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + movq m4, [srcq+ssq*0] ; 6 _ _ _ + add srcq, ssq +%else + movq m4, [srcq+ssq*2] ; 6 _ _ _ + add srcq, ss3q +%endif + pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ + pmaddubsw m3, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m3, m0 ;H 4 5 6 7 + pmulhrsw m3, w8192reg ;H pw_8192 + SAVELINE_W4 m3, 3, 0 + ; upper shuffle + mova m6, [base+subpel_h_shuf4+16] + pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ + pmaddubsw m3, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m3, m0 ;H 4 5 6 7 + pmulhrsw m3, w8192reg ;H pw_8192 + ;process high + palignr m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + ;process low + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + palignr m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 +.hv_w4_loop: + ;process low + pmaddwd m5, m1, subpelv0 ; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 + mova m6, [base+subpel_h_shuf4] + movq m4, [srcq+ssq*0] ; 7 + movhps m4, [srcq+ssq*1] ; 7 _ 8 _ + pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ + pmaddubsw m4, m7 ;H subpel_filters + phaddw m4, m4 ;H 7 8 7 8 + pmulhrsw m4, w8192reg ;H pw_8192 + palignr m3, m4, m0, 12 ; 6 7 8 7 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 + paddd m5, d512reg ; pd_512 + paddd m5, m4 + psrad m5, 10 + SAVELINE_W4 m0, 0, 0 + SAVELINE_W4 m1, 1, 0 + SAVELINE_W4 m2, 2, 0 + SAVELINE_W4 m3, 3, 0 + SAVELINE_W4 m5, 5, 0 + ;process high + RESTORELINE_W4 m0, 0, 1 + RESTORELINE_W4 m1, 1, 1 + RESTORELINE_W4 m2, 2, 1 + RESTORELINE_W4 m3, 3, 1 + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 + mova m6, [base+subpel_h_shuf4+16] + movq m4, [srcq+ssq*0] ; 7 + movhps m4, [srcq+ssq*1] ; 7 _ 8 _ + lea srcq, [srcq+ssq*2] + pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ + pmaddubsw m4, m7 ;H subpel_filters + phaddw m4, m4 ;H 7 8 7 8 + pmulhrsw m4, w8192reg ;H pw_8192 + palignr m3, m4, m0, 12 ; 6 7 8 7 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 + paddd m5, d512reg ; pd_512 + paddd m5, m4 + psrad m4, m5, 10 + RESTORELINE_W4 m5, 5, 0 + packssdw m5, m4 ; d -> w + packuswb m5, m5 ; w -> b + pshuflw m5, m5, q3120 + movd [dstq+dsq*0], m5 + psrlq m5, 32 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + RESTORELINE_W4 m0, 0, 0 + RESTORELINE_W4 m1, 1, 0 + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + jg .hv_w4_loop + RET +%undef subpelv0 +%undef subpelv1 +%undef subpelv2 +%undef subpelv3 +.hv_w8: + %assign stack_offset org_stack_offset +%define hv8_line_1 0 +%define hv8_line_2 1 +%define hv8_line_3 2 +%define hv8_line_4 3 +%define hv8_line_6 4 +%macro SAVELINE_W8 2 + mova [rsp+hv8_line_%1*mmsize], %2 +%endmacro +%macro RESTORELINE_W8 2 + mova %2, [rsp+hv8_line_%1*mmsize] +%endmacro + shr mxd, 16 + sub srcq, 3 +%if ARCH_X86_32 + %define base_reg r1 + %define subpelh0 [rsp+mmsize*5] + %define subpelh1 [rsp+mmsize*6] + %define subpelv0 [rsp+mmsize*7] + %define subpelv1 [rsp+mmsize*8] + %define subpelv2 [rsp+mmsize*9] + %define subpelv3 [rsp+mmsize*10] + %define accuv0 [rsp+mmsize*11] + %define accuv1 [rsp+mmsize*12] + movq m1, [base_reg+mxq*8+subpel_filters-put_ssse3] + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + movq m5, [base_reg+ssq*8+subpel_filters-put_ssse3] + mov ssq, ssmp + ALLOC_STACK -mmsize*13 +%if STACK_ALIGNMENT < 16 + %define dstm [rsp+mmsize*13+gprsize*1] + %define dsm [rsp+mmsize*13+gprsize*2] + mov r6, [rstk+stack_offset+gprsize*2] + mov dsm, r6 +%endif + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + punpcklbw m5, m5 + psraw m5, 8 ; sign-extend + pshufd m2, m5, q0000 + pshufd m3, m5, q1111 + pshufd m4, m5, q2222 + pshufd m5, m5, q3333 + mova subpelh0, m0 + mova subpelh1, m1 + mova subpelv0, m2 + mova subpelv1, m3 + mova subpelv2, m4 + mova subpelv3, m5 + lea r6, [ssq*3] + mov dstm, dstq + sub srcq, r6 +%else + ALLOC_STACK 16*5, 16 + %define subpelh0 m10 + %define subpelh1 m11 + %define subpelv0 m12 + %define subpelv1 m13 + %define subpelv2 m14 + %define subpelv3 m15 + %define accuv0 m8 + %define accuv1 m9 + movq m0, [base_reg+mxq*8+subpel_filters-put_ssse3] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m1, [base_reg+myq*8+subpel_filters-put_ssse3] + pshufd subpelh0, m0, q0000 + pshufd subpelh1, m0, q1111 + punpcklbw m1, m1 + psraw m1, 8 ; sign-extend + pshufd subpelv0, m1, q0000 + pshufd subpelv1, m1, q1111 + pshufd subpelv2, m1, q2222 + pshufd subpelv3, m1, q3333 + lea ss3q, [ssq*3] + mov r7, dstq + sub srcq, ss3q +%endif + shl wd, 14 + lea r6d, [hq+wq-(1<<16)] + mov r4, srcq +.hv_w8_loop0: + movu m4, [srcq+ssq*0] ; 0 = _ _ + movu m5, [srcq+ssq*1] ; 1 = _ _ +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] +%endif +%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] + %if ARCH_X86_32 + pshufb %3, %1, [base+subpel_h_shufB] + pshufb %4, %1, [base+subpel_h_shufC] + pshufb %1, [base+subpel_h_shufA] + %else + pshufb %3, %1, %6 ; subpel_h_shufB + pshufb %4, %1, %7 ; subpel_h_shufC + pshufb %1, %5 ; subpel_h_shufA + %endif + pmaddubsw %2, %3, subpelh0 ; subpel +0 C0 + pmaddubsw %4, subpelh1; subpel +4 B4 + pmaddubsw %3, subpelh1; C4 + pmaddubsw %1, subpelh0; A0 + paddw %2, %4 ; C0+B4 + paddw %1, %3 ; A0+C4 + phaddw %1, %2 +%endmacro +%if ARCH_X86_64 + mova m7, [base+subpel_h_shufA] + mova m8, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] +%endif + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 0 ~ ~ ~ + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 1 ~ ~ ~ +%if ARCH_X86_32 + movu m6, [srcq+ssq*0] ; 2 = _ _ + movu m0, [srcq+ssq*1] ; 3 = _ _ + lea srcq, [srcq+ssq*2] +%else + movu m6, [srcq+ssq*2] ; 2 = _ _ + add srcq, ss3q + movu m0, [srcq+ssq*0] ; 3 = _ _ +%endif + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 2 ~ ~ ~ + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 ; 3 ~ ~ ~ + mova m7, [base+pw_8192] + pmulhrsw m4, m7 ; H pw_8192 + pmulhrsw m5, m7 ; H pw_8192 + pmulhrsw m6, m7 ; H pw_8192 + pmulhrsw m0, m7 ; H pw_8192 + punpcklwd m1, m4, m5 ; 0 1 ~ + punpcklwd m2, m5, m6 ; 1 2 ~ + punpcklwd m3, m6, m0 ; 2 3 ~ + SAVELINE_W8 1, m1 + SAVELINE_W8 2, m2 + SAVELINE_W8 3, m3 + mova m7, [base+subpel_h_shufA] +%if ARCH_X86_32 + movu m4, [srcq+ssq*0] ; 4 = _ _ + movu m5, [srcq+ssq*1] ; 5 = _ _ + lea srcq, [srcq+ssq*2] +%else + movu m4, [srcq+ssq*1] ; 4 = _ _ + movu m5, [srcq+ssq*2] ; 5 = _ _ + add srcq, ss3q +%endif + movu m6, [srcq+ssq*0] ; 6 = _ _ + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 4 ~ ~ ~ + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 5 ~ ~ ~ + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 6 ~ ~ ~ + mova m7, [base+pw_8192] + pmulhrsw m1, m4, m7 ; H pw_8192 4 ~ + pmulhrsw m2, m5, m7 ; H pw_8192 5 ~ + pmulhrsw m3, m6, m7 ; H pw_8192 6 ~ + punpcklwd m4, m0, m1 ; 3 4 ~ + punpcklwd m5, m1, m2 ; 4 5 ~ + punpcklwd m6, m2, m3 ; 5 6 ~ + SAVELINE_W8 6, m3 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 +.hv_w8_loop: + ; m8 accu for V a + ; m9 accu for V b + SAVELINE_W8 1, m3 + SAVELINE_W8 2, m4 + SAVELINE_W8 3, m5 + SAVELINE_W8 4, m6 +%if ARCH_X86_32 + pmaddwd m0, m1, subpelv0 ; a0 + pmaddwd m7, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m0, m3 + paddd m7, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m0, m5 + paddd m7, m6 + mova m5, [base+pd_512] + paddd m0, m5 ; pd_512 + paddd m7, m5 ; pd_512 + mova accuv0, m0 + mova accuv1, m7 +%else + pmaddwd m8, m1, subpelv0 ; a0 + pmaddwd m9, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m8, m3 + paddd m9, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m8, m5 + paddd m9, m6 + mova m7, [base+pd_512] + paddd m8, m7 ; pd_512 + paddd m9, m7 ; pd_512 + mova m7, [base+subpel_h_shufB] + mova m6, [base+subpel_h_shufC] + mova m5, [base+subpel_h_shufA] +%endif + movu m0, [srcq+ssq*1] ; 7 + movu m4, [srcq+ssq*2] ; 8 + lea srcq, [srcq+ssq*2] + HV_H_W8 m0, m1, m2, m3, m5, m7, m6 + HV_H_W8 m4, m1, m2, m3, m5, m7, m6 + mova m5, [base+pw_8192] + pmulhrsw m0, m5 ; H pw_8192 + pmulhrsw m4, m5 ; H pw_8192 + RESTORELINE_W8 6, m6 + punpcklwd m5, m6, m0 ; 6 7 ~ + punpcklwd m6, m0, m4 ; 7 8 ~ + pmaddwd m1, m5, subpelv3 ; a3 + paddd m2, m1, accuv0 + pmaddwd m1, m6, subpelv3 ; b3 + paddd m1, m1, accuv1 ; H + V + psrad m2, 10 + psrad m1, 10 + packssdw m2, m1 ; d -> w + packuswb m2, m1 ; w -> b + movd [dstq+dsq*0], m2 + psrlq m2, 32 +%if ARCH_X86_32 + add dstq, dsm + movd [dstq+dsq*0], m2 + add dstq, dsm +%else + movd [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] +%endif + sub hd, 2 + jle .hv_w8_outer + SAVELINE_W8 6, m4 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 + RESTORELINE_W8 4, m4 + jmp .hv_w8_loop +.hv_w8_outer: +%if ARCH_X86_32 + mov dstq, dstm + add r4, 4 + movzx hd, r6w + add dstq, 4 + mov srcq, r4 + mov dstm, dstq +%else + add r4, 4 + add r7, 4 + movzx hd, r6b + mov srcq, r4 + mov dstq, r7 +%endif + sub r6d, 1<<16 + jg .hv_w8_loop0 + RET + +%macro PSHUFB_SUBPEL_H_4 5 ; dst/src1, src2/mask, tmp1, tmp2, reset_mask + %if cpuflag(ssse3) + pshufb %1, %2 + %else + %if %5 == 1 + pcmpeqd %2, %2 + psrlq %2, 32 + %endif + psrldq %3, %1, 1 + pshufd %3, %3, q2301 + pand %1, %2 + pandn %4, %2, %3 + por %1, %4 + %endif +%endmacro + +%macro PSHUFB_SUBPEL_H_4a 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask + %ifnidn %1, %2 + mova %1, %2 + %endif + PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6 +%endmacro + +%macro PSHUFB_SUBPEL_H_4b 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask + %if notcpuflag(ssse3) + psrlq %1, %2, 16 + %elifnidn %1, %2 + mova %1, %2 + %endif + PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6 +%endmacro + +%macro PALIGNR 4-5 ; dst, src1, src2, shift[, tmp] + %if cpuflag(ssse3) + palignr %1, %2, %3, %4 + %else + %if %0 == 4 + %assign %%i regnumof%+%1 + 1 + %define %%tmp m %+ %%i + %else + %define %%tmp %5 + %endif + psrldq %1, %3, %4 + pslldq %%tmp, %2, 16-%4 + por %1, %%tmp + %endif +%endmacro + +%macro PHADDW 4 ; dst, src, pw_1/tmp, load_pw_1 + %if cpuflag(ssse3) + phaddw %1, %2 + %elifnidn %1, %2 + %if %4 == 1 + mova %3, [base+pw_1] + %endif + pmaddwd %1, %3 + pmaddwd %2, %3 + packssdw %1, %2 + %else + %if %4 == 1 + pmaddwd %1, [base+pw_1] + %else + pmaddwd %1, %3 + %endif + packssdw %1, %1 + %endif +%endmacro + +%macro PMULHRSW_POW2 4 ; dst, src1, src2, shift + %if cpuflag(ssse3) + pmulhrsw %1, %2, %3 + %else + paddw %1, %2, %3 + psraw %1, %4 + %endif +%endmacro + +%macro PMULHRSW_8192 3 ; dst, src1, src2 + PMULHRSW_POW2 %1, %2, %3, 2 +%endmacro + +%macro PREP_8TAP_H_LOAD4 5 ; dst, src_memloc, tmp[1-2] + movd %1, [%2+0] + movd %3, [%2+1] + movd %4, [%2+2] + movd %5, [%2+3] + punpckldq %1, %3 + punpckldq %4, %5 + punpcklqdq %1, %4 +%endmacro + +%macro PREP_8TAP_H_LOAD 2 ; dst0, src_memloc + %if cpuflag(ssse3) + movu m%1, [%2] + pshufb m2, m%1, m11 ; subpel_h_shufB + pshufb m3, m%1, m9 ; subpel_h_shufC + pshufb m%1, m10 ; subpel_h_shufA + %else + %if ARCH_X86_64 + SWAP m12, m5 + SWAP m13, m6 + SWAP m14, m7 + %define %%mx0 m%+%%i + %define %%mx1 m%+%%j + %assign %%i 0 + %rep 12 + movd %%mx0, [%2+%%i] + %assign %%i %%i+1 + %endrep + %assign %%i 0 + %rep 6 + %assign %%j %%i+1 + punpckldq %%mx0, %%mx1 + %assign %%i %%i+2 + %endrep + %assign %%i 0 + %rep 3 + %assign %%j %%i+2 + punpcklqdq %%mx0, %%mx1 + %assign %%i %%i+4 + %endrep + SWAP m%1, m0 + SWAP m2, m4 + SWAP m3, m8 + SWAP m5, m12 + SWAP m6, m13 + SWAP m7, m14 + %else + PREP_8TAP_H_LOAD4 m0, %2+0, m1, m4, m7 + PREP_8TAP_H_LOAD4 m2, %2+4, m1, m4, m7 + PREP_8TAP_H_LOAD4 m3, %2+8, m1, m4, m7 + SWAP m%1, m0 + %endif + %endif +%endmacro + +%macro PREP_8TAP_H 2 ; dst, src_memloc + PREP_8TAP_H_LOAD %1, %2 + %if ARCH_X86_64 && notcpuflag(ssse3) + SWAP m8, m1 + SWAP m9, m7 + %endif + %xdefine mX m%+%1 + %assign %%i regnumof%+mX + %define mX m%+%%i + mova m4, m2 + PMADDUBSW m4, m5, m1, m7, 1 ; subpel +0 B0 + PMADDUBSW m2, m6, m1, m7, 0 ; subpel +4 B4 + PMADDUBSW m3, m6, m1, m7, 0 ; subpel +4 C4 + PMADDUBSW mX, m5, m1, m7, 0 ; subpel +0 A0 + %undef mX + %if ARCH_X86_64 && notcpuflag(ssse3) + SWAP m1, m8 + SWAP m7, m9 + %endif + paddw m3, m4 + paddw m%1, m2 + PHADDW m%1, m3, m15, ARCH_X86_32 + %if ARCH_X86_64 || cpuflag(ssse3) + PMULHRSW_8192 m%1, m%1, m7 + %else + PMULHRSW_8192 m%1, m%1, [base+pw_2] + %endif +%endmacro + +%macro PREP_8TAP_HV 4 ; dst, src_memloc, tmp[1-2] + %if cpuflag(ssse3) + movu %1, [%2] + pshufb m2, %1, shufB + pshufb m3, %1, shufC + pshufb %1, shufA + %else + PREP_8TAP_H_LOAD4 %1, %2+0, m1, %3, %4 + PREP_8TAP_H_LOAD4 m2, %2+4, m1, %3, %4 + PREP_8TAP_H_LOAD4 m3, %2+8, m1, %3, %4 + %endif + mova m1, m2 + PMADDUBSW m1, subpelh0, %3, %4, 1 ; subpel +0 C0 + PMADDUBSW m3, subpelh1, %3, %4, 0 ; subpel +4 B4 + PMADDUBSW m2, subpelh1, %3, %4, 0 ; C4 + PMADDUBSW %1, subpelh0, %3, %4, 0 ; A0 + paddw m1, m3 ; C0+B4 + paddw %1, m2 ; A0+C4 + PHADDW %1, m1, %3, 1 +%endmacro + +%macro PREP_8TAP 0 +%if ARCH_X86_32 + DECLARE_REG_TMP 1, 2 +%elif WIN64 + DECLARE_REG_TMP 6, 4 +%else + DECLARE_REG_TMP 6, 7 +%endif + +FN prep_8tap, sharp, SHARP, SHARP +FN prep_8tap, sharp_smooth, SHARP, SMOOTH +FN prep_8tap, smooth_sharp, SMOOTH, SHARP +FN prep_8tap, smooth, SMOOTH, SMOOTH +FN prep_8tap, sharp_regular, SHARP, REGULAR +FN prep_8tap, regular_sharp, REGULAR, SHARP +FN prep_8tap, smooth_regular, SMOOTH, REGULAR +FN prep_8tap, regular_smooth, REGULAR, SMOOTH +FN prep_8tap, regular, REGULAR, REGULAR + +%if ARCH_X86_32 + %define base_reg r2 + %define base base_reg-prep%+SUFFIX +%else + %define base_reg r7 + %define base 0 +%endif +cglobal prep_8tap_8bpc, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3 +%assign org_stack_offset stack_offset + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + mov wd, wm + movifnidn srcd, srcm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + LEA base_reg, prep_ssse3 + tzcnt wd, wd + movzx wd, word [base_reg-prep_ssse3+prep_ssse3_table+wq*2] + pxor m4, m4 + add wq, base_reg + movifnidn strided, stridem + lea r6, [strideq*3] + %assign stack_offset org_stack_offset +%if WIN64 + pop r8 + pop r7 +%endif + jmp wq +.h: + LEA base_reg, prep%+SUFFIX + test myd, 0xf00 + jnz .hv +%if cpuflag(ssse3) + WIN64_SPILL_XMM 12 +%else + WIN64_SPILL_XMM 16 +%endif +%if ARCH_X86_32 + %define strideq r6 + mov strideq, stridem +%endif + cmp wd, 4 + je .h_w4 + tzcnt wd, wd +%if cpuflag(ssse3) + %if ARCH_X86_64 + mova m10, [base+subpel_h_shufA] + mova m11, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] + %else + %define m10 [base+subpel_h_shufA] + %define m11 [base+subpel_h_shufB] + %define m9 [base+subpel_h_shufC] + %endif +%endif + shr mxd, 16 + sub srcq, 3 + movzx wd, word [base_reg+wq*2+table_offset(prep, _8tap_h)] + movq m6, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX] +%if cpuflag(ssse3) + mova m7, [base+pw_8192] + pshufd m5, m6, q0000 + pshufd m6, m6, q1111 +%else + punpcklbw m6, m6 + psraw m6, 8 + %if ARCH_X86_64 + mova m7, [pw_2] + mova m15, [pw_1] + %else + %define m15 m4 + %endif + pshufd m5, m6, q1010 + punpckhqdq m6, m6 +%endif + add wq, base_reg + jmp wq +.h_w4: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m4, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2] +%if cpuflag(ssse3) + mova m6, [base+pw_8192] + mova m5, [base+subpel_h_shufA] + pshufd m4, m4, q0000 +%else + mova m6, [base+pw_2] + %if ARCH_X86_64 + mova m14, [pw_1] + %else + %define m14 m7 + %endif + punpcklbw m4, m4 + psraw m4, 8 + punpcklqdq m4, m4 +%endif +%if ARCH_X86_64 + lea stride3q, [strideq*3] +%endif +.h_w4_loop: +%if cpuflag(ssse3) + movq m0, [srcq+strideq*0] ; 0 + movq m1, [srcq+strideq*1] ; 1 + %if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movq m2, [srcq+strideq*0] ; 2 + movq m3, [srcq+strideq*1] ; 3 + lea srcq, [srcq+strideq*2] + %else + movq m2, [srcq+strideq*2] ; 2 + movq m3, [srcq+stride3q ] ; 3 + lea srcq, [srcq+strideq*4] + %endif + pshufb m0, m5 + pshufb m1, m5 + pshufb m2, m5 + pshufb m3, m5 +%elif ARCH_X86_64 + movd m0, [srcq+strideq*0+0] + movd m12, [srcq+strideq*0+1] + movd m1, [srcq+strideq*1+0] + movd m5, [srcq+strideq*1+1] + movd m2, [srcq+strideq*2+0] + movd m13, [srcq+strideq*2+1] + movd m3, [srcq+stride3q +0] + movd m7, [srcq+stride3q +1] + punpckldq m0, m12 + punpckldq m1, m5 + punpckldq m2, m13 + punpckldq m3, m7 + movd m12, [srcq+strideq*0+2] + movd m8, [srcq+strideq*0+3] + movd m5, [srcq+strideq*1+2] + movd m9, [srcq+strideq*1+3] + movd m13, [srcq+strideq*2+2] + movd m10, [srcq+strideq*2+3] + movd m7, [srcq+stride3q +2] + movd m11, [srcq+stride3q +3] + lea srcq, [srcq+strideq*4] + punpckldq m12, m8 + punpckldq m5, m9 + punpckldq m13, m10 + punpckldq m7, m11 + punpcklqdq m0, m12 ; 0 + punpcklqdq m1, m5 ; 1 + punpcklqdq m2, m13 ; 2 + punpcklqdq m3, m7 ; 3 +%else + movd m0, [srcq+strideq*0+0] + movd m1, [srcq+strideq*0+1] + movd m2, [srcq+strideq*0+2] + movd m3, [srcq+strideq*0+3] + punpckldq m0, m1 + punpckldq m2, m3 + punpcklqdq m0, m2 ; 0 + movd m1, [srcq+strideq*1+0] + movd m2, [srcq+strideq*1+1] + movd m3, [srcq+strideq*1+2] + movd m7, [srcq+strideq*1+3] + lea srcq, [srcq+strideq*2] + punpckldq m1, m2 + punpckldq m3, m7 + punpcklqdq m1, m3 ; 1 + movd m2, [srcq+strideq*0+0] + movd m3, [srcq+strideq*0+1] + movd m7, [srcq+strideq*0+2] + movd m5, [srcq+strideq*0+3] + punpckldq m2, m3 + punpckldq m7, m5 + punpcklqdq m2, m7 ; 2 + movd m3, [srcq+strideq*1+0] + movd m7, [srcq+strideq*1+1] + punpckldq m3, m7 + movd m7, [srcq+strideq*1+2] + movd m5, [srcq+strideq*1+3] + lea srcq, [srcq+strideq*2] + punpckldq m7, m5 + punpcklqdq m3, m7 ; 3 +%endif + PMADDUBSW m0, m4, m5, m7, 1 ; subpel_filters + 2 + PMADDUBSW m1, m4, m5, m7, 0 + PMADDUBSW m2, m4, m5, m7, 0 + PMADDUBSW m3, m4, m5, m7, 0 + PHADDW m0, m1, m14, ARCH_X86_32 + PHADDW m2, m3, m14, 0 + PMULHRSW_8192 m0, m0, m6 + PMULHRSW_8192 m2, m2, m6 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m2 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: +%if cpuflag(ssse3) + PREP_8TAP_H 0, srcq+strideq*0 + PREP_8TAP_H 1, srcq+strideq*1 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + lea srcq, [srcq+strideq*2] + add tmpq, 32 + sub hd, 2 +%else + PREP_8TAP_H 0, srcq + mova [tmpq], m0 + add srcq, strideq + add tmpq, 16 + dec hd +%endif + jg .h_w8 + RET +.h_w16: + mov r3, -16*1 + jmp .h_start +.h_w32: + mov r3, -16*2 + jmp .h_start +.h_w64: + mov r3, -16*4 + jmp .h_start +.h_w128: + mov r3, -16*8 +.h_start: + sub srcq, r3 + mov r5, r3 +.h_loop: +%if cpuflag(ssse3) + PREP_8TAP_H 0, srcq+r3+8*0 + PREP_8TAP_H 1, srcq+r3+8*1 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + add tmpq, 32 + add r3, 16 +%else + PREP_8TAP_H 0, srcq+r3 + mova [tmpq], m0 + add tmpq, 16 + add r3, 8 +%endif + jl .h_loop + add srcq, strideq + mov r3, r5 + dec hd + jg .h_loop + RET +.v: + LEA base_reg, prep%+SUFFIX +%if ARCH_X86_32 + mov mxd, myd + and mxd, 0x7f +%else + %assign stack_offset org_stack_offset + WIN64_SPILL_XMM 16 + movzx mxd, myb +%endif + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] +%if cpuflag(ssse3) + mova m2, [base+pw_512] + mova m7, [base+pw_8192] + punpcklwd m0, m0 +%else + punpcklbw m0, m0 + psraw m0, 8 +%endif +%if ARCH_X86_32 + %define subpel0 [rsp+mmsize*0] + %define subpel1 [rsp+mmsize*1] + %define subpel2 [rsp+mmsize*2] + %define subpel3 [rsp+mmsize*3] +%assign regs_used 6 ; use r5 (mx) as tmp for stack alignment if needed + %if cpuflag(ssse3) + ALLOC_STACK -mmsize*4 + %else + ALLOC_STACK -mmsize*5 + %endif +%assign regs_used 7 + mov strideq, [rstk+stack_offset+gprsize*3] + pshufd m1, m0, q0000 + mova subpel0, m1 + pshufd m1, m0, q1111 + mova subpel1, m1 + lea r5, [strideq*3] + pshufd m1, m0, q2222 + mova subpel2, m1 + pshufd m1, m0, q3333 + mova subpel3, m1 + sub srcq, r5 +%else + %define subpel0 m8 + %define subpel1 m9 + %define subpel2 m10 + %define subpel3 m11 + pshufd m8, m0, q0000 + pshufd m9, m0, q1111 + lea stride3q, [strideq*3] + pshufd m10, m0, q2222 + pshufd m11, m0, q3333 + sub srcq, stride3q + cmp wd, 8 + jns .v_w8 +%endif +.v_w4: +%if notcpuflag(ssse3) + pxor m6, m6 + %if ARCH_X86_64 + mova m7, [base+pw_2] + %endif +%endif +%if ARCH_X86_32 + %if STACK_ALIGNMENT < mmsize + %define srcm [esp+stack_size+gprsize*1] + %define tmpm [esp+stack_size+gprsize*2] + %endif + mov tmpm, tmpq + mov srcm, srcq + lea r5d, [wq - 4] ; horizontal loop + shl r5d, (16 - 2) ; (wq / 4) << 16 + mov r5w, hw +.v_w4_loop0: +%endif + movd m1, [srcq+strideq*0] + movd m0, [srcq+strideq*1] +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movd m2, [srcq+strideq*0] + movd m4, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movd m3, [srcq+strideq*0] + movd m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] +%else + movd m2, [srcq+strideq*2] + add srcq, stride3q + movd m4, [srcq+strideq*0] + movd m3, [srcq+strideq*1] + movd m5, [srcq+strideq*2] + add srcq, stride3q +%endif + punpckldq m1, m0 ; 0 1 + punpckldq m0, m2 ; 1 2 + punpcklbw m1, m0 ; 01 12 + movd m0, [srcq+strideq*0] + punpckldq m2, m4 ; 2 3 + punpckldq m4, m3 ; 3 4 + punpckldq m3, m5 ; 4 5 + punpckldq m5, m0 ; 5 6 + punpcklbw m2, m4 ; 23 34 + punpcklbw m3, m5 ; 45 56 +.v_w4_loop: +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel0 + %define subpel0 m7 +%endif + mova m5, m1 + PMADDUBSW m5, subpel0, m6, m4, 0 ; a0 b0 +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel1 + %define subpel1 m7 +%endif + mova m1, m2 + PMADDUBSW m2, subpel1, m6, m4, 0 ; a1 b1 + paddw m5, m2 +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel2 + %define subpel2 m7 +%endif + mova m2, m3 + PMADDUBSW m3, subpel2, m6, m4, 0 ; a2 b2 + movd m4, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + paddw m5, m3 + punpckldq m3, m0, m4 ; 6 7 _ _ + movd m0, [srcq+strideq*0] + punpckldq m4, m0 ; 7 8 _ _ + punpcklbw m3, m4 ; 67 78 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m12, m0 + %else + mova [esp+mmsize*4], m0 + mova m7, subpel3 + %define subpel3 m7 + %endif +%endif + mova m4, m3 + PMADDUBSW m4, subpel3, m6, m0, 0 ; a3 b3 + paddw m5, m4 +%if ARCH_X86_64 || cpuflag(ssse3) + %if notcpuflag(ssse3) + SWAP m0, m12 + %endif + PMULHRSW_8192 m5, m5, m7 +%else + mova m0, [esp+mmsize*4] + PMULHRSW_8192 m5, m5, [base+pw_2] +%endif + movq [tmpq+wq*0], m5 + movhps [tmpq+wq*2], m5 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w4_loop +%if ARCH_X86_32 + mov srcq, srcm + mov tmpq, tmpm + movzx hd, r5w + add srcq, 4 + add tmpq, 8 + mov srcm, srcq + mov tmpm, tmpq + sub r5d, 1<<16 ; horizontal-- + jg .v_w4_loop0 +%endif + RET +%if ARCH_X86_64 +.v_w8: + lea r6d, [wq*8-64] + mov r5, srcq + mov r8, tmpq + lea r6d, [hq+r6*4] +.v_w8_loop0: + movq m1, [srcq+strideq*0] + movq m2, [srcq+strideq*1] + movq m3, [srcq+strideq*2] + add srcq, stride3q + movq m4, [srcq+strideq*0] + movq m5, [srcq+strideq*1] + movq m6, [srcq+strideq*2] + add srcq, stride3q + movq m0, [srcq+strideq*0] + punpcklbw m1, m2 ; 01 + punpcklbw m2, m3 ; 12 + punpcklbw m3, m4 ; 23 + punpcklbw m4, m5 ; 34 + punpcklbw m5, m6 ; 45 + punpcklbw m6, m0 ; 56 +.v_w8_loop: + movq m13, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] +%if cpuflag(ssse3) + pmaddubsw m14, m1, subpel0 ; a0 + pmaddubsw m15, m2, subpel0 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, subpel1 ; a1 + pmaddubsw m4, subpel1 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, subpel2 ; a2 + pmaddubsw m6, subpel2 ; b2 + punpcklbw m12, m0, m13 ; 67 + movq m0, [srcq+strideq*0] + punpcklbw m13, m0 ; 78 + paddw m14, m5 + mova m5, m12 + pmaddubsw m12, subpel3 ; a3 + paddw m15, m6 + mova m6, m13 + pmaddubsw m13, subpel3 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 +%else + mova m14, m1 + PMADDUBSW m14, subpel0, m7, m12, 1 ; a0 + mova m15, m2 + PMADDUBSW m15, subpel0, m7, m12, 0 ; b0 + mova m1, m3 + PMADDUBSW m3, subpel1, m7, m12, 0 ; a1 + mova m2, m4 + PMADDUBSW m4, subpel1, m7, m12, 0 ; b1 + paddw m14, m3 + mova m3, m5 + PMADDUBSW m5, subpel2, m7, m12, 0 ; a2 + paddw m15, m4 + mova m4, m6 + PMADDUBSW m6, subpel2, m7, m12, 0 ; b2 + paddw m15, m6 + punpcklbw m12, m0, m13 ; 67 + movq m0, [srcq+strideq*0] + punpcklbw m13, m0 ; 78 + paddw m14, m5 + mova m5, m12 + PMADDUBSW m12, subpel3, m7, m6, 0 ; a3 + paddw m14, m12 + mova m6, m13 + PMADDUBSW m13, subpel3, m7, m12, 0 ; b3 + paddw m15, m13 + PMULHRSW_8192 m14, m14, [base+pw_2] + PMULHRSW_8192 m15, m15, [base+pw_2] +%endif + movu [tmpq+wq*0], m14 + movu [tmpq+wq*2], m15 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w8_loop + add r5, 8 + add r8, 16 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r8 + sub r6d, 1<<8 + jg .v_w8_loop0 + RET +%endif ;ARCH_X86_64 +%undef subpel0 +%undef subpel1 +%undef subpel2 +%undef subpel3 +.hv: + %assign stack_offset org_stack_offset + cmp wd, 4 + jg .hv_w8 + and mxd, 0x7f + movd m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2] +%if ARCH_X86_32 + mov mxd, myd + shr myd, 16 + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + mov strideq, stridem + %assign regs_used 6 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + lea r5, [strideq*3+1] + sub srcq, r5 + %define subpelv0 [rsp+mmsize*0] + %define subpelv1 [rsp+mmsize*1] + %define subpelv2 [rsp+mmsize*2] + %define subpelv3 [rsp+mmsize*3] + punpcklbw m0, m0 + psraw m0, 8 + pshufd m6, m0, q0000 + mova subpelv0, m6 + pshufd m6, m0, q1111 + mova subpelv1, m6 + pshufd m6, m0, q2222 + mova subpelv2, m6 + pshufd m6, m0, q3333 + mova subpelv3, m6 +%else + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + %if cpuflag(ssse3) + ALLOC_STACK mmsize*14, 14 + %else + ALLOC_STACK mmsize*14, 16 + %endif + lea stride3q, [strideq*3] + sub srcq, stride3q + dec srcq + %define subpelv0 m10 + %define subpelv1 m11 + %define subpelv2 m12 + %define subpelv3 m13 + punpcklbw m0, m0 + psraw m0, 8 + %if cpuflag(ssse3) + mova m8, [base+pw_8192] + %else + mova m8, [base+pw_2] + %endif + mova m9, [base+pd_32] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 +%endif + pshufd m7, m1, q0000 +%if notcpuflag(ssse3) + punpcklbw m7, m7 + psraw m7, 8 +%endif +%define hv4_line_0_0 4 +%define hv4_line_0_1 5 +%define hv4_line_0_2 6 +%define hv4_line_0_3 7 +%define hv4_line_0_4 8 +%define hv4_line_0_5 9 +%define hv4_line_1_0 10 +%define hv4_line_1_1 11 +%define hv4_line_1_2 12 +%define hv4_line_1_3 13 +%if ARCH_X86_32 + %if cpuflag(ssse3) + %define w8192reg [base+pw_8192] + %else + %define w8192reg [base+pw_2] + %endif + %define d32reg [base+pd_32] +%else + %define w8192reg m8 + %define d32reg m9 +%endif + ; lower shuffle 0 1 2 3 4 +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%else + %if ARCH_X86_64 + mova m15, [pw_1] + %else + %define m15 m1 + %endif +%endif + movq m5, [srcq+strideq*0] ; 0 _ _ _ + movhps m5, [srcq+strideq*1] ; 0 _ 1 _ +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movq m4, [srcq+strideq*0] ; 2 _ _ _ + movhps m4, [srcq+strideq*1] ; 2 _ 3 _ + lea srcq, [srcq+strideq*2] +%else + movq m4, [srcq+strideq*2] ; 2 _ _ _ + movhps m4, [srcq+stride3q ] ; 2 _ 3 _ + lea srcq, [srcq+strideq*4] +%endif + PSHUFB_SUBPEL_H_4a m2, m5, m6, m1, m3, 1 ;H subpel_h_shuf4 0~1~ + PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~ + PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters + PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3 + PMULHRSW_8192 m2, m2, w8192reg + SAVELINE_W4 m2, 2, 0 + ; upper shuffle 2 3 4 5 6 +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + PSHUFB_SUBPEL_H_4b m2, m5, m6, m1, m3, 0 ;H subpel_h_shuf4 0~1~ + PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~ + PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters + PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3 + PMULHRSW_8192 m2, m2, w8192reg +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m2 + %else + mova [esp+mmsize*4], m2 + %endif +%endif + ; lower shuffle +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%endif + movq m5, [srcq+strideq*0] ; 4 _ _ _ + movhps m5, [srcq+strideq*1] ; 4 _ 5 _ +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movq m4, [srcq+strideq*0] ; 6 _ _ _ + add srcq, strideq +%else + movq m4, [srcq+strideq*2] ; 6 _ _ _ + add srcq, stride3q +%endif + PSHUFB_SUBPEL_H_4a m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~ + PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~ + PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters + PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7 + PMULHRSW_8192 m3, m3, w8192reg + SAVELINE_W4 m3, 3, 0 + ; upper shuffle +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + PSHUFB_SUBPEL_H_4b m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~ + PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~ + PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters + PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7 + PMULHRSW_8192 m3, m3, w8192reg +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m2, m14 + %else + mova m2, [esp+mmsize*4] + %endif +%endif + ;process high + PALIGNR m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + ;process low + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + PALIGNR m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 +.hv_w4_loop: + ;process low + pmaddwd m5, m1, subpelv0 ; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m5 + %else + mova [esp+mmsize*4], m5 + %define m15 m3 + %endif +%endif +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%endif + movq m4, [srcq+strideq*0] ; 7 + movhps m4, [srcq+strideq*1] ; 7 _ 8 _ + PSHUFB_SUBPEL_H_4a m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~ + PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters + PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878 + PMULHRSW_8192 m4, m4, w8192reg + PALIGNR m3, m4, m0, 12, m5 ; 6787 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m5, m14 + %else + mova m5, [esp+mmsize*4] + %endif +%endif + paddd m5, d32reg ; pd_32 + paddd m5, m4 + psrad m5, 6 + SAVELINE_W4 m0, 0, 0 + SAVELINE_W4 m1, 1, 0 + SAVELINE_W4 m2, 2, 0 + SAVELINE_W4 m3, 3, 0 + SAVELINE_W4 m5, 5, 0 + ;process high + RESTORELINE_W4 m0, 0, 1 + RESTORELINE_W4 m1, 1, 1 + RESTORELINE_W4 m2, 2, 1 + RESTORELINE_W4 m3, 3, 1 + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m5 + %else + mova [esp+0xA0], m5 + %endif +%endif +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + movq m4, [srcq+strideq*0] ; 7 + movhps m4, [srcq+strideq*1] ; 7 _ 8 _ + PSHUFB_SUBPEL_H_4b m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~ + PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters + PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878 + PMULHRSW_8192 m4, m4, w8192reg + PALIGNR m3, m4, m0, 12, m5 ; 6787 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m5, m14 + %else + mova m5, [esp+0xA0] + %endif +%endif + paddd m5, d32reg ; pd_32 + paddd m5, m4 + psrad m4, m5, 6 + RESTORELINE_W4 m5, 5, 0 + packssdw m5, m4 + pshufd m5, m5, q3120 + movu [tmpq], m5 + lea srcq, [srcq+strideq*2] + add tmpq, 16 + sub hd, 2 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + RESTORELINE_W4 m0, 0, 0 + RESTORELINE_W4 m1, 1, 0 + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + jg .hv_w4_loop + RET +%undef subpelv0 +%undef subpelv1 +%undef subpelv2 +%undef subpelv3 +.hv_w8: + %assign stack_offset org_stack_offset +%define hv8_line_1 0 +%define hv8_line_2 1 +%define hv8_line_3 2 +%define hv8_line_4 3 +%define hv8_line_6 4 + shr mxd, 16 +%if ARCH_X86_32 + %define subpelh0 [rsp+mmsize*5] + %define subpelh1 [rsp+mmsize*6] + %define subpelv0 [rsp+mmsize*7] + %define subpelv1 [rsp+mmsize*8] + %define subpelv2 [rsp+mmsize*9] + %define subpelv3 [rsp+mmsize*10] + %define accuv0 [rsp+mmsize*11] + %define accuv1 [rsp+mmsize*12] + movq m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX] + mov mxd, myd + shr myd, 16 + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd + movq m5, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + mov strideq, stridem + %assign regs_used 6 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + %if STACK_ALIGNMENT < mmsize + %define tmpm [rsp+mmsize*13+gprsize*1] + %define srcm [rsp+mmsize*13+gprsize*2] + %define stridem [rsp+mmsize*13+gprsize*3] + mov tmpm, tmpq + mov stridem, strideq + %endif + %if cpuflag(ssse3) + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + %else + punpcklbw m1, m1 + psraw m1, 8 + pshufd m0, m1, q1010 + punpckhqdq m1, m1 + %endif + punpcklbw m5, m5 + psraw m5, 8 + pshufd m2, m5, q0000 + pshufd m3, m5, q1111 + pshufd m4, m5, q2222 + pshufd m5, m5, q3333 + mova subpelh0, m0 + mova subpelh1, m1 + mova subpelv0, m2 + mova subpelv1, m3 + mova subpelv2, m4 + mova subpelv3, m5 + lea r5, [strideq*3+3] + sub srcq, r5 + mov srcm, srcq +%else + ALLOC_STACK mmsize*5, 16 + %define subpelh0 m10 + %define subpelh1 m11 + %define subpelv0 m12 + %define subpelv1 m13 + %define subpelv2 m14 + %define subpelv3 m15 + %define accuv0 m8 + %define accuv1 m9 + movq m0, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m1, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + %if cpuflag(ssse3) + pshufd subpelh0, m0, q0000 + pshufd subpelh1, m0, q1111 + %else + punpcklbw m0, m0 + psraw m0, 8 + pshufd subpelh0, m0, q1010 + pshufd subpelh1, m0, q3232 + mova m7, [base+pw_2] + %endif + punpcklbw m1, m1 + psraw m1, 8 + pshufd subpelv0, m1, q0000 + pshufd subpelv1, m1, q1111 + pshufd subpelv2, m1, q2222 + pshufd subpelv3, m1, q3333 + lea stride3q, [strideq*3] + sub srcq, 3 + sub srcq, stride3q + mov r6, srcq + mov r8, tmpq +%endif + lea r5d, [wq-4] + shl r5d, 14 + add r5d, hd +.hv_w8_loop0: +%if cpuflag(ssse3) + %if ARCH_X86_64 + mova m7, [base+subpel_h_shufA] + mova m8, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] + %define shufA m7 + %define shufB m8 + %define shufC m9 + %else + %define shufA [base+subpel_h_shufA] + %define shufB [base+subpel_h_shufB] + %define shufC [base+subpel_h_shufC] + %endif +%endif + PREP_8TAP_HV m4, srcq+strideq*0, m7, m0 + PREP_8TAP_HV m5, srcq+strideq*1, m7, m0 +%if ARCH_X86_64 + PREP_8TAP_HV m6, srcq+strideq*2, m7, m0 + add srcq, stride3q + PREP_8TAP_HV m0, srcq+strideq*0, m7, m9 +%else + lea srcq, [srcq+strideq*2] + %if notcpuflag(ssse3) + mova [esp], m4 + %endif + PREP_8TAP_HV m6, srcq+strideq*0, m7, m4 + PREP_8TAP_HV m0, srcq+strideq*1, m7, m4 + lea srcq, [srcq+strideq*2] +%endif +%if cpuflag(ssse3) + mova m7, [base+pw_8192] +%else + mova m7, [base+pw_2] + %if ARCH_X86_32 + mova m4, [esp] + %endif +%endif + PMULHRSW_8192 m4, m4, m7 + PMULHRSW_8192 m5, m5, m7 + PMULHRSW_8192 m6, m6, m7 + PMULHRSW_8192 m0, m0, m7 + punpcklwd m1, m4, m5 ; 01 + punpcklwd m2, m5, m6 ; 12 + punpcklwd m3, m6, m0 ; 23 + SAVELINE_W8 1, m1 + SAVELINE_W8 2, m2 + SAVELINE_W8 3, m3 +%if cpuflag(ssse3) + mova m7, [base+subpel_h_shufA] +%endif +%if ARCH_X86_64 + PREP_8TAP_HV m4, srcq+strideq*1, m8, m9 + PREP_8TAP_HV m5, srcq+strideq*2, m8, m9 + add srcq, stride3q + PREP_8TAP_HV m6, srcq+strideq*0, m8, m9 +%else + %if notcpuflag(ssse3) + mova [esp+0x30], m0 + %endif + PREP_8TAP_HV m4, srcq+strideq*0, m7, m0 + PREP_8TAP_HV m5, srcq+strideq*1, m7, m0 + lea srcq, [srcq+strideq*2] + PREP_8TAP_HV m6, srcq+strideq*0, m7, m0 +%endif +%if cpuflag(ssse3) + mova m7, [base+pw_8192] +%elif ARCH_X86_32 + mova m0, [esp+0x30] + mova m7, [base+pw_2] +%endif + PMULHRSW_8192 m1, m4, m7 + PMULHRSW_8192 m2, m5, m7 + PMULHRSW_8192 m3, m6, m7 + punpcklwd m4, m0, m1 ; 34 + punpcklwd m5, m1, m2 ; 45 + punpcklwd m6, m2, m3 ; 56 + SAVELINE_W8 6, m3 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 +.hv_w8_loop: + SAVELINE_W8 1, m3 + SAVELINE_W8 2, m4 + SAVELINE_W8 3, m5 + SAVELINE_W8 4, m6 +%if ARCH_X86_32 + pmaddwd m0, m1, subpelv0 ; a0 + pmaddwd m7, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m0, m3 + paddd m7, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m0, m5 + paddd m7, m6 + mova m5, [base+pd_32] + paddd m0, m5 + paddd m7, m5 + mova accuv0, m0 + mova accuv1, m7 +%else + pmaddwd accuv0, m1, subpelv0 ; a0 + pmaddwd accuv1, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd accuv0, m3 + paddd accuv1, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd accuv0, m5 + paddd accuv1, m6 + mova m7, [base+pd_32] + paddd accuv0, m7 + paddd accuv1, m7 + %if cpuflag(ssse3) + mova m7, [base+subpel_h_shufB] + mova m6, [base+subpel_h_shufC] + mova m5, [base+subpel_h_shufA] + %define shufA m5 + %define shufB m7 + %define shufC m6 + %endif +%endif + PREP_8TAP_HV m0, srcq+strideq*1, m5, m6 + lea srcq, [srcq+strideq*2] + PREP_8TAP_HV m4, srcq+strideq*0, m5, m6 +%if cpuflag(ssse3) + mova m5, [base+pw_8192] +%else + mova m5, [base+pw_2] +%endif + PMULHRSW_8192 m0, m0, m5 + PMULHRSW_8192 m4, m4, m5 + RESTORELINE_W8 6, m6 + punpcklwd m5, m6, m0 ; 67 + punpcklwd m6, m0, m4 ; 78 + pmaddwd m1, m5, subpelv3 ; a3 + paddd m2, m1, accuv0 + pmaddwd m1, m6, subpelv3 ; b3 + paddd m1, m1, accuv1 + psrad m2, 6 + psrad m1, 6 + packssdw m2, m1 + movq [tmpq+wq*0], m2 + movhps [tmpq+wq*2], m2 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jle .hv_w8_outer + SAVELINE_W8 6, m4 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 + RESTORELINE_W8 4, m4 + jmp .hv_w8_loop +.hv_w8_outer: +%if ARCH_X86_32 + mov srcq, srcm + mov tmpq, tmpm + movzx hd, r5w + add srcq, 4 + add tmpq, 8 + mov srcm, srcq + mov tmpm, tmpq +%else + add r6, 4 + add r8, 8 + movzx hd, r5b + mov srcq, r6 + mov tmpq, r8 +%endif + sub r5d, 1<<16 + jg .hv_w8_loop0 + RET +%endmacro + +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro SAVE_REG 1 + %xdefine r%1_save r%1 + %xdefine r%1q_save r%1q + %xdefine r%1d_save r%1d + %if ARCH_X86_32 + %define r%1m_save [rstk+stack_offset+(%1+1)*4] + %endif +%endmacro + +%macro LOAD_REG 1 + %xdefine r%1 r%1_save + %xdefine r%1q r%1q_save + %xdefine r%1d r%1d_save + %if ARCH_X86_32 + %define r%1m r%1m_save + %endif + %undef r%1d_save + %undef r%1q_save + %undef r%1_save +%endmacro + +%macro REMAP_REG 2-3 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d + %if ARCH_X86_32 + %if %3 == 0 + %xdefine r%1m r%2m + %else + %define r%1m [rstk+stack_offset+(%1+1)*4] + %endif + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %if ARCH_X86_64 + SAVE_REG 14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %else + SAVE_REG 5 + %assign %%i 5 + %rep 5 + %assign %%j %%i-1 + REMAP_REG %%i, %%j, 0 + %assign %%i %%i-1 + %endrep + %endif + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %if ARCH_X86_64 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + LOAD_REG 14 + %else + %rep 4 + %assign %%j %%i+1 + REMAP_REG %%i, %%j, 1 + %assign %%i %%i+1 + %endrep + LOAD_REG 5 + %endif + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%if ARCH_X86_64 + %macro MC_8TAP_SCALED_H 12 ; dst[0-1], tmp[0-5], weights[0-3] + SWAP m%2, m%5 + movq m%1, [srcq+ r4] + movq m%2, [srcq+ r6] + movhps m%1, [srcq+ r7] + movhps m%2, [srcq+ r9] + movq m%3, [srcq+r10] + movq m%4, [srcq+r11] + movhps m%3, [srcq+r13] + movhps m%4, [srcq+ rX] + add srcq, ssq + movq m%5, [srcq+ r4] + movq m%6, [srcq+ r6] + movhps m%5, [srcq+ r7] + movhps m%6, [srcq+ r9] + movq m%7, [srcq+r10] + movq m%8, [srcq+r11] + movhps m%7, [srcq+r13] + movhps m%8, [srcq+ rX] + add srcq, ssq + pmaddubsw m%1, m%9 + pmaddubsw m%5, m%9 + pmaddubsw m%2, m%10 + pmaddubsw m%6, m%10 + pmaddubsw m%3, m%11 + pmaddubsw m%7, m%11 + pmaddubsw m%4, m%12 + pmaddubsw m%8, m%12 + phaddw m%1, m%2 + phaddw m%5, m%6 + phaddw m%3, m%4 + phaddw m%7, m%8 + phaddw m%1, m%3 + phaddw m%5, m%7 + pmulhrsw m%1, m12 + pmulhrsw m%5, m12 + SWAP m%2, m%5 + %endmacro +%else + %macro MC_8TAP_SCALED_H 2-3 1 ; weights_mem_start, h_mem_start, load_fh_offsets + %if %3 == 1 + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + %endif + movq m0, [srcq+r0] + movq m1, [srcq+rX] + movhps m0, [srcq+r4] + movhps m1, [srcq+r5] + add srcq, ssq + movq m4, [srcq+r0] + movq m5, [srcq+rX] + movhps m4, [srcq+r4] + movhps m5, [srcq+r5] + mov r0, [esp+16] + mov rX, [esp+24] + mov r4, [esp+20] + mov r5, [esp+28] + sub srcq, ssq + movq m2, [srcq+r0] + movq m3, [srcq+rX] + movhps m2, [srcq+r4] + movhps m3, [srcq+r5] + add srcq, ssq + movq m6, [srcq+r0] + movq m7, [srcq+rX] + movhps m6, [srcq+r4] + movhps m7, [srcq+r5] + add srcq, ssq + pmaddubsw m0, [esp+%1+ 0] + pmaddubsw m4, [esp+%1+ 0] + pmaddubsw m1, [esp+%1+16] + pmaddubsw m5, [esp+%1+16] + pmaddubsw m2, [esp+%1+32] + pmaddubsw m6, [esp+%1+32] + pmaddubsw m3, [esp+%1+48] + pmaddubsw m7, [esp+%1+48] + phaddw m0, m1 + phaddw m4, m5 + phaddw m2, m3 + phaddw m6, m7 + phaddw m0, m2 + phaddw m4, m6 + pmulhrsw m0, m12 + pmulhrsw m4, m12 + %if %2 != 0 + mova [esp+%2+ 0], m0 + mova [esp+%2+16], m4 + %endif + %endmacro +%endif + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isprep 0 + %if ARCH_X86_64 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled_8bpc, 2, 15, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy + %else +cglobal put_8tap_scaled_8bpc, 2, 14, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy + %endif + %else ; ARCH_X86_32 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled_8bpc, 0, 7, 8, 0x200, dst, ds, src, ss, w, h, mx, my, dx, dy + %else +cglobal put_8tap_scaled_8bpc, 0, 7, 8, -0x200-0x20, dst, ds, src, ss, w, h, mx, my, dx, dy + %endif + %endif + %xdefine base_reg r12 + %define rndshift 10 +%else ; prep + %assign isprep 1 + %if ARCH_X86_64 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled_8bpc, 2, 15, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy + %xdefine tmp_stridem r14q + %else +cglobal prep_8tap_scaled_8bpc, 2, 14, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy + %define tmp_stridem qword [rsp+0x138] + %endif + %xdefine base_reg r11 + %else ; ARCH_X86_32 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled_8bpc, 0, 7, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy + %else +cglobal prep_8tap_scaled_8bpc, 0, 6, 8, 0x200, tmp, src, ss, w, h, mx, my, dx, dy + %endif + %define tmp_stridem dword [esp+0x138] + %endif + %define rndshift 6 +%endif +%if ARCH_X86_32 + mov [esp+0x1f0], t0d + mov [esp+0x1f4], t1d + %if !isprep && required_stack_alignment > STACK_ALIGNMENT + mov dstd, dstm + mov dsd, dsm + mov srcd, srcm + mov ssd, ssm + mov hd, hm + mov r4, mxm + %define r0m [esp+0x200] + %define dsm [esp+0x204] + %define dsmp dsm + %define r1m dsm + %define r2m [esp+0x208] + %define ssm [esp+0x20c] + %define r3m ssm + %define hm [esp+0x210] + %define mxm [esp+0x214] + mov r0m, dstd + mov dsm, dsd + mov r2m, srcd + mov ssm, ssd + mov hm, hd + mov r0, mym + mov r1, dxm + mov r2, dym + %define mym [esp+0x218] + %define dxm [esp+0x09c] + %define dym [esp+0x21c] + mov mxm, r4 + mov mym, r0 + mov dxm, r1 + mov dym, r2 + tzcnt wd, wm + %endif + %if isprep && required_stack_alignment > STACK_ALIGNMENT + %xdefine base_reg r5 + %else + %xdefine base_reg r6 + %endif + mov ssd, ssm +%endif + LEA base_reg, %1_8tap_scaled_8bpc_ssse3 +%xdefine base base_reg-%1_8tap_scaled_8bpc_ssse3 +%if ARCH_X86_64 || isprep || required_stack_alignment <= STACK_ALIGNMENT + tzcnt wd, wm +%endif +%if ARCH_X86_32 + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 +%endif + movd m8, dxm + movd m14, mxm + pshufd m8, m8, q0000 + pshufd m14, m14, q0000 +%if isprep && UNIX64 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 +%endif +%if ARCH_X86_64 + mov dyd, dym +%endif +%ifidn %1, put + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %elif ARCH_X86_64 + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %endif + %if ARCH_X86_64 + %if required_stack_alignment > STACK_ALIGNMENT + %define dsm [rsp+0x138] + %define rX r1 + %define rXd r1d + %else + %define dsm dsq + %define rX r14 + %define rXd r14d + %endif + %else + %define rX r1 + %endif +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %elif ARCH_X86_64 + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %define hm [rsp+0x94] + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %if ARCH_X86_64 + %define rX r14 + %define rXd r14d + %else + %define rX r3 + %endif +%endif +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] + mova m12, [base+pw_8192] + %ifidn %1, put + mova m13, [base+pd_512] + %else + mova m13, [base+pd_32] + %endif +%else + %define m10 [base+pd_0x3ff] + %define m12 [base+pw_8192] + %ifidn %1, put + %define m13 [base+pd_512] + %else + %define m13 [base+pd_32] + %endif +%endif + pxor m9, m9 +%if ARCH_X86_64 + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q +%else + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + mov r1, [esp+0x1f4] + lea r0, [ssq*3] + movzx r2, r1b + shr r1, 16 + cmp dword hm, 6 + cmovs r1, r2 + mov [esp+0x1f4], r1 + mov r1, r1m + mov r2, r2m + sub srcq, r0 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define ss3q r0 + %define myd r4 + %define dyd dword dym + %define hd dword hm +%endif + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_ssse3_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.w2: + %if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d + %else + movzx r4, byte [esp+0x1f0] + dec srcq + movd m15, r4 + %endif + punpckldq m9, m8 + SWAP m8, m9 + paddd m14, m8 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m11, [base+pd_0x4000] + %else + %define m11 [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + psrldq m15, 4 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_dw] + mova m6, [base+subpel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m9, m9 + pcmpeqd m8, m9 + psrld m14, 10 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [rsp+0x180], m14 + SWAP m5, m0 + SWAP m6, m3 + %define m8 m5 + %define m15 m6 + %endif + movq m0, [srcq+ssq*0] + movq m2, [srcq+ssq*2] + movhps m0, [srcq+ssq*1] + movhps m2, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + %if ARCH_X86_64 + pshufb m14, m5 + paddb m14, m6 + %endif + movq m1, [srcq+ssq*0] + movq m3, [srcq+ssq*2] + movhps m1, [srcq+ssq*1] + movhps m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + punpckldq m15, m7 + punpcklqdq m15, m15 + %if ARCH_X86_64 + pand m11, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m11 + %else + pand m7, m8, m11 + pandn m8, m15 + %define m8 m6 + %define m15 m5 + por m15, m7 + mova [rsp+0x190], m15 + %endif + pshufb m0, m14 + pshufb m2, m14 + pshufb m1, m14 + pshufb m3, m14 + pmaddubsw m0, m15 + pmaddubsw m2, m15 + pmaddubsw m1, m15 + pmaddubsw m3, m15 + phaddw m0, m2 + phaddw m1, m3 + pmulhrsw m0, m12 ; 0 1 2 3 + pmulhrsw m1, m12 ; 4 5 6 7 + palignr m2, m1, m0, 4 ; 1 2 3 4 + punpcklwd m3, m0, m2 ; 01 12 + punpckhwd m0, m2 ; 23 34 + pshufd m5, m1, q0321 ; 5 6 7 _ + punpcklwd m2, m1, m5 ; 45 56 + punpckhwd m4, m1, m5 ; 67 __ + %if ARCH_X86_32 + mov myd, mym + mov r0, r0m + mova [rsp+0x1a0], m3 + mova [rsp+0x1b0], m0 + mova [rsp+0x1c0], m2 + mova [rsp+0x1d0], m4 + %endif +.w2_loop: + and myd, 0x3ff + %if ARCH_X86_64 + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq m11, r6q + punpcklbw m11, m11 + psraw m11, 8 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m5, m3, m8 + pmaddwd m6, m0, m9 + pmaddwd m7, m2, m10 + pmaddwd m8, m4, m11 + paddd m5, m6 + paddd m7, m8 + %else + mov mym, myd + mov r1, [esp+0x1f4] + xor r3, r3 + shr r4, 6 + lea r1, [r1+r4] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r1*8+0] + cmovnz r3, [base+subpel_filters+r1*8+4] + movd m7, r4 + movd m6, r3 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m5, m7, q0000 + pshufd m6, m7, q1111 + pmaddwd m3, m5 + pmaddwd m0, m6 + pshufd m5, m7, q2222 + pshufd m7, m7, q3333 + pmaddwd m2, m5 + pmaddwd m4, m7 + paddd m3, m0 + paddd m2, m4 + SWAP m5, m3 + SWAP m7, m2 + %endif + paddd m5, m13 + paddd m5, m7 + psrad m5, 10 + packssdw m5, m5 + packuswb m5, m5 + %if ARCH_X86_64 + pextrw r6d, m5, 0 + mov [dstq], r6w + add dstq, dsq + dec hd + jz .ret + add myd, dyd + %else + pextrw r3d, m5, 0 + mov [dstq], r3w + add dstq, dsm + dec hd + jz .ret + mov myd, mym + add myd, dym + %endif + test myd, ~0x3ff + %if ARCH_X86_32 + SWAP m3, m5 + SWAP m2, m7 + mova m3, [rsp+0x1a0] + mova m0, [rsp+0x1b0] + mova m2, [rsp+0x1c0] + mova m4, [rsp+0x1d0] + %define m14 [esp+0x180] + %define m15 [esp+0x190] + %endif + jz .w2_loop + %if ARCH_X86_32 + mov r3, r3m + %endif + movq m5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps m3, m0, q1032 ; 01 12 + shufps m0, m2, q1032 ; 23 34 + shufps m2, m4, q1032 ; 45 56 + pshufb m5, m14 + pmaddubsw m5, m15 + phaddw m5, m5 + pmulhrsw m5, m12 + palignr m4, m5, m1, 12 + punpcklqdq m1, m4, m4 ; 6 7 6 7 + punpcklwd m4, m1, m5 ; 67 __ + %if ARCH_X86_32 + mova [rsp+0x1a0], m3 + mova [rsp+0x1b0], m0 + mova [rsp+0x1c0], m2 + mova [rsp+0x1d0], m4 + %endif + jmp .w2_loop +.w2_skip_line: + movhps m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m3, m0 ; 01 12 + mova m0, m2 ; 23 34 + pshufb m5, m14 + pmaddubsw m5, m15 + phaddw m5, m5 + pmulhrsw m5, m12 ; 6 7 6 7 + palignr m4, m5, m1, 8 ; 4 5 6 7 + pshufd m5, m4, q0321 ; 5 6 7 _ + mova m1, m4 + punpcklwd m2, m4, m5 ; 45 56 + punpckhwd m4, m5 ; 67 __ + %if ARCH_X86_32 + mova [rsp+0x1a0], m3 + mova [rsp+0x1b0], m0 + mova [rsp+0x1c0], m2 + mova [rsp+0x1d0], m4 + %endif + jmp .w2_loop +%endif +INIT_XMM ssse3 +.w4: +%if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d +%else + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + movzx r4, byte [esp+0x1f0] + dec srcq + movd m15, r4 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m11, [base+pd_0x4000] +%else + %define m11 [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd m15, m0 + psrldq m7, m15, 8 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r6d, m15 + movd r13d, m7 + movd m15, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+r11*8+2] + movd m3, [base+subpel_filters+ r6*8+2] + movd m4, [base+subpel_filters+r13*8+2] +%else + movd r0, m15 + movd rX, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r4, m15 + movd r5, m7 + movd m1, [base+subpel_filters+r0*8+2] + movd m2, [base+subpel_filters+rX*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m7, [base+subpel_filters+r5*8+2] + movifprep r3, r3m + SWAP m4, m7 + %define m15 m1 +%endif + mova m5, [base+bdct_lb_dw] + movq m6, [base+subpel_s_shuf2] + psrld m14, 10 + punpckldq m15, m3 + punpckldq m2, m4 + punpcklqdq m15, m2 + punpcklqdq m6, m6 + pshufb m14, m5 + paddb m14, m6 +%if ARCH_X86_64 + pcmpeqd m0, m9 + pand m11, m0 +%else + mova [esp+0x180], m14 + SWAP m7, m4 + pxor m3, m3 + pcmpeqd m0, m3 + pand m2, m11, m0 + %define m11 m2 +%endif + pandn m0, m15 +%if ARCH_X86_64 + SWAP m15, m0 +%else + %define m15 m0 +%endif + por m15, m11 +%if ARCH_X86_64 + movu m7, [srcq+ssq*0] + movu m9, [srcq+ssq*1] + movu m8, [srcq+ssq*2] + movu m10, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + movu m2, [srcq+ssq*0] + movu m4, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + movu m5, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m7, m14 + pshufb m9, m14 + pshufb m8, m14 + pshufb m10, m14 + pshufb m2, m14 + pshufb m4, m14 + pshufb m3, m14 + pshufb m5, m14 + pmaddubsw m7, m15 + pmaddubsw m9, m15 + pmaddubsw m8, m15 + pmaddubsw m10, m15 + pmaddubsw m2, m15 + pmaddubsw m4, m15 + pmaddubsw m3, m15 + pmaddubsw m5, m15 + phaddw m7, m9 + phaddw m8, m10 + phaddw m9, m2, m4 + phaddw m3, m5 + pmulhrsw m7, m12 ; 0 1 + pmulhrsw m8, m12 ; 2 3 + pmulhrsw m9, m12 ; 4 5 + pmulhrsw m3, m12 ; 6 7 + shufps m4, m7, m8, q1032 ; 1 2 + shufps m5, m8, m9, q1032 ; 3 4 + shufps m6, m9, m3, q1032 ; 5 6 + psrldq m11, m3, 8 ; 7 _ + punpcklwd m0, m7, m4 ; 01 + punpckhwd m7, m4 ; 12 + punpcklwd m1, m8, m5 ; 23 + punpckhwd m8, m5 ; 34 + punpcklwd m2, m9, m6 ; 45 + punpckhwd m9, m6 ; 56 + punpcklwd m3, m11 ; 67 + mova [rsp+0x00], m7 + mova [rsp+0x10], m8 + mova [rsp+0x20], m9 +%else + mova [esp+0x190], m15 + lea ss3q, [ssq*3] + movu m2, [srcq+ssq*0] + movu m3, [srcq+ssq*1] + movu m7, [srcq+ssq*2] + movu m6, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m2, m14 + pshufb m3, m14 + pshufb m7, m14 + pshufb m6, m14 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + pmaddubsw m7, m15 + pmaddubsw m6, m15 + phaddw m2, m3 + phaddw m7, m6 + movu m1, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + movu m6, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m1, m14 + pshufb m5, m14 + pshufb m3, m14 + pshufb m6, m14 + pmaddubsw m1, m15 + pmaddubsw m5, m15 + pmaddubsw m3, m15 + pmaddubsw m6, m15 + phaddw m1, m5 + phaddw m3, m6 + pmulhrsw m2, m12 + pmulhrsw m7, m12 + pmulhrsw m1, m12 + pmulhrsw m3, m12 + shufps m4, m2, m7, q1032 ; 1 2 + shufps m5, m7, m1, q1032 ; 3 4 + shufps m6, m1, m3, q1032 ; 5 6 + psrldq m0, m3, 8 ; 7 _ + mova [esp+0x1a0], m0 + %define m11 [esp+0x1a0] + punpcklwd m0, m2, m4 ; 01 + punpckhwd m2, m4 ; 12 + punpcklwd m4, m7, m5 ; 23 + punpckhwd m7, m5 ; 34 + punpcklwd m5, m1, m6 ; 45 + punpckhwd m1, m6 ; 56 + punpcklwd m3, [esp+0x1a0] ; 67 + mov myd, mym + mov r0, r0m + mova [esp+0x1b0], m0 ; 01 + mova [esp+0x1c0], m4 ; 23 + mova [esp+0x1d0], m5 ; 45 + mova [esp+0x1e0], m3 ; 67 + mova [rsp+0x00], m2 ; 12 + mova [rsp+0x10], m7 ; 34 + mova [rsp+0x20], m1 ; 56 + SWAP m1, m4 + SWAP m2, m5 +%endif +.w4_loop: + and myd, 0x3ff +%if ARCH_X86_64 + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq m10, r6q + punpcklbw m10, m10 + psraw m10, 8 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pmaddwd m6, m2, m9 + pmaddwd m7, m3, m10 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 +%else + mov mym, myd + mov r5, [esp+0x1f4] + xor r3, r3 + shr r4, 6 + lea r5, [r5+r4] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + movd m7, r4 + movd m6, r3 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m4, m7, q0000 + pshufd m5, m7, q1111 + pshufd m6, m7, q2222 + pshufd m7, m7, q3333 + pmaddwd m0, m4 + pmaddwd m1, m5 + pmaddwd m2, m6 + pmaddwd m3, m7 + paddd m0, m1 + paddd m2, m3 + paddd m0, m13 + paddd m0, m2 + SWAP m4, m0 +%endif + psrad m4, rndshift + packssdw m4, m4 +%ifidn %1, put + packuswb m4, m4 + movd [dstq], m4 + add dstq, dsmp +%else + movq [tmpq], m4 + add tmpq, 8 +%endif + dec hd + jz .ret +%if ARCH_X86_64 + add myd, dyd + test myd, ~0x3ff + jz .w4_loop +%else + SWAP m0, m4 + mov myd, mym + mov r3, r3m + add myd, dym + test myd, ~0x3ff + jnz .w4_next_line + mova m0, [esp+0x1b0] + mova m1, [esp+0x1c0] + mova m2, [esp+0x1d0] + mova m3, [esp+0x1e0] + jmp .w4_loop +.w4_next_line: + %define m14 [esp+0x180] + %define m15 [esp+0x190] +%endif + movu m4, [srcq] + test myd, 0x400 + jz .w4_skip_line +%if ARCH_X86_64 + mova m0, [rsp+0x00] + mova [rsp+0x00], m1 + mova m1, [rsp+0x10] + mova [rsp+0x10], m2 + mova m2, [rsp+0x20] + mova [rsp+0x20], m3 +%else + mova m5, [esp+0x1c0] + mova m0, [rsp+0x000] + mova [rsp+0x00], m5 + mova [esp+0x1b0], m0 + mova m6, [esp+0x1d0] + mova m1, [rsp+0x010] + mova [rsp+0x10], m6 + mova [esp+0x1c0], m1 + mova m7, [esp+0x1e0] + mova m2, [rsp+0x020] + mova [rsp+0x20], m7 + mova [esp+0x1d0], m2 +%endif + pshufb m4, m14 + pmaddubsw m4, m15 + phaddw m4, m4 + pmulhrsw m4, m12 + punpcklwd m3, m11, m4 +%if ARCH_X86_32 + mova [esp+0x1e0], m3 +%endif + mova m11, m4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: +%if ARCH_X86_32 + mova m0, [esp+0x1c0] + mova m1, [esp+0x1d0] + mova m2, [esp+0x1e0] +%endif + movu m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova m6, [rsp+0x10] + mova m7, [rsp+0x20] + pshufb m4, m14 + pshufb m5, m14 + pmaddubsw m4, m15 + pmaddubsw m5, m15 + phaddw m4, m5 + pmulhrsw m4, m12 + punpcklwd m5, m11, m4 + mova [rsp+0x00], m6 + mova [rsp+0x10], m7 + mova [rsp+0x20], m5 +%if ARCH_X86_64 + psrldq m11, m4, 8 + mova m0, m1 + mova m1, m2 + mova m2, m3 + punpcklwd m3, m4, m11 +%else + psrldq m6, m4, 8 + punpcklwd m3, m4, m6 + mova [esp+0x1a0], m6 + mova [esp+0x1b0], m0 + mova [esp+0x1c0], m1 + mova [esp+0x1d0], m2 + mova [esp+0x1e0], m3 +%endif + jmp .w4_loop +INIT_XMM ssse3 +.w8: + mov dword [rsp+0x90], 1 + movifprep tmp_stridem, 16 + jmp .w_start +.w16: + mov dword [rsp+0x90], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [rsp+0x90], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [rsp+0x90], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [rsp+0x90], 16 + movifprep tmp_stridem, 256 +.w_start: +%ifidn %1, put + movifnidn dsm, dsq +%endif +%if ARCH_X86_64 + shr t0d, 16 + movd m15, t0d +%else + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %if isprep + %define ssq ssm + %endif + mov r4, [esp+0x1f0] + shr r4, 16 + movd m15, r4 + mov r0, r0m + mov myd, mym +%endif + sub srcq, 3 + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + mova [rsp+0x100], m7 + mova [rsp+0x120], m15 + mov [rsp+0x098], srcq + mov [rsp+0x130], r0q ; dstq / tmpq +%if ARCH_X86_64 && UNIX64 + mov hm, hd +%elif ARCH_X86_32 + mov r5, hm + mov [esp+0x094], myd + mov [esp+0x134], r5 +%endif + jmp .hloop +.hloop_prep: + dec dword [rsp+0x090] + jz .ret +%if ARCH_X86_64 + add qword [rsp+0x130], 8*(isprep+1) + mov hd, hm +%else + add dword [esp+0x130], 8*(isprep+1) + mov myd, [esp+0x094] + mov r5, [esp+0x134] + mov r0, [esp+0x130] +%endif + mova m7, [rsp+0x100] + mova m14, [rsp+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] +%endif + mova m15, [rsp+0x120] + pxor m9, m9 + mov srcq, [rsp+0x098] +%if ARCH_X86_64 + mov r0q, [rsp+0x130] ; dstq / tmpq +%else + mov mym, myd + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.hloop: +%if ARCH_X86_64 + mova m11, [base+pq_0x40000000] +%else + %define m11 [base+pq_0x40000000] +%endif + psrld m2, m14, 10 + mova [rsp], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m9 + psrldq m2, m5, 8 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] + pxor m2, m2 + %define m9 m2 +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + mova [rsp+0x110], m14 + psrldq m4, m15, 8 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + psrldq m4, m14, 8 + movd r10d, m14 + movd r11d, m4 + psrldq m14, 4 + psrldq m4, 4 + movd r13d, m14 + movd rXd, m4 + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m14, m5, q1100 + pshufd m5, m5, q3322 + pand m7, m11, m4 + pand m8, m11, m6 + pand m15, m11, m14 + pand m11, m11, m5 + pandn m4, m0 + pandn m6, m1 + pandn m14, m2 + pandn m5, m3 + por m7, m4 + por m8, m6 + por m15, m14 + por m11, m5 + mova [rsp+0x10], m7 + mova [rsp+0x20], m8 + mova [rsp+0x30], m15 + mova [rsp+0x40], m11 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 9, 10, 7, 8, 15, 11 ; 0-1 + mova [rsp+0x50], m1 + mova [rsp+0x60], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 9, 10, 7, 8, 15, 11 ; 2-3 + mova [rsp+0x70], m3 + mova [rsp+0x80], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 9, 10, 7, 8, 15, 11 ; 4-5 + MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 9, 10, 7, 8, 15, 11 ; 6-7 + SWAP m7, m0 + SWAP m8, m14 + mova m1, [rsp+0x50] + mova m2, [rsp+0x60] + mova m3, [rsp+0x70] + mova m9, [rsp+0x80] + mov myd, mym + mov dyd, dym + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m9 ; 23a + punpckhwd m3, m9 ; 23b + mova [rsp+0x50], m4 + mova [rsp+0x60], m5 + mova [rsp+0x70], m6 + mova [rsp+0x80], m7 + SWAP m14, m8 +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq m11, r6q + punpcklbw m11, m11 + psraw m11, 8 + pshufd m5, m11, q0000 + pshufd m7, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m4, m5, m0 + pmaddwd m5, m5, m1 + pmaddwd m6, m7, m2 + pmaddwd m7, m7, m3 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 + pmaddwd m6, [rsp+0x50], m10 + pmaddwd m7, [rsp+0x60], m10 + pmaddwd m8, [rsp+0x70], m11 + pmaddwd m9, [rsp+0x80], m11 + paddd m4, m6 + paddd m5, m7 + paddd m4, m8 + paddd m5, m9 +%else + movd r0, m15 + movd rX, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r4, m15 + movd r5, m4 + mova m14, [esp+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [esp+16], m14 + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m11, m4 + pand m1, m11, m6 + pand m2, m11, m7 + pand m3, m11, m5 + pandn m4, [esp+0x20] + pandn m6, [esp+0x30] + pandn m7, [esp+0x40] + pandn m5, [esp+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + MC_8TAP_SCALED_H 0x20, 0x140, 0 ; 0-1 + MC_8TAP_SCALED_H 0x20, 0x160 ; 2-3 + MC_8TAP_SCALED_H 0x20, 0x180 ; 4-5 + MC_8TAP_SCALED_H 0x20, 0x1a0 ; 6-7 + mova m5, [esp+0x180] + mova m6, [esp+0x190] + mova m7, [esp+0x1a0] + mova m0, [esp+0x1b0] + mov myd, mym + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [esp+0x180], m4 + mova [esp+0x190], m5 + mova [esp+0x1a0], m6 + mova [esp+0x1b0], m7 + mova m1, [esp+0x140] + mova m2, [esp+0x150] + mova m3, [esp+0x160] + mova m4, [esp+0x170] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova [esp+0x140], m0 + mova [esp+0x150], m1 + mova [esp+0x160], m2 + mova [esp+0x170], m3 +.vloop: + mov r0, r0m + mov r5, [esp+0x1f4] + and myd, 0x3ff + mov mym, myd + xor r3, r3 + shr r4, 6 + lea r5, [r5+r4] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + movd m7, r4 + movd m6, r3 + punpckldq m7, m6 + punpcklbw m7, m7 + psraw m7, 8 + pshufd m4, m7, q0000 + pshufd m5, m7, q1111 + pmaddwd m0, m4 + pmaddwd m1, m4 + pmaddwd m2, m5 + pmaddwd m3, m5 + pshufd m6, m7, q2222 + pshufd m7, m7, q3333 + paddd m0, m2 + paddd m1, m3 + pmaddwd m2, [esp+0x180], m6 + pmaddwd m3, [esp+0x190], m6 + pmaddwd m4, [esp+0x1a0], m7 + pmaddwd m5, [esp+0x1b0], m7 + paddd m0, m2 + paddd m1, m3 + paddd m0, m13 + paddd m1, m13 + paddd m4, m0 + paddd m5, m1 +%endif + psrad m4, rndshift + psrad m5, rndshift + packssdw m4, m5 +%ifidn %1, put + packuswb m4, m4 + movq [dstq], m4 + add dstq, dsm +%else + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep +%if ARCH_X86_64 + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [rsp+0x140], myd + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + jz .skip_line + mova m14, [base+unpckw] + movq m6, [srcq+r10] + movq m7, [srcq+r11] + movhps m6, [srcq+r13] + movhps m7, [srcq+ rX] + movq m4, [srcq+ r4] + movq m5, [srcq+ r6] + movhps m4, [srcq+ r7] + movhps m5, [srcq+ r9] + add srcq, ssq + mov myd, [rsp+0x140] + mov dyd, dym + pshufd m9, m14, q1032 + pshufb m0, m14 ; 0a 1a + pshufb m1, m14 ; 0b 1b + pshufb m2, m9 ; 3a 2a + pshufb m3, m9 ; 3b 2b + pmaddubsw m6, [rsp+0x30] + pmaddubsw m7, [rsp+0x40] + pmaddubsw m4, [rsp+0x10] + pmaddubsw m5, [rsp+0x20] + phaddw m6, m7 + phaddw m4, m5 + phaddw m4, m6 + pmulhrsw m4, m12 + pshufb m5, [rsp+0x50], m14 ; 4a 5a + pshufb m6, [rsp+0x60], m14 ; 4b 5b + pshufb m7, [rsp+0x70], m9 ; 7a 6a + pshufb m8, [rsp+0x80], m9 ; 7b 6b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m5 ; 34a + punpcklwd m3, m6 ; 34b + punpckhwd m5, m7 ; 56a + punpckhwd m6, m8 ; 56b + punpcklwd m7, m4 ; 78a + punpckhqdq m4, m4 + punpcklwd m8, m4 ; 78b + mova [rsp+0x50], m5 + mova [rsp+0x60], m6 + mova [rsp+0x70], m7 + mova [rsp+0x80], m8 + jmp .vloop +.skip_line: + mova m0, [rsp+0x10] + mova m1, [rsp+0x20] + mova m14, [rsp+0x30] + mova m15, [rsp+0x40] + MC_8TAP_SCALED_H 4, 8, 5, 6, 7, 9, 10, 11, 0, 1, 14, 15 + mov myd, [rsp+0x140] + mov dyd, dym + mova m0, m2 ; 01a + mova m1, m3 ; 01b + mova m2, [rsp+0x50] ; 23a + mova m3, [rsp+0x60] ; 23b + mova m5, [rsp+0x70] ; 45a + mova m6, [rsp+0x80] ; 45b + punpcklwd m7, m4, m8 ; 67a + punpckhwd m4, m8 ; 67b + mova [rsp+0x50], m5 + mova [rsp+0x60], m6 + mova [rsp+0x70], m7 + mova [rsp+0x80], m4 +%else + mov r0m, r0 + mov myd, mym + mov r3, r3m + add myd, dym + test myd, ~0x3ff + mov mym, myd + jnz .next_line + mova m0, [esp+0x140] + mova m1, [esp+0x150] + mova m2, [esp+0x160] + mova m3, [esp+0x170] + jmp .vloop +.next_line: + test myd, 0x400 + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + jz .skip_line + mova m6, [base+unpckw] + mova m0, [esp+0x140] + mova m1, [esp+0x150] + mova m7, [esp+0x180] + movq m4, [srcq+r0] + movq m5, [srcq+rX] + movhps m4, [srcq+r4] + movhps m5, [srcq+r5] + pshufb m0, m6 ; 0a 1a + pshufb m1, m6 ; 0b 1b + pshufb m7, m6 ; 4a 5a + mov r0, [esp+16] + mov rX, [esp+24] + mov r4, [esp+20] + mov r5, [esp+28] + movq m3, [srcq+r0] + movq m2, [srcq+rX] + movhps m3, [srcq+r4] + movhps m2, [srcq+r5] + add srcq, ssq + pmaddubsw m4, [esp+0x20] + pmaddubsw m5, [esp+0x30] + pmaddubsw m3, [esp+0x40] + pmaddubsw m2, [esp+0x50] + phaddw m4, m5 + phaddw m3, m2 + mova m5, [esp+0x190] + mova m2, [esp+0x160] + phaddw m4, m3 + mova m3, [esp+0x170] + pmulhrsw m4, m12 ; 8a 8b + mov myd, mym + pshufb m5, m6 ; 4b 5b + pshufd m6, m6, q1032 + pshufb m2, m6 ; 3a 2a + pshufb m3, m6 ; 3b 2b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + mova [esp+0x140], m0 + mova [esp+0x150], m1 + mova m0, [esp+0x1a0] + mova m1, [esp+0x1b0] + punpcklwd m2, m7 ; 34a + punpcklwd m3, m5 ; 34b + mova [esp+0x160], m2 + mova [esp+0x170], m3 + pshufb m0, m6 ; 7a 6a + pshufb m1, m6 ; 7b 6b + punpckhwd m7, m0 ; 56a + punpckhwd m5, m1 ; 56b + punpcklwd m0, m4 + punpckhqdq m4, m4 + punpcklwd m1, m4 + mova [esp+0x180], m7 + mova [esp+0x190], m5 + mova [esp+0x1a0], m0 + mova [esp+0x1b0], m1 + mova m0, [esp+0x140] + mova m1, [esp+0x150] + jmp .vloop +.skip_line: + MC_8TAP_SCALED_H 0x20, 0x1c0, 0 + mov myd, mym + mova m0, [esp+0x160] + mova m1, [esp+0x170] + mova m2, [esp+0x180] + mova m3, [esp+0x190] + mova [esp+0x140], m0 + mova [esp+0x150], m1 + mova m4, [esp+0x1a0] + mova m5, [esp+0x1b0] + mova [esp+0x160], m2 + mova [esp+0x170], m3 + mova m6, [esp+0x1c0] + mova m7, [esp+0x1d0] + mova [esp+0x180], m4 + mova [esp+0x190], m5 + punpcklwd m4, m6, m7 + punpckhwd m6, m7 + mova [esp+0x1a0], m4 + mova [esp+0x1b0], m6 +%endif + jmp .vloop +INIT_XMM ssse3 +.dy1: + movzx wd, word [base+%1_8tap_scaled_ssse3_dy1_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy1_w2: + %if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d + %else + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 + movzx r5, byte [esp+0x1f0] + dec srcd + movd m15, r5 + %endif + punpckldq m9, m8 + SWAP m8, m9 + paddd m14, m8 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m11, [base+pd_0x4000] + %else + %define m11 [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + psrldq m15, 4 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_dw] + mova m6, [base+subpel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m9, m9 + pcmpeqd m8, m9 + psrld m14, 10 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [esp+0x00], m14 + %define m14 [esp+0x00] + SWAP m5, m0 + SWAP m6, m3 + %define m8 m5 + %define m15 m6 + %endif + movq m0, [srcq+ssq*0] + movq m2, [srcq+ssq*2] + movhps m0, [srcq+ssq*1] + movhps m2, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + %if ARCH_X86_64 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + movq m10, r4 + %else + mov myd, mym + mov r5, [esp+0x1f4] + xor r3, r3 + shr myd, 6 + lea r5, [r5+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r5*8+0] + cmovnz r3, [base+subpel_filters+r5*8+4] + %define m10 m4 + movd m10, r4 + movd m3, r3 + mov r3, r3m + punpckldq m10, m3 + %endif + movq m1, [srcq+ssq*0] + movq m3, [srcq+ssq*2] + movhps m1, [srcq+ssq*1] + add srcq, ss3q + punpcklbw m10, m10 + psraw m10, 8 + punpckldq m15, m7 + punpcklqdq m15, m15 + %if ARCH_X86_64 + pand m11, m8 + %else + pand m7, m11, m8 + %define m11 m7 + %endif + pandn m8, m15 + SWAP m15, m8 + por m15, m11 + %if ARCH_X86_64 + pshufd m8, m10, q0000 + pshufd m9, m10, q1111 + pshufd m11, m10, q3333 + pshufd m10, m10, q2222 + %else + mova [esp+0x10], m15 + %define m15 [esp+0x10] + mov r0, r0m + pshufd m5, m4, q0000 + pshufd m6, m4, q1111 + pshufd m7, m4, q2222 + pshufd m4, m4, q3333 + %define m8 [esp+0x20] + %define m9 [esp+0x30] + %define m10 [esp+0x40] + %define m11 [esp+0x50] + mova m8, m5 + mova m9, m6 + mova m10, m7 + mova m11, m4 + %endif + pshufb m0, m14 + pshufb m2, m14 + pshufb m1, m14 + pshufb m3, m14 + pmaddubsw m0, m15 + pmaddubsw m2, m15 + pmaddubsw m1, m15 + pmaddubsw m3, m15 + phaddw m0, m2 + phaddw m1, m3 + pmulhrsw m0, m12 + pmulhrsw m1, m12 + palignr m2, m1, m0, 4 + pshufd m4, m1, q2121 + punpcklwd m3, m0, m2 ; 01 12 + punpckhwd m0, m2 ; 23 34 + punpcklwd m2, m1, m4 ; 45 56 +.dy1_w2_loop: + movq m1, [srcq+ssq*0] + movhps m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd m5, m3, m8 + pmaddwd m6, m0, m9 + pmaddwd m7, m2, m10 + mova m3, m0 + mova m0, m2 + paddd m5, m13 + paddd m6, m7 + pshufb m1, m14 + pmaddubsw m1, m15 + phaddw m1, m1 + pmulhrsw m1, m12 + palignr m7, m1, m4, 12 + punpcklwd m2, m7, m1 ; 67 78 + pmaddwd m7, m2, m11 + mova m4, m1 + paddd m5, m6 + paddd m5, m7 + psrad m5, rndshift + packssdw m5, m5 + packuswb m5, m5 + movd r4d, m5 + mov [dstq+dsq*0], r4w + shr r4d, 16 + mov [dstq+dsq*1], r4w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +INIT_XMM ssse3 +.dy1_w4: +%if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d +%else + %define m10 [base+pd_0x3ff] + %define m11 [base+pd_0x4000] + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %if isprep + %define ssq r3 + %endif + movzx r4, byte [esp+0x1f0] + dec srcq + movd m15, r4 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m11, [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + psrldq m7, m15, 8 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r6d, m15 + movd r13d, m7 + movd m15, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+r11*8+2] + movd m3, [base+subpel_filters+ r6*8+2] + movd m4, [base+subpel_filters+r13*8+2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] +%else + movd r1, m15 + movd r3, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r4, m15 + movd r5, m7 + %define m15 m5 + SWAP m4, m7 + movd m15, [base+subpel_filters+r1*8+2] + movd m2, [base+subpel_filters+r3*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m4, [base+subpel_filters+r5*8+2] + mov myd, mym + mov rX, [esp+0x1f4] + xor r5, r5 + shr myd, 6 + lea rX, [rX+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+rX*8+0] + cmovnz r5, [base+subpel_filters+rX*8+4] + mov r3, r3m + %if isprep + lea ss3q, [ssq*3] + %endif +%endif + punpckldq m15, m3 + punpckldq m2, m4 + punpcklqdq m15, m2 + movq m6, [base+subpel_s_shuf2] +%if ARCH_X86_64 + pcmpeqd m8, m9 + psrld m14, 10 + pshufb m14, [base+bdct_lb_dw] + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + movu m2, [srcq+ssq*2] + movu m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + punpcklqdq m6, m6 + movu m4, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + movu m7, [srcq+ssq*2] + add srcq, ss3q + pand m11, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m11 + paddb m14, m6 + movq m10, r4q + punpcklbw m10, m10 + psraw m10, 8 + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + pshufb m4, m14 + pshufb m5, m14 + pshufb m7, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + pmaddubsw m4, m15 + pmaddubsw m5, m15 + pmaddubsw m7, m15 + phaddw m0, m1 + phaddw m2, m3 + phaddw m4, m5 + phaddw m6, m7, m7 + pmulhrsw m0, m12 ; 0 1 + pmulhrsw m2, m12 ; 2 3 + pmulhrsw m4, m12 ; 4 5 + pmulhrsw m6, m12 ; 6 _ + shufps m1, m0, m2, q1032 ; 1 2 + shufps m3, m2, m4, q1032 ; 3 4 + shufps m5, m4, m6, q1032 ; 5 6 + punpcklwd m7, m0, m1 ; 01 + punpckhwd m0, m1 ; 12 + punpcklwd m8, m2, m3 ; 23 + punpckhwd m2, m3 ; 34 + punpcklwd m9, m4, m5 ; 45 + punpckhwd m4, m5 ; 56 +%else + pxor m3, m3 + pcmpeqd m8, m3 + psrld m14, 10 + pshufb m14, [base+bdct_lb_dw] + movu m1, [srcq+ssq*0] + movu m2, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + add srcq, ss3q + punpcklqdq m6, m6 + SWAP m4, m7 + pand m7, m11, m8 + pandn m8, m15 + SWAP m5, m0 + por m15, m7 + paddb m14, m6 + movu m0, [srcq+ssq*0] + movu m7, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + pshufb m0, m14 + pshufb m7, m14 + pshufb m6, m14 + pmaddubsw m1, m15 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + mova [esp+0x00], m14 + mova [esp+0x10], m15 + pmaddubsw m0, m15 + pmaddubsw m7, m15 + pmaddubsw m6, m15 + phaddw m1, m2 + movu m2, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + mov r0, r0m + phaddw m3, m0 + pshufb m2, m14 + pmaddubsw m2, m15 + %define m14 [esp+0x00] + %define m15 [esp+0x10] + phaddw m7, m6 + phaddw m2, m2 + movd m6, r4 + movd m0, r5 + punpckldq m6, m0 + punpcklbw m6, m6 + psraw m6, 8 + mova [esp+0x20], m6 + pmulhrsw m1, m12 ; 0 1 + pmulhrsw m3, m12 ; 2 3 + pmulhrsw m7, m12 ; 4 5 + pmulhrsw m2, m12 ; 6 _ + shufps m0, m1, m3, q1032 ; 1 2 + shufps m4, m3, m7, q1032 ; 3 4 + shufps m5, m7, m2, q1032 ; 5 6 + punpcklwd m6, m1, m0 ; 01 + punpckhwd m1, m0 ; 12 + mova [esp+0x30], m1 + punpcklwd m1, m3, m4 ; 23 + punpckhwd m3, m4 ; 34 + mova [esp+0x40], m3 + punpcklwd m3, m7, m5 ; 45 + punpckhwd m7, m5 ; 56 + mova [esp+0x50], m7 + mova [esp+0x60], m2 + mova m0, [esp+0x20] + %xdefine m8 m1 + %xdefine m9 m3 + %xdefine m10 m0 + SWAP m7, m6 + SWAP m1, m4 + SWAP m3, m2 +%endif + pshufd m1, m10, q0000 + pshufd m3, m10, q1111 + pshufd m5, m10, q2222 + pshufd m10, m10, q3333 +%if ARCH_X86_64 + mova [rsp+0x00], m8 + mova [rsp+0x10], m2 + mova [rsp+0x20], m9 + mova [rsp+0x30], m4 +%else + mova [esp+0x70], m8 + mova [esp+0x80], m9 + mova [esp+0x90], m1 + mova [esp+0xa0], m3 + mova [esp+0xb0], m5 + mova [esp+0xc0], m10 + %ifidn %1, put + mov dsd, dsm + %endif + %define m11 m6 +%endif +.dy1_w4_loop: +%if ARCH_X86_64 + movu m11, [srcq+ssq*0] + pmaddwd m7, m1 + pmaddwd m8, m3 + pmaddwd m0, m1 + pmaddwd m2, m3 + pmaddwd m9, m5 + pmaddwd m4, m5 + paddd m7, m8 + paddd m0, m2 + movu m8, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m11, m14 + pmaddubsw m11, m15 + paddd m7, m13 + paddd m0, m13 + paddd m7, m9 + paddd m0, m4 + pshufb m8, m14 + pmaddubsw m8, m15 + phaddw m11, m8 + mova m8, [rsp+0x20] + pmulhrsw m11, m12 + punpcklwd m9, m6, m11 ; 67 + psrldq m6, m11, 8 + punpcklwd m4, m11, m6 ; 78 + pmaddwd m2, m9, m10 + pmaddwd m11, m4, m10 + paddd m7, m2 + mova m2, [rsp+0x30] + paddd m0, m11 +%else + SWAP m7, m6 + SWAP m1, m4 + SWAP m3, m2 + movu m5, [srcq+ssq*0] + mova m0, [esp+0x30] + mova m2, [esp+0x40] + mova m4, [esp+0x50] + pmaddwd m6, [esp+0x90] + pmaddwd m1, [esp+0xa0] + pmaddwd m0, [esp+0x90] + pmaddwd m2, [esp+0xa0] + pmaddwd m3, [esp+0xb0] + pmaddwd m4, [esp+0xb0] + paddd m6, m1 + paddd m0, m2 + movu m7, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m5, m14 + pmaddubsw m5, m15 + paddd m6, m13 + paddd m0, m13 + paddd m6, m3 + paddd m0, m4 + pshufb m7, m14 + pmaddubsw m7, m15 + phaddw m5, m7 + mova m7, [rsp+0x80] + pmulhrsw m5, m12 + punpcklwd m3, [esp+0x60], m5 ; 67 + psrldq m1, m5, 8 + punpcklwd m4, m5, m1 ; 78 + pmaddwd m2, m3, [esp+0xc0] + pmaddwd m5, m4, [esp+0xc0] + mova [esp+0x60], m1 + paddd m6, m2 + mova m2, [esp+0x50] + paddd m0, m5 + SWAP m7, m6 +%endif + psrad m7, rndshift + psrad m0, rndshift + packssdw m7, m0 +%if ARCH_X86_64 + mova m0, [rsp+0x10] +%else + mova m0, [esp+0x40] +%define m11 m5 +%endif +%ifidn %1, put + packuswb m7, m7 + psrldq m11, m7, 4 + movd [dstq+dsq*0], m7 + movd [dstq+dsq*1], m11 + lea dstq, [dstq+dsq*2] +%else + mova [tmpq], m7 + add tmpq, 16 +%endif + sub hd, 2 + jz .ret +%if ARCH_X86_64 + mova m7, [rsp+0x00] + mova [rsp+0x00], m8 + mova [rsp+0x10], m2 + mova [rsp+0x20], m9 + mova [rsp+0x30], m4 +%else + mova m7, [esp+0x70] ; 01 + mova m1, [esp+0x80] ; 23 + mova m2, [esp+0x50] ; 34 + mova [esp+0x30], m0 + mova [esp+0x70], m1 + mova [esp+0x40], m2 + mova [esp+0x80], m3 + mova [esp+0x50], m4 +%endif + jmp .dy1_w4_loop +INIT_XMM ssse3 +.dy1_w8: + mov dword [rsp+0x90], 1 + movifprep tmp_stridem, 16 + jmp .dy1_w_start +.dy1_w16: + mov dword [rsp+0x90], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [rsp+0x90], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [rsp+0x90], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [rsp+0x90], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: + mov myd, mym +%ifidn %1, put + movifnidn dsm, dsq +%endif +%if ARCH_X86_64 + shr t0d, 16 + sub srcq, 3 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + movd m15, t0d +%else + %define m8 m0 + %define m9 m1 + %xdefine m14 m4 + %xdefine m15 m3 + %if isprep + %define ssq ssm + %endif + mov r5, [esp+0x1f0] + mov r3, [esp+0x1f4] + shr r5, 16 + sub srcq, 3 + movd m15, r5 + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r0, r0m + mov r3, r3m +%endif + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] +%if ARCH_X86_64 + movq m3, r4q + punpcklbw m3, m3 + psraw m3, 8 +%else + movd m5, r4 + movd m6, r5 + punpckldq m5, m6 + punpcklbw m5, m5 + psraw m5, 8 + SWAP m3, m5 +%endif + mova [rsp+0x100], m7 + mova [rsp+0x120], m15 + mov [rsp+0x098], srcq + mov [rsp+0x130], r0q ; dstq / tmpq + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova [rsp+0x140], m0 + mova [rsp+0x150], m1 + mova [rsp+0x160], m2 + mova [rsp+0x170], m3 +%if ARCH_X86_64 && UNIX64 + mov hm, hd +%elif ARCH_X86_32 + SWAP m5, m3 + mov r5, hm + mov [esp+0x134], r5 +%endif + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [rsp+0x090] + jz .ret +%if ARCH_X86_64 + add qword [rsp+0x130], 8*(isprep+1) + mov hd, hm +%else + add dword [rsp+0x130], 8*(isprep+1) + mov r5, [esp+0x134] + mov r0, [esp+0x130] +%endif + mova m7, [rsp+0x100] + mova m14, [rsp+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] +%else + %define m10 [base+pd_0x3ff] +%endif + mova m15, [rsp+0x120] + mov srcq, [rsp+0x098] +%if ARCH_X86_64 + mov r0q, [rsp+0x130] ; dstq / tmpq +%else + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.dy1_hloop: + pxor m9, m9 +%if ARCH_X86_64 + mova m11, [base+pq_0x40000000] +%else + %define m11 [base+pq_0x40000000] +%endif + psrld m2, m14, 10 + mova [rsp], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m9 + psrldq m2, m5, 8 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] + pxor m2, m2 + %define m9 m2 +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + mova [rsp+0x110], m14 + psrldq m4, m15, 8 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + psrldq m4, m14, 8 + movd r10d, m14 + movd r11d, m4 + psrldq m14, 4 + psrldq m4, 4 + movd r13d, m14 + movd rXd, m4 + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m8, m11, m4 + pand m9, m11, m6 + pand m15, m11, m7 + pand m11, m11, m5 + pandn m4, m0 + pandn m6, m1 + pandn m7, m2 + pandn m5, m3 + por m8, m4 + por m9, m6 + por m15, m7 + por m11, m5 + mova [rsp+0x10], m8 + mova [rsp+0x20], m9 + mova [rsp+0x30], m15 + mova [rsp+0x40], m11 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 10, 8, 9, 15, 11 ; 0-1 + mova [rsp+0x50], m1 + mova [rsp+0x60], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 7, 10, 8, 9, 15, 11 ; 2-3 + mova [rsp+0x70], m3 + mova [rsp+0x80], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 4-5 + MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 6-7 + SWAP m7, m0 + SWAP m8, m14 + mova m1, [rsp+0x50] + mova m2, [rsp+0x60] + mova m3, [rsp+0x70] + mova m15, [rsp+0x80] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + SWAP m14, m8 + mova m8, [rsp+0x140] + mova m9, [rsp+0x150] + mova m10, [rsp+0x160] + mova m11, [rsp+0x170] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m15; 23a + punpckhwd m3, m15 ; 23b + mova [rsp+0x50], m4 + mova [rsp+0x60], m5 + mova [rsp+0x70], m6 + mova [rsp+0x80], m7 + mova m14, [base+unpckw] +%else + movd r0, m15 + movd rX, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r4, m15 + movd r5, m4 + mova m14, [esp+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [esp+16], m14 + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m11, m4 + pand m1, m11, m6 + pand m2, m11, m7 + pand m3, m11, m5 + pandn m4, [esp+0x20] + pandn m6, [esp+0x30] + pandn m7, [esp+0x40] + pandn m5, [esp+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + MC_8TAP_SCALED_H 0x20, 0x60, 0 ; 0-1 + MC_8TAP_SCALED_H 0x20, 0x180 ; 2-3 + MC_8TAP_SCALED_H 0x20, 0x1a0 ; 4-5 + MC_8TAP_SCALED_H 0x20, 0x1c0 ; 6-7 + mova m5, [esp+0x1a0] + mova m6, [esp+0x1b0] + mova m7, [esp+0x1c0] + mova m0, [esp+0x1d0] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [esp+0x1a0], m4 + mova [esp+0x1b0], m5 + mova [esp+0x1c0], m6 + mova [esp+0x1d0], m7 + mova m1, [esp+0x060] + mova m2, [esp+0x070] + mova m3, [esp+0x180] + mova m4, [esp+0x190] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova [esp+0x060], m0 + mova [esp+0x070], m1 + mova [esp+0x180], m2 + mova [esp+0x190], m3 + %define m8 [esp+0x140] + %define m9 [esp+0x150] + %define m10 [esp+0x160] + %define m11 [esp+0x170] +%endif +.dy1_vloop: +%if ARCH_X86_32 + mov r0, r0m +%endif + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m8 + pmaddwd m6, m2, m9 + pmaddwd m7, m3, m9 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 +%if ARCH_X86_64 + pmaddwd m6, [rsp+0x50], m10 + pmaddwd m7, [rsp+0x60], m10 +%else + pmaddwd m6, [rsp+0x1a0], m10 + pmaddwd m7, [rsp+0x1b0], m10 +%endif + paddd m4, m6 + paddd m5, m7 +%if ARCH_X86_64 + pmaddwd m6, [rsp+0x70], m11 + pmaddwd m7, [rsp+0x80], m11 +%else + pmaddwd m6, [rsp+0x1c0], m11 + pmaddwd m7, [rsp+0x1d0], m11 +%endif + paddd m4, m6 + paddd m5, m7 + psrad m4, rndshift + psrad m5, rndshift + packssdw m4, m5 +%ifidn %1, put + packuswb m4, m4 + movq [dstq], m4 + add dstq, dsm +%else + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif +%if ARCH_X86_32 + mov r0m, r0 +%endif + dec hd + jz .dy1_hloop_prep +%if ARCH_X86_64 + movq m4, [srcq+ r4] + movq m5, [srcq+ r6] + movhps m4, [srcq+ r7] + movhps m5, [srcq+ r9] + movq m6, [srcq+r10] + movq m7, [srcq+r11] + movhps m6, [srcq+r13] + movhps m7, [srcq+ rX] + add srcq, ssq + pshufd m15, m14, q1032 + pshufb m0, m14 ; 0a 1a + pshufb m1, m14 ; 0b 1b + pshufb m2, m15 ; 3a 2a + pshufb m3, m15 ; 3b 2b + pmaddubsw m4, [rsp+0x10] + pmaddubsw m5, [rsp+0x20] + pmaddubsw m6, [rsp+0x30] + pmaddubsw m7, [rsp+0x40] + phaddw m4, m5 + phaddw m6, m7 + phaddw m4, m6 + pmulhrsw m4, m12 + pshufb m5, [rsp+0x70], m15 ; 7a 6a + pshufb m7, [rsp+0x80], m15 ; 7b 6b + pshufb m6, [rsp+0x50], m14 ; 4a 5a + pshufb m15, [rsp+0x60], m14 ; 4b 5b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + punpcklwd m2, m6 ; 34a + punpcklwd m3, m15 ; 34b + punpckhwd m6, m5 ; 56a + punpckhwd m15, m7 ; 56b + punpcklwd m5, m4 ; 78a + psrldq m4, 8 + punpcklwd m7, m4 ; 78b + mova [rsp+0x50], m6 + mova [rsp+0x60], m15 + mova [rsp+0x70], m5 + mova [rsp+0x80], m7 +%else + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + mova m6, [base+unpckw] + mova m0, [esp+0x060] + mova m1, [esp+0x070] + mova m7, [esp+0x1a0] + movq m4, [srcq+r0] + movq m5, [srcq+rX] + movhps m4, [srcq+r4] + movhps m5, [srcq+r5] + pshufb m0, m6 ; 0a 1a + pshufb m1, m6 ; 0b 1b + pshufb m7, m6 ; 4a 5a + mov r0, [esp+16] + mov rX, [esp+24] + mov r4, [esp+20] + mov r5, [esp+28] + movq m3, [srcq+r0] + movq m2, [srcq+rX] + movhps m3, [srcq+r4] + movhps m2, [srcq+r5] + add srcq, ssq + pmaddubsw m4, [esp+0x20] + pmaddubsw m5, [esp+0x30] + pmaddubsw m3, [esp+0x40] + pmaddubsw m2, [esp+0x50] + phaddw m4, m5 + phaddw m3, m2 + mova m5, [esp+0x1b0] + mova m2, [esp+0x180] + phaddw m4, m3 + mova m3, [esp+0x190] + pmulhrsw m4, m12 ; 8a 8b + pshufb m5, m6 ; 4b 5b + pshufd m6, m6, q1032 + pshufb m2, m6 ; 3a 2a + pshufb m3, m6 ; 3b 2b + punpckhwd m0, m2 ; 12a + punpckhwd m1, m3 ; 12b + mova [esp+0x60], m0 + mova [esp+0x70], m1 + mova m0, [esp+0x1c0] + mova m1, [esp+0x1d0] + punpcklwd m2, m7 ; 34a + punpcklwd m3, m5 ; 34b + mova [esp+0x180], m2 + mova [esp+0x190], m3 + pshufb m0, m6 ; 7a 6a + pshufb m1, m6 ; 7b 6b + punpckhwd m7, m0 ; 56a + punpckhwd m5, m1 ; 56b + punpcklwd m0, m4 + punpckhqdq m4, m4 + punpcklwd m1, m4 + mova [esp+0x1a0], m7 + mova [esp+0x1b0], m5 + mova [esp+0x1c0], m0 + mova [esp+0x1d0], m1 + mova m0, [esp+0x60] + mova m1, [esp+0x70] +%endif + jmp .dy1_vloop +INIT_XMM ssse3 +.dy2: + movzx wd, word [base+%1_8tap_scaled_ssse3_dy2_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy2_w2: + %if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d + %else + %define m10 [base+pd_0x3ff] + %define m11 [base+pd_0x4000] + %define m8 m0 + %define m9 m1 + %define m14 m4 + %define m15 m3 + movzx r5, byte [esp+0x1f0] + dec srcd + movd m15, r5 + %endif + punpckldq m9, m8 + SWAP m8, m9 + paddd m14, m8 ; mx+dx*[0-1] + %if ARCH_X86_64 + mova m11, [base+pd_0x4000] + %endif + pshufd m15, m15, q0000 + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + movd r4d, m15 + psrldq m15, 4 + %if ARCH_X86_64 + movd r6d, m15 + %else + movd r3d, m15 + %endif + mova m5, [base+bdct_lb_dw] + mova m6, [base+subpel_s_shuf2] + movd m15, [base+subpel_filters+r4*8+2] + %if ARCH_X86_64 + movd m7, [base+subpel_filters+r6*8+2] + %else + movd m7, [base+subpel_filters+r3*8+2] + %endif + pxor m9, m9 + pcmpeqd m8, m9 + psrld m14, 10 + %if ARCH_X86_32 + mov r3, r3m + pshufb m14, m5 + paddb m14, m6 + mova [esp+0x00], m14 + %define m14 [esp+0x00] + SWAP m5, m0 + SWAP m6, m3 + %define m8 m5 + %define m15 m6 + %endif + movq m0, [srcq+ssq*0] + movq m1, [srcq+ssq*1] + movhps m0, [srcq+ssq*2] + movhps m1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + %if ARCH_X86_64 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + movq m10, r4q + %else + mov myd, mym + mov r3, [esp+0x1f4] + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r3, r3m + %define m10 m4 + movd m10, r4 + movd m3, r5 + punpckldq m10, m3 + %endif + movq m3, [srcq+ssq*0] + movhps m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m10, m10 + psraw m10, 8 + punpckldq m15, m7 + punpcklqdq m15, m15 + %if ARCH_X86_64 + pand m11, m8 + %else + pand m7, m11, m8 + %define m11 m7 + %endif + pandn m8, m15 + SWAP m15, m8 + por m15, m11 + %if ARCH_X86_64 + pshufd m8, m10, q0000 + pshufd m9, m10, q1111 + pshufd m11, m10, q3333 + pshufd m10, m10, q2222 + %else + mova [esp+0x10], m15 + %define m15 [esp+0x10] + mov r5, r0m + %define dstq r5 + mov dsd, dsm + pshufd m5, m4, q0000 + pshufd m6, m4, q1111 + pshufd m7, m4, q2222 + pshufd m4, m4, q3333 + %define m8 [esp+0x20] + %define m9 [esp+0x30] + %define m10 [esp+0x40] + %define m11 [esp+0x50] + mova m8, m5 + mova m9, m6 + mova m10, m7 + mova m11, m4 + %endif + pshufb m0, m14 + pshufb m1, m14 + pshufb m3, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + pmaddubsw m3, m15 + pslldq m2, m3, 8 + phaddw m0, m2 + phaddw m1, m3 + pmulhrsw m0, m12 ; 0 2 _ 4 + pmulhrsw m1, m12 ; 1 3 _ 5 + pshufd m2, m0, q3110 ; 0 2 2 4 + pshufd m1, m1, q3110 ; 1 3 3 5 + punpcklwd m3, m2, m1 ; 01 23 + punpckhwd m2, m1 ; 23 45 +.dy2_w2_loop: + movq m6, [srcq+ssq*0] + movq m7, [srcq+ssq*1] + movhps m6, [srcq+ssq*2] + movhps m7, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pmaddwd m4, m3, m8 + pmaddwd m5, m2, m9 + pshufb m6, m14 + pshufb m7, m14 + pmaddubsw m6, m15 + pmaddubsw m7, m15 + phaddw m6, m7 + pmulhrsw m6, m12 + psrldq m7, m6, 8 + palignr m6, m0, 8 + palignr m7, m1, 8 + mova m0, m6 + mova m1, m7 + pshufd m6, m6, q3221 + pshufd m7, m7, q3221 + punpcklwd m3, m6, m7 ; 45 67 + punpckhwd m2, m6, m7 ; 67 89 + pmaddwd m6, m3, m10 + pmaddwd m7, m2, m11 + paddd m4, m5 + paddd m4, m13 + paddd m6, m7 + paddd m4, m6 + psrad m4, rndshift + packssdw m4, m4 + packuswb m4, m4 + movd r4d, m4 + mov [dstq+dsq*0], r4w + shr r4d, 16 + mov [dstq+dsq*1], r4w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +INIT_XMM ssse3 +.dy2_w4: +%if ARCH_X86_64 + mov myd, mym + movzx t0d, t0b + dec srcq + movd m15, t0d +%else + %define m10 [base+pd_0x3ff] + %define m11 [base+pd_0x4000] + %define m8 m0 + %xdefine m14 m4 + %define m15 m3 + %define dstq r0 + %if isprep + %define ssq r3 + %endif + movzx r4, byte [esp+0x1f0] + dec srcq + movd m15, r4 +%endif + pmaddwd m8, [base+rescale_mul] +%if ARCH_X86_64 + mova m11, [base+pd_0x4000] +%endif + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd m15, m8 + psrldq m7, m15, 8 +%if ARCH_X86_64 + movd r4d, m15 + movd r11d, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r6d, m15 + movd r13d, m7 + movd m15, [base+subpel_filters+ r4*8+2] + movd m2, [base+subpel_filters+r11*8+2] + movd m3, [base+subpel_filters+ r6*8+2] + movd m4, [base+subpel_filters+r13*8+2] + movq m6, [base+subpel_s_shuf2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] +%else + movd r1, m15 + movd r3, m7 + psrldq m15, 4 + psrldq m7, 4 + movd r4, m15 + movd r5, m7 + %define m15 m5 + SWAP m4, m7 + movd m15, [base+subpel_filters+r1*8+2] + movd m2, [base+subpel_filters+r3*8+2] + movd m3, [base+subpel_filters+r4*8+2] + movd m4, [base+subpel_filters+r5*8+2] + movq m6, [base+subpel_s_shuf2] + mov myd, mym + mov r3, [esp+0x1f4] + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r3, r3m + %if isprep + lea ss3q, [ssq*3] + %endif +%endif + punpckldq m15, m3 + punpckldq m2, m4 + punpcklqdq m15, m2 +%if ARCH_X86_64 + pcmpeqd m8, m9 + psrld m14, 10 + movu m0, [srcq+ssq*0] + movu m2, [srcq+ssq*2] + movu m1, [srcq+ssq*1] + movu m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + punpcklqdq m6, m6 + pshufb m14, [base+bdct_lb_dw] + movu m4, [srcq+ssq*0] + movu m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pand m11, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m11 + paddb m14, m6 + movq m11, r4q + punpcklbw m11, m11 + psraw m11, 8 + pshufb m0, m14 + pshufb m2, m14 + pshufb m1, m14 + pshufb m3, m14 + pshufb m4, m14 + pshufb m5, m14 + pmaddubsw m0, m15 + pmaddubsw m2, m15 + pmaddubsw m1, m15 + pmaddubsw m3, m15 + pmaddubsw m4, m15 + pmaddubsw m5, m15 + phaddw m0, m2 + phaddw m1, m3 + phaddw m4, m5 + pmulhrsw m0, m12 ; 0 2 + pmulhrsw m1, m12 ; 1 3 + pmulhrsw m4, m12 ; 4 5 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 +%else + pxor m3, m3 + pcmpeqd m8, m3 + psrld m14, 10 + pshufb m14, [base+bdct_lb_dw] + movu m1, [srcq+ssq*0] + movu m2, [srcq+ssq*2] + movu m3, [srcq+ssq*1] + add srcq, ss3q + punpcklqdq m6, m6 + SWAP m4, m7 + pand m7, m11, m8 + pandn m8, m15 + SWAP m15, m8 + por m15, m7 + paddb m14, m6 + movu m0, [srcq+ssq*0] + movu m7, [srcq+ssq*1] + movu m6, [srcq+ssq*2] + add srcq, ss3q + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + pshufb m0, m14 + pshufb m7, m14 + pshufb m6, m14 + pmaddubsw m1, m15 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + mova [esp+0x00], m14 + mova [esp+0x10], m15 + pmaddubsw m0, m15 + pmaddubsw m7, m15 + pmaddubsw m6, m15 + %define m14 [esp+0x00] + %define m15 [esp+0x10] + phaddw m1, m2 + phaddw m3, m0 + phaddw m7, m6 + %ifidn %1, put + mov dsd, dsm + %define dstq r5 + %else + %define tmpq r5 + %endif + movd m6, r4 + movd m0, r5 + punpckldq m6, m0 + punpcklbw m6, m6 + psraw m6, 8 + mov r5, r0m + pmulhrsw m1, m12 ; 0 2 + pmulhrsw m3, m12 ; 1 3 + pmulhrsw m7, m12 ; 4 5 + SWAP m0, m1, m3 + SWAP m4, m7 + pshufd m2, m6, q0000 + pshufd m3, m6, q1111 + pshufd m7, m6, q2222 + pshufd m6, m6, q3333 + mova [esp+0x30], m2 + mova [esp+0x40], m3 + mova [esp+0x50], m7 + mova [esp+0x60], m6 + %define m8 [esp+0x30] + %define m9 [esp+0x40] + %define m10 [esp+0x50] + %define m11 [esp+0x60] +%endif + psrldq m5, m4, 8 ; 5 _ + punpckhwd m2, m0, m1 ; 23 + punpcklwd m0, m1 ; 01 + punpcklwd m4, m5 ; 45 +.dy2_w4_loop: + pmaddwd m0, m8 ; a0 + pmaddwd m5, m2, m8 ; b0 + pmaddwd m2, m9 ; a1 + pmaddwd m7, m4, m9 ; b1 + pmaddwd m3, m4, m10 ; a2 + paddd m0, m13 + paddd m5, m13 + paddd m0, m2 + paddd m5, m7 + paddd m0, m3 + movu m6, [srcq+ssq*0] + movu m7, [srcq+ssq*1] + movu m3, [srcq+ssq*2] + movu m1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m6, m14 + pshufb m7, m14 + pshufb m3, m14 + pshufb m1, m14 + pmaddubsw m6, m15 + pmaddubsw m7, m15 + pmaddubsw m3, m15 + pmaddubsw m1, m15 + phaddw m6, m7 + phaddw m3, m1 + pmulhrsw m6, m12 ; 6 7 + pmulhrsw m3, m12 ; 8 9 + psrldq m7, m6, 8 + psrldq m1, m3, 8 + punpcklwd m6, m7 ; 67 + punpcklwd m3, m1 ; 89 + mova m2, m6 + pmaddwd m1, m6, m10 ; b2 + pmaddwd m6, m11 ; a3 + pmaddwd m7, m3, m11 ; b3 + paddd m5, m1 + paddd m0, m6 + paddd m5, m7 + psrad m0, rndshift + psrad m5, rndshift + packssdw m0, m5 +%ifidn %1, put + packuswb m0, m0 + psrldq m1, m0, 4 + movd [dstq+dsq*0], m0 + movd [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] +%else + mova [tmpq], m0 + add tmpq, 16 +%endif + mova m0, m4 + mova m4, m3 + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET +INIT_XMM ssse3 +.dy2_w8: + mov dword [rsp+0x90], 1 + movifprep tmp_stridem, 16 + jmp .dy2_w_start +.dy2_w16: + mov dword [rsp+0x90], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [rsp+0x90], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [rsp+0x90], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [rsp+0x90], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: + mov myd, mym +%ifidn %1, put + movifnidn dsm, dsq +%endif +%if ARCH_X86_64 + shr t0d, 16 + sub srcq, 3 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + movd m15, t0d +%else + %define m10 [base+pd_0x3ff] + %define m11 [base+pd_0x4000] + %define m8 m0 + %define m9 m1 + %xdefine m14 m4 + %xdefine m15 m3 + %if isprep + %define tmpq r0 + %define ssq ssm + %else + %define dstq r0 + %endif + mov r5, [esp+0x1f0] + mov r3, [esp+0x1f4] + shr r5, 16 + sub srcq, 3 + movd m15, r5 + xor r5, r5 + shr myd, 6 + lea r3, [r3+myd] + mov r4, 64 << 24 + cmovnz r4, [base+subpel_filters+r3*8+0] + cmovnz r5, [base+subpel_filters+r3*8+4] + mov r0, r0m + mov r3, r3m +%endif + pslld m7, m8, 2 ; dx*4 + pmaddwd m8, [base+rescale_mul] ; dx*[0-3] + pshufd m15, m15, q0000 + paddd m14, m8 ; mx+dx*[0-3] +%if ARCH_X86_64 + movq m3, r4q + punpcklbw m3, m3 + psraw m3, 8 +%else + movd m5, r4 + movd m6, r5 + punpckldq m5, m6 + punpcklbw m5, m5 + psraw m5, 8 + SWAP m3, m5 +%endif + mova [rsp+0x100], m7 + mova [rsp+0x120], m15 + mov [rsp+0x098], srcq + mov [rsp+0x130], r0q ; dstq / tmpq + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + mova [rsp+0x140], m0 + mova [rsp+0x150], m1 + mova [rsp+0x160], m2 + mova [rsp+0x170], m3 +%if ARCH_X86_64 && UNIX64 + mov hm, hd +%elif ARCH_X86_32 + SWAP m5, m3 + mov r5, hm + mov [esp+0x134], r5 +%endif + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [rsp+0x090] + jz .ret +%if ARCH_X86_64 + add qword [rsp+0x130], 8*(isprep+1) + mov hd, hm +%else + add dword [rsp+0x130], 8*(isprep+1) + mov r5, [esp+0x134] + mov r0, [esp+0x130] +%endif + mova m7, [rsp+0x100] + mova m14, [rsp+0x110] +%if ARCH_X86_64 + mova m10, [base+pd_0x3ff] +%else + %define m10 [base+pd_0x3ff] +%endif + mova m15, [rsp+0x120] + mov srcq, [rsp+0x098] +%if ARCH_X86_64 + mov r0q, [rsp+0x130] ; dstq / tmpq +%else + mov hm, r5 + mov r0m, r0 + mov r3, r3m +%endif + paddd m14, m7 +.dy2_hloop: + pxor m9, m9 +%if ARCH_X86_64 + mova m11, [base+pq_0x40000000] +%else + %define m11 [base+pq_0x40000000] +%endif + psrld m2, m14, 10 + mova [rsp], m2 + pand m6, m14, m10 + psrld m6, 6 + paddd m5, m15, m6 + pcmpeqd m6, m9 + psrldq m2, m5, 8 +%if ARCH_X86_64 + movd r4d, m5 + movd r6d, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r7d, m5 + movd r9d, m2 + movq m0, [base+subpel_filters+r4*8] + movq m1, [base+subpel_filters+r6*8] + movhps m0, [base+subpel_filters+r7*8] + movhps m1, [base+subpel_filters+r9*8] +%else + movd r0, m5 + movd rX, m2 + psrldq m5, 4 + psrldq m2, 4 + movd r4, m5 + movd r5, m2 + movq m0, [base+subpel_filters+r0*8] + movq m1, [base+subpel_filters+rX*8] + movhps m0, [base+subpel_filters+r4*8] + movhps m1, [base+subpel_filters+r5*8] + pxor m2, m2 + %define m9 m2 +%endif + paddd m14, m7 ; mx+dx*[4-7] + pand m5, m14, m10 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + mova [rsp+0x110], m14 + psrldq m4, m15, 8 +%if ARCH_X86_64 + movd r10d, m15 + movd r11d, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r13d, m15 + movd rXd, m4 + movq m2, [base+subpel_filters+r10*8] + movq m3, [base+subpel_filters+r11*8] + movhps m2, [base+subpel_filters+r13*8] + movhps m3, [base+subpel_filters+ rX*8] + psrld m14, 10 + psrldq m4, m14, 8 + movd r10d, m14 + movd r11d, m4 + psrldq m14, 4 + psrldq m4, 4 + movd r13d, m14 + movd rXd, m4 + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m8, m11, m4 + pand m9, m11, m6 + pand m15, m11, m7 + pand m11, m11, m5 + pandn m4, m0 + pandn m6, m1 + pandn m7, m2 + pandn m5, m3 + por m8, m4 + por m9, m6 + por m15, m7 + por m11, m5 + mova [rsp+0x10], m8 + mova [rsp+0x20], m9 + mova [rsp+0x30], m15 + mova [rsp+0x40], m11 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 10, 8, 9, 15, 11 ; 0-1 + mova [rsp+0x50], m1 + mova [rsp+0x60], m2 + MC_8TAP_SCALED_H 3, 4, 5, 6, 1, 2, 7, 10, 8, 9, 15, 11 ; 2-3 + mova [rsp+0x70], m3 + mova [rsp+0x80], m4 + MC_8TAP_SCALED_H 5, 6, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 4-5 + MC_8TAP_SCALED_H 0,14, 1, 2, 3, 4, 7, 10, 8, 9, 15, 11 ; 6-7 + SWAP m7, m0 + SWAP m8, m14 + mova m1, [rsp+0x50] + mova m2, [rsp+0x60] + mova m3, [rsp+0x70] + mova m15, [rsp+0x80] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m8 ; 67a + punpckhwd m7, m8 ; 67b + SWAP m14, m8 + mova m8, [rsp+0x140] + mova m9, [rsp+0x150] + mova m10, [rsp+0x160] + mova m11, [rsp+0x170] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m15; 23a + punpckhwd m3, m15 ; 23b + mova [rsp+0x50], m4 + mova [rsp+0x60], m5 + mova [rsp+0x70], m6 + mova [rsp+0x80], m7 +%else + movd r0, m15 + movd rX, m4 + psrldq m15, 4 + psrldq m4, 4 + movd r4, m15 + movd r5, m4 + mova m14, [esp+0x110] + movq m2, [base+subpel_filters+r0*8] + movq m3, [base+subpel_filters+rX*8] + movhps m2, [base+subpel_filters+r4*8] + movhps m3, [base+subpel_filters+r5*8] + psrld m14, 10 + mova [esp+16], m14 + mov r0, [esp+ 0] + mov rX, [esp+ 8] + mov r4, [esp+ 4] + mov r5, [esp+12] + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + pshufd m4, m6, q1100 + pshufd m6, m6, q3322 + pshufd m7, m5, q1100 + pshufd m5, m5, q3322 + pand m0, m11, m4 + pand m1, m11, m6 + pand m2, m11, m7 + pand m3, m11, m5 + pandn m4, [esp+0x20] + pandn m6, [esp+0x30] + pandn m7, [esp+0x40] + pandn m5, [esp+0x50] + por m0, m4 + por m1, m6 + por m2, m7 + por m3, m5 + mova [esp+0x20], m0 + mova [esp+0x30], m1 + mova [esp+0x40], m2 + mova [esp+0x50], m3 + MC_8TAP_SCALED_H 0x20, 0x60, 0 ; 0-1 + MC_8TAP_SCALED_H 0x20, 0x180 ; 2-3 + MC_8TAP_SCALED_H 0x20, 0x1a0 ; 4-5 + MC_8TAP_SCALED_H 0x20, 0x1c0 ; 6-7 + mova m5, [esp+0x1a0] + mova m6, [esp+0x1b0] + mova m7, [esp+0x1c0] + mova m0, [esp+0x1d0] + punpcklwd m4, m5, m6 ; 45a + punpckhwd m5, m6 ; 45b + punpcklwd m6, m7, m0 ; 67a + punpckhwd m7, m0 ; 67b + mova [esp+0x1a0], m4 + mova [esp+0x1b0], m5 + mova [esp+0x1c0], m6 + mova [esp+0x1d0], m7 + mova m1, [esp+0x060] + mova m2, [esp+0x070] + mova m3, [esp+0x180] + mova m4, [esp+0x190] + punpcklwd m0, m1, m2 ; 01a + punpckhwd m1, m2 ; 01b + punpcklwd m2, m3, m4 ; 23a + punpckhwd m3, m4 ; 23b + mova [esp+0x180], m2 + mova [esp+0x190], m3 + %define m8 [esp+0x140] + %define m9 [esp+0x150] + %define m10 [esp+0x160] + %define m11 [esp+0x170] +%endif +.dy2_vloop: +%if ARCH_X86_32 + mov r0, r0m +%endif + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m8 + pmaddwd m6, m2, m9 + pmaddwd m7, m3, m9 + paddd m4, m13 + paddd m5, m13 + paddd m4, m6 + paddd m5, m7 +%if ARCH_X86_64 + pmaddwd m6, [rsp+0x50], m10 + pmaddwd m7, [rsp+0x60], m10 +%else + pmaddwd m6, [esp+0x1a0], m10 + pmaddwd m7, [esp+0x1b0], m10 +%endif + paddd m4, m6 + paddd m5, m7 +%if ARCH_X86_64 + pmaddwd m6, [rsp+0x70], m11 + pmaddwd m7, [rsp+0x80], m11 +%else + pmaddwd m6, [esp+0x1c0], m11 + pmaddwd m7, [esp+0x1d0], m11 +%endif + paddd m4, m6 + paddd m5, m7 + psrad m4, rndshift + psrad m5, rndshift + packssdw m4, m5 +%ifidn %1, put + packuswb m4, m4 + movq [dstq], m4 + add dstq, dsm +%else + mova [tmpq], m4 + add tmpq, tmp_stridem +%endif +%if ARCH_X86_32 + mov r0m, r0 +%endif + dec hd + jz .dy2_hloop_prep +%if ARCH_X86_64 + mova m8, [rsp+0x10] + mova m9, [rsp+0x20] + mova m10, [rsp+0x30] + mova m11, [rsp+0x40] + mova m0, m2 ; 01a + mova m1, m3 ; 01b + MC_8TAP_SCALED_H 2, 6, 3, 4, 5, 7, 14, 15, 8, 9, 10, 11 + mova m3, [rsp+0x50] ; 23a + mova m4, [rsp+0x60] ; 23b + mova m5, [rsp+0x70] ; 45a + mova m7, [rsp+0x80] ; 45b + mova m8, [rsp+0x140] + mova m9, [rsp+0x150] + mova m10, [rsp+0x160] + mova m11, [rsp+0x170] + punpcklwd m14, m2, m6 ; 67a + punpckhwd m2, m6 ; 67b + mova [rsp+0x50], m5 + mova [rsp+0x60], m7 + mova [rsp+0x70], m14 + mova [rsp+0x80], m2 + mova m2, m3 + mova m3, m4 +%else + MC_8TAP_SCALED_H 0x20, 0 + punpcklwd m6, m0, m4 + punpckhwd m7, m0, m4 + mova m0, [esp+0x180] ; 01a + mova m1, [esp+0x190] ; 01b + mova m2, [rsp+0x1a0] ; 23a + mova m3, [esp+0x1b0] ; 23b + mova m4, [esp+0x1c0] ; 45a + mova m5, [esp+0x1d0] ; 45b + mova [esp+0x180], m2 + mova [esp+0x190], m3 + mova [esp+0x1a0], m4 + mova [esp+0x1b0], m5 + mova [esp+0x1c0], m6 ; 67a + mova [esp+0x1d0], m7 ; 67b +%endif + jmp .dy2_vloop +.ret: + MC_8TAP_SCALED_RET 0 +%if ARCH_X86_32 && !isprep && required_stack_alignment > STACK_ALIGNMENT + %define r0m [rstk+stack_offset+ 4] + %define r1m [rstk+stack_offset+ 8] + %define r2m [rstk+stack_offset+12] + %define r3m [rstk+stack_offset+16] +%endif +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled_8bpc + mov t0d, (5*15 << 16) | 5*15 + mov t1d, (5*15 << 16) | 5*15 + jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%elif ARCH_X86_64 +DECLARE_REG_TMP 6, 8 +%else +DECLARE_REG_TMP 1, 2 +%endif +BILIN_SCALED_FN put +FN put_8tap_scaled, sharp, SHARP, SHARP +FN put_8tap_scaled, sharp_smooth, SHARP, SMOOTH +FN put_8tap_scaled, smooth_sharp, SMOOTH, SHARP +FN put_8tap_scaled, smooth, SMOOTH, SMOOTH +FN put_8tap_scaled, sharp_regular, SHARP, REGULAR +FN put_8tap_scaled, regular_sharp, REGULAR, SHARP +FN put_8tap_scaled, smooth_regular, SMOOTH, REGULAR +FN put_8tap_scaled, regular_smooth, REGULAR, SMOOTH +FN put_8tap_scaled, regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%elif ARCH_X86_64 +DECLARE_REG_TMP 6, 7 +%else +DECLARE_REG_TMP 1, 2 +%endif +BILIN_SCALED_FN prep +FN prep_8tap_scaled, sharp, SHARP, SHARP +FN prep_8tap_scaled, sharp_smooth, SHARP, SMOOTH +FN prep_8tap_scaled, smooth_sharp, SMOOTH, SHARP +FN prep_8tap_scaled, smooth, SMOOTH, SMOOTH +FN prep_8tap_scaled, sharp_regular, SHARP, REGULAR +FN prep_8tap_scaled, regular_sharp, REGULAR, SHARP +FN prep_8tap_scaled, smooth_regular, SMOOTH, REGULAR +FN prep_8tap_scaled, regular_smooth, REGULAR, SMOOTH +FN prep_8tap_scaled, regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + +%if ARCH_X86_32 + %macro SAVE_ALPHA_BETA 0 + mov alpham, alphad + mov betam, betad + %endmacro + + %macro SAVE_DELTA_GAMMA 0 + mov deltam, deltad + mov gammam, gammad + %endmacro + + %macro LOAD_ALPHA_BETA_MX 0 + mov mym, myd + mov alphad, alpham + mov betad, betam + mov mxd, mxm + %endmacro + + %macro LOAD_DELTA_GAMMA_MY 0 + mov mxm, mxd + mov deltad, deltam + mov gammad, gammam + mov myd, mym + %endmacro + + %define PIC_reg r2 + %define PIC_base_offset $$ + %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) +%else + %define SAVE_ALPHA_BETA + %define SAVE_DELTA_GAMMA + %define PIC_sym(sym) sym +%endif + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < required_stack_alignment + %assign copy_args 8*4 + %else + %assign copy_args 0 + %endif +%endif + +%macro RELOC_ARGS 0 + %if copy_args + mov r0, r0m + mov r1, r1m + mov r2, r2m + mov r3, r3m + mov r5, r5m + mov dstm, r0 + mov dsm, r1 + mov srcm, r2 + mov ssm, r3 + mov mxm, r5 + mov r0, r6m + mov mym, r0 + %endif +%endmacro + +%macro BLENDHWDW 2 ; blend high words from dwords, src1, src2 + %if cpuflag(sse4) + pblendw %1, %2, 0xAA + %else + pand %2, m10 + por %1, %2 + %endif +%endmacro + +%macro WARP_V 10 ; dst0, dst1, 0, 2, 4, 6, 1, 3, 5, 7 + %if ARCH_X86_32 + %define m8 m4 + %define m9 m5 + %define m14 m6 + %define m15 m7 + %define m11 m7 + %endif + %if notcpuflag(ssse3) || ARCH_X86_32 + pxor m11, m11 + %endif + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq m2, [filterq+myq *8] ; a + movq m8, [filterq+tmp1q*8] ; e + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+deltaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq m3, [filterq+tmp2q*8] ; b + movq m0, [filterq+tmp1q*8] ; f + punpcklwd m2, m3 + punpcklwd m8, m0 + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq m0, [filterq+myq *8] ; c + movq m9, [filterq+tmp1q*8] ; g + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+gammaq] ; my += gamma + shr tmp2d, 10 + shr tmp1d, 10 + movq m3, [filterq+tmp2q*8] ; d + movq m1, [filterq+tmp1q*8] ; h + punpcklwd m0, m3 + punpcklwd m9, m1 + punpckldq m1, m2, m0 + punpckhdq m2, m0 + punpcklbw m0, m11, m1 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 + punpckhbw m3, m11, m1 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 + punpcklbw m1, m11, m2 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 + punpckhbw m14, m11, m2 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 + pmaddwd m0, %3 + pmaddwd m3, %5 + pmaddwd m1, %7 + pmaddwd m14, %9 + paddd m0, m3 + paddd m1, m14 + paddd m0, m1 + mova %1, m0 + %if ARCH_X86_64 + SWAP m3, m14 + %endif + punpckldq m0, m8, m9 + punpckhdq m8, m9 + punpcklbw m1, m11, m0 ; e0 e2 f0 f2 g0 g2 h0 h2 << 8 + punpckhbw m14, m11, m0 ; e4 e6 f4 f6 g4 g6 h4 h6 << 8 + punpcklbw m2, m11, m8 ; e1 e3 f1 f3 g1 g3 h1 h3 << 8 + punpckhbw m15, m11, m8 ; e5 e7 f5 f7 g5 g7 h5 h7 << 8 + pmaddwd m1, %4 + pmaddwd m14, %6 + pmaddwd m2, %8 + pmaddwd m15, %10 + paddd m1, m14 + paddd m2, m15 + paddd m1, m2 + mova %2, m1 + %if ARCH_X86_64 + SWAP m14, m3 + %endif +%endmacro + +%if ARCH_X86_64 + %define counterd r4d +%else + %if copy_args == 0 + %define counterd dword r4m + %else + %define counterd dword [esp+stack_size-4*7] + %endif +%endif + +%macro WARP_AFFINE_8X8T 0 +%if ARCH_X86_64 +cglobal warp_affine_8x8t_8bpc, 6, 14, 16, 0x90, tmp, ts +%else +cglobal warp_affine_8x8t_8bpc, 0, 7, 16, -0x130-copy_args, tmp, ts + %if copy_args + %define tmpm [esp+stack_size-4*1] + %define tsm [esp+stack_size-4*2] + %endif +%endif + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main +.loop: +%if ARCH_X86_32 + %define m12 m4 + %define m13 m5 + %define m14 m6 + %define m15 m7 + mova m12, [esp+0xC0] + mova m13, [esp+0xD0] + mova m14, [esp+0xE0] + mova m15, [esp+0xF0] +%endif +%if cpuflag(ssse3) + psrad m12, 13 + psrad m13, 13 + psrad m14, 13 + psrad m15, 13 + packssdw m12, m13 + packssdw m14, m15 + mova m13, [PIC_sym(pw_8192)] + pmulhrsw m12, m13 ; (x + (1 << 6)) >> 7 + pmulhrsw m14, m13 +%else + %if ARCH_X86_32 + %define m10 m0 + %endif + mova m10, [PIC_sym(pd_16384)] + paddd m12, m10 + paddd m13, m10 + paddd m14, m10 + paddd m15, m10 + psrad m12, 15 + psrad m13, 15 + psrad m14, 15 + psrad m15, 15 + packssdw m12, m13 + packssdw m14, m15 +%endif + mova [tmpq+tsq*0], m12 + mova [tmpq+tsq*2], m14 + dec counterd + jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).end +%if ARCH_X86_32 + mov tmpm, tmpd + mov r0, [esp+0x100] + mov r1, [esp+0x104] +%endif + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main2 + lea tmpq, [tmpq+tsq*4] + jmp .loop +%endmacro + +%macro WARP_AFFINE_8X8 0 +%if ARCH_X86_64 +cglobal warp_affine_8x8_8bpc, 6, 14, 16, 0x90, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma +%else +cglobal warp_affine_8x8_8bpc, 0, 7, 16, -0x130-copy_args, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma + %define alphaq r0 + %define alphad r0 + %define alpham [esp+gprsize+0x100] + %define betaq r1 + %define betad r1 + %define betam [esp+gprsize+0x104] + %define deltaq r0 + %define deltad r0 + %define deltam [esp+gprsize+0x108] + %define gammaq r1 + %define gammad r1 + %define gammam [esp+gprsize+0x10C] + %define filterq r3 + %define tmp1q r4 + %define tmp1d r4 + %define tmp1m [esp+gprsize+0x110] + %define myq r5 + %define myd r5 + %define mym r6m + %if copy_args + %define dstm [esp+stack_size-4*1] + %define dsm [esp+stack_size-4*2] + %define srcm [esp+stack_size-4*3] + %define ssm [esp+stack_size-4*4] + %define mxm [esp+stack_size-4*5] + %define mym [esp+stack_size-4*6] + %endif +%endif + call .main + jmp .start +.loop: +%if ARCH_X86_32 + mov dstm, dstd + mov alphad, [esp+0x100] + mov betad, [esp+0x104] +%endif + call .main2 + lea dstq, [dstq+dsq*2] +.start: +%if notcpuflag(sse4) + %if cpuflag(ssse3) + %define roundval pw_8192 + %else + %define roundval pd_262144 + %endif + %if ARCH_X86_64 + mova m10, [PIC_sym(roundval)] + %else + %define m10 [PIC_sym(roundval)] + %endif +%endif +%if ARCH_X86_32 + %define m12 m5 + %define m13 m6 + mova m12, [esp+0xC0] + mova m13, [esp+0xD0] +%endif +%if cpuflag(sse4) + %if ARCH_X86_32 + %define m11 m4 + pxor m11, m11 + %endif + psrad m12, 18 + psrad m13, 18 + packusdw m12, m13 + pavgw m12, m11 ; (x + (1 << 10)) >> 11 +%else + %if cpuflag(ssse3) + psrad m12, 17 + psrad m13, 17 + packssdw m12, m13 + pmulhrsw m12, m10 + %else + paddd m12, m10 + paddd m13, m10 + psrad m12, 19 + psrad m13, 19 + packssdw m12, m13 + %endif +%endif +%if ARCH_X86_32 + %define m14 m6 + %define m15 m7 + mova m14, [esp+0xE0] + mova m15, [esp+0xF0] +%endif +%if cpuflag(sse4) + psrad m14, 18 + psrad m15, 18 + packusdw m14, m15 + pavgw m14, m11 ; (x + (1 << 10)) >> 11 +%else + %if cpuflag(ssse3) + psrad m14, 17 + psrad m15, 17 + packssdw m14, m15 + pmulhrsw m14, m10 + %else + paddd m14, m10 + paddd m15, m10 + psrad m14, 19 + psrad m15, 19 + packssdw m14, m15 + %endif +%endif + packuswb m12, m14 + movq [dstq+dsq*0], m12 + movhps [dstq+dsq*1], m12 + dec counterd + jg .loop +.end: + RET +ALIGN function_align +.main: +%assign stack_offset stack_offset+gprsize +%if ARCH_X86_32 + %assign stack_size stack_size+4 + %if copy_args + %assign stack_offset stack_offset-4 + %endif + RELOC_ARGS + LEA PIC_reg, $$ + %define PIC_mem [esp+gprsize+0x114] + mov abcdd, abcdm + %if copy_args == 0 + mov ssd, ssm + mov mxd, mxm + %endif + mov PIC_mem, PIC_reg + mov srcd, srcm +%endif + movsx deltad, word [abcdq+2*2] + movsx gammad, word [abcdq+2*3] + lea tmp1d, [deltaq*3] + sub gammad, tmp1d ; gamma -= delta*3 + SAVE_DELTA_GAMMA +%if ARCH_X86_32 + mov abcdd, abcdm +%endif + movsx alphad, word [abcdq+2*0] + movsx betad, word [abcdq+2*1] + lea tmp1q, [ssq*3+3] + add mxd, 512+(64<<10) + lea tmp2d, [alphaq*3] + sub srcq, tmp1q ; src -= src_stride*3 + 3 +%if ARCH_X86_32 + mov srcm, srcd + mov PIC_reg, PIC_mem +%endif + sub betad, tmp2d ; beta -= alpha*3 + lea filterq, [PIC_sym(mc_warp_filter2)] +%if ARCH_X86_64 + mov myd, r6m + %if cpuflag(ssse3) + pxor m11, m11 + %endif +%endif + call .h + psrld m2, m0, 16 + psrld m3, m1, 16 +%if ARCH_X86_32 + %if notcpuflag(ssse3) + mova [esp+gprsize+0x00], m2 + %endif + mova [esp+gprsize+0x10], m3 +%endif + call .h + psrld m4, m0, 16 + psrld m5, m1, 16 +%if ARCH_X86_32 + mova [esp+gprsize+0x20], m4 + mova [esp+gprsize+0x30], m5 +%endif + call .h +%if ARCH_X86_64 + %define blendmask [rsp+gprsize+0x80] +%else + %if notcpuflag(ssse3) + mova m2, [esp+gprsize+0x00] + %endif + mova m3, [esp+gprsize+0x10] + %define blendmask [esp+gprsize+0x120] + %define m10 m7 +%endif + pcmpeqd m10, m10 + pslld m10, 16 + mova blendmask, m10 + BLENDHWDW m2, m0 ; 0 + BLENDHWDW m3, m1 ; 2 + mova [rsp+gprsize+0x00], m2 + mova [rsp+gprsize+0x10], m3 + call .h +%if ARCH_X86_32 + mova m4, [esp+gprsize+0x20] + mova m5, [esp+gprsize+0x30] +%endif + mova m10, blendmask + BLENDHWDW m4, m0 ; 1 + BLENDHWDW m5, m1 ; 3 + mova [rsp+gprsize+0x20], m4 + mova [rsp+gprsize+0x30], m5 + call .h +%if ARCH_X86_32 + %if notcpuflag(ssse3) + mova m2, [esp+gprsize+0x00] + %endif + mova m3, [esp+gprsize+0x10] + %define m10 m5 +%endif + psrld m6, m2, 16 + psrld m7, m3, 16 + mova m10, blendmask + BLENDHWDW m6, m0 ; 2 + BLENDHWDW m7, m1 ; 4 + mova [rsp+gprsize+0x40], m6 + mova [rsp+gprsize+0x50], m7 + call .h +%if ARCH_X86_32 + mova m4, [esp+gprsize+0x20] + mova m5, [esp+gprsize+0x30] +%endif + psrld m2, m4, 16 + psrld m3, m5, 16 + mova m10, blendmask + BLENDHWDW m2, m0 ; 3 + BLENDHWDW m3, m1 ; 5 + mova [rsp+gprsize+0x60], m2 + mova [rsp+gprsize+0x70], m3 + call .h +%if ARCH_X86_32 + mova m6, [esp+gprsize+0x40] + mova m7, [esp+gprsize+0x50] + %define m10 m7 +%endif + psrld m4, m6, 16 + psrld m5, m7, 16 + mova m10, blendmask + BLENDHWDW m4, m0 ; 4 + BLENDHWDW m5, m1 ; 6 +%if ARCH_X86_64 + add myd, 512+(64<<10) + mova m6, m2 + mova m7, m3 +%else + mova [esp+gprsize+0x80], m4 + mova [esp+gprsize+0x90], m5 + add dword mym, 512+(64<<10) +%endif + mov counterd, 4 + SAVE_ALPHA_BETA +.main2: + call .h +%if ARCH_X86_32 + mova m6, [esp+gprsize+0x60] + mova m7, [esp+gprsize+0x70] + %define m10 m5 +%endif + psrld m6, 16 + psrld m7, 16 + mova m10, blendmask + BLENDHWDW m6, m0 ; 5 + BLENDHWDW m7, m1 ; 7 +%if ARCH_X86_64 + WARP_V m12, m13, [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ + m4, m5, \ + [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ + m6, m7 +%else + mova [esp+gprsize+0xA0], m6 + mova [esp+gprsize+0xB0], m7 + LOAD_DELTA_GAMMA_MY + WARP_V [esp+gprsize+0xC0], [esp+gprsize+0xD0], \ + [esp+gprsize+0x00], [esp+gprsize+0x10], \ + [esp+gprsize+0x80], [esp+gprsize+0x90], \ + [esp+gprsize+0x20], [esp+gprsize+0x30], \ + [esp+gprsize+0xA0], [esp+gprsize+0xB0] + LOAD_ALPHA_BETA_MX +%endif + call .h + mova m2, [rsp+gprsize+0x40] + mova m3, [rsp+gprsize+0x50] +%if ARCH_X86_32 + mova m4, [rsp+gprsize+0x80] + mova m5, [rsp+gprsize+0x90] + %define m10 m7 +%endif + mova [rsp+gprsize+0x00], m2 + mova [rsp+gprsize+0x10], m3 + mova [rsp+gprsize+0x40], m4 + mova [rsp+gprsize+0x50], m5 + psrld m4, 16 + psrld m5, 16 + mova m10, blendmask + BLENDHWDW m4, m0 ; 6 + BLENDHWDW m5, m1 ; 8 +%if ARCH_X86_64 + WARP_V m14, m15, [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ + m6, m7, \ + [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ + m4, m5 +%else + mova [esp+gprsize+0x80], m4 + mova [esp+gprsize+0x90], m5 + LOAD_DELTA_GAMMA_MY + WARP_V [esp+gprsize+0xE0], [esp+gprsize+0xF0], \ + [esp+gprsize+0x20], [esp+gprsize+0x30], \ + [esp+gprsize+0xA0], [esp+gprsize+0xB0], \ + [esp+gprsize+0x00], [esp+gprsize+0x10], \ + [esp+gprsize+0x80], [esp+gprsize+0x90] + mov mym, myd + mov dstd, dstm + mov dsd, dsm + mov mxd, mxm +%endif + mova m2, [rsp+gprsize+0x60] + mova m3, [rsp+gprsize+0x70] +%if ARCH_X86_32 + mova m6, [esp+gprsize+0xA0] + mova m7, [esp+gprsize+0xB0] +%endif + mova [rsp+gprsize+0x20], m2 + mova [rsp+gprsize+0x30], m3 + mova [rsp+gprsize+0x60], m6 + mova [rsp+gprsize+0x70], m7 + ret +ALIGN function_align +.h: +%if ARCH_X86_32 + %define m8 m3 + %define m9 m4 + %define m10 m5 + %define m14 m6 + %define m15 m7 +%endif + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] +%if ARCH_X86_32 + %assign stack_offset stack_offset+4 + %assign stack_size stack_size+4 + %define PIC_mem [esp+gprsize*2+0x114] + mov PIC_mem, PIC_reg + mov srcd, srcm +%endif + movu m10, [srcq] +%if ARCH_X86_32 + add srcd, ssm + mov srcm, srcd + mov PIC_reg, PIC_mem +%else + add srcq, ssq +%endif + shr mxd, 10 + shr tmp1d, 10 + movq m1, [filterq+mxq *8] ; 0 X + movq m8, [filterq+tmp1q*8] ; 4 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+alphaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movhps m1, [filterq+tmp2q*8] ; 0 1 + movhps m8, [filterq+tmp1q*8] ; 4 5 + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + shr mxd, 10 + shr tmp1d, 10 +%if cpuflag(ssse3) + movq m14, [filterq+mxq *8] ; 2 X + movq m9, [filterq+tmp1q*8] ; 6 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + movhps m14, [filterq+tmp2q*8] ; 2 3 + movhps m9, [filterq+tmp1q*8] ; 6 7 + pshufb m0, m10, [PIC_sym(warp_8x8_shufA)] + pmaddubsw m0, m1 + pshufb m1, m10, [PIC_sym(warp_8x8_shufB)] + pmaddubsw m1, m8 + pshufb m15, m10, [PIC_sym(warp_8x8_shufC)] + pmaddubsw m15, m14 + pshufb m10, m10, [PIC_sym(warp_8x8_shufD)] + pmaddubsw m10, m9 + phaddw m0, m15 + phaddw m1, m10 +%else + %if ARCH_X86_32 + %define m11 m2 + %endif + pcmpeqw m0, m0 + psrlw m14, m0, 8 + psrlw m15, m10, 8 ; 01 03 05 07 09 11 13 15 + pand m14, m10 ; 00 02 04 06 08 10 12 14 + packuswb m14, m15 ; 00 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 + psrldq m9, m0, 4 + pshufd m0, m14, q0220 + pand m0, m9 + psrldq m14, 1 ; 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ + pslldq m15, m14, 12 + por m0, m15 ; shufA + psrlw m15, m0, 8 + psraw m11, m1, 8 + psllw m0, 8 + psllw m1, 8 + psrlw m0, 8 + psraw m1, 8 + pmullw m15, m11 + pmullw m0, m1 + paddw m0, m15 ; pmaddubsw m0, m1 + pshufd m15, m14, q0220 + pand m15, m9 + psrldq m14, 1 ; 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ + pslldq m1, m14, 12 + por m15, m1 ; shufC + pshufd m1, m14, q0220 + pand m1, m9 + psrldq m14, 1 ; 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ + pslldq m11, m14, 12 + por m1, m11 ; shufB + pshufd m10, m14, q0220 + pand m10, m9 + psrldq m14, 1 ; 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ __ + pslldq m14, m14, 12 + por m10, m14 ; shufD + psrlw m9, m1, 8 + psraw m11, m8, 8 + psllw m1, 8 + psllw m8, 8 + psrlw m1, 8 + psraw m8, 8 + pmullw m9, m11 + pmullw m1, m8 + paddw m1, m9 ; pmaddubsw m1, m8 + movq m14, [filterq+mxq *8] ; 2 X + movq m9, [filterq+tmp1q*8] ; 6 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + movhps m14, [filterq+tmp2q*8] ; 2 3 + movhps m9, [filterq+tmp1q*8] ; 6 7 + psrlw m8, m15, 8 + psraw m11, m14, 8 + psllw m15, 8 + psllw m14, 8 + psrlw m15, 8 + psraw m14, 8 + pmullw m8, m11 + pmullw m15, m14 + paddw m15, m8 ; pmaddubsw m15, m14 + psrlw m8, m10, 8 + psraw m11, m9, 8 + psllw m10, 8 + psllw m9, 8 + psrlw m10, 8 + psraw m9, 8 + pmullw m8, m11 + pmullw m10, m9 + paddw m10, m8 ; pmaddubsw m10, m9 + pslld m8, m0, 16 + pslld m9, m1, 16 + pslld m14, m15, 16 + pslld m11, m10, 16 + paddw m0, m8 + paddw m1, m9 + paddw m15, m14 + paddw m10, m11 + psrad m0, 16 + psrad m1, 16 + psrad m15, 16 + psrad m10, 16 + packssdw m0, m15 ; phaddw m0, m15 + packssdw m1, m10 ; phaddw m1, m10 +%endif + mova m14, [PIC_sym(pw_8192)] + mova m9, [PIC_sym(pd_32768)] + pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 + pmaddwd m1, m14 + paddd m0, m9 ; rounded 14-bit result in upper 16 bits of dword + paddd m1, m9 + ret +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%macro BIDIR_FN 1 ; op + %1 0 + lea stride3q, [strideq*3] + jmp wq +.w4_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*4] +.w4: ; tile 4x + movd [dstq ], m0 ; copy dw[0] + pshuflw m1, m0, q1032 ; swap dw[1] and dw[0] + movd [dstq+strideq*1], m1 ; copy dw[1] + punpckhqdq m0, m0 ; swap dw[3,2] with dw[1,0] + movd [dstq+strideq*2], m0 ; dw[2] + psrlq m0, 32 ; shift right in dw[3] + movd [dstq+stride3q ], m0 ; copy + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*2] +.w8: + movq [dstq ], m0 + movhps [dstq+strideq*1], m0 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq] +.w16: + mova [dstq ], m0 + dec hd + jg .w16_loop + RET +.w32_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq] +.w32: + mova [dstq ], m0 + %1 2 + mova [dstq + 16 ], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + %1_INC_PTR 8 + %1 0 + add dstq, strideq +.w64: + %assign i 0 + %rep 4 + mova [dstq + i*16 ], m0 + %assign i i+1 + %if i < 4 + %1 2*i + %endif + %endrep + dec hd + jg .w64_loop + RET +.w128_loop: + %1_INC_PTR 16 + %1 0 + add dstq, strideq +.w128: + %assign i 0 + %rep 8 + mova [dstq + i*16 ], m0 + %assign i i+1 + %if i < 8 + %1 2*i + %endif + %endrep + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + ; writes AVG of tmp1 tmp2 uint16 coeffs into uint8 pixel + mova m0, [tmp1q+(%1+0)*mmsize] ; load 8 coef(2bytes) from tmp1 + paddw m0, [tmp2q+(%1+0)*mmsize] ; load/add 8 coef(2bytes) tmp2 + mova m1, [tmp1q+(%1+1)*mmsize] + paddw m1, [tmp2q+(%1+1)*mmsize] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + packuswb m0, m1 ; pack/trunc 16 bits from m0 & m1 to 8 bit +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 + LEA r6, avg_ssse3_table + tzcnt wd, wm ; leading zeros + movifnidn hd, hm ; move h(stack) to h(register) if not already that register + movsxd wq, dword [r6+wq*4] ; push table entry matching the tile width (tzcnt) in widen reg + mova m2, [pw_1024+r6-avg_ssse3_table] ; fill m2 with shift/align + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m2, [tmp1q+(%1+0)*mmsize] + mova m0, m2 + psubw m2, [tmp2q+(%1+0)*mmsize] + mova m3, [tmp1q+(%1+1)*mmsize] + mova m1, m3 + psubw m3, [tmp2q+(%1+1)*mmsize] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 + LEA r6, w_avg_ssse3_table + tzcnt wd, wm + movd m4, r6m + movifnidn hd, hm + pxor m0, m0 + movsxd wq, dword [r6+wq*4] + mova m5, [pw_2048+r6-w_avg_ssse3_table] + pshufb m4, m0 + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + psubw m0, m4 + mov tmp1q, tmp2q + mova m4, m0 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 + mova m3, [maskq+(%1+0)*(mmsize/2)] + mova m0, [tmp2q+(%1+0)*mmsize] ; b + psubw m1, m0, [tmp1q+(%1+0)*mmsize] ; b - a + mova m6, m3 ; m + psubb m3, m4, m6 ; -m + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 ; -m << 1 + punpcklbw m2, m4, m3 ; -m << 9 (<< 8 when ext as uint16) + pmulhw m1, m2 ; (-m * (b - a)) << 10 + paddw m0, m1 ; + b + mova m1, [tmp2q+(%1+1)*mmsize] ; b + psubw m2, m1, [tmp1q+(%1+1)*mmsize] ; b - a + paddw m2, m2 ; (b - a) << 1 + mova m6, m3 ; (-m << 1) + punpckhbw m3, m4, m6 ; (-m << 9) + pmulhw m2, m3 ; (-m << 9) + paddw m1, m2 ; (-m * (b - a)) << 10 + pmulhrsw m0, m5 ; round + pmulhrsw m1, m5 ; round + packuswb m0, m1 ; interleave 16 -> 8 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*mmsize/2 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +%if ARCH_X86_64 +cglobal mask_8bpc, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3 + movifnidn hd, hm +%else +cglobal mask_8bpc, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3 +%define hd dword r5m +%endif +%define base r6-mask_ssse3_table + LEA r6, mask_ssse3_table + tzcnt wd, wm + movsxd wq, dword [r6+wq*4] + pxor m4, m4 + mova m5, [base+pw_2048] + add wq, r6 + mov maskq, r6m + BIDIR_FN MASK +%undef hd + +%macro W_MASK_420_END 1-* +%rep %0 + call .main + paddw m2, [maskq+16*%1] + mova [maskq+16*%1], m2 + mova [dstq+strideq*1+16*(2*%1+0)], m0 + call .main + psubw m3, m7, m2 + psubw m1, m7, [maskq+16*%1] + psubw m3, [dstq+strideq*1+16*(2*%1+1)] + psrlw m1, 2 + psrlw m3, 2 + packuswb m1, m3 + mova [maskq+16*%1], m1 + mova [dstq+strideq*1+16*(2*%1+1)], m0 + %rotate 1 +%endrep +%endmacro + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal w_mask_420_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_420_ssse3_table + LEA t0, w_mask_420_ssse3_table + tzcnt wd, wm + mov r6d, r7m ; sign + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_2048] + movddup m7, [base+wm_420_sign+r6*8] ; 258 - sign + add wq, t0 +%if ARCH_X86_64 + mova m8, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + movifnidn hd, hm +%else + %define m8 [base+pw_6903] + %define hd dword hm +%endif + mov maskq, maskmp + call .main + jmp wq +.w4_loop: + call .main + add maskq, 4 + lea dstq, [dstq+strideq*2] +.w4: + pshufd m3, m2, q2020 + pshufd m2, m2, q3131 + psubw m1, m7, m3 + psubw m1, m2 + psrlw m1, 2 + packuswb m1, m1 + movd [maskq], m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call .main + add maskq, 4 + lea dstq, [dstq+strideq*2] +.w8: + movhlps m3, m2 + psubw m1, m7, m2 + psubw m1, m3 + psrlw m1, 2 + packuswb m1, m1 + movd [maskq], m1 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + call .main + add maskq, 8 + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*1], m2 + mova [dstq+strideq*0], m0 + call .main + psubw m1, m7, [dstq+strideq*1] + psubw m1, m2 + psrlw m1, 2 + packuswb m1, m1 + movq [maskq], m1 + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w32: + mova [maskq], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + W_MASK_420_END 0 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add maskq, 16*2 + lea dstq, [dstq+strideq*2] +.w64: + mova [maskq+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + call .main + mova [maskq+16*1], m2 + mova [dstq+strideq*0+16*2], m0 + call .main + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*3], m0 + W_MASK_420_END 0, 1 + sub hd, 2 + jg .w64_loop + RET +.w128_loop: + call .main + add maskq, 16*4 + lea dstq, [dstq+strideq*2] +.w128: + mova [maskq+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + call .main + mova [maskq+16*1], m2 + mova [dstq+strideq*0+16*2], m0 + call .main + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*3], m0 + call .main + mova [maskq+16*2], m2 + mova [dstq+strideq*0+16*4], m0 + call .main + mova [dstq+strideq*1+16*5], m2 + mova [dstq+strideq*0+16*5], m0 + call .main + mova [maskq+16*3], m2 + mova [dstq+strideq*0+16*6], m0 + call .main + mova [dstq+strideq*1+16*7], m2 + mova [dstq+strideq*0+16*7], m0 + W_MASK_420_END 0, 1, 2, 3 + sub hd, 2 + jg .w128_loop + RET +ALIGN function_align +.main: + mova m0, [tmp1q +16*0] + mova m3, [tmp1q+tmp2q+16*0] + mova m1, [tmp1q +16*1] + mova m4, [tmp1q+tmp2q+16*1] + add tmp1q, 16*2 + psubw m3, m0 + psubw m4, m1 + pabsw m5, m3 + psubusw m2, m8, m5 + psrlw m2, 8 ; 64 - m + psllw m5, m2, 10 + pmulhw m3, m5 + pabsw m5, m4 + paddw m0, m3 + psubusw m3, m8, m5 + psrlw m3, 8 + phaddw m2, m3 + psllw m3, 10 + pmulhw m4, m3 + paddw m1, m4 + pmulhrsw m0, m6 + pmulhrsw m1, m6 + packuswb m0, m1 + ret + +%macro W_MASK_422_BACKUP 1 ; mask_offset +%if ARCH_X86_64 + mova m10, m2 +%else + mova [maskq+16*%1], m2 +%endif +%endmacro + +%macro W_MASK_422_END 1 ; mask_offset +%if ARCH_X86_64 + packuswb m10, m2 + psubb m1, m7, m10 + pavgb m1, m9 +%else + mova m3, [maskq+16*%1] + packuswb m3, m2 + pxor m2, m2 + psubb m1, m7, m3 + pavgb m1, m2 +%endif + mova [maskq+16*%1], m1 +%endmacro + +cglobal w_mask_422_8bpc, 4, 7, 11, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_422_ssse3_table + LEA t0, w_mask_422_ssse3_table + tzcnt wd, wm + mov r6d, r7m ; sign + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_2048] + movddup m7, [base+wm_422_sign+r6*8] ; 128 - sign + add wq, t0 +%if ARCH_X86_64 + mova m8, [base+pw_6903] + pxor m9, m9 + movifnidn hd, hm +%else + add t0, w_mask_420_ssse3_table-w_mask_422_ssse3_table + %define hd dword hm +%endif + mov maskq, maskmp + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + jmp wq +.w4_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 8 + lea dstq, [dstq+strideq*2] +.w4: + packuswb m2, m2 + psubb m1, m7, m2 +%if ARCH_X86_64 + pavgb m1, m9 +%else + pxor m2, m2 + pavgb m1, m2 +%endif + movq [maskq], m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w8: + W_MASK_422_BACKUP 0 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + lea dstq, [dstq+strideq*2] + W_MASK_422_END 0 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w16: + W_MASK_422_BACKUP 0 + mova [dstq+strideq*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + add dstq, strideq +.w32: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16*2 + add dstq, strideq +.w64: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 1 + mova [dstq+16*2], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 1 + mova [dstq+16*3], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16*4 + add dstq, strideq +.w128: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 1 + mova [dstq+16*2], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 1 + mova [dstq+16*3], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 2 + mova [dstq+16*4], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 2 + mova [dstq+16*5], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 3 + mova [dstq+16*6], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 3 + mova [dstq+16*7], m0 + dec hd + jg .w128_loop + RET + +cglobal w_mask_444_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_444_ssse3_table + LEA t0, w_mask_444_ssse3_table + tzcnt wd, wm + mov maskq, maskmp + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_6903] + mova m7, [base+pw_2048] + add wq, t0 +%if ARCH_X86_64 + mova m8, [base+pb_64] + movifnidn hd, hm +%else + %define m8 [base+pb_64] + %define hd dword hm +%endif + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] +.w4: + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] +.w8: + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*0], m0 + call .main + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + call .main + mova [dstq+16*2], m0 + call .main + mova [dstq+16*3], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + call .main + mova [dstq+16*2], m0 + call .main + mova [dstq+16*3], m0 + call .main + mova [dstq+16*4], m0 + call .main + mova [dstq+16*5], m0 + call .main + mova [dstq+16*6], m0 + call .main + mova [dstq+16*7], m0 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + mova m0, [tmp1q +16*0] + mova m3, [tmp1q+tmp2q+16*0] + mova m1, [tmp1q +16*1] + mova m4, [tmp1q+tmp2q+16*1] + add tmp1q, 16*2 + psubw m3, m0 + psubw m4, m1 + pabsw m5, m3 + psubusw m2, m6, m5 + psrlw m2, 8 ; 64 - m + psllw m5, m2, 10 + pmulhw m3, m5 + pabsw m5, m4 + paddw m0, m3 + psubusw m3, m6, m5 + psrlw m3, 8 + packuswb m2, m3 + psllw m3, 10 + pmulhw m4, m3 + psubb m3, m8, m2 + paddw m1, m4 + pmulhrsw m0, m7 + pmulhrsw m1, m7 + mova [maskq], m3 + add maskq, 16 + packuswb m0, m1 + ret + +%macro BLEND_64M 4; a, b, mask1, mask2 + punpcklbw m0, %1, %2; {b;a}[7..0] + punpckhbw %1, %2 ; {b;a}[15..8] + pmaddubsw m0, %3 ; {b*m[0] + (64-m[0])*a}[7..0] u16 + pmaddubsw %1, %4 ; {b*m[1] + (64-m[1])*a}[15..8] u16 + pmulhrsw m0, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 + pmulhrsw %1, m5 ; {((b*m[1] + (64-m[0])*a) + 1) / 32}[15..8] u16 + packuswb m0, %1 ; {blendpx}[15..0] u8 +%endmacro + +%macro BLEND 2; a, b + psubb m3, m4, m0 ; m3 = (64 - m) + punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] + punpckhbw m3, m0 ; {m;(64-m)}[15..8] + BLEND_64M %1, %2, m2, m3 +%endmacro + +cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_ssse3_table + LEA r6, blend_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movifnidn maskq, maskmp + movsxd wq, dword [r6+wq*4] + mova m4, [base+pb_64] + mova m5, [base+pw_512] + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + movq m0, [maskq]; m + movd m1, [dstq+dsq*0] ; a + movd m6, [dstq+dsq*1] + punpckldq m1, m6 + movq m6, [tmpq] ; b + psubb m3, m4, m0 ; m3 = (64 - m) + punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] + punpcklbw m1, m6 ; {b;a}[7..0] + pmaddubsw m1, m2 ; {b*m[0] + (64-m[0])*a}[7..0] u16 + pmulhrsw m1, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 + packuswb m1, m0 ; {blendpx}[15..0] u8 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + add maskq, 8 + add tmpq, 8 + lea dstq, [dstq+dsq*2] ; dst_stride * 2 + sub hd, 2 + jg .w4 + RET +.w8: + mova m0, [maskq]; m + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m6, [tmpq] ; b + BLEND m1, m6 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + add maskq, 16 + add tmpq, 16 + lea dstq, [dstq+dsq*2] ; dst_stride * 2 + sub hd, 2 + jg .w8 + RET +.w16: + mova m0, [maskq]; m + mova m1, [dstq] ; a + mova m6, [tmpq] ; b + BLEND m1, m6 + mova [dstq], m0 + add maskq, 16 + add tmpq, 16 + add dstq, dsq ; dst_stride + dec hd + jg .w16 + RET +.w32: + %assign i 0 + %rep 2 + mova m0, [maskq+16*i]; m + mova m1, [dstq+16*i] ; a + mova m6, [tmpq+16*i] ; b + BLEND m1, m6 + mova [dstq+i*16], m0 + %assign i i+1 + %endrep + add maskq, 32 + add tmpq, 32 + add dstq, dsq ; dst_stride + dec hd + jg .w32 + RET + +cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_v_ssse3_table + LEA r5, blend_v_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r5+wq*4] + mova m5, [base+pw_512] + add wq, r5 + add maskq, obmc_masks-blend_v_ssse3_table + jmp wq +.w2: + movd m3, [maskq+4] + punpckldq m3, m3 + ; 2 mask blend is provided for 4 pixels / 2 lines +.w2_loop: + movd m1, [dstq+dsq*0] ; a {..;a;a} + pinsrw m1, [dstq+dsq*1], 1 + movd m2, [tmpq] ; b + punpcklbw m0, m1, m2; {b;a}[7..0] + pmaddubsw m0, m3 ; {b*m + (64-m)*a}[7..0] u16 + pmulhrsw m0, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 + packuswb m0, m1 ; {blendpx}[8..0] u8 + movd r3d, m0 + mov [dstq+dsq*0], r3w + shr r3d, 16 + mov [dstq+dsq*1], r3w + add tmpq, 2*2 + lea dstq, [dstq + dsq * 2] + sub hd, 2 + jg .w2_loop + RET +.w4: + movddup m3, [maskq+8] + ; 4 mask blend is provided for 8 pixels / 2 lines +.w4_loop: + movd m1, [dstq+dsq*0] ; a + movd m2, [dstq+dsq*1] ; + punpckldq m1, m2 + movq m2, [tmpq] ; b + punpcklbw m1, m2 ; {b;a}[7..0] + pmaddubsw m1, m3 ; {b*m + (64-m)*a}[7..0] u16 + pmulhrsw m1, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 + packuswb m1, m1 ; {blendpx}[8..0] u8 + movd [dstq], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + add tmpq, 2*4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +.w8: + mova m3, [maskq+16] + ; 8 mask blend is provided for 16 pixels +.w8_loop: + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m2, [tmpq]; b + BLEND_64M m1, m2, m3, m3 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + add tmpq, 16 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + ; 16 mask blend is provided for 32 pixels + mova m3, [maskq+32] ; obmc_masks_16[0] (64-m[0]) + mova m4, [maskq+48] ; obmc_masks_16[1] (64-m[1]) +.w16_loop: + mova m1, [dstq] ; a + mova m2, [tmpq] ; b + BLEND_64M m1, m2, m3, m4 + mova [dstq], m0 + add tmpq, 16 + add dstq, dsq + dec hd + jg .w16_loop + RET +.w32: +%if WIN64 + mova [rsp+8], xmm6 +%endif + mova m3, [maskq+64] ; obmc_masks_32[0] (64-m[0]) + mova m4, [maskq+80] ; obmc_masks_32[1] (64-m[1]) + mova m6, [maskq+96] ; obmc_masks_32[2] (64-m[2]) + ; 16 mask blend is provided for 64 pixels +.w32_loop: + mova m1, [dstq+16*0] ; a + mova m2, [tmpq+16*0] ; b + BLEND_64M m1, m2, m3, m4 + movq m1, [dstq+16*1] ; a + punpcklbw m1, [tmpq+16*1] ; b + pmaddubsw m1, m6 + pmulhrsw m1, m5 + packuswb m1, m1 + mova [dstq+16*0], m0 + movq [dstq+16*1], m1 + add tmpq, 32 + add dstq, dsq + dec hd + jg .w32_loop +%if WIN64 + mova xmm6, [rsp+8] +%endif + RET + +cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask +%define base t0-blend_h_ssse3_table +%if ARCH_X86_32 + ; We need to keep the PIC pointer for w4, reload wd from stack instead + DECLARE_REG_TMP 6 +%else + DECLARE_REG_TMP 5 + mov r6d, wd +%endif + LEA t0, blend_h_ssse3_table + tzcnt wd, wm + mov hd, hm + movsxd wq, dword [t0+wq*4] + mova m5, [base+pw_512] + add wq, t0 + lea maskq, [base+obmc_masks+hq*2] + lea hd, [hq*3] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd m0, [dstq+dsq*0] + pinsrw m0, [dstq+dsq*1], 1 + movd m2, [maskq+hq*2] + movd m1, [tmpq] + punpcklwd m2, m2 + punpcklbw m0, m1 + pmaddubsw m0, m2 + pmulhrsw m0, m5 + packuswb m0, m0 + movd r3d, m0 + mov [dstq+dsq*0], r3w + shr r3d, 16 + mov [dstq+dsq*1], r3w + lea dstq, [dstq+dsq*2] + add tmpq, 2*2 + add hq, 2 + jl .w2 + RET +.w4: +%if ARCH_X86_32 + mova m3, [base+blend_shuf] +%else + mova m3, [blend_shuf] +%endif +.w4_loop: + movd m0, [dstq+dsq*0] + movd m2, [dstq+dsq*1] + punpckldq m0, m2 ; a + movq m1, [tmpq] ; b + movq m2, [maskq+hq*2] ; m + pshufb m2, m3 + punpcklbw m0, m1 + pmaddubsw m0, m2 + pmulhrsw m0, m5 + packuswb m0, m0 + movd [dstq+dsq*0], m0 + psrlq m0, 32 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add tmpq, 4*2 + add hq, 2 + jl .w4_loop + RET +.w8: + movd m4, [maskq+hq*2] + punpcklwd m4, m4 + pshufd m3, m4, q0000 + pshufd m4, m4, q1111 + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m2, [tmpq] + BLEND_64M m1, m2, m3, m4 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add tmpq, 8*2 + add hq, 2 + jl .w8 + RET +; w16/w32/w64/w128 +.w16: +%if ARCH_X86_32 + mov r6d, wm +%endif + sub dsq, r6 +.w16_loop0: + movd m3, [maskq+hq*2] + pshuflw m3, m3, q0000 + punpcklqdq m3, m3 + mov wd, r6d +.w16_loop: + mova m1, [dstq] ; a + mova m2, [tmpq] ; b + BLEND_64M m1, m2, m3, m3 + mova [dstq], m0 + add dstq, 16 + add tmpq, 16 + sub wd, 16 + jg .w16_loop + add dstq, dsq + inc hq + jl .w16_loop0 + RET + +; emu_edge args: +; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih, +; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride, +; const pixel *ref, const ptrdiff_t ref_stride +; +; bw, bh total filled size +; iw, ih, copied block -> fill bottom, right +; x, y, offset in bw/bh -> fill top, left +cglobal emu_edge_8bpc, 10, 13, 2, bw, bh, iw, ih, x, \ + y, dst, dstride, src, sstride, \ + bottomext, rightext, blk + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + pxor m1, m1 + +%if ARCH_X86_64 + %define reg_zero r12q + %define reg_tmp r10 + %define reg_src srcq + %define reg_bottomext bottomextq + %define reg_rightext rightextq + %define reg_blkm r9m +%else + %define reg_zero r6 + %define reg_tmp r0 + %define reg_src r1 + %define reg_bottomext r0 + %define reg_rightext r1 + %define reg_blkm r2m +%endif + ; + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor reg_zero, reg_zero + lea reg_tmp, [ihq-1] + cmp yq, ihq + cmovs reg_tmp, yq + test yq, yq + cmovs reg_tmp, reg_zero +%if ARCH_X86_64 + imul reg_tmp, sstrideq + add srcq, reg_tmp +%else + imul reg_tmp, sstridem + mov reg_src, srcm + add reg_src, reg_tmp +%endif + ; + ; ref += iclip(x, 0, iw - 1) + lea reg_tmp, [iwq-1] + cmp xq, iwq + cmovs reg_tmp, xq + test xq, xq + cmovs reg_tmp, reg_zero + add reg_src, reg_tmp +%if ARCH_X86_32 + mov srcm, reg_src +%endif + ; + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) +%if ARCH_X86_32 + mov r1, r1m ; restore bh +%endif + lea reg_bottomext, [yq+bhq] + sub reg_bottomext, ihq + lea r3, [bhq-1] + cmovs reg_bottomext, reg_zero + ; + + DEFINE_ARGS bw, bh, iw, ih, x, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, reg_zero + cmp reg_bottomext, bhq + cmovns reg_bottomext, r3 + cmp topextq, bhq + cmovg topextq, r3 + %if ARCH_X86_32 + mov r4m, reg_bottomext + ; + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + mov r0, r0m ; restore bw + %endif + lea reg_rightext, [xq+bwq] + sub reg_rightext, iwq + lea r2, [bwq-1] + cmovs reg_rightext, reg_zero + + DEFINE_ARGS bw, bh, iw, ih, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, reg_zero + cmp reg_rightext, bwq + cmovns reg_rightext, r2 + %if ARCH_X86_32 + mov r3m, r1 + %endif + cmp leftextq, bwq + cmovns leftextq, r2 + +%undef reg_zero +%undef reg_tmp +%undef reg_src +%undef reg_bottomext +%undef reg_rightext + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; center_h = bh - top_ext - bottom_ext +%if ARCH_X86_64 + lea r3, [bottomextq+topextq] + sub centerhq, r3 +%else + mov r1, centerhm ; restore r1 + sub centerhq, topextq + sub centerhq, r4m + mov r1m, centerhq +%endif + ; + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq +%if ARCH_X86_64 + imul r2, dstrideq +%else + mov r6, r6m ; restore dstq + imul r2, dstridem +%endif + add dstq, r2 + mov reg_blkm, dstq ; save pointer for ext + ; + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq +%if ARCH_X86_64 + lea r3, [rightextq+leftextq] + sub centerwq, r3 +%else + sub centerwq, r3m + sub centerwq, leftextq +%endif + +; vloop Macro +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix + %if ARCH_X86_64 + %define reg_tmp r12 + %else + %define reg_tmp r0 + %endif +.v_loop_%3: + %if ARCH_X86_32 + mov r0, r0m + mov r1, r1m + %endif +%if %1 + ; left extension + %if ARCH_X86_64 + movd m0, [srcq] + %else + mov r3, srcm + movd m0, [r3] + %endif + pshufb m0, m1 + xor r3, r3 +.left_loop_%3: + mova [dstq+r3], m0 + add r3, mmsize + cmp r3, leftextq + jl .left_loop_%3 + ; body + lea reg_tmp, [dstq+leftextq] +%endif + xor r3, r3 +.body_loop_%3: + %if ARCH_X86_64 + movu m0, [srcq+r3] + %else + mov r1, srcm + movu m0, [r1+r3] + %endif +%if %1 + movu [reg_tmp+r3], m0 +%else + movu [dstq+r3], m0 +%endif + add r3, mmsize + cmp r3, centerwq + jl .body_loop_%3 +%if %2 + ; right extension +%if %1 + add reg_tmp, centerwq +%else + lea reg_tmp, [dstq+centerwq] +%endif + %if ARCH_X86_64 + movd m0, [srcq+centerwq-1] + %else + mov r3, srcm + movd m0, [r3+centerwq-1] + %endif + pshufb m0, m1 + xor r3, r3 +.right_loop_%3: + movu [reg_tmp+r3], m0 + add r3, mmsize + %if ARCH_X86_64 + cmp r3, rightextq + %else + cmp r3, r3m + %endif + jl .right_loop_%3 +%endif + %if ARCH_X86_64 + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 + %else + add dstq, dstridem + mov r0, sstridem + add srcm, r0 + sub dword centerhm, 1 + jg .v_loop_%3 + mov r0, r0m ; restore r0 + %endif +%endmacro ; vloop MACRO + + test leftextq, leftextq + jnz .need_left_ext + %if ARCH_X86_64 + test rightextq, rightextq + jnz .need_right_ext + %else + cmp leftextq, r3m ; leftextq == 0 + jne .need_right_ext + %endif + v_loop 0, 0, 0 + jmp .body_done + + ;left right extensions +.need_left_ext: + %if ARCH_X86_64 + test rightextq, rightextq + %else + mov r3, r3m + test r3, r3 + %endif + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: +; r0 ; bw +; r1 ;; x loop +; r4 ;; y loop +; r5 ; topextq +; r6 ;dstq +; r7 ;dstrideq +; r8 ; srcq +%if ARCH_X86_64 + %define reg_dstride dstrideq +%else + %define reg_dstride r2 +%endif + ; + ; bottom edge extension + %if ARCH_X86_64 + test bottomextq, bottomextq + jz .top + %else + xor r1, r1 + cmp r1, r4m + je .top + %endif + ; + %if ARCH_X86_64 + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 + %else + mov r3, dstq + mov reg_dstride, dstridem + sub r3, reg_dstride + mov srcm, r3 + %endif + ; +.bottom_x_loop: + %if ARCH_X86_64 + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, bottomextq + %else + mov r3, srcm + mova m0, [r3+r1] + lea r3, [dstq+r1] + mov r4, r4m + %endif + ; +.bottom_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .bottom_y_loop + add r1, mmsize + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end +%if ARCH_X86_64 + mov srcq, reg_blkm +%else + mov r3, reg_blkm + mov reg_dstride, dstridem +%endif + mov dstq, dstm + xor r1, r1 + ; +.top_x_loop: +%if ARCH_X86_64 + mova m0, [srcq+r1] +%else + mov r3, reg_blkm + mova m0, [r3+r1] +%endif + lea r3, [dstq+r1] + mov r4, topextq + ; +.top_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .top_y_loop + add r1, mmsize + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +%undef reg_dstride +%undef reg_blkm +%undef reg_tmp + +cextern resize_filter + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +%if ARCH_X86_64 +cglobal resize_8bpc, 0, 12, 14, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%elif STACK_ALIGNMENT >= 16 +cglobal resize_8bpc, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%else +cglobal resize_8bpc, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%endif + movifnidn dstq, dstmp + movifnidn srcq, srcmp +%if STACK_ALIGNMENT >= 16 + movifnidn dst_wd, dst_wm +%endif +%if ARCH_X86_64 + movifnidn hd, hm +%endif + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + movd m7, dxm + movd m6, mx0m + movd m5, src_wm + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + pshufd m5, m5, q0000 + +%if ARCH_X86_64 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x + LEA r7, $$ +%define base r7-$$ +%else + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x +%define hd dword r5m +%if STACK_ALIGNMENT >= 16 + LEA r6, $$ +%define base r6-$$ +%else + LEA r4, $$ +%define base r4-$$ +%endif +%endif + +%if ARCH_X86_64 + mova m10, [base+pw_m256] + mova m9, [base+pd_63] + mova m8, [base+pb_8x0_8x8] +%else +%define m10 [base+pw_m256] +%define m9 [base+pd_63] +%define m8 [base+pb_8x0_8x8] +%endif + pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3] + pslld m7, 2 ; dx*4 + pslld m5, 14 + paddd m6, m4 ; mx+[0..3]*dx + SCRATCH 7, 13, 0 + SCRATCH 6, 12, 1 + SCRATCH 5, 11, 2 + + ; m10 = pmulhrsw constant for x=(x+64)>>7 + ; m12 = mx+[0..3]*dx, m13 = dx*4, m11 = src_w, m9 = 0x3f, m8=0,8 + +.loop_y: + xor xd, xd + mova m0, m12 ; per-line working version of mx + +.loop_x: + pxor m1, m1 + pcmpgtd m1, m0 + pandn m1, m0 + psrad m2, m0, 8 ; filter offset (unmasked) + pcmpgtd m3, m11, m1 + pand m1, m3 + pandn m3, m11 + por m1, m3 + psubd m3, m0, m1 ; pshufb offset + psrad m1, 14 ; clipped src_x offset + psrad m3, 14 ; pshufb edge_emu offset + pand m2, m9 ; filter offset (masked) + + ; load source pixels +%if ARCH_X86_64 + movd r8d, m1 + pshuflw m1, m1, q3232 + movd r9d, m1 + punpckhqdq m1, m1 + movd r10d, m1 + psrlq m1, 32 + movd r11d, m1 + movq m4, [srcq+r8] + movq m5, [srcq+r10] + movhps m4, [srcq+r9] + movhps m5, [srcq+r11] +%else + movd r3d, m1 + pshufd m1, m1, q3312 + movd r1d, m1 + pshuflw m1, m1, q3232 + movq m4, [srcq+r3] + movq m5, [srcq+r1] + movd r3d, m1 + punpckhqdq m1, m1 + movd r1d, m1 + movhps m4, [srcq+r3] + movhps m5, [srcq+r1] +%endif + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + pxor m6, m6 + pcmpeqb m6, m3 +%if ARCH_X86_64 + pmovmskb r8d, m6 + cmp r8d, 0xffff +%else + pmovmskb r3d, m6 + cmp r3d, 0xffff +%endif + je .filter + +%if ARCH_X86_64 + movd r8d, m3 + pshuflw m3, m3, q3232 + movd r9d, m3 + punpckhqdq m3, m3 + movd r10d, m3 + psrlq m3, 32 + movd r11d, m3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + movq m6, [base+resize_shuf+4+r8] + movq m7, [base+resize_shuf+4+r10] + movhps m6, [base+resize_shuf+4+r9] + movhps m7, [base+resize_shuf+4+r11] +%else + movd r3d, m3 + pshufd m3, m3, q3312 + movd r1d, m3 + pshuflw m3, m3, q3232 + movq m6, [base+resize_shuf+4+r3] + movq m7, [base+resize_shuf+4+r1] + movd r3d, m3 + punpckhqdq m3, m3 + movd r1d, m3 + movhps m6, [base+resize_shuf+4+r3] + movhps m7, [base+resize_shuf+4+r1] +%endif + + paddb m6, m8 + paddb m7, m8 + pshufb m4, m6 + pshufb m5, m7 + +.filter: +%if ARCH_X86_64 + movd r8d, m2 + pshuflw m2, m2, q3232 + movd r9d, m2 + punpckhqdq m2, m2 + movd r10d, m2 + psrlq m2, 32 + movd r11d, m2 + movq m6, [base+resize_filter+r8*8] + movq m7, [base+resize_filter+r10*8] + movhps m6, [base+resize_filter+r9*8] + movhps m7, [base+resize_filter+r11*8] +%else + movd r3d, m2 + pshufd m2, m2, q3312 + movd r1d, m2 + pshuflw m2, m2, q3232 + movq m6, [base+resize_filter+r3*8] + movq m7, [base+resize_filter+r1*8] + movd r3d, m2 + punpckhqdq m2, m2 + movd r1d, m2 + movhps m6, [base+resize_filter+r3*8] + movhps m7, [base+resize_filter+r1*8] +%endif + + pmaddubsw m4, m6 + pmaddubsw m5, m7 + phaddw m4, m5 + phaddsw m4, m4 + pmulhrsw m4, m10 ; x=(x+64)>>7 + packuswb m4, m4 + movd [dstq+xq], m4 + + paddd m0, m13 + add xd, 4 +%if STACK_ALIGNMENT >= 16 + cmp xd, dst_wd +%else + cmp xd, dst_wm +%endif + jl .loop_x + + add dstq, dst_stridemp + add srcq, src_stridemp + dec hd + jg .loop_y + RET + +INIT_XMM ssse3 +PREP_BILIN +PREP_8TAP +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T + +INIT_XMM sse4 +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T + +INIT_XMM sse2 +PREP_BILIN +PREP_8TAP +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T diff --git a/third_party/dav1d/src/x86/msac.asm b/third_party/dav1d/src/x86/msac.asm new file mode 100644 index 0000000000..9f05c921a6 --- /dev/null +++ b/third_party/dav1d/src/x86/msac.asm @@ -0,0 +1,667 @@ +; Copyright © 2019, VideoLAN and dav1d authors +; Copyright © 2019, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 64 ; avoids cacheline splits + +min_prob: dw 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0 +pw_0xff00: times 8 dw 0xff00 +pw_32: times 8 dw 32 + +%if ARCH_X86_64 +%define resp resq +%define movp movq +%define c_shuf q3333 +%macro DECODE_SYMBOL_ADAPT_INIT 0-1 +%endmacro +%else +%define resp resd +%define movp movd +%define c_shuf q1111 +%macro DECODE_SYMBOL_ADAPT_INIT 0-1 0 ; hi_tok + mov t0, r0m + mov t1, r1m +%if %1 == 0 + mov t2, r2m +%endif +%if STACK_ALIGNMENT >= 16 + sub esp, 40-%1*4 +%else + mov eax, esp + and esp, ~15 + sub esp, 40-%1*4 + mov [esp], eax +%endif +%endmacro +%endif + +struc msac + .buf: resp 1 + .end: resp 1 + .dif: resp 1 + .rng: resd 1 + .cnt: resd 1 + .update_cdf: resd 1 +endstruc + +%define m(x, y) mangle(private_prefix %+ _ %+ x %+ y) + +SECTION .text + +%if WIN64 +DECLARE_REG_TMP 0, 1, 2, 3, 4, 5, 7, 3, 8 +%define buf rsp+stack_offset+8 ; shadow space +%elif UNIX64 +DECLARE_REG_TMP 0, 1, 2, 3, 4, 5, 7, 0, 8 +%define buf rsp-40 ; red zone +%else +DECLARE_REG_TMP 2, 3, 4, 1, 5, 6, 5, 2, 3 +%define buf esp+8 +%endif + +INIT_XMM sse2 +cglobal msac_decode_symbol_adapt4, 0, 6, 6 + DECODE_SYMBOL_ADAPT_INIT + LEA rax, pw_0xff00 + movd m2, [t0+msac.rng] + movq m1, [t1] + movp m3, [t0+msac.dif] + mov t3d, [t0+msac.update_cdf] + mov t4d, t2d + not t2 ; -(n_symbols + 1) + pshuflw m2, m2, q0000 + movd [buf+12], m2 + pand m2, [rax] + mova m0, m1 + psrlw m1, 6 + psllw m1, 7 + pmulhuw m1, m2 + movq m2, [rax+t2*2] + pshuflw m3, m3, c_shuf + paddw m1, m2 + mova [buf+16], m1 + psubusw m1, m3 + pxor m2, m2 + pcmpeqw m1, m2 ; c >= v + pmovmskb eax, m1 + test t3d, t3d + jz .renorm ; !allow_update_cdf + +; update_cdf: + movzx t3d, word [t1+t4*2] ; count + pcmpeqw m2, m2 + mov t2d, t3d + shr t3d, 4 + cmp t4d, 3 + sbb t3d, -5 ; (count >> 4) + (n_symbols > 2) + 4 + cmp t2d, 32 + adc t2d, 0 ; count + (count < 32) + movd m3, t3d + pavgw m2, m1 ; i >= val ? -1 : 32768 + psubw m2, m0 ; for (i = 0; i < val; i++) + psubw m0, m1 ; cdf[i] += (32768 - cdf[i]) >> rate; + psraw m2, m3 ; for (; i < n_symbols; i++) + paddw m0, m2 ; cdf[i] += (( -1 - cdf[i]) >> rate) + 1; + movq [t1], m0 + mov [t1+t4*2], t2w + +.renorm: + tzcnt eax, eax + mov t4, [t0+msac.dif] + movzx t1d, word [buf+rax+16] ; v + movzx t2d, word [buf+rax+14] ; u + shr eax, 1 +.renorm2: +%if ARCH_X86_64 == 0 +%if STACK_ALIGNMENT >= 16 + add esp, 40 +%else + mov esp, [esp] +%endif +%endif + not t4 + sub t2d, t1d ; rng + shl t1, gprsize*8-16 + add t4, t1 ; ~dif +.renorm3: + mov t1d, [t0+msac.cnt] + movifnidn t7, t0 +.renorm4: + bsr ecx, t2d + xor ecx, 15 ; d +.renorm5: + shl t2d, cl + shl t4, cl + mov [t7+msac.rng], t2d + not t4 + sub t1d, ecx + jae .end ; no refill required + +; refill: + mov t2, [t7+msac.buf] + mov rcx, [t7+msac.end] +%if ARCH_X86_64 == 0 + push t5 +%endif + lea t5, [t2+gprsize] + cmp t5, rcx + ja .refill_eob + mov t2, [t2] + lea ecx, [t1+23] + add t1d, 16 + shr ecx, 3 ; shift_bytes + bswap t2 + sub t5, rcx + shl ecx, 3 ; shift_bits + shr t2, cl + sub ecx, t1d ; shift_bits - 16 - cnt + mov t1d, gprsize*8-16 + shl t2, cl + mov [t7+msac.buf], t5 + sub t1d, ecx ; cnt + gprsize*8 - shift_bits + xor t4, t2 +%if ARCH_X86_64 == 0 + pop t5 +%endif +.end: + mov [t7+msac.cnt], t1d + mov [t7+msac.dif], t4 + RET +.refill_eob: ; avoid overreading the input buffer + mov t5, rcx + mov ecx, gprsize*8-24 + sub ecx, t1d ; c +.refill_eob_loop: + cmp t2, t5 + jae .refill_eob_end ; eob reached + movzx t1d, byte [t2] + inc t2 + shl t1, cl + xor t4, t1 + sub ecx, 8 + jge .refill_eob_loop +.refill_eob_end: + mov t1d, gprsize*8-24 +%if ARCH_X86_64 == 0 + pop t5 +%endif + sub t1d, ecx + mov [t7+msac.buf], t2 + mov [t7+msac.dif], t4 + mov [t7+msac.cnt], t1d + RET + +cglobal msac_decode_symbol_adapt8, 0, 6, 6 + DECODE_SYMBOL_ADAPT_INIT + LEA rax, pw_0xff00 + movd m2, [t0+msac.rng] + mova m1, [t1] + movp m3, [t0+msac.dif] + mov t3d, [t0+msac.update_cdf] + mov t4d, t2d + not t2 + pshuflw m2, m2, q0000 + movd [buf+12], m2 + punpcklqdq m2, m2 + mova m0, m1 + psrlw m1, 6 + pand m2, [rax] + psllw m1, 7 + pmulhuw m1, m2 + movu m2, [rax+t2*2] + pshuflw m3, m3, c_shuf + paddw m1, m2 + punpcklqdq m3, m3 + mova [buf+16], m1 + psubusw m1, m3 + pxor m2, m2 + pcmpeqw m1, m2 + pmovmskb eax, m1 + test t3d, t3d + jz m(msac_decode_symbol_adapt4, SUFFIX).renorm + movzx t3d, word [t1+t4*2] + pcmpeqw m2, m2 + mov t2d, t3d + shr t3d, 4 + cmp t4d, 3 ; may be called with n_symbols <= 2 + sbb t3d, -5 + cmp t2d, 32 + adc t2d, 0 + movd m3, t3d + pavgw m2, m1 + psubw m2, m0 + psubw m0, m1 + psraw m2, m3 + paddw m0, m2 + mova [t1], m0 + mov [t1+t4*2], t2w + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm + +cglobal msac_decode_symbol_adapt16, 0, 6, 6 + DECODE_SYMBOL_ADAPT_INIT + LEA rax, pw_0xff00 + movd m4, [t0+msac.rng] + mova m2, [t1] + mova m3, [t1+16] + movp m5, [t0+msac.dif] + mov t3d, [t0+msac.update_cdf] + mov t4d, t2d + not t2 +%if WIN64 + sub rsp, 48 ; need 36 bytes, shadow space is only 32 +%endif + pshuflw m4, m4, q0000 + movd [buf-4], m4 + punpcklqdq m4, m4 + mova m0, m2 + psrlw m2, 6 + mova m1, m3 + psrlw m3, 6 + pand m4, [rax] + psllw m2, 7 + psllw m3, 7 + pmulhuw m2, m4 + pmulhuw m3, m4 + movu m4, [rax+t2*2] + pshuflw m5, m5, c_shuf + paddw m2, m4 + psubw m4, [rax-pw_0xff00+pw_32] + punpcklqdq m5, m5 + paddw m3, m4 + mova [buf], m2 + psubusw m2, m5 + mova [buf+16], m3 + psubusw m3, m5 + pxor m4, m4 + pcmpeqw m2, m4 + pcmpeqw m3, m4 + packsswb m5, m2, m3 + pmovmskb eax, m5 + test t3d, t3d + jz .renorm + movzx t3d, word [t1+t4*2] + pcmpeqw m4, m4 + mova m5, m4 + lea t2d, [t3+80] ; only support n_symbols > 2 + shr t2d, 4 + cmp t3d, 32 + adc t3d, 0 + pavgw m4, m2 + pavgw m5, m3 + psubw m4, m0 + psubw m0, m2 + movd m2, t2d + psubw m5, m1 + psubw m1, m3 + psraw m4, m2 + psraw m5, m2 + paddw m0, m4 + paddw m1, m5 + mova [t1], m0 + mova [t1+16], m1 + mov [t1+t4*2], t3w +.renorm: + tzcnt eax, eax + mov t4, [t0+msac.dif] + movzx t1d, word [buf+rax*2] + movzx t2d, word [buf+rax*2-2] +%if WIN64 + add rsp, 48 +%endif + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm2 + +cglobal msac_decode_bool_adapt, 0, 6, 0 + movifnidn t1, r1mp + movifnidn t0, r0mp + movzx eax, word [t1] + movzx t3d, byte [t0+msac.rng+1] + mov t4, [t0+msac.dif] + mov t2d, [t0+msac.rng] +%if ARCH_X86_64 + mov t5d, eax +%endif + and eax, ~63 + imul eax, t3d +%if UNIX64 + mov t6, t4 +%endif + shr eax, 7 + add eax, 4 ; v + mov t3d, eax + shl rax, gprsize*8-16 ; vw + sub t2d, t3d ; r - v + sub t4, rax ; dif - vw + setb al + cmovb t2d, t3d + mov t3d, [t0+msac.update_cdf] +%if UNIX64 + cmovb t4, t6 +%else + cmovb t4, [t0+msac.dif] +%endif +%if ARCH_X86_64 == 0 + movzx eax, al +%endif + not t4 + test t3d, t3d + jz m(msac_decode_symbol_adapt4, SUFFIX).renorm3 +%if UNIX64 == 0 + push t6 +%endif + movzx t6d, word [t1+2] +%if ARCH_X86_64 == 0 + push t5 + movzx t5d, word [t1] +%endif + movifnidn t7, t0 + lea ecx, [t6+64] + cmp t6d, 32 + adc t6d, 0 + mov [t1+2], t6w + imul t6d, eax, -32769 + shr ecx, 4 ; rate + add t6d, t5d ; if (bit) + sub t5d, eax ; cdf[0] -= ((cdf[0] - 32769) >> rate) + 1; + sar t6d, cl ; else + sub t5d, t6d ; cdf[0] -= cdf[0] >> rate; + mov [t1], t5w +%if WIN64 + mov t1d, [t7+msac.cnt] + pop t6 + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm4 +%else +%if ARCH_X86_64 == 0 + pop t5 + pop t6 +%endif + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3 +%endif + +cglobal msac_decode_bool_equi, 0, 6, 0 + movifnidn t0, r0mp + mov t1d, [t0+msac.rng] + mov t4, [t0+msac.dif] + mov t2d, t1d + mov t1b, 8 + mov t3, t4 + mov eax, t1d + shr t1d, 1 ; v + shl rax, gprsize*8-17 ; vw + sub t2d, t1d ; r - v + sub t4, rax ; dif - vw + cmovb t2d, t1d + mov t1d, [t0+msac.cnt] + cmovb t4, t3 + movifnidn t7, t0 + mov ecx, 0xbfff + setb al ; the upper 32 bits contains garbage but that's OK + sub ecx, t2d + not t4 + ; In this case of this function, (d =) 16 - clz(v) = 2 - (v >> 14) + ; i.e. (0 <= d <= 2) and v < (3 << 14) + shr ecx, 14 ; d +%if ARCH_X86_64 == 0 + movzx eax, al +%endif + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm5 + +cglobal msac_decode_bool, 0, 6, 0 + movifnidn t0, r0mp + movifnidn t1d, r1m + movzx eax, byte [t0+msac.rng+1] ; r >> 8 + mov t4, [t0+msac.dif] + mov t2d, [t0+msac.rng] + and t1d, ~63 + imul eax, t1d + mov t3, t4 + shr eax, 7 + add eax, 4 ; v + mov t1d, eax + shl rax, gprsize*8-16 ; vw + sub t2d, t1d ; r - v + sub t4, rax ; dif - vw + cmovb t2d, t1d + cmovb t4, t3 + setb al + not t4 +%if ARCH_X86_64 == 0 + movzx eax, al +%endif + jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3 + +%macro HI_TOK 1 ; update_cdf +%if ARCH_X86_64 == 0 + mov eax, -24 +%endif +%%loop: +%if %1 + movzx t2d, word [t1+3*2] +%endif + mova m1, m0 + pshuflw m2, m2, q0000 + psrlw m1, 6 + movd [buf+12], m2 + pand m2, m4 + psllw m1, 7 + pmulhuw m1, m2 +%if ARCH_X86_64 == 0 + add eax, 5 + mov [buf+8], eax +%endif + pshuflw m3, m3, c_shuf + paddw m1, m5 + movq [buf+16], m1 + psubusw m1, m3 + pxor m2, m2 + pcmpeqw m1, m2 + pmovmskb eax, m1 +%if %1 + lea ecx, [t2+80] + pcmpeqw m2, m2 + shr ecx, 4 + cmp t2d, 32 + adc t2d, 0 + movd m3, ecx + pavgw m2, m1 + psubw m2, m0 + psubw m0, m1 + psraw m2, m3 + paddw m0, m2 + movq [t1], m0 + mov [t1+3*2], t2w +%endif + tzcnt eax, eax + movzx ecx, word [buf+rax+16] + movzx t2d, word [buf+rax+14] + not t4 +%if ARCH_X86_64 + add t6d, 5 +%endif + sub eax, 5 ; setup for merging the tok_br and tok branches + sub t2d, ecx + shl rcx, gprsize*8-16 + add t4, rcx + bsr ecx, t2d + xor ecx, 15 + shl t2d, cl + shl t4, cl + movd m2, t2d + mov [t7+msac.rng], t2d + not t4 + sub t5d, ecx + jae %%end + mov t2, [t7+msac.buf] + mov rcx, [t7+msac.end] +%if UNIX64 == 0 + push t8 +%endif + lea t8, [t2+gprsize] + cmp t8, rcx + ja %%refill_eob + mov t2, [t2] + lea ecx, [t5+23] + add t5d, 16 + shr ecx, 3 + bswap t2 + sub t8, rcx + shl ecx, 3 + shr t2, cl + sub ecx, t5d + mov t5d, gprsize*8-16 + shl t2, cl + mov [t7+msac.buf], t8 +%if UNIX64 == 0 + pop t8 +%endif + sub t5d, ecx + xor t4, t2 +%%end: + movp m3, t4 +%if ARCH_X86_64 + add t6d, eax ; CF = tok_br < 3 || tok == 15 + jnc %%loop + lea eax, [t6+30] +%else + add eax, [buf+8] + jnc %%loop + add eax, 30 +%if STACK_ALIGNMENT >= 16 + add esp, 36 +%else + mov esp, [esp] +%endif +%endif + mov [t7+msac.dif], t4 + shr eax, 1 + mov [t7+msac.cnt], t5d + RET +%%refill_eob: + mov t8, rcx + mov ecx, gprsize*8-24 + sub ecx, t5d +%%refill_eob_loop: + cmp t2, t8 + jae %%refill_eob_end + movzx t5d, byte [t2] + inc t2 + shl t5, cl + xor t4, t5 + sub ecx, 8 + jge %%refill_eob_loop +%%refill_eob_end: +%if UNIX64 == 0 + pop t8 +%endif + mov t5d, gprsize*8-24 + mov [t7+msac.buf], t2 + sub t5d, ecx + jmp %%end +%endmacro + +cglobal msac_decode_hi_tok, 0, 7 + ARCH_X86_64, 6 + DECODE_SYMBOL_ADAPT_INIT 1 +%if ARCH_X86_64 == 0 && PIC + LEA t2, min_prob+12*2 + %define base t2-(min_prob+12*2) +%else + %define base 0 +%endif + movq m0, [t1] + movd m2, [t0+msac.rng] + mov eax, [t0+msac.update_cdf] + movq m4, [base+pw_0xff00] + movp m3, [t0+msac.dif] + movq m5, [base+min_prob+12*2] + mov t4, [t0+msac.dif] + mov t5d, [t0+msac.cnt] +%if ARCH_X86_64 + mov t6d, -24 +%endif + movifnidn t7, t0 + test eax, eax + jz .no_update_cdf + HI_TOK 1 +.no_update_cdf: + HI_TOK 0 + +%if ARCH_X86_64 +INIT_YMM avx2 +cglobal msac_decode_symbol_adapt16, 3, 6, 6 + lea rax, [pw_0xff00] + vpbroadcastw m2, [t0+msac.rng] + mova m0, [t1] + vpbroadcastw m3, [t0+msac.dif+6] + vbroadcasti128 m4, [rax] + mov t3d, [t0+msac.update_cdf] + mov t4d, t2d + not t2 + mov r5, rsp +%if WIN64 + and rsp, ~31 + sub rsp, 40 +%else + and r5, ~31 + %define buf r5-32 +%endif + psrlw m1, m0, 6 + movd [buf-4], xm2 + pand m2, m4 + psllw m1, 7 + pmulhuw m1, m2 + paddw m1, [rax+t2*2] + mova [buf], m1 + pmaxuw m1, m3 + pcmpeqw m1, m3 + pmovmskb eax, m1 + test t3d, t3d + jz .renorm + movzx t3d, word [t1+t4*2] + pcmpeqw m2, m2 + lea t2d, [t3+80] + shr t2d, 4 + cmp t3d, 32 + adc t3d, 0 + movd xm3, t2d + pavgw m2, m1 + psubw m2, m0 + psubw m0, m1 + psraw m2, xm3 + paddw m0, m2 + mova [t1], m0 + mov [t1+t4*2], t3w +.renorm: + tzcnt eax, eax + mov t4, [t0+msac.dif] + movzx t1d, word [buf+rax-0] + movzx t2d, word [buf+rax-2] + shr eax, 1 +%if WIN64 + mov rsp, r5 +%endif + vzeroupper + jmp m(msac_decode_symbol_adapt4, _sse2).renorm2 +%endif diff --git a/third_party/dav1d/src/x86/msac.h b/third_party/dav1d/src/x86/msac.h new file mode 100644 index 0000000000..0bb632fb31 --- /dev/null +++ b/third_party/dav1d/src/x86/msac.h @@ -0,0 +1,75 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2019, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_X86_MSAC_H +#define DAV1D_SRC_X86_MSAC_H + +#include "src/cpu.h" + +unsigned dav1d_msac_decode_symbol_adapt4_sse2(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_symbol_adapt8_sse2(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_symbol_adapt16_sse2(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_symbol_adapt16_avx2(MsacContext *s, uint16_t *cdf, + size_t n_symbols); +unsigned dav1d_msac_decode_bool_adapt_sse2(MsacContext *s, uint16_t *cdf); +unsigned dav1d_msac_decode_bool_equi_sse2(MsacContext *s); +unsigned dav1d_msac_decode_bool_sse2(MsacContext *s, unsigned f); +unsigned dav1d_msac_decode_hi_tok_sse2(MsacContext *s, uint16_t *cdf); + +#if ARCH_X86_64 || defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +#define dav1d_msac_decode_symbol_adapt4 dav1d_msac_decode_symbol_adapt4_sse2 +#define dav1d_msac_decode_symbol_adapt8 dav1d_msac_decode_symbol_adapt8_sse2 +#define dav1d_msac_decode_hi_tok dav1d_msac_decode_hi_tok_sse2 +#endif + +#define dav1d_msac_decode_bool_adapt dav1d_msac_decode_bool_adapt_sse2 +#define dav1d_msac_decode_bool_equi dav1d_msac_decode_bool_equi_sse2 +#define dav1d_msac_decode_bool dav1d_msac_decode_bool_sse2 + +#if ARCH_X86_64 +#define dav1d_msac_decode_symbol_adapt16(ctx, cdf, symb) ((ctx)->symbol_adapt16(ctx, cdf, symb)) + +static ALWAYS_INLINE void msac_init_x86(MsacContext *const s) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (flags & DAV1D_X86_CPU_FLAG_SSE2) { + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_sse2; + } + + if (flags & DAV1D_X86_CPU_FLAG_AVX2) { + s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_avx2; + } +} + +#elif defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +#define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt16_sse2 +#endif + +#endif /* DAV1D_SRC_X86_MSAC_H */ diff --git a/third_party/dav1d/src/x86/refmvs.asm b/third_party/dav1d/src/x86/refmvs.asm new file mode 100644 index 0000000000..06f555db11 --- /dev/null +++ b/third_party/dav1d/src/x86/refmvs.asm @@ -0,0 +1,688 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 64 + +%macro JMP_TABLE 2-* + %xdefine %%prefix mangle(private_prefix %+ _%1) + %1_table: + %xdefine %%base %1_table + %rep %0 - 1 + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%macro SAVE_TMVS_TABLE 3 ; num_entries, w, suffix + %rep %1 + db %2*3 + db mangle(private_prefix %+ _save_tmvs_%3).write%2 - \ + mangle(private_prefix %+ _save_tmvs_%3).write1 + %endrep +%endmacro + +%if ARCH_X86_64 +splat_mv_shuf: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3 + db 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 + db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3 +%endif +save_pack0: db 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0 + db 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1 +save_pack1: db 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2 + db 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3 +save_ref_shuf: db 0, -1, -1, -1, 1, -1, -1, -1, 8, -1, -1, -1, 9, -1, -1, -1 +cond_shuf512: db 3, 3, 3, 3, 7, 7, 7, 7, 7, 7, 7, 7, 3, 3, 3, 3 +save_cond0: db 0x80, 0x81, 0x82, 0x83, 0x89, 0x84, 0x00, 0x00 +save_cond1: db 0x84, 0x85, 0x86, 0x87, 0x88, 0x80, 0x00, 0x00 +pb_128: times 16 db 128 + +save_tmvs_ssse3_table: SAVE_TMVS_TABLE 2, 16, ssse3 + SAVE_TMVS_TABLE 4, 8, ssse3 + SAVE_TMVS_TABLE 4, 4, ssse3 + SAVE_TMVS_TABLE 5, 2, ssse3 + SAVE_TMVS_TABLE 7, 1, ssse3 + +%if ARCH_X86_64 +save_tmvs_avx2_table: SAVE_TMVS_TABLE 2, 16, avx2 + SAVE_TMVS_TABLE 4, 8, avx2 + SAVE_TMVS_TABLE 4, 4, avx2 + SAVE_TMVS_TABLE 5, 2, avx2 + SAVE_TMVS_TABLE 7, 1, avx2 + +save_tmvs_avx512icl_table: SAVE_TMVS_TABLE 2, 16, avx512icl + SAVE_TMVS_TABLE 4, 8, avx512icl + SAVE_TMVS_TABLE 4, 4, avx512icl + SAVE_TMVS_TABLE 5, 2, avx512icl + SAVE_TMVS_TABLE 7, 1, avx512icl + +JMP_TABLE splat_mv_avx512icl, 1, 2, 4, 8, 16, 32 +JMP_TABLE splat_mv_avx2, 1, 2, 4, 8, 16, 32 +%endif + +JMP_TABLE splat_mv_sse2, 1, 2, 4, 8, 16, 32 + +SECTION .text + +%macro movif32 2 +%if ARCH_X86_32 + mov %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +; refmvs_temporal_block *rp, ptrdiff_t stride, +; refmvs_block **rr, uint8_t *ref_sign, +; int col_end8, int row_end8, int col_start8, int row_start8 +%if ARCH_X86_64 +cglobal save_tmvs, 4, 13, 11, rp, stride, rr, ref_sign, \ + xend, yend, xstart, ystart +%define base_reg r12 +%else +cglobal save_tmvs, 6, 7, 8, rp, stride, rr, ref_sign, \ + xend, yend, xstart, ystart + movq m5, [ref_signq] + lea strided, [strided*5] + mov stridem, strided + mov r3, xstartm + mov r1, ystartm + DEFINE_ARGS b, ystart, rr, cand, xend, x +%define stridemp r1m +%define m8 [base+pb_128] +%define m9 [base+save_pack0+ 0] +%define m10 [base+save_pack0+16] +%define base_reg r6 +%endif +%define base base_reg-.write1 + LEA base_reg, .write1 +%if ARCH_X86_64 + movifnidn xendd, xendm + movifnidn yendd, yendm + mov xstartd, xstartm + mov ystartd, ystartm + movq m5, [ref_signq] +%endif + movu m4, [base+save_ref_shuf] + movddup m6, [base+save_cond0] + movddup m7, [base+save_cond1] +%if ARCH_X86_64 + mova m8, [base+pb_128] + mova m9, [base+save_pack0+ 0] + mova m10, [base+save_pack0+16] +%endif + psllq m5, 8 +%if ARCH_X86_64 + lea r9d, [xendq*5] + lea xstartd, [xstartq*5] + sub yendd, ystartd + add ystartd, ystartd + lea strideq, [strideq*5] + sub xstartq, r9 + add xendd, r9d + add rpq, r9 + DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand +%else + lea r0, [xendd*5] ; xend5 + lea r3, [r3*5] ; xstart5 + sub r3, r0 ; -w5 + mov r6m, r3 +%define xstartq r6m + add xendd, r0 ; xend6 + add r0m, r0 ; rp+xend5 + mov xendm, xendd + sub r5, r1 ; h + add r1, r1 + mov r7m, r1 + mov r5m, r5 +%define hd r5mp + jmp .loop_y_noload +%endif +.loop_y: + movif32 ystartd, r7m + movif32 xendd, xendm +.loop_y_noload: + and ystartd, 30 + mov xq, xstartq + mov bq, [rrq+ystartq*gprsize] + add ystartd, 2 + movif32 r7m, ystartd + lea bq, [bq+xendq*4] +.loop_x: +%if ARCH_X86_32 +%define rpq r3 +%define r10 r1 +%define r10d r1 +%define r11 r4 +%define r11d r4 +%endif + imul candq, xq, 0x9999 ; x / 5 * 3 + sar candq, 16 + movzx r10d, byte [bq+candq*8+22] ; cand_b->bs + movu m0, [bq+candq*8+12] ; cand_b + movzx r11d, byte [base+save_tmvs_ssse3_table+r10*2+0] + movzx r10d, byte [base+save_tmvs_ssse3_table+r10*2+1] + add r10, base_reg + add candq, r11 + jge .calc + movu m1, [bq+candq*8+12] + movzx r11d, byte [bq+candq*8+22] + movzx r11d, byte [base+save_tmvs_ssse3_table+r11*2+1] + add r11, base_reg +.calc: + movif32 rpq, r0m + ; ref check + punpckhqdq m2, m0, m1 + pshufb m2, m4 ; b0.ref0 b0.ref1 b1.ref0 b1.ref1 | ... + pshufb m3, m5, m2 ; ref > 0 && res_sign[ref - 1] + ; mv check + punpcklqdq m2, m0, m1 ; b0.mv0 b0.mv1 b1.mv0 b1.mv1 | ... + pabsw m2, m2 + psrlw m2, 12 ; (abs(mv.x) | abs(mv.y)) < 4096 + ; res + pcmpgtd m3, m2 + pshufd m2, m3, q2301 + pand m3, m6 ; b0c0 b0c1 b1c0 b1c1 | ... + pand m2, m7 ; b0c1 b0c0 b1c1 b1c0 | ... + por m3, m2 ; b0.shuf b1.shuf | ... + pxor m3, m8 ; if cond0|cond1 == 0 => zero out + pshufb m0, m3 + pshufb m1, m3 + call r10 + jge .next_line + pshufd m0, m1, q3232 + call r11 + jl .loop_x +.next_line: + add rpq, stridemp + movif32 r0m, rpq + dec hd + jg .loop_y + RET +.write1: + movd [rpq+xq+0], m0 + psrlq m0, 8 + movd [rpq+xq+1], m0 + add xq, 5*1 + ret +.write2: + movq [rpq+xq+0], m0 + psrlq m0, 8 + movd [rpq+xq+6], m0 + add xq, 5*2 + ret +.write4: + pshufb m0, m9 + movu [rpq+xq+ 0], m0 + psrlq m0, 8 + movd [rpq+xq+16], m0 + add xq, 5*4 + ret +.write8: + pshufb m2, m0, m9 + movu [rpq+xq+ 0], m2 + pshufb m0, m10 + movu [rpq+xq+16], m0 + psrldq m2, 2 + movq [rpq+xq+32], m2 + add xq, 5*8 + ret +.write16: + pshufb m2, m0, m9 + movu [rpq+xq+ 0], m2 + pshufb m0, m10 + movu [rpq+xq+16], m0 + shufps m2, m0, q1032 + movu [rpq+xq+48], m2 + shufps m2, m0, q2121 + movu [rpq+xq+32], m2 + shufps m0, m2, q1032 + movu [rpq+xq+64], m0 + add xq, 5*16 + ret + +INIT_XMM sse2 +; refmvs_block **rr, refmvs_block *a, int bx4, int bw4, int bh4 +cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4 + add bx4d, bw4d + tzcnt bw4d, bw4d + mova m2, [aq] + LEA aq, splat_mv_sse2_table + lea bx4q, [bx4q*3-32] + movsxd bw4q, [aq+bw4q*4] + movifnidn bh4d, bh4m + pshufd m0, m2, q0210 + pshufd m1, m2, q1021 + pshufd m2, m2, q2102 + add bw4q, aq +.loop: + mov aq, [rrq] + add rrq, gprsize + lea aq, [aq+bx4q*4] + jmp bw4q +.w32: + mova [aq-16*16], m0 + mova [aq-16*15], m1 + mova [aq-16*14], m2 + mova [aq-16*13], m0 + mova [aq-16*12], m1 + mova [aq-16*11], m2 + mova [aq-16*10], m0 + mova [aq-16* 9], m1 + mova [aq-16* 8], m2 + mova [aq-16* 7], m0 + mova [aq-16* 6], m1 + mova [aq-16* 5], m2 +.w16: + mova [aq-16* 4], m0 + mova [aq-16* 3], m1 + mova [aq-16* 2], m2 + mova [aq-16* 1], m0 + mova [aq+16* 0], m1 + mova [aq+16* 1], m2 +.w8: + mova [aq+16* 2], m0 + mova [aq+16* 3], m1 + mova [aq+16* 4], m2 +.w4: + mova [aq+16* 5], m0 + mova [aq+16* 6], m1 + mova [aq+16* 7], m2 + dec bh4d + jg .loop + RET +.w2: + movu [aq+104], m0 + movq [aq+120], m1 + dec bh4d + jg .loop + RET +.w1: + movq [aq+116], m0 + movd [aq+124], m2 + dec bh4d + jg .loop + RET + +%if ARCH_X86_64 +INIT_YMM avx2 +; refmvs_temporal_block *rp, ptrdiff_t stride, +; refmvs_block **rr, uint8_t *ref_sign, +; int col_end8, int row_end8, int col_start8, int row_start8 +cglobal save_tmvs, 4, 13, 10, rp, stride, rr, ref_sign, \ + xend, yend, xstart, ystart +%define base r12-.write1 + lea r12, [.write1] + movifnidn xendd, xendm + movifnidn yendd, yendm + mov xstartd, xstartm + mov ystartd, ystartm + vpbroadcastq m4, [ref_signq] + vpbroadcastq m3, [base+save_ref_shuf+8] + vpbroadcastq m5, [base+save_cond0] + vpbroadcastq m6, [base+save_cond1] + vpbroadcastd m7, [base+pb_128] + mova m8, [base+save_pack0] + mova m9, [base+save_pack1] + psllq m4, 8 + lea r9d, [xendq*5] + lea xstartd, [xstartq*5] + sub yendd, ystartd + add ystartd, ystartd + lea strideq, [strideq*5] + sub xstartq, r9 + add xendd, r9d + add rpq, r9 + DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand +.loop_y: + and ystartd, 30 + mov xq, xstartq + mov bq, [rrq+ystartq*8] + add ystartd, 2 + lea bq, [bq+xendq*4] +.loop_x: + imul candq, xq, 0x9999 + sar candq, 16 ; x / 5 * 3 + movzx r10d, byte [bq+candq*8+22] ; cand_b->bs + movu xm0, [bq+candq*8+12] ; cand_b + movzx r11d, byte [base+save_tmvs_avx2_table+r10*2+0] + movzx r10d, byte [base+save_tmvs_avx2_table+r10*2+1] + add r10, r12 + add candq, r11 + jge .calc + vinserti128 m0, [bq+candq*8+12], 1 + movzx r11d, byte [bq+candq*8+22] + movzx r11d, byte [base+save_tmvs_avx2_table+r11*2+1] + add r11, r12 +.calc: + pshufb m1, m0, m3 + pabsw m2, m0 + pshufb m1, m4, m1 ; ref > 0 && res_sign[ref - 1] + psrlw m2, 12 ; (abs(mv.x) | abs(mv.y)) < 4096 + pcmpgtd m1, m2 + pshufd m2, m1, q2301 + pand m1, m5 ; b0.cond0 b1.cond0 + pand m2, m6 ; b0.cond1 b1.cond1 + por m1, m2 ; b0.shuf b1.shuf + pxor m1, m7 ; if cond0|cond1 == 0 => zero out + pshufb m0, m1 + call r10 + jge .next_line + vextracti128 xm0, m0, 1 + call r11 + jl .loop_x +.next_line: + add rpq, strideq + dec hd + jg .loop_y + RET +.write1: + movd [rpq+xq+ 0], xm0 + pextrb [rpq+xq+ 4], xm0, 4 + add xq, 5*1 + ret +.write2: + movq [rpq+xq+0], xm0 + psrlq xm1, xm0, 8 + movd [rpq+xq+6], xm1 + add xq, 5*2 + ret +.write4: + pshufb xm1, xm0, xm8 + movu [rpq+xq+ 0], xm1 + psrlq xm1, 8 + movd [rpq+xq+16], xm1 + add xq, 5*4 + ret +.write8: + vinserti128 m1, m0, xm0, 1 + pshufb m1, m8 + movu [rpq+xq+ 0], m1 + psrldq xm1, 2 + movq [rpq+xq+32], xm1 + add xq, 5*8 + ret +.write16: + vinserti128 m1, m0, xm0, 1 + pshufb m2, m1, m8 + movu [rpq+xq+ 0], m2 + pshufb m1, m9 + movu [rpq+xq+32], m1 + shufps xm2, xm1, q1021 + movu [rpq+xq+64], xm2 + add xq, 5*16 + ret + +cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4 + add bx4d, bw4d + tzcnt bw4d, bw4d + vbroadcasti128 m0, [aq] + lea aq, [splat_mv_avx2_table] + lea bx4q, [bx4q*3-32] + movsxd bw4q, [aq+bw4q*4] + pshufb m0, [splat_mv_shuf] + movifnidn bh4d, bh4m + pshufd m1, m0, q2102 + pshufd m2, m0, q1021 + add bw4q, aq +.loop: + mov aq, [rrq] + add rrq, gprsize + lea aq, [aq+bx4q*4] + jmp bw4q +.w32: + mova [aq-32*8], m0 + mova [aq-32*7], m1 + mova [aq-32*6], m2 + mova [aq-32*5], m0 + mova [aq-32*4], m1 + mova [aq-32*3], m2 +.w16: + mova [aq-32*2], m0 + mova [aq-32*1], m1 + mova [aq+32*0], m2 +.w8: + mova [aq+32*1], m0 + mova [aq+32*2], m1 + mova [aq+32*3], m2 + dec bh4d + jg .loop + RET +.w4: + movu [aq+ 80], m0 + mova [aq+112], xm1 + dec bh4d + jg .loop + RET +.w2: + movu [aq+104], xm0 + movq [aq+120], xm2 + dec bh4d + jg .loop + RET +.w1: + movq [aq+116], xm0 + movd [aq+124], xm1 + dec bh4d + jg .loop + RET + +INIT_ZMM avx512icl +; refmvs_temporal_block *rp, ptrdiff_t stride, +; refmvs_block **rr, uint8_t *ref_sign, +; int col_end8, int row_end8, int col_start8, int row_start8 +cglobal save_tmvs, 4, 15, 10, rp, stride, rr, ref_sign, \ + xend, yend, xstart, ystart +%define base r14-.write1 + lea r14, [.write1] + movifnidn xendd, xendm + movifnidn yendd, yendm + mov xstartd, xstartm + mov ystartd, ystartm + psllq m4, [ref_signq]{bcstq}, 8 + vpbroadcastq m3, [base+save_ref_shuf+8] + vbroadcasti32x4 m5, [base+cond_shuf512] + vbroadcasti32x4 m6, [base+save_cond0] + vpbroadcastd m7, [base+pb_128] + mova m8, [base+save_pack0] + movu xm9, [base+save_pack0+4] + lea r9d, [xendq*5] + lea xstartd, [xstartq*5] + sub yendd, ystartd + add ystartd, ystartd + lea strideq, [strideq*5] + sub xstartq, r9 + add xendd, r9d + add rpq, r9 + mov r10d, 0x1f + kmovb k2, r10d + DEFINE_ARGS rp, stride, rr, x, xend, h, xstart, ystart, b, cand +.loop_y: + and ystartd, 30 + mov xq, xstartq + mov bq, [rrq+ystartq*8] + add ystartd, 2 + lea bq, [bq+xendq*4] +.loop_x: + imul candq, xq, 0x9999 + sar candq, 16 ; x / 5 * 3 + movzx r10d, byte [bq+candq*8+22] ; cand_b->bs + movu xm0, [bq+candq*8+12] ; cand_b + movzx r11d, byte [base+save_tmvs_avx512icl_table+r10*2+0] + movzx r10d, byte [base+save_tmvs_avx512icl_table+r10*2+1] + add r10, r14 + add candq, r11 + jge .calc + movzx r11d, byte [bq+candq*8+22] + vinserti32x4 ym0, [bq+candq*8+12], 1 + movzx r12d, byte [base+save_tmvs_avx512icl_table+r11*2+0] + movzx r11d, byte [base+save_tmvs_avx512icl_table+r11*2+1] + add r11, r14 + add candq, r12 + jge .calc + movzx r12d, byte [bq+candq*8+22] + vinserti32x4 m0, [bq+candq*8+12], 2 + movzx r13d, byte [base+save_tmvs_avx512icl_table+r12*2+0] + movzx r12d, byte [base+save_tmvs_avx512icl_table+r12*2+1] + add r12, r14 + add candq, r13 + jge .calc + vinserti32x4 m0, [bq+candq*8+12], 3 + movzx r13d, byte [bq+candq*8+22] + movzx r13d, byte [base+save_tmvs_avx512icl_table+r13*2+1] + add r13, r14 +.calc: + pshufb m1, m0, m3 + pabsw m2, m0 + pshufb m1, m4, m1 ; ref > 0 && res_sign[ref - 1] + psrlw m2, 12 ; (abs(mv.x) | abs(mv.y)) < 4096 + psubd m2, m1 + pshufb m2, m5 ; c0 c1 c1 c0 + pand m2, m6 + punpckhqdq m1, m2, m2 + vpternlogd m1, m2, m7, 0x56 ; (c0shuf | c1shuf) ^ 0x80 + pshufb m2, m0, m1 + mova xm0, xm2 + call r10 + jge .next_line + vextracti32x4 xm0, m2, 1 + call r11 + jge .next_line + vextracti32x4 xm0, m2, 2 + call r12 + jge .next_line + vextracti32x4 xm0, m2, 3 + call r13 + jl .loop_x +.next_line: + add rpq, strideq + dec hd + jg .loop_y + RET +.write1: + vmovdqu8 [rpq+xq]{k2}, xm0 + add xq, 5*1 + ret +.write2: + pshufb xm0, xm8 + vmovdqu16 [rpq+xq]{k2}, xm0 + add xq, 5*2 + ret +.write4: + vpermb ym0, ym8, ym0 + vmovdqu32 [rpq+xq]{k2}, ym0 + add xq, 5*4 + ret +.write8: + vpermb m0, m8, m0 + vmovdqu64 [rpq+xq]{k2}, m0 + add xq, 5*8 + ret +.write16: + vpermb m1, m8, m0 + movu [rpq+xq+ 0], m1 + pshufb xm0, xm9 + movu [rpq+xq+64], xm0 + add xq, 5*16 + ret + +INIT_ZMM avx512icl +cglobal splat_mv, 4, 7, 3, rr, a, bx4, bw4, bh4 + vbroadcasti32x4 m0, [aq] + lea r1, [splat_mv_avx512icl_table] + tzcnt bw4d, bw4d + lea bx4d, [bx4q*3] + pshufb m0, [splat_mv_shuf] + movsxd bw4q, [r1+bw4q*4] + mov r6d, bh4m + add bw4q, r1 + lea rrq, [rrq+r6*8] + mov r1d, 0x3f + neg r6 + kmovb k1, r1d + jmp bw4q +.w1: + mov r1, [rrq+r6*8] + vmovdqu16 [r1+bx4q*4]{k1}, xm0 + inc r6 + jl .w1 + RET +.w2: + mov r1, [rrq+r6*8] + vmovdqu32 [r1+bx4q*4]{k1}, ym0 + inc r6 + jl .w2 + RET +.w4: + mov r1, [rrq+r6*8] + vmovdqu64 [r1+bx4q*4]{k1}, m0 + inc r6 + jl .w4 + RET +.w8: + pshufd ym1, ym0, q1021 +.w8_loop: + mov r1, [rrq+r6*8+0] + mov r3, [rrq+r6*8+8] + movu [r1+bx4q*4+ 0], m0 + mova [r1+bx4q*4+64], ym1 + movu [r3+bx4q*4+ 0], m0 + mova [r3+bx4q*4+64], ym1 + add r6, 2 + jl .w8_loop + RET +.w16: + pshufd m1, m0, q1021 + pshufd m2, m0, q2102 +.w16_loop: + mov r1, [rrq+r6*8+0] + mov r3, [rrq+r6*8+8] + mova [r1+bx4q*4+64*0], m0 + mova [r1+bx4q*4+64*1], m1 + mova [r1+bx4q*4+64*2], m2 + mova [r3+bx4q*4+64*0], m0 + mova [r3+bx4q*4+64*1], m1 + mova [r3+bx4q*4+64*2], m2 + add r6, 2 + jl .w16_loop + RET +.w32: + pshufd m1, m0, q1021 + pshufd m2, m0, q2102 +.w32_loop: + mov r1, [rrq+r6*8] + lea r1, [r1+bx4q*4] + mova [r1+64*0], m0 + mova [r1+64*1], m1 + mova [r1+64*2], m2 + mova [r1+64*3], m0 + mova [r1+64*4], m1 + mova [r1+64*5], m2 + inc r6 + jl .w32_loop + RET +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/refmvs.h b/third_party/dav1d/src/x86/refmvs.h new file mode 100644 index 0000000000..9dafa78b13 --- /dev/null +++ b/third_party/dav1d/src/x86/refmvs.h @@ -0,0 +1,61 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/refmvs.h" + +decl_save_tmvs_fn(dav1d_save_tmvs_ssse3); +decl_save_tmvs_fn(dav1d_save_tmvs_avx2); +decl_save_tmvs_fn(dav1d_save_tmvs_avx512icl); + +decl_splat_mv_fn(dav1d_splat_mv_sse2); +decl_splat_mv_fn(dav1d_splat_mv_avx2); +decl_splat_mv_fn(dav1d_splat_mv_avx512icl); + +static ALWAYS_INLINE void refmvs_dsp_init_x86(Dav1dRefmvsDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; + + c->splat_mv = dav1d_splat_mv_sse2; + + if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; + + c->save_tmvs = dav1d_save_tmvs_ssse3; + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->save_tmvs = dav1d_save_tmvs_avx2; + c->splat_mv = dav1d_splat_mv_avx2; + + if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; + + c->save_tmvs = dav1d_save_tmvs_avx512icl; + c->splat_mv = dav1d_splat_mv_avx512icl; +#endif +} -- cgit v1.2.3