summaryrefslogtreecommitdiffstats
path: root/third_party/dav1d/src/x86/itx_avx512.asm
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/dav1d/src/x86/itx_avx512.asm')
-rw-r--r--third_party/dav1d/src/x86/itx_avx512.asm7389
1 files changed, 7389 insertions, 0 deletions
diff --git a/third_party/dav1d/src/x86/itx_avx512.asm b/third_party/dav1d/src/x86/itx_avx512.asm
new file mode 100644
index 0000000000..31c60fdd45
--- /dev/null
+++ b/third_party/dav1d/src/x86/itx_avx512.asm
@@ -0,0 +1,7389 @@
+; Copyright © 2020-2023, VideoLAN and dav1d authors
+; Copyright © 2020-2023, Two Orioles, LLC
+; All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions are met:
+;
+; 1. Redistributions of source code must retain the above copyright notice, this
+; list of conditions and the following disclaimer.
+;
+; 2. Redistributions in binary form must reproduce the above copyright notice,
+; this list of conditions and the following disclaimer in the documentation
+; and/or other materials provided with the distribution.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+%include "config.asm"
+%include "ext/x86/x86inc.asm"
+
+%if ARCH_X86_64
+
+SECTION_RODATA 64
+const \
+dup16_perm, db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7
+ db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15
+ db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23
+ db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31
+const \
+int8_permA, db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51
+ db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55
+ db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59
+ db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63
+int8_permB: db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51
+ db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59
+ db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55
+ db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63
+int16_perm: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39
+ db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47
+ db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55
+ db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63
+idtx_16x4p: db 0, 1, 4, 5, 16, 17, 20, 21, 2, 3, 6, 7, 18, 19, 22, 23
+ db 32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55
+ db 8, 9, 12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31
+ db 40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63
+idct_8x32p: db 60, 61, 4, 5, 32, 33, 0, 1, 28, 29, 36, 37, 56, 57, 8, 9
+ db 12, 13, 52, 53, 24, 25, 40, 41, 44, 45, 20, 21, 48, 49, 16, 17
+ db 62, 63, 2, 3, 6, 7, 58, 59, 54, 55, 10, 11, 14, 15, 50, 51
+ db 46, 47, 18, 19, 22, 23, 42, 43, 38, 39, 26, 27, 30, 31, 34, 35
+idct_16x32p: db 6, 7, 58, 59, 38, 39, 26, 27, 32, 33, 0, 1, 30, 31, 34, 35
+ db 46, 47, 18, 19, 22, 23, 42, 43, 24, 25, 40, 41, 44, 45, 20, 21
+ db 62, 63, 2, 3, 48, 49, 16, 17, 56, 57, 8, 9, 14, 15, 50, 51
+ db 54, 55, 10, 11, 60, 61, 4, 5, 12, 13, 52, 53, 28, 29, 36, 37
+end_16x32p: db 0, 32, 1, 48, 2, 36, 3, 52, 16, 40, 17, 56, 18, 44, 19, 60
+ db 4, 33, 5, 49, 6, 37, 7, 53, 20, 41, 21, 57, 22, 45, 23, 61
+ db 8, 35, 9, 51, 10, 39, 11, 55, 24, 43, 25, 59, 26, 47, 27, 63
+ db 12, 34, 13, 50, 14, 38, 15, 54, 28, 42, 29, 58, 30, 46, 31, 62
+
+; packed 4-bit qword shuffle indices
+permA: dq 0x1c0d0d1ce0d94040, 0x5849495868fb6262
+ dq 0x3e2f2f3ef1c85151, 0x7a6b6b7a79ea7373
+ dq 0x94858594a451c8d9, 0xd0c1c1d02c73eafb
+ dq 0xb6a7a7b6b540d9c8, 0xf2e3e3f23d62fbea
+permB: dq 0x40acbd0fcadb0f40, 0x518e9f3ce8f99604
+ dq 0xc824352d56128751, 0xd906171e74301e15
+ dq 0x6271604b03472d62, 0x735342782165b426
+ dq 0xeaf9e8699f8ea573, 0xfbdbca5abdac3c37
+permC: dq 0x9d409d041551c2e0, 0xbf62bf263773a486
+ dq 0xc88c8c15409dd3f1, 0xeaaeae3762bfb597
+ dq 0x04d9158c8cc84a68, 0x26fb37aeaeea2c0e
+ dq 0x5115049dd9045b79, 0x733726bffb263d1f
+permD: dq 0x0cda098800041504, 0x0edb09b2028c3726
+ dq 0x0f11fa9c01150415, 0x0988f326039d2637
+ dq 0x05640f1108269d8c, 0x05290edb0aaebfae
+ dq 0x0005000509378c9d, 0xffffffff0bbfaebf
+
+pd_0to15: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+gather8a: dd 0, 2, 1, 3, 8, 10, 9, 11
+gather8b: dd 0, 1, 4, 5, 8, 9, 12, 13
+gather8c: dd 0, 4, 2, 6, 12, 8, 14, 10
+gather8d: dd 0, 19, 1, 18, 2, 17, 3, 16
+
+int_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
+int_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7
+int_shuf3: db 0, 1, 8, 9, 4, 5, 12, 13, 2, 3, 10, 11, 6, 7, 14, 15
+int_shuf4: db 8, 9, 0, 1, 12, 13, 4, 5, 10, 11, 2, 3, 14, 15, 6, 7
+deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
+int_mshift: db 12, 20, 0, 0, 44, 52, 0, 0
+
+pb_32: times 4 db 32
+pw_2048: times 2 dw 2048
+pw_4096: times 2 dw 4096
+pw_8192: times 2 dw 8192
+pw_16384: times 2 dw 16384
+pw_1697x16: times 2 dw 1697*16
+pw_1697x8: times 2 dw 1697*8
+pw_2896x8: times 2 dw 2896*8
+pd_2048: dd 2048
+
+%define pw_5 (permD+52)
+%define pd_m1 (permD+60)
+%define pw_3803_1321 (permD+44)
+%define pw_2482_3803 (permD+12)
+%define pw_2440_3290 (permD+ 4)
+%define pw_m3290_2440 (permD+28)
+%define pw_3857_1380 (permD+36)
+%define pw_m1380_3857 (permD+20)
+
+pw_8192_m8192: dw 8192, -8192
+pw_m8192_8192: dw -8192, 8192
+pw_16384_m16384: dw 16384, -16384
+pw_m16384_16384: dw -16384, 16384
+
+pw_m1321_2482: dw -1321, 2482
+pw_m3344_3344: dw -3344, 3344
+pw_2482_3344: dw 2482, 3344
+pw_m3803_3344: dw -3803, 3344
+pd_3344: dd 3344
+pw_m1321_m3344: dw -1321, -3344
+pw_2896_m2896: dw 2896, -2896
+
+pw_1567_m3784: dw 1567, -3784
+pw_3784_m1567: dw 3784, -1567
+pw_4017_m799: dw 4017, -799
+pw_2276_m3406: dw 2276, -3406
+pw_m799_m4017: dw -799, -4017
+pw_m3406_m2276: dw -3406, -2276
+
+%macro COEF_PAIR 2-3 0
+pw_%1_%2: dw %1, %2
+pw_m%2_%1: dw -%2, %1
+%if %3
+pw_m%1_m%2: dw -%1, -%2
+%endif
+%endmacro
+
+COEF_PAIR 2896, 2896
+COEF_PAIR 1567, 3784, 1
+COEF_PAIR 3784, 1567
+COEF_PAIR 201, 4091
+COEF_PAIR 995, 3973
+COEF_PAIR 1751, 3703
+COEF_PAIR 3035, 2751
+COEF_PAIR 3513, 2106
+COEF_PAIR 4052, 601
+COEF_PAIR 3166, 2598, 1
+COEF_PAIR 3920, 1189, 1
+COEF_PAIR 2276, 3406
+COEF_PAIR 4017, 799
+
+%macro COEF_X8 1-*
+%rep %0
+ dw %1*8, %1*8
+ %rotate 1
+%endrep
+%endmacro
+
+pw_m2276x8: COEF_X8 -2276
+pw_3406x8: COEF_X8 3406
+pw_4017x8: COEF_X8 4017
+pw_799x8: COEF_X8 799
+pw_3784x8: COEF_X8 3784
+pw_1567x8: COEF_X8 1567
+
+pw_4076x8: COEF_X8 4076
+pw_401x8: COEF_X8 401
+pw_m2598x8: COEF_X8 -2598
+pw_3166x8: COEF_X8 3166
+pw_3612x8: COEF_X8 3612
+pw_1931x8: COEF_X8 1931
+pw_m1189x8: COEF_X8 -1189
+pw_3920x8: COEF_X8 3920
+
+pw_4091x8: COEF_X8 4091
+pw_201x8: COEF_X8 201
+pw_m2751x8: COEF_X8 -2751
+pw_3035x8: COEF_X8 3035
+pw_3703x8: COEF_X8 3703
+pw_1751x8: COEF_X8 1751
+pw_m1380x8: COEF_X8 -1380
+pw_3857x8: COEF_X8 3857
+pw_3973x8: COEF_X8 3973
+pw_995x8: COEF_X8 995
+pw_m2106x8: COEF_X8 -2106
+pw_3513x8: COEF_X8 3513
+pw_3290x8: COEF_X8 3290
+pw_2440x8: COEF_X8 2440
+pw_m601x8: COEF_X8 -601
+pw_4052x8: COEF_X8 4052
+
+pw_401_4076x8: dw 401*8, 4076*8
+pw_m2598_3166x8: dw -2598*8, 3166*8
+pw_1931_3612x8: dw 1931*8, 3612*8
+pw_m1189_3920x8: dw -1189*8, 3920*8
+pw_799_4017x8: dw 799*8, 4017*8
+pw_m2276_3406x8: dw -2276*8, 3406*8
+
+pw_201_4091x8: dw 201*8, 4091*8
+pw_m601_4052x8: dw -601*8, 4052*8
+pw_995_3973x8: dw 995*8, 3973*8
+pw_m1380_3857x8: dw -1380*8, 3857*8
+pw_1751_3703x8: dw 1751*8, 3703*8
+pw_m2106_3513x8: dw -2106*8, 3513*8
+pw_2440_3290x8: dw 2440*8, 3290*8
+pw_m2751_3035x8: dw -2751*8, 3035*8
+
+pw_101_4095x8: dw 101*8, 4095*8
+pw_m2824_2967x8: dw -2824*8, 2967*8
+pw_1660_3745x8: dw 1660*8, 3745*8
+pw_m1474_3822x8: dw -1474*8, 3822*8
+pw_897_3996x8: dw 897*8, 3996*8
+pw_m2191_3461x8: dw -2191*8, 3461*8
+pw_2359_3349x8: dw 2359*8, 3349*8
+pw_m700_4036x8: dw -700*8, 4036*8
+pw_501_4065x8: dw 501*8, 4065*8
+pw_m2520_3229x8: dw -2520*8, 3229*8
+pw_2019_3564x8: dw 2019*8, 3564*8
+pw_m1092_3948x8: dw -1092*8, 3948*8
+pw_1285_3889x8: dw 1285*8, 3889*8
+pw_m1842_3659x8: dw -1842*8, 3659*8
+pw_2675_3102x8: dw 2675*8, 3102*8
+pw_m301_4085x8: dw -301*8, 4085*8
+
+idct64_mul: COEF_X8 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474
+COEF_PAIR 401, 4076, 1
+COEF_PAIR 799, 4017
+ COEF_X8 -700, 4036, 2359, 3349, -2191, 3461, 897, 3996
+dw -2598, -3166, 3166, -2598, 2598, 3166, -4017, -799, 799, -4017
+ COEF_X8 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092
+COEF_PAIR 1931, 3612, 1
+COEF_PAIR 3406, 2276
+ COEF_X8 -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889
+dw -1189, -3920, 3920, -1189, 1189, 3920, -2276, -3406, 3406, -2276
+
+SECTION .text
+
+%define o_base int8_permA+64*18
+%define o(x) (r5 - (o_base) + (x))
+%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
+
+; flags: 1 = swap, 2 = interleave (l), 4 = interleave (t), 8 = no_pack,
+; 16 = special_mul1, 32 = special_mul2
+%macro ITX_MUL2X_PACK 6-7 0 ; dst/src, tmp[1-2], rnd, coef[1-2], flags
+ mova m%2, m%4
+%if %7 & 16
+ vpdpwssd m%2, m%1, [o(pw_%5)] {bcstd}
+ mova m%3, m%4
+%if %7 & 32
+ vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd}
+%else
+ vpdpwssd m%3, m%1, m%6
+%endif
+%elif %7 & 32
+ vpdpwssd m%2, m%1, m%5
+ mova m%3, m%4
+ vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd}
+%elif %6 < 32
+ vpdpwssd m%2, m%1, m%5
+ mova m%3, m%4
+ vpdpwssd m%3, m%1, m%6
+%elif %7 & 1
+ vpdpwssd m%2, m%1, [o(pw_%5_%6)] {bcstd}
+ mova m%3, m%4
+ vpdpwssd m%3, m%1, [o(pw_m%6_%5)] {bcstd}
+%else
+ vpdpwssd m%2, m%1, [o(pw_m%6_%5)] {bcstd}
+ mova m%3, m%4
+ vpdpwssd m%3, m%1, [o(pw_%5_%6)] {bcstd}
+%endif
+%if %7 & 2
+ psrld m%2, 12
+ pslld m%3, 4
+ vpshrdd m%1, m%3, m%2, 16
+%elif %7 & 4
+ ; compared to using shifts (as above) this has better throughput,
+ ; but worse latency and requires setting up the opmask/index
+ ; registers, so only use this method for the larger transforms
+ pslld m%1, m%2, 4
+ vpmultishiftqb m%1{k7}, m13, m%3
+%else
+ psrad m%2, 12
+ psrad m%3, 12
+%if %7 & 8 == 0
+ packssdw m%1, m%3, m%2
+%endif
+%endif
+%endmacro
+
+; flags: same as ITX_MUL2X_PACK
+%macro ITX_MUL4X_PACK 10-11 0 ; dst/src, tmp[1-2], coef_tmp[1-2], rnd, coef[1-4], flags
+%if %11 & 1
+ vpbroadcastd m%4, [o(pw_%9_%10)]
+ vpbroadcastd m%4{k1}, [o(pw_%7_%8)]
+ vpbroadcastd m%5, [o(pw_m%10_%9)]
+ vpbroadcastd m%5{k1}, [o(pw_m%8_%7)]
+%else
+ vpbroadcastd m%4, [o(pw_m%10_%9)]
+ vpbroadcastd m%4{k1}, [o(pw_m%8_%7)]
+ vpbroadcastd m%5, [o(pw_%9_%10)]
+ vpbroadcastd m%5{k1}, [o(pw_%7_%8)]
+%endif
+ ITX_MUL2X_PACK %1, %2, %3, %6, %4, %5, %11
+%endmacro
+
+; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12
+; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12
+%macro ITX_MULSUB_2W 7-8 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2], dst2
+ punpcklwd m%3, m%2, m%1
+ punpckhwd m%2, m%1
+%if %7 < 32
+ mova m%1, m%5
+ vpdpwssd m%1, m%3, m%7
+ mova m%4, m%5
+ vpdpwssd m%4, m%2, m%7
+%else
+ mova m%1, m%5
+ vpdpwssd m%1, m%3, [o(pw_m%7_%6)] {bcstd}
+ mova m%4, m%5
+ vpdpwssd m%4, m%2, [o(pw_m%7_%6)] {bcstd}
+%endif
+ psrad m%1, 12
+ psrad m%4, 12
+ packssdw m%1, m%4
+ mova m%4, m%5
+%if %7 < 32
+ vpdpwssd m%4, m%2, m%6
+ mova m%2, m%5
+ vpdpwssd m%2, m%3, m%6
+%else
+ vpdpwssd m%4, m%2, [o(pw_%6_%7)] {bcstd}
+ mova m%2, m%5
+ vpdpwssd m%2, m%3, [o(pw_%6_%7)] {bcstd}
+%endif
+ psrad m%4, 12
+ psrad m%2, 12
+%if %0 == 8
+ packssdw m%8, m%2, m%4
+%else
+ packssdw m%2, m%4
+%endif
+%endmacro
+
+%macro WRAP_XMM 1+
+ %xdefine %%reset RESET_MM_PERMUTATION
+ INIT_XMM cpuname
+ DEFINE_MMREGS xmm
+ AVX512_MM_PERMUTATION
+ %1
+ %%reset
+%endmacro
+
+%macro WRAP_YMM 1+
+ INIT_YMM cpuname
+ %1
+ INIT_ZMM cpuname
+%endmacro
+
+%macro ITX4_END 4-5 2048 ; row[1-4], rnd
+%if %5
+ vpbroadcastd m2, [o(pw_%5)]
+ pmulhrsw m0, m2
+ pmulhrsw m1, m2
+%endif
+ lea r2, [dstq+strideq*2]
+%assign %%i 1
+%rep 4
+ %if %1 & 2
+ CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1)
+ %else
+ CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1)
+ %endif
+ %assign %%i %%i + 1
+ %rotate 1
+%endrep
+ movd m2, [%%row_adr1]
+ pinsrd m2, [%%row_adr2], 1
+ movd m3, [%%row_adr3]
+ pinsrd m3, [%%row_adr4], 1
+ pmovzxbw m2, m2
+ pmovzxbw m3, m3
+ paddw m0, m2
+ paddw m1, m3
+ packuswb m0, m1
+ movd [%%row_adr1], m0
+ pextrd [%%row_adr2], m0, 1
+ pextrd [%%row_adr3], m0, 2
+ pextrd [%%row_adr4], m0, 3
+ ret
+%endmacro
+
+%macro INV_TXFM_FN 3 ; type1, type2, size
+cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, 0, dst, stride, c, eob, tx2, base
+ %define %%p1 m(i%1_%3_internal_8bpc)
+ lea baseq, [o_base]
+ ; Jump to the 1st txfm function if we're not taking the fast path, which
+ ; in turn performs an indirect jump to the 2nd txfm function.
+ lea tx2q, [m(i%2_%3_internal_8bpc).pass2]
+%ifidn %1_%2, dct_dct
+ test eobd, eobd
+ jnz %%p1
+%else
+ ; jump to the 1st txfm function unless it's located directly after this
+ times ((%%end - %%p1) >> 31) & 1 jmp %%p1
+ALIGN function_align
+%%end:
+%endif
+%endmacro
+
+%macro INV_TXFM_4X4_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 4x4
+%ifidn %1_%2, dct_dct
+ vpbroadcastw m0, [cq]
+ vpbroadcastd m1, [o(pw_2896x8)]
+ pmulhrsw m0, m1
+ mov [cq], eobd
+ pmulhrsw m0, m1
+ mova m1, m0
+ jmp m(iadst_4x4_internal_8bpc).end2
+%endif
+%endmacro
+
+%macro IDCT4_1D_PACKED 0
+ vpbroadcastd m4, [o(pd_2048)]
+ punpckhwd m2, m1, m0
+ punpcklwd m1, m0
+ ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784
+ ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896
+ paddsw m0, m1, m2 ; out0 out1
+ psubsw m1, m2 ; out3 out2
+%endmacro
+
+%macro IADST4_1D_PACKED 0
+ punpcklwd m4, m1, m0 ; in2 in0
+ punpckhwd m5, m1, m0 ; in3 in1
+.main2:
+ vpbroadcastd m3, [o(pd_2048)]
+ mova m0, m3
+ vpdpwssd m0, m4, [o(pw_3803_1321)] {bcstd}
+ mova m2, m3
+ vpdpwssd m2, m4, [o(pw_m1321_2482)] {bcstd}
+ mova m1, m3
+ vpdpwssd m1, m4, [o(pw_m3344_3344)] {bcstd}
+ vpdpwssd m3, m4, [o(pw_2482_3803)] {bcstd}
+ vpdpwssd m0, m5, [o(pw_2482_3344)] {bcstd}
+ vpdpwssd m2, m5, [o(pw_m3803_3344)] {bcstd}
+ vpdpwssd m1, m5, [o(pd_3344)] {bcstd}
+ vpdpwssd m3, m5, [o(pw_m1321_m3344)] {bcstd}
+ REPX {psrad x, 12}, m0, m2, m1, m3
+ packssdw m0, m2 ; out0 out1
+ packssdw m1, m3 ; out2 out3
+%endmacro
+
+INIT_XMM avx512icl
+INV_TXFM_4X4_FN dct, dct
+INV_TXFM_4X4_FN dct, adst
+INV_TXFM_4X4_FN dct, flipadst
+INV_TXFM_4X4_FN dct, identity
+
+cglobal idct_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+16*0]
+ mova m1, [cq+16*1]
+ IDCT4_1D_PACKED
+ mova m2, [o(deint_shuf)]
+ shufps m3, m0, m1, q1331
+ shufps m0, m0, m1, q0220
+ pshufb m0, m2
+ pshufb m1, m3, m2
+ jmp tx2q
+.pass2:
+ IDCT4_1D_PACKED
+ pxor ymm16, ymm16
+ mova [cq], ymm16
+ ITX4_END 0, 1, 3, 2
+
+INV_TXFM_4X4_FN adst, dct
+INV_TXFM_4X4_FN adst, adst
+INV_TXFM_4X4_FN adst, flipadst
+INV_TXFM_4X4_FN adst, identity
+
+cglobal iadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+16*0]
+ mova m1, [cq+16*1]
+ call .main
+ punpckhwd m3, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m0, m3
+ punpcklwd m0, m3
+ jmp tx2q
+.pass2:
+ call .main
+.end:
+ pxor ymm16, ymm16
+ mova [cq], ymm16
+.end2:
+ ITX4_END 0, 1, 2, 3
+ALIGN function_align
+.main:
+ IADST4_1D_PACKED
+ ret
+
+INV_TXFM_4X4_FN flipadst, dct
+INV_TXFM_4X4_FN flipadst, adst
+INV_TXFM_4X4_FN flipadst, flipadst
+INV_TXFM_4X4_FN flipadst, identity
+
+cglobal iflipadst_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+16*0]
+ mova m1, [cq+16*1]
+ call m(iadst_4x4_internal_8bpc).main
+ punpcklwd m2, m1, m0
+ punpckhwd m1, m0
+ punpcklwd m0, m1, m2
+ punpckhwd m1, m2
+ jmp tx2q
+.pass2:
+ call m(iadst_4x4_internal_8bpc).main
+.end:
+ pxor ymm16, ymm16
+ mova [cq], ymm16
+.end2:
+ ITX4_END 3, 2, 1, 0
+
+INV_TXFM_4X4_FN identity, dct
+INV_TXFM_4X4_FN identity, adst
+INV_TXFM_4X4_FN identity, flipadst
+INV_TXFM_4X4_FN identity, identity
+
+cglobal iidentity_4x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+16*0]
+ mova m1, [cq+16*1]
+ vpbroadcastd m3, [o(pw_1697x8)]
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ paddsw m0, m2
+ paddsw m1, m3
+ punpckhwd m2, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m0, m2
+ punpcklwd m0, m2
+ jmp tx2q
+.pass2:
+ vpbroadcastd m3, [o(pw_1697x8)]
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ paddsw m0, m2
+ paddsw m1, m3
+ jmp m(iadst_4x4_internal_8bpc).end
+
+%macro INV_TXFM_4X8_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 4x8
+%ifidn %1_%2, dct_dct
+ movd xmm1, [o(pw_2896x8)]
+ pmulhrsw xmm0, xmm1, [cq]
+ movd xmm2, [o(pw_2048)]
+ pmulhrsw xmm0, xmm1
+ pmulhrsw xmm0, xmm1
+ pmulhrsw xmm0, xmm2
+ vpbroadcastw ym0, xmm0
+ mova ym1, ym0
+ jmp m(iadst_4x8_internal_8bpc).end3
+%endif
+%endmacro
+
+%macro IDCT8_1D_PACKED 0
+ punpckhwd m5, m3, m0 ; in7 in1
+ punpckhwd m4, m1, m2 ; in3 in5
+ punpcklwd m3, m1 ; in6 in2
+ punpcklwd m2, m0 ; in4 in0
+.main2:
+ vpbroadcastd m6, [o(pd_2048)]
+ ITX_MUL2X_PACK 5, 0, 1, 6, 799, 4017, 3 ; t4a t7a
+ ITX_MUL2X_PACK 4, 0, 1, 6, 3406, 2276, 3 ; t5a t6a
+ ITX_MUL2X_PACK 3, 0, 1, 6, 1567, 3784 ; t3 t2
+ psubsw m0, m5, m4 ; t5a t6a (interleaved)
+ paddsw m4, m5 ; t4 t7 (interleaved)
+ ITX_MUL2X_PACK 2, 1, 5, 6, 2896, 2896 ; t0 t1
+ ITX_MUL2X_PACK 0, 1, 5, 6, 2896, 2896, 1 ; t6 t5
+%if mmsize > 16
+ vbroadcasti32x4 m1, [o(deint_shuf)]
+ pshufb m4, m1
+%else
+ pshufb m4, [o(deint_shuf)]
+%endif
+ psubsw m1, m2, m3 ; tmp3 tmp2
+ paddsw m3, m2 ; tmp0 tmp1
+ punpckhqdq m2, m4, m0 ; t7 t6
+ punpcklqdq m4, m0 ; t4 t5
+ paddsw m0, m3, m2 ; out0 out1
+ psubsw m3, m2 ; out7 out6
+ psubsw m2, m1, m4 ; out4 out5
+ paddsw m1, m4 ; out3 out2
+%endmacro
+
+%macro IADST8_1D_PACKED 1 ; pass
+ vpbroadcastd m6, [o(pd_2048)]
+%if %1 == 1
+ ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076, 3 ; t1a t0a
+ ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612, 2 ; t2a t3a
+ ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598, 3 ; t5a t4a
+ ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189, 2 ; t6a t7a
+ psubsw m4, m0, m2 ; t5 t4
+ paddsw m0, m2 ; t1 t0
+ psubsw m5, m1, m3 ; t6 t7
+ paddsw m1, m3 ; t2 t3
+ ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784, 3 ; t5a t4a
+ ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 2 ; t7a t6a
+%if mmsize > 16
+ vbroadcasti32x4 m2, [o(deint_shuf)]
+%else
+ mova m2, [o(deint_shuf)]
+%endif
+ vprord m1, 16
+ psubsw m3, m0, m1 ; t3 t2
+ paddsw m0, m1 ; -out7 out0
+ psubsw m1, m4, m5 ; t7 t6
+ paddsw m4, m5 ; out6 -out1
+ pshufb m0, m2
+ pshufb m4, m2
+ mova m2, m6
+ vpdpwssd m2, m3, [o(pw_m2896_2896)] {bcstd}
+ mova m5, m6
+ vpdpwssd m5, m1, [o(pw_m2896_2896)] {bcstd}
+ psrad m2, 12
+ psrad m5, 12
+ packssdw m2, m5 ; out4 -out5
+ mova m5, m6
+ vpdpwssd m5, m3, [o(pw_2896_2896)] {bcstd}
+ mova m3, m6
+ vpdpwssd m3, m1, [o(pw_2896_2896)] {bcstd}
+ psrad m5, 12
+ psrad m3, 12
+ packssdw m1, m3, m5 ; out2 -out3
+%else
+ punpckhwd m0, m4, m3 ; 0 7
+ punpckhwd m1, m5, m2 ; 2 5
+ punpcklwd m2, m5 ; 4 3
+ punpcklwd m3, m4 ; 6 1
+ ITX_MUL2X_PACK 0, 4, 5, 6, 401, 4076 ; t0a t1a
+ ITX_MUL2X_PACK 1, 4, 5, 6, 1931, 3612 ; t2a t3a
+ ITX_MUL2X_PACK 2, 4, 5, 6, 3166, 2598 ; t4a t5a
+ ITX_MUL2X_PACK 3, 4, 5, 6, 3920, 1189 ; t6a t7a
+ psubsw m4, m0, m2 ; t4 t5
+ paddsw m0, m2 ; t0 t1
+ psubsw m5, m1, m3 ; t6 t7
+ paddsw m1, m3 ; t2 t3
+ shufps m2, m5, m4, q1032
+ punpckhwd m4, m2
+ punpcklwd m5, m2
+ ITX_MUL2X_PACK 4, 2, 3, 6, 1567, 3784 ; t4a t5a
+ ITX_MUL2X_PACK 5, 2, 3, 6, 3784, 1567, 1 ; t6a t7a
+ psubsw m2, m0, m1 ; t2 t3
+ paddsw m0, m1 ; out0 -out7
+ psubsw m1, m4, m5 ; t6 t7
+ paddsw m4, m5 ; -out1 out6
+ vpbroadcastd m5, [o(pw_2896x8)]
+ punpckhqdq m3, m2, m1 ; t3 t7
+ punpcklqdq m2, m1 ; t2 t6
+ paddsw m1, m2, m3 ; t2+t3 t6+t7
+ psubsw m2, m3 ; t2-t3 t6-t7
+ punpckhqdq m3, m4, m0 ; out6 -out7
+ punpcklqdq m0, m4 ; out0 -out1
+ pmulhrsw m2, m5 ; out4 -out5
+ pshufd m1, m1, q1032
+ pmulhrsw m1, m5 ; out2 -out3
+%endif
+%endmacro
+
+INIT_YMM avx512icl
+INV_TXFM_4X8_FN dct, dct
+INV_TXFM_4X8_FN dct, identity
+INV_TXFM_4X8_FN dct, adst
+INV_TXFM_4X8_FN dct, flipadst
+
+cglobal idct_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m0, [cq+32*0], q3120
+ vpermq m1, [cq+32*1], q3120
+ vpbroadcastd m2, [o(pw_2896x8)]
+ pmulhrsw m0, m2
+ pmulhrsw m1, m2
+ IDCT4_1D_PACKED
+ vbroadcasti32x4 m2, [o(deint_shuf)]
+ shufps m3, m0, m1, q1331
+ shufps m0, m0, m1, q0220
+ pshufb m0, m2
+ pshufb m1, m3, m2
+ jmp tx2q
+.pass2:
+ vextracti32x4 xm2, m0, 1
+ vextracti32x4 xm3, m1, 1
+ call .main
+ vpbroadcastd m4, [o(pw_2048)]
+ vinserti32x4 m0, m0, xm2, 1
+ vinserti32x4 m1, m1, xm3, 1
+ pshufd m1, m1, q1032
+ jmp m(iadst_4x8_internal_8bpc).end2
+ALIGN function_align
+.main:
+ WRAP_XMM IDCT8_1D_PACKED
+ ret
+
+INV_TXFM_4X8_FN adst, dct
+INV_TXFM_4X8_FN adst, adst
+INV_TXFM_4X8_FN adst, flipadst
+INV_TXFM_4X8_FN adst, identity
+
+cglobal iadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m0, [cq+32*0], q3120
+ vpermq m1, [cq+32*1], q3120
+ vpbroadcastd m2, [o(pw_2896x8)]
+ pmulhrsw m0, m2
+ pmulhrsw m1, m2
+ call m(iadst_8x4_internal_8bpc).main
+ punpckhwd m3, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m0, m3
+ punpcklwd m0, m3
+ jmp tx2q
+.pass2:
+ vextracti32x4 xm2, m0, 1
+ vextracti32x4 xm3, m1, 1
+ pshufd xm4, xm0, q1032
+ pshufd xm5, xm1, q1032
+ call .main_pass2
+ vpbroadcastd m4, [o(pw_2048)]
+ vinserti32x4 m0, xm2, 1
+ vinserti32x4 m1, xm3, 1
+ pxor m5, m5
+ psubw m5, m4
+.end:
+ punpcklqdq m4, m5
+.end2:
+ pmulhrsw m0, m4
+ pmulhrsw m1, m4
+.end3:
+ vpbroadcastd m3, strided
+ pmulld m5, m3, [o(pd_0to15)]
+ kxnorb k1, k1, k1
+ kmovb k2, k1
+ vpgatherdd m3{k1}, [dstq+m5]
+ pxor m4, m4
+ mova [cq], zmm20
+ punpcklbw m2, m3, m4
+ punpckhbw m3, m4
+ paddw m0, m2
+ paddw m1, m3
+ packuswb m0, m1
+ vpscatterdd [dstq+m5]{k2}, m0
+ RET
+ALIGN function_align
+.main_pass1:
+ punpckhwd xm0, xm4, xm3 ; 0 7
+ punpckhwd xm1, xm5, xm2 ; 2 5
+ punpcklwd xm2, xm5 ; 4 3
+ punpcklwd xm3, xm4 ; 6 1
+ WRAP_XMM IADST8_1D_PACKED 1
+ punpcklqdq xm3, xm4, xm0 ; out6 -out7
+ punpckhqdq xm0, xm4 ; out0 -out1
+ ret
+ALIGN function_align
+.main_pass2:
+ WRAP_XMM IADST8_1D_PACKED 2
+ ret
+
+INV_TXFM_4X8_FN flipadst, dct
+INV_TXFM_4X8_FN flipadst, adst
+INV_TXFM_4X8_FN flipadst, flipadst
+INV_TXFM_4X8_FN flipadst, identity
+
+cglobal iflipadst_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m0, [cq+32*0], q3120
+ vpermq m1, [cq+32*1], q3120
+ vpbroadcastd m2, [o(pw_2896x8)]
+ pmulhrsw m0, m2
+ pmulhrsw m1, m2
+ call m(iadst_8x4_internal_8bpc).main
+ punpcklwd m3, m1, m0
+ punpckhwd m1, m0
+ punpcklwd m0, m1, m3
+ punpckhwd m1, m3
+ jmp tx2q
+.pass2:
+ vextracti32x4 xm2, m0, 1
+ vextracti32x4 xm3, m1, 1
+ pshufd xm4, xm0, q1032
+ pshufd xm5, xm1, q1032
+ call m(iadst_4x8_internal_8bpc).main_pass2
+ vpbroadcastd m5, [o(pw_2048)]
+ vinserti32x4 m3, xm1, 1
+ vinserti32x4 m2, xm0, 1
+ pxor m4, m4
+ psubw m4, m5
+ pshufd m0, m3, q1032
+ pshufd m1, m2, q1032
+ jmp m(iadst_4x8_internal_8bpc).end
+
+INIT_ZMM avx512icl
+INV_TXFM_4X8_FN identity, dct
+INV_TXFM_4X8_FN identity, adst
+INV_TXFM_4X8_FN identity, flipadst
+INV_TXFM_4X8_FN identity, identity
+
+cglobal iidentity_4x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd m0, [o(pw_2896x8)]
+ pmulhrsw m0, [cq]
+ mova m1, [o(int8_permB)]
+ vpbroadcastd m2, [o(pw_1697x8)]
+ vpermb m0, m1, m0
+ pmulhrsw m2, m0
+ paddsw m0, m2
+ vextracti32x8 ym1, m0, 1
+ jmp tx2q
+.pass2:
+ vpbroadcastd ym4, [o(pw_4096)]
+ jmp m(iadst_4x8_internal_8bpc).end2
+
+%macro INV_TXFM_4X16_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 4x16
+%ifidn %1_%2, dct_dct
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+ imul r6d, 181
+ add r6d, 128+2048
+ sar r6d, 8+4
+ vpbroadcastw m0, r6d
+ mova m1, m0
+ jmp m(iadst_4x16_internal_8bpc).end3
+%endif
+%endmacro
+
+%macro IDCT16_1D_PACKED 0
+ punpckhwd m8, m7, m0 ; dct16 in15 in1
+ punpcklwd m9, m4, m0 ; dct4 in2 in0
+ punpckhwd m0, m3, m4 ; dct16 in7 in9
+ punpcklwd m7, m1 ; dct8 in7 in1
+ punpckhwd m1, m6 ; dct16 in3 in13
+ punpcklwd m3, m5 ; dct8 in3 in5
+ punpckhwd m5, m2 ; dct16 in11 in5
+ punpcklwd m6, m2 ; dct4 in3 in1
+cglobal_label .main2
+ vpbroadcastd m10, [o(pd_2048)]
+.main3:
+ vpbroadcastq m13, [o(int_mshift)]
+ vpcmpub k7, m13, m10, 6 ; 0x33...
+ ITX_MUL2X_PACK 8, 2, 4, 10, 401, 4076, 5 ; t8a t15a
+ ITX_MUL2X_PACK 0, 2, 4, 10, 3166, 2598, 5 ; t9a t14a
+ ITX_MUL2X_PACK 5, 2, 4, 10, 1931, 3612, 5 ; t10a t13a
+ ITX_MUL2X_PACK 1, 2, 4, 10, 3920, 1189, 5 ; t11a t12a
+ ITX_MUL2X_PACK 7, 2, 4, 10, 799, 4017, 5 ; t4a t7a
+ ITX_MUL2X_PACK 3, 2, 4, 10, 3406, 2276, 5 ; t5a t6a
+.main4:
+ psubsw m2, m8, m0 ; t9 t14
+ paddsw m8, m0 ; t8 t15
+ psubsw m4, m1, m5 ; t10 t13
+ paddsw m1, m5 ; t11 t12
+ ITX_MUL2X_PACK 6, 0, 5, 10, 1567, 3784 ; t3 t2
+ psubsw m0, m8, m1 ; t11a t12a
+ paddsw m8, m1 ; t8a t15a
+ psubsw m1, m7, m3 ; t5a t6a
+ paddsw m7, m3 ; t4 t7
+.main5:
+ ITX_MUL2X_PACK 2, 3, 5, 10, 1567, 3784, 5 ; t9a t14a
+ ITX_MUL2X_PACK 4, 3, 5, 10, m3784, 1567, 5 ; t10a t13a
+%if mmsize > 16
+ vbroadcasti32x4 m5, [o(deint_shuf)]
+%else
+ mova m5, [o(deint_shuf)]
+%endif
+ vpbroadcastd m11, [o(pw_m2896_2896)]
+ vpbroadcastd m12, [o(pw_2896_2896)]
+ paddsw m3, m2, m4 ; t9 t14
+ psubsw m2, m4 ; t10 t13
+ pshufb m8, m5
+ pshufb m7, m5
+ pshufb m3, m5
+ ITX_MUL2X_PACK 9, 4, 5, 10, 11, 12 ; t0 t1
+ ITX_MUL2X_PACK 1, 4, 5, 10, 12, 11 ; t5 t6
+ ITX_MUL2X_PACK 0, 4, 5, 10, 11, 12, 8 ; t11 t12
+ ITX_MUL2X_PACK 2, 0, 11, 10, 11, 12, 8 ; t10a t13a
+ punpckhqdq m2, m7, m1 ; t7 t6
+ punpcklqdq m7, m1 ; t4 t5
+ psubsw m1, m9, m6 ; dct4 out3 out2
+ paddsw m9, m6 ; dct4 out0 out1
+ packssdw m5, m11 ; t12 t13a
+ packssdw m4, m0 ; t11 t10a
+ punpckhqdq m0, m8, m3 ; t15a t14
+ punpcklqdq m8, m3 ; t8a t9
+ psubsw m3, m9, m2 ; dct8 out7 out6
+ paddsw m9, m2 ; dct8 out0 out1
+ psubsw m2, m1, m7 ; dct8 out4 out5
+ paddsw m1, m7 ; dct8 out3 out2
+ psubsw m7, m9, m0 ; out15 out14
+ paddsw m0, m9 ; out0 out1
+ psubsw m6, m1, m5 ; out12 out13
+ paddsw m1, m5 ; out3 out2
+ psubsw m5, m2, m4 ; out11 out10
+ paddsw m2, m4 ; out4 out5
+ psubsw m4, m3, m8 ; out8 out9
+ paddsw m3, m8 ; out7 out6
+%endmacro
+
+INV_TXFM_4X16_FN dct, dct
+INV_TXFM_4X16_FN dct, identity
+INV_TXFM_4X16_FN dct, adst
+INV_TXFM_4X16_FN dct, flipadst
+
+cglobal idct_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova ym1, [cq+32*2]
+ vinserti32x8 m1, [cq+32*0], 1
+ mova m0, [o(int16_perm)]
+ mova ym2, [cq+32*3]
+ vinserti32x8 m2, [cq+32*1], 1
+ vpbroadcastd m4, [o(pd_2048)]
+ vpermb m1, m0, m1 ; c0 a0 c1 a1 c2 a2 c3 a3
+ vpermb m2, m0, m2 ; d0 b0 d1 b1 d2 b2 d3 b3
+ ITX_MUL2X_PACK 1, 0, 3, 4, 2896, 2896, 2
+ ITX_MUL2X_PACK 2, 0, 3, 4, 1567, 3784, 2
+ vpbroadcastd m4, [o(pw_16384)]
+ psubsw m3, m1, m2
+ paddsw m1, m2 ; out0 out1
+ vprord m3, 16 ; out2 out3
+ punpckldq m0, m1, m3
+ punpckhdq m1, m3
+ pmulhrsw m0, m4
+ pmulhrsw m1, m4
+ jmp tx2q
+.pass2:
+ vextracti32x4 xm2, ym0, 1
+ vextracti32x4 xm3, ym1, 1
+ vextracti32x4 xm4, m0, 2
+ vextracti32x4 xm5, m1, 2
+ vextracti32x4 xm6, m0, 3
+ vextracti32x4 xm7, m1, 3
+ call .main
+ vinserti32x4 ym0, xm2, 1
+ vinserti32x4 ym1, xm3, 1
+ vinserti32x4 ym4, xm6, 1
+ vinserti32x4 ym5, xm7, 1
+ vinserti32x8 m0, ym4, 1
+ vinserti32x8 m1, ym5, 1
+ vpbroadcastd m5, [o(pw_2048)]
+ pshufd m1, m1, q1032
+ jmp m(iadst_4x16_internal_8bpc).end2
+ALIGN function_align
+.main:
+ WRAP_XMM IDCT16_1D_PACKED
+ ret
+
+INV_TXFM_4X16_FN adst, dct
+INV_TXFM_4X16_FN adst, adst
+INV_TXFM_4X16_FN adst, flipadst
+INV_TXFM_4X16_FN adst, identity
+
+cglobal iadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m1, [o(permB)]
+ vpermq m0, m1, [cq+64*0]
+ vpermq m1, m1, [cq+64*1]
+ call m(iadst_16x4_internal_8bpc).main
+ vpbroadcastd m3, [o(pw_16384)]
+ punpckhwd m2, m0, m1
+ punpcklwd m0, m1
+ pmulhrsw m2, m3
+ pmulhrsw m0, m3
+ punpckhwd m1, m0, m2
+ punpcklwd m0, m2
+ jmp tx2q
+.pass2:
+ call .main
+ vpbroadcastd m5, [o(pw_2048)]
+ psrlq m10, 4
+ psubw m6, m8, m5
+.end:
+ vpbroadcastd m7, [o(pw_2896x8)]
+ paddsw ym1, ym2, ym4
+ psubsw ym2, ym4
+ vinserti32x8 m1, ym2, 1
+ pmulhrsw m1, m7 ; -out7 out4 out6 -out5 out8 -out11 -out9 out10
+ psrlq m0, m10, 4
+ vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d
+ vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f
+ punpcklqdq m5, m6
+.end2:
+ pmulhrsw m0, m5
+ pmulhrsw m1, m5
+.end3:
+ vpbroadcastd m3, strided
+ pmulld m5, m3, [o(pd_0to15)]
+ kxnorw k1, k1, k1
+ kmovw k2, k1
+ vpgatherdd m3{k1}, [dstq+m5]
+ pxor m4, m4
+ mova [cq+64*0], m4
+ mova [cq+64*1], m4
+ punpcklbw m2, m3, m4
+ punpckhbw m3, m4
+ paddw m0, m2
+ paddw m1, m3
+ packuswb m0, m1
+ vpscatterdd [dstq+m5]{k2}, m0
+ RET
+ALIGN function_align
+.main:
+ movu m3, [o(permB+1)]
+ psrlq m10, m3, 4
+.main2:
+ vpermi2q m3, m0, m1 ; in15 in12 in13 in14 in11 in8 in9 in10
+ vpermt2q m0, m10, m1 ; in0 in3 in2 in1 in4 in7 in6 in5
+ vpbroadcastd m9, [o(pd_2048)]
+ vpbroadcastq ym13, [o(int_mshift)]
+ kxnorb k1, k1, k1
+ punpckhwd m4, m3, m0 ; in12 in3 in14 in1
+ punpcklwd m0, m3 ; in0 in15 in2 in13
+ kshiftrb k1, k1, 4
+ vextracti32x8 ym3, m4, 1 ; in8 in7 in10 in5
+ vextracti32x8 ym1, m0, 1 ; in4 in11 in6 in9
+INIT_YMM avx512icl
+ vpcmpub k7, m13, m9, 6 ; 0x33...
+ pxor m8, m8
+ ITX_MUL4X_PACK 0, 2, 5, 6, 7, 9, 201, 4091, 995, 3973, 5
+ ITX_MUL4X_PACK 1, 2, 5, 6, 7, 9, 1751, 3703, 2440, 3290, 5
+ ITX_MUL4X_PACK 3, 2, 5, 6, 7, 9, 3035, 2751, 3513, 2106, 5
+ ITX_MUL4X_PACK 4, 2, 5, 6, 7, 9, 3857, 1380, 4052, 601, 5
+ psubsw m2, m0, m3 ; t9a t8a t11a t10a
+ paddsw m0, m3 ; t1a t0a t3a t2a
+ psubsw m3, m1, m4 ; t13a t12a t15a t14a
+ paddsw m4, m1 ; t5a t4a t7a t6a
+ ITX_MUL4X_PACK 2, 1, 5, 6, 7, 9, 799, 4017, 3406, 2276, 5
+ psubw m7, m8, m7
+ ITX_MUL2X_PACK 3, 1, 5, 9, 7, 6, 4
+ vpbroadcastd m6, [o(pw_3784_m1567)]
+ vpbroadcastd m6{k1}, [o(pw_m3784_1567)]
+ psubsw m1, m0, m4 ; t5 t4 t7 t6
+ paddsw m0, m4 ; t1 t0 t3 t2
+ psubsw m4, m2, m3 ; t13a t12a t15a t14a
+ paddsw m2, m3 ; t9a t8a t11a t10a
+ ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t4a t5a t7a t6a
+ ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t12 t13 t15 t14
+ vbroadcasti32x4 m5, [o(deint_shuf)]
+ pshufb m0, m5
+ pshufb m2, m5
+ vshufi32x4 m3, m0, m2, 0x03 ; t3 t2 t11a t10a
+ vinserti32x4 m0, xm2, 1 ; t1 t0 t9a t8a
+ vshufi32x4 m2, m1, m4, 0x03 ; t7a t6a t15 t14
+ vinserti32x4 m1, xm4, 1 ; t4a t5a t12 t13
+ pshufd m2, m2, q1032 ; t6a t7a t14 t15
+ psubsw m4, m0, m3 ; t3a t2a t11 t10
+ paddsw m0, m3 ; -out15 out0 out14 -out1
+ paddsw m3, m1, m2 ; out12 -out3 -out13 out2
+ psubsw m1, m2 ; t7 t6 t15a t14a
+ punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a
+ punpcklqdq m4, m1 ; t3a t7 t11 t15a
+INIT_ZMM avx512icl
+ vinserti32x8 m3, ym0, 1 ; out12 -out3 -out13 out2 -out15 out0 out14 -out1
+ ret
+
+INV_TXFM_4X16_FN flipadst, dct
+INV_TXFM_4X16_FN flipadst, adst
+INV_TXFM_4X16_FN flipadst, flipadst
+INV_TXFM_4X16_FN flipadst, identity
+
+cglobal iflipadst_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m1, [o(permB)]
+ vpermq m0, m1, [cq+64*0]
+ vpermq m1, m1, [cq+64*1]
+ call m(iadst_16x4_internal_8bpc).main
+ vpbroadcastd m3, [o(pw_16384)]
+ punpcklwd m2, m1, m0
+ punpckhwd m1, m0
+ pmulhrsw m2, m3
+ pmulhrsw m1, m3
+ punpcklwd m0, m1, m2
+ punpckhwd m1, m2
+ jmp tx2q
+.pass2:
+ call m(iadst_4x16_internal_8bpc).main
+ vpbroadcastd m6, [o(pw_2048)]
+ psrlq m10, 12
+ psubw m5, m8, m6
+ jmp m(iadst_4x16_internal_8bpc).end
+
+INV_TXFM_4X16_FN identity, dct
+INV_TXFM_4X16_FN identity, adst
+INV_TXFM_4X16_FN identity, flipadst
+INV_TXFM_4X16_FN identity, identity
+
+cglobal iidentity_4x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m2, [o(int16_perm)]
+ vpermb m1, m2, [cq+64*0]
+ vpermb m2, m2, [cq+64*1]
+ vpbroadcastd m4, [o(pw_1697x8)]
+ vpbroadcastd m0, [o(pd_m1)]
+ pmulhrsw m3, m4, m1 ; we want to do a signed avg, but pavgw is
+ vpcmpw k1, m1, m0, 4 ; unsigned. as long as both signs are equal
+ pmulhrsw m4, m2 ; it still works, but if the input is -1 the
+ vpcmpw k2, m2, m0, 4 ; pmulhrsw result will become 0 which causes
+ vpavgw m1{k1}{z}, m3 ; pavgw to output -32768 instead of 0 unless
+ vpavgw m2{k2}{z}, m4 ; we explicitly deal with that case here.
+ punpckldq m0, m1, m2
+ punpckhdq m1, m2
+ jmp tx2q
+.pass2:
+ vpbroadcastd m3, [o(pw_1697x16)]
+ vpbroadcastd m5, [o(pw_2048)]
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ paddsw m0, m0
+ paddsw m1, m1
+ paddsw m0, m2
+ paddsw m1, m3
+ jmp m(iadst_4x16_internal_8bpc).end2
+
+%macro WRITE_8X4 4-7 strideq*1, strideq*2, r6 ; coefs[1-2], tmp[1-2], off[1-3]
+ movq xm%3, [dstq ]
+ movhps xm%3, [dstq+%5]
+ movq xm%4, [dstq+%6]
+ movhps xm%4, [dstq+%7]
+ pmovzxbw m%3, xm%3
+ pmovzxbw m%4, xm%4
+%ifnum %1
+ paddw m%3, m%1
+%else
+ paddw m%3, %1
+%endif
+%ifnum %2
+ paddw m%4, m%2
+%else
+ paddw m%4, %2
+%endif
+ packuswb m%3, m%4
+ vextracti32x4 xm%4, m%3, 1
+ movq [dstq ], xm%3
+ movhps [dstq+%6], xm%3
+ movq [dstq+%5], xm%4
+ movhps [dstq+%7], xm%4
+%endmacro
+
+%macro INV_TXFM_8X4_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 8x4
+%ifidn %1_%2, dct_dct
+ movd xm1, [o(pw_2896x8)]
+ pmulhrsw xm0, xm1, [cq]
+ movd xm2, [o(pw_2048)]
+ pmulhrsw xm0, xm1
+ pmulhrsw xm0, xm1
+ pmulhrsw xm0, xm2
+ vpbroadcastw m0, xm0
+ mova m1, m0
+ jmp m(iadst_8x4_internal_8bpc).end3
+%endif
+%endmacro
+
+INIT_YMM avx512icl
+INV_TXFM_8X4_FN dct, dct
+INV_TXFM_8X4_FN dct, adst
+INV_TXFM_8X4_FN dct, flipadst
+INV_TXFM_8X4_FN dct, identity
+
+cglobal idct_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd xm3, [o(pw_2896x8)]
+ pmulhrsw xm0, xm3, [cq+16*0]
+ pmulhrsw xm1, xm3, [cq+16*1]
+ pmulhrsw xm2, xm3, [cq+16*2]
+ pmulhrsw xm3, [cq+16*3]
+ call m(idct_4x8_internal_8bpc).main
+ vbroadcasti32x4 m4, [o(deint_shuf)]
+ vinserti32x4 m3, m1, xm3, 1
+ vinserti32x4 m1, m0, xm2, 1
+ shufps m0, m1, m3, q0220
+ shufps m1, m3, q1331
+ pshufb m0, m4
+ pshufb m1, m4
+ jmp tx2q
+.pass2:
+ IDCT4_1D_PACKED
+ vpermq m0, m0, q3120
+ vpermq m1, m1, q2031
+ jmp m(iadst_8x4_internal_8bpc).end2
+
+INV_TXFM_8X4_FN adst, dct
+INV_TXFM_8X4_FN adst, adst
+INV_TXFM_8X4_FN adst, flipadst
+INV_TXFM_8X4_FN adst, identity
+
+cglobal iadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd xm0, [o(pw_2896x8)]
+ pshufd xm4, [cq+16*0], q1032
+ pmulhrsw xm3, xm0, [cq+16*3]
+ pshufd xm5, [cq+16*1], q1032
+ pmulhrsw xm2, xm0, [cq+16*2]
+ pmulhrsw xm4, xm0
+ pmulhrsw xm5, xm0
+ call m(iadst_4x8_internal_8bpc).main_pass1
+ vinserti32x4 m0, xm2, 1
+ vinserti32x4 m1, xm3, 1
+ pxor m3, m3
+ punpckhwd m2, m0, m1
+ punpcklwd m0, m1
+ psubsw m3, m2
+ punpckhwd m1, m0, m3
+ punpcklwd m0, m3
+ jmp tx2q
+.pass2:
+ call .main
+.end:
+ vpermq m0, m0, q3120
+ vpermq m1, m1, q3120
+.end2:
+ vpbroadcastd m2, [o(pw_2048)]
+ pmulhrsw m0, m2
+ pmulhrsw m1, m2
+.end3:
+ pxor m2, m2
+ mova [cq], zmm18
+ lea r6, [strideq*3]
+ WRITE_8X4 0, 1, 4, 5
+ RET
+ALIGN function_align
+.main:
+ IADST4_1D_PACKED
+ ret
+
+INV_TXFM_8X4_FN flipadst, dct
+INV_TXFM_8X4_FN flipadst, adst
+INV_TXFM_8X4_FN flipadst, flipadst
+INV_TXFM_8X4_FN flipadst, identity
+
+cglobal iflipadst_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd xm0, [o(pw_2896x8)]
+ pshufd xm4, [cq+16*0], q1032
+ pmulhrsw xm3, xm0, [cq+16*3]
+ pshufd xm5, [cq+16*1], q1032
+ pmulhrsw xm2, xm0, [cq+16*2]
+ pmulhrsw xm4, xm0
+ pmulhrsw xm5, xm0
+ call m(iadst_4x8_internal_8bpc).main_pass1
+ vinserti32x4 m3, m3, xm1, 1
+ vinserti32x4 m2, m2, xm0, 1
+ punpckhwd m1, m3, m2
+ punpcklwd m3, m2
+ pxor m0, m0
+ psubsw m0, m1
+ punpckhwd m1, m0, m3
+ punpcklwd m0, m3
+ jmp tx2q
+.pass2:
+ call m(iadst_8x4_internal_8bpc).main
+ mova m2, m1
+ vpermq m1, m0, q2031
+ vpermq m0, m2, q2031
+ jmp m(iadst_8x4_internal_8bpc).end2
+
+INV_TXFM_8X4_FN identity, dct
+INV_TXFM_8X4_FN identity, adst
+INV_TXFM_8X4_FN identity, flipadst
+INV_TXFM_8X4_FN identity, identity
+
+cglobal iidentity_8x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova xm2, [cq+16*0]
+ mova xm0, [cq+16*1]
+ vinserti32x4 m2, [cq+16*2], 1
+ vinserti32x4 m0, [cq+16*3], 1
+ vpbroadcastd m3, [o(pw_2896x8)]
+ punpcklwd m1, m2, m0
+ punpckhwd m2, m0
+ pmulhrsw m1, m3
+ pmulhrsw m2, m3
+ punpcklwd m0, m1, m2
+ punpckhwd m1, m2
+ paddsw m0, m0
+ paddsw m1, m1
+ jmp tx2q
+.pass2:
+ vpbroadcastd m3, [o(pw_1697x8)]
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ paddsw m0, m2
+ paddsw m1, m3
+ jmp m(iadst_8x4_internal_8bpc).end
+
+%macro INV_TXFM_8X8_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 8x8
+%ifidn %1_%2, dct_dct
+INIT_ZMM avx512icl
+ movsx r6d, word [cq]
+ mov [cq], eobd
+.dconly:
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+.dconly2:
+ vpbroadcastd ym2, strided
+ imul r6d, 181
+ pmulld ym5, ym2, [o(pd_0to15)]
+ kxnorb k1, k1, k1
+ add r6d, 128+2048
+ sar r6d, 8+4
+ pxor m3, m3
+ vpbroadcastw m4, r6d
+.dconly_loop:
+ kmovb k2, k1
+ vpgatherdq m2{k1}, [dstq+ym5]
+ punpcklbw m0, m2, m3
+ punpckhbw m1, m2, m3
+ paddw m0, m4
+ paddw m1, m4
+ packuswb m0, m1
+ kmovb k1, k2
+ vpscatterdq [dstq+ym5]{k2}, m0
+ lea dstq, [dstq+strideq*8]
+ sub r3d, 8
+ jg .dconly_loop
+ RET
+INIT_YMM avx512icl
+%endif
+%endmacro
+
+INV_TXFM_8X8_FN dct, dct
+INV_TXFM_8X8_FN dct, identity
+INV_TXFM_8X8_FN dct, adst
+INV_TXFM_8X8_FN dct, flipadst
+
+cglobal idct_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m0, [cq+32*0], q3120 ; 0 1
+ vpermq m3, [cq+32*3], q3120 ; 6 7
+ vpermq m2, [cq+32*2], q3120 ; 4 5
+ vpermq m1, [cq+32*1], q3120 ; 2 3
+ call .main
+ shufps m4, m0, m1, q0220
+ shufps m5, m0, m1, q1331
+ shufps m1, m2, m3, q0220
+ shufps m3, m2, m3, q1331
+ vbroadcasti32x4 m0, [o(deint_shuf)]
+ vpbroadcastd m2, [o(pw_16384)]
+ REPX {pshufb x, m0}, m4, m5, m1, m3
+ REPX {pmulhrsw x, m2}, m4, m5, m1, m3
+ vinserti32x4 m0, m4, xm1, 1
+ vshufi32x4 m2, m4, m1, 0x03
+ vinserti32x4 m1, m5, xm3, 1
+ vshufi32x4 m3, m5, m3, 0x03
+ jmp tx2q
+.pass2:
+ call .main
+ vpbroadcastd m4, [o(pw_2048)]
+ vpermq m0, m0, q3120
+ vpermq m1, m1, q2031
+ vpermq m2, m2, q3120
+ vpermq m3, m3, q2031
+ jmp m(iadst_8x8_internal_8bpc).end2
+ALIGN function_align
+cglobal_label .main
+ IDCT8_1D_PACKED
+ ret
+
+INV_TXFM_8X8_FN adst, dct
+INV_TXFM_8X8_FN adst, adst
+INV_TXFM_8X8_FN adst, flipadst
+INV_TXFM_8X8_FN adst, identity
+
+cglobal iadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m4, [cq+32*0], q1302 ; 1 0
+ vpermq m3, [cq+32*3], q3120 ; 6 7
+ vpermq m5, [cq+32*1], q1302 ; 3 2
+ vpermq m2, [cq+32*2], q3120 ; 4 5
+ call .main_pass1
+ vpbroadcastd m5, [o(pw_16384_m16384)]
+ punpcklwd m4, m0, m1
+ punpckhwd m0, m1
+ punpcklwd m1, m2, m3
+ punpckhwd m2, m3
+ punpcklwd m3, m4, m0
+ punpckhwd m4, m0
+ punpcklwd m0, m1, m2
+ punpckhwd m1, m2
+ REPX {pmulhrsw x, m5}, m3, m4, m0, m1
+ vshufi32x4 m2, m3, m0, 0x03
+ vinserti32x4 m0, m3, xm0, 1
+ vshufi32x4 m3, m4, m1, 0x03
+ vinserti32x4 m1, m4, xm1, 1
+ jmp tx2q
+.pass2:
+ pshufd m4, m0, q1032
+ pshufd m5, m1, q1032
+ call .main_pass2
+ vpbroadcastd m5, [o(pw_2048)]
+ vpbroadcastd xm4, [o(pw_4096)]
+ psubw m4, m5 ; lower half = 2048, upper half = -2048
+.end:
+ REPX {vpermq x, x, q3120}, m0, m1, m2, m3
+.end2:
+ pmulhrsw m0, m4
+ pmulhrsw m1, m4
+.end3:
+ pmulhrsw m2, m4
+ pmulhrsw m3, m4
+.end4:
+ pxor m4, m4
+ mova [cq+32*0], m4
+ mova [cq+32*1], m4
+ mova [cq+32*2], m4
+ mova [cq+32*3], m4
+ lea r6, [strideq*3]
+ WRITE_8X4 0, 1, 4, 5
+ lea dstq, [dstq+strideq*4]
+ WRITE_8X4 2, 3, 4, 5
+ RET
+ALIGN function_align
+.main_pass1:
+ punpckhwd m0, m4, m3 ; 0 7
+ punpckhwd m1, m5, m2 ; 2 5
+ punpcklwd m2, m5 ; 4 3
+ punpcklwd m3, m4 ; 6 1
+ IADST8_1D_PACKED 1
+ punpcklqdq m3, m4, m0 ; out6 -out7
+ punpckhqdq m0, m4 ; out0 -out1
+ ret
+ALIGN function_align
+cglobal_label .main_pass2
+ IADST8_1D_PACKED 2
+ ret
+
+INV_TXFM_8X8_FN flipadst, dct
+INV_TXFM_8X8_FN flipadst, adst
+INV_TXFM_8X8_FN flipadst, flipadst
+INV_TXFM_8X8_FN flipadst, identity
+
+cglobal iflipadst_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpermq m4, [cq+32*0], q1302 ; 1 0
+ vpermq m3, [cq+32*3], q3120 ; 6 7
+ vpermq m5, [cq+32*1], q1302 ; 3 2
+ vpermq m2, [cq+32*2], q3120 ; 4 5
+ call m(iadst_8x8_internal_8bpc).main_pass1
+ vpbroadcastd m5, [o(pw_m16384_16384)]
+ punpckhwd m4, m3, m2
+ punpcklwd m3, m2
+ punpckhwd m2, m1, m0
+ punpcklwd m1, m0
+ punpckhwd m0, m4, m3
+ punpcklwd m4, m3
+ punpckhwd m3, m2, m1
+ punpcklwd m2, m1
+ REPX {pmulhrsw x, m5}, m0, m4, m3, m2
+ vinserti32x4 m1, m0, xm3, 1
+ vshufi32x4 m3, m0, m3, 0x03
+ vinserti32x4 m0, m4, xm2, 1
+ vshufi32x4 m2, m4, m2, 0x03
+ jmp tx2q
+.pass2:
+ pshufd m4, m0, q1032
+ pshufd m5, m1, q1032
+ call m(iadst_8x8_internal_8bpc).main_pass2
+ vpbroadcastd m4, [o(pw_2048)]
+ vpbroadcastd xm5, [o(pw_4096)]
+ psubw m4, m5 ; lower half = -2048, upper half = 2048
+ vpermq m5, m3, q2031
+ vpermq m3, m0, q2031
+ vpermq m0, m2, q2031
+ vpermq m2, m1, q2031
+ pmulhrsw m1, m0, m4
+ pmulhrsw m0, m5, m4
+ jmp m(iadst_8x8_internal_8bpc).end3
+
+INV_TXFM_8X8_FN identity, dct
+INV_TXFM_8X8_FN identity, adst
+INV_TXFM_8X8_FN identity, flipadst
+INV_TXFM_8X8_FN identity, identity
+
+cglobal iidentity_8x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova xm3, [cq+16*0]
+ mova xm2, [cq+16*1]
+ vinserti32x4 m3, [cq+16*4], 1
+ vinserti32x4 m2, [cq+16*5], 1
+ mova xm4, [cq+16*2]
+ mova xm0, [cq+16*3]
+ vinserti32x4 m4, [cq+16*6], 1
+ vinserti32x4 m0, [cq+16*7], 1
+ punpcklwd m1, m3, m2
+ punpckhwd m3, m2
+ punpcklwd m2, m4, m0
+ punpckhwd m4, m0
+ punpckldq m0, m1, m2
+ punpckhdq m1, m2
+ punpckldq m2, m3, m4
+ punpckhdq m3, m4
+ jmp tx2q
+.pass2:
+ vpbroadcastd m4, [o(pw_4096)]
+ jmp m(iadst_8x8_internal_8bpc).end
+
+%macro INV_TXFM_8X16_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 8x16
+%ifidn %1_%2, dct_dct
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 16
+ imul r6d, 181
+ add r6d, 128
+ sar r6d, 8
+ jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly
+%endif
+%endmacro
+
+%macro ITX_8X16_LOAD_COEFS 0
+ vpbroadcastd m4, [o(pw_2896x8)]
+ pmulhrsw m0, m4, [cq+32*0]
+ add cq, 32*4
+ pmulhrsw m7, m4, [cq+32*3]
+ pmulhrsw m1, m4, [cq-32*3]
+ pmulhrsw m6, m4, [cq+32*2]
+ pmulhrsw m2, m4, [cq-32*2]
+ pmulhrsw m5, m4, [cq+32*1]
+ pmulhrsw m3, m4, [cq-32*1]
+ pmulhrsw m4, [cq+32*0]
+%endmacro
+
+INIT_ZMM avx512icl
+INV_TXFM_8X16_FN dct, dct
+INV_TXFM_8X16_FN dct, identity
+INV_TXFM_8X16_FN dct, adst
+INV_TXFM_8X16_FN dct, flipadst
+
+cglobal idct_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m3, [o(permB)]
+ vpermq m0, m3, [cq+64*0]
+ vpbroadcastd m4, [o(pw_2896x8)]
+ vpermq m1, m3, [cq+64*1]
+ vpermq m2, m3, [cq+64*2]
+ vpermq m3, m3, [cq+64*3]
+ REPX {pmulhrsw x, m4}, m0, m1, m2, m3
+ call m(idct_16x8_internal_8bpc).main
+ vpbroadcastd m5, [o(pw_16384)]
+ punpckhwd m4, m0, m2 ; b0 f0 b1 f1 b2 f2 b3 f3
+ punpcklwd m0, m2 ; a0 e0 a1 e1 a2 e2 a3 e3
+ punpckhwd m2, m1, m3 ; c0 g0 c1 g1 c2 g2 c3 g3
+ punpcklwd m1, m3 ; d0 h0 d1 h1 d2 h2 d3 h3
+ REPX {pmulhrsw x, m5}, m4, m0, m2, m1
+ punpckhwd m3, m0, m4 ; a2 b2 e2 f2 a3 b3 e3 f3
+ punpcklwd m0, m4 ; a0 b0 e0 f0 a1 b1 e1 f1
+ punpckhwd m4, m2, m1 ; c2 d2 g2 h2 c3 d3 g3 h3
+ punpcklwd m2, m1 ; c0 d0 g0 h0 c1 d1 g1 h1
+ punpckhdq m1, m0, m2 ; 1 5 9 13
+ punpckldq m0, m2 ; 0 4 8 12
+ punpckldq m2, m3, m4 ; 2 6 10 14
+ punpckhdq m3, m4 ; 3 7 11 15
+ jmp tx2q
+.pass2:
+ vprord m5, [o(int16_perm)], 16
+ vshufi32x4 m2, m2, q1320 ; 2 10 14 6
+ vshufi32x4 m4, m1, m3, q2310 ; 1 5 15 11
+ vshufi32x4 m1, m3, q0132 ; 9 13 7 3
+ vpermb m9, m5, m0
+ vpermb m7, m5, m2
+ vpermb m8, m5, m4
+ vpermb m0, m5, m1
+ vextracti32x8 ym6, m9, 1
+ vextracti32x8 ym3, m7, 1
+ vextracti32x8 ym5, m8, 1
+ vextracti32x8 ym1, m0, 1
+ call .main2
+ mova ym8, [o(gather8a)]
+ lea r3, [dstq+strideq*4]
+ pmovzxdq m9, ym8
+ pshufd ym8, ym8, q1230
+ vpermt2q m0, m9, m4
+ vpermt2q m1, m9, m5
+ vpermt2q m2, m9, m6
+ vpermt2q m3, m9, m7
+.end:
+ vpbroadcastd m7, [o(pw_2048)]
+.end2:
+ pmulhrsw m0, m7
+ pmulhrsw m1, m7
+.end3:
+ pmulhrsw m2, m7
+ pmulhrsw m3, m7
+.end4:
+ vpbroadcastd ym6, strided
+ kxnorb k1, k1, k1
+ pxor m4, m4
+ pmulld ym8, ym6
+ kmovb k2, k1
+ vpgatherdq m6{k1}, [dstq+ym8]
+ kmovb k1, k2
+ vpgatherdq m7{k2}, [r3+ym8]
+ mova [cq+64*0], m4
+ mova [cq+64*1], m4
+ kmovb k2, k1
+ mova [cq+64*2], m4
+ mova [cq+64*3], m4
+ punpcklbw m5, m6, m4
+ punpckhbw m6, m4
+ paddw m0, m5
+ paddw m1, m6
+ packuswb m0, m1
+ vpscatterdq [dstq+ym8]{k1}, m0
+ punpcklbw m6, m7, m4
+ punpckhbw m7, m4
+ paddw m2, m6
+ paddw m3, m7
+ packuswb m2, m3
+ vpscatterdq [r3+ym8]{k2}, m2
+ RET
+ALIGN function_align
+cglobal_label .main_fast2 ; bottom three-quarters are zero
+ vpbroadcastd ym10, [o(pd_2048)]
+ vpbroadcastq ym13, [o(int_mshift)]
+ vpbroadcastd ym3, [o(pw_401_4076x8)]
+ vpbroadcastd ym5, [o(pw_799_4017x8)]
+ vpbroadcastd ym4, [o(pw_m1189_3920x8)]
+ pxor ym6, ym6
+ punpckhwd ym2, ym0, ym0
+ pmulhrsw ym2, ym3 ; t8a t15a
+ punpcklwd ym7, ym1, ym1
+ pmulhrsw ym7, ym5 ; t4a t7a
+ punpckhwd ym1, ym1
+ pmulhrsw ym4, ym1 ; t11a t12a
+ vpcmpub k7, ym13, ym10, 6
+ punpcklwd ym9, ym6, ym0
+ psubsw ym0, ym2, ym4 ; t11a t12a
+ paddsw ym8, ym2, ym4 ; t8a t15a
+ mova ym1, ym7
+ jmp .main5
+ALIGN function_align
+cglobal_label .main_fast ; bottom half is zero
+ vpbroadcastd ym10, [o(pd_2048)]
+ vpbroadcastq ym13, [o(int_mshift)]
+ pxor ym6, ym6
+ punpckhwd ym8, ym0, ym0
+ punpckhwd ym4, ym3, ym3
+ punpckhwd ym5, ym2, ym2
+ punpcklwd ym7, ym1, ym1
+ punpckhwd ym1, ym1
+ punpcklwd ym3, ym3
+ punpcklwd ym9, ym6, ym0
+ punpcklwd ym6, ym2
+ vpbroadcastd ym2, [o(pw_401_4076x8)]
+ vpbroadcastd ym0, [o(pw_m2598_3166x8)]
+ vpbroadcastd ym11, [o(pw_1931_3612x8)]
+ vpbroadcastd ym12, [o(pw_m1189_3920x8)]
+ pmulhrsw ym8, ym2 ; t8a t15a
+ vpbroadcastd ym2, [o(pw_799_4017x8)]
+ pmulhrsw ym0, ym4 ; t9a t14a
+ vpbroadcastd ym4, [o(pw_m2276_3406x8)]
+ pmulhrsw ym5, ym11 ; t10a t13a
+ pmulhrsw ym1, ym12 ; t11a t12a
+ pmulhrsw ym7, ym2 ; t4a t7a
+ pmulhrsw ym3, ym4 ; t5a t6a
+ vpcmpub k7, ym13, ym10, 6
+ jmp .main4
+ALIGN function_align
+cglobal_label .main
+ WRAP_YMM IDCT16_1D_PACKED
+ ret
+
+INV_TXFM_8X16_FN adst, dct
+INV_TXFM_8X16_FN adst, adst
+INV_TXFM_8X16_FN adst, flipadst
+INV_TXFM_8X16_FN adst, identity
+
+cglobal iadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call m(iadst_16x8_internal_8bpc).main_pass1
+ vbroadcasti32x4 m6, [o(int_shuf1)]
+ vpbroadcastd m7, [o(pw_16384_m16384)]
+ punpckhwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpcklwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3
+ pshufb m5, m1, m6 ; c0 d0 c1 d1 c2 d2 c3 d3
+ pshufb m2, m6 ; e0 f0 e1 f1 e2 f2 e3 f3
+.pass1_end:
+ REPX {pmulhrsw x, m7}, m3, m5, m4, m2
+ punpckldq m0, m3, m5 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m3, m5 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckhdq m5, m2, m4 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpckldq m2, m4 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhqdq m1, m0, m2
+ punpcklqdq m0, m2
+ punpcklqdq m2, m3, m5
+ punpckhqdq m3, m5
+ jmp tx2q
+.pass2:
+ call .main_pass2
+ vpbroadcastd m6, [o(pw_2048)]
+ psrlq m10, 4
+ psubw m7, m8, m6
+.pass2_end:
+ vpbroadcastd m5, [o(pw_2896x8)]
+ paddsw m1, m2, m4
+ psubsw m2, m4
+ pmulhrsw m1, m5 ; -out7 out4 out6 -out5
+ pmulhrsw m5, m2 ; out8 -out11 -out9 out10
+ mova ym8, [o(gather8c)]
+ lea r3, [dstq+strideq]
+ psrlq m2, m10, 4
+ vpermi2q m2, m0, m3 ; 1 3 13 15
+ vpermt2q m0, m10, m3 ; 0 2 12 14
+ psrlq m3, m10, 8
+ vpermi2q m3, m1, m5 ; 5 7 9 11
+ psrlq m10, 12
+ vpermt2q m1, m10, m5 ; 4 6 8 10
+ pmulhrsw m0, m6
+ pmulhrsw m1, m6
+ jmp m(idct_8x16_internal_8bpc).end3
+ALIGN function_align
+.main_pass1:
+ vpbroadcastd m2, [o(pw_2896x8)]
+ pmulhrsw m5, m2, [cq+64*0]
+ pmulhrsw m3, m2, [cq+64*3]
+ pmulhrsw m1, m2, [cq+64*1]
+ pmulhrsw m2, [cq+64*2]
+ movu m4, [o(permA+3)]
+ psrlq m10, m4, 4
+ mova m6, m4
+ vpermi2q m4, m5, m3 ; in0 in12 in2 in14
+ vpermt2q m5, m10, m3 ; in15 in3 in13 in1
+ vpermi2q m6, m1, m2 ; in4 in8 in6 in10
+ vpermt2q m1, m10, m2 ; in11 in7 in9 in5
+ jmp .main
+ALIGN function_align
+.main_pass2:
+ mova m4, [o(permC)]
+ psrlq m5, m4, 4
+ vpermi2q m4, m0, m2 ; in0 in12 in2 in14
+ psrlq m6, m5, 4
+ vpermi2q m5, m1, m3 ; in15 in3 in13 in1
+ psrlq m10, m6, 4
+ vpermi2q m6, m0, m2 ; in4 in8 in6 in10
+ vpermt2q m1, m10, m3 ; in11 in7 in9 in5
+.main:
+ punpcklwd m0, m4, m5 ; in0 in15 in2 in13
+ punpckhwd m4, m5 ; in12 in3 in14 in1
+ punpcklwd m5, m6, m1 ; in4 in11 in6 in9
+ punpckhwd m6, m1 ; in8 in7 in10 in5
+cglobal_label .main2
+ vpbroadcastd m9, [o(pd_2048)]
+ vpbroadcastq m13, [o(int_mshift)]
+ kxnorb k1, k1, k1
+ vpcmpub k7, m13, m9, 6 ; 0x33...
+ pxor m8, m8
+ ITX_MUL4X_PACK 0, 1, 2, 3, 7, 9, 201, 4091, 995, 3973, 5
+ ITX_MUL4X_PACK 6, 1, 2, 3, 7, 9, 3035, 2751, 3513, 2106, 5
+ ITX_MUL4X_PACK 4, 1, 2, 3, 7, 9, 3857, 1380, 4052, 601, 5
+ ITX_MUL4X_PACK 5, 1, 2, 3, 7, 9, 1751, 3703, 2440, 3290, 5
+ psubsw m2, m0, m6 ; t9a t8a t11a t10a
+ paddsw m0, m6 ; t1a t0a t3a t2a
+ psubsw m3, m5, m4 ; t13a t12a t15a t14a
+ paddsw m5, m4 ; t5a t4a t7a t6a
+ ITX_MUL4X_PACK 2, 4, 1, 6, 7, 9, 799, 4017, 3406, 2276, 5
+ psubw m7, m8, m7
+ ITX_MUL2X_PACK 3, 4, 1, 9, 7, 6, 4
+ vpbroadcastd m6, [o(pw_3784_m1567)]
+ vpbroadcastd m6{k1}, [o(pw_m3784_1567)]
+ psubsw m1, m0, m5 ; t5 t4 t7 t6
+ paddsw m0, m5 ; t1 t0 t3 t2
+ psubsw m4, m2, m3 ; t13a t12a t15a t14a
+ paddsw m2, m3 ; t9a t8a t11a t10a
+ ITX_MUL2X_PACK 1, 3, 5, 9, 1567_3784, 6, 16 ; t5a t4a t6a t7a
+ ITX_MUL2X_PACK 4, 3, 5, 9, 1567_3784, 6, 16 ; t13 t12 t14 t15
+ vbroadcasti32x4 m5, [o(deint_shuf)]
+ pshufb m0, m5
+ pshufb m2, m5
+ vshufi32x4 m3, m0, m2, q3232 ; t3 t2 t11a t10a
+ vinserti32x8 m0, ym2, 1 ; t1 t0 t9a t8a
+ vshufi32x4 m2, m1, m4, q3232 ; t6a t7a t14 t15
+ vinserti32x8 m1, ym4, 1 ; t5a t4a t13 t12
+ pshufd m2, m2, q1032 ; t7a t6a t15 t14
+ psubsw m4, m0, m3 ; t3a t2a t11 t10
+ paddsw m0, m3 ; -out15 out0 out14 -out1
+ paddsw m3, m1, m2 ; out12 -out3 -out13 out2
+ psubsw m1, m2 ; t7 t6 t15a t14a
+ punpckhqdq m2, m4, m1 ; t2a t6 t10 t14a
+ punpcklqdq m4, m1 ; t3a t7 t11 t15a
+ ret
+
+INV_TXFM_8X16_FN flipadst, dct
+INV_TXFM_8X16_FN flipadst, adst
+INV_TXFM_8X16_FN flipadst, flipadst
+INV_TXFM_8X16_FN flipadst, identity
+
+cglobal iflipadst_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call m(iadst_16x8_internal_8bpc).main_pass1
+ vbroadcasti32x4 m6, [o(int_shuf2)]
+ vpbroadcastd m7, [o(pw_m16384_16384)]
+ punpcklwd m3, m0, m4 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpckhwd m4, m0 ; g0 h0 g1 h1 g2 h2 g3 h3
+ pshufb m5, m2, m6 ; c0 d0 c1 d1 c2 d2 c3 d3
+ pshufb m2, m1, m6 ; e0 f0 e1 f1 e2 f2 e3 f3
+ jmp m(iadst_8x16_internal_8bpc).pass1_end
+.pass2:
+ call m(iadst_8x16_internal_8bpc).main_pass2
+ vpbroadcastd m7, [o(pw_2048)]
+ psrlq m10, 36
+ psubw m6, m8, m7
+ jmp m(iadst_8x16_internal_8bpc).pass2_end
+
+INV_TXFM_8X16_FN identity, dct
+INV_TXFM_8X16_FN identity, adst
+INV_TXFM_8X16_FN identity, flipadst
+INV_TXFM_8X16_FN identity, identity
+
+cglobal iidentity_8x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [o(int16_perm)]
+ vpermb m3, m0, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3
+ vpermb m2, m0, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3
+ vpermb m4, m0, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3
+ vpermb m0, m0, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3
+ vpbroadcastd m5, [o(pw_2896x8)]
+ punpckldq m1, m3, m2 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m3, m2 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m2, m4, m0 ; e0 f0 g0 h0 a1 f1 g1 h1
+ punpckhdq m4, m0 ; e2 f2 g2 h2 e3 f3 g3 h3
+ REPX {pmulhrsw x, m5}, m1, m2, m3, m4
+ punpcklqdq m0, m1, m2 ; a0 b0 c0 d0 e0 f0 g0 h0
+ punpckhqdq m1, m2 ; a1 b1 c1 d1 e1 f1 g1 h1
+ punpcklqdq m2, m3, m4 ; a2 b2 c2 d2 e2 f2 g2 h2
+ punpckhqdq m3, m4 ; a3 b3 c3 d3 e3 f3 g3 h3
+ jmp tx2q
+.pass2:
+ vpbroadcastd m7, [o(pw_1697x16)]
+ mova ym8, [o(gather8b)]
+ lea r3, [dstq+strideq*2]
+ pmulhrsw m4, m7, m0
+ pmulhrsw m5, m7, m1
+ pmulhrsw m6, m7, m2
+ pmulhrsw m7, m3
+ REPX {paddsw x, x}, m0, m1, m2, m3
+ paddsw m0, m4
+ paddsw m1, m5
+ paddsw m2, m6
+ paddsw m3, m7
+ jmp m(idct_8x16_internal_8bpc).end
+
+%macro WRITE_16X2 6 ; coefs[1-2], tmp[1-2], offset[1-2]
+ pmovzxbw m%3, [dstq+%5]
+%ifnum %1
+ paddw m%3, m%1
+%else
+ paddw m%3, %1
+%endif
+ pmovzxbw m%4, [dstq+%6]
+%ifnum %2
+ paddw m%4, m%2
+%else
+ paddw m%4, %2
+%endif
+ packuswb m%3, m%4
+ vpermq m%3, m%3, q3120
+ mova [dstq+%5], xm%3
+ vextracti32x4 [dstq+%6], m%3, 1
+%endmacro
+
+%macro INV_TXFM_16X4_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 16x4
+%ifidn %1_%2, dct_dct
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly2
+%endif
+%endmacro
+
+INIT_ZMM avx512icl
+INV_TXFM_16X4_FN dct, dct
+INV_TXFM_16X4_FN dct, adst
+INV_TXFM_16X4_FN dct, flipadst
+INV_TXFM_16X4_FN dct, identity
+
+cglobal idct_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova xm0, [cq+16*0]
+ mova xm1, [cq+16*1]
+ mova xm2, [cq+16*2]
+ mova xm3, [cq+16*3]
+ mova xm4, [cq+16*4]
+ mova xm5, [cq+16*5]
+ mova xm6, [cq+16*6]
+ mova xm7, [cq+16*7]
+ call m(idct_4x16_internal_8bpc).main
+ vpbroadcastd m8, [o(pw_16384)]
+ vinserti32x4 ym1, xm3, 1 ; 3 2 7 6
+ vinserti32x4 ym5, xm7, 1 ; b a f e
+ vinserti32x4 ym0, xm2, 1 ; 0 1 4 5
+ vinserti32x4 ym4, xm6, 1 ; 8 9 c d
+ vinserti32x8 m1, ym5, 1 ; 3 2 7 6 b a f e
+ vinserti32x8 m0, ym4, 1 ; 0 1 4 5 8 9 c d
+ pmulhrsw m1, m8
+ pmulhrsw m0, m8
+ pshufd m1, m1, q1032
+ punpckhwd m2, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m0, m2
+ punpcklwd m0, m2
+ jmp tx2q
+.pass2:
+ IDCT4_1D_PACKED
+ mova m2, [o(permA)]
+ jmp m(iadst_16x4_internal_8bpc).end
+
+INV_TXFM_16X4_FN adst, dct
+INV_TXFM_16X4_FN adst, adst
+INV_TXFM_16X4_FN adst, flipadst
+INV_TXFM_16X4_FN adst, identity
+
+cglobal iadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+64*0]
+ mova m1, [cq+64*1]
+ movshdup m3, [o(permB)]
+ psrlq m10, m3, 4
+ call m(iadst_4x16_internal_8bpc).main2
+ vpbroadcastd m6, [o(pw_16384_m16384)]
+ psrlq m0, m10, 4
+ psrlq m10, 8
+.pass1_end:
+ punpcklwd ym5, ym4, ym2
+ punpckhwd ym4, ym2
+ vinserti32x8 m5, ym4, 1
+ mova m1, m9
+ vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16}
+ mova m4, m9
+ vpdpwssd m4, m5, [o(pw_2896_2896)] {1to16}
+ psrad m1, 12
+ psrad m4, 12
+ packssdw m1, m4 ; out8 -out7 -out9 out6 -out11 out4 out10 -out5
+ vpermi2q m0, m1, m3 ; 0 1 4 5 8 9 c d
+ vpermt2q m1, m10, m3 ; 2 3 6 7 a b e f
+ punpckhwd m2, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m0, m2
+ punpcklwd m0, m2
+ pmulhrsw m0, m6
+ pmulhrsw m1, m6
+ jmp tx2q
+.pass2:
+ call .main
+ movu m2, [o(permA+1)]
+.end:
+ vpbroadcastd m3, [o(pw_2048)]
+ pmulhrsw m0, m3
+ pmulhrsw m1, m3
+.end2:
+ psrlq m3, m2, 4
+ vpermi2q m2, m0, m1
+ vpermi2q m3, m0, m1
+.end3:
+ lea r3, [dstq+strideq*2]
+ mova xm1, [dstq+strideq*0]
+ vinserti32x4 ym1, [dstq+strideq*1], 1
+ vinserti32x4 m1, [r3 +strideq*0], 2
+ vinserti32x4 m1, [r3 +strideq*1], 3
+ pxor m4, m4
+ mova [cq+64*0], m4
+ mova [cq+64*1], m4
+ punpcklbw m0, m1, m4
+ punpckhbw m1, m4
+ paddw m0, m2
+ paddw m1, m3
+ packuswb m0, m1
+ mova [dstq+strideq*0], xm0
+ vextracti32x4 [dstq+strideq*1], ym0, 1
+ vextracti32x4 [r3 +strideq*0], m0, 2
+ vextracti32x4 [r3 +strideq*1], m0, 3
+ RET
+ALIGN function_align
+.main:
+ IADST4_1D_PACKED
+ ret
+
+INV_TXFM_16X4_FN flipadst, dct
+INV_TXFM_16X4_FN flipadst, adst
+INV_TXFM_16X4_FN flipadst, flipadst
+INV_TXFM_16X4_FN flipadst, identity
+
+cglobal iflipadst_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m0, [cq+64*0]
+ mova m1, [cq+64*1]
+ movshdup m3, [o(permB)]
+ psrlq m10, m3, 4
+ call m(iadst_4x16_internal_8bpc).main2
+ vpbroadcastd m6, [o(pw_m16384_16384)]
+ psrlq m0, m10, 12
+ psrlq m10, 16
+ jmp m(iadst_16x4_internal_8bpc).pass1_end
+.pass2:
+ call m(iadst_16x4_internal_8bpc).main
+ movu m2, [o(permA+2)]
+ jmp m(iadst_16x4_internal_8bpc).end
+
+INV_TXFM_16X4_FN identity, dct
+INV_TXFM_16X4_FN identity, adst
+INV_TXFM_16X4_FN identity, flipadst
+INV_TXFM_16X4_FN identity, identity
+
+cglobal iidentity_16x4_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m1, [cq+64*0]
+ mova m2, [cq+64*1]
+ vpbroadcastd m3, [o(pw_1697x16)]
+ vpbroadcastd m4, [o(pw_16384)]
+ mova m5, [o(idtx_16x4p)]
+ shufps m0, m1, m2, q2020
+ shufps m1, m2, q3131
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ pmulhrsw m2, m4
+ pmulhrsw m3, m4
+ paddsw m0, m2
+ paddsw m1, m3
+ vpermb m0, m5, m0
+ vpermb m1, m5, m1
+ jmp tx2q
+.pass2:
+ vpbroadcastd m3, [o(pw_1697x8)]
+ pmulhrsw m2, m3, m0
+ pmulhrsw m3, m1
+ paddsw m0, m2
+ paddsw m1, m3
+ movu m2, [o(permA+1)]
+ jmp m(iadst_16x4_internal_8bpc).end
+
+%macro INV_TXFM_16X8_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 16x8
+%ifidn %1_%2, dct_dct
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 8
+.dconly:
+ imul r6d, 181
+ add r6d, 128
+ sar r6d, 8
+.dconly2:
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+.dconly3:
+ imul r6d, 181
+ lea r2, [strideq*3]
+ add r6d, 128+2048
+ sar r6d, 8+4
+ pxor m2, m2
+ vpbroadcastw m3, r6d
+.dconly_loop:
+ mova xm1, [dstq+strideq*0]
+ vinserti32x4 ym1, [dstq+strideq*1], 1
+ vinserti32x4 m1, [dstq+strideq*2], 2
+ vinserti32x4 m1, [dstq+r2 ], 3
+ punpcklbw m0, m1, m2
+ punpckhbw m1, m2
+ paddw m0, m3
+ paddw m1, m3
+ packuswb m0, m1
+ mova [dstq+strideq*0], xm0
+ vextracti32x4 [dstq+strideq*1], ym0, 1
+ vextracti32x4 [dstq+strideq*2], m0, 2
+ vextracti32x4 [dstq+r2 ], m0, 3
+ lea dstq, [dstq+strideq*4]
+ sub r3d, 4
+ jg .dconly_loop
+ RET
+%endif
+%endmacro
+
+%macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd
+ vpbroadcastd m8, [o(pw_2896x8)]
+ vpermq m0, [cq+32*0], q3120
+ add cq, 32*4
+ vpermq m7, [cq+32*3], q%1
+ vpermq m1, [cq-32*3], q%1
+ vpermq m6, [cq+32*2], q3120
+ vpermq m2, [cq-32*2], q3120
+ vpermq m5, [cq+32*1], q%1
+ vpermq m3, [cq-32*1], q%1
+ vpermq m4, [cq+32*0], q3120
+ REPX {pmulhrsw x, m8}, m0, m7, m1, m6, m2, m5, m3, m4
+%endmacro
+
+INV_TXFM_16X8_FN dct, dct
+INV_TXFM_16X8_FN dct, identity
+INV_TXFM_16X8_FN dct, adst
+INV_TXFM_16X8_FN dct, flipadst
+
+cglobal idct_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd m1, [o(pw_2896x8)]
+ vpermq m0, [cq+64*0], q3120
+ vpermq m2, [cq+64*1], q3120
+ vpermq m4, [cq+64*2], q3120
+ vpermq m6, [cq+64*3], q3120
+ REPX {pmulhrsw x, m1}, m0, m2, m4, m6
+ vextracti32x8 ym1, m0, 1
+ vextracti32x8 ym3, m2, 1
+ vextracti32x8 ym5, m4, 1
+ vextracti32x8 ym7, m6, 1
+ call m(idct_8x16_internal_8bpc).main
+ vbroadcasti32x4 m8, [o(int_shuf1)]
+ vbroadcasti32x4 m9, [o(int_shuf2)]
+ vinserti32x8 m0, ym2, 1 ; a0 a1 a2 a3 b0 b1 b2 b3
+ vinserti32x8 m1, ym3, 1 ; d0 d1 d2 d3 c0 c1 c2 c3
+ vinserti32x8 m4, ym6, 1 ; i0 i1 i2 i3 j0 j1 j2 j3
+ vinserti32x8 m5, ym7, 1 ; l0 l1 l2 l3 k0 k1 k2 k3
+ vpbroadcastd m2, [o(pw_16384)]
+ pshufb m0, m8 ; a0 b0 a1 b1 a2 b2 a3 b3
+ pshufb m1, m9 ; c0 d0 c1 d1 c2 d2 c3 d3
+ pshufb m6, m4, m8 ; i0 j0 i1 j1 i2 j2 i3 j3
+ pshufb m7, m5, m9 ; m0 n0 m1 n1 m2 n2 m3 n3
+ REPX {pmulhrsw x, m2}, m0, m1, m6, m7
+ punpckldq m2, m0, m1 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m3, m0, m1 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m4, m6, m7 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhdq m5, m6, m7 ; i2 j2 k2 l2 i3 j3 k3 l3
+ jmp tx2q
+.pass2:
+ vshufi32x4 m0, m2, m4, q2020 ; 0 1
+ vshufi32x4 m2, m4, q3131 ; 4 5
+ vshufi32x4 m1, m3, m5, q2020 ; 2 3
+ vshufi32x4 m3, m5, q3131 ; 6 7
+ call .main
+ movshdup m4, [o(permC)]
+ psrlq m6, m4, 4
+ vpermq m5, m4, q1032
+ vpermi2q m4, m0, m2 ; a2 a3 b2 b3 e2 e3 f2 f3
+ vpermt2q m0, m6, m2 ; a0 a1 b0 b1 e0 e1 f0 f1
+ psrlq m6, m5, 4
+ vpermi2q m5, m1, m3 ; c2 c3 d2 d3 g2 g3 h2 h3
+ vpermt2q m1, m6, m3 ; c0 c1 d0 d1 g0 g1 h0 h1
+ vpbroadcastd m6, [o(pw_2048)]
+.end:
+ REPX {pmulhrsw x, m6}, m0, m4, m1, m5
+.end2:
+ lea r3, [dstq+strideq*4]
+ lea r4, [strideq*3]
+ mova xm3, [dstq+strideq*0]
+ mova xm6, [dstq+strideq*2]
+ vinserti32x4 ym3, [dstq+strideq*1], 1
+ vinserti32x4 ym6, [dstq+r4 ], 1
+ vinserti32x4 m3, [r3 +strideq*0], 2
+ vinserti32x4 m6, [r3 +strideq*2], 2
+ vinserti32x4 m3, [r3 +strideq*1], 3
+ vinserti32x4 m6, [r3 +r4 ], 3
+ pxor m7, m7
+ mova [cq+64*0], m7
+ mova [cq+64*1], m7
+ mova [cq+64*2], m7
+ mova [cq+64*3], m7
+ punpcklbw m2, m3, m7
+ punpckhbw m3, m7
+ paddw m0, m2
+ paddw m4, m3
+ packuswb m0, m4
+ mova [dstq+strideq*0], xm0
+ vextracti32x4 [dstq+strideq*1], ym0, 1
+ vextracti32x4 [r3 +strideq*0], m0, 2
+ vextracti32x4 [r3 +strideq*1], m0, 3
+ punpcklbw m3, m6, m7
+ punpckhbw m6, m7
+ paddw m1, m3
+ paddw m5, m6
+ packuswb m1, m5
+ mova [dstq+strideq*2], xm1
+ vextracti32x4 [dstq+r4 ], ym1, 1
+ vextracti32x4 [r3 +strideq*2], m1, 2
+ vextracti32x4 [r3 +r4 ], m1, 3
+ RET
+ALIGN function_align
+cglobal_label .main
+ IDCT8_1D_PACKED
+ ret
+
+INV_TXFM_16X8_FN adst, dct
+INV_TXFM_16X8_FN adst, adst
+INV_TXFM_16X8_FN adst, flipadst
+INV_TXFM_16X8_FN adst, identity
+
+cglobal iadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call m(iadst_8x16_internal_8bpc).main_pass1
+ vpbroadcastd m7, [o(pw_16384_m16384)]
+ psrlq m10, 4
+.pass1_end:
+ punpcklwd m5, m4, m2
+ punpckhwd m4, m2
+ mova m1, m9
+ vpdpwssd m1, m5, [o(pw_m2896_2896)] {1to16}
+ mova m6, m9
+ vpdpwssd m6, m5, [o(pw_2896_2896)] {1to16}
+ mova m2, m9
+ vpdpwssd m2, m4, [o(pw_m2896_2896)] {1to16}
+ vpdpwssd m9, m4, [o(pw_2896_2896)] {1to16}
+ psrad m1, 12
+ psrad m6, 12
+ packssdw m1, m6 ; out8 -out7 -out9 out6
+ psrad m2, 12
+ psrad m9, 12
+ packssdw m2, m9 ; -out11 out4 out10 -out5
+ psrlq m4, m10, 4
+ vpermi2q m4, m0, m2
+ vpermt2q m0, m10, m2
+ psrlq m5, m10, 8
+ vpermi2q m5, m1, m3
+ psrlq m10, 12
+ vpermt2q m1, m10, m3
+ punpcklwd m3, m4, m5 ; a0 c0 a1 c1 a2 c2 a3 c3
+ punpckhwd m4, m5 ; b0 d0 b1 d1 b2 d2 b3 d3
+ punpcklwd m5, m1, m0 ; i0 k0 i1 k1 2i k2 i3 k3
+ punpckhwd m1, m0 ; j0 l0 j1 l1 j2 l2 j3 l3
+ punpcklwd m2, m3, m4 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhwd m3, m4 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpcklwd m4, m5, m1 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhwd m5, m1 ; i2 j2 k2 l2 i3 j3 k3 l3
+ REPX {pmulhrsw x, m7}, m2, m3, m4, m5
+ jmp tx2q
+.pass2:
+ vshufi32x4 m0, m2, m4, q2020
+ vshufi32x4 m2, m4, q3131 ; 4 5
+ vshufi32x4 m1, m3, m5, q2020
+ vshufi32x4 m3, m5, q3131 ; 6 7
+ pshufd m4, m0, q1032 ; 1 0
+ pshufd m5, m1, q1032 ; 3 2
+ call .main_pass2
+ movshdup m4, [o(permC)]
+ pmulhrsw m0, m6
+ pmulhrsw m1, m6
+ psrlq m6, m4, 4
+ mova m5, m4
+ vpermi2q m4, m0, m2
+ vpermt2q m0, m6, m2
+ vpermi2q m5, m1, m3
+ vpermt2q m1, m6, m3
+ jmp m(idct_16x8_internal_8bpc).end2
+ALIGN function_align
+.main_pass1:
+ vpbroadcastd m4, [o(pw_2896x8)]
+ pmulhrsw m3, m4, [cq+64*0]
+ pmulhrsw m1, m4, [cq+64*3]
+ pmulhrsw m2, m4, [cq+64*1]
+ pmulhrsw m4, [cq+64*2]
+ mova m5, [o(int16_perm)]
+ kxnorb k1, k1, k1
+ vpblendmd m0{k1}, m1, m3 ; 0 7
+ vmovdqa32 m3{k1}, m1 ; 6 1
+ vpblendmd m1{k1}, m4, m2 ; 2 5
+ vmovdqa32 m2{k1}, m4 ; 4 3
+ REPX {vpermb x, m5, x}, m0, m1, m2, m3
+ IADST8_1D_PACKED 1
+ ret
+ALIGN function_align
+cglobal_label .main_pass2
+ IADST8_1D_PACKED 2
+ pxor m5, m5
+ psubd m5, m6
+ packssdw m6, m5
+ pmulhrsw m2, m6
+ pmulhrsw m3, m6
+ ret
+
+INV_TXFM_16X8_FN flipadst, dct
+INV_TXFM_16X8_FN flipadst, adst
+INV_TXFM_16X8_FN flipadst, flipadst
+INV_TXFM_16X8_FN flipadst, identity
+
+cglobal iflipadst_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call m(iadst_8x16_internal_8bpc).main_pass1
+ vpbroadcastd m7, [o(pw_m16384_16384)]
+ psrlq m10, 20
+ jmp m(iadst_16x8_internal_8bpc).pass1_end
+.pass2:
+ vshufi32x4 m0, m2, m4, q2020
+ vshufi32x4 m2, m4, q3131 ; 4 5
+ vshufi32x4 m1, m3, m5, q2020
+ vshufi32x4 m3, m5, q3131 ; 6 7
+ pshufd m4, m0, q1032 ; 1 0
+ pshufd m5, m1, q1032 ; 3 2
+ call m(iadst_16x8_internal_8bpc).main_pass2
+ movshdup m4, [o(permC)]
+ pmulhrsw m5, m6, m0
+ pmulhrsw m0, m6, m1
+ psrlq m1, m4, 12
+ psrlq m4, 8
+ mova m7, m4
+ vpermi2q m4, m0, m3
+ vpermt2q m0, m1, m3
+ vpermi2q m1, m5, m2
+ vpermt2q m5, m7, m2
+ jmp m(idct_16x8_internal_8bpc).end2
+
+INV_TXFM_16X8_FN identity, dct
+INV_TXFM_16X8_FN identity, adst
+INV_TXFM_16X8_FN identity, flipadst
+INV_TXFM_16X8_FN identity, identity
+
+cglobal iidentity_16x8_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ vpbroadcastd m0, [o(pw_2896x8)]
+ pmulhrsw m3, m0, [cq+64*0]
+ pmulhrsw m4, m0, [cq+64*1]
+ pmulhrsw m5, m0, [cq+64*2]
+ pmulhrsw m0, [cq+64*3]
+ vpbroadcastd m7, [o(pw_1697x16)]
+ vpbroadcastd m8, [o(pw_16384)]
+ shufps m2, m3, m4, q2020 ; a0 a1 a4 a5 e0 e1 e4 e5
+ shufps m3, m4, q3131 ; a2 a3 a6 a7 e2 e3 e6 e7
+ shufps m4, m5, m0, q2020 ; i0 i1 i4 i5 m0 m1 m4 m5
+ shufps m5, m0, q3131 ; i2 i3 i6 i7 m2 m3 m6 m7
+ mova m9, [o(int8_permA)]
+ pmulhrsw m0, m7, m2
+ pmulhrsw m1, m7, m3
+ pmulhrsw m6, m7, m4
+ pmulhrsw m7, m5
+ REPX {pmulhrsw x, m8}, m0, m1, m6, m7
+ paddsw m2, m0
+ paddsw m3, m1
+ paddsw m4, m6
+ paddsw m5, m7
+ REPX {vpermb x, m9, x}, m2, m3, m4, m5
+ jmp tx2q
+.pass2:
+ mova m7, [o(permB)]
+ vpbroadcastd m6, [o(pw_4096)]
+ vpermq m0, m7, m2
+ vpermq m4, m7, m4
+ vpermq m1, m7, m3
+ vpermq m5, m7, m5
+ jmp m(idct_16x8_internal_8bpc).end
+
+%macro INV_TXFM_16X16_FN 2 ; type1, type2
+ INV_TXFM_FN %1, %2, 16x16
+%ifidn %1_%2, dct_dct
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 16
+ imul r6d, 181
+ add r6d, 128+512
+ sar r6d, 8+2
+ jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3
+%endif
+%endmacro
+
+INV_TXFM_16X16_FN dct, dct
+INV_TXFM_16X16_FN dct, identity
+INV_TXFM_16X16_FN dct, adst
+INV_TXFM_16X16_FN dct, flipadst
+
+cglobal idct_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m7, [o(permB)]
+ vpermq m0, m7, [cq+64*0]
+ vpermq m1, m7, [cq+64*1]
+ vpermq m2, m7, [cq+64*2]
+ vpermq m3, m7, [cq+64*3]
+ vpermq m4, m7, [cq+64*4]
+ vpermq m5, m7, [cq+64*5]
+ vpermq m6, m7, [cq+64*6]
+ vpermq m7, m7, [cq+64*7]
+ call .main
+ vbroadcasti32x4 m12, [o(int_shuf1)]
+ vbroadcasti32x4 m11, [o(int_shuf2)]
+ vpbroadcastd m13, [o(pw_8192)]
+ pshufb m0, m12
+ pshufb m8, m1, m11
+ pshufb m2, m12
+ pshufb m9, m3, m11
+ pshufb m4, m12
+ pshufb m10, m5, m11
+ pshufb m6, m12
+ pshufb m11, m7, m11
+ REPX {pmulhrsw x, m13}, m0, m8, m2, m9, m4, m10, m6, m11
+ punpckhdq m1, m0, m8
+ punpckldq m0, m8
+ punpckhdq m3, m2, m9
+ punpckldq m2, m9
+ punpckhdq m5, m4, m10
+ punpckldq m4, m10
+ punpckhdq m7, m6, m11
+ punpckldq m6, m11
+ jmp tx2q
+.pass2:
+ vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc
+ vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4
+ vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec
+ vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4
+ vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me
+ vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6
+ vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee
+ vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6
+ vshufi32x4 m2, m0, m4, q3131 ; 4 5
+ vshufi32x4 m0, m4, q2020 ; 0 1
+ vshufi32x4 m4, m6, m8, q2020 ; 8 9
+ vshufi32x4 m6, m8, q3131 ; 12 13
+ vshufi32x4 m3, m1, m5, q3131 ; 6 7
+ vshufi32x4 m1, m5, q2020 ; 2 3
+ vshufi32x4 m5, m7, m9, q2020 ; 10 11
+ vshufi32x4 m7, m9, q3131 ; 14 15
+ call .main
+ mova m8, [o(permD)]
+ psrlq m12, m8, 4
+ psrlq m9, m8, 8
+ psrlq m13, m8, 12
+ mova m10, m8
+ vpermi2q m8, m0, m2 ; 0 1 4 5
+ vpermt2q m0, m12, m2
+ mova m11, m9
+ vpermi2q m9, m1, m3 ; 2 3 6 7
+ vpermt2q m1, m13, m3
+ vpermi2q m10, m4, m6 ; 8 9 12 13
+ vpermt2q m4, m12, m6
+ vpermi2q m11, m5, m7 ; 10 11 14 15
+ vpermt2q m5, m13, m7
+.end:
+ vpbroadcastd m12, [o(pw_2048)]
+.end2:
+ REPX {pmulhrsw x, m12}, m0, m1, m4, m5
+.end3:
+ REPX {pmulhrsw x, m12}, m8, m9, m10, m11
+ lea r3, [strideq*3]
+ lea r4, [dstq+strideq*4]
+ lea r5, [dstq+strideq*8]
+ lea r6, [r4 +strideq*8]
+ mova xm3, [dstq+strideq*0]
+ mova xm6, [dstq+strideq*2]
+ vinserti32x4 ym3, [dstq+strideq*1], 1
+ vinserti32x4 ym6, [dstq+r3 ], 1
+ vinserti32x4 m3, [r4+strideq*0], 2
+ vinserti32x4 m6, [r4+strideq*2], 2
+ vinserti32x4 m3, [r4+strideq*1], 3
+ vinserti32x4 m6, [r4+r3 ], 3
+ mova xm12, [r5+strideq*0]
+ mova xm13, [r5+strideq*2]
+ vinserti32x4 ym12, [r5+strideq*1], 1
+ vinserti32x4 ym13, [r5+r3 ], 1
+ vinserti32x4 m12, [r6+strideq*0], 2
+ vinserti32x4 m13, [r6+strideq*2], 2
+ vinserti32x4 m12, [r6+strideq*1], 3
+ vinserti32x4 m13, [r6+r3 ], 3
+ pxor m7, m7
+ REPX {mova [cq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7
+ punpcklbw m2, m3, m7
+ punpckhbw m3, m7
+ paddw m0, m2
+ paddw m8, m3
+ packuswb m0, m8
+ punpcklbw m2, m6, m7
+ punpckhbw m6, m7
+ paddw m1, m2
+ paddw m9, m6
+ packuswb m1, m9
+ punpcklbw m2, m12, m7
+ punpckhbw m12, m7
+ paddw m2, m4
+ paddw m10, m12
+ packuswb m2, m10
+ punpcklbw m3, m13, m7
+ punpckhbw m13, m7
+ paddw m3, m5
+ paddw m11, m13
+ packuswb m3, m11
+ mova [dstq+strideq*0], xm0
+ vextracti32x4 [dstq+strideq*1], ym0, 1
+ mova [dstq+strideq*2], xm1
+ vextracti32x4 [dstq+r3 ], ym1, 1
+ vextracti32x4 [r4+strideq*0], m0, 2
+ vextracti32x4 [r4+strideq*1], m0, 3
+ vextracti32x4 [r4+strideq*2], m1, 2
+ vextracti32x4 [r4+r3 ], m1, 3
+ mova [r5+strideq*0], xm2
+ vextracti32x4 [r5+strideq*1], ym2, 1
+ mova [r5+strideq*2], xm3
+ vextracti32x4 [r5+r3 ], ym3, 1
+ vextracti32x4 [r6+strideq*0], m2, 2
+ vextracti32x4 [r6+strideq*1], m2, 3
+ vextracti32x4 [r6+strideq*2], m3, 2
+ vextracti32x4 [r6+r3 ], m3, 3
+ RET
+ALIGN function_align
+cglobal_label .main_fast2 ; bottom three-quarters are zero
+ vpbroadcastd m10, [o(pd_2048)]
+ vpbroadcastq m13, [o(int_mshift)]
+ vpcmpub k7, m13, m10, 6
+.main_fast4:
+ vpbroadcastd m2, [o(pw_401_4076x8)]
+ vpbroadcastd m4, [o(pw_m1189_3920x8)]
+ vpbroadcastd m3, [o(pw_799_4017x8)]
+ pmulhrsw m2, m8 ; t8a t15a
+ pmulhrsw m4, m1 ; t11a t12a
+ pmulhrsw m7, m3 ; t4a t7a
+ pxor m6, m6
+ psubsw m0, m2, m4 ; t11a t12a
+ paddsw m8, m2, m4 ; t8a t15a
+ mova m1, m7
+ jmp .main5
+ALIGN function_align
+cglobal_label .main_fast ; bottom half is zero
+ vpbroadcastd m10, [o(pd_2048)]
+.main_fast3:
+ vpbroadcastq m13, [o(int_mshift)]
+ vpcmpub k7, m13, m10, 6
+.main_fast5:
+ vpbroadcastd m2, [o(pw_401_4076x8)]
+ vpbroadcastd m4, [o(pw_m2598_3166x8)]
+ vpbroadcastd m11, [o(pw_1931_3612x8)]
+ vpbroadcastd m12, [o(pw_m1189_3920x8)]
+ pmulhrsw m8, m2 ; t8a t15a
+ vpbroadcastd m2, [o(pw_799_4017x8)]
+ pmulhrsw m0, m4 ; t9a t14a
+ vpbroadcastd m4, [o(pw_m2276_3406x8)]
+ pmulhrsw m5, m11 ; t10a t13a
+ pmulhrsw m1, m12 ; t11a t12a
+ pmulhrsw m7, m2 ; t4a t7a
+ pmulhrsw m3, m4 ; t5a t6a
+ jmp .main4
+ALIGN function_align
+cglobal_label .main
+ IDCT16_1D_PACKED
+ ret
+
+INV_TXFM_16X16_FN adst, dct
+INV_TXFM_16X16_FN adst, adst
+INV_TXFM_16X16_FN adst, flipadst
+
+cglobal iadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call .main_pass1
+ vpbroadcastd m10, [o(pw_8192_m8192)]
+ punpcklwd m8, m0, m1 ; b0 d0 b1 d1 b2 d2 b3 d3
+ punpckhwd m0, m1 ; a0 c0 a1 c1 a2 c2 a3 c3
+ punpckhwd m1, m0, m8 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpcklwd m0, m8 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpcklwd m8, m2, m3 ; f0 h0 f1 h1 f2 h2 f3 h3
+ punpckhwd m2, m3 ; e0 g0 e1 g1 e2 g2 e3 g3
+ punpckhwd m3, m2, m8 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpcklwd m2, m8 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhwd m8, m4, m5 ; i0 k0 i1 k1 i2 k2 i3 k3
+ punpcklwd m4, m5 ; j0 l0 j1 l1 j2 l2 j3 l3
+ punpckhwd m5, m4, m8 ; i2 j2 k2 l2 i3 j3 k3 l3
+ punpcklwd m4, m8 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhwd m8, m6, m7 ; m0 o0 m1 o1 m2 o2 m3 o3
+ punpcklwd m6, m7 ; n0 p0 n1 p1 n2 p2 n3 p3
+ punpckhwd m7, m6, m8 ; m2 n2 o2 p2 m3 n3 o3 p3
+ punpcklwd m6, m8 ; m0 n0 o0 p0 m1 n1 o1 p1
+.pass1_end:
+ REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7
+ jmp tx2q
+.pass2:
+ call .main_pass2
+ mova m10, [o(permD)]
+ psrlq m8, m10, 8
+ psrlq m12, m10, 12
+ psrlq m13, m10, 4
+ mova m9, m8
+ vpermi2q m8, m0, m2 ; 0 1 4 5
+ vpermt2q m0, m12, m2
+ vpermi2q m9, m1, m3 ; 2 3 6 7
+ vpermt2q m1, m12, m3
+ vpbroadcastd m12, [o(pw_2048)]
+ mov r3d, 0xff00ff00
+ mova m11, m10
+ vpermi2q m10, m4, m6 ; 8 9 12 13
+ vpermt2q m4, m13, m6
+ kmovd k1, r3d
+ vpermi2q m11, m5, m7 ; 10 11 14 15
+ vpermt2q m5, m13, m7
+ pxor m7, m7
+ vpsubw m12{k1}, m7, m12
+ jmp m(idct_16x16_internal_8bpc).end2
+ALIGN function_align
+.main_pass1:
+ mova m4, [o(permB)]
+ psrlq m3, m4, 4
+ vpermq m0, m4, [cq+64*0]
+ vpermq m7, m3, [cq+64*7]
+ vpermq m6, m4, [cq+64*6]
+ vpermq m1, m3, [cq+64*1]
+ vpermq m2, m4, [cq+64*2]
+ vpermq m5, m3, [cq+64*5]
+ vpermq m4, m4, [cq+64*4]
+ vpermq m3, m3, [cq+64*3]
+ call .main
+ vpbroadcastd m13, [o(pw_2896_2896)]
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ mova m2, m10
+ vpdpwssd m2, m5, m13 ; -out5
+ mova m8, m10
+ vpdpwssd m8, m11, m13 ; out4
+ mova m9, m10
+ vpdpwssd m9, m5, m12 ; out10
+ mova m5, m10
+ vpdpwssd m5, m11, m12 ; -out11
+ mova m11, m10
+ vpdpwssd m11, m3, m13 ; -out7
+ mova m14, m10
+ vpdpwssd m14, m4, m13 ; out6
+ mova m13, m10
+ vpdpwssd m13, m3, m12 ; out8
+ vpdpwssd m10, m4, [o(pw_2896_m2896)] {1to16} ; -out9
+ REPX {psrad x, 12}, m2, m8, m9, m5, m11, m14, m13, m10
+ packssdw m2, m8 ; -out5 out4
+ packssdw m5, m9, m5 ; out10 -out11
+ packssdw m3, m11, m14 ; -out7 out6
+ packssdw m4, m13, m10 ; out8 -out9
+ ret
+ALIGN function_align
+.main_pass2:
+ vshufi32x4 m8, m4, m6, q3232 ; i8 ic m8 mc
+ vinserti32x8 m4, ym6, 1 ; i0 i4 m0 m4
+ vshufi32x4 m6, m0, m2, q3232 ; a8 ac e8 ec
+ vinserti32x8 m0, ym2, 1 ; a0 a4 e0 e4
+ vshufi32x4 m9, m5, m7, q3232 ; ia ie ma me
+ vinserti32x8 m5, ym7, 1 ; i2 i6 m2 m6
+ vshufi32x4 m7, m1, m3, q3232 ; aa ae ea ee
+ vinserti32x8 m1, ym3, 1 ; a2 a6 e2 e6
+ vshufi32x4 m2, m0, m4, q3131 ; 4 5
+ vshufi32x4 m0, m4, q2020 ; 0 1
+ vshufi32x4 m4, m6, m8, q2020 ; 8 9
+ vshufi32x4 m6, m8, q3131 ; 12 13
+ vshufi32x4 m3, m1, m5, q3131 ; 6 7
+ vshufi32x4 m1, m5, q2020 ; 2 3
+ vshufi32x4 m5, m7, m9, q2020 ; 10 11
+ vshufi32x4 m7, m9, q3131 ; 14 15
+cglobal_label .main_pass2b
+ REPX {pshufd x, x, q1032}, m1, m3, m5, m7
+ call .main
+ vpbroadcastd m8, [o(pw_2896x8)]
+ pshufb m2, m11, m12
+ pshufb m5, m12
+ pshufb m3, m12
+ pshufb m4, m12
+ punpcklqdq m9, m5, m2 ; t15a t7
+ punpckhqdq m5, m2 ; t14a t6
+ shufps m2, m3, m4, q1032 ; t2a t10
+ shufps m3, m4, q3210 ; t3a t11
+ psubsw m4, m2, m3 ; out8 -out9
+ paddsw m3, m2 ; -out7 out6
+ paddsw m2, m5, m9 ; -out5 out4
+ psubsw m5, m9 ; out10 -out11
+ REPX {pmulhrsw x, m8}, m2, m3, m4, m5
+ ret
+ALIGN function_align
+.main:
+ vpbroadcastd m10, [o(pd_2048)]
+ vpbroadcastq m13, [o(int_mshift)]
+ punpckhwd m8, m7, m0 ; in14 in1
+ punpcklwd m0, m7 ; in0 in15
+ punpcklwd m7, m6, m1 ; in12 in3
+ punpckhwd m1, m6 ; in2 in13
+ punpckhwd m6, m5, m2 ; in10 in5
+ punpcklwd m2, m5 ; in4 in11
+ punpcklwd m5, m4, m3 ; in8 in7
+ punpckhwd m3, m4 ; in6 in9
+ vpcmpub k7, m13, m10, 6 ; 0x33...
+ ITX_MUL2X_PACK 0, 4, 9, 10, 201, 4091, 5 ; t0 t1
+ ITX_MUL2X_PACK 1, 4, 9, 10, 995, 3973, 5 ; t2 t3
+ ITX_MUL2X_PACK 2, 4, 9, 10, 1751, 3703, 5 ; t4 t5
+ ITX_MUL2X_PACK 3, 4, 9, 10, 2440, 3290, 5 ; t6 t7
+ ITX_MUL2X_PACK 5, 4, 9, 10, 3035, 2751, 5 ; t8 t9
+ ITX_MUL2X_PACK 6, 4, 9, 10, 3513, 2106, 5 ; t10 t11
+ ITX_MUL2X_PACK 7, 4, 9, 10, 3857, 1380, 5 ; t12 t13
+ ITX_MUL2X_PACK 8, 4, 9, 10, 4052, 601, 5 ; t14 t15
+ psubsw m4, m0, m5 ; t9a t8a
+ paddsw m0, m5 ; t1a t0a
+ psubsw m5, m1, m6 ; t11a t10a
+ paddsw m1, m6 ; t3a t2a
+ psubsw m6, m2, m7 ; t13a t12a
+ paddsw m2, m7 ; t5a t4a
+ psubsw m7, m3, m8 ; t15a t14a
+ paddsw m3, m8 ; t7a t6a
+ ITX_MUL2X_PACK 4, 8, 9, 10, 799, 4017, 4 ; t8 t9
+ ITX_MUL2X_PACK 6, 8, 9, 10, 799_4017, 4017_m799, 52 ; t12 t13
+ ITX_MUL2X_PACK 5, 8, 9, 10, 3406, 2276, 4 ; t10 t11
+ ITX_MUL2X_PACK 7, 8, 9, 10, 3406_2276, 2276_m3406, 52 ; t14 t15
+ psubsw m8, m1, m3 ; t7 t6
+ paddsw m1, m3 ; t3 t2
+ psubsw m3, m0, m2 ; t5 t4
+ paddsw m0, m2 ; t1 t0
+ psubsw m2, m5, m7 ; t14a t15a
+ paddsw m7, m5 ; t10a t11a
+ psubsw m5, m4, m6 ; t12a t13a
+ paddsw m4, m6 ; t8a t9a
+ ITX_MUL2X_PACK 3, 6, 9, 10, 1567, 3784, 5 ; t5a t4a
+ ITX_MUL2X_PACK 8, 6, 9, 10, 3784_m1567, 1567_3784, 52 ; t7a t6a
+ ITX_MUL2X_PACK 2, 6, 9, 10, 3784, 1567, 4 ; t15 t14
+ ITX_MUL2X_PACK 5, 6, 9, 10, 3784_1567, 1567_m3784, 52 ; t13 t12
+ vbroadcasti32x4 m12, [o(deint_shuf)]
+ paddsw m6, m4, m7 ; -out1 out14
+ psubsw m4, m7 ; t10 t11
+ psubsw m11, m3, m8 ; t7 t6
+ paddsw m8, m3 ; out12 -out3
+ psubsw m3, m0, m1 ; t3a t2a
+ paddsw m0, m1 ; -out15 out0
+ paddsw m1, m2, m5 ; -out13 out2
+ psubsw m5, m2 ; t15a t14a
+ pshufb m0, m12
+ pshufb m6, m12
+ pshufb m8, m12
+ pshufb m1, m12
+ shufps m7, m6, m0, q1032 ; out14 -out15
+ shufps m0, m6, m0, q3210 ; -out1 out0
+ punpcklqdq m6, m8, m1 ; out12 -out13
+ punpckhqdq m1, m8, m1 ; -out3 out2
+ ret
+
+INV_TXFM_16X16_FN flipadst, dct
+INV_TXFM_16X16_FN flipadst, adst
+INV_TXFM_16X16_FN flipadst, flipadst
+
+cglobal iflipadst_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ call m(iadst_16x16_internal_8bpc).main_pass1
+ vpbroadcastd m10, [o(pw_m8192_8192)]
+ punpcklwd m8, m1, m0 ; m0 o0 m1 o1 m2 o2 m3 o3
+ punpckhwd m9, m1, m0 ; n0 p0 n1 p1 n2 p2 n3 p3
+ punpckhwd m1, m7, m6 ; a0 c0 a1 c1 a2 c2 a3 c3
+ punpcklwd m7, m6 ; b0 d0 b1 d1 b2 d2 b3 d3
+ punpcklwd m0, m1, m7 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhwd m1, m7 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpcklwd m6, m8, m9 ; m0 n0 o0 p0 m1 n1 o1 p1
+ punpckhwd m7, m8, m9 ; m2 n2 o2 p2 m3 n3 o3 p3
+ punpcklwd m8, m3, m2 ; i0 k0 i1 k1 i2 k2 i3 k3
+ punpckhwd m9, m3, m2 ; j0 l0 j1 l1 j2 l2 j3 l3
+ punpckhwd m3, m5, m4 ; e0 g0 e1 g1 e2 g2 e3 g3
+ punpcklwd m5, m4 ; f0 h0 f1 h1 f2 h2 f3 h3
+ punpcklwd m2, m3, m5 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhwd m3, m5 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpcklwd m4, m8, m9 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhwd m5, m8, m9 ; i2 j2 k2 l2 i3 j3 k3 l3
+ jmp m(iadst_16x16_internal_8bpc).pass1_end
+.pass2:
+ call m(iadst_16x16_internal_8bpc).main_pass2
+ mova m10, [o(permD)]
+ psrlq m8, m10, 8
+ psrlq m12, m10, 12
+ psrlq m13, m10, 4
+ mova m9, m8
+ vpermi2q m8, m7, m5 ; 0 1 4 5
+ vpermt2q m7, m12, m5
+ vpermi2q m9, m6, m4 ; 2 3 6 7
+ vpermt2q m6, m12, m4
+ vpbroadcastd m12, [o(pw_2048)]
+ mov r3d, 0x00ff00ff
+ mova m11, m10
+ vpermi2q m10, m3, m1 ; 8 9 12 13
+ vpermt2q m3, m13, m1
+ kmovd k1, r3d
+ vpermi2q m11, m2, m0 ; 10 11 14 15
+ vpermt2q m2, m13, m0
+ pxor m0, m0
+ vpsubw m12{k1}, m0, m12
+ pmulhrsw m0, m7, m12
+ pmulhrsw m1, m6, m12
+ pmulhrsw m4, m3, m12
+ pmulhrsw m5, m2, m12
+ jmp m(idct_16x16_internal_8bpc).end3
+
+INV_TXFM_16X16_FN identity, dct
+INV_TXFM_16X16_FN identity, identity
+
+cglobal iidentity_16x16_internal_8bpc, 0, 6, 0, dst, stride, c, eob, tx2
+ mova m8, [o(int16_perm)]
+ vpermb m1, m8, [cq+64*0] ; a0 b0 a1 b1 a2 b2 a3 b3
+ vpermb m2, m8, [cq+64*1] ; c0 d0 c1 d1 c2 d2 c3 d3
+ vpbroadcastd m0, [o(pw_1697x16)]
+ vpermb m3, m8, [cq+64*2] ; e0 f0 e1 f1 e2 f2 e3 f3
+ vpermb m4, m8, [cq+64*3] ; g0 h0 g1 h1 g2 h2 g3 h3
+ vpermb m5, m8, [cq+64*4] ; i0 j0 i1 j1 i2 j2 i3 j3
+ vpermb m6, m8, [cq+64*5] ; k0 l0 k1 l1 k2 l2 k3 l3
+ vpermb m7, m8, [cq+64*6] ; m0 n0 m1 n1 m2 n2 m3 n3
+ vpermb m8, m8, [cq+64*7] ; o0 p0 o1 p1 o2 p2 o3 p3
+ pmulhrsw m9, m0, m1
+ pmulhrsw m10, m0, m2
+ pmulhrsw m11, m0, m3
+ pmulhrsw m12, m0, m4
+ pmulhrsw m13, m0, m5
+ pmulhrsw m14, m0, m6
+ pmulhrsw m15, m0, m7
+ pmulhrsw m0, m8
+ REPX {psraw x, 1}, m9, m10, m11, m12
+ pavgw m1, m9
+ pavgw m2, m10
+ pavgw m3, m11
+ pavgw m4, m12
+ REPX {psraw x, 1}, m13, m14, m15, m0
+ pavgw m5, m13
+ pavgw m6, m14
+ pavgw m7, m15
+ pavgw m8, m0
+ punpckldq m0, m1, m2 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m1, m2 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m2, m3, m4 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhdq m3, m4 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpckldq m4, m5, m6 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhdq m5, m6 ; i2 j2 k2 l2 i3 j3 k3 l3
+ punpckldq m6, m7, m8 ; m0 n0 o0 p0 m1 n1 o1 p1
+ punpckhdq m7, m8 ; m2 n2 o2 p2 m3 n3 o3 p3
+ jmp tx2q
+ALIGN function_align
+.pass2:
+ vpbroadcastd m11, [o(pw_1697x16)]
+ pmulhrsw m12, m11, m0
+ pmulhrsw m13, m11, m1
+ pmulhrsw m14, m11, m2
+ pmulhrsw m15, m11, m3
+ pmulhrsw m8, m11, m4
+ pmulhrsw m9, m11, m5
+ pmulhrsw m10, m11, m6
+ pmulhrsw m11, m7
+ REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7
+ paddsw m0, m12
+ paddsw m1, m13
+ paddsw m2, m14
+ paddsw m3, m15
+ paddsw m8, m4
+ movu m4, [o(permD+2)]
+ paddsw m9, m5
+ paddsw m6, m10
+ paddsw m7, m11
+ psrlq m12, m4, 4
+ mova m5, m4
+ mova m10, m4
+ mova m11, m4
+ vpermi2q m4, m0, m2 ; 8 9 12 13
+ vpermt2q m0, m12, m2 ; 0 1 4 5
+ vpermi2q m5, m1, m3 ; 10 11 14 15
+ vpermt2q m1, m12, m3 ; 2 3 6 7
+ vpermi2q m10, m8, m6
+ vpermt2q m8, m12, m6
+ vpermi2q m11, m9, m7
+ vpermt2q m9, m12, m7
+ jmp m(idct_16x16_internal_8bpc).end
+
+%macro ITX_UNPACK_MULHRSW 8 ; dst[1-2], src, tmp, coef[1-4]
+ vpbroadcastd m%4, [o(pw_%5_%6x8)]
+ punpcklwd m%1, m%3, m%3
+ pmulhrsw m%1, m%4
+ vpbroadcastd m%4, [o(pw_%7_%8x8)]
+ punpckhwd m%2, m%3, m%3
+ pmulhrsw m%2, m%4
+%endmacro
+
+cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ cmp eobd, 107
+ jb .fast
+ mova m5, [cq+64*5]
+ mova m3, [cq+64*3]
+ mova m1, [cq+64*1]
+ mova m7, [cq+64*7]
+ mova m2, [cq+64*2]
+ mova m6, [cq+64*6]
+ mova m0, [cq+64*0]
+ mova m4, [cq+64*4]
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ mova m8, [o(idct_8x32p)]
+ vpbroadcastd m9, [o(pw_8192)]
+ REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7
+ punpckldq m8, m0, m1 ; ab
+ punpckhdq m0, m1
+ punpckldq m1, m2, m3 ; cd
+ punpckhdq m2, m3
+ punpckldq m3, m4, m5 ; ef
+ punpckhdq m4, m5
+ punpckldq m5, m6, m7 ; gh
+ punpckhdq m6, m7
+ REPX {pmulhrsw x, m9}, m8, m0, m1, m2, m3, m4, m5, m6
+ punpcklqdq m18, m8, m1 ; 30 2 6 26 31 1 23 9
+ punpckhqdq m14, m8, m1 ; 16 0 12 20 3 29 11 21
+ punpcklqdq m21, m0, m2 ; 14 18 22 10 27 5 19 13
+ punpckhqdq m15, m0, m2 ; 18 4 24 8 7 25 15 17
+ punpcklqdq m20, m3, m5
+ punpckhqdq m16, m3, m5
+ punpcklqdq m19, m4, m6
+ punpckhqdq m17, m4, m6
+ vinserti32x4 ym8, ym18, xm20, 1
+ vshufi32x4 ym1, ym18, ym20, 0x03
+ vinserti32x4 ym9, ym14, xm16, 1
+ vshufi32x4 ym3, ym14, ym16, 0x03
+ vinserti32x4 ym0, ym21, xm19, 1
+ vshufi32x4 ym5, ym21, ym19, 0x03
+ vinserti32x4 ym7, ym15, xm17, 1
+ vshufi32x4 ym6, ym15, ym17, 0x03
+ call m(idct_8x16_internal_8bpc).main2
+ psrlq m12, [o(permB)], 60
+ vpermt2q m14, m12, m16
+ vpermt2q m21, m12, m19
+ vpermt2q m15, m12, m17
+ vpermi2q m12, m18, m20
+ vextracti32x8 ym16, m14, 1
+ vextracti32x8 ym19, m21, 1
+ vextracti32x8 ym17, m15, 1
+ vextracti32x8 ym20, m12, 1
+ call .main2
+ jmp .end
+.fast: ; right half is zero
+ mova m0, [o(int16_perm)]
+ mova ym2, [cq+64*4]
+ vinserti32x8 m2, [cq+64*0], 1
+ mova ym3, [cq+64*6]
+ vinserti32x8 m3, [cq+64*2], 1
+ mova ym4, [cq+64*3]
+ vinserti32x8 m4, [cq+64*5], 1
+ mova ym5, [cq+64*7]
+ vinserti32x8 m5, [cq+64*1], 1
+ REPX {vpermb x, m0, x}, m2, m3, m4, m5
+ call m(idct_16x8_internal_8bpc).main2
+ vbroadcasti32x4 m4, [o(int_shuf3)]
+ vbroadcasti32x4 m5, [o(int_shuf4)]
+ pshufb m2, m4 ; e0 f0 e2 f2 e1 f1 e3 f3
+ pshufb m3, m5 ; g0 h0 g2 h2 g1 h1 g3 h3
+ pshufb m0, m4 ; a0 b0 a2 b2 a1 b1 a3 b3
+ pshufb m1, m5 ; c0 d0 c2 d2 c1 d1 c3 d3
+ vpbroadcastd m4, [o(pw_8192)]
+ psrlq m5, [o(permB)], 60
+ punpckldq m6, m2, m3 ; e0 f0 g0 h0 e2 f2 g2 h2
+ punpckhdq m17, m2, m3 ; e1 f1 g1 h1 e3 f3 g3 h3
+ punpckldq m2, m0, m1 ; a0 b0 c0 d0 a2 b2 c2 d2
+ punpckhdq m16, m0, m1 ; a1 b1 c1 d1 a3 b3 c3 d3
+ REPX {pmulhrsw x, m4}, m6, m17, m2, m16
+ vinserti32x4 ym0, ym2, xm6, 1 ; 0 2
+ vshufi32x4 ym1, ym2, ym6, 0x03 ; 4 6
+ vinserti32x4 ym14, ym16, xm17, 1 ; 1 3
+ vshufi32x4 ym15, ym16, ym17, 0x03 ; 5 7
+ vpermt2q m2, m5, m6 ; 8 10
+ vpermt2q m16, m5, m17 ; 9 11
+ vextracti32x8 ym3, m2, 1 ; 12 14
+ vextracti32x8 ym17, m16, 1 ; 13 15
+ call m(idct_8x16_internal_8bpc).main_fast
+ call .main_fast
+.end:
+ vpbroadcastd ym8, strided
+ pmulld ym8, [o(gather8d)]
+ call .main_end
+ lea r3, [dstq+strideq*4]
+ kxnorb k1, k1, k1
+ lea r4, [dstq+strideq*8]
+ pxor m9, m9
+ lea r1, [r3+strideq*8]
+ kmovb k2, k1
+ vpgatherdq m12{k1}, [r0+ym8]
+ kmovb k1, k2
+ vpgatherdq m13{k2}, [r3+ym8]
+ kmovb k2, k1
+ vpgatherdq m14{k1}, [r4+ym8]
+ kmovb k1, k2
+ vpgatherdq m15{k2}, [r1+ym8]
+ REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7
+ REPX {mova [cq+64*x], m9}, 0, 1, 2, 3, 4, 5, 6, 7
+ punpcklbw m11, m12, m9
+ punpckhbw m12, m9
+ paddw m0, m11
+ paddw m1, m12
+ packuswb m0, m1
+ kmovb k2, k1
+ vpscatterdq [r0+ym8]{k1}, m0
+ punpcklbw m12, m13, m9
+ punpckhbw m13, m9
+ paddw m2, m12
+ paddw m3, m13
+ packuswb m2, m3
+ kmovb k1, k2
+ vpscatterdq [r3+ym8]{k2}, m2
+ punpcklbw m13, m14, m9
+ punpckhbw m14, m9
+ paddw m4, m13
+ paddw m5, m14
+ packuswb m4, m5
+ kmovb k2, k1
+ vpscatterdq [r4+ym8]{k1}, m4
+ punpcklbw m14, m15, m9
+ punpckhbw m15, m9
+ paddw m6, m14
+ paddw m7, m15
+ packuswb m6, m7
+ vpscatterdq [r1+ym8]{k2}, m6
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 32
+ imul r6d, 181
+ add r6d, 128+512
+ sar r6d, 8+2
+ jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2
+INIT_YMM avx512icl
+ALIGN function_align
+cglobal_label .main_fast2 ; bottom three-quarters are zero
+ ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a
+ ITX_UNPACK_MULHRSW 21, 20, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a
+ mova m11, m12
+ mova m17, m20
+ mova m15, m21
+ mova m16, m14
+ jmp .main4
+ALIGN function_align
+cglobal_label .main_fast ; bottom half is zero
+ ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a
+ ITX_UNPACK_MULHRSW 21, 15, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a
+ ITX_UNPACK_MULHRSW 20, 16, 16, 8, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a
+ ITX_UNPACK_MULHRSW 19, 17, 17, 8, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a
+ jmp .main3
+ALIGN function_align
+cglobal_label .main
+ punpcklwd m12, m21, m14 ; in31 in1
+ punpckhwd m14, m21 ; in3 in29
+ punpcklwd m21, m20, m15 ; in27 in5
+ punpckhwd m15, m20 ; in7 in25
+ punpcklwd m20, m19, m16 ; in23 in9
+ punpckhwd m16, m19 ; in11 in21
+ punpcklwd m19, m18, m17 ; in19 in13
+ punpckhwd m17, m18 ; in15 in17
+.main2:
+ ITX_MUL2X_PACK 12, 8, 9, 10, 201, 4091, 5 ; t16a, t31a
+ ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a
+ ITX_MUL2X_PACK 21, 8, 9, 10, 995, 3973, 5 ; t20a, t27a
+ ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a
+ ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a
+ ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a
+ ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a
+ ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a
+.main3:
+ psubsw m11, m12, m17 ; t17 t30
+ paddsw m12, m17 ; t16 t31
+ psubsw m17, m15, m20 ; t18 t29
+ paddsw m20, m15 ; t19 t28
+ psubsw m15, m21, m16 ; t21 t26
+ paddsw m21, m16 ; t20 t27
+ psubsw m16, m14, m19 ; t22 t25
+ paddsw m14, m19 ; t23 t24
+.main4:
+ ITX_MUL2X_PACK 11, 18, 19, 10, 799, 4017, 5 ; t17a t30a
+ ITX_MUL2X_PACK 17, 18, 19, 10, m4017, 799, 5 ; t18a t29a
+ ITX_MUL2X_PACK 15, 18, 19, 10, 3406, 2276, 5 ; t21a t26a
+ ITX_MUL2X_PACK 16, 18, 19, 10, m2276, 3406, 5 ; t22a t25a
+ vpbroadcastd m8, [o(pw_m3784_1567)]
+ psubsw m19, m12, m20 ; t19a t28a
+ paddsw m20, m12 ; t16a t31a
+ psubsw m12, m14, m21 ; t20a t27a
+ paddsw m14, m21 ; t23a t24a
+ psubsw m21, m11, m17 ; t18 t29
+ paddsw m11, m17 ; t17 t30
+ psubsw m17, m16, m15 ; t21 t26
+ paddsw m16, m15 ; t22 t25
+ ITX_MUL2X_PACK 21, 18, 15, 10, 1567_3784, 8, 20 ; t18a t29a
+ ITX_MUL2X_PACK 19, 18, 15, 10, 1567_3784, 8, 20 ; t19 t28
+ ITX_MUL2X_PACK 12, 18, 15, 10, 8, m1567_m3784, 36 ; t20 t27
+ ITX_MUL2X_PACK 17, 18, 15, 10, 8, m1567_m3784, 36 ; t21a t26a
+ vbroadcasti32x4 m18, [o(deint_shuf)]
+ vpbroadcastd m8, [o(pw_m2896_2896)]
+ vpbroadcastd m9, [o(pw_2896_2896)]
+ psubsw m15, m20, m14 ; t23 t24
+ paddsw m20, m14 ; t16 t31
+ psubsw m14, m11, m16 ; t22a t25a
+ paddsw m11, m16 ; t17a t30a
+ psubsw m16, m21, m17 ; t21 t26
+ paddsw m21, m17 ; t18 t29
+ psubsw m17, m19, m12 ; t20a t27a
+ paddsw m19, m12 ; t19a t28a
+ REPX {pshufb x, m18}, m20, m11, m21, m19
+ ITX_MUL2X_PACK 15, 18, 12, 10, 8, 9, 8 ; t23a t22a
+ ITX_MUL2X_PACK 14, 13, 15, 10, 8, 9, 8 ; t22 t25
+ packssdw m18, m13 ; t23a t22
+ packssdw m12, m15 ; t24a t25
+ ITX_MUL2X_PACK 16, 13, 15, 10, 8, 9, 8 ; t21a t26a
+ ITX_MUL2X_PACK 17, 16, 14, 10, 8, 9, 8 ; t20 t27
+ packssdw m16, m13 ; t20 t21a
+ packssdw m14, m15 ; t27 t26a
+ punpcklqdq m13, m19, m21 ; t19a t18
+ punpckhqdq m19, m21 ; t28a t29
+ punpcklqdq m21, m20, m11 ; t16 t17a
+ punpckhqdq m20, m11 ; t31 t30a
+INIT_ZMM avx512icl
+ mova m15, [o(permA)]
+ ret
+cglobal_label .main_end
+ vpbroadcastd m10, [o(pw_2048)]
+ vpermt2q m0, m15, m1 ; t0 t1 t2 t3
+ vpermt2q m20, m15, m19 ; t31 t30a t29 t28a
+ vpermt2q m2, m15, m3 ; t4 t5 t6 t7
+ vpermt2q m14, m15, m12 ; t27 t26a t25 t24a
+ vpermt2q m4, m15, m5 ; t8 t9 t10 t11
+ vpermt2q m18, m15, m16 ; t23a t22 t21a t20
+ vpermt2q m6, m15, m7 ; t12 t13 t14 t15
+ vpermt2q m13, m15, m21 ; t19a t18 t17a t16
+ psubsw m7, m0, m20 ; out31 out30 out29 out28
+ paddsw m0, m20 ; out0 out1 out2 out3
+ psubsw m5, m2, m14 ; out27 out26 out25 out24
+ paddsw m2, m14 ; out4 out5 out6 out7
+ psubsw m3, m4, m18 ; out23 out22 out21 out20
+ paddsw m4, m18 ; out8 out9 out10 out11
+ psubsw m1, m6, m13 ; out19 out18 out17 out16
+ paddsw m6, m13 ; out12 out13 out14 out15
+ vzeroupper
+ ret
+
+%macro LOAD_PACKED_16X2 3 ; dst, row[1-2]
+ vbroadcasti32x4 ym%1, [cq+16*%2]
+ vbroadcasti32x4 ym8, [cq+16*%3]
+ shufpd ym%1, ym8, 0x0c
+%endmacro
+
+cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 4, 0, dst, stride, c, eob
+%undef cmp
+ test eobd, eobd
+ jz .dconly
+ lea r5, [o_base]
+ LOAD_PACKED_16X2 0, 0, 2 ; in0 in2
+ LOAD_PACKED_16X2 1, 4, 6 ; in4 in6
+ LOAD_PACKED_16X2 2, 8, 10 ; in8 in10
+ LOAD_PACKED_16X2 3, 12, 14 ; in12 in14
+ LOAD_PACKED_16X2 14, 1, 3 ; in1 in3
+ LOAD_PACKED_16X2 15, 5, 7 ; in5 in7
+ LOAD_PACKED_16X2 16, 9, 11 ; in9 in11
+ LOAD_PACKED_16X2 17, 13, 15 ; in13 in15
+ pxor m4, m4
+ REPX {mova [cq+64*x], m4}, 0, 1, 2, 3
+ cmp eobd, 107
+ jb .fast
+ LOAD_PACKED_16X2 4, 16, 18 ; in16 in18
+ LOAD_PACKED_16X2 5, 20, 22 ; in20 in22
+ LOAD_PACKED_16X2 6, 24, 26 ; in24 in26
+ LOAD_PACKED_16X2 7, 28, 30 ; in28 in30
+ call m(idct_8x16_internal_8bpc).main
+ LOAD_PACKED_16X2 18, 19, 17 ; in19 in17
+ LOAD_PACKED_16X2 19, 23, 21 ; in23 in21
+ LOAD_PACKED_16X2 20, 27, 25 ; in27 in25
+ LOAD_PACKED_16X2 21, 31, 29 ; in31 in29
+ pxor m8, m8
+ REPX {mova [cq+64*x], m8}, 4, 5, 6, 7
+ call m(inv_txfm_add_dct_dct_8x32_8bpc).main
+ jmp .pass2
+.fast: ; bottom half is zero
+ mova ym5, ym4
+ mova ym6, ym4
+ mova ym7, ym4
+ call m(idct_8x16_internal_8bpc).main
+ call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast
+.pass2:
+ vpbroadcastd m10, [o(pw_8192)]
+ vpermt2q m0, m15, m4 ; t0 t1 t9 t8
+ vpermt2q m20, m15, m18 ; t31 t30a t23a t22
+ vpermt2q m3, m15, m7 ; t7 t6 t14 t15
+ vpermt2q m12, m15, m21 ; t25 t24a t17a t16
+ vpermt2q m2, m15, m6 ; t4 t5 t13 t12
+ vpermt2q m14, m15, m13 ; t23a t22 t21a t20
+ vpermt2q m1, m15, m5 ; t3 t2 t10 t11
+ vpermt2q m19, m15, m16 ; t27 t26a t19a t18
+ psubsw m8, m0, m20 ; out31 out30 out22 out23
+ paddsw m0, m20 ; out0 out1 out9 out8
+ paddsw m6, m3, m12 ; out7 out6 out14 out15
+ psubsw m3, m12 ; out24 out25 out17 out16
+ psubsw m5, m2, m14 ; out27 out26 out18 out19
+ paddsw m4, m2, m14 ; out4 out5 out13 out12
+ psubsw m7, m1, m19 ; out28 out29 out21 out20
+ paddsw m2, m1, m19 ; out3 out2 out10 out11
+ vzeroupper
+ vshufi32x4 m1, m0, m3, q1221 ; out1 out9 out17 out25
+ vshufi32x4 m0, m3, q0330 ; out0 out8 out16 out24
+ vshufi32x4 m3, m2, m5, q0330 ; out3 out11 out19 out27
+ vshufi32x4 m2, m5, q1221 ; out2 out10 out18 out26
+ vshufi32x4 m5, m4, m7, q1221 ; out5 out13 out21 out29
+ vshufi32x4 m4, m7, q0330 ; out4 out12 out20 out28
+ vshufi32x4 m7, m6, m8, q0330 ; out7 out15 out23 out31
+ vshufi32x4 m6, m8, q1221 ; out6 out14 out22 out30
+ REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8
+ call .main
+ vpbroadcastd m8, [o(pw_2048)]
+ REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7
+ lea r2, [strideq*3]
+ lea r3, [dstq+strideq*4]
+ movshdup m12, [o(permD)]
+ pmovzxbw m8, [dstq+strideq*0]
+ pmovzxbw m9, [dstq+strideq*1]
+ pmovzxbw m10, [dstq+strideq*2]
+ pmovzxbw m11, [dstq+r2 ]
+ paddw m0, m8
+ paddw m1, m9
+ paddw m2, m10
+ paddw m3, m11
+ pmovzxbw m8, [r3+strideq*0]
+ pmovzxbw m9, [r3+strideq*1]
+ pmovzxbw m10, [r3+strideq*2]
+ pmovzxbw m11, [r3+r2 ]
+ paddw m4, m8
+ paddw m5, m9
+ paddw m6, m10
+ paddw m7, m11
+ packuswb m0, m1
+ packuswb m2, m3
+ vpermq m0, m12, m0
+ vpermq m2, m12, m2
+ mova [dstq+strideq*0], ym0
+ vextracti32x8 [dstq+strideq*1], m0, 1
+ mova [dstq+strideq*2], ym2
+ vextracti32x8 [dstq+r2 ], m2, 1
+ packuswb m4, m5
+ packuswb m6, m7
+ vpermq m4, m12, m4
+ vpermq m6, m12, m6
+ mova [r3+strideq*0], ym4
+ vextracti32x8 [r3+strideq*1], m4, 1
+ mova [r3+strideq*2], ym6
+ vextracti32x8 [r3+r2 ], m6, 1
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 8
+.dconly2:
+ imul r6d, 181
+ add r6d, 128+512
+ sar r6d, 8+2
+.dconly3:
+ imul r6d, 181
+ add r6d, 128+2048
+ sar r6d, 8+4
+ pxor m2, m2
+ vpbroadcastw m3, r6d
+.dconly_loop:
+ mova ym1, [dstq+strideq*0]
+ vinserti32x8 m1, [dstq+strideq*1], 1
+ punpcklbw m0, m1, m2
+ punpckhbw m1, m2
+ paddw m0, m3
+ paddw m1, m3
+ packuswb m0, m1
+ mova [dstq+strideq*0], ym0
+ vextracti32x8 [dstq+strideq*1], m0, 1
+ lea dstq, [dstq+strideq*2]
+ sub r3d, 2
+ jg .dconly_loop
+ RET
+ALIGN function_align
+cglobal_label .main
+ vpbroadcastd m10, [o(pd_2048)]
+.main2:
+ ITX_MULSUB_2W 5, 3, 8, 9, 10, 3406, 2276 ; t5a, t6a
+ ITX_MULSUB_2W 1, 7, 8, 9, 10, 799, 4017 ; t4a, t7a
+ ITX_MULSUB_2W 2, 6, 8, 9, 10, 1567, 3784 ; t2, t3
+ vpbroadcastd m11, [o(pw_2896_2896)]
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ ITX_MULSUB_2W 0, 4, 8, 9, 10, 11, 12 ; t1, t0
+.main3:
+ paddsw m8, m1, m5 ; t4
+ psubsw m1, m5 ; t5a
+ paddsw m9, m7, m3 ; t7
+ psubsw m7, m3 ; t6a
+ ITX_MULSUB_2W 7, 1, 3, 5, 10, 11, 12 ; t5, t6
+ psubsw m5, m0, m2 ; dct4 out2
+ paddsw m2, m0 ; dct4 out1
+ paddsw m0, m4, m6 ; dct4 out0
+ psubsw m4, m6 ; dct4 out3
+ psubsw m6, m2, m1 ; out6
+ paddsw m1, m2 ; out1
+ paddsw m2, m5, m7 ; out2
+ psubsw m5, m7 ; out5
+ psubsw m7, m0, m9 ; out7
+ paddsw m0, m9 ; out0
+ paddsw m3, m4, m8 ; out3
+ psubsw m4, m8 ; out4
+ ret
+
+cglobal inv_txfm_add_identity_identity_8x32_8bpc, 3, 5, 0, dst, stride, c
+ vpbroadcastd m7, [pw_5]
+ paddsw m0, m7, [cq+64*0]
+ paddsw m1, m7, [cq+64*1]
+ vpbroadcastd ym9, strided
+ paddsw m2, m7, [cq+64*2]
+ paddsw m3, m7, [cq+64*3]
+ paddsw m4, m7, [cq+64*4]
+ paddsw m5, m7, [cq+64*5]
+ paddsw m6, m7, [cq+64*6]
+ paddsw m7, [cq+64*7]
+ pmulld ym14, ym9, [pd_0to15]
+ lea r3, [dstq+strideq*1]
+ lea r4, [dstq+strideq*2]
+ kxnorb k1, k1, k1
+ pxor m13, m13
+ add r1, r4 ; dstq+strideq*3
+ kmovb k2, k1
+ vpgatherdq m9{k1}, [r0+ym14*4]
+ kmovb k1, k2
+ vpgatherdq m10{k2}, [r3+ym14*4]
+ kmovb k2, k1
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8
+ REPX {psraw x, 3}, m0, m1, m2, m3, m4, m5, m6, m7
+ vpgatherdq m11{k1}, [r4+ym14*4]
+ kmovb k1, k2
+ vpgatherdq m12{k2}, [r1+ym14*4]
+ REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7
+ punpcklbw m8, m9, m13 ; 0 8 16 24
+ punpckhbw m9, m13 ; 4 12 20 28
+ paddw m0, m8
+ paddw m4, m9
+ packuswb m0, m4
+ kmovb k2, k1
+ vpscatterdq [r0+ym14*4]{k1}, m0
+ punpcklbw m8, m10, m13 ; 1 9 17 25
+ punpckhbw m10, m13 ; 5 13 21 29
+ paddw m1, m8
+ paddw m5, m10
+ packuswb m1, m5
+ kmovb k1, k2
+ vpscatterdq [r3+ym14*4]{k2}, m1
+ punpcklbw m8, m11, m13 ; 2 10 18 26
+ punpckhbw m11, m13 ; 6 14 22 30
+ paddw m2, m8
+ paddw m6, m11
+ packuswb m2, m6
+ kmovb k2, k1
+ vpscatterdq [r4+ym14*4]{k1}, m2
+ punpcklbw m8, m12, m13 ; 3 11 19 27
+ punpckhbw m12, m13 ; 7 15 23 31
+ paddw m3, m8
+ paddw m7, m12
+ packuswb m3, m7
+ vpscatterdq [r1+ym14*4]{k2}, m3
+ RET
+
+cglobal inv_txfm_add_identity_identity_32x8_8bpc, 3, 5, 0, dst, stride, c
+ vpbroadcastd m0, [pw_4096]
+ pmulhrsw m3, m0, [cq+64*0]
+ pmulhrsw m4, m0, [cq+64*4]
+ pmulhrsw m6, m0, [cq+64*1]
+ pmulhrsw m5, m0, [cq+64*5]
+ pmulhrsw m7, m0, [cq+64*2]
+ pmulhrsw m2, m0, [cq+64*6]
+ pmulhrsw m8, m0, [cq+64*3]
+ pmulhrsw m0, [cq+64*7]
+ mova m13, [int8_permA]
+ lea r3, [strideq*3]
+ lea r4, [dstq+strideq*4]
+ punpckldq m1, m3, m4
+ punpckhdq m3, m4
+ punpckldq m4, m6, m5
+ punpckhdq m6, m5
+ punpckldq m5, m7, m2
+ punpckhdq m7, m2
+ punpckldq m2, m8, m0
+ punpckhdq m8, m0
+ mova ym9, [dstq+strideq*0]
+ vinserti32x8 m9, [dstq+strideq*2], 1
+ mova ym10, [dstq+strideq*1]
+ vinserti32x8 m10, [dstq+r3 ], 1
+ mova ym11, [r4+strideq*0]
+ vinserti32x8 m11, [r4+strideq*2], 1
+ mova ym12, [r4+strideq*1]
+ vinserti32x8 m12, [r4+r3 ], 1
+ REPX {vpermb x, m13, x}, m1, m4, m5, m2, m3, m6, m7, m8
+ pxor m13, m13
+ REPX {mova [cq+64*x], m13}, 0, 1, 2, 3, 4, 5, 6, 7
+ punpcklqdq m0, m1, m4 ; a0 a2 c0 c2
+ punpckhqdq m1, m4 ; b0 b2 d0 d2
+ punpcklqdq m4, m5, m2 ; a1 a3 c1 c3
+ punpckhqdq m5, m2 ; b1 b3 d1 d3
+ punpcklqdq m2, m3, m6 ; e0 e2 g0 g2
+ punpckhqdq m3, m6 ; f0 f2 h0 h2
+ punpcklqdq m6, m7, m8 ; e1 e3 g1 g3
+ punpckhqdq m7, m8 ; f1 f3 h1 h3
+ punpcklbw m8, m9, m13
+ punpckhbw m9, m13
+ paddw m0, m8
+ paddw m4, m9
+ packuswb m0, m4
+ mova [dstq+strideq*0], ym0
+ vextracti32x8 [dstq+strideq*2], m0, 1
+ punpcklbw m8, m10, m13
+ punpckhbw m10, m13
+ paddw m1, m8
+ paddw m5, m10
+ packuswb m1, m5
+ mova [dstq+strideq*1], ym1
+ vextracti32x8 [dstq+r3 ], m1, 1
+ punpcklbw m8, m11, m13
+ punpckhbw m11, m13
+ paddw m2, m8
+ paddw m6, m11
+ packuswb m2, m6
+ mova [r4+strideq*0], ym2
+ vextracti32x8 [r4+strideq*2], m2, 1
+ punpcklbw m8, m12, m13
+ punpckhbw m12, m13
+ paddw m3, m8
+ paddw m7, m12
+ packuswb m3, m7
+ mova [r4+strideq*1], ym3
+ vextracti32x8 [r4+r3 ], m3, 1
+ RET
+
+%macro IDCT_16x32_END 3 ; src[1-2], row
+ mova xm8, [dstq+strideq*0]
+ vinserti32x4 ym8, [dstq+strideq*1], 1
+ mova xm9, [dstq+r3 ]
+ vinserti32x4 ym9, [dstq+strideq*2], 1
+ pmulhrsw m%1, m10
+ pmulhrsw m%2, m10
+ vpermb m8, m11, m8
+ vpermb m9, m11, m9
+ mova [cq+64*(%3*2+0)], m13
+ mova [cq+64*(%3*2+1)], m13
+ paddw m8, m%1
+ paddw m9, m%2
+ packuswb m8, m9
+ vpermd m8, m12, m8
+ mova [dstq+strideq*0], xm8
+ vextracti32x4 [dstq+strideq*1], ym8, 1
+ vextracti32x4 [dstq+strideq*2], m8, 2
+ vextracti32x4 [dstq+r3 ], m8, 3
+%if %1 != 20
+ lea dstq, [dstq+strideq*4]
+%endif
+%endmacro
+
+cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 4, 22, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ vpbroadcastd m15, [o(pw_2896x8)]
+ cmp eobd, 151
+ jb .fast
+ pmulhrsw m5, m15, [cq+64*10]
+ pmulhrsw m3, m15, [cq+64* 6]
+ pmulhrsw m1, m15, [cq+64* 2]
+ pmulhrsw m7, m15, [cq+64*14]
+ pmulhrsw m2, m15, [cq+64* 4]
+ pmulhrsw m6, m15, [cq+64*12]
+ pmulhrsw m0, m15, [cq+64* 0]
+ pmulhrsw m4, m15, [cq+64* 8]
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ pmulhrsw m14, m15, [cq+64* 1]
+ pmulhrsw m21, m15, [cq+64*15]
+ pmulhrsw m18, m15, [cq+64* 9]
+ pmulhrsw m17, m15, [cq+64* 7]
+ pmulhrsw m16, m15, [cq+64* 5]
+ pmulhrsw m19, m15, [cq+64*11]
+ pmulhrsw m20, m15, [cq+64*13]
+ pmulhrsw m15, [cq+64* 3]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ mova m8, [o(idct_16x32p)]
+ vpbroadcastd m9, [o(pw_16384)]
+ REPX {vpermb x, m8, x}, m0, m1, m2, m3, m4, m5, m6, m7, \
+ m14, m15, m16, m17, m18, m19, m20, m21
+ punpckldq m8, m0, m1
+ punpckhdq m0, m1
+ punpckldq m1, m2, m3
+ punpckhdq m2, m3
+ REPX {pmulhrsw x, m9}, m8, m0, m1, m2
+ punpckldq m3, m4, m5
+ punpckhdq m4, m5
+ punpckldq m5, m6, m7
+ punpckhdq m6, m7
+ REPX {pmulhrsw x, m9}, m3, m4, m5, m6
+ punpckldq m7, m14, m15
+ punpckhdq m14, m15
+ punpckldq m15, m16, m17
+ punpckhdq m16, m17
+ REPX {pmulhrsw x, m9}, m7, m14, m15, m16
+ punpckldq m17, m18, m19
+ punpckhdq m18, m19
+ punpckldq m19, m20, m21
+ punpckhdq m20, m21
+ REPX {pmulhrsw x, m9}, m17, m18, m19, m20
+ punpcklqdq m21, m8, m1
+ punpckhqdq m8, m1
+ punpcklqdq m1, m0, m2
+ punpckhqdq m0, m2
+ punpcklqdq m2, m3, m5
+ punpckhqdq m3, m5
+ punpcklqdq m5, m4, m6
+ punpckhqdq m4, m6
+ punpcklqdq m6, m7, m15
+ punpckhqdq m7, m15
+ punpcklqdq m15, m14, m16
+ punpckhqdq m14, m16
+ punpcklqdq m16, m17, m19
+ punpckhqdq m17, m19
+ punpcklqdq m19, m18, m20
+ punpckhqdq m18, m20
+ vinserti32x8 m20, m21, ym2, 1
+ vshufi32x4 m21, m2, q3232
+ vinserti32x8 m2, m8, ym3, 1
+ vshufi32x4 m8, m3, q3232
+ vinserti32x8 m3, m1, ym5, 1
+ vshufi32x4 m1, m5, q3232
+ vinserti32x8 m5, m0, ym4, 1
+ vshufi32x4 m0, m4, q3232
+ vinserti32x8 m4, m6, ym16, 1
+ vshufi32x4 m6, m16, q3232
+ vinserti32x8 m16, m7, ym17, 1
+ vshufi32x4 m7, m17, q3232
+ vinserti32x8 m17, m15, ym19, 1
+ vshufi32x4 m15, m19, q3232
+ vinserti32x8 m19, m14, ym18, 1
+ vshufi32x4 m14, m18, q3232
+ vshufi32x4 m18, m21, m6, q3131 ; 27 5
+ vshufi32x4 m21, m6, q2020 ; 31 1
+ vshufi32x4 m6, m8, m7, q2020 ; 24 8
+ vshufi32x4 m8, m7, q3131 ; 30 2
+ vshufi32x4 m7, m1, m15, q2020 ; 28 4
+ vshufi32x4 m1, m15, q3131 ; 6 26
+ vshufi32x4 m15, m0, m14, q2020 ; 7 25
+ vshufi32x4 m0, m14, q3131 ; 14 18
+ vshufi32x4 m14, m20, m4, q2020 ; 3 29
+ vshufi32x4 m20, m4, q3131 ; 23 9
+ vshufi32x4 m9, m3, m17, q2020 ; 16 0
+ vshufi32x4 m3, m17, q3131 ; 12 20
+ vshufi32x4 m17, m5, m19, q2020 ; 15 17
+ vshufi32x4 m5, m19, q3131 ; 22 10
+ vshufi32x4 m19, m2, m16, q2020 ; 19 13
+ vshufi32x4 m16, m2, m16, q3131 ; 11 21
+ call m(idct_16x16_internal_8bpc).main3
+ call .main_oddhalf
+ jmp .pass2
+.fast: ; right half is zero
+ mova ym8, [cq+64*15]
+ vinserti32x8 m8, [cq+64* 1], 1
+ mova m2, [o(int16_perm)]
+ mova ym9, [cq+64* 8]
+ vinserti32x8 m9, [cq+64* 0], 1
+ mova ym0, [cq+64* 7]
+ vinserti32x8 m0, [cq+64* 9], 1
+ mova ym7, [cq+64*14]
+ vinserti32x8 m7, [cq+64* 2], 1
+ mova ym1, [cq+64* 3]
+ vinserti32x8 m1, [cq+64*13], 1
+ mova ym3, [cq+64* 6]
+ vinserti32x8 m3, [cq+64*10], 1
+ mova ym5, [cq+64*11]
+ vinserti32x8 m5, [cq+64* 5], 1
+ mova ym6, [cq+64*12]
+ vinserti32x8 m6, [cq+64* 4], 1
+ REPX {pmulhrsw x, m15}, m8, m9, m0, m7, m1, m3, m5, m6
+ REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6
+ call m(idct_16x16_internal_8bpc).main2
+ vbroadcasti32x4 m8, [o(int_shuf3)]
+ vbroadcasti32x4 m9, [o(int_shuf4)]
+ vpbroadcastd m11, [o(pw_16384)]
+ pshufb m0, m8
+ pshufb m1, m9
+ pshufb m2, m8
+ pshufb m3, m9
+ REPX {pmulhrsw x, m11}, m0, m1, m2, m3
+ pshufb m4, m8
+ pshufb m5, m9
+ pshufb m6, m8
+ pshufb m7, m9
+ REPX {pmulhrsw x, m11}, m4, m5, m6, m7
+ punpckhdq m17, m0, m1
+ punpckldq m0, m1
+ punpckhdq m16, m2, m3
+ punpckldq m2, m3
+ punpckhdq m18, m4, m5
+ punpckldq m4, m5
+ punpckhdq m5, m6, m7
+ punpckldq m6, m7
+ vinserti32x8 m1, m0, ym2, 1
+ vshufi32x4 m3, m0, m2, q3232
+ vinserti32x8 m2, m4, ym6, 1
+ vshufi32x4 m4, m6, q3232
+ vinserti32x8 m15, m17, ym16, 1
+ vshufi32x4 m17, m16, q3232
+ vinserti32x8 m16, m18, ym5, 1
+ vshufi32x4 m18, m5, q3232
+ vshufi32x4 m0, m1, m2, q2020 ; 0 2
+ vshufi32x4 m1, m2, q3131 ; 4 6
+ vshufi32x4 m2, m3, m4, q2020 ; 8 10
+ vshufi32x4 m3, m4, q3131 ; 12 14
+ vshufi32x4 m14, m15, m16, q2020 ; 1 3
+ vshufi32x4 m15, m16, q3131 ; 5 7
+ vshufi32x4 m16, m17, m18, q2020 ; 9 11
+ vshufi32x4 m17, m18, q3131 ; 13 15
+ pxor m6, m6
+ punpckhwd m8, m0, m0
+ punpcklwd m9, m6, m0
+ punpckhwd m0, m3, m3
+ punpckhwd m5, m2, m2
+ punpcklwd m7, m1, m1
+ punpckhwd m1, m1
+ punpcklwd m3, m3
+ punpcklwd m6, m2
+ call m(idct_16x16_internal_8bpc).main_fast5
+ punpcklwd m21, m14, m14
+ punpckhwd m14, m14
+ punpcklwd m18, m15, m15
+ punpckhwd m15, m15
+ punpcklwd m20, m16, m16
+ punpckhwd m16, m16
+ punpcklwd m19, m17, m17
+ punpckhwd m17, m17
+ call .main_oddhalf_fast
+.pass2:
+ vpbroadcastd m10, [o(pw_2048)]
+ mova m11, [o(end_16x32p)]
+ lea r3, [strideq*3]
+ pxor m13, m13
+ psrld m12, m11, 8
+ IDCT_16x32_END 0, 1, 0
+ IDCT_16x32_END 2, 3, 1
+ IDCT_16x32_END 4, 5, 2
+ IDCT_16x32_END 6, 7, 3
+ IDCT_16x32_END 14, 15, 4
+ IDCT_16x32_END 16, 17, 5
+ IDCT_16x32_END 18, 19, 6
+ IDCT_16x32_END 20, 21, 7
+ RET
+ALIGN function_align
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 32
+ jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly
+ALIGN function_align
+cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero
+ vpbroadcastd m8, [o(pw_201_4091x8)]
+ vpbroadcastd m20, [o(pw_m1380_3857x8)]
+ vpbroadcastd m9, [o(pw_995_3973x8)]
+ vpbroadcastd m16, [o(pw_m601_4052x8)]
+ pmulhrsw m21, m8 ; t16a, t31a
+ pmulhrsw m20, m15 ; t19a, t28a
+ pmulhrsw m18, m9 ; t20a, t27a
+ pmulhrsw m14, m16 ; t23a, t24a
+ mova m8, m21
+ mova m17, m20
+ mova m15, m18
+ mova m16, m14
+ jmp .main3
+ALIGN function_align
+cglobal_label .main_oddhalf_fast ; bottom half is zero
+ vpbroadcastd m8, [o(pw_201_4091x8)]
+ vpbroadcastd m9, [o(pw_m2751_3035x8)]
+ vpbroadcastd m11, [o(pw_1751_3703x8)]
+ vpbroadcastd m12, [o(pw_m1380_3857x8)]
+ pmulhrsw m21, m8 ; t16a, t31a
+ vpbroadcastd m8, [o(pw_995_3973x8)]
+ pmulhrsw m17, m9 ; t17a, t30a
+ vpbroadcastd m9, [o(pw_m2106_3513x8)]
+ pmulhrsw m20, m11 ; t18a, t29a
+ vpbroadcastd m11, [o(pw_2440_3290x8)]
+ pmulhrsw m15, m12 ; t19a, t28a
+ vpbroadcastd m12, [o(pw_m601_4052x8)]
+ pmulhrsw m18, m8 ; t20a, t27a
+ pmulhrsw m16, m9 ; t21a, t26a
+ pmulhrsw m19, m11 ; t22a, t25a
+ pmulhrsw m14, m12 ; t23a, t24a
+ jmp .main2
+ALIGN function_align
+cglobal_label .main_oddhalf
+ ITX_MUL2X_PACK 21, 8, 9, 10, 201, 4091, 5 ; t16a, t31a
+ ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a
+ ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a
+ ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a
+ ITX_MUL2X_PACK 18, 8, 9, 10, 995, 3973, 5 ; t20a, t27a
+ ITX_MUL2X_PACK 16, 8, 9, 10, 3513, 2106, 5 ; t21a, t26a
+ ITX_MUL2X_PACK 19, 8, 9, 10, 2440, 3290, 5 ; t22a, t25a
+ ITX_MUL2X_PACK 14, 8, 9, 10, 4052, 601, 5 ; t23a, t24a
+.main2:
+ psubsw m8, m21, m17 ; t17 t30
+ paddsw m21, m17 ; t16 t31
+ psubsw m17, m15, m20 ; t18 t29
+ paddsw m20, m15 ; t19 t28
+ psubsw m15, m18, m16 ; t21 t26
+ paddsw m18, m16 ; t20 t27
+ psubsw m16, m14, m19 ; t22 t25
+ paddsw m14, m19 ; t23 t24
+.main3:
+ ITX_MUL2X_PACK 8, 9, 19, 10, 799, 4017, 5 ; t17a t30a
+ ITX_MUL2X_PACK 17, 9, 19, 10, m4017, 799, 5 ; t18a t29a
+ ITX_MUL2X_PACK 15, 9, 19, 10, 3406, 2276, 5 ; t21a t26a
+ ITX_MUL2X_PACK 16, 9, 19, 10, m2276, 3406, 5 ; t22a t25a
+ vpbroadcastd m11, [o(pw_m3784_1567)]
+ psubsw m19, m21, m20 ; t19a t28a
+ paddsw m21, m20 ; t16a t31a
+ psubsw m20, m14, m18 ; t20a t27a
+ paddsw m14, m18 ; t23a t24a
+ psubsw m18, m8, m17 ; t18 t29
+ paddsw m8, m17 ; t17 t30
+ psubsw m17, m16, m15 ; t21 t26
+ paddsw m15, m16 ; t22 t25
+ ITX_MUL2X_PACK 18, 9, 16, 10, 1567_3784, 11, 20 ; t18a t29a
+ ITX_MUL2X_PACK 19, 9, 16, 10, 1567_3784, 11, 20 ; t19 t28
+ ITX_MUL2X_PACK 20, 9, 16, 10, 11, m1567_m3784, 36 ; t20 t27
+ ITX_MUL2X_PACK 17, 9, 16, 10, 11, m1567_m3784, 36 ; t21a t26a
+ vbroadcasti32x4 m9, [o(deint_shuf)]
+ psubsw m16, m21, m14 ; t23 t24
+ paddsw m14, m21 ; t16 t31
+ psubsw m21, m8, m15 ; t22a t25a
+ paddsw m15, m8 ; t17a t30a
+ psubsw m8, m18, m17 ; t21 t26
+ paddsw m18, m17 ; t18 t29
+ paddsw m17, m19, m20 ; t19a t28a
+ psubsw m19, m20 ; t20a t27a
+ vpbroadcastd m11, [o(pw_m2896_2896)]
+ vpbroadcastd m12, [o(pw_2896_2896)]
+ REPX {pshufb x, m9}, m14, m15, m18, m17
+ mova m9, m10
+ vpdpwssd m9, m16, m11
+ mova m20, m10
+ vpdpwssd m20, m21, m11
+ psrad m9, 12
+ psrad m20, 12
+ packssdw m9, m20 ; t23a t22
+ mova m20, m10
+ vpdpwssd m20, m16, m12
+ mova m16, m10
+ vpdpwssd m16, m21, m12
+ psrad m20, 12
+ psrad m16, 12
+ packssdw m16, m20, m16 ; t24a t25
+ ITX_MUL2X_PACK 8, 21, 20, 10, 11, 12, 8 ; t21a t26a
+ ITX_MUL2X_PACK 19, 8, 11, 10, 11, 12, 8 ; t20 t27
+ packssdw m11, m20 ; t27 t26a
+ packssdw m8, m21 ; t20 t21a
+ punpcklqdq m20, m14, m15 ; t16 t17a
+ punpckhqdq m14, m15 ; t31 t30a
+ punpckhqdq m15, m17, m18 ; t28a t29
+ punpcklqdq m17, m18 ; t19a t18
+ psubsw m21, m0, m14 ; out31 out30
+ paddsw m0, m14 ; out0 out1
+ psubsw m14, m7, m20 ; out16 out17
+ paddsw m7, m20 ; out15 out14
+ psubsw m20, m1, m15 ; out28 out29
+ paddsw m1, m15 ; out3 out2
+ psubsw m15, m6, m17 ; out19 out18
+ paddsw m6, m17 ; out12 out13
+ psubsw m17, m4, m9 ; out23 out22
+ paddsw m4, m9 ; out8 out9
+ psubsw m18, m3, m16 ; out24 out25
+ paddsw m3, m16 ; out7 out6
+ psubsw m16, m5, m8 ; out20 out21
+ paddsw m5, m8 ; out11 out10
+ psubsw m19, m2, m11 ; out27 out26
+ paddsw m2, m11 ; out4 out5
+ ret
+
+cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 22, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ mova m21, [o(permB)]
+ vpermq m1, m21, [cq+64* 0] ; 0 1
+ vpermq m14, m21, [cq+64* 1] ; 2 3
+ vpermq m20, m21, [cq+64* 2] ; 4 5
+ vpermq m15, m21, [cq+64* 3] ; 6 7
+ vpbroadcastd m8, [o(pw_2896x8)]
+ vpermq m2, m21, [cq+64* 4] ; 8 9
+ vpermq m16, m21, [cq+64* 5] ; 10 11
+ vpermq m3, m21, [cq+64* 6] ; 12 13
+ vpermq m17, m21, [cq+64* 7] ; 14 15
+ REPX {pmulhrsw x, m8}, m1, m14, m20, m15, m2, m16, m3, m17
+ pxor m12, m12
+ REPX {mova [cq+64*x], m12}, 0, 1, 2, 3, 4, 5, 6, 7
+ cmp eobd, 151
+ jb .fast
+ vpermq m9, m21, [cq+64* 8] ; 16 17
+ vpermq m19, m21, [cq+64* 9] ; 18 19
+ vpermq m4, m21, [cq+64*10] ; 20 21
+ vpermq m5, m21, [cq+64*11] ; 22 23
+ vpermq m6, m21, [cq+64*12] ; 24 25
+ vpermq m18, m21, [cq+64*13] ; 26 27
+ vpermq m7, m21, [cq+64*14] ; 28 29
+ vpermq m21, m21, [cq+64*15] ; 30 31
+ REPX {pmulhrsw x, m8}, m9, m19, m4, m5, m6, m18, m7, m21
+ REPX {mova [cq+64*x], m12}, 8, 9, 10, 11, 12, 13, 14, 15
+ punpcklwd m8, m21, m14 ; 30 2
+ punpckhwd m21, m1 ; 31 1
+ punpcklwd m0, m17, m19 ; 14 18
+ punpckhwd m17, m9 ; 15 17
+ punpcklwd m9, m1 ; 16 0
+ punpckhwd m14, m7 ; 3 29
+ punpcklwd m1, m15, m18 ; 6 26
+ punpckhwd m15, m6 ; 7 25
+ punpcklwd m6, m2 ; 24 8
+ punpckhwd m19, m3 ; 19 13
+ punpcklwd m3, m4 ; 12 20
+ punpckhwd m18, m20 ; 27 5
+ punpcklwd m7, m20 ; 28 4
+ punpckhwd m20, m5, m2 ; 23 9
+ punpcklwd m5, m16 ; 22 10
+ punpckhwd m16, m4 ; 11 21
+ call m(idct_16x16_internal_8bpc).main2
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf
+ jmp .pass2
+.fast: ; bottom half zero
+ punpcklwd m8, m14, m14 ; 2
+ punpcklwd m0, m17, m17 ; 14
+ punpcklwd m5, m16, m16 ; 10
+ punpcklwd m9, m12, m1 ; __ 0
+ punpckhwd m21, m1, m1 ; 1
+ punpcklwd m1, m15, m15 ; 6
+ punpcklwd m7, m20, m20 ; 4
+ punpckhwd m19, m3, m3 ; 13
+ punpcklwd m3, m3 ; 12
+ punpcklwd m6, m12, m2 ; __ 8
+ punpckhwd m18, m20, m20 ; 5
+ punpckhwd m20, m2, m2 ; 9
+ call m(idct_16x16_internal_8bpc).main_fast
+ punpckhwd m15, m15 ; 7
+ punpckhwd m14, m14 ; 3
+ punpckhwd m16, m16 ; 11
+ punpckhwd m17, m17 ; 15
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast
+.pass2:
+ vpbroadcastd m9, [o(pw_16384)]
+ call .transpose_round
+ vshufi32x4 m16, m14, m2, q3131 ; 5
+ vshufi32x4 m14, m2, q2020 ; 1
+ vshufi32x4 m2, m0, m3, q3131 ; 4
+ vshufi32x4 m0, m3, q2020 ; 0
+ vshufi32x4 m3, m1, m18, q3131 ; 6
+ vshufi32x4 m1, m18, q2020 ; 2
+ vshufi32x4 m18, m20, m6, q2020 ; 9
+ vshufi32x4 m20, m6, q3131 ; 13
+ vshufi32x4 m6, m21, m4, q3131 ; 12
+ vshufi32x4 m4, m21, m4, q2020 ; 8
+ vshufi32x4 m21, m19, m7, q3131 ; 15
+ vshufi32x4 m19, m7, q2020 ; 11
+ vshufi32x4 m7, m5, m15, q3131 ; 14
+ vshufi32x4 m5, m15, q2020 ; 10
+ vshufi32x4 m15, m17, m9, q2020 ; 3
+ vshufi32x4 m17, m9, q3131 ; 7
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main2
+ call .main_oddhalf
+ vpbroadcastd m12, [o(pw_2048)]
+ movshdup m13, [o(permD)]
+ lea r2, [strideq*3]
+ pmovzxbw m8, [dstq+strideq*0]
+ pmovzxbw m9, [dstq+strideq*1]
+ pmovzxbw m10, [dstq+strideq*2]
+ pmovzxbw m11, [dstq+r2 ]
+ REPX {pmulhrsw x, m12}, m0, m1, m2, m3
+ lea r3, [dstq+strideq*4]
+ paddw m0, m8
+ paddw m1, m9
+ paddw m2, m10
+ paddw m3, m11
+ pmovzxbw m8, [r3+strideq*0]
+ pmovzxbw m9, [r3+strideq*1]
+ pmovzxbw m10, [r3+strideq*2]
+ pmovzxbw m11, [r3+r2 ]
+ REPX {pmulhrsw x, m12}, m4, m5, m6, m7
+ lea r4, [dstq+strideq*8]
+ packuswb m0, m1
+ paddw m4, m8
+ paddw m5, m9
+ packuswb m2, m3
+ paddw m6, m10
+ paddw m7, m11
+ pmovzxbw m8, [r4+strideq*0]
+ pmovzxbw m9, [r4+strideq*1]
+ pmovzxbw m10, [r4+strideq*2]
+ pmovzxbw m11, [r4+r2 ]
+ REPX {pmulhrsw x, m12}, m14, m15, m16, m17
+ lea r5, [r3+strideq*8]
+ packuswb m4, m5
+ paddw m14, m8
+ paddw m15, m9
+ packuswb m6, m7
+ paddw m16, m10
+ paddw m17, m11
+ pmovzxbw m8, [r5+strideq*0]
+ pmovzxbw m9, [r5+strideq*1]
+ pmovzxbw m10, [r5+strideq*2]
+ pmovzxbw m11, [r5+r2 ]
+ REPX {pmulhrsw x, m12}, m18, m19, m20, m21
+ packuswb m14, m15
+ paddw m18, m8
+ paddw m19, m9
+ packuswb m16, m17
+ paddw m20, m10
+ paddw m21, m11
+ packuswb m18, m19
+ packuswb m20, m21
+ REPX {vpermq x, m13, x}, m0, m2, m4, m6, m14, m16, m18, m20
+ mova [dstq+strideq*0], ym0
+ vextracti32x8 [dstq+strideq*1], m0, 1
+ mova [dstq+strideq*2], ym2
+ vextracti32x8 [dstq+r2 ], m2, 1
+ mova [r3+strideq*0], ym4
+ vextracti32x8 [r3+strideq*1], m4, 1
+ mova [r3+strideq*2], ym6
+ vextracti32x8 [r3+r2 ], m6, 1
+ mova [r4+strideq*0], ym14
+ vextracti32x8 [r4+strideq*1], m14, 1
+ mova [r4+strideq*2], ym16
+ vextracti32x8 [r4+r2 ], m16, 1
+ mova [r5+strideq*0], ym18
+ vextracti32x8 [r5+strideq*1], m18, 1
+ mova [r5+strideq*2], ym20
+ vextracti32x8 [r5+r2 ], m20, 1
+ RET
+ALIGN function_align
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 16
+ imul r6d, 181
+ add r6d, 128
+ sar r6d, 8
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+ jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3
+ALIGN function_align
+cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero
+ vpbroadcastd m9, [o(pw_2896x8)]
+ vpbroadcastd m2, [o(pw_4017x8)]
+ vpbroadcastd m3, [o(pw_799x8)]
+ vpbroadcastd m18, [o(pw_4076x8)]
+ vpbroadcastd m19, [o(pw_401x8)]
+ vpbroadcastd m20, [o(pw_m1189x8)]
+ vpbroadcastd m16, [o(pw_3920x8)]
+ pmulhrsw m9, m0 ; t0
+ pmulhrsw m2, m1 ; t7a
+ pmulhrsw m1, m3 ; t4a
+ pmulhrsw m18, m14 ; t15a
+ pmulhrsw m14, m19 ; t8a
+ pmulhrsw m20, m15 ; t11a
+ pmulhrsw m15, m16 ; t12a
+ psubsw m7, m9, m2 ; idct8 out7
+ paddsw m0, m9, m2 ; idct8 out0
+ psubsw m4, m9, m1 ; idct8 out4
+ paddsw m3, m9, m1 ; idct8 out3
+ ITX_MULSUB_2W 2, 1, 5, 6, 10, 2896, 2896 ; t5, t6
+ mova m21, m18
+ mova m19, m14
+ mova m16, m15
+ mova m8, m20
+ psubsw m6, m9, m1 ; idct8 out6
+ paddsw m1, m9 ; idct8 out1
+ psubsw m5, m9, m2 ; idct8 out5
+ paddsw m2, m9 ; idct8 out2
+ jmp .main3
+ALIGN function_align
+cglobal_label .main_oddhalf_fast ; bottom half is zero
+ vpbroadcastd m5, [o(pw_m2276x8)]
+ vpbroadcastd m11, [o(pw_3406x8)]
+ vpbroadcastd m7, [o(pw_4017x8)]
+ vpbroadcastd m12, [o(pw_799x8)]
+ vpbroadcastd m6, [o(pw_3784x8)]
+ vpbroadcastd m10, [o(pw_1567x8)]
+ vpbroadcastd m4, [o(pw_2896x8)]
+ pmulhrsw m5, m3 ; t5a
+ pmulhrsw m3, m11 ; t6a
+ pmulhrsw m7, m1 ; t7a
+ pmulhrsw m1, m12 ; t4a
+ pmulhrsw m6, m2 ; t3
+ pmulhrsw m2, m10 ; t2
+ pmulhrsw m4, m0 ; t0
+ vpbroadcastd m11, [o(pw_2896_2896)]
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ vpbroadcastd m10, [o(pd_2048)]
+ mova m0, m4 ; t1
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main3
+ vpbroadcastd m21, [o(pw_4076x8)]
+ vpbroadcastd m8, [o(pw_401x8)]
+ vpbroadcastd m18, [o(pw_m2598x8)]
+ vpbroadcastd m9, [o(pw_3166x8)]
+ vpbroadcastd m19, [o(pw_3612x8)]
+ vpbroadcastd m11, [o(pw_1931x8)]
+ vpbroadcastd m20, [o(pw_m1189x8)]
+ vpbroadcastd m12, [o(pw_3920x8)]
+ pmulhrsw m21, m14 ; t15a
+ pmulhrsw m14, m8 ; t8a
+ pmulhrsw m18, m17 ; t9a
+ pmulhrsw m17, m9 ; t14a
+ pmulhrsw m19, m16 ; t13a
+ pmulhrsw m16, m11 ; t10a
+ pmulhrsw m20, m15 ; t11a
+ pmulhrsw m15, m12 ; t12a
+ jmp .main2
+ALIGN function_align
+cglobal_label .main_oddhalf
+ ITX_MULSUB_2W 14, 21, 8, 9, 10, 401, 4076 ; t8a, t15a
+ ITX_MULSUB_2W 18, 17, 8, 9, 10, 3166, 2598 ; t9a, t14a
+ ITX_MULSUB_2W 16, 19, 8, 9, 10, 1931, 3612 ; t10a, t13a
+ ITX_MULSUB_2W 20, 15, 8, 9, 10, 3920, 1189 ; t11a, t12a
+.main2:
+ paddsw m8, m20, m16 ; t11
+ psubsw m20, m16 ; t10
+ paddsw m16, m15, m19 ; t12
+ psubsw m15, m19 ; t13
+ psubsw m19, m14, m18 ; t9
+ paddsw m14, m18 ; t8
+ psubsw m18, m21, m17 ; t14
+ paddsw m21, m17 ; t15
+.main3:
+ vpbroadcastd m11, [o(pw_1567_3784)]
+ vpbroadcastd m12, [o(pw_m3784_1567)]
+ ITX_MULSUB_2W 18, 19, 9, 17, 10, 11, 12 ; t9a, t14a
+ vpbroadcastd m11, [o(pw_m1567_m3784)]
+ ITX_MULSUB_2W 15, 20, 9, 17, 10, 12, 11 ; t10a, t13a
+ vpbroadcastd m11, [o(pw_2896_2896)]
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ psubsw m17, m14, m8 ; t11a
+ paddsw m8, m14 ; t8a
+ paddsw m14, m18, m15 ; t9
+ psubsw m18, m15 ; t10
+ psubsw m15, m19, m20 ; t13
+ paddsw m19, m20 ; t14
+ paddsw m20, m21, m16 ; t15a
+ psubsw m16, m21, m16 ; t12a
+ ITX_MULSUB_2W 15, 18, 9, 21, 10, 11, 12 ; t10a, t13a
+ ITX_MULSUB_2W 16, 17, 9, 21, 10, 11, 12 ; t11, t12
+ psubsw m21, m0, m20 ; out15
+ paddsw m0, m20 ; out0
+ psubsw m20, m1, m19 ; out14
+ paddsw m1, m19 ; out1
+ psubsw m19, m2, m18 ; out13
+ paddsw m2, m18 ; out2
+ psubsw m18, m3, m17 ; out12
+ paddsw m3, m17 ; out3
+ psubsw m17, m4, m16 ; out11
+ paddsw m4, m16 ; out4
+ psubsw m16, m5, m15 ; out10
+ paddsw m5, m15 ; out5
+ psubsw m15, m6, m14 ; out9
+ paddsw m6, m14 ; out6
+ psubsw m14, m7, m8 ; out8
+ paddsw m7, m8 ; out7
+ ret
+.transpose_round:
+ punpcklwd m8, m0, m2
+ punpckhwd m0, m2
+ punpcklwd m2, m1, m3
+ punpckhwd m1, m3
+ punpcklwd m3, m4, m6
+ punpckhwd m4, m6
+ punpcklwd m6, m5, m7
+ punpckhwd m5, m7
+ punpcklwd m7, m14, m16
+ punpckhwd m14, m16
+ punpcklwd m16, m15, m17
+ punpckhwd m15, m17
+ punpcklwd m17, m19, m21
+ punpckhwd m19, m21
+ punpckhwd m21, m18, m20
+ punpcklwd m18, m20
+ punpcklwd m20, m8, m1
+ punpckhwd m8, m1
+ punpcklwd m1, m0, m2
+ punpckhwd m0, m2
+ punpcklwd m2, m3, m5
+ punpckhwd m3, m5
+ punpcklwd m5, m4, m6
+ punpckhwd m4, m6
+ REPX {pmulhrsw x, m9}, m20, m8, m1, m0
+ punpcklwd m6, m7, m15
+ punpckhwd m7, m15
+ punpcklwd m15, m14, m16
+ punpckhwd m14, m16
+ REPX {pmulhrsw x, m9}, m2, m3, m5, m4
+ punpckhwd m16, m18, m19
+ punpcklwd m18, m19
+ punpcklwd m19, m21, m17
+ punpckhwd m21, m17
+ REPX {pmulhrsw x, m9}, m6, m7, m15, m14
+ punpcklwd m17, m8, m0 ; a2 a6 aa ae
+ punpckhwd m8, m0 ; a3 a7 ab af
+ punpcklwd m0, m20, m1 ; a0 a4 a8 ac
+ punpckhwd m20, m1 ; a1 a5 a9 ad
+ REPX {pmulhrsw x, m9}, m16, m18, m19, m21
+ punpcklwd m1, m2, m5 ; b0 b4 b8 bc
+ punpckhwd m2, m5 ; b1 b5 b9 bd
+ punpcklwd m5, m3, m4 ; b2 b6 ba be
+ punpckhwd m3, m4 ; b3 b7 bb bf
+ punpcklwd m4, m6, m15 ; c0 c4 c8 cc
+ punpckhwd m6, m15 ; c1 c5 c9 cd
+ punpcklwd m15, m7, m14 ; c2 c6 ca ce
+ punpckhwd m7, m14 ; c3 c7 cb cf
+ punpcklwd m14, m18, m19 ; d0 d4 d8 dc
+ punpckhwd m18, m19 ; d1 d5 d9 dd
+ punpcklwd m9, m16, m21 ; d2 d6 da de
+ punpckhwd m16, m21 ; d3 d7 db df
+ vshufi32x4 m21, m0, m1, q3232 ; a8 ac b8 bc
+ vinserti32x8 m0, ym1, 1 ; a0 a4 b0 b4
+ vinserti32x8 m1, m17, ym5, 1 ; a2 a6 b2 b6
+ vshufi32x4 m5, m17, m5, q3232 ; aa ae ba be
+ vinserti32x8 m17, m8, ym3, 1 ; a3 a7 b3 b7
+ vshufi32x4 m19, m8, m3, q3232 ; ab af bb bf
+ vinserti32x8 m3, m4, ym14, 1 ; c0 c4 d0 d4
+ vshufi32x4 m4, m14, q3232 ; c8 cc d8 dc
+ vinserti32x8 m14, m20, ym2, 1 ; a1 a5 b1 b5
+ vshufi32x4 m20, m2, q3232 ; a9 ad b9 bd
+ vinserti32x8 m2, m6, ym18, 1 ; c1 c5 d1 d5
+ vshufi32x4 m6, m18, q3232 ; c9 cd d9 dd
+ vinserti32x8 m18, m15, ym9, 1 ; c2 c6 d2 d6
+ vshufi32x4 m15, m9, q3232 ; ca ce da de
+ vinserti32x8 m9, m7, ym16, 1 ; c3 c7 d3 d7
+ vshufi32x4 m7, m16, q3232 ; cb cf db df
+ ret
+
+%macro IDTX_16x32 4 ; src/dst[1-4]
+ pmulhrsw m%1, m15, [cq+64*%1]
+ pmulhrsw m%2, m15, [cq+64*%2]
+ pmulhrsw m%3, m15, [cq+64*%3]
+ pmulhrsw m%4, m15, [cq+64*%4]
+ pmulhrsw m18, m16, m%1
+ pmulhrsw m19, m16, m%2
+ pmulhrsw m20, m16, m%3
+ pmulhrsw m21, m16, m%4
+ REPX {pmulhrsw x, m17}, m18, m19, m20, m21
+ paddsw m%1, m18
+ paddsw m%2, m19
+ paddsw m%3, m20
+ paddsw m%4, m21
+%endmacro
+
+%macro IDTX_16x32_STORE 2 ; src[1-2]
+ mova xm17, [dstq+r3*0]
+ vinserti128 ym17, [dstq+r3*4], 1
+ vinserti32x4 m17, [dstq+r3*8], 2
+ vinserti32x4 m17, [dstq+r4*8], 3
+ mova [cq+64*(%1*2+0)], m18
+ mova [cq+64*(%1*2+1)], m18
+ punpcklbw m16, m17, m18
+ punpckhbw m17, m18
+ paddw m16, m%1
+ paddw m17, m%2
+ packuswb m16, m17
+ mova [dstq+r3*0], xm16
+ vextracti128 [dstq+r3*4], ym16, 1
+ vextracti32x4 [dstq+r3*8], m16, 2
+ vextracti32x4 [dstq+r4*8], m16, 3
+%if %1 != 7
+ add dstq, strideq
+%endif
+%endmacro
+
+cglobal inv_txfm_add_identity_identity_16x32_8bpc, 3, 5, 22, dst, stride, c
+ vpbroadcastd m15, [pw_2896x8]
+ vpbroadcastd m16, [pw_1697x16]
+ vpbroadcastd m17, [pw_16384]
+ IDTX_16x32 0, 1, 2, 3
+ IDTX_16x32 4, 5, 6, 7
+ IDTX_16x32 8, 9, 10, 11
+ IDTX_16x32 12, 13, 14, 15
+ vpbroadcastd m16, [pw_8192]
+ call .transpose_2x8x8_round
+ lea r3, [strideq*2]
+ lea r4, [strideq*3]
+ pxor m18, m18
+ IDTX_16x32_STORE 0, 8
+ IDTX_16x32_STORE 1, 9
+ IDTX_16x32_STORE 2, 10
+ IDTX_16x32_STORE 3, 11
+ IDTX_16x32_STORE 4, 12
+ IDTX_16x32_STORE 5, 13
+ IDTX_16x32_STORE 6, 14
+ IDTX_16x32_STORE 7, 15
+ RET
+ALIGN function_align
+.transpose_2x8x8_round:
+ punpckhwd m17, m4, m5
+ punpcklwd m4, m5
+ punpckhwd m5, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m6, m7
+ punpcklwd m6, m7
+ punpckhwd m7, m2, m3
+ punpcklwd m2, m3
+ punpckhdq m3, m0, m2
+ punpckldq m0, m2
+ punpckldq m2, m4, m6
+ punpckhdq m4, m6
+ punpckhdq m6, m5, m7
+ punpckldq m5, m7
+ punpckldq m7, m17, m1
+ punpckhdq m17, m1
+ REPX {pmulhrsw x, m16}, m0, m2, m3, m4, m5, m7, m6, m17
+ punpckhqdq m1, m0, m2
+ punpcklqdq m0, m2
+ punpcklqdq m2, m3, m4
+ punpckhqdq m3, m4
+ punpcklqdq m4, m5, m7
+ punpckhqdq m5, m7
+ punpckhqdq m7, m6, m17
+ punpcklqdq m6, m17
+ punpckhwd m17, m12, m13
+ punpcklwd m12, m13
+ punpckhwd m13, m8, m9
+ punpcklwd m8, m9
+ punpckhwd m9, m14, m15
+ punpcklwd m14, m15
+ punpckhwd m15, m10, m11
+ punpcklwd m10, m11
+ punpckhdq m11, m8, m10
+ punpckldq m8, m10
+ punpckldq m10, m12, m14
+ punpckhdq m12, m14
+ punpckhdq m14, m13, m15
+ punpckldq m13, m15
+ punpckldq m15, m17, m9
+ punpckhdq m17, m9
+ REPX {pmulhrsw x, m16}, m8, m10, m11, m12, m13, m15, m14, m17
+ punpckhqdq m9, m8, m10
+ punpcklqdq m8, m10
+ punpcklqdq m10, m11, m12
+ punpckhqdq m11, m12
+ punpcklqdq m12, m13, m15
+ punpckhqdq m13, m15
+ punpckhqdq m15, m14, m17
+ punpcklqdq m14, m17
+ ret
+
+%macro IDTX_32x16 4 ; dst[1-4]
+ pmulhrsw m%2, m12, [cq+32*(%1+ 0)]
+ pmulhrsw m18, m12, [cq+32*(%1+16)]
+ pmulhrsw m%4, m12, [cq+32*(%3+ 0)]
+ pmulhrsw m19, m12, [cq+32*(%3+16)]
+ REPX {paddsw x, x}, m%2, m18, m%4, m19
+ mova m%1, m14
+ vpermi2q m%1, m%2, m18
+ vpermt2q m%2, m16, m18
+%if %3 != 14
+ mova m%3, m14
+%endif
+ vpermi2q m%3, m%4, m19
+ vpermt2q m%4, m16, m19
+ pmulhrsw m18, m17, m%1
+ pmulhrsw m19, m17, m%2
+ pmulhrsw m20, m17, m%3
+ pmulhrsw m21, m17, m%4
+ REPX {paddsw x, x}, m%1, m%2, m%3, m%4
+ paddsw m%1, m18
+ paddsw m%2, m19
+ paddsw m%3, m20
+ paddsw m%4, m21
+%endmacro
+
+%macro IDTX_32x16_STORE 2-3 0 ; src[1-2], 32x32
+ mova ym19, [dstq+strideq*0]
+ vinserti32x8 m19, [dstq+strideq*8], 1
+%if %3 == 0
+ mova [cq+64*(%1*2+0)], m20
+ mova [cq+64*(%1*2+1)], m20
+%endif
+ punpcklbw m18, m19, m20
+ punpckhbw m19, m20
+ paddw m18, m%1
+ paddw m19, m%2
+ packuswb m18, m19
+ mova [dstq+strideq*0], ym18
+ vextracti32x8 [dstq+strideq*8], m18, 1
+%if %3 || %1 != 7
+ add dstq, strideq
+%endif
+%endmacro
+
+cglobal inv_txfm_add_identity_identity_32x16_8bpc, 3, 3, 22, dst, stride, c
+ vpbroadcastd m12, [pw_2896x8]
+ movu m14, [permB+7]
+ vpbroadcastd m17, [pw_1697x16]
+ psrlq m16, m14, 4
+ IDTX_32x16 0, 1, 2, 3
+ IDTX_32x16 4, 5, 6, 7
+ IDTX_32x16 8, 9, 10, 11
+ IDTX_32x16 12, 13, 14, 15
+ vpbroadcastd m16, [pw_2048]
+ call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round
+ pxor m20, m20
+ IDTX_32x16_STORE 0, 8
+ IDTX_32x16_STORE 1, 9
+ IDTX_32x16_STORE 2, 10
+ IDTX_32x16_STORE 3, 11
+ IDTX_32x16_STORE 4, 12
+ IDTX_32x16_STORE 5, 13
+ IDTX_32x16_STORE 6, 14
+ IDTX_32x16_STORE 7, 15
+ RET
+
+%macro IDCT_32x32_END 4 ; src, mem, stride[1-2]
+ pmovzxbw m10, [dstq+%3]
+ pmovzxbw m11, [r3 +%4]
+%if %2 < 8
+ paddsw m8, m%2, m%1
+ psubsw m9, m%2, m%1
+%else
+ mova m9, [cq+64*(%2*2-16)]
+ paddsw m8, m9, m%1
+ psubsw m9, m%1
+%endif
+ pmulhrsw m8, m12
+ pmulhrsw m9, m12
+%if %2 >= 8
+%if %2 == 8
+ pxor m0, m0
+%endif
+ mova [cq+64*(%2*2-16)], m0
+ mova [cq+64*(%2*2-15)], m0
+%endif
+ paddw m8, m10
+ paddw m9, m11
+ packuswb m8, m9
+ vpermq m8, m13, m8
+ mova [dstq+%3], ym8
+ vextracti32x8 [r3 +%4], m8, 1
+%if %2 == 3 || %2 == 7 || %2 == 11
+ add dstq, r5
+ sub r3, r5
+%endif
+%endmacro
+
+cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 0, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ WIN64_SPILL_XMM 30
+ cmp eobd, 136
+ jb .fast
+ mova m5, [cq+64*20]
+ mova m3, [cq+64*12]
+ mova m1, [cq+64* 4]
+ mova m7, [cq+64*28]
+ mova m2, [cq+64* 8]
+ mova m6, [cq+64*24]
+ mova m0, [cq+64* 0]
+ mova m4, [cq+64*16]
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ mova m14, [cq+64* 2]
+ mova m21, [cq+64*30]
+ mova m18, [cq+64*18]
+ mova m17, [cq+64*14]
+ mova m16, [cq+64*10]
+ mova m19, [cq+64*22]
+ mova m20, [cq+64*26]
+ mova m15, [cq+64* 6]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ mova [cq+64* 0], m14
+ mova [cq+64* 2], m15
+ mova [cq+64* 4], m16
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64*12], m20
+ mova [cq+64*14], m21
+ mova m22, [cq+64* 1]
+ mova m21, [cq+64*31]
+ mova m14, [cq+64*17]
+ mova m29, [cq+64*15]
+ mova m26, [cq+64* 9]
+ mova m17, [cq+64*23]
+ mova m18, [cq+64*25]
+ mova m25, [cq+64* 7]
+ mova m24, [cq+64* 5]
+ mova m19, [cq+64*27]
+ mova m16, [cq+64*21]
+ mova m27, [cq+64*11]
+ mova m28, [cq+64*13]
+ mova m15, [cq+64*19]
+ mova m20, [cq+64*29]
+ mova m23, [cq+64* 3]
+ call .main_oddhalf
+ vpbroadcastd m10, [o(pw_8192)]
+ psubsw m13, m0, m29 ; 31
+ paddsw m0, m29 ; 0
+ psubsw m29, m1, m28 ; 30
+ paddsw m1, m28 ; 1
+ psubsw m28, m2, m27 ; 29
+ paddsw m2, m27 ; 2
+ psubsw m27, m3, m26 ; 28
+ paddsw m3, m26 ; 3
+ psubsw m26, m4, m25 ; 27
+ paddsw m4, m25 ; 4
+ psubsw m25, m5, m24 ; 26
+ paddsw m5, m24 ; 5
+ psubsw m24, m6, m23 ; 25
+ paddsw m6, m23 ; 6
+ psubsw m23, m7, m22 ; 24
+ paddsw m7, m22 ; 7
+ pxor m9, m9
+ punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7
+ punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7
+ punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3
+ REPX {mova [cq+64*x], m9}, 16, 17, 18, 19
+ punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7
+ punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3
+ punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7
+ punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3
+ REPX {mova [cq+64*x], m9}, 20, 21, 22, 23
+ punpckhwd m3, m23, m24
+ punpcklwd m23, m24
+ punpckhwd m24, m25, m26
+ punpcklwd m25, m26
+ REPX {mova [cq+64*x], m9}, 24, 25, 26, 27
+ punpckhwd m26, m27, m28
+ punpcklwd m27, m28
+ punpckhwd m28, m29, m13
+ punpcklwd m29, m13
+ REPX {mova [cq+64*x], m9}, 28, 29, 30, 31
+ punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7
+ punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5
+ punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7
+ punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5
+ REPX {pmulhrsw x, m10}, m0, m4, m8, m22
+ punpckhdq m13, m23, m25
+ punpckldq m23, m25
+ punpckhdq m25, m27, m29
+ punpckldq m27, m29
+ REPX {pmulhrsw x, m10}, m13, m23, m25, m27
+ punpckhdq m9, m3, m24
+ punpckldq m3, m24
+ punpckhdq m24, m26, m28
+ punpckldq m26, m28
+ punpcklqdq m5, m23, m27 ; d00 d08 d16 d24
+ punpckhqdq m23, m27 ; d01 d09 d17 d25
+ punpckhqdq m27, m13, m25 ; d03 d11 d19 d27
+ punpcklqdq m13, m25 ; d02 d10 d18 d26
+ punpckhqdq m25, m3, m26 ; d05 d13 d21 d29
+ punpcklqdq m3, m26 ; d04 d12 d20 d28
+ punpckhqdq m26, m9, m24 ; d07 d15 d23 d31
+ punpcklqdq m9, m24 ; d06 d14 d22 d30
+ REPX {pmulhrsw x, m10}, m25, m3, m26
+ mova [cq+64* 9], m23
+ mova [cq+64*11], m27
+ mova [cq+64*13], m25
+ mova [cq+64*15], m26
+ punpckhqdq m24, m8, m22 ; a05 a13 a21 a29
+ punpcklqdq m8, m22 ; a04 a12 a20 a28
+ punpckhqdq m22, m0, m4 ; a01 a09 a17 a25
+ punpcklqdq m0, m4 ; a00 a08 a16 a24
+ punpckhqdq m23, m7, m2 ; a03 a11 a19 a27
+ punpcklqdq m7, m2 ; a02 a10 a18 a26
+ punpckhqdq m25, m6, m1 ; a07 a15 a23 a31
+ punpcklqdq m6, m1 ; a06 a14 a22 a30
+ mova m2, [cq+64* 0]
+ mova m11, [cq+64* 2]
+ mova m12, [cq+64* 4]
+ mova m29, [cq+64* 6]
+ mova m27, [cq+64* 8]
+ mova m26, [cq+64*10]
+ mova m4, [cq+64*12]
+ mova m28, [cq+64*14]
+ psubsw m1, m2, m21 ; 23
+ paddsw m2, m21 ; 8
+ psubsw m21, m11, m20 ; 22
+ paddsw m11, m20 ; 9
+ psubsw m20, m12, m19 ; 21
+ paddsw m12, m19 ; 10
+ psubsw m19, m29, m18 ; 20
+ paddsw m29, m18 ; 11
+ psubsw m18, m27, m17 ; 19
+ paddsw m27, m17 ; 12
+ psubsw m17, m26, m16 ; 18
+ paddsw m26, m16 ; 13
+ paddsw m16, m4, m15 ; 14
+ psubsw m4, m15 ; 17
+ pmulhrsw m15, m6, m10
+ psubsw m6, m28, m14 ; 16
+ paddsw m28, m14 ; 15
+ pmulhrsw m14, m7, m10
+ punpcklwd m7, m6, m4
+ punpckhwd m6, m4
+ punpckhwd m4, m17, m18
+ punpcklwd m17, m18
+ punpckhwd m18, m19, m20
+ punpcklwd m19, m20
+ punpckhwd m20, m21, m1
+ punpcklwd m21, m1
+ punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7
+ punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3
+ punpckhwd m11, m12, m29 ; k4 l4 k5 l5 k6 l6 k7 l7
+ punpcklwd m12, m29 ; k0 l0 k1 l1 k2 l2 k3 l3
+ punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7
+ punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3
+ punpckhwd m26, m16, m28 ; o4 p4 o5 p5 o6 p6 o7 p7
+ punpcklwd m16, m28 ; o0 p0 o1 p1 o2 p2 o3 p3
+ pmulhrsw m23, m10
+ pmulhrsw m25, m10
+ punpckhdq m28, m2, m12 ; i2 j2 k2 l2 i3 j3 k3 l3
+ punpckldq m2, m12 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhdq m12, m27, m16 ; m2 n2 o2 p2 m3 n3 o3 p3
+ punpckldq m27, m16 ; m0 n0 o0 p0 m1 n1 o1 p1
+ REPX {pmulhrsw x, m10}, m28, m2, m12, m27
+ punpckhdq m16, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7
+ punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5
+ punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7
+ punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5
+ REPX {pmulhrsw x, m10}, m16, m1, m11, m29
+ punpckhdq m26, m19, m21
+ punpckldq m19, m21
+ punpckhdq m21, m6, m4
+ punpckldq m6, m4
+ REPX {pmulhrsw x, m10}, m26, m19, m21, m6
+ punpckhdq m4, m18, m20
+ punpckldq m18, m20
+ punpckhdq m20, m7, m17
+ punpckldq m7, m17
+ REPX {pmulhrsw x, m10}, m4, m18, m20, m7
+ punpcklqdq m17, m28, m12 ; b02 b10 b18 b26
+ punpckhqdq m28, m12 ; b03 b11 b19 b27
+ punpckhqdq m12, m2, m27 ; b01 b09 b17 b25
+ punpcklqdq m2, m27 ; b00 b08 b16 b24
+ punpckhqdq m27, m1, m29 ; b05 b13 b21 b29
+ punpcklqdq m1, m29 ; b04 b12 b20 b28
+ punpckhqdq m29, m16, m11 ; b07 b15 b23 b31
+ punpcklqdq m16, m11 ; b06 b14 b22 b30
+ mova [cq+64* 1], m12
+ mova [cq+64* 3], m28
+ mova [cq+64* 5], m27
+ mova [cq+64* 7], m29
+ punpckhqdq m27, m20, m26 ; c03 c11 c19 c27
+ punpcklqdq m20, m26 ; c02 c10 c18 c26
+ punpckhqdq m26, m7, m19 ; c01 c09 c17 c25
+ punpcklqdq m7, m19 ; c00 c08 c16 c24
+ punpckhqdq m28, m6, m18 ; c05 c13 c21 c29
+ punpcklqdq m6, m18 ; c04 c12 c20 c28
+ punpckhqdq m29, m21, m4 ; c07 c15 c23 c31
+ punpcklqdq m21, m4 ; c06 c14 c22 c30
+ pmulhrsw m19, m9, m10
+ vshufi32x4 m4, m0, m2, q3232 ; a16 a24 b16 b24
+ vinserti32x8 m0, ym2, 1 ; a00 a08 b00 b08
+ vshufi32x4 m2, m7, m5, q3232 ; c16 c24 d16 d24
+ vinserti32x8 m7, ym5, 1 ; c00 c08 d00 d08
+ vshufi32x4 m5, m8, m1, q3232 ; a20 a28 b20 b28
+ vinserti32x8 m1, m8, ym1, 1 ; a04 a12 b04 b12
+ vshufi32x4 m8, m6, m3, q3232 ; c20 c28 d20 d28
+ vinserti32x8 m6, ym3, 1 ; c04 c12 d04 d12
+ vshufi32x4 m3, m1, m6, q3131 ; 12
+ vshufi32x4 m1, m6, q2020 ; 4
+ vshufi32x4 m6, m4, m2, q3131 ; 24
+ vshufi32x4 m4, m2, q2020 ; 16
+ vshufi32x4 m2, m0, m7, q3131 ; 8
+ vshufi32x4 m0, m7, q2020 ; 0
+ vshufi32x4 m7, m5, m8, q3131 ; 28
+ vshufi32x4 m5, m8, q2020 ; 20
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ vshufi32x4 m18, m14, m17, q3232 ; a18 a26 b18 b26
+ vinserti32x8 m14, ym17, 1 ; a02 a10 b02 b10
+ vshufi32x4 m17, m20, m13, q3232 ; c18 c26 d18 d26
+ vinserti32x8 m20, ym13, 1 ; c02 c10 d02 d10
+ vshufi32x4 m13, m21, m19, q3232 ; c22 c30 d22 d30
+ vinserti32x8 m21, ym19, 1 ; c06 c14 d06 d14
+ vshufi32x4 m19, m15, m16, q3232 ; a22 a30 b22 b30
+ vinserti32x8 m15, ym16, 1 ; a06 a14 b06 b14
+ vshufi32x4 m16, m14, m20, q3131 ; 10
+ vshufi32x4 m14, m20, q2020 ; 2
+ vshufi32x4 m20, m18, m17, q3131 ; 26
+ vshufi32x4 m18, m17, q2020 ; 18
+ vshufi32x4 m17, m15, m21, q3131 ; 14
+ vshufi32x4 m15, m21, q2020 ; 6
+ vshufi32x4 m21, m19, m13, q3131 ; 30
+ vshufi32x4 m19, m13, q2020 ; 22
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ mova [cq+64* 0], m14
+ mova [cq+64* 2], m15
+ mova [cq+64* 4], m16
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64*12], m20
+ mova [cq+64*14], m21
+ mova m15, [cq+64* 1]
+ mova m16, [cq+64* 3]
+ mova m17, [cq+64* 5]
+ mova m19, [cq+64* 7]
+ mova m20, [cq+64* 9]
+ mova m21, [cq+64*11]
+ mova m13, [cq+64*13]
+ mova m18, [cq+64*15]
+ vshufi32x4 m14, m22, m15, q3232 ; a17 a25 b17 b25
+ vinserti32x8 m22, ym15, 1 ; a01 a09 b01 b09
+ vshufi32x4 m15, m23, m16, q3232 ; a19 a27 b19 b27
+ vinserti32x8 m23, ym16, 1 ; a03 a11 b03 b11
+ vshufi32x4 m16, m24, m17, q3232 ; a21 a29 b21 b29
+ vinserti32x8 m24, ym17, 1 ; a05 a13 b05 b13
+ vshufi32x4 m17, m25, m19, q3232 ; a23 a31 b23 b31
+ vinserti32x8 m25, ym19, 1 ; a07 a15 b07 b15
+ vinserti32x8 m8, m26, ym20, 1 ; c01 c09 d01 d09
+ vshufi32x4 m26, m20, q3232 ; c17 c25 d17 d25
+ vinserti32x8 m9, m27, ym21, 1 ; c03 c11 d03 d11
+ vshufi32x4 m27, m21, q3232 ; c19 c27 d19 d27
+ vinserti32x8 m11, m28, ym13, 1 ; c05 c13 d05 d13
+ vshufi32x4 m28, m13, q3232 ; c21 c29 d21 d29
+ vinserti32x8 m12, m29, ym18, 1 ; c07 c15 d07 d15
+ vshufi32x4 m29, m18, q3232 ; c23 c31 d23 d31
+ vshufi32x4 m18, m14, m26, q3131 ; 25
+ vshufi32x4 m14, m26, q2020 ; 17
+ vshufi32x4 m19, m15, m27, q3131 ; 27
+ vshufi32x4 m15, m27, q2020 ; 19
+ vshufi32x4 m20, m16, m28, q3131 ; 29
+ vshufi32x4 m16, m28, q2020 ; 21
+ vshufi32x4 m21, m17, m29, q3131 ; 31
+ vshufi32x4 m17, m29, q2020 ; 23
+ vshufi32x4 m26, m22, m8, q3131 ; 9
+ vshufi32x4 m22, m8, q2020 ; 1
+ vshufi32x4 m27, m23, m9, q3131 ; 11
+ vshufi32x4 m23, m9, q2020 ; 3
+ vshufi32x4 m28, m24, m11, q3131 ; 13
+ vshufi32x4 m24, m11, q2020 ; 5
+ vshufi32x4 m29, m25, m12, q3131 ; 15
+ vshufi32x4 m25, m12, q2020 ; 7
+ call .main_oddhalf
+ jmp .end
+.fast: ; bottom/right halves are zero
+ mova m14, [o(dup16_perm)]
+ pmovzxwd m9, [cq+64* 0]
+ pmovzxwd m6, [cq+64* 8]
+ vpermb m8, m14, [cq+64* 2]
+ vpermb ym0, ym14, [cq+64*14]
+ vpermb ym5, ym14, [cq+64*10]
+ vpermb m1, m14, [cq+64* 6]
+ vpermb m7, m14, [cq+64* 4]
+ vpermb ym3, ym14, [cq+64*12]
+ pslld m9, 16
+ pslld m6, 16
+ call m(idct_16x16_internal_8bpc).main_fast
+ vpermb m21, m14, [cq+64* 1]
+ vpermb ym17, ym14, [cq+64*15]
+ vpermb ym20, ym14, [cq+64* 9]
+ vpermb m15, m14, [cq+64* 7]
+ vpermb m18, m14, [cq+64* 5]
+ vpermb ym16, ym14, [cq+64*11]
+ vpermb ym19, ym14, [cq+64*13]
+ vpermb m14, m14, [cq+64* 3]
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast
+ vpbroadcastd m9, [o(pw_8192)]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round
+ vshufi32x4 m22, m14, m2, q2020 ; 1
+ vshufi32x4 m24, m14, m2, q3131 ; 5
+ vshufi32x4 m23, m17, m9, q2020 ; 3
+ vshufi32x4 m25, m17, m9, q3131 ; 7
+ vshufi32x4 m16, m5, m15, q2020 ; 10
+ vshufi32x4 m17, m5, m15, q3131 ; 14
+ vshufi32x4 m14, m1, m18, q2020 ; 2
+ vshufi32x4 m15, m1, m18, q3131 ; 6
+ vshufi32x4 m1, m0, m3, q3131 ; 4
+ vshufi32x4 m0, m3, q2020 ; 0
+ vshufi32x4 m3, m21, m4, q3131 ; 12
+ vshufi32x4 m2, m21, m4, q2020 ; 8
+ vshufi32x4 m26, m20, m6, q2020 ; 9
+ vshufi32x4 m28, m20, m6, q3131 ; 13
+ vshufi32x4 m27, m19, m7, q2020 ; 11
+ vshufi32x4 m29, m19, m7, q3131 ; 15
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+ mova [cq+64* 0], m14
+ mova [cq+64* 2], m15
+ mova [cq+64* 4], m16
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64*12], m20
+ mova [cq+64*14], m21
+ call .main_oddhalf_fast
+.end:
+ lea r4, [strideq*3]
+ vpbroadcastd m12, [o(pw_2048)]
+ movshdup m13, [o(permD)]
+ lea r3, [dstq+r4*8]
+ lea r5, [strideq+r4] ; stride*4
+ add r3, r5 ; dst+stride*28
+ IDCT_32x32_END 29, 0, strideq*0, r4
+ IDCT_32x32_END 28, 1, strideq*1, strideq*2
+ IDCT_32x32_END 27, 2, strideq*2, strideq*1
+ IDCT_32x32_END 26, 3, r4 , strideq*0
+ IDCT_32x32_END 25, 4, strideq*0, r4
+ IDCT_32x32_END 24, 5, strideq*1, strideq*2
+ IDCT_32x32_END 23, 6, strideq*2, strideq*1
+ IDCT_32x32_END 22, 7, r4 , strideq*0
+ IDCT_32x32_END 21, 8, strideq*0, r4
+ IDCT_32x32_END 20, 9, strideq*1, strideq*2
+ IDCT_32x32_END 19, 10, strideq*2, strideq*1
+ IDCT_32x32_END 18, 11, r4 , strideq*0
+ IDCT_32x32_END 17, 12, strideq*0, r4
+ IDCT_32x32_END 16, 13, strideq*1, strideq*2
+ IDCT_32x32_END 15, 14, strideq*2, strideq*1
+ IDCT_32x32_END 14, 15, r4 , strideq*0
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 32
+ jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly2
+ALIGN function_align
+cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero
+ vpbroadcastd m21, [o(pw_4091x8)]
+ vpbroadcastd m8, [o(pw_201x8)]
+ vpbroadcastd m18, [o(pw_m1380x8)]
+ vpbroadcastd m9, [o(pw_3857x8)]
+ vpbroadcastd m19, [o(pw_3973x8)]
+ vpbroadcastd m11, [o(pw_995x8)]
+ vpbroadcastd m28, [o(pw_m601x8)]
+ vpbroadcastd m12, [o(pw_4052x8)]
+ pmulhrsw m21, m22 ; t31a
+ pmulhrsw m22, m8 ; t16a
+ pmulhrsw m18, m25 ; t19a
+ pmulhrsw m25, m9 ; t28a
+ pmulhrsw m19, m24 ; t27a
+ pmulhrsw m24, m11 ; t20a
+ pmulhrsw m28, m23 ; t23a
+ pmulhrsw m23, m12 ; t24a
+ mova m15, m21
+ mova m8, m22
+ mova m14, m18
+ mova m27, m25
+ mova m29, m19
+ mova m26, m24
+ mova m16, m28
+ mova m20, m23
+ jmp .main3
+ALIGN function_align
+cglobal_label .main_oddhalf_fast ; bottom half is zero
+ vpbroadcastd m21, [o(pw_4091x8)]
+ vpbroadcastd m8, [o(pw_201x8)]
+ vpbroadcastd m14, [o(pw_m2751x8)]
+ vpbroadcastd m9, [o(pw_3035x8)]
+ vpbroadcastd m17, [o(pw_3703x8)]
+ vpbroadcastd m11, [o(pw_1751x8)]
+ vpbroadcastd m18, [o(pw_m1380x8)]
+ vpbroadcastd m12, [o(pw_3857x8)]
+ pmulhrsw m21, m22 ; t31a
+ vpbroadcastd m19, [o(pw_3973x8)]
+ pmulhrsw m22, m8 ; t16a
+ vpbroadcastd m8, [o(pw_995x8)]
+ pmulhrsw m14, m29 ; t30a
+ vpbroadcastd m16, [o(pw_m2106x8)]
+ pmulhrsw m29, m9 ; t17a
+ vpbroadcastd m9, [o(pw_3513x8)]
+ pmulhrsw m17, m26 ; t29a
+ vpbroadcastd m15, [o(pw_3290x8)]
+ pmulhrsw m26, m11 ; t18a
+ vpbroadcastd m11, [o(pw_2440x8)]
+ pmulhrsw m18, m25 ; t19a
+ vpbroadcastd m20, [o(pw_m601x8)]
+ pmulhrsw m25, m12 ; t28a
+ vpbroadcastd m12, [o(pw_4052x8)]
+ pmulhrsw m19, m24 ; t27a
+ pmulhrsw m24, m8 ; t20a
+ pmulhrsw m16, m27 ; t21a
+ pmulhrsw m27, m9 ; t26a
+ pmulhrsw m15, m28 ; t25a
+ pmulhrsw m28, m11 ; t22a
+ pmulhrsw m20, m23 ; t23a
+ pmulhrsw m23, m12 ; t24a
+ jmp .main2
+ALIGN function_align
+cglobal_label .main_oddhalf
+ ITX_MULSUB_2W 22, 21, 8, 9, 10, 201, 4091 ; t16a, t31a
+ ITX_MULSUB_2W 14, 29, 8, 9, 10, 3035, 2751 ; t17a, t30a
+ ITX_MULSUB_2W 26, 17, 8, 9, 10, 1751, 3703 ; t18a, t29a
+ ITX_MULSUB_2W 18, 25, 8, 9, 10, 3857, 1380 ; t19a, t28a
+ ITX_MULSUB_2W 24, 19, 8, 9, 10, 995, 3973 ; t20a, t27a
+ ITX_MULSUB_2W 16, 27, 8, 9, 10, 3513, 2106 ; t21a, t26a
+ ITX_MULSUB_2W 28, 15, 8, 9, 10, 2440, 3290 ; t22a, t25a
+ ITX_MULSUB_2W 20, 23, 8, 9, 10, 4052, 601 ; t23a, t24a
+.main2:
+ psubsw m8, m22, m14 ; t17
+ paddsw m22, m14 ; t16
+ paddsw m14, m18, m26 ; t19
+ psubsw m18, m26 ; t18
+ psubsw m26, m24, m16 ; t21
+ paddsw m24, m16 ; t20
+ psubsw m16, m20, m28 ; t22
+ paddsw m28, m20 ; t23
+ psubsw m20, m23, m15 ; t25
+ paddsw m23, m15 ; t24
+ psubsw m15, m21, m29 ; t30
+ paddsw m21, m29 ; t31
+ psubsw m29, m19, m27 ; t26
+ paddsw m19, m27 ; t27
+ paddsw m27, m25, m17 ; t28
+ psubsw m25, m17 ; t29
+.main3:
+ ITX_MULSUB_2W 15, 8, 9, 17, 10, 799, 4017 ; t17a, t30a
+ ITX_MULSUB_2W 25, 18, 9, 17, 10, m4017, 799 ; t18a, t29a
+ ITX_MULSUB_2W 29, 26, 9, 17, 10, 3406, 2276 ; t21a, t26a
+ ITX_MULSUB_2W 20, 16, 9, 17, 10, m2276, 3406 ; t22a, t25a
+ vpbroadcastd m12, [o(pw_m3784_1567)]
+ vpbroadcastd m11, [o(pw_1567_3784)]
+ psubsw m17, m21, m27 ; t28a
+ paddsw m21, m27 ; t31a
+ psubsw m27, m15, m25 ; t18
+ paddsw m15, m25 ; t17
+ psubsw m25, m20, m29 ; t21
+ paddsw m20, m29 ; t22
+ psubsw m29, m8, m18 ; t29
+ paddsw m8, m18 ; t30
+ psubsw m18, m22, m14 ; t19a
+ paddsw m22, m14 ; t16a
+ psubsw m14, m28, m24 ; t20a
+ paddsw m24, m28 ; t23a
+ paddsw m28, m16, m26 ; t25
+ psubsw m16, m26 ; t26
+ psubsw m26, m23, m19 ; t27a
+ paddsw m23, m19 ; t24a
+ ITX_MULSUB_2W 29, 27, 9, 19, 10, 11, 12 ; t18a, t29a
+ ITX_MULSUB_2W 17, 18, 9, 19, 10, 11, 12 ; t19, t28
+ vpbroadcastd m11, [o(pw_m1567_m3784)]
+ ITX_MULSUB_2W 16, 25, 9, 19, 10, 12, 11 ; t21a, t26a
+ ITX_MULSUB_2W 26, 14, 9, 19, 10, 12, 11 ; t20, t27
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ vpbroadcastd m11, [o(pw_2896_2896)]
+ psubsw m19, m27, m25 ; t26
+ paddsw m27, m25 ; t29
+ psubsw m25, m17, m26 ; t20a
+ paddsw m17, m26 ; t19a
+ paddsw m26, m18, m14 ; t28a
+ psubsw m18, m14 ; t27a
+ paddsw m14, m22, m24 ; t16
+ psubsw m22, m24 ; t23
+ psubsw m24, m29, m16 ; t21
+ paddsw m16, m29 ; t18
+ paddsw m29, m21, m23 ; t31
+ psubsw m21, m23 ; t24
+ psubsw m23, m15, m20 ; t22a
+ paddsw m15, m20 ; t17a
+ psubsw m20, m8, m28 ; t25a
+ paddsw m28, m8 ; t30a
+ ITX_MULSUB_2W 18, 25, 8, 9, 10, 11, 12 ; t20, t27
+ ITX_MULSUB_2W 19, 24, 8, 9, 10, 11, 12 ; t21a, t26a
+ ITX_MULSUB_2W 21, 22, 8, 9, 10, 11, 12 ; t23a, t24a
+ ITX_MULSUB_2W 20, 23, 8, 9, 10, 11, 12 ; t22, t25
+ ret
+
+%macro IDTX_32x32 2 ; dst[1-2]
+ vmovdqa32 ym%1, [cq+64*(%1+ 0)] ; force EVEX encoding, which
+ vmovdqa32 ym17, [cq+64*(%1+16)] ; reduces code size due to
+ vmovdqa32 ym%2, [cq+64*(%2+ 0)] ; compressed displacements
+ vmovdqa32 ym18, [cq+64*(%2+16)]
+ vpermt2q m%1, m21, m17
+ vpermt2q m%2, m21, m18
+%endmacro
+
+cglobal inv_txfm_add_identity_identity_32x32_8bpc, 3, 3, 22, dst, stride, c
+ movu m21, [permB+7]
+ vpbroadcastd m16, [pw_8192]
+ pxor m20, m20
+.loop:
+ IDTX_32x32 0, 1
+ IDTX_32x32 2, 3
+ IDTX_32x32 4, 5
+ IDTX_32x32 6, 7
+ IDTX_32x32 8, 9
+ IDTX_32x32 10, 11
+ IDTX_32x32 12, 13
+ IDTX_32x32 14, 15
+ call m(inv_txfm_add_identity_identity_16x32_8bpc).transpose_2x8x8_round
+ IDTX_32x16_STORE 0, 8, 1
+ IDTX_32x16_STORE 1, 9, 1
+ IDTX_32x16_STORE 2, 10, 1
+ IDTX_32x16_STORE 3, 11, 1
+ IDTX_32x16_STORE 4, 12, 1
+ IDTX_32x16_STORE 5, 13, 1
+ IDTX_32x16_STORE 6, 14, 1
+ IDTX_32x16_STORE 7, 15, 1
+ lea dstq, [dstq+strideq*8]
+ btc cq, 5
+ jnc .loop
+ mov r0d, 8
+.zero_loop:
+ mova [cq+64*0], m20
+ mova [cq+64*1], m20
+ mova [cq+64*2], m20
+ mova [cq+64*3], m20
+ add cq, 64*4
+ dec r0d
+ jg .zero_loop
+ RET
+
+cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 7, 0, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ WIN64_SPILL_XMM 30
+ cmp eobd, 151
+ jb .fast
+ mova m5, [cq+64*10]
+ mova m3, [cq+64* 6]
+ mova m1, [cq+64* 2]
+ mova m7, [cq+64*14]
+ mova m2, [cq+64* 4]
+ mova m6, [cq+64*12]
+ mova m0, [cq+64* 0]
+ mova m4, [cq+64* 8]
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ mova m14, [cq+64* 1]
+ mova m21, [cq+64*15]
+ mova m18, [cq+64* 9]
+ mova m17, [cq+64* 7]
+ mova m16, [cq+64* 5]
+ mova m19, [cq+64*11]
+ mova m20, [cq+64*13]
+ mova m15, [cq+64* 3]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ vpbroadcastd m9, [o(pw_8192)]
+%macro TRANSPOSE_8x4_ROUND 4
+ punpckhwd m8, m%3, m%4 ; c4 d4 c5 d5 c6 d6 c7 d7
+ punpcklwd m%3, m%4 ; c0 d0 c1 d1 c2 d2 c3 d3
+ punpckhwd m%4, m%1, m%2 ; a4 b4 a5 b5 a6 b6 a7 b7
+ punpcklwd m%1, m%2 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpckhdq m%2, m%1, m%3 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m%1, m%3 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckldq m%3, m%4, m8 ; a4 b4 c4 d4 a5 b5 c5 d5
+ punpckhdq m%4, m8 ; a6 b6 c6 d6 a7 b7 c7 d7
+ REPX {pmulhrsw x, m9}, m%2, m%1, m%3, m%4
+%endmacro
+ TRANSPOSE_8x4_ROUND 0, 1, 2, 3
+ TRANSPOSE_8x4_ROUND 4, 5, 6, 7
+ TRANSPOSE_8x4_ROUND 14, 15, 16, 17
+ TRANSPOSE_8x4_ROUND 18, 19, 20, 21
+ vinserti32x8 m26, m0, ym4, 1 ; a0 a4 b0 b4
+ vshufi32x4 m0, m4, q3232 ; a8 a12 b8 b12
+ vinserti32x8 m27, m1, ym5, 1 ; a1 a5 b1 b5
+ vshufi32x4 m1, m5, q3232 ; a9 a13 b9 b13
+ vinserti32x8 m28, m2, ym6, 1 ; a2 a6 b2 b6
+ vshufi32x4 m2, m6, q3232 ; a10 a14 b10 b14
+ vinserti32x8 m29, m3, ym7, 1 ; a3 a7 b3 b7
+ vshufi32x4 m8, m3, m7, q3232 ; a11 a15 b11 b15
+ vinserti32x8 m4, m14, ym18, 1 ; c0 c4 d0 d4
+ vshufi32x4 m14, m18, q3232 ; c8 c12 d8 d12
+ vinserti32x8 m5, m15, ym19, 1 ; c1 c5 d1 d5
+ vshufi32x4 m15, m19, q3232 ; c9 c13 d9 d13
+ vinserti32x8 m6, m16, ym20, 1 ; c2 c6 d2 d6
+ vshufi32x4 m16, m20, q3232 ; c10 c14 d10 d14
+ vinserti32x8 m7, m17, ym21, 1 ; c3 c7 d3 d7
+ vshufi32x4 m17, m21, q3232 ; c11 c15 d11 d15
+ vshufi32x4 m22, m26, m4, q2020 ; 0 1
+ vshufi32x4 m26, m4, q3131 ; 8 9
+ vshufi32x4 m23, m27, m5, q2020 ; 2 3
+ vshufi32x4 m27, m5, q3131 ; 10 11
+ vshufi32x4 m24, m28, m6, q2020 ; 4 5
+ vshufi32x4 m28, m6, q3131 ; 12 13
+ vshufi32x4 m25, m29, m7, q2020 ; 6 7
+ vshufi32x4 m29, m7, q3131 ; 14 15
+ vshufi32x4 m4, m0, m14, q2020 ; 16 17
+ vshufi32x4 m3, m0, m14, q3131 ; 24 25
+ vshufi32x4 m20, m1, m15, q2020 ; 18 19
+ vshufi32x4 m19, m1, m15, q3131 ; 26 27
+ vshufi32x4 m5, m2, m16, q2020 ; 20 21
+ vshufi32x4 m0, m2, m16, q3131 ; 28 29
+ vshufi32x4 m16, m8, m17, q2020 ; 22 23
+ vshufi32x4 m17, m8, m17, q3131 ; 30 31
+ pxor m6, m6
+ mova [cq+64* 0], m4
+ mova [cq+64* 2], m5
+ mova [cq+64* 4], m3
+ mova [cq+64* 6], m0
+ punpcklwd m8, m24, m24 ; 4
+ punpcklwd m0, m0 ; 28
+ punpcklwd m5, m5 ; 20
+ punpcklwd m1, m28, m28 ; 12
+ punpcklwd m7, m26, m26 ; 8
+ punpcklwd m3, m3 ; 24
+ punpcklwd m9, m6, m22 ; __ 0
+ punpcklwd m6, m4 ; __ 16
+ call m(idct_16x16_internal_8bpc).main_fast3
+ mova [cq+64* 1], m20
+ mova [cq+64* 3], m16
+ mova [cq+64* 5], m19
+ mova [cq+64* 7], m17
+ punpcklwd m21, m23, m23 ; 2
+ punpcklwd m17, m17 ; 30
+ punpcklwd m20, m20 ; 18
+ punpcklwd m15, m29, m29 ; 14
+ punpcklwd m18, m27, m27 ; 10
+ punpcklwd m16, m16 ; 22
+ punpcklwd m19, m19 ; 26
+ punpcklwd m14, m25, m25 ; 6
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast
+ mova [cq+64* 8], m14
+ mova [cq+64* 9], m15
+ mova [cq+64*10], m16
+ mova [cq+64*11], m17
+ mova [cq+64*12], m18
+ mova [cq+64*13], m19
+ mova [cq+64*14], m20
+ mova [cq+64*15], m21
+ mova m21, [cq+64* 7]
+ mova m14, [cq+64* 0]
+ mova m17, [cq+64* 3]
+ mova m18, [cq+64* 4]
+ mova m19, [cq+64* 5]
+ mova m16, [cq+64* 2]
+ mova m15, [cq+64* 1]
+ mova m20, [cq+64* 6]
+ REPX {punpckhwd x, x}, m22, m21, m14, m29, m26, m17, m18, m25, \
+ m24, m19, m16, m27, m28, m15, m20, m23
+ call .main_oddhalf
+ jmp .end
+.fast: ; right half is zero
+ mova ym8, [cq+64*15]
+ vinserti32x8 m8, [cq+64* 1], 1
+ mova m2, [o(int16_perm)]
+ mova ym9, [cq+64* 8]
+ vinserti32x8 m9, [cq+64* 0], 1
+ mova ym0, [cq+64* 7]
+ vinserti32x8 m0, [cq+64* 9], 1
+ mova ym7, [cq+64*14]
+ vinserti32x8 m7, [cq+64* 2], 1
+ mova ym1, [cq+64* 3]
+ vinserti32x8 m1, [cq+64*13], 1
+ mova ym3, [cq+64* 6]
+ vinserti32x8 m3, [cq+64*10], 1
+ mova ym5, [cq+64*11]
+ vinserti32x8 m5, [cq+64* 5], 1
+ mova ym6, [cq+64*12]
+ vinserti32x8 m6, [cq+64* 4], 1
+ REPX {vpermb x, m2, x}, m8, m9, m0, m7, m1, m3, m5, m6
+ call m(idct_16x16_internal_8bpc).main2
+ vbroadcasti32x4 m8, [o(int_shuf3)]
+ vbroadcasti32x4 m9, [o(int_shuf4)]
+ vpbroadcastd m11, [o(pw_8192)]
+ pshufb m0, m8
+ pshufb m1, m9
+ pshufb m2, m8
+ pshufb m3, m9
+ REPX {pmulhrsw x, m11}, m0, m1, m2, m3
+ pshufb m4, m8
+ pshufb m5, m9
+ pshufb m6, m8
+ pshufb m7, m9
+ REPX {pmulhrsw x, m11}, m4, m5, m6, m7
+ punpckhdq m28, m0, m1
+ punpckldq m0, m1
+ punpckhdq m27, m2, m3
+ punpckldq m2, m3
+ punpckhdq m22, m4, m5
+ punpckldq m4, m5
+ punpckhdq m23, m6, m7
+ punpckldq m6, m7
+ vinserti32x8 m14, m0, ym2, 1
+ vshufi32x4 m15, m0, m2, q3232
+ vinserti32x8 m2, m4, ym6, 1
+ vshufi32x4 m4, m6, q3232
+ vshufi32x4 m21, m14, m2, q2020 ; 0 2
+ vshufi32x4 m14, m2, q3131 ; 4 6
+ vshufi32x4 m18, m15, m4, q2020 ; 8 10
+ vshufi32x4 m15, m4, q3131 ; 12 14
+ pxor m9, m9
+ punpcklwd m8, m14, m14 ; 4
+ punpcklwd m1, m15, m15 ; 12
+ punpcklwd m7, m18, m18 ; 8
+ punpcklwd m9, m21 ; __ 0
+ call m(idct_16x16_internal_8bpc).main_fast4
+ punpckhwd m21, m21 ; 2
+ punpckhwd m15, m15 ; 14
+ punpckhwd m18, m18 ; 10
+ punpckhwd m14, m14 ; 6
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2
+ vinserti32x8 m24, m28, ym27, 1
+ vshufi32x4 m28, m27, q3232
+ vinserti32x8 m27, m22, ym23, 1
+ vshufi32x4 m22, m23, q3232
+ vshufi32x4 m23, m24, m27, q2020 ; 1 3
+ vshufi32x4 m24, m27, q3131 ; 5 7
+ vshufi32x4 m27, m28, m22, q2020 ; 9 11
+ vshufi32x4 m28, m22, q3131 ; 13 15
+ punpcklwd m22, m23, m23 ; 1
+ punpckhwd m29, m28, m28 ; 15
+ punpcklwd m26, m27, m27 ; 9
+ punpckhwd m25, m24, m24 ; 7
+ mova [cq+64* 8], m14
+ mova [cq+64* 9], m15
+ mova [cq+64*10], m16
+ mova [cq+64*11], m17
+ punpcklwd m24, m24 ; 5
+ punpckhwd m27, m27 ; 11
+ punpcklwd m28, m28 ; 13
+ punpckhwd m23, m23 ; 3
+ mova [cq+64*12], m18
+ mova [cq+64*13], m19
+ mova [cq+64*14], m20
+ mova [cq+64*15], m21
+ call .main_oddhalf_fast
+.end:
+ imul r6, strideq, 60
+ mova m10, [o(end_16x32p)]
+ vpbroadcastd m11, [o(pw_2048)]
+ lea r3, [strideq*3]
+ pxor m12, m12
+ add r6, dstq ; dst+stride*60
+ psrldq m13, m10, 1
+ lea r4, [strideq+r3] ; stride*4
+%macro IDCT_16x64_END 3 ; idct32, idct64, tmp
+%if %1 & 1
+ %define %%s0 r3
+ %define %%s1 strideq*2
+ %define %%s2 strideq*1
+ %define %%s3 strideq*0
+%else
+ %define %%s0 strideq*0
+ %define %%s1 strideq*1
+ %define %%s2 strideq*2
+ %define %%s3 r3
+%if %1
+ add dstq, r4
+ sub r6, r4
+%endif
+%endif
+%if %1 < 8
+ pmulhrsw m8, m11, m%1
+ pmulhrsw m9, m11, m%2
+%else
+ mova m9, [cq+64*%1]
+ paddsw m8, m9, m%2 ; out 0+n, 1+n
+ psubsw m9, m%2 ; out 63-n, 62-n
+ pmulhrsw m8, m11
+ pmulhrsw m9, m11
+%endif
+ mova xm29, [dstq+%%s0]
+ vinserti128 ym29, [dstq+%%s1], 1
+ mova xm%3, [r6 +%%s3]
+ vinserti128 ym%3, [r6 +%%s2], 1
+ vpermb m29, m10, m29
+ vpermb m%3, m10, m%3
+ mova [cq+64*%1], m12
+ paddw m29, m8
+ paddw m%3, m9
+ packuswb m29, m%3
+ vpermd m29, m13, m29
+ mova [dstq+%%s0], xm29
+ vextracti128 [dstq+%%s1], ym29, 1
+ vextracti32x4 [r6 +%%s2], m29, 2
+ vextracti32x4 [r6 +%%s3], m29, 3
+%endmacro
+ IDCT_16x64_END 0, 29, 0
+ IDCT_16x64_END 1, 28, 28
+ IDCT_16x64_END 2, 27, 28
+ IDCT_16x64_END 3, 26, 28
+ IDCT_16x64_END 4, 25, 28
+ IDCT_16x64_END 5, 24, 28
+ IDCT_16x64_END 6, 23, 28
+ IDCT_16x64_END 7, 22, 28
+ IDCT_16x64_END 8, 21, 28
+ IDCT_16x64_END 9, 20, 28
+ IDCT_16x64_END 10, 19, 28
+ IDCT_16x64_END 11, 18, 28
+ IDCT_16x64_END 12, 17, 28
+ IDCT_16x64_END 13, 16, 28
+ IDCT_16x64_END 14, 15, 28
+ IDCT_16x64_END 15, 14, 28
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 64
+ imul r6d, 181
+ add r6d, 128+512
+ sar r6d, 8+2
+ jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3
+ALIGN function_align
+cglobal_label .main_oddhalf_fast ; bottom three-quarters are zero
+ vpbroadcastd m8, [o(pw_101_4095x8)]
+ vpbroadcastd m21, [o(pw_m1474_3822x8)]
+ vpbroadcastd m14, [o(pw_897_3996x8)]
+ vpbroadcastd m17, [o(pw_m700_4036x8)]
+ vpbroadcastd m18, [o(pw_501_4065x8)]
+ vpbroadcastd m19, [o(pw_m1092_3948x8)]
+ vpbroadcastd m16, [o(pw_1285_3889x8)]
+ vpbroadcastd m15, [o(pw_m301_4085x8)]
+ pmulhrsw m8, m22 ; t32a t63a
+ pmulhrsw m21, m29 ; t35a t60a
+ pmulhrsw m14, m26 ; t36a t59a
+ pmulhrsw m17, m25 ; t39a t56
+ pmulhrsw m18, m24 ; t40a t55a
+ pmulhrsw m19, m27 ; t43a t52a
+ pmulhrsw m16, m28 ; t44a t51a
+ pmulhrsw m15, m23 ; t47a t48a
+ mova m22, m8
+ mova m29, m21
+ mova m26, m14
+ mova m25, m17
+ mova m24, m18
+ mova m27, m19
+ mova m28, m16
+ mova m20, m15
+ jmp .main_oddhalf2
+ALIGN function_align
+cglobal_label .main_oddhalf
+ vpbroadcastd m8, [o(pw_101_4095x8)]
+ vpbroadcastd m9, [o(pw_m2824_2967x8)]
+ vpbroadcastd m11, [o(pw_1660_3745x8)]
+ vpbroadcastd m12, [o(pw_m1474_3822x8)]
+ pmulhrsw m22, m8 ; t32a t63a
+ vpbroadcastd m8, [o(pw_897_3996x8)]
+ pmulhrsw m21, m9 ; t33a t62a
+ vpbroadcastd m9, [o(pw_m2191_3461x8)]
+ pmulhrsw m14, m11 ; t34a t61a
+ vpbroadcastd m11, [o(pw_2359_3349x8)]
+ pmulhrsw m29, m12 ; t35a t60a
+ vpbroadcastd m12, [o(pw_m700_4036x8)]
+ pmulhrsw m26, m8 ; t36a t59a
+ vpbroadcastd m8, [o(pw_501_4065x8)]
+ pmulhrsw m17, m9 ; t37a t58a
+ vpbroadcastd m9, [o(pw_m2520_3229x8)]
+ pmulhrsw m18, m11 ; t38a t57a
+ vpbroadcastd m11, [o(pw_2019_3564x8)]
+ pmulhrsw m25, m12 ; t39a t56a
+ vpbroadcastd m12, [o(pw_m1092_3948x8)]
+ pmulhrsw m24, m8 ; t40a t55a
+ vpbroadcastd m8, [o(pw_1285_3889x8)]
+ pmulhrsw m19, m9 ; t41a t54a
+ vpbroadcastd m9, [o(pw_m1842_3659x8)]
+ pmulhrsw m16, m11 ; t42a t53a
+ vpbroadcastd m11, [o(pw_2675_3102x8)]
+ pmulhrsw m27, m12 ; t43a t52a
+ vpbroadcastd m12, [o(pw_m301_4085x8)]
+ pmulhrsw m28, m8 ; t44a t51a
+ pmulhrsw m15, m9 ; t45a t50a
+ pmulhrsw m20, m11 ; t46a t49a
+ pmulhrsw m23, m12 ; t47a t48a
+ psubsw m8, m22, m21 ; t33 t62
+ paddsw m22, m21 ; t32 t63
+ psubsw m21, m29, m14 ; t34 t61
+ paddsw m29, m14 ; t35 t60
+ psubsw m14, m26, m17 ; t37 t58
+ paddsw m26, m17 ; t36 t59
+ psubsw m17, m25, m18 ; t38 t57
+ paddsw m25, m18 ; t39 t56
+ psubsw m18, m24, m19 ; t41 t54
+ paddsw m24, m19 ; t40 t55
+ psubsw m19, m27, m16 ; t42 t53
+ paddsw m27, m16 ; t43 t52
+ psubsw m16, m28, m15 ; t45 t50
+ paddsw m28, m15 ; t44 t51
+ psubsw m15, m23, m20 ; t46 t49
+ paddsw m20, m23 ; t47 t48
+.main_oddhalf2:
+ ITX_MUL2X_PACK 8, 9, 23, 10, 401, 4076, 5 ; t33a t62a
+ ITX_MUL2X_PACK 21, 9, 23, 10, m4076, 401, 5 ; t34a t61a
+ ITX_MUL2X_PACK 14, 9, 23, 10, 3166, 2598, 5 ; t37a t58a
+ ITX_MUL2X_PACK 17, 9, 23, 10, m2598, 3166, 5 ; t38a t57a
+ ITX_MUL2X_PACK 18, 9, 23, 10, 1931, 3612, 5 ; t41a t54a
+ ITX_MUL2X_PACK 19, 9, 23, 10, m3612, 1931, 5 ; t42a t53a
+ ITX_MUL2X_PACK 16, 9, 23, 10, 3920, 1189, 5 ; t45a t50a
+ ITX_MUL2X_PACK 15, 9, 23, 10, m1189, 3920, 5 ; t46a t49a
+ vpbroadcastd m11, [o(pw_m4017_799)]
+ psubsw m23, m25, m26 ; t36a t59a
+ paddsw m25, m26 ; t39a t56a
+ psubsw m26, m24, m27 ; t43a t52a
+ paddsw m27, m24 ; t40a t55a
+ psubsw m24, m20, m28 ; t44a t51a
+ paddsw m20, m28 ; t47a t48a
+ psubsw m28, m8, m21 ; t34 t61
+ paddsw m8, m21 ; t33 t62
+ psubsw m21, m17, m14 ; t37 t58
+ paddsw m17, m14 ; t38 t57
+ psubsw m14, m18, m19 ; t42 t53
+ paddsw m18, m19 ; t41 t54
+ psubsw m19, m15, m16 ; t45 t50
+ paddsw m15, m16 ; t46 t49
+ psubsw m16, m22, m29 ; t35a t60a
+ paddsw m22, m29 ; t32a t63a
+ ITX_MUL2X_PACK 16, 9, 29, 10, 799_4017, 11, 20 ; t35 t60
+ ITX_MUL2X_PACK 28, 9, 29, 10, 799_4017, 11, 20 ; t34a t61a
+ ITX_MUL2X_PACK 23, 9, 29, 10, 11, m799_m4017, 36 ; t36 t59
+ ITX_MUL2X_PACK 21, 9, 29, 10, 11, m799_m4017, 36 ; t37a t58a
+ vpbroadcastd m11, [o(pw_m2276_3406)]
+ ITX_MUL2X_PACK 26, 9, 29, 10, 3406_2276, 11, 20 ; t43 t52
+ ITX_MUL2X_PACK 14, 9, 29, 10, 3406_2276, 11, 20 ; t42a t53a
+ ITX_MUL2X_PACK 24, 9, 29, 10, 11, m3406_m2276, 36 ; t44 t51
+ ITX_MUL2X_PACK 19, 9, 29, 10, 11, m3406_m2276, 36 ; t45a t50a
+ vpbroadcastd m11, [o(pw_1567_3784)]
+ vpbroadcastd m12, [o(pw_m3784_1567)]
+ psubsw m29, m22, m25 ; t39 t56
+ paddsw m22, m25 ; t32 t63
+ psubsw m25, m20, m27 ; t40 t55
+ paddsw m20, m27 ; t47 t48
+ psubsw m27, m8, m17 ; t38a t57a
+ paddsw m8, m17 ; t33a t62a
+ psubsw m17, m15, m18 ; t41a t54a
+ paddsw m15, m18 ; t46a t49a
+ paddsw m18, m16, m23 ; t35a t60a
+ psubsw m16, m23 ; t36a t59a
+ psubsw m23, m24, m26 ; t43a t52a
+ paddsw m24, m26 ; t44a t51a
+ paddsw m26, m28, m21 ; t34 t61
+ psubsw m28, m21 ; t37 t58
+ psubsw m21, m19, m14 ; t42 t53
+ paddsw m19, m14 ; t45 t50
+ ITX_MUL2X_PACK 29, 9, 14, 10, 11, 12, 4 ; t39a t56a
+ ITX_MUL2X_PACK 27, 9, 14, 10, 11, 12, 4 ; t38 t57
+ ITX_MUL2X_PACK 16, 9, 14, 10, 11, 12, 4 ; t36 t59
+ ITX_MUL2X_PACK 28, 9, 14, 10, 11, 12, 4 ; t37a t58a
+ vpbroadcastd m11, [o(pw_m1567_m3784)]
+ ITX_MUL2X_PACK 25, 9, 14, 10, 12, 11, 4 ; t40a t55a
+ ITX_MUL2X_PACK 17, 9, 14, 10, 12, 11, 4 ; t41 t54
+ ITX_MUL2X_PACK 23, 9, 14, 10, 12, 11, 4 ; t43 t52
+ ITX_MUL2X_PACK 21, 9, 14, 10, 12, 11, 4 ; t42a t53a
+ vbroadcasti32x4 m13, [o(deint_shuf)]
+ vpbroadcastd m11, [o(pw_2896_2896)]
+ vpbroadcastd m12, [o(pw_m2896_2896)]
+ paddsw m14, m22, m20 ; t32a t63a
+ psubsw m22, m20 ; t47a t48a
+ psubsw m20, m8, m15 ; t46 t49
+ paddsw m8, m15 ; t33 t62
+ paddsw m15, m18, m24 ; t35 t60
+ psubsw m18, m24 ; t44 t51
+ psubsw m24, m26, m19 ; t45a t50a
+ paddsw m26, m19 ; t34a t61a
+ REPX {pshufb x, m13}, m14, m8, m15, m26
+ psubsw m19, m29, m25 ; t40 t55
+ paddsw m25, m29 ; t39 t56
+ psubsw m29, m27, m17 ; t41a t54a
+ paddsw m27, m17 ; t38a t57a
+ psubsw m17, m16, m23 ; t43a t52a
+ paddsw m16, m23 ; t36a t59a
+ psubsw m9, m28, m21 ; t42 t53
+ paddsw m28, m21 ; t37 t58
+ REPX {pshufb x, m13}, m25, m27, m16, m28
+ ITX_MUL2X_PACK 22, 13, 21, 10, 11, 12, 8 ; t47 t48
+ ITX_MUL2X_PACK 20, 23, 22, 10, 11, 12, 8 ; t46a t49a
+ packssdw m21, m22 ; t47 t46a
+ packssdw m13, m23 ; t48 t49a
+ ITX_MUL2X_PACK 18, 22, 20, 10, 11, 12, 8 ; t44a t51a
+ ITX_MUL2X_PACK 24, 23, 18, 10, 11, 12, 8 ; t45 t50
+ packssdw m20, m18 ; t44a t45
+ packssdw m22, m23 ; t51a t50
+ ITX_MUL2X_PACK 19, 24, 18, 10, 11, 12, 8 ; t40a t55a
+ ITX_MUL2X_PACK 29, 23, 19, 10, 11, 12, 8 ; t41 t54
+ packssdw m18, m19 ; t40a t41
+ packssdw m24, m23 ; t55a t54
+ ITX_MUL2X_PACK 17, 23, 19, 10, 11, 12, 8 ; t43 t52
+ ITX_MUL2X_PACK 9, 29, 17, 10, 11, 12, 8 ; t42a t53a
+ packssdw m19, m17 ; t43 t42a
+ packssdw m23, m29 ; t52 t53a
+ punpcklqdq m17, m25, m27 ; t39 t38a
+ punpckhqdq m25, m27 ; t56 t57a
+ punpckhqdq m27, m15, m26 ; t60 t61a
+ punpcklqdq m15, m26 ; t35 t34a
+ punpckhqdq m26, m16, m28 ; t59a t58
+ punpcklqdq m16, m28 ; t36a t37
+ punpckhqdq m28, m14, m8 ; t63a t62
+ punpcklqdq m14, m8 ; t32a t33
+ psubsw m29, m0, m28 ; out63 out62
+ paddsw m0, m28 ; out0 out1
+ psubsw m28, m1, m27 ; out60 out61
+ paddsw m1, m27 ; out3 out2
+ psubsw m27, m2, m26 ; out59 out58
+ paddsw m2, m26 ; out4 out5
+ psubsw m26, m3, m25 ; out56 out57
+ paddsw m3, m25 ; out7 out6
+ psubsw m25, m4, m24 ; out55 out54
+ paddsw m4, m24 ; out8 out9
+ psubsw m24, m5, m23 ; out52 out53
+ paddsw m5, m23 ; out11 out10
+ psubsw m23, m6, m22 ; out51 out50
+ paddsw m6, m22 ; out12 out13
+ psubsw m22, m7, m13 ; out48 out49
+ paddsw m7, m13 ; out15 out14
+ ret
+
+cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 7, 0, dst, stride, c, eob
+%undef cmp
+ lea r5, [o_base]
+ test eobd, eobd
+ jnz .normal
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 16
+.dconly:
+ imul r6d, 181
+ add r6d, 128+512
+ sar r6d, 8+2
+.dconly2:
+ imul r6d, 181
+ add r6d, 128+2048
+ sar r6d, 8+4
+ pxor m2, m2
+ vpbroadcastw m3, r6d
+.dconly_loop:
+ mova m1, [dstq]
+ punpcklbw m0, m1, m2
+ punpckhbw m1, m2
+ paddw m0, m3
+ paddw m1, m3
+ packuswb m0, m1
+ mova [dstq], m0
+ add dstq, strideq
+ dec r3d
+ jg .dconly_loop
+ RET
+.normal:
+ WIN64_SPILL_XMM 31
+ mova m19, [o(dup16_perm)]
+ mova m24, [cq+64* 2]
+ mova m28, [cq+64* 6]
+ mova m26, [cq+64* 4]
+ mova m22, [cq+64* 0]
+ mova m23, [cq+64* 1]
+ mova m29, [cq+64* 7]
+ mova m27, [cq+64* 5]
+ mova m25, [cq+64* 3]
+ vpermb m8, m19, m24 ; 4
+ vpermb m1, m19, m28 ; 12
+ vpermb m7, m19, m26 ; 8
+ vpermb m9, m19, m22 ; __ 0
+ vpermb m21, m19, m23 ; 2
+ vpermb m15, m19, m29 ; 14
+ vpermb m18, m19, m27 ; 10
+ vpermb m14, m19, m25 ; 6
+ pslld m9, 16
+ vpord m30, m19, [o(pb_32)] {1to16}
+ REPX {vpermb x, m30, x}, m22, m29, m26, m25, m24, m27, m28, m23
+ cmp eobd, 151
+ jb .fast
+ vpermb m0, m19, [cq+64*14] ; 28
+ vpermb m5, m19, [cq+64*10] ; 20
+ vpermb m3, m19, [cq+64*12] ; 24
+ vpermb m6, m19, [cq+64* 8] ; __ 16
+ pslld m6, 16
+ call m(idct_16x16_internal_8bpc).main_fast
+ vpermb m17, m19, [cq+64*15] ; 30
+ vpermb m20, m19, [cq+64* 9] ; 18
+ vpermb m16, m19, [cq+64*11] ; 22
+ vpermb m19, m19, [cq+64*13] ; 26
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ vpermb m21, m30, [cq+64*15]
+ vpermb m14, m30, [cq+64* 8]
+ vpermb m17, m30, [cq+64*11]
+ vpermb m18, m30, [cq+64*12]
+ vpermb m19, m30, [cq+64*13]
+ vpermb m16, m30, [cq+64*10]
+ vpermb m15, m30, [cq+64* 9]
+ vpermb m20, m30, [cq+64*14]
+ call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf
+ jmp .end
+.fast: ; bottom half is zero
+ call m(idct_16x16_internal_8bpc).main_fast2
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast
+.end:
+ mova [cq+64* 8], m4
+ mova [cq+64* 9], m5
+ mova [cq+64*10], m6
+ mova [cq+64*11], m7
+ mova [cq+64*12], m26
+ mova [cq+64*13], m27
+ mova [cq+64*14], m28
+ mova [cq+64*15], m29
+ vpbroadcastd m13, [o(pw_8192)]
+ call .pass1_end
+ call .pass2
+ mova [cq+64* 0], m0
+ mova [cq+64* 1], m1
+ mova [cq+64* 2], m2
+ mova [cq+64* 3], m3
+ mova [cq+64* 4], m4
+ mova [cq+64* 5], m5
+ mova [cq+64* 6], m6
+ mova [cq+64* 7], m7
+ pmulhrsw m0, m13, [cq+64* 8]
+ pmulhrsw m1, m13, [cq+64* 9]
+ pmulhrsw m2, m13, [cq+64*10]
+ pmulhrsw m3, m13, [cq+64*11]
+ vpbroadcastd m30, [o(pw_2048)]
+ pmulhrsw m4, m13, m22
+ pmulhrsw m5, m13, m23
+ pmulhrsw m6, m13, m24
+ pmulhrsw m7, m13, m25
+ pmulhrsw m22, m30, m14
+ pmulhrsw m14, m13, m26
+ pmulhrsw m23, m30, m15
+ pmulhrsw m15, m13, m27
+ pmulhrsw m24, m30, m16
+ pmulhrsw m16, m13, m28
+ pmulhrsw m25, m30, m17
+ pmulhrsw m17, m13, m29
+ pmulhrsw m26, m30, m18
+ pmulhrsw m18, m13, [cq+64*12]
+ pmulhrsw m27, m30, m19
+ pmulhrsw m19, m13, [cq+64*13]
+ pmulhrsw m28, m30, m20
+ pmulhrsw m20, m13, [cq+64*14]
+ pmulhrsw m29, m30, m21
+ pmulhrsw m21, m13, [cq+64*15]
+ call .transpose_round
+ call .pass2
+ pxor m10, m10
+ lea r3, [strideq*3]
+%macro IDCT_64x16_END 4
+ mova m9, [dstq+%4]
+%if %1 < 8
+ pmulhrsw m%3, m30, [cq+64*%1]
+%endif
+ pmulhrsw m%2, m30
+ mova [cq+64*%1], m10
+ punpcklbw m8, m9, m10
+ punpckhbw m9, m10
+ paddw m8, m%3
+ paddw m9, m%2
+ packuswb m8, m9
+ mova [dstq+%4], m8
+%if %1 == 3 || %1 == 7 || %1 == 11
+ lea dstq, [dstq+strideq*4]
+%endif
+%endmacro
+ IDCT_64x16_END 0, 0, 11, strideq*0
+ IDCT_64x16_END 1, 1, 11, strideq*1
+ IDCT_64x16_END 2, 2, 11, strideq*2
+ IDCT_64x16_END 3, 3, 11, r3
+ IDCT_64x16_END 4, 4, 11, strideq*0
+ IDCT_64x16_END 5, 5, 11, strideq*1
+ IDCT_64x16_END 6, 6, 11, strideq*2
+ IDCT_64x16_END 7, 7, 11, r3
+ IDCT_64x16_END 8, 14, 22, strideq*0
+ IDCT_64x16_END 9, 15, 23, strideq*1
+ IDCT_64x16_END 10, 16, 24, strideq*2
+ IDCT_64x16_END 11, 17, 25, r3
+ IDCT_64x16_END 12, 18, 26, strideq*0
+ IDCT_64x16_END 13, 19, 27, strideq*1
+ IDCT_64x16_END 14, 20, 28, strideq*2
+ IDCT_64x16_END 15, 21, 29, r3
+ RET
+ALIGN function_align
+.pass1_end:
+ mova m4, [cq+64* 0]
+ mova m5, [cq+64* 1]
+ mova m6, [cq+64* 2]
+ mova m7, [cq+64* 3]
+ mova m8, [cq+64* 4]
+ mova m9, [cq+64* 5]
+ mova m11, [cq+64* 6]
+ mova m12, [cq+64* 7]
+ psubsw m29, m4, m21 ; out47 out46
+ paddsw m4, m21 ; out16 out17
+ psubsw m28, m5, m20 ; out44 out45
+ paddsw m5, m20 ; out19 out18
+ REPX {pmulhrsw x, m13}, m0, m1, m2, m3
+ psubsw m27, m6, m19 ; out43 out42
+ paddsw m6, m19 ; out20 out21
+ psubsw m26, m7, m18 ; out40 out41
+ paddsw m7, m18 ; out23 out22
+ pmulhrsw m18, m13, m22
+ pmulhrsw m19, m13, m23
+ pmulhrsw m20, m13, m24
+ pmulhrsw m21, m13, m25
+ paddsw m25, m12, m14 ; out31 out30
+ psubsw m14, m12, m14 ; out32 out33
+ paddsw m24, m11, m15 ; out28 out29
+ psubsw m15, m11, m15 ; out35 out34
+ REPX {pmulhrsw x, m13}, m4, m5, m6, m7
+ paddsw m23, m9, m16 ; out27 out26
+ psubsw m16, m9, m16 ; out36 out37
+ paddsw m22, m8, m17 ; out24 out25
+ psubsw m17, m8, m17 ; out39 out38
+ REPX {pmulhrsw x, m13}, m14, m15, m16, m17
+.transpose_round:
+%macro TRANSPOSE_8x4_PACKED 4
+ punpckhwd m8, m%1, m%3 ; b0 f0 b1 f1 b2 f2 b3 f3
+ punpcklwd m%1, m%3 ; a0 e0 a1 e1 a2 e2 a3 e3
+ punpcklwd m%3, m%2, m%4 ; d0 h0 d1 h1 d2 h2 d3 h3
+ punpckhwd m%2, m%4 ; c0 g0 c1 g1 c2 g2 c3 g3
+ punpckhwd m%4, m%1, m%2 ; a2 c2 e2 g2 a3 c3 e3 g3
+ punpcklwd m%1, m%2 ; a0 c0 e0 g0 a1 c1 e1 g1
+ punpckhwd m%2, m8, m%3 ; b2 d2 f2 h2 b3 d3 f3 h3
+ punpcklwd m8, m%3 ; b0 d0 f0 h0 b1 d1 f1 h1
+ punpcklwd m%3, m%4, m%2 ; 2
+ punpckhwd m%4, m%2 ; 3
+ punpckhwd m%2, m%1, m8 ; 1
+ punpcklwd m%1, m8 ; 0
+%endmacro
+ TRANSPOSE_8x4_PACKED 0, 1, 2, 3
+ TRANSPOSE_8x4_PACKED 18, 19, 20, 21
+ TRANSPOSE_8x4_PACKED 4, 5, 6, 7
+ TRANSPOSE_8x4_PACKED 14, 15, 16, 17
+ vshufi32x4 m8, m0, m4, q3232 ; a02 a03 b02 b03
+ vinserti32x8 m0, ym4, 1 ; a00 a01 b00 b01
+ vshufi32x4 m4, m1, m5, q3232 ; a12 a13 b12 b13
+ vinserti32x8 m9, m1, ym5, 1 ; a10 a11 b10 b11
+ vshufi32x4 m5, m2, m6, q3232 ; a22 a23 b22 b23
+ vinserti32x8 m1, m2, ym6, 1 ; a20 a21 b20 b21
+ vshufi32x4 m6, m3, m7, q3232 ; a32 a33 b32 b33
+ vinserti32x8 m11, m3, ym7, 1 ; a30 a31 b30 b31
+ vshufi32x4 m2, m14, m18, q3232 ; c02 c03 d02 d03
+ vinserti32x8 m3, m14, ym18, 1 ; c00 c01 d00 d01
+ vshufi32x4 m18, m15, m19, q3232 ; c12 c13 d12 d13
+ vinserti32x8 m15, ym19, 1 ; c10 c11 d10 d11
+ vshufi32x4 m19, m16, m20, q3232 ; c22 c23 d22 d23
+ vinserti32x8 m16, ym20, 1 ; c20 c21 d20 d21
+ vshufi32x4 m20, m17, m21, q3232 ; c32 c33 d32 d33
+ vinserti32x8 m17, ym21, 1 ; c30 c31 d30 d31
+ ret
+.pass2:
+ vshufi32x4 m7, m5, m19, q3131 ; 14
+ vshufi32x4 m5, m19, q2020 ; 10
+ vshufi32x4 m21, m6, m20, q3131 ; 15
+ vshufi32x4 m19, m6, m20, q2020 ; 11
+ vshufi32x4 m20, m4, m18, q3131 ; 13
+ vshufi32x4 m18, m4, m18, q2020 ; 9
+ vshufi32x4 m6, m8, m2, q3131 ; 12
+ vshufi32x4 m4, m8, m2, q2020 ; 8
+ vshufi32x4 m2, m0, m3, q3131 ; 4
+ vshufi32x4 m0, m3, q2020 ; 0
+ vshufi32x4 m3, m1, m16, q3131 ; 6
+ vshufi32x4 m1, m16, q2020 ; 2
+ vshufi32x4 m16, m9, m15, q3131 ; 5
+ vshufi32x4 m14, m9, m15, q2020 ; 1
+ vshufi32x4 m15, m11, m17, q2020 ; 3
+ vshufi32x4 m17, m11, m17, q3131 ; 7
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main2
+ jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+
+cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 7, 0, dst, stride, c, eob
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ PROLOGUE 0, 9, 30, 64*32, dst, stride, c, eob
+ vpbroadcastd m23, [o(pw_2896x8)]
+%undef cmp
+ cmp eobd, 136
+ jb .fast
+ pmulhrsw m5, m23, [cq+64*20]
+ pmulhrsw m3, m23, [cq+64*12]
+ pmulhrsw m1, m23, [cq+64* 4]
+ pmulhrsw m7, m23, [cq+64*28]
+ pmulhrsw m2, m23, [cq+64* 8]
+ pmulhrsw m6, m23, [cq+64*24]
+ pmulhrsw m0, m23, [cq+64* 0]
+ pmulhrsw m4, m23, [cq+64*16]
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ pmulhrsw m14, m23, [cq+64* 2]
+ pmulhrsw m21, m23, [cq+64*30]
+ pmulhrsw m18, m23, [cq+64*18]
+ pmulhrsw m17, m23, [cq+64*14]
+ pmulhrsw m16, m23, [cq+64*10]
+ pmulhrsw m19, m23, [cq+64*22]
+ pmulhrsw m20, m23, [cq+64*26]
+ pmulhrsw m15, m23, [cq+64* 6]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ mova [cq+64* 0], m14
+ mova [cq+64* 2], m15
+ mova [cq+64* 4], m16
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64*12], m20
+ mova [cq+64*14], m21
+ pmulhrsw m22, m23, [cq+64* 1]
+ pmulhrsw m21, m23, [cq+64*31]
+ pmulhrsw m14, m23, [cq+64*17]
+ pmulhrsw m29, m23, [cq+64*15]
+ pmulhrsw m26, m23, [cq+64* 9]
+ pmulhrsw m17, m23, [cq+64*23]
+ pmulhrsw m18, m23, [cq+64*25]
+ pmulhrsw m25, m23, [cq+64* 7]
+ pmulhrsw m24, m23, [cq+64* 5]
+ pmulhrsw m19, m23, [cq+64*27]
+ pmulhrsw m16, m23, [cq+64*21]
+ pmulhrsw m27, m23, [cq+64*11]
+ pmulhrsw m28, m23, [cq+64*13]
+ pmulhrsw m15, m23, [cq+64*19]
+ pmulhrsw m20, m23, [cq+64*29]
+ pmulhrsw m23, [cq+64* 3]
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf
+ vpbroadcastd m12, [o(pw_16384)]
+ psubsw m13, m0, m29 ; 31
+ paddsw m0, m29 ; 0
+ psubsw m29, m1, m28 ; 30
+ paddsw m1, m28 ; 1
+ psubsw m28, m2, m27 ; 29
+ paddsw m2, m27 ; 2
+ psubsw m27, m3, m26 ; 28
+ paddsw m3, m26 ; 3
+ psubsw m26, m4, m25 ; 27
+ paddsw m4, m25 ; 4
+ psubsw m25, m5, m24 ; 26
+ paddsw m5, m24 ; 5
+ psubsw m24, m6, m23 ; 25
+ paddsw m6, m23 ; 6
+ psubsw m23, m7, m22 ; 24
+ paddsw m7, m22 ; 7
+ pxor m9, m9
+ punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7
+ punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7
+ punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3
+ REPX {mova [cq+64*x], m9}, 16, 17, 18, 19
+ punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7
+ punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3
+ punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7
+ punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3
+ REPX {mova [cq+64*x], m9}, 20, 21, 22, 23
+ punpckhwd m3, m23, m24
+ punpcklwd m23, m24
+ punpckhwd m24, m25, m26
+ punpcklwd m25, m26
+ REPX {mova [cq+64*x], m9}, 24, 25, 26, 27
+ punpckhwd m26, m27, m28
+ punpcklwd m27, m28
+ punpckhwd m28, m29, m13
+ punpcklwd m29, m13
+ REPX {mova [cq+64*x], m9}, 28, 29, 30, 31
+ punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1
+ REPX {pmulhrsw x, m12}, m7, m0, m2, m4
+ punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7
+ punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5
+ punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7
+ punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5
+ REPX {pmulhrsw x, m12}, m6, m8, m1, m22
+ punpckhdq m13, m23, m25
+ punpckldq m23, m25
+ punpckhdq m25, m27, m29
+ punpckldq m27, m29
+ REPX {pmulhrsw x, m12}, m13, m23, m25, m27
+ punpckhdq m9, m3, m24
+ punpckldq m3, m24
+ punpckhdq m24, m26, m28
+ punpckldq m26, m28
+ REPX {pmulhrsw x, m12}, m9, m3, m24, m26
+ punpckhqdq m5, m23, m27 ; d01 d09 d17 d25
+ punpcklqdq m23, m27 ; d00 d08 d16 d24
+ punpcklqdq m27, m13, m25 ; d02 d10 d18 d26
+ punpckhqdq m13, m25 ; d03 d11 d19 d27
+ punpcklqdq m25, m3, m26 ; d04 d12 d20 d28
+ punpckhqdq m3, m26 ; d05 d13 d21 d29
+ punpcklqdq m26, m9, m24 ; d06 d14 d22 d30
+ punpckhqdq m9, m24 ; d07 d15 d23 d31
+ mova [cq+64* 3], m23
+ mova [cq+64*13], m27
+ mova [cq+64* 7], m25
+ mova [cq+64*15], m26
+ punpckhqdq m24, m8, m22 ; a05 a13 a21 a29
+ punpcklqdq m8, m22 ; a04 a12 a20 a28
+ punpckhqdq m22, m0, m4 ; a01 a09 a17 a25
+ punpcklqdq m0, m4 ; a00 a08 a16 a24
+ punpckhqdq m23, m7, m2 ; a03 a11 a19 a27
+ punpcklqdq m7, m2 ; a02 a10 a18 a26
+ punpckhqdq m25, m6, m1 ; a07 a15 a23 a31
+ punpcklqdq m6, m1 ; a06 a14 a22 a30
+ mova [cq+64* 1], m0
+ mova [cq+64* 9], m7
+ mova [cq+64* 5], m8
+ mova [cq+64*11], m6
+ mova m2, [cq+64* 0]
+ mova m11, [cq+64* 2]
+ mova m8, [cq+64* 4]
+ mova m29, [cq+64* 6]
+ mova m27, [cq+64* 8]
+ mova m26, [cq+64*10]
+ mova m4, [cq+64*12]
+ mova m28, [cq+64*14]
+ psubsw m1, m2, m21 ; 23
+ paddsw m2, m21 ; 8
+ psubsw m21, m11, m20 ; 22
+ paddsw m11, m20 ; 9
+ psubsw m20, m8, m19 ; 21
+ paddsw m8, m19 ; 10
+ psubsw m19, m29, m18 ; 20
+ paddsw m29, m18 ; 11
+ psubsw m18, m27, m17 ; 19
+ paddsw m27, m17 ; 12
+ psubsw m17, m26, m16 ; 18
+ paddsw m26, m16 ; 13
+ psubsw m16, m4, m15 ; 17
+ paddsw m4, m15 ; 14
+ psubsw m15, m28, m14 ; 16
+ paddsw m28, m14 ; 15
+ punpcklwd m14, m15, m16
+ punpckhwd m15, m16
+ punpckhwd m16, m17, m18
+ punpcklwd m17, m18
+ punpckhwd m18, m19, m20
+ punpcklwd m19, m20
+ punpckhwd m20, m21, m1
+ punpcklwd m21, m1
+ punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7
+ punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3
+ punpckhwd m11, m8, m29 ; k4 l4 k5 l5 k6 l6 k7 l7
+ punpcklwd m8, m29 ; k0 l0 k1 l1 k2 l2 k3 l3
+ punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7
+ punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3
+ punpckhwd m26, m4, m28 ; o4 p4 o5 p5 o6 p6 o7 p7
+ punpcklwd m4, m28 ; o0 p0 o1 p1 o2 p2 o3 p3
+ punpckhdq m28, m2, m8 ; i2 j2 k2 l2 i3 j3 k3 l3
+ punpckldq m2, m8 ; i0 j0 k0 l0 i1 j1 k1 l1
+ punpckhdq m8, m27, m4 ; m2 n2 o2 p2 m3 n3 o3 p3
+ punpckldq m27, m4 ; m0 n0 o0 p0 m1 n1 o1 p1
+ REPX {pmulhrsw x, m12}, m28, m2, m8, m27
+ punpckhdq m4, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7
+ punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5
+ punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7
+ punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5
+ REPX {pmulhrsw x, m12}, m4, m1, m11, m29
+ punpckhdq m26, m19, m21
+ punpckldq m19, m21
+ punpckhdq m21, m15, m16
+ punpckldq m15, m16
+ REPX {pmulhrsw x, m12}, m26, m19, m21, m15
+ punpckhdq m16, m18, m20
+ punpckldq m18, m20
+ punpckhdq m20, m14, m17
+ punpckldq m14, m17
+ REPX {pmulhrsw x, m12}, m16, m18, m20, m14
+ punpckhqdq m17, m28, m8 ; b03 b11 b19 b27
+ punpcklqdq m28, m8 ; b02 b10 b18 b26
+ punpckhqdq m8, m2, m27 ; b01 b09 b17 b25
+ punpcklqdq m2, m27 ; b00 b08 b16 b24
+ punpcklqdq m27, m1, m29 ; b04 b12 b20 b28
+ punpckhqdq m1, m29 ; b05 b13 b21 b29
+ punpcklqdq m29, m4, m11 ; b06 b14 b22 b30
+ punpckhqdq m4, m11 ; b07 b15 b23 b31
+ mova [cq+64* 0], m2
+ mova [cq+64* 8], m28
+ mova [cq+64* 4], m27
+ mova [cq+64*10], m29
+ punpckhqdq m27, m20, m26 ; c03 c11 c19 c27
+ punpcklqdq m20, m26 ; c02 c10 c18 c26
+ punpckhqdq m26, m14, m19 ; c01 c09 c17 c25
+ punpcklqdq m14, m19 ; c00 c08 c16 c24
+ punpckhqdq m28, m15, m18 ; c05 c13 c21 c29
+ punpcklqdq m15, m18 ; c04 c12 c20 c28
+ punpckhqdq m29, m21, m16 ; c07 c15 c23 c31
+ punpcklqdq m21, m16 ; c06 c14 c22 c30
+ mova [cq+64* 2], m14
+ mova [cq+64*12], m20
+ mova [cq+64* 6], m15
+ mova [cq+64*14], m21
+ vshufi32x4 m14, m22, m8, q3232 ; a17 a25 b17 b25
+ vinserti32x8 m22, ym8, 1 ; a01 a09 b01 b09
+ vshufi32x4 m15, m23, m17, q3232 ; a19 a27 b19 b27
+ vinserti32x8 m23, ym17, 1 ; a03 a11 b03 b11
+ vshufi32x4 m16, m24, m1, q3232 ; a21 a29 b21 b29
+ vinserti32x8 m24, ym1, 1 ; a05 a13 b05 b13
+ vshufi32x4 m17, m25, m4, q3232 ; a23 a31 b23 b31
+ vinserti32x8 m25, ym4, 1 ; a07 a15 b07 b15
+ vinserti32x8 m19, m26, ym5, 1 ; c01 c09 d01 d09
+ vshufi32x4 m26, m5, q3232 ; c17 c25 d17 d25
+ vinserti32x8 m20, m27, ym13, 1 ; c03 c11 d03 d11
+ vshufi32x4 m27, m13, q3232 ; c19 c27 d19 d27
+ vinserti32x8 m21, m28, ym3, 1 ; c05 c13 d05 d13
+ vshufi32x4 m28, m3, q3232 ; c21 c29 d21 d29
+ vinserti32x8 m18, m29, ym9, 1 ; c07 c15 d07 d15
+ vshufi32x4 m29, m9, q3232 ; c23 c31 d23 d31
+ mov r4, rsp
+ vshufi32x4 m0, m22, m19, q2020 ; 1
+ vshufi32x4 m1, m17, m29, q3131 ; 31
+ vshufi32x4 m2, m14, m26, q2020 ; 17
+ vshufi32x4 m3, m25, m18, q3131 ; 15
+ call .main_part1
+ vshufi32x4 m0, m25, m18, q2020 ; 7
+ vshufi32x4 m1, m14, m26, q3131 ; 25
+ vshufi32x4 m2, m17, m29, q2020 ; 23
+ vshufi32x4 m3, m22, m19, q3131 ; 9
+ call .main_part1
+ vshufi32x4 m0, m24, m21, q2020 ; 5
+ vshufi32x4 m1, m15, m27, q3131 ; 27
+ vshufi32x4 m2, m16, m28, q2020 ; 21
+ vshufi32x4 m3, m23, m20, q3131 ; 11
+ call .main_part1
+ vshufi32x4 m0, m23, m20, q2020 ; 3
+ vshufi32x4 m1, m16, m28, q3131 ; 29
+ vshufi32x4 m2, m15, m27, q2020 ; 19
+ vshufi32x4 m3, m24, m21, q3131 ; 13
+ call .main_part1
+ call .main_part2
+ mova m0, [cq+64* 1] ; a0
+ mova m15, [cq+64* 0] ; b0
+ mova m3, [cq+64* 2] ; c0
+ mova m16, [cq+64* 3] ; d0
+ mova m14, [cq+64* 5] ; a4
+ mova m8, [cq+64* 4] ; b4
+ mova m17, [cq+64* 6] ; c4
+ mova m1, [cq+64* 7] ; d4
+ vshufi32x4 m2, m0, m15, q3232 ; a16 a24 b16 b24
+ vinserti32x8 m0, ym15, 1 ; a00 a08 b00 b08
+ vshufi32x4 m15, m3, m16, q3232 ; c16 c24 d16 d24
+ vinserti32x8 m3, ym16, 1 ; c00 c08 d00 d08
+ vshufi32x4 m16, m14, m8, q3232 ; a20 a28 b20 b28
+ vinserti32x8 m14, ym8, 1 ; a04 a12 b04 b12
+ vshufi32x4 m8, m17, m1, q3232 ; c20 c28 d20 d28
+ vinserti32x8 m17, ym1, 1 ; c04 c12 d04 d12
+ vshufi32x4 m1, m0, m3, q3131 ; 8
+ vshufi32x4 m0, m3, q2020 ; 0
+ vshufi32x4 m3, m2, m15, q3131 ; 24
+ vshufi32x4 m2, m15, q2020 ; 16
+ vshufi32x4 m15, m14, m17, q3131 ; 12
+ vshufi32x4 m14, m17, q2020 ; 4
+ vshufi32x4 m17, m16, m8, q3131 ; 28
+ vshufi32x4 m16, m8, q2020 ; 20
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+ mova m8, [cq+64* 8]
+ mova m9, [cq+64*12]
+ mova m11, [cq+64*10]
+ mova m12, [cq+64*14]
+ mova [cq+64* 0], m14
+ mova [cq+64* 2], m15
+ mova [cq+64* 4], m16
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64*12], m20
+ mova [cq+64*14], m21
+ mova m22, [cq+64* 9]
+ mova m27, [cq+64*13]
+ mova m23, [cq+64*11]
+ mova m24, [cq+64*15]
+ vshufi32x4 m26, m22, m8, q3232 ; a18 a26 b18 b26
+ vinserti32x8 m22, ym8, 1 ; a02 a10 b02 b10
+ vshufi32x4 m8, m9, m27, q3232 ; c18 c26 d18 d26
+ vinserti32x8 m9, ym27, 1 ; c02 c10 d02 d10
+ vshufi32x4 m27, m23, m11, q3232 ; a22 a30 b22 b30
+ vinserti32x8 m23, ym11, 1 ; a06 a14 b06 b14
+ vshufi32x4 m11, m12, m24, q3232 ; c22 c30 d22 d30
+ vinserti32x8 m12, ym24, 1 ; c06 c14 d06 d14
+ vshufi32x4 m28, m26, m8, q3131 ; 26
+ vshufi32x4 m26, m8, q2020 ; 18
+ vshufi32x4 m24, m22, m9, q3131 ; 10
+ vshufi32x4 m22, m9, q2020 ; 2
+ vshufi32x4 m29, m27, m11, q3131 ; 30
+ vshufi32x4 m27, m11, q2020 ; 22
+ vshufi32x4 m25, m23, m12, q3131 ; 14
+ vshufi32x4 m23, m12, q2020 ; 6
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+ jmp .end
+.fast: ; bottom/right halves are zero
+ pmulhrsw ym9, ym23, [cq+64* 0]
+ pmulhrsw ym6, ym23, [cq+64* 8]
+ mova m14, [o(dup16_perm)]
+ pmulhrsw ym8, ym23, [cq+64* 2]
+ pmulhrsw xm0, xm23, [cq+64*14]
+ pmulhrsw xm5, xm23, [cq+64*10]
+ pmulhrsw ym1, ym23, [cq+64* 6]
+ pmulhrsw ym7, ym23, [cq+64* 4]
+ pmulhrsw xm3, xm23, [cq+64*12]
+ pmovzxwd m9, ym9
+ pmovzxwd m6, ym6
+ vpermb m8, m14, m8
+ punpcklwd xm0, xm0
+ vpermb ym5, ym14, ym5
+ vpermb m1, m14, m1
+ vpermb m7, m14, m7
+ punpcklwd xm3, xm3
+ pslld m9, 16
+ pslld m6, 16
+ call m(idct_16x16_internal_8bpc).main_fast
+ vpmulhrsw ym21, ym23, [cq+64* 1]
+ {evex}vpmulhrsw xm17, xm23, [cq+64*15] ; force EVEX encoding, which
+ {evex}vpmulhrsw xm20, xm23, [cq+64* 9] ; reduces code size due to
+ {evex}vpmulhrsw ym15, ym23, [cq+64* 7] ; compressed displacements
+ {evex}vpmulhrsw ym18, ym23, [cq+64* 5]
+ {evex}vpmulhrsw xm16, xm23, [cq+64*11]
+ {evex}vpmulhrsw xm19, xm23, [cq+64*13]
+ {evex}vpmulhrsw ym23, [cq+64* 3]
+ vpermb m21, m14, m21
+ punpcklwd xm17, xm17
+ vpermb ym20, ym14, ym20
+ vpermb m15, m14, m15
+ vpermb m18, m14, m18
+ vpermb ym16, ym14, ym16
+ punpcklwd xm19, xm19
+ vpermb m14, m14, m23
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast
+ vpbroadcastd m9, [o(pw_16384)]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).transpose_round
+ vshufi32x4 m16, m0, m3, q2020 ; 0
+ vshufi32x4 m26, m0, m3, q3131 ; 4
+ vshufi32x4 m0, m14, m2, q2020 ; 1
+ vshufi32x4 m14, m2, q3131 ; 5
+ vshufi32x4 m3, m19, m7, q3131 ; 15
+ vshufi32x4 m19, m7, q2020 ; 11
+ vshufi32x4 m27, m17, m9, q2020 ; 3
+ vshufi32x4 m17, m9, q3131 ; 7
+ vshufi32x4 m28, m20, m6, q2020 ; 9
+ vshufi32x4 m20, m6, q3131 ; 13
+ vshufi32x4 m22, m1, m18, q2020 ; 2
+ vshufi32x4 m23, m1, m18, q3131 ; 6
+ vshufi32x4 m24, m5, m15, q2020 ; 10
+ vshufi32x4 m25, m5, m15, q3131 ; 14
+ vshufi32x4 m15, m21, m4, q3131 ; 12
+ vshufi32x4 m21, m21, m4, q2020 ; 8
+ mov r4, rsp
+ call .main_part1_fast
+ mova m0, m17
+ mova m3, m28
+ call .main_part1_fast
+ mova m0, m14
+ mova m3, m19
+ call .main_part1_fast
+ mova m0, m27
+ mova m3, m20
+ call .main_part1_fast
+ call .main_part2
+ mova m0, m16
+ mova m1, m21
+ mova m14, m26
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2
+ mova [cq+64*14], m21
+ mova [cq+64* 0], m14
+ mova [cq+64* 6], m17
+ mova [cq+64* 8], m18
+ mova [cq+64*10], m19
+ mova [cq+64* 4], m16
+ mova [cq+64* 2], m15
+ mova [cq+64*12], m20
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2
+.end:
+ lea r4, [strideq*3]
+ vpbroadcastd m12, [o(pw_2048)]
+ movshdup m13, [o(permD)]
+ lea r5, [r4+strideq] ; stride*4
+ lea r3, [dstq+r4*8]
+ lea r6, [strideq+r5*8] ; stride*33
+ lea r8, [r4+r5*8] ; stride*35
+ add r3, r5 ; dst+stride*28
+ lea r7, [r6+strideq] ; stride*34
+%macro IDCT_32x64_END 6 ; src, mem, stride[1-4]
+%if %2 < 8
+ paddsw m10, m%2, m%1
+ psubsw m11, m%2, m%1
+%else
+ mova m11, [cq+64*(%2*2-16)]
+ paddsw m10, m11, m%1
+ psubsw m11, m%1
+%endif
+ mova m9, [rsp+64*(31-%2)]
+ mova m%1, [rsp+64*%2]
+ paddsw m8, m10, m9
+ psubsw m10, m9
+ paddsw m9, m11, m%1
+ pmovzxbw m0, [dstq+%3]
+ psubsw m11, m%1
+ pmovzxbw m%1, [r3 +%4]
+ REPX {pmulhrsw x, m12}, m8, m10, m9, m11
+ paddw m8, m0
+ pmovzxbw m0, [r3 +%5]
+ paddw m10, m%1
+ pmovzxbw m%1, [dstq+%6]
+ paddw m9, m0
+ paddw m11, m%1
+%if %2 >= 8
+%if %2 == 8
+ pxor m1, m1
+%endif
+ mova [cq+64*(%2*2-16)], m1
+ mova [cq+64*(%2*2-15)], m1
+%endif
+ packuswb m8, m10
+ packuswb m9, m11
+ vpermq m8, m13, m8
+ vpermq m9, m13, m9
+ mova [dstq+%3], ym8
+ vextracti32x8 [r3 +%4], m8, 1
+ mova [r3 +%5], ym9
+ vextracti32x8 [dstq+%6], m9, 1
+%if %2 == 3 || %2 == 7 || %2 == 11
+ add dstq, r5
+ sub r3, r5
+%endif
+%endmacro
+ IDCT_32x64_END 29, 0, strideq*0, r8, r4 , r5*8
+ IDCT_32x64_END 28, 1, strideq*1, r7, strideq*2, r6
+ IDCT_32x64_END 27, 2, strideq*2, r6, strideq*1, r7
+ IDCT_32x64_END 26, 3, r4 , r5*8, strideq*0, r8
+ IDCT_32x64_END 25, 4, strideq*0, r8, r4 , r5*8
+ IDCT_32x64_END 24, 5, strideq*1, r7, strideq*2, r6
+ IDCT_32x64_END 23, 6, strideq*2, r6, strideq*1, r7
+ IDCT_32x64_END 22, 7, r4 , r5*8, strideq*0, r8
+ IDCT_32x64_END 21, 8, strideq*0, r8, r4 , r5*8
+ IDCT_32x64_END 20, 9, strideq*1, r7, strideq*2, r6
+ IDCT_32x64_END 19, 10, strideq*2, r6, strideq*1, r7
+ IDCT_32x64_END 18, 11, r4 , r5*8, strideq*0, r8
+ IDCT_32x64_END 17, 12, strideq*0, r8, r4 , r5*8
+ IDCT_32x64_END 16, 13, strideq*1, r7, strideq*2, r6
+ IDCT_32x64_END 15, 14, strideq*2, r6, strideq*1, r7
+ IDCT_32x64_END 14, 15, r4 , r5*8, strideq*0, r8
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 64
+ imul r6d, 181
+ add r6d, 128
+ sar r6d, 8
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+ jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3
+ALIGN function_align ; bottom three-quarters are zero
+.main_part1_fast:
+ vpbroadcastd m1, [o(idct64_mul+4*0)]
+ vpbroadcastd m8, [o(idct64_mul+4*1)]
+ vpbroadcastd m2, [o(idct64_mul+4*6)]
+ vpbroadcastd m9, [o(idct64_mul+4*7)]
+ pmulhrsw m1, m0 ; t63a
+ pmulhrsw m0, m8 ; t32a
+ pmulhrsw m2, m3 ; t60a
+ pmulhrsw m3, m9 ; t35a
+ mova m8, m0
+ mova m7, m1
+ mova m6, m3
+ mova m5, m2
+ jmp .main_part1b
+.main_part1:
+ ; idct64 steps 1-5:
+ ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a
+ ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a
+ ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a
+ ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a
+ vpbroadcastd m7, [o(idct64_mul+4*0)]
+ vpbroadcastd m8, [o(idct64_mul+4*1)]
+ vpbroadcastd m6, [o(idct64_mul+4*2)]
+ vpbroadcastd m9, [o(idct64_mul+4*3)]
+ pmulhrsw m7, m0 ; t63a
+ vpbroadcastd m5, [o(idct64_mul+4*4)]
+ pmulhrsw m0, m8 ; t32a
+ vpbroadcastd m8, [o(idct64_mul+4*5)]
+ pmulhrsw m6, m1 ; t62a
+ vpbroadcastd m4, [o(idct64_mul+4*6)]
+ pmulhrsw m1, m9 ; t33a
+ vpbroadcastd m9, [o(idct64_mul+4*7)]
+ pmulhrsw m5, m2 ; t61a
+ pmulhrsw m2, m8 ; t34a
+ pmulhrsw m4, m3 ; t60a
+ pmulhrsw m3, m9 ; t35a
+ psubsw m8, m0, m1 ; t33
+ paddsw m0, m1 ; t32
+ psubsw m1, m7, m6 ; t62
+ paddsw m7, m6 ; t63
+ psubsw m6, m3, m2 ; t34
+ paddsw m3, m2 ; t35
+ psubsw m2, m4, m5 ; t61
+ paddsw m5, m4 ; t60
+.main_part1b:
+ vpbroadcastd m11, [o(idct64_mul+4*8)]
+ vpbroadcastd m12, [o(idct64_mul+4*9)]
+ ITX_MULSUB_2W 1, 8, 4, 9, 10, 11, 12 ; t33a, t62a
+ vpbroadcastd m11, [o(idct64_mul+4*10)]
+ ITX_MULSUB_2W 2, 6, 4, 9, 10, 12, 11 ; t34a, t61a
+ vpbroadcastd m11, [o(idct64_mul+4*11)]
+ vpbroadcastd m12, [o(idct64_mul+4*12)]
+ psubsw m4, m0, m3 ; t35a
+ paddsw m0, m3 ; t32a
+ psubsw m3, m7, m5 ; t60a
+ paddsw m7, m5 ; t63a
+ psubsw m5, m1, m2 ; t34
+ paddsw m1, m2 ; t33
+ psubsw m2, m8, m6 ; t61
+ paddsw m6, m8 ; t62
+ add r5, 4*13
+ ITX_MULSUB_2W 3, 4, 8, 9, 10, 11, 12 ; t35, t60
+ ITX_MULSUB_2W 2, 5, 8, 9, 10, 11, 12 ; t34a, t61a
+ mova [r4+64*0], m0
+ mova [r4+64*7], m7
+ mova [r4+64*1], m1
+ mova [r4+64*6], m6
+ mova [r4+64*3], m3
+ mova [r4+64*4], m4
+ mova [r4+64*2], m2
+ mova [r4+64*5], m5
+ add r4, 64*8
+ ret
+.main_part2:
+ vpbroadcastd m11, [o(pw_1567_3784 -16*13)]
+ vpbroadcastd m12, [o(pw_m3784_1567 -16*13)]
+ lea r6, [r4+64*7]
+ vpbroadcastd m17, [o(pw_m1567_m3784-16*13)]
+ vpbroadcastd m18, [o(pw_2896_2896 -16*13)]
+ vpbroadcastd m19, [o(pw_m2896_2896 -16*13)]
+ sub r5, 16*13
+.main_part2_loop:
+ mova m0, [r4-64*32] ; t32a
+ mova m1, [r6-64*24] ; t39a
+ mova m2, [r6-64*32] ; t63a
+ mova m3, [r4-64*24] ; t56a
+ mova m4, [r4-64*16] ; t40a
+ mova m5, [r6-64* 8] ; t47a
+ mova m6, [r6-64*16] ; t55a
+ mova m7, [r4-64* 8] ; t48a
+ psubsw m8, m0, m1 ; t39
+ paddsw m0, m1 ; t32
+ psubsw m1, m2, m3 ; t56
+ paddsw m2, m3 ; t63
+ psubsw m3, m5, m4 ; t40
+ paddsw m5, m4 ; t47
+ psubsw m4, m7, m6 ; t55
+ paddsw m7, m6 ; t48
+ ITX_MULSUB_2W 1, 8, 6, 9, 10, 11, 12 ; t39a, t56a
+ ITX_MULSUB_2W 4, 3, 6, 9, 10, 12, 17 ; t40a, t55a
+ psubsw m6, m2, m7 ; t48a
+ paddsw m2, m7 ; t63a
+ psubsw m7, m0, m5 ; t47a
+ paddsw m0, m5 ; t32a
+ psubsw m5, m8, m3 ; t55
+ paddsw m8, m3 ; t56
+ psubsw m3, m1, m4 ; t40
+ paddsw m1, m4 ; t39
+ ITX_MULSUB_2W 6, 7, 4, 9, 10, 18, 19 ; t47, t48
+ ITX_MULSUB_2W 5, 3, 4, 9, 10, 18, 19 ; t40a, t55a
+ mova [r6-64* 8], m2
+ mova [r4-64*32], m0
+ mova [r4-64* 8], m8
+ mova [r6-64*32], m1
+ mova [r6-64*24], m6
+ mova [r4-64*16], m7
+ mova [r4-64*24], m5
+ mova [r6-64*16], m3
+ add r4, 64
+ sub r6, 64
+ cmp r4, r6
+ jb .main_part2_loop
+ ret
+
+cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 7, 0, dst, stride, c, eob
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ PROLOGUE 0, 7, 30, 64*32, dst, stride, c, eob
+ vpbroadcastd m23, [o(pw_2896x8)]
+%undef cmp
+ cmp eobd, 136
+ jb .fast
+ pmulhrsw m0, m23, [cq+64* 1]
+ pmulhrsw m1, m23, [cq+64*31]
+ pmulhrsw m2, m23, [cq+64*17]
+ pmulhrsw m3, m23, [cq+64*15]
+ vpbroadcastd m10, [o(pd_2048)]
+ mov r4, rsp
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ pmulhrsw m0, m23, [cq+64* 7]
+ pmulhrsw m1, m23, [cq+64*25]
+ pmulhrsw m2, m23, [cq+64*23]
+ pmulhrsw m3, m23, [cq+64* 9]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ pmulhrsw m0, m23, [cq+64* 5]
+ pmulhrsw m1, m23, [cq+64*27]
+ pmulhrsw m2, m23, [cq+64*21]
+ pmulhrsw m3, m23, [cq+64*11]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ pmulhrsw m0, m23, [cq+64* 3]
+ pmulhrsw m1, m23, [cq+64*29]
+ pmulhrsw m2, m23, [cq+64*19]
+ pmulhrsw m3, m23, [cq+64*13]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2
+ pmulhrsw m3, m23, [cq+64*24]
+ pmulhrsw m1, m23, [cq+64* 8]
+ pmulhrsw m2, m23, [cq+64*16]
+ pmulhrsw m0, m23, [cq+64* 0]
+ pmulhrsw m14, m23, [cq+64* 4]
+ pmulhrsw m17, m23, [cq+64*28]
+ pmulhrsw m16, m23, [cq+64*20]
+ pmulhrsw m15, m23, [cq+64*12]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+ pmulhrsw m22, m23, [cq+64* 2]
+ pmulhrsw m29, m23, [cq+64*30]
+ pmulhrsw m26, m23, [cq+64*18]
+ pmulhrsw m25, m23, [cq+64*14]
+ pmulhrsw m24, m23, [cq+64*10]
+ pmulhrsw m27, m23, [cq+64*22]
+ pmulhrsw m28, m23, [cq+64*26]
+ pmulhrsw m23, [cq+64* 6]
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+ vpbroadcastd m13, [o(pw_16384)]
+ call .pass1_end_part1
+ mova [cq+64*16], m1
+ mova [cq+64*17], m3
+ mova [cq+64*18], m5
+ mova [cq+64*19], m7
+ mova [cq+64*24], m23
+ mova [cq+64*25], m25
+ mova [cq+64*26], m27
+ mova [cq+64*27], m29
+ pmulhrsw m23, m13, m0 ; a0
+ pmulhrsw m25, m13, m2 ; a2
+ pmulhrsw m27, m13, m4 ; a4
+ pmulhrsw m29, m13, m6 ; a6
+ REPX {pmulhrsw x, m13}, m22, m24, m26, m28 ; e0 e2 e4 e6
+ call .pass1_end_part2
+ mova [cq+64*20], m15
+ mova [cq+64*21], m17
+ mova [cq+64*22], m19
+ mova [cq+64*23], m21
+ mova [cq+64*28], m1
+ mova [cq+64*29], m3
+ mova [cq+64*30], m5
+ mova [cq+64*31], m7
+ REPX {pmulhrsw x, m13}, m14, m16, m18, m20 ; c0 c2 c4 c6
+ REPX {pmulhrsw x, m13}, m0, m2, m4, m6 ; g0 g2 g4 g6
+ vinserti32x8 m3, m23, ym14, 1 ; a00 a01 c00 c01
+ vshufi32x4 m23, m14, q3232 ; a02 a03 c02 c03
+ vinserti32x8 m15, m22, ym0, 1 ; e00 e01 g00 g01
+ vshufi32x4 m22, m0, q3232 ; e02 e03 g02 g03
+ vinserti32x8 m1, m27, ym18, 1 ; a40 a41 c40 c41
+ vshufi32x4 m27, m18, q3232 ; a42 a43 c42 c43
+ vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41
+ vshufi32x4 m26, m4, q3232 ; e42 e43 g42 g43
+ vinserti32x8 m14, m25, ym16, 1 ; a20 a21 c20 c21
+ vshufi32x4 m25, m16, q3232 ; a22 a23 c22 c23
+ vinserti32x8 m17, m24, ym2, 1 ; e20 e21 g20 g21
+ vshufi32x4 m24, m2, q3232 ; e22 e23 g22 g23
+ vinserti32x8 m19, m29, ym20, 1 ; a60 a61 c60 c61
+ vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63
+ vinserti32x8 m20, m28, ym6, 1 ; e60 e61 g60 g61
+ vshufi32x4 m28, m6, q3232 ; e62 e63 g62 g63
+ vshufi32x4 m2, m3, m15, q3131 ; 8
+ vshufi32x4 m0, m3, m15, q2020 ; 0
+ vshufi32x4 m6, m23, m22, q3131 ; 24
+ vshufi32x4 m4, m23, m22, q2020 ; 16
+ vshufi32x4 m3, m1, m18, q3131 ; 12
+ vshufi32x4 m1, m18, q2020 ; 4
+ vshufi32x4 m7, m27, m26, q3131 ; 28
+ vshufi32x4 m5, m27, m26, q2020 ; 20
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ vshufi32x4 m16, m14, m17, q3131 ; 10
+ vshufi32x4 m14, m17, q2020 ; 2
+ vshufi32x4 m17, m19, m20, q3131 ; 14
+ vshufi32x4 m15, m19, m20, q2020 ; 6
+ vshufi32x4 m20, m25, m24, q3131 ; 26
+ vshufi32x4 m18, m25, m24, q2020 ; 18
+ vshufi32x4 m21, m29, m28, q3131 ; 30
+ vshufi32x4 m19, m29, m28, q2020 ; 22
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ pmulhrsw m22, m13, [cq+64*16] ; a1
+ pmulhrsw m23, m13, [cq+64*20] ; c1
+ pmulhrsw m24, m13, [cq+64*24] ; e1
+ pmulhrsw m25, m13, [cq+64*28] ; g1
+ pmulhrsw m26, m13, [cq+64*17] ; a3
+ pmulhrsw m27, m13, [cq+64*21] ; c3
+ pmulhrsw m28, m13, [cq+64*25] ; e3
+ pmulhrsw m29, m13, [cq+64*29] ; g3
+ mova [cq+64* 8], m14
+ mova [cq+64* 9], m15
+ mova [cq+64*10], m16
+ mova [cq+64*11], m17
+ mova [cq+64*12], m18
+ mova [cq+64*13], m19
+ mova [cq+64*14], m20
+ mova [cq+64*15], m21
+ pmulhrsw m14, m13, [cq+64*18] ; a5
+ pmulhrsw m15, m13, [cq+64*22] ; c5
+ pmulhrsw m16, m13, [cq+64*26] ; e5
+ pmulhrsw m17, m13, [cq+64*30] ; g5
+ pmulhrsw m18, m13, [cq+64*19] ; a7
+ pmulhrsw m19, m13, [cq+64*23] ; c7
+ pmulhrsw m20, m13, [cq+64*27] ; e7
+ pmulhrsw m21, m13, [cq+64*31] ; g7
+ vinserti32x8 m8, m22, ym23, 1 ; a10 a11 c10 c11
+ vshufi32x4 m22, m23, q3232 ; a12 a13 c12 c13
+ vinserti32x8 m9, m24, ym25, 1 ; e10 e11 g10 g11
+ vshufi32x4 m24, m25, q3232 ; e12 e13 g12 g13
+ vinserti32x8 m23, m26, ym27, 1 ; a30 a31 c30 c31
+ vshufi32x4 m26, m27, q3232 ; a32 a33 c32 c33
+ vinserti32x8 m11, m28, ym29, 1 ; e30 e31 g30 g31
+ vshufi32x4 m28, m29, q3232 ; e32 e33 g32 g33
+ mova [cq+64* 0], m0
+ mova [cq+64* 1], m1
+ mova [cq+64* 2], m2
+ mova [cq+64* 3], m3
+ mova [cq+64* 4], m4
+ mova [cq+64* 5], m5
+ mova [cq+64* 6], m6
+ mova [cq+64* 7], m7
+ vinserti32x8 m12, m14, ym15, 1 ; a50 a51 c50 c51
+ vshufi32x4 m14, m15, q3232 ; a52 a53 c52 c53
+ vinserti32x8 m13, m16, ym17, 1 ; e50 e51 g50 g51
+ vshufi32x4 m16, m17, q3232 ; e52 e53 g52 g53
+ vinserti32x8 m25, m18, ym19, 1 ; a70 a71 c70 c71
+ vshufi32x4 m18, m19, q3232 ; a72 a73 c72 c73
+ vinserti32x8 m17, m20, ym21, 1 ; e70 e71 g70 g71
+ vshufi32x4 m20, m21, q3232 ; e72 e73 g72 g73
+ vshufi32x4 m27, m23, m11, q3131 ; 11 m27
+ vshufi32x4 m23, m11, q2020 ; 3 m23
+ vshufi32x4 m19, m26, m28, q3131 ; 27 m19
+ vshufi32x4 m15, m26, m28, q2020 ; 19 m15
+ vshufi32x4 m29, m25, m17, q3131 ; 15 m29
+ vshufi32x4 m25, m17, q2020 ; 7 m25
+ vshufi32x4 m21, m18, m20, q3131 ; 31 m21
+ vshufi32x4 m17, m18, m20, q2020 ; 23 m17
+ vshufi32x4 m20, m14, m16, q3131 ; 29 m20
+ vshufi32x4 m16, m14, m16, q2020 ; 21 m16
+ vshufi32x4 m18, m22, m24, q3131 ; 25 m18
+ vshufi32x4 m14, m22, m24, q2020 ; 17 m14
+ vshufi32x4 m26, m8, m9, q3131 ; 9 m26
+ vshufi32x4 m22, m8, m9, q2020 ; 1 m22
+ vshufi32x4 m28, m12, m13, q3131 ; 13 m28
+ vshufi32x4 m24, m12, m13, q2020 ; 5 m24
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf
+ vpbroadcastd m13, [o(pw_16384)]
+ pmulhrsw m0, m13, [r4-64*21]
+ pmulhrsw m1, m13, [r4-64*22]
+ pmulhrsw m2, m13, [r4-64*23]
+ pmulhrsw m3, m13, [r4-64*24]
+ pmulhrsw m4, m13, [r4-64*25]
+ pmulhrsw m5, m13, [r4-64*26]
+ pmulhrsw m6, m13, [r4-64*27]
+ pmulhrsw m7, m13, [r4-64*28]
+ mova [cq+64*16], m14
+ mova [cq+64*17], m15
+ mova [cq+64*18], m16
+ mova [cq+64*19], m17
+ mova [cq+64*20], m18
+ mova [cq+64*21], m19
+ mova [cq+64*22], m20
+ mova [cq+64*23], m21
+ pmulhrsw m14, m13, [r4-64*12]
+ pmulhrsw m15, m13, [r4-64*11]
+ pmulhrsw m16, m13, [r4-64*10]
+ pmulhrsw m17, m13, [r4-64* 9]
+ pmulhrsw m18, m13, [r4-64* 8]
+ pmulhrsw m19, m13, [r4-64* 7]
+ pmulhrsw m20, m13, [r4-64* 6]
+ pmulhrsw m21, m13, [r4-64* 5]
+ mova [cq+64*24], m22
+ mova [cq+64*25], m23
+ mova [cq+64*26], m24
+ mova [cq+64*27], m25
+ mova [cq+64*28], m26
+ mova [cq+64*29], m27
+ mova [cq+64*30], m28
+ mova [cq+64*31], m29
+ call .transpose_2x8x8_lo
+ mova [r4-64*12], m1
+ mova [r4-64*11], m3
+ mova [r4-64*10], m5
+ mova [r4-64* 9], m7
+ mova [r4-64* 8], m15
+ mova [r4-64* 7], m17
+ mova [r4-64* 6], m19
+ mova [r4-64* 5], m21
+ vinserti32x8 m22, m0, ym14, 1 ; f00 f01 h00 h01
+ vshufi32x4 m23, m0, m14, q3232 ; f02 f03 h02 h03
+ vinserti32x8 m24, m2, ym16, 1 ; f20 f21 h20 h21
+ vshufi32x4 m25, m2, m16, q3232 ; f22 f23 h22 h23
+ vinserti32x8 m26, m4, ym18, 1 ; f40 f41 h40 h41
+ vshufi32x4 m27, m4, m18, q3232 ; f42 f43 h42 h43
+ vinserti32x8 m28, m6, ym20, 1 ; f60 f61 h60 h61
+ vshufi32x4 m29, m6, m20, q3232 ; f62 f63 h62 h63
+ pmulhrsw m0, m13, [r4-64*20]
+ pmulhrsw m1, m13, [r4-64*19]
+ pmulhrsw m2, m13, [r4-64*18]
+ pmulhrsw m3, m13, [r4-64*17]
+ pmulhrsw m4, m13, [r4-64*16]
+ pmulhrsw m5, m13, [r4-64*15]
+ pmulhrsw m6, m13, [r4-64*14]
+ pmulhrsw m7, m13, [r4-64*13]
+ pmulhrsw m14, m13, [r4-64*29]
+ pmulhrsw m15, m13, [r4-64*30]
+ pmulhrsw m16, m13, [r4-64*31]
+ pmulhrsw m17, m13, [r4-64*32]
+ pmulhrsw m18, m13, [r4-64*33]
+ pmulhrsw m19, m13, [r4-64*34]
+ pmulhrsw m20, m13, [r4-64*35]
+ pmulhrsw m21, m13, [r4-64*36]
+ call .transpose_2x8x8_lo
+ mova [r4-64*20], m1
+ mova [r4-64*19], m3
+ mova [r4-64*18], m5
+ mova [r4-64*17], m7
+ mova [r4-64*16], m15
+ mova [r4-64*15], m17
+ mova [r4-64*14], m19
+ mova [r4-64*13], m21
+ vinserti32x8 m1, m4, ym18, 1 ; b40 b41 d40 d41
+ vshufi32x4 m5, m4, m18, q3232 ; b42 b43 d42 d43
+ vshufi32x4 m4, m0, m14, q3232 ; b02 b03 d02 d03
+ vinserti32x8 m0, ym14, 1 ; b00 b01 d00 d01
+ vinserti32x8 m14, m2, ym16, 1 ; b20 b21 d20 d21
+ vshufi32x4 m18, m2, m16, q3232 ; b22 b23 d22 d23
+ vinserti32x8 m15, m6, ym20, 1 ; b60 b61 d60 d61
+ vshufi32x4 m19, m6, m20, q3232 ; b62 b63 d62 d63
+ vshufi32x4 m2, m0, m22, q3131 ; 8
+ vshufi32x4 m0, m22, q2020 ; 0
+ vshufi32x4 m3, m1, m26, q3131 ; 12
+ vshufi32x4 m1, m26, q2020 ; 4
+ vshufi32x4 m6, m4, m23, q3131 ; 24
+ vshufi32x4 m4, m23, q2020 ; 16
+ vshufi32x4 m7, m5, m27, q3131 ; 28
+ vshufi32x4 m5, m27, q2020 ; 20
+ call m(inv_txfm_add_dct_dct_32x8_8bpc).main
+ vshufi32x4 m16, m14, m24, q3131 ; 10
+ vshufi32x4 m14, m24, q2020 ; 2
+ vshufi32x4 m17, m15, m28, q3131 ; 14
+ vshufi32x4 m15, m28, q2020 ; 6
+ vshufi32x4 m20, m18, m25, q3131 ; 26
+ vshufi32x4 m18, m25, q2020 ; 18
+ vshufi32x4 m21, m19, m29, q3131 ; 30
+ vshufi32x4 m19, m29, q2020 ; 22
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf
+ mova m22, [r4-64*20]
+ mova m26, [r4-64*16]
+ mova m23, [r4-64*19]
+ mova m27, [r4-64*15]
+ mova m24, [r4-64*18]
+ mova m28, [r4-64*14]
+ mova m25, [r4-64*17]
+ mova m29, [r4-64*13]
+ mova [r4-64*20], m14
+ mova [r4-64*19], m15
+ mova [r4-64*18], m16
+ mova [r4-64*17], m17
+ mova [r4-64*16], m18
+ mova [r4-64*15], m19
+ mova [r4-64*14], m20
+ mova [r4-64*13], m21
+ mova m19, [r4-64*12]
+ mova m11, [r4-64* 8]
+ mova m20, [r4-64*11]
+ mova m12, [r4-64* 7]
+ mova m21, [r4-64*10]
+ mova m8, [r4-64* 6]
+ mova m9, [r4-64* 9]
+ mova m18, [r4-64* 5]
+ vshufi32x4 m14, m22, m26, q3232 ; b12 b13 d12 d13
+ vinserti32x8 m22, ym26, 1 ; b10 b11 d10 d11
+ vshufi32x4 m15, m23, m27, q3232 ; b32 b33 d32 d33
+ vinserti32x8 m23, ym27, 1 ; b30 b31 d30 d31
+ vshufi32x4 m16, m24, m28, q3232 ; b52 b53 d52 d53
+ vinserti32x8 m24, ym28, 1 ; b50 b51 d50 d51
+ vshufi32x4 m17, m25, m29, q3232 ; b72 b73 d72 d73
+ vinserti32x8 m25, ym29, 1 ; b70 b71 d70 d71
+ vinserti32x8 m27, m19, ym11, 1 ; f10 f11 h10 h11
+ vshufi32x4 m19, m11, q3232 ; f12 f13 h12 h13
+ vinserti32x8 m28, m20, ym12, 1 ; f30 f31 h30 h31
+ vshufi32x4 m20, m12, q3232 ; f32 f33 h32 h33
+ vinserti32x8 m29, m21, ym8, 1 ; f50 f51 h50 h51
+ vshufi32x4 m21, m8, q3232 ; f52 f53 h52 h53
+ vinserti32x8 m8, m9, ym18, 1 ; f70 f71 h70 h71
+ vshufi32x4 m9, m18, q3232 ; f72 f73 h72 h73
+ vshufi32x4 m26, m22, m27, q3131 ; 9
+ vshufi32x4 m22, m27, q2020 ; 1
+ vshufi32x4 m27, m23, m28, q3131 ; 11
+ vshufi32x4 m23, m28, q2020 ; 3
+ vshufi32x4 m28, m24, m29, q3131 ; 13
+ vshufi32x4 m24, m29, q2020 ; 5
+ vshufi32x4 m29, m25, m8, q3131 ; 15
+ vshufi32x4 m25, m8, q2020 ; 7
+ vshufi32x4 m18, m14, m19, q3131 ; 25
+ vshufi32x4 m14, m19, q2020 ; 17
+ vshufi32x4 m19, m15, m20, q3131 ; 27
+ vshufi32x4 m15, m20, q2020 ; 19
+ vshufi32x4 m20, m16, m21, q3131 ; 29
+ vshufi32x4 m16, m21, q2020 ; 21
+ vshufi32x4 m21, m17, m9, q3131 ; 31
+ vshufi32x4 m17, m9, q2020 ; 23
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf
+ jmp .end
+.fast: ; bottom/right halves are zero
+ {evex}vpmulhrsw ym8, ym23, [cq+64* 4]
+ {evex}vpmulhrsw xm1, xm23, [cq+64*12]
+ mova m28, [o(dup16_perm)]
+ {evex}vpmulhrsw ym7, ym23, [cq+64* 8]
+ vpmulhrsw ym22, ym23, [cq+64* 0]
+ vpermb m8, m28, m8
+ vpermb ym1, ym28, ym1
+ vpermb m7, m28, m7
+ pmovzxwd m9, ym22
+ pslld m9, 16
+ call m(idct_16x16_internal_8bpc).main_fast2
+ {evex}vpmulhrsw ym21, ym23, [cq+64* 2]
+ {evex}vpmulhrsw xm15, xm23, [cq+64*14]
+ {evex}vpmulhrsw xm18, xm23, [cq+64*10]
+ {evex}vpmulhrsw ym14, ym23, [cq+64* 6]
+ vpermb m21, m28, m21
+ punpcklwd xm15, xm15
+ vpermb ym18, ym28, ym18
+ vpermb m14, m28, m14
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2
+ vpmulhrsw ym22, ym23, [cq+64* 1]
+ {evex}vpmulhrsw xm29, xm23, [cq+64*15]
+ {evex}vpmulhrsw xm26, xm23, [cq+64* 9]
+ {evex}vpmulhrsw ym25, ym23, [cq+64* 7]
+ {evex}vpmulhrsw ym24, ym23, [cq+64* 5]
+ {evex}vpmulhrsw xm27, xm23, [cq+64*11]
+ {evex}vpmulhrsw xm8, xm23, [cq+64*13]
+ {evex}vpmulhrsw ym23, [cq+64* 3]
+ vpermb m22, m28, m22
+ punpcklwd xm29, xm29
+ vpermb ym26, ym28, ym26
+ vpermb m25, m28, m25
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ REPX {vpermb x, m28, x}, m24, m27, m23
+ punpcklwd xm28, xm8, xm8
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast
+ mov r4, rsp
+ vpbroadcastd m13, [o(pw_16384)]
+ mova [r4+64*16], m4
+ mova [r4+64*17], m5
+ mova [r4+64*18], m6
+ mova [r4+64*19], m7
+ mova [r4+64*28], m26
+ mova [r4+64*29], m27
+ mova [r4+64*30], m28
+ mova [r4+64*31], m29
+ call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end
+ mova [r4+64*20], m22
+ mova [r4+64*21], m23
+ mova [r4+64*22], m24
+ mova [r4+64*23], m25
+ mova [r4+64*24], m26
+ mova [r4+64*25], m27
+ mova [r4+64*26], m28
+ mova [r4+64*27], m29
+ call .pass2_fast
+ mova [cq+64* 8], m14
+ mova [cq+64* 9], m15
+ mova [cq+64*10], m16
+ mova [cq+64*11], m17
+ mova [cq+64*12], m18
+ mova [cq+64*13], m19
+ mova [cq+64*14], m20
+ mova [cq+64*15], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+ mova [cq+64* 0], m0
+ mova [cq+64* 1], m1
+ mova [cq+64* 2], m2
+ mova [cq+64* 3], m3
+ mova [cq+64* 4], m4
+ mova [cq+64* 5], m5
+ mova [cq+64* 6], m6
+ mova [cq+64* 7], m7
+ pmulhrsw m0, m13, [r4+64*16]
+ pmulhrsw m1, m13, [r4+64*17]
+ pmulhrsw m2, m13, [r4+64*18]
+ pmulhrsw m3, m13, [r4+64*19]
+ pmulhrsw m4, m13, [r4+64*20]
+ pmulhrsw m5, m13, [r4+64*21]
+ pmulhrsw m6, m13, [r4+64*22]
+ pmulhrsw m7, m13, [r4+64*23]
+ mova [cq+64*16], m14
+ mova [cq+64*17], m15
+ mova [cq+64*18], m16
+ mova [cq+64*19], m17
+ mova [cq+64*20], m18
+ mova [cq+64*21], m19
+ mova [cq+64*22], m20
+ mova [cq+64*23], m21
+ pmulhrsw m14, m13, [r4+64*24]
+ pmulhrsw m15, m13, [r4+64*25]
+ pmulhrsw m16, m13, [r4+64*26]
+ pmulhrsw m17, m13, [r4+64*27]
+ pmulhrsw m18, m13, [r4+64*28]
+ pmulhrsw m19, m13, [r4+64*29]
+ pmulhrsw m20, m13, [r4+64*30]
+ pmulhrsw m21, m13, [r4+64*31]
+ mova [cq+64*24], m22
+ mova [cq+64*25], m23
+ mova [cq+64*26], m24
+ mova [cq+64*27], m25
+ mova [cq+64*28], m26
+ mova [cq+64*29], m27
+ mova [cq+64*30], m28
+ mova [cq+64*31], m29
+ call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round
+ call .pass2_fast
+ mova [r4+64*16], m14
+ mova [r4+64*17], m15
+ mova [r4+64*18], m16
+ mova [r4+64*19], m17
+ mova [r4+64*20], m18
+ mova [r4+64*21], m19
+ mova [r4+64*22], m20
+ mova [r4+64*23], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+.end:
+ vpbroadcastd m13, [o(pw_2048)]
+ lea r5, [strideq*3]
+ pxor m12, m12
+ lea r3, [dstq+r5*8]
+ lea r6, [strideq+r5] ; stride*4
+ add r3, r6 ; dst+stride*28
+%macro IDCT_64x32_END 5 ; src16, src32, mem, off_lo, off_hi
+ mova m11, [cq+64*( %3)] ; 0
+ mova m9, [cq+64*(31-%3)] ; 31
+%if %3 >= 8
+ mova m%1, [rsp+64*(%1+16)]
+%endif
+ mova m10, [dstq+%4]
+ paddsw m8, m11, m9
+ psubsw m11, m9
+ paddsw m9, m%1, m%2
+ psubsw m%1, m%2
+ punpcklbw m%2, m10, m12
+ punpckhbw m10, m12
+ pmulhrsw m8, m13
+ pmulhrsw m9, m13
+ paddw m8, m%2
+ paddw m9, m10
+ mova m10, [r3+%5]
+ pmulhrsw m11, m13
+ pmulhrsw m%1, m13
+ mova [cq+64*( %3)], m12
+ mova [cq+64*(31-%3)], m12
+ punpcklbw m%2, m10, m12
+ punpckhbw m10, m12
+ packuswb m8, m9
+ paddw m11, m%2
+ paddw m%1, m10
+ packuswb m11, m%1
+ mova [dstq+%4], m8
+ mova [r3 +%5], m11
+%if %3 == 3 || %3 == 7 || %3 == 11
+ add dstq, r6
+ sub r3, r6
+%endif
+%endmacro
+ IDCT_64x32_END 0, 29, 0, strideq*0, r5
+ IDCT_64x32_END 1, 28, 1, strideq*1, strideq*2
+ IDCT_64x32_END 2, 27, 2, strideq*2, strideq*1
+ IDCT_64x32_END 3, 26, 3, r5 , strideq*0
+ IDCT_64x32_END 4, 25, 4, strideq*0, r5
+ IDCT_64x32_END 5, 24, 5, strideq*1, strideq*2
+ IDCT_64x32_END 6, 23, 6, strideq*2, strideq*1
+ IDCT_64x32_END 7, 22, 7, r5 , strideq*0
+ IDCT_64x32_END 0, 21, 8, strideq*0, r5
+ IDCT_64x32_END 1, 20, 9, strideq*1, strideq*2
+ IDCT_64x32_END 2, 19, 10, strideq*2, strideq*1
+ IDCT_64x32_END 3, 18, 11, r5 , strideq*0
+ IDCT_64x32_END 4, 17, 12, strideq*0, r5
+ IDCT_64x32_END 5, 16, 13, strideq*1, strideq*2
+ IDCT_64x32_END 6, 15, 14, strideq*2, strideq*1
+ IDCT_64x32_END 7, 14, 15, r5 , strideq*0
+ RET
+ALIGN function_align
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 32
+ imul r6d, 181
+ add r6d, 128
+ sar r6d, 8
+ imul r6d, 181
+ add r6d, 128+256
+ sar r6d, 8+1
+ jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly2
+ALIGN function_align
+.pass1_end_part1:
+%macro IDCT_64x32_PASS1_END 3 ; src16, src32, src64
+%if %1 != %3
+ mova m%1, [cq+64*%1]
+%endif
+ mova m9, [r4+64*(%3-36)] ; idct64 32+n
+ mova m11, [r4+64*(-5-%3)] ; idct64 63-n
+ psubsw m8, m%1, m%2 ; idct32 31-n
+ paddsw m%1, m%2 ; idct32 0+n
+%if %1 == %3
+ psubsw m%2, m8, m9 ; out 32+n e
+ paddsw m8, m9 ; out 31-n d
+ psubsw m9, m%1, m11 ; out 63-n h
+ paddsw m%1, m11 ; out 0+n a
+%else
+ paddsw m%2, m8, m9 ; out 23-n c
+ psubsw m8, m9 ; out 40+n f
+ paddsw m9, m%1, m11 ; out 8+n b
+ psubsw m%1, m11 ; out 55-n g
+%endif
+ mova [r4+64*(%3-36)], m8
+ mova [r4+64*(-5-%3)], m9
+%endmacro
+ IDCT_64x32_PASS1_END 0, 29, 0
+ IDCT_64x32_PASS1_END 1, 28, 1
+ IDCT_64x32_PASS1_END 2, 27, 2
+ IDCT_64x32_PASS1_END 3, 26, 3
+ IDCT_64x32_PASS1_END 4, 25, 4
+ IDCT_64x32_PASS1_END 5, 24, 5
+ IDCT_64x32_PASS1_END 6, 23, 6
+ IDCT_64x32_PASS1_END 7, 22, 7
+.transpose_2x8x8_hi: ; m0-m7 + m22-m29 (inverted)
+ punpcklwd m8, m25, m24 ; e0 f0 e1 f1 e2 f2 e3 f3
+ punpckhwd m25, m24 ; e4 f4 e5 f5 e6 f6 e7 f7
+ punpcklwd m24, m23, m22 ; g0 h0 g1 h1 g2 h2 g3 h3
+ punpckhwd m23, m22 ; g4 h4 g5 h5 g6 h6 g7 h7
+ punpcklwd m22, m29, m28 ; a0 b0 a1 b1 a2 b2 a3 b3
+ punpckhwd m29, m28 ; a4 b4 a5 b5 a6 b6 a7 b7
+ punpcklwd m28, m27, m26 ; c0 d0 c1 d1 c2 d2 c3 d3
+ punpckhwd m27, m26 ; c4 d4 c5 d5 c6 d6 c7 d7
+ punpckldq m26, m29, m27 ; a4 b4 c4 d4 a5 b5 c5 d5
+ punpckhdq m29, m27 ; a6 b6 c6 d6 a7 b7 c7 d7
+ punpckldq m27, m8, m24 ; e0 f0 g0 h0 e1 f1 g1 h1
+ punpckhdq m8, m24 ; e2 f2 g2 h2 e3 f3 g3 h3
+ punpckhdq m24, m22, m28 ; a2 b2 c2 d2 a3 b3 c3 d3
+ punpckldq m22, m28 ; a0 b0 c0 d0 a1 b1 c1 d1
+ punpckldq m28, m25, m23 ; e4 f4 g4 h4 e5 f5 g5 h5
+ punpckhdq m25, m23 ; e6 f6 g6 h6 e7 f7 g7 h7
+ punpckhqdq m23, m22, m27 ; 1 23
+ punpcklqdq m22, m27 ; 0 22
+ punpckhqdq m27, m26, m28 ; 5 27
+ punpcklqdq m26, m28 ; 4 26
+ punpcklqdq m28, m29, m25 ; 6 28
+ punpckhqdq m29, m25 ; 7 29
+ punpckhqdq m25, m24, m8 ; 3 25
+ punpcklqdq m24, m8 ; 2 24
+.transpose_8x8:
+ punpckhwd m8, m4, m5
+ punpcklwd m4, m5
+ punpckhwd m5, m0, m1
+ punpcklwd m0, m1
+ punpckhwd m1, m6, m7
+ punpcklwd m6, m7
+ punpckhwd m7, m2, m3
+ punpcklwd m2, m3
+ punpckhdq m3, m0, m2
+ punpckldq m0, m2
+ punpckldq m2, m4, m6
+ punpckhdq m4, m6
+ punpckhdq m6, m5, m7
+ punpckldq m5, m7
+ punpckldq m7, m8, m1
+ punpckhdq m8, m1
+ punpckhqdq m1, m0, m2
+ punpcklqdq m0, m2
+ punpcklqdq m2, m3, m4
+ punpckhqdq m3, m4
+ punpcklqdq m4, m5, m7
+ punpckhqdq m5, m7
+ punpckhqdq m7, m6, m8
+ punpcklqdq m6, m8
+ ret
+.pass1_end_part2:
+ IDCT_64x32_PASS1_END 0, 21, 8
+ IDCT_64x32_PASS1_END 1, 20, 9
+ IDCT_64x32_PASS1_END 2, 19, 10
+ IDCT_64x32_PASS1_END 3, 18, 11
+ IDCT_64x32_PASS1_END 4, 17, 12
+ IDCT_64x32_PASS1_END 5, 16, 13
+ IDCT_64x32_PASS1_END 6, 15, 14
+ IDCT_64x32_PASS1_END 7, 14, 15
+.transpose_2x8x8_lo: ; m0-m7 (inverted) + m14-m21
+ punpcklwd m8, m3, m2
+ punpckhwd m3, m2
+ punpcklwd m2, m1, m0
+ punpckhwd m1, m0
+ punpcklwd m0, m7, m6
+ punpckhwd m7, m6
+ punpcklwd m6, m5, m4
+ punpckhwd m5, m4
+ punpckldq m4, m7, m5
+ punpckhdq m7, m5
+ punpckldq m5, m8, m2
+ punpckhdq m8, m2
+ punpckhdq m2, m0, m6
+ punpckldq m0, m6
+ punpckldq m6, m3, m1
+ punpckhdq m3, m1
+ punpckhqdq m1, m0, m5
+ punpcklqdq m0, m5
+ punpckhqdq m5, m4, m6
+ punpcklqdq m4, m6
+ punpcklqdq m6, m7, m3
+ punpckhqdq m7, m3
+ punpckhqdq m3, m2, m8
+ punpcklqdq m2, m8
+ punpckhwd m8, m18, m19
+ punpcklwd m18, m19
+ punpckhwd m19, m14, m15
+ punpcklwd m14, m15
+ punpckhwd m15, m20, m21
+ punpcklwd m20, m21
+ punpckhwd m21, m16, m17
+ punpcklwd m16, m17
+ punpckhdq m17, m14, m16
+ punpckldq m14, m16
+ punpckldq m16, m18, m20
+ punpckhdq m18, m20
+ punpckhdq m20, m19, m21
+ punpckldq m19, m21
+ punpckldq m21, m8, m15
+ punpckhdq m8, m15
+ punpckhqdq m15, m14, m16
+ punpcklqdq m14, m16
+ punpcklqdq m16, m17, m18
+ punpckhqdq m17, m18
+ punpcklqdq m18, m19, m21
+ punpckhqdq m19, m21
+ punpckhqdq m21, m20, m8
+ punpcklqdq m20, m8
+ ret
+.pass2_fast:
+ vshufi32x4 m24, m9, m15, q3131 ; 5
+ vshufi32x4 m22, m9, m15, q2020 ; 1
+ vshufi32x4 m15, m1, m16, q3131 ; 6
+ vshufi32x4 m14, m1, m16, q2020 ; 2
+ vshufi32x4 m1, m0, m3, q3131 ; 4
+ vshufi32x4 m0, m3, q2020 ; 0
+ vshufi32x4 m3, m8, m2, q3131 ; 12
+ vshufi32x4 m2, m8, m2, q2020 ; 8
+ vshufi32x4 m25, m11, m17, q3131 ; 7
+ vshufi32x4 m23, m11, m17, q2020 ; 3
+ vshufi32x4 m17, m5, m19, q3131 ; 14
+ vshufi32x4 m16, m5, m19, q2020 ; 10
+ vshufi32x4 m29, m6, m20, q3131 ; 15
+ vshufi32x4 m27, m6, m20, q2020 ; 11
+ vshufi32x4 m28, m4, m18, q3131 ; 13
+ vshufi32x4 m26, m4, m18, q2020 ; 9
+ jmp m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+
+cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 7, 0, dst, stride, c, eob
+ lea r5, [o_base]
+ test eobd, eobd
+ jz .dconly
+ PROLOGUE 0, 7, 30, 64*96, dst, stride, c, eob
+%undef cmp
+ cmp eobd, 136
+ jb .fast
+ mova m0, [cq+64* 1]
+ mova m1, [cq+64*31]
+ mova m2, [cq+64*17]
+ mova m3, [cq+64*15]
+ vpbroadcastd m10, [o(pd_2048)]
+ mov r4, rsp
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ mova m0, [cq+64* 7]
+ mova m1, [cq+64*25]
+ mova m2, [cq+64*23]
+ mova m3, [cq+64* 9]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ mova m0, [cq+64* 5]
+ mova m1, [cq+64*27]
+ mova m2, [cq+64*21]
+ mova m3, [cq+64*11]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ mova m0, [cq+64* 3]
+ mova m1, [cq+64*29]
+ mova m2, [cq+64*19]
+ mova m3, [cq+64*13]
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2
+ mova m0, [cq+64* 0]
+ mova m1, [cq+64* 8]
+ mova m2, [cq+64*16]
+ mova m3, [cq+64*24]
+ mova m14, [cq+64* 4]
+ mova m15, [cq+64*12]
+ mova m16, [cq+64*20]
+ mova m17, [cq+64*28]
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+ mova m22, [cq+64* 2]
+ mova m29, [cq+64*30]
+ mova m26, [cq+64*18]
+ mova m25, [cq+64*14]
+ mova m24, [cq+64*10]
+ mova m27, [cq+64*22]
+ mova m28, [cq+64*26]
+ mova m23, [cq+64* 6]
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+ vpbroadcastd m13, [o(pw_8192)]
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part1
+ mova [r4+64*36], m1
+ mova [r4+64*37], m3
+ mova [r4+64*38], m5
+ mova [r4+64*39], m7
+ mova [r4+64*44], m23
+ mova [r4+64*45], m25
+ mova [r4+64*46], m27
+ mova [r4+64*47], m29
+ pmulhrsw m23, m13, m0 ; a0
+ pmulhrsw m25, m13, m2 ; a2
+ pmulhrsw m27, m13, m4 ; a4
+ pmulhrsw m29, m13, m6 ; a6
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).pass1_end_part2
+ lea r6, [r4-64*4]
+ add r4, 64*28
+ call .pass2_end
+ mov r4, rsp
+ mova m0, [r4+64*23]
+ mova m1, [r4+64*22]
+ mova m2, [r4+64*21]
+ mova m3, [r4+64*20]
+ mova m4, [r4+64*19]
+ mova m5, [r4+64*18]
+ mova m6, [r4+64*17]
+ mova m7, [r4+64*16]
+ mova m22, [r4+64*15]
+ mova m23, [r4+64*14]
+ mova m24, [r4+64*13]
+ mova m25, [r4+64*12]
+ mova m26, [r4+64*11]
+ mova m27, [r4+64*10]
+ mova m28, [r4+64* 9]
+ mova m29, [r4+64* 8]
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_hi
+ vpbroadcastd m13, [o(pw_8192)]
+ mova [r4+64* 8], m1
+ mova [r4+64* 9], m3
+ mova [r4+64*10], m5
+ mova [r4+64*11], m7
+ mova [r4+64*16], m23
+ mova [r4+64*17], m25
+ mova [r4+64*18], m27
+ mova [r4+64*19], m29
+ pmulhrsw m23, m13, m0 ; b0
+ pmulhrsw m25, m13, m2 ; b2
+ pmulhrsw m27, m13, m4 ; b4
+ pmulhrsw m29, m13, m6 ; b6
+ mova m0, [r4+64*31]
+ mova m1, [r4+64*30]
+ mova m2, [r4+64*29]
+ mova m3, [r4+64*28]
+ mova m4, [r4+64*27]
+ mova m5, [r4+64*26]
+ mova m6, [r4+64*25]
+ mova m7, [r4+64*24]
+ mova m14, [r4+64* 7]
+ mova m15, [r4+64* 6]
+ mova m16, [r4+64* 5]
+ mova m17, [r4+64* 4]
+ mova m18, [r4+64* 3]
+ mova m19, [r4+64* 2]
+ mova m20, [r4+64* 1]
+ mova m21, [r4+64* 0]
+ call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_2x8x8_lo
+ mov r6, cq
+ call .pass2_end
+ jmp .end
+.fast: ; bottom/right halves are zero
+ mova m28, [o(dup16_perm)]
+ pmovzxwd m9, [cq+64* 0]
+ vpermb m8, m28, [cq+64* 4]
+ vpermb ym1, ym28, [cq+64*12]
+ vpermb m7, m28, [cq+64* 8]
+ pslld m9, 16
+ call m(idct_16x16_internal_8bpc).main_fast2
+ vpermb m21, m28, [cq+64* 2]
+ vpermb ym15, ym28, [cq+64*14]
+ vpermb ym18, ym28, [cq+64*10]
+ vpermb m14, m28, [cq+64* 6]
+ call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2
+ vpermb m22, m28, [cq+64* 1]
+ vpermb ym29, ym28, [cq+64*15]
+ vpermb ym26, ym28, [cq+64* 9]
+ vpermb m25, m28, [cq+64* 7]
+ vpermb m24, m28, [cq+64* 5]
+ vpermb ym27, ym28, [cq+64*11]
+ vpermb m23, m28, [cq+64* 3]
+ vpermb ym28, ym28, [cq+64*13]
+ mova [cq+64* 0], m14
+ mova [cq+64* 1], m15
+ mova [cq+64* 2], m16
+ mova [cq+64* 3], m17
+ mova [cq+64* 4], m18
+ mova [cq+64* 5], m19
+ mova [cq+64* 6], m20
+ mova [cq+64* 7], m21
+ call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast
+ vpbroadcastd m13, [o(pw_8192)]
+ mova [cq+64*16], m4
+ mova [cq+64*17], m5
+ mova [cq+64*18], m6
+ mova [cq+64*19], m7
+ mova [cq+64*28], m26
+ mova [cq+64*29], m27
+ mova [cq+64*30], m28
+ mova [cq+64*31], m29
+ call m(inv_txfm_add_dct_dct_64x16_8bpc).pass1_end
+ mova [cq+64*20], m22
+ mova [cq+64*21], m23
+ mova [cq+64*22], m24
+ mova [cq+64*23], m25
+ mova [cq+64*24], m26
+ mova [cq+64*25], m27
+ mova [cq+64*26], m28
+ mova [cq+64*27], m29
+ lea r4, [rsp+64*64]
+ lea r3, [rsp+64*32]
+ call .pass2_fast
+ pmulhrsw m0, m13, [cq+64*16]
+ pmulhrsw m1, m13, [cq+64*17]
+ pmulhrsw m2, m13, [cq+64*18]
+ pmulhrsw m3, m13, [cq+64*19]
+ pmulhrsw m4, m13, [cq+64*20]
+ pmulhrsw m5, m13, [cq+64*21]
+ pmulhrsw m6, m13, [cq+64*22]
+ pmulhrsw m7, m13, [cq+64*23]
+ pmulhrsw m14, m13, [cq+64*24]
+ pmulhrsw m15, m13, [cq+64*25]
+ pmulhrsw m16, m13, [cq+64*26]
+ pmulhrsw m17, m13, [cq+64*27]
+ pmulhrsw m18, m13, [cq+64*28]
+ pmulhrsw m19, m13, [cq+64*29]
+ pmulhrsw m20, m13, [cq+64*30]
+ pmulhrsw m21, m13, [cq+64*31]
+ call m(inv_txfm_add_dct_dct_64x16_8bpc).transpose_round
+ mov r4, rsp
+ mov r3, cq
+ call .pass2_fast
+.end:
+ vpbroadcastd m17, [o(pw_2048)]
+ lea r5, [strideq*8]
+ mov r3, dstq
+ pxor m16, m16
+ sub r4, 64*5 ; rsp+64*31
+ mov r6, rsp
+.end_loop:
+ mova m2, [r6+64*32] ; idct16 0+n lo
+ mova m7, [r6+64*48] ; idct32 31-n lo
+ mova m6, [cq+64* 0] ; idct16 0+n hi
+ mova m0, [cq+64*16] ; idct32 31-n hi
+ mova m4, [r4+64*64] ; idct64 63-n lo
+ mova m1, [r4+64* 0] ; idct64 63-n hi
+ mova m5, [r6+64*64] ; idct64 32+n lo
+ mova m8, [r6+64* 0] ; idct64 32+n hi
+ sub r3, strideq
+ paddsw m3, m2, m7 ; idct32 0+n lo
+ mova m12, [dstq+r5*0]
+ psubsw m2, m7 ; idct32 31-n lo
+ mova m15, [r3 +r5*8]
+ paddsw m7, m6, m0 ; idct32 0+n hi
+ mova m13, [r3 +r5*4]
+ psubsw m6, m0 ; idct32 31-n hi
+ mova m14, [dstq+r5*4]
+ paddsw m0, m3, m4 ; out 0+n lo
+ add r6, 64
+ psubsw m3, m4 ; out 63-n lo
+ sub r4, 64
+ paddsw m4, m7, m1 ; out 0+n hi
+ mova [cq+64* 0], m16
+ psubsw m7, m1 ; out 63-n hi
+ mova [cq+64*16], m16
+ paddsw m1, m2, m5 ; out 31-n lo
+ add cq, 64
+ psubsw m2, m5 ; out 32+n lo
+ paddsw m5, m6, m8 ; out 31-n hi
+ psubsw m6, m8 ; out 32+n hi
+ pmulhrsw m0, m17
+ punpcklbw m8, m12, m16
+ pmulhrsw m4, m17
+ punpckhbw m12, m16
+ pmulhrsw m3, m17
+ punpcklbw m11, m15, m16
+ pmulhrsw m7, m17
+ punpckhbw m15, m16
+ pmulhrsw m1, m17
+ punpcklbw m9, m13, m16
+ pmulhrsw m5, m17
+ punpckhbw m13, m16
+ pmulhrsw m2, m17
+ punpcklbw m10, m14, m16
+ pmulhrsw m6, m17
+ punpckhbw m14, m16
+ paddw m0, m8
+ paddw m4, m12
+ packuswb m0, m4
+ paddw m3, m11
+ paddw m7, m15
+ packuswb m3, m7
+ paddw m1, m9
+ paddw m5, m13
+ packuswb m1, m5
+ paddw m2, m10
+ paddw m6, m14
+ packuswb m2, m6
+ mova [dstq+r5*0], m0
+ mova [r3 +r5*8], m3
+ mova [r3 +r5*4], m1
+ mova [dstq+r5*4], m2
+ add dstq, strideq
+ cmp r6, r4
+ jb .end_loop
+ RET
+.dconly:
+ movsx r6d, word [cq]
+ mov [cq], eobd
+ or r3d, 64
+ jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly
+ALIGN function_align
+.pass2_end:
+ REPX {pmulhrsw x, m13}, m22, m24, m26, m28, m14, m16, m18, m20, m0, m2, m4, m6
+ mova [r4+64*20], m1
+ mova [r4+64*21], m3
+ mova [r4+64*22], m5
+ mova [r4+64*23], m7
+ vinserti32x8 m1, m23, ym14, 1 ; a00 a01 c00 c01
+ vshufi32x4 m3, m23, m14, q3232 ; a02 a03 c02 c03
+ vinserti32x8 m5, m22, ym0, 1 ; e00 e01 g00 g01
+ vshufi32x4 m14, m22, m0, q3232 ; e02 e03 g02 g03
+ mova [r4+64*12], m15
+ mova [r4+64*13], m17
+ mova [r4+64*14], m19
+ mova [r4+64*15], m21
+ vinserti32x8 m15, m27, ym18, 1 ; a40 a41 c40 c41
+ vshufi32x4 m17, m27, m18, q3232 ; a42 a43 c42 c43
+ vinserti32x8 m18, m26, ym4, 1 ; e40 e41 g40 g41
+ vshufi32x4 m19, m26, m4, q3232 ; e42 e43 g42 g43
+ vinserti32x8 m22, m25, ym16, 1 ; a20 a21 c20 c21
+ vshufi32x4 m26, m25, m16, q3232 ; a22 a23 c22 c23
+ vinserti32x8 m25, m24, ym2, 1 ; e20 e21 g20 g21
+ vshufi32x4 m27, m24, m2, q3232 ; e22 e23 g22 g23
+ vinserti32x8 m23, m29, ym20, 1 ; a60 a61 c60 c61
+ vshufi32x4 m29, m20, q3232 ; a62 a63 c62 c63
+ vshufi32x4 m13, m28, m6, q3232 ; e62 e63 g62 g63
+ vinserti32x8 m28, ym6, 1 ; e60 e61 g60 g61
+ vshufi32x4 m0, m1, m5, q2020 ; 0
+ vshufi32x4 m1, m5, q3131 ; 8
+ vshufi32x4 m2, m3, m14, q2020 ; 16
+ vshufi32x4 m3, m14, q3131 ; 24
+ vshufi32x4 m14, m15, m18, q2020 ; 4
+ vshufi32x4 m15, m18, q3131 ; 12
+ vshufi32x4 m16, m17, m19, q2020 ; 20
+ vshufi32x4 m17, m19, q3131 ; 28
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast
+ vshufi32x4 m24, m22, m25, q3131 ; 10
+ vshufi32x4 m22, m25, q2020 ; 2
+ vshufi32x4 m25, m23, m28, q3131 ; 14
+ vshufi32x4 m23, m28, q2020 ; 6
+ vshufi32x4 m28, m26, m27, q3131 ; 26
+ vshufi32x4 m26, m27, q2020 ; 18
+ vshufi32x4 m27, m29, m13, q2020 ; 22
+ vshufi32x4 m29, m13, q3131 ; 30
+ mova [r6+64* 0], m0
+ mova [r6+64* 1], m1
+ mova [r6+64* 2], m2
+ mova [r6+64* 3], m3
+ mova [r6+64* 4], m4
+ mova [r6+64* 5], m5
+ mova [r6+64* 6], m6
+ mova [r6+64* 7], m7
+ mova [r6+64* 8], m14
+ mova [r6+64* 9], m15
+ mova [r6+64*10], m16
+ mova [r6+64*11], m17
+ mova [r6+64*12], m18
+ mova [r6+64*13], m19
+ mova [r6+64*14], m20
+ mova [r6+64*15], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast
+ vpbroadcastd m13, [o(pw_8192)]
+ mova [r6+64*16], m29
+ mova [r6+64*17], m28
+ mova [r6+64*18], m27
+ mova [r6+64*19], m26
+ mova [r6+64*20], m25
+ mova [r6+64*21], m24
+ mova [r6+64*22], m23
+ mova [r6+64*23], m22
+ mova [r6+64*24], m21
+ mova [r6+64*25], m20
+ mova [r6+64*26], m19
+ mova [r6+64*27], m18
+ mova [r6+64*28], m17
+ mova [r6+64*29], m16
+ mova [r6+64*30], m15
+ mova [r6+64*31], m14
+ pmulhrsw m15, m13, [r4+64* 8] ; 1 9 17 25
+ pmulhrsw m16, m13, [r4+64*12]
+ pmulhrsw m17, m13, [r4+64*16]
+ pmulhrsw m18, m13, [r4+64*20]
+ pmulhrsw m19, m13, [r4+64*11] ; 7 15 23 31
+ pmulhrsw m20, m13, [r4+64*15]
+ pmulhrsw m21, m13, [r4+64*19]
+ pmulhrsw m22, m13, [r4+64*23]
+ vinserti32x8 m14, m15, ym16, 1 ; a1 a9 c1 c9
+ vshufi32x4 m15, m16, q3232 ; a17 a25 c17 c25
+ vinserti32x8 m16, m17, ym18, 1 ; e1 e9 g1 g9
+ vshufi32x4 m17, m18, q3232 ; e17 e25 g17 g25
+ pmulhrsw m23, m13, [r4+64*10] ; 5 13 21 29
+ pmulhrsw m24, m13, [r4+64*14]
+ pmulhrsw m25, m13, [r4+64*18]
+ pmulhrsw m26, m13, [r4+64*22]
+ vinserti32x8 m18, m19, ym20, 1 ; a7 a15 c7 c15
+ vshufi32x4 m19, m20, q3232 ; a23 a31 c23 c31
+ vinserti32x8 m20, m21, ym22, 1 ; e7 e15 g7 g15
+ vshufi32x4 m21, m22, q3232 ; e23 e31 g23 g31
+ pmulhrsw m27, m13, [r4+64* 9] ; 3 11 19 27
+ pmulhrsw m28, m13, [r4+64*13]
+ pmulhrsw m29, m13, [r4+64*17]
+ pmulhrsw m13, [r4+64*21]
+ vshufi32x4 m0, m14, m16, q2020 ; 1
+ vshufi32x4 m1, m19, m21, q3131 ; 31
+ vshufi32x4 m2, m15, m17, q2020 ; 17
+ vshufi32x4 m3, m18, m20, q3131 ; 15
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ vshufi32x4 m0, m18, m20, q2020 ; 7
+ vshufi32x4 m1, m15, m17, q3131 ; 25
+ vshufi32x4 m2, m19, m21, q2020 ; 23
+ vshufi32x4 m3, m14, m16, q3131 ; 9
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ vinserti32x8 m22, m23, ym24, 1 ; a5 a13 c5 c13
+ vshufi32x4 m23, m24, q3232 ; a21 a29 c21 c29
+ vinserti32x8 m24, m25, ym26, 1 ; e5 e13 g5 g13
+ vshufi32x4 m25, m26, q3232 ; e21 e29 g21 g29
+ vinserti32x8 m26, m27, ym28, 1 ; a3 a11 c3 c11
+ vshufi32x4 m27, m28, q3232 ; a19 a27 c19 c27
+ vinserti32x8 m28, m29, ym13, 1 ; e3 e11 g3 g11
+ vshufi32x4 m29, m13, q3232 ; e19 e17 g19 g27
+ vshufi32x4 m0, m22, m24, q2020 ; 5
+ vshufi32x4 m1, m27, m29, q3131 ; 27
+ vshufi32x4 m2, m23, m25, q2020 ; 21
+ vshufi32x4 m3, m26, m28, q3131 ; 11
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ vshufi32x4 m0, m26, m28, q2020 ; 3
+ vshufi32x4 m1, m23, m25, q3131 ; 29
+ vshufi32x4 m2, m27, m29, q2020 ; 19
+ vshufi32x4 m3, m22, m24, q3131 ; 13
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1
+ jmp m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2
+ALIGN function_align
+.pass2_fast:
+ vshufi32x4 m23, m1, m16, q3131 ; 6
+ vshufi32x4 m22, m1, m16, q2020 ; 2
+ vshufi32x4 m14, m0, m3, q3131 ; 4
+ vshufi32x4 m26, m0, m3, q2020 ; 0
+ vshufi32x4 m28, m9, m15, q3131 ; 5
+ vshufi32x4 m0, m9, m15, q2020 ; 1
+ vshufi32x4 m16, m11, m17, q3131 ; 7
+ vshufi32x4 m29, m11, m17, q2020 ; 3
+ vshufi32x4 m15, m8, m2, q3131 ; 12
+ vshufi32x4 m27, m8, m2, q2020 ; 8
+ vshufi32x4 m25, m5, m19, q3131 ; 14
+ vshufi32x4 m24, m5, m19, q2020 ; 10
+ vshufi32x4 m3, m6, m20, q3131 ; 15
+ vshufi32x4 m19, m6, m20, q2020 ; 11
+ vshufi32x4 m17, m4, m18, q3131 ; 13
+ vshufi32x4 m18, m4, m18, q2020 ; 9
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast
+ mova m0, m16
+ mova m3, m18
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast
+ mova m0, m28
+ mova m3, m19
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast
+ mova m0, m29
+ mova m3, m17
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast
+ call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2
+ mova m0, m26
+ mova m1, m27
+ call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2
+ mova [r3+64* 0], m0
+ mova [r3+64* 1], m1
+ mova [r3+64* 2], m2
+ mova [r3+64* 3], m3
+ mova [r3+64* 4], m4
+ mova [r3+64* 5], m5
+ mova [r3+64* 6], m6
+ mova [r3+64* 7], m7
+ mova [r3+64* 8], m14
+ mova [r3+64* 9], m15
+ mova [r3+64*10], m16
+ mova [r3+64*11], m17
+ mova [r3+64*12], m18
+ mova [r3+64*13], m19
+ mova [r3+64*14], m20
+ mova [r3+64*15], m21
+ call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2
+ mova [r3+64*16], m29
+ mova [r3+64*17], m28
+ mova [r3+64*18], m27
+ mova [r3+64*19], m26
+ mova [r3+64*20], m25
+ mova [r3+64*21], m24
+ mova [r3+64*22], m23
+ mova [r3+64*23], m22
+ mova [r3+64*24], m21
+ mova [r3+64*25], m20
+ mova [r3+64*26], m19
+ mova [r3+64*27], m18
+ mova [r3+64*28], m17
+ mova [r3+64*29], m16
+ mova [r3+64*30], m15
+ mova [r3+64*31], m14
+ ret
+
+%endif ; ARCH_X86_64