Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 3270 insertions, 0 deletions

diff --git a/third_party/dav1d/src/arm/64/itx.S b/third_party/dav1d/src/arm/64/itx.S
new file mode 100644
index 0000000000..b1b2f8fe65
--- /dev/null
+++ b/third_party/dav1d/src/arm/64/itx.S
@@ -0,0 +1,3270 @@
+/******************************************************************************
+ * Copyright © 2018, VideoLAN and dav1d authors
+ * Copyright © 2019, Martin Storsjo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *****************************************************************************/
+
+#include "src/arm/asm.S"
+#include "util.S"
+
+// The exported functions in this file have got the following signature:
+// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob);
+
+// Most of the functions use the following register layout:
+// x0-x3  external parameters
+// x4     function pointer to first transform
+// x5     function pointer to second transform
+// x6     output parameter for helper function
+// x7     input parameter for helper function
+// x8     input stride for helper function
+// x9-x12 scratch variables for helper functions
+// x13    pointer to list of eob thresholds
+// x14    return pointer for helper function
+// x15    return pointer for main function
+
+// The SIMD registers most often use the following layout:
+// v0-v1   multiplication coefficients
+// v2-v7   scratch registers
+// v8-v15  unused
+// v16-v31 inputs/outputs of transforms
+
+// Potential further optimizations, that are left unimplemented for now:
+// - Trying to keep multiplication coefficients in registers across multiple
+//   transform functions. (The register layout is designed to potentially
+//   allow this.)
+// - Use a simplified version of the transforms themselves for cases where
+//   we know a significant number of inputs are zero. E.g. if the eob value
+//   indicates only a quarter of input values are set, for idct16 and up,
+//   a significant amount of calculation can be skipped, at the cost of more
+//   code duplication and special casing.
+
+const idct_coeffs, align=4
+        // idct4
+        .short          2896, 2896*8, 1567, 3784
+        // idct8
+        .short          799, 4017, 3406, 2276
+        // idct16
+        .short          401, 4076, 3166, 2598
+        .short          1931, 3612, 3920, 1189
+        // idct32
+        .short          201, 4091, 3035, 2751
+        .short          1751, 3703, 3857, 1380
+        .short          995, 3973, 3513, 2106
+        .short          2440, 3290, 4052, 601
+endconst
+
+const idct64_coeffs, align=4
+        .short          101*8, 4095*8, 2967*8, -2824*8
+        .short          1660*8, 3745*8, 3822*8, -1474*8
+        .short          4076, 401, 4017, 799
+        .short          0, 0, 0, 0
+
+        .short          4036*8, -700*8, 2359*8, 3349*8
+        .short          3461*8, -2191*8, 897*8, 3996*8
+        .short          -3166, -2598, -799, -4017
+        .short          0, 0, 0, 0
+
+        .short          501*8, 4065*8, 3229*8, -2520*8
+        .short          2019*8, 3564*8, 3948*8, -1092*8
+        .short          3612, 1931, 2276, 3406
+        .short          0, 0, 0, 0
+
+        .short          4085*8, -301*8, 2675*8, 3102*8
+        .short          3659*8, -1842*8, 1285*8, 3889*8
+        .short          -3920, -1189, -3406, -2276
+        .short          0, 0, 0, 0
+endconst
+
+const iadst4_coeffs, align=4
+        // .h[4-5] can be interpreted as .s[2]
+        .short          1321, 3803, 2482, 3344, 3344, 0
+endconst
+
+const iadst8_coeffs, align=4
+        .short          4076, 401, 3612, 1931
+        .short          2598, 3166, 1189, 3920
+        // idct_coeffs
+        .short          2896, 0, 1567, 3784, 0, 0, 0, 0
+endconst
+
+const iadst16_coeffs, align=4
+        .short          4091, 201, 3973, 995
+        .short          3703, 1751, 3290, 2440
+        .short          2751, 3035, 2106, 3513
+        .short          1380, 3857, 601, 4052
+endconst
+
+.macro smull_smlal d0, d1, s0, s1, c0, c1, sz
+        smull           \d0\().4s, \s0\().4h, \c0
+        smlal           \d0\().4s, \s1\().4h, \c1
+.ifc \sz, .8h
+        smull2          \d1\().4s, \s0\().8h, \c0
+        smlal2          \d1\().4s, \s1\().8h, \c1
+.endif
+.endm
+
+.macro smull_smlsl d0, d1, s0, s1, c0, c1, sz
+        smull           \d0\().4s, \s0\().4h, \c0
+        smlsl           \d0\().4s, \s1\().4h, \c1
+.ifc \sz, .8h
+        smull2          \d1\().4s, \s0\().8h, \c0
+        smlsl2          \d1\().4s, \s1\().8h, \c1
+.endif
+.endm
+
+.macro sqrshrn_sz d0, s0, s1, shift, sz
+        sqrshrn         \d0\().4h, \s0\().4s, \shift
+.ifc \sz, .8h
+        sqrshrn2        \d0\().8h, \s1\().4s, \shift
+.endif
+.endm
+
+.macro scale_input sz, c, r0, r1, r2 r3, r4, r5, r6, r7
+        sqrdmulh        \r0\sz,  \r0\sz,  \c
+        sqrdmulh        \r1\sz,  \r1\sz,  \c
+        sqrdmulh        \r2\sz,  \r2\sz,  \c
+        sqrdmulh        \r3\sz,  \r3\sz,  \c
+.ifnb \r4
+        sqrdmulh        \r4\sz,  \r4\sz,  \c
+        sqrdmulh        \r5\sz,  \r5\sz,  \c
+        sqrdmulh        \r6\sz,  \r6\sz,  \c
+        sqrdmulh        \r7\sz,  \r7\sz,  \c
+.endif
+.endm
+
+.macro load_add_store load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src, shiftbits=4
+.ifnb \load
+        ld1             {\load},  [\src], x1
+.endif
+.ifnb \shift
+        srshr           \shift,  \shift,  #\shiftbits
+.endif
+.ifnb \addsrc
+        uaddw           \adddst, \adddst, \addsrc
+.endif
+.ifnb \narrowsrc
+        sqxtun          \narrowdst, \narrowsrc
+.endif
+.ifnb \store
+        st1             {\store},  [\dst], x1
+.endif
+.endm
+.macro load_add_store_8x16 dst, src
+        mov             \src, \dst
+        load_add_store  v2.8b, v16.8h,      ,       ,       ,      ,      , \dst, \src
+        load_add_store  v3.8b, v17.8h,      ,       ,       ,      ,      , \dst, \src
+        load_add_store  v4.8b, v18.8h, v2.8b, v16.8h,       ,      ,      , \dst, \src
+        load_add_store  v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b,      , \dst, \src
+        load_add_store  v6.8b, v20.8h, v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src
+        load_add_store  v7.8b, v21.8h, v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src
+        load_add_store  v2.8b, v22.8h, v6.8b, v20.8h, v19.8h, v5.8b, v4.8b, \dst, \src
+        load_add_store  v3.8b, v23.8h, v7.8b, v21.8h, v20.8h, v6.8b, v5.8b, \dst, \src
+        load_add_store  v4.8b, v24.8h, v2.8b, v22.8h, v21.8h, v7.8b, v6.8b, \dst, \src
+        load_add_store  v5.8b, v25.8h, v3.8b, v23.8h, v22.8h, v2.8b, v7.8b, \dst, \src
+        load_add_store  v6.8b, v26.8h, v4.8b, v24.8h, v23.8h, v3.8b, v2.8b, \dst, \src
+        load_add_store  v7.8b, v27.8h, v5.8b, v25.8h, v24.8h, v4.8b, v3.8b, \dst, \src
+        load_add_store  v2.8b, v28.8h, v6.8b, v26.8h, v25.8h, v5.8b, v4.8b, \dst, \src
+        load_add_store  v3.8b, v29.8h, v7.8b, v27.8h, v26.8h, v6.8b, v5.8b, \dst, \src
+        load_add_store  v4.8b, v30.8h, v2.8b, v28.8h, v27.8h, v7.8b, v6.8b, \dst, \src
+        load_add_store  v5.8b, v31.8h, v3.8b, v29.8h, v28.8h, v2.8b, v7.8b, \dst, \src
+        load_add_store       ,       , v4.8b, v30.8h, v29.8h, v3.8b, v2.8b, \dst, \src
+        load_add_store       ,       , v5.8b, v31.8h, v30.8h, v4.8b, v3.8b, \dst, \src
+        load_add_store       ,       ,      ,       , v31.8h, v5.8b, v4.8b, \dst, \src
+        load_add_store       ,       ,      ,       ,       ,      , v5.8b, \dst, \src
+.endm
+.macro load_add_store_8x8 dst, src, shiftbits=4
+        mov             \src, \dst
+        load_add_store  v2.8b, v16.8h,      ,       ,       ,      ,      , \dst, \src, \shiftbits
+        load_add_store  v3.8b, v17.8h,      ,       ,       ,      ,      , \dst, \src, \shiftbits
+        load_add_store  v4.8b, v18.8h, v2.8b, v16.8h,       ,      ,      , \dst, \src, \shiftbits
+        load_add_store  v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b,      , \dst, \src, \shiftbits
+        load_add_store  v6.8b, v20.8h, v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src, \shiftbits
+        load_add_store  v7.8b, v21.8h, v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src, \shiftbits
+        load_add_store  v2.8b, v22.8h, v6.8b, v20.8h, v19.8h, v5.8b, v4.8b, \dst, \src, \shiftbits
+        load_add_store  v3.8b, v23.8h, v7.8b, v21.8h, v20.8h, v6.8b, v5.8b, \dst, \src, \shiftbits
+        load_add_store       ,       , v2.8b, v22.8h, v21.8h, v7.8b, v6.8b, \dst, \src, \shiftbits
+        load_add_store       ,       , v3.8b, v23.8h, v22.8h, v2.8b, v7.8b, \dst, \src, \shiftbits
+        load_add_store       ,       ,      ,       , v23.8h, v3.8b, v2.8b, \dst, \src, \shiftbits
+        load_add_store       ,       ,      ,       ,       ,      , v3.8b, \dst, \src, \shiftbits
+.endm
+.macro load_add_store_8x4 dst, src
+        mov             \src, \dst
+        load_add_store  v2.8b, v16.8h,      ,       ,       ,      ,      , \dst, \src
+        load_add_store  v3.8b, v17.8h,      ,       ,       ,      ,      , \dst, \src
+        load_add_store  v4.8b, v18.8h, v2.8b, v16.8h,       ,      ,      , \dst, \src
+        load_add_store  v5.8b, v19.8h, v3.8b, v17.8h, v16.8h, v2.8b,      , \dst, \src
+        load_add_store       ,       , v4.8b, v18.8h, v17.8h, v3.8b, v2.8b, \dst, \src
+        load_add_store       ,       , v5.8b, v19.8h, v18.8h, v4.8b, v3.8b, \dst, \src
+        load_add_store       ,       ,      ,       , v19.8h, v5.8b, v4.8b, \dst, \src
+        load_add_store       ,       ,      ,       ,       ,      , v5.8b, \dst, \src
+.endm
+.macro load_add_store4 load, inssrc, insdst, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src
+.ifnb \load
+        ld1             {\load}[0],  [\src], x1
+.endif
+.ifnb \inssrc
+        ins             \insdst\().d[1],   \inssrc\().d[0]
+.endif
+.ifnb \shift
+        srshr           \shift,  \shift,  #4
+.endif
+.ifnb \load
+        ld1             {\load}[1],  [\src], x1
+.endif
+.ifnb \addsrc
+        uaddw           \adddst, \adddst, \addsrc
+.endif
+.ifnb \store
+        st1             {\store}[0],  [\dst], x1
+.endif
+.ifnb \narrowsrc
+        sqxtun          \narrowdst, \narrowsrc
+.endif
+.ifnb \store
+        st1             {\store}[1],  [\dst], x1
+.endif
+.endm
+.macro load_add_store_4x16 dst, src
+        mov             \src, \dst
+        load_add_store4 v0.s, v17, v16,       ,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v1.s, v19, v18,       ,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v2.s, v21, v20, v16.8h,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v3.s, v23, v22, v18.8h, v0.8b, v16.8h,       ,      ,     , \dst, \src
+        load_add_store4 v4.s, v25, v24, v20.8h, v1.8b, v18.8h, v16.8h, v0.8b,     , \dst, \src
+        load_add_store4 v5.s, v27, v26, v22.8h, v2.8b, v20.8h, v18.8h, v1.8b, v0.s, \dst, \src
+        load_add_store4 v6.s, v29, v28, v24.8h, v3.8b, v22.8h, v20.8h, v2.8b, v1.s, \dst, \src
+        load_add_store4 v7.s, v31, v30, v26.8h, v4.8b, v24.8h, v22.8h, v3.8b, v2.s, \dst, \src
+        load_add_store4     ,    ,    , v28.8h, v5.8b, v26.8h, v24.8h, v4.8b, v3.s, \dst, \src
+        load_add_store4     ,    ,    , v30.8h, v6.8b, v28.8h, v26.8h, v5.8b, v4.s, \dst, \src
+        load_add_store4     ,    ,    ,       , v7.8b, v30.8h, v28.8h, v6.8b, v5.s, \dst, \src
+        load_add_store4     ,    ,    ,       ,      ,       , v30.8h, v7.8b, v6.s, \dst, \src
+        load_add_store4     ,    ,    ,       ,      ,       ,       ,      , v7.s, \dst, \src
+.endm
+.macro load_add_store_4x8 dst, src
+        mov             \src, \dst
+        load_add_store4 v0.s, v17, v16,       ,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v1.s, v19, v18,       ,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v2.s, v21, v20, v16.8h,      ,       ,       ,      ,     , \dst, \src
+        load_add_store4 v3.s, v23, v22, v18.8h, v0.8b, v16.8h,       ,      ,     , \dst, \src
+        load_add_store4     ,    ,    , v20.8h, v1.8b, v18.8h, v16.8h, v0.8b,     , \dst, \src
+        load_add_store4     ,    ,    , v22.8h, v2.8b, v20.8h, v18.8h, v1.8b, v0.s, \dst, \src
+        load_add_store4     ,    ,    ,       , v3.8b, v22.8h, v20.8h, v2.8b, v1.s, \dst, \src
+        load_add_store4     ,    ,    ,       ,      ,       , v22.8h, v3.8b, v2.s, \dst, \src
+        load_add_store4     ,    ,    ,       ,      ,       ,       ,      , v3.s, \dst, \src
+.endm
+
+.macro idct_dc w, h, shift
+        cbnz            w3,  1f
+        mov             w16, #2896*8
+        ld1r            {v16.8h}, [x2]
+        dup             v0.4h,   w16
+        sqrdmulh        v16.8h,  v16.8h,  v0.h[0]
+        strh            wzr, [x2]
+.if (\w == 2*\h) || (2*\w == \h)
+        sqrdmulh        v16.8h,  v16.8h,  v0.h[0]
+.endif
+.if \shift > 0
+        srshr           v16.8h,  v16.8h,  #\shift
+.endif
+        sqrdmulh        v16.8h,  v16.8h,  v0.h[0]
+        srshr           v16.8h,  v16.8h,  #4
+        mov             w4,  #\h
+        b               idct_dc_w\w\()_neon
+1:
+.endm
+
+function idct_dc_w4_neon
+1:
+        ld1             {v0.s}[0], [x0], x1
+        ld1             {v0.s}[1], [x0], x1
+        ld1             {v1.s}[0], [x0], x1
+        ld1             {v1.s}[1], [x0], x1
+        subs            w4,  w4,  #4
+        sub             x0,  x0,  x1, lsl #2
+        uaddw           v0.8h,   v16.8h,  v0.8b
+        sqxtun          v0.8b,   v0.8h
+        uaddw           v1.8h,   v16.8h,  v1.8b
+        st1             {v0.s}[0], [x0], x1
+        sqxtun          v1.8b,   v1.8h
+        st1             {v0.s}[1], [x0], x1
+        st1             {v1.s}[0], [x0], x1
+        st1             {v1.s}[1], [x0], x1
+        b.gt            1b
+        ret
+endfunc
+
+function idct_dc_w8_neon
+1:
+        ld1             {v0.8b}, [x0], x1
+        ld1             {v1.8b}, [x0], x1
+        ld1             {v2.8b}, [x0], x1
+        uaddw           v20.8h,  v16.8h, v0.8b
+        ld1             {v3.8b}, [x0], x1
+        sub             x0,  x0,  x1, lsl #2
+        subs            w4,  w4,  #4
+        uaddw           v21.8h,  v16.8h, v1.8b
+        sqxtun          v0.8b,   v20.8h
+        uaddw           v22.8h,  v16.8h, v2.8b
+        sqxtun          v1.8b,   v21.8h
+        uaddw           v23.8h,  v16.8h, v3.8b
+        st1             {v0.8b}, [x0], x1
+        sqxtun          v2.8b,   v22.8h
+        st1             {v1.8b}, [x0], x1
+        sqxtun          v3.8b,   v23.8h
+        st1             {v2.8b}, [x0], x1
+        st1             {v3.8b}, [x0], x1
+        b.gt            1b
+        ret
+endfunc
+
+function idct_dc_w16_neon
+1:
+        ld1             {v0.16b}, [x0], x1
+        ld1             {v1.16b}, [x0], x1
+        ld1             {v2.16b}, [x0], x1
+        subs            w4,  w4,  #4
+        uaddw           v20.8h,  v16.8h, v0.8b
+        uaddw2          v21.8h,  v16.8h, v0.16b
+        ld1             {v3.16b}, [x0], x1
+        uaddw           v22.8h,  v16.8h, v1.8b
+        uaddw2          v23.8h,  v16.8h, v1.16b
+        sub             x0,  x0,  x1, lsl #2
+        uaddw           v24.8h,  v16.8h, v2.8b
+        uaddw2          v25.8h,  v16.8h, v2.16b
+        sqxtun          v0.8b,   v20.8h
+        sqxtun2         v0.16b,  v21.8h
+        uaddw           v26.8h,  v16.8h, v3.8b
+        uaddw2          v27.8h,  v16.8h, v3.16b
+        sqxtun          v1.8b,   v22.8h
+        sqxtun2         v1.16b,  v23.8h
+        sqxtun          v2.8b,   v24.8h
+        sqxtun2         v2.16b,  v25.8h
+        st1             {v0.16b}, [x0], x1
+        sqxtun          v3.8b,   v26.8h
+        sqxtun2         v3.16b,  v27.8h
+        st1             {v1.16b}, [x0], x1
+        st1             {v2.16b}, [x0], x1
+        st1             {v3.16b}, [x0], x1
+        b.gt            1b
+        ret
+endfunc
+
+function idct_dc_w32_neon
+1:
+        ld1             {v0.16b, v1.16b},  [x0], x1
+        subs            w4,  w4,  #2
+        uaddw           v20.8h,  v16.8h, v0.8b
+        uaddw2          v21.8h,  v16.8h, v0.16b
+        ld1             {v2.16b, v3.16b},  [x0]
+        uaddw           v22.8h,  v16.8h, v1.8b
+        uaddw2          v23.8h,  v16.8h, v1.16b
+        sub             x0,  x0,  x1
+        uaddw           v24.8h,  v16.8h, v2.8b
+        uaddw2          v25.8h,  v16.8h, v2.16b
+        sqxtun          v0.8b,   v20.8h
+        sqxtun2         v0.16b,  v21.8h
+        uaddw           v26.8h,  v16.8h, v3.8b
+        uaddw2          v27.8h,  v16.8h, v3.16b
+        sqxtun          v1.8b,   v22.8h
+        sqxtun2         v1.16b,  v23.8h
+        sqxtun          v2.8b,   v24.8h
+        sqxtun2         v2.16b,  v25.8h
+        st1             {v0.16b, v1.16b},  [x0], x1
+        sqxtun          v3.8b,   v26.8h
+        sqxtun2         v3.16b,  v27.8h
+        st1             {v2.16b, v3.16b},  [x0], x1
+        b.gt            1b
+        ret
+endfunc
+
+function idct_dc_w64_neon
+1:
+        ld1             {v0.16b, v1.16b, v2.16b, v3.16b},  [x0]
+        subs            w4,  w4,  #1
+        uaddw           v20.8h,  v16.8h, v0.8b
+        uaddw2          v21.8h,  v16.8h, v0.16b
+        uaddw           v22.8h,  v16.8h, v1.8b
+        uaddw2          v23.8h,  v16.8h, v1.16b
+        uaddw           v24.8h,  v16.8h, v2.8b
+        uaddw2          v25.8h,  v16.8h, v2.16b
+        sqxtun          v0.8b,   v20.8h
+        sqxtun2         v0.16b,  v21.8h
+        uaddw           v26.8h,  v16.8h, v3.8b
+        uaddw2          v27.8h,  v16.8h, v3.16b
+        sqxtun          v1.8b,   v22.8h
+        sqxtun2         v1.16b,  v23.8h
+        sqxtun          v2.8b,   v24.8h
+        sqxtun2         v2.16b,  v25.8h
+        sqxtun          v3.8b,   v26.8h
+        sqxtun2         v3.16b,  v27.8h
+        st1             {v0.16b, v1.16b, v2.16b, v3.16b},  [x0], x1
+        b.gt            1b
+        ret
+endfunc
+
+.macro iwht4
+        add             v16.4h,  v16.4h,  v17.4h
+        sub             v21.4h,  v18.4h,  v19.4h
+        sub             v20.4h,  v16.4h,  v21.4h
+        sshr            v20.4h,  v20.4h,  #1
+        sub             v18.4h,  v20.4h,  v17.4h
+        sub             v17.4h,  v20.4h,  v19.4h
+        add             v19.4h,  v21.4h,  v18.4h
+        sub             v16.4h,  v16.4h,  v17.4h
+.endm
+
+.macro idct_4 r0, r1, r2, r3, sz
+        smull_smlal     v6,  v7,  \r1, \r3, v0.h[3], v0.h[2], \sz
+        smull_smlsl     v4,  v5,  \r1, \r3, v0.h[2], v0.h[3], \sz
+        smull_smlal     v2,  v3,  \r0, \r2, v0.h[0], v0.h[0], \sz
+        sqrshrn_sz      v6,  v6,  v7,  #12, \sz
+        sqrshrn_sz      v7,  v4,  v5,  #12, \sz
+        smull_smlsl     v4,  v5,  \r0, \r2, v0.h[0], v0.h[0], \sz
+        sqrshrn_sz      v2,  v2,  v3,  #12, \sz
+        sqrshrn_sz      v3,  v4,  v5,  #12, \sz
+        sqadd           \r0\sz,  v2\sz,   v6\sz
+        sqsub           \r3\sz,  v2\sz,   v6\sz
+        sqadd           \r1\sz,  v3\sz,   v7\sz
+        sqsub           \r2\sz,  v3\sz,   v7\sz
+.endm
+
+function inv_dct_4h_x4_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.4h}, [x16]
+        idct_4          v16, v17, v18, v19, .4h
+        ret
+endfunc
+
+function inv_dct_8h_x4_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.4h}, [x16]
+        idct_4          v16, v17, v18, v19, .8h
+        ret
+endfunc
+
+.macro iadst_4x4 o0, o1, o2, o3
+        movrel          x16, iadst4_coeffs
+        ld1             {v0.8h}, [x16]
+
+        ssubl           v3.4s,   v16.4h,  v18.4h
+        smull           v4.4s,   v16.4h,  v0.h[0]
+        smlal           v4.4s,   v18.4h,  v0.h[1]
+        smlal           v4.4s,   v19.4h,  v0.h[2]
+        smull           v7.4s,   v17.4h,  v0.h[3]
+        saddw           v3.4s,   v3.4s,   v19.4h
+        smull           v5.4s,   v16.4h,  v0.h[2]
+        smlsl           v5.4s,   v18.4h,  v0.h[0]
+        smlsl           v5.4s,   v19.4h,  v0.h[1]
+
+        add             \o3\().4s, v4.4s,     v5.4s
+        mul             \o2\().4s, v3.4s,     v0.s[2]
+        add             \o0\().4s, v4.4s,     v7.4s
+        add             \o1\().4s, v5.4s,     v7.4s
+        sub             \o3\().4s, \o3\().4s, v7.4s
+
+        sqrshrn         \o0\().4h, \o0\().4s, #12
+        sqrshrn         \o2\().4h, \o2\().4s, #12
+        sqrshrn         \o1\().4h, \o1\().4s, #12
+        sqrshrn         \o3\().4h, \o3\().4s, #12
+.endm
+
+function inv_adst_4h_x4_neon, export=1
+        iadst_4x4       v16, v17, v18, v19
+        ret
+endfunc
+
+function inv_flipadst_4h_x4_neon, export=1
+        iadst_4x4       v19, v18, v17, v16
+        ret
+endfunc
+
+.macro iadst_8x4 o0, o1, o2, o3
+        movrel          x16, iadst4_coeffs
+        ld1             {v0.8h}, [x16]
+
+        ssubl           v2.4s,   v16.4h,  v18.4h
+        ssubl2          v3.4s,   v16.8h,  v18.8h
+        smull           v4.4s,   v16.4h,  v0.h[0]
+        smlal           v4.4s,   v18.4h,  v0.h[1]
+        smlal           v4.4s,   v19.4h,  v0.h[2]
+        smull2          v5.4s,   v16.8h,  v0.h[0]
+        smlal2          v5.4s,   v18.8h,  v0.h[1]
+        smlal2          v5.4s,   v19.8h,  v0.h[2]
+        saddw           v2.4s,   v2.4s,   v19.4h
+        saddw2          v3.4s,   v3.4s,   v19.8h
+        smull           v6.4s,   v16.4h,  v0.h[2]
+        smlsl           v6.4s,   v18.4h,  v0.h[0]
+        smlsl           v6.4s,   v19.4h,  v0.h[1]
+        smull2          v7.4s,   v16.8h,  v0.h[2]
+        smlsl2          v7.4s,   v18.8h,  v0.h[0]
+        smlsl2          v7.4s,   v19.8h,  v0.h[1]
+
+        mul             v18.4s,  v2.4s,   v0.s[2]
+        mul             v19.4s,  v3.4s,   v0.s[2]
+
+        smull           v2.4s,   v17.4h,  v0.h[3]
+        smull2          v3.4s,   v17.8h,  v0.h[3]
+
+        add             v16.4s,  v4.4s,   v2.4s // out0
+        add             v17.4s,  v5.4s,   v3.4s
+
+        add             v4.4s,   v4.4s,   v6.4s // out3
+        add             v5.4s,   v5.4s,   v7.4s
+
+        add             v6.4s,   v6.4s,   v2.4s // out1
+        add             v7.4s,   v7.4s,   v3.4s
+
+        sub             v4.4s,   v4.4s,   v2.4s // out3
+        sub             v5.4s,   v5.4s,   v3.4s
+
+        sqrshrn         v18.4h,  v18.4s, #12
+        sqrshrn2        v18.8h,  v19.4s, #12
+
+        sqrshrn         \o0\().4h, v16.4s, #12
+        sqrshrn2        \o0\().8h, v17.4s, #12
+
+.ifc \o2, v17
+        mov             v17.16b,   v18.16b
+.endif
+
+        sqrshrn         \o1\().4h, v6.4s,  #12
+        sqrshrn2        \o1\().8h, v7.4s,  #12
+
+        sqrshrn         \o3\().4h, v4.4s,  #12
+        sqrshrn2        \o3\().8h, v5.4s,  #12
+.endm
+
+function inv_adst_8h_x4_neon, export=1
+        iadst_8x4       v16, v17, v18, v19
+        ret
+endfunc
+
+function inv_flipadst_8h_x4_neon, export=1
+        iadst_8x4       v19, v18, v17, v16
+        ret
+endfunc
+
+function inv_identity_4h_x4_neon, export=1
+        mov             w16, #(5793-4096)*8
+        dup             v0.4h,   w16
+        sqrdmulh        v4.4h,   v16.4h,  v0.h[0]
+        sqrdmulh        v5.4h,   v17.4h,  v0.h[0]
+        sqrdmulh        v6.4h,   v18.4h,  v0.h[0]
+        sqrdmulh        v7.4h,   v19.4h,  v0.h[0]
+        sqadd           v16.4h,  v16.4h,  v4.4h
+        sqadd           v17.4h,  v17.4h,  v5.4h
+        sqadd           v18.4h,  v18.4h,  v6.4h
+        sqadd           v19.4h,  v19.4h,  v7.4h
+        ret
+endfunc
+
+function inv_identity_8h_x4_neon, export=1
+        mov             w16, #(5793-4096)*8
+        dup             v0.4h,   w16
+        sqrdmulh        v4.8h,   v16.8h,  v0.h[0]
+        sqrdmulh        v5.8h,   v17.8h,  v0.h[0]
+        sqrdmulh        v6.8h,   v18.8h,  v0.h[0]
+        sqrdmulh        v7.8h,   v19.8h,  v0.h[0]
+        sqadd           v16.8h,  v16.8h,  v4.8h
+        sqadd           v17.8h,  v17.8h,  v5.8h
+        sqadd           v18.8h,  v18.8h,  v6.8h
+        sqadd           v19.8h,  v19.8h,  v7.8h
+        ret
+endfunc
+
+.macro identity_8x4_shift1 r0, r1, r2, r3, c
+.irp i, \r0\().8h, \r1\().8h, \r2\().8h, \r3\().8h
+        sqrdmulh        v2.8h,  \i,  \c
+        srhadd          \i,     \i,  v2.8h
+.endr
+.endm
+
+function inv_txfm_add_wht_wht_4x4_8bpc_neon, export=1
+        mov             x15, x30
+        movi            v31.8h,  #0
+        ld1             {v16.4h,v17.4h,v18.4h,v19.4h}, [x2]
+        st1             {v31.8h}, [x2], #16
+
+        sshr            v16.4h,  v16.4h,  #2
+        sshr            v17.4h,  v17.4h,  #2
+        sshr            v18.4h,  v18.4h,  #2
+        sshr            v19.4h,  v19.4h,  #2
+
+        iwht4
+
+        st1             {v31.8h}, [x2], #16
+        transpose_4x4h  v16, v17, v18, v19, v20, v21, v22, v23
+
+        iwht4
+
+        ld1             {v0.s}[0], [x0], x1
+        ld1             {v0.s}[1], [x0], x1
+        ins             v16.d[1], v17.d[0]
+        ins             v18.d[1], v19.d[0]
+        ld1             {v1.s}[0], [x0], x1
+        ld1             {v1.s}[1], [x0], x1
+
+        b               L(itx_4x4_end)
+endfunc
+
+function inv_txfm_add_4x4_neon
+        movi            v31.8h,  #0
+        ld1             {v16.4h,v17.4h,v18.4h,v19.4h}, [x2]
+        st1             {v31.8h}, [x2], #16
+
+        blr             x4
+
+        st1             {v31.8h}, [x2], #16
+        transpose_4x4h  v16, v17, v18, v19, v20, v21, v22, v23
+
+        blr             x5
+
+        ld1             {v0.s}[0], [x0], x1
+        ld1             {v0.s}[1], [x0], x1
+        ins             v16.d[1], v17.d[0]
+        ins             v18.d[1], v19.d[0]
+        ld1             {v1.s}[0], [x0], x1
+        ld1             {v1.s}[1], [x0], x1
+        srshr           v16.8h,  v16.8h,  #4
+        srshr           v18.8h,  v18.8h,  #4
+
+L(itx_4x4_end):
+        sub             x0,  x0,  x1, lsl #2
+        uaddw           v16.8h,  v16.8h,  v0.8b
+        sqxtun          v0.8b,   v16.8h
+        uaddw           v18.8h,  v18.8h,  v1.8b
+        st1             {v0.s}[0], [x0], x1
+        sqxtun          v1.8b,   v18.8h
+        st1             {v0.s}[1], [x0], x1
+        st1             {v1.s}[0], [x0], x1
+        st1             {v1.s}[1], [x0], x1
+
+        ret             x15
+endfunc
+
+.macro def_fn_4x4 txfm1, txfm2
+function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_8bpc_neon, export=1
+        mov             x15, x30
+
+.ifc \txfm1\()_\txfm2, dct_dct
+        cbnz            w3,  1f
+        mov             w16, #2896*8
+        ld1r            {v16.8h}, [x2]
+        dup             v4.8h,   w16
+        strh            wzr, [x2]
+        sqrdmulh        v16.8h,  v16.8h,  v4.h[0]
+        ld1             {v0.s}[0], [x0], x1
+        sqrdmulh        v20.8h,  v16.8h,  v4.h[0]
+        ld1             {v0.s}[1], [x0], x1
+        srshr           v16.8h,  v20.8h,  #4
+        ld1             {v1.s}[0], [x0], x1
+        srshr           v18.8h,  v20.8h,  #4
+        ld1             {v1.s}[1], [x0], x1
+        b               L(itx_4x4_end)
+1:
+.endif
+        adr             x4,  inv_\txfm1\()_4h_x4_neon
+        adr             x5,  inv_\txfm2\()_4h_x4_neon
+        b               inv_txfm_add_4x4_neon
+endfunc
+.endm
+
+def_fn_4x4 dct, dct
+def_fn_4x4 identity, identity
+def_fn_4x4 dct, adst
+def_fn_4x4 dct, flipadst
+def_fn_4x4 dct, identity
+def_fn_4x4 adst, dct
+def_fn_4x4 adst, adst
+def_fn_4x4 adst, flipadst
+def_fn_4x4 flipadst, dct
+def_fn_4x4 flipadst, adst
+def_fn_4x4 flipadst, flipadst
+def_fn_4x4 identity, dct
+
+def_fn_4x4 adst, identity
+def_fn_4x4 flipadst, identity
+def_fn_4x4 identity, adst
+def_fn_4x4 identity, flipadst
+
+.macro idct_8 r0, r1, r2, r3, r4, r5, r6, r7, sz, szb
+        idct_4          \r0, \r2, \r4, \r6, \sz
+
+        smull_smlsl     v2,  v3,  \r1, \r7, v0.h[4], v0.h[5], \sz // -> t4a
+        smull_smlal     v4,  v5,  \r1, \r7, v0.h[5], v0.h[4], \sz // -> t7a
+        smull_smlsl     v6,  v7,  \r5, \r3, v0.h[6], v0.h[7], \sz // -> t5a
+        sqrshrn_sz      \r1, v2,  v3,  #12, \sz                   // t4a
+        sqrshrn_sz      \r7, v4,  v5,  #12, \sz                   // t7a
+        smull_smlal     v2,  v3,  \r5, \r3, v0.h[7], v0.h[6], \sz // -> t6a
+        sqrshrn_sz      \r3, v6,  v7,  #12, \sz                   // t5a
+        sqrshrn_sz      \r5, v2,  v3,  #12, \sz                   // t6a
+
+        sqadd           v2\sz,   \r1\sz,  \r3\sz // t4
+        sqsub           \r1\sz,  \r1\sz,  \r3\sz // t5a
+        sqadd           v3\sz,   \r7\sz,  \r5\sz // t7
+        sqsub           \r3\sz,  \r7\sz,  \r5\sz // t6a
+
+        smull_smlsl     v4,  v5,  \r3, \r1, v0.h[0], v0.h[0], \sz // -> t5
+        smull_smlal     v6,  v7,  \r3, \r1, v0.h[0], v0.h[0], \sz // -> t6
+        sqrshrn_sz      v4,  v4,  v5,  #12, \sz // t5
+        sqrshrn_sz      v5,  v6,  v7,  #12, \sz // t6
+
+        sqsub           \r7\sz,  \r0\sz,  v3\sz // out7
+        sqadd           \r0\sz,  \r0\sz,  v3\sz // out0
+        sqadd           \r1\sz,  \r2\sz,  v5\sz // out1
+        sqsub           v6\sz,   \r2\sz,  v5\sz // out6
+        sqadd           \r2\sz,  \r4\sz,  v4\sz // out2
+        sqsub           \r5\sz,  \r4\sz,  v4\sz // out5
+        sqadd           \r3\sz,  \r6\sz,  v2\sz // out3
+        sqsub           \r4\sz,  \r6\sz,  v2\sz // out4
+        mov             \r6\szb, v6\szb         // out6
+.endm
+
+function inv_dct_8h_x8_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.8h}, [x16]
+        idct_8          v16, v17, v18, v19, v20, v21, v22, v23, .8h, .16b
+        ret
+endfunc
+
+function inv_dct_4h_x8_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.8h}, [x16]
+        idct_8          v16, v17, v18, v19, v20, v21, v22, v23, .4h, .8b
+        ret
+endfunc
+
+.macro iadst_8 o0, o1, o2, o3, o4, o5, o6, o7, sz
+        movrel          x16, iadst8_coeffs
+        ld1             {v0.8h, v1.8h}, [x16]
+
+        smull_smlal     v2,  v3,  v23, v16, v0.h[0], v0.h[1], \sz
+        smull_smlsl     v4,  v5,  v23, v16, v0.h[1], v0.h[0], \sz
+        smull_smlal     v6,  v7,  v21, v18, v0.h[2], v0.h[3], \sz
+        sqrshrn_sz      v16, v2,  v3,  #12, \sz  // t0a
+        sqrshrn_sz      v23, v4,  v5,  #12, \sz  // t1a
+        smull_smlsl     v2,  v3,  v21, v18, v0.h[3], v0.h[2], \sz
+        smull_smlal     v4,  v5,  v19, v20, v0.h[4], v0.h[5], \sz
+        sqrshrn_sz      v18, v6,  v7,  #12, \sz  // t2a
+        sqrshrn_sz      v21, v2,  v3,  #12, \sz  // t3a
+        smull_smlsl     v6,  v7,  v19, v20, v0.h[5], v0.h[4], \sz
+        smull_smlal     v2,  v3,  v17, v22, v0.h[6], v0.h[7], \sz
+        sqrshrn_sz      v20, v4,  v5,  #12, \sz  // t4a
+        sqrshrn_sz      v19, v6,  v7,  #12, \sz  // t5a
+        smull_smlsl     v4,  v5,  v17, v22, v0.h[7], v0.h[6], \sz
+        sqrshrn_sz      v22, v2,  v3,  #12, \sz  // t6a
+        sqrshrn_sz      v17, v4,  v5,  #12, \sz  // t7a
+
+        sqadd           v2\sz,   v16\sz,  v20\sz // t0
+        sqsub           v3\sz,   v16\sz,  v20\sz // t4
+        sqadd           v4\sz,   v23\sz,  v19\sz // t1
+        sqsub           v5\sz,   v23\sz,  v19\sz // t5
+        sqadd           v6\sz,   v18\sz,  v22\sz // t2
+        sqsub           v7\sz,   v18\sz,  v22\sz // t6
+        sqadd           v18\sz,  v21\sz,  v17\sz // t3
+        sqsub           v19\sz,  v21\sz,  v17\sz // t7
+
+        smull_smlal     v16, v17, v3,  v5,  v1.h[3], v1.h[2], \sz
+        smull_smlsl     v20, v21, v3,  v5,  v1.h[2], v1.h[3], \sz
+        smull_smlsl     v22, v23, v19, v7,  v1.h[3], v1.h[2], \sz
+
+        sqrshrn_sz      v3,  v16, v17, #12, \sz  // t4a
+        sqrshrn_sz      v5,  v20, v21, #12, \sz  // t5a
+
+        smull_smlal     v16, v17, v19, v7,  v1.h[2], v1.h[3], \sz
+
+        sqrshrn_sz      v7,  v22, v23, #12, \sz  // t6a
+        sqrshrn_sz      v19, v16, v17, #12, \sz  // t7a
+
+        sqadd           \o0\()\sz, v2\sz, v6\sz  // out0
+        sqsub           v2\sz,     v2\sz, v6\sz  // t2
+        sqadd           \o7\()\sz, v4\sz, v18\sz // out7
+        sqsub           v4\sz,     v4\sz, v18\sz // t3
+        sqneg           \o7\()\sz, \o7\()\sz     // out7
+
+        sqadd           \o1\()\sz, v3\sz, v7\sz  // out1
+        sqsub           v3\sz,     v3\sz, v7\sz  // t6
+        sqadd           \o6\()\sz, v5\sz, v19\sz // out6
+        sqsub           v5\sz,     v5\sz, v19\sz // t7
+        sqneg           \o1\()\sz, \o1\()\sz     // out1
+
+        smull_smlal     v18, v19, v2,  v4,  v1.h[0], v1.h[0], \sz // -> out3 (v19 or v20)
+        smull_smlsl     v6,  v7,  v2,  v4,  v1.h[0], v1.h[0], \sz // -> out4 (v20 or v19)
+        smull_smlsl     v20, v21, v3,  v5,  v1.h[0], v1.h[0], \sz // -> out5 (v21 or v18)
+        sqrshrn_sz      v2,  v18, v19, #12, \sz // out3
+        smull_smlal     v18, v19, v3,  v5,  v1.h[0], v1.h[0], \sz // -> out2 (v18 or v21)
+        sqrshrn_sz      v3,  v20, v21, #12, \sz // out5
+        sqrshrn_sz      \o2, v18, v19, #12, \sz // out2 (v18 or v21)
+        sqrshrn_sz      \o4, v6,  v7,  #12, \sz // out4 (v20 or v19)
+
+        sqneg           \o3\()\sz, v2\sz     // out3
+        sqneg           \o5\()\sz, v3\sz     // out5
+.endm
+
+function inv_adst_8h_x8_neon, export=1
+        iadst_8         v16, v17, v18, v19, v20, v21, v22, v23, .8h
+        ret
+endfunc
+
+function inv_flipadst_8h_x8_neon, export=1
+        iadst_8         v23, v22, v21, v20, v19, v18, v17, v16, .8h
+        ret
+endfunc
+
+function inv_adst_4h_x8_neon, export=1
+        iadst_8         v16, v17, v18, v19, v20, v21, v22, v23, .4h
+        ret
+endfunc
+
+function inv_flipadst_4h_x8_neon, export=1
+        iadst_8         v23, v22, v21, v20, v19, v18, v17, v16, .4h
+        ret
+endfunc
+
+function inv_identity_8h_x8_neon, export=1
+        sqshl           v16.8h,  v16.8h,  #1
+        sqshl           v17.8h,  v17.8h,  #1
+        sqshl           v18.8h,  v18.8h,  #1
+        sqshl           v19.8h,  v19.8h,  #1
+        sqshl           v20.8h,  v20.8h,  #1
+        sqshl           v21.8h,  v21.8h,  #1
+        sqshl           v22.8h,  v22.8h,  #1
+        sqshl           v23.8h,  v23.8h,  #1
+        ret
+endfunc
+
+function inv_identity_4h_x8_neon, export=1
+        sqshl           v16.4h,  v16.4h,  #1
+        sqshl           v17.4h,  v17.4h,  #1
+        sqshl           v18.4h,  v18.4h,  #1
+        sqshl           v19.4h,  v19.4h,  #1
+        sqshl           v20.4h,  v20.4h,  #1
+        sqshl           v21.4h,  v21.4h,  #1
+        sqshl           v22.4h,  v22.4h,  #1
+        sqshl           v23.4h,  v23.4h,  #1
+        ret
+endfunc
+
+.macro def_fn_8x8_base variant
+function inv_txfm_\variant\()add_8x8_neon
+        movi            v28.8h,  #0
+        movi            v29.8h,  #0
+        movi            v30.8h,  #0
+        movi            v31.8h,  #0
+        ld1             {v16.8h,v17.8h,v18.8h,v19.8h}, [x2]
+        st1             {v28.8h,v29.8h,v30.8h,v31.8h}, [x2], #64
+        ld1             {v20.8h,v21.8h,v22.8h,v23.8h}, [x2]
+        st1             {v28.8h,v29.8h,v30.8h,v31.8h}, [x2]
+
+.ifc \variant, identity_
+        // The identity shl #1 and downshift srshr #1 cancel out
+.else
+        blr             x4
+
+        srshr           v16.8h,  v16.8h,  #1
+        srshr           v17.8h,  v17.8h,  #1
+        srshr           v18.8h,  v18.8h,  #1
+        srshr           v19.8h,  v19.8h,  #1
+        srshr           v20.8h,  v20.8h,  #1
+        srshr           v21.8h,  v21.8h,  #1
+        srshr           v22.8h,  v22.8h,  #1
+        srshr           v23.8h,  v23.8h,  #1
+.endif
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v24, v25
+
+        blr             x5
+
+        load_add_store_8x8 x0, x7
+        ret             x15
+endfunc
+.endm
+
+def_fn_8x8_base
+def_fn_8x8_base identity_
+
+.macro def_fn_8x8 txfm1, txfm2
+function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_8bpc_neon, export=1
+        mov             x15, x30
+
+.ifc \txfm1\()_\txfm2, dct_dct
+        idct_dc         8,   8,   1
+.endif
+        adr             x5,  inv_\txfm2\()_8h_x8_neon
+.ifc \txfm1, identity
+        b               inv_txfm_identity_add_8x8_neon
+.else
+        adr             x4,  inv_\txfm1\()_8h_x8_neon
+        b               inv_txfm_add_8x8_neon
+.endif
+endfunc
+.endm
+
+def_fn_8x8 dct, dct
+def_fn_8x8 identity, identity
+def_fn_8x8 dct, adst
+def_fn_8x8 dct, flipadst
+def_fn_8x8 dct, identity
+def_fn_8x8 adst, dct
+def_fn_8x8 adst, adst
+def_fn_8x8 adst, flipadst
+def_fn_8x8 flipadst, dct
+def_fn_8x8 flipadst, adst
+def_fn_8x8 flipadst, flipadst
+def_fn_8x8 identity, dct
+def_fn_8x8 adst, identity
+def_fn_8x8 flipadst, identity
+def_fn_8x8 identity, adst
+def_fn_8x8 identity, flipadst
+
+function inv_txfm_add_8x4_neon
+        movi            v30.8h,  #0
+        movi            v31.8h,  #0
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+        ld1             {v16.4h,v17.4h,v18.4h,v19.4h}, [x2]
+        st1             {v30.8h,v31.8h}, [x2], #32
+        ld1             {v20.4h,v21.4h,v22.4h,v23.4h}, [x2]
+        st1             {v30.8h,v31.8h}, [x2]
+
+        scale_input     .4h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+
+        blr             x4
+
+        transpose_4x4h  v16, v17, v18, v19, v4,  v5,  v6,  v7
+        transpose_4x4h  v20, v21, v22, v23, v4,  v5,  v6,  v7
+        ins             v16.d[1], v20.d[0]
+        ins             v17.d[1], v21.d[0]
+        ins             v18.d[1], v22.d[0]
+        ins             v19.d[1], v23.d[0]
+
+        blr             x5
+
+        load_add_store_8x4 x0, x7
+        ret             x15
+endfunc
+
+function inv_txfm_add_4x8_neon
+        movi            v28.8h,  #0
+        movi            v29.8h,  #0
+        movi            v30.8h,  #0
+        movi            v31.8h,  #0
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+        ld1             {v16.8h,v17.8h,v18.8h,v19.8h}, [x2]
+        st1             {v28.8h,v29.8h,v30.8h,v31.8h}, [x2]
+
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19
+
+        blr             x4
+
+        transpose_4x8h  v16, v17, v18, v19, v4,  v5,  v6,  v7
+        ins             v20.d[0], v16.d[1]
+        ins             v21.d[0], v17.d[1]
+        ins             v22.d[0], v18.d[1]
+        ins             v23.d[0], v19.d[1]
+
+        blr             x5
+
+        load_add_store_4x8 x0, x7
+        ret             x15
+endfunc
+
+.macro def_fn_48 w, h, txfm1, txfm2
+function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1
+        mov             x15, x30
+
+.ifc \txfm1\()_\txfm2, dct_dct
+        idct_dc         \w,  \h,  0
+.endif
+        adr             x4,  inv_\txfm1\()_\h\()h_x\w\()_neon
+        adr             x5,  inv_\txfm2\()_\w\()h_x\h\()_neon
+        b               inv_txfm_add_\w\()x\h\()_neon
+endfunc
+.endm
+
+.macro def_fns_48 w, h
+def_fn_48 \w, \h, dct, dct
+def_fn_48 \w, \h, identity, identity
+def_fn_48 \w, \h, dct, adst
+def_fn_48 \w, \h, dct, flipadst
+def_fn_48 \w, \h, dct, identity
+def_fn_48 \w, \h, adst, dct
+def_fn_48 \w, \h, adst, adst
+def_fn_48 \w, \h, adst, flipadst
+def_fn_48 \w, \h, flipadst, dct
+def_fn_48 \w, \h, flipadst, adst
+def_fn_48 \w, \h, flipadst, flipadst
+def_fn_48 \w, \h, identity, dct
+def_fn_48 \w, \h, adst, identity
+def_fn_48 \w, \h, flipadst, identity
+def_fn_48 \w, \h, identity, adst
+def_fn_48 \w, \h, identity, flipadst
+.endm
+
+def_fns_48 4, 8
+def_fns_48 8, 4
+
+
+.macro idct_16 sz, szb
+        idct_8          v16, v18, v20, v22, v24, v26, v28, v30, \sz, \szb
+
+        smull_smlsl     v2,  v3,  v17, v31, v1.h[0], v1.h[1], \sz // -> t8a
+        smull_smlal     v4,  v5,  v17, v31, v1.h[1], v1.h[0], \sz // -> t15a
+        smull_smlsl     v6,  v7,  v25, v23, v1.h[2], v1.h[3], \sz // -> t9a
+        sqrshrn_sz      v17, v2,  v3,  #12, \sz                   // t8a
+        sqrshrn_sz      v31, v4,  v5,  #12, \sz                   // t15a
+        smull_smlal     v2,  v3,  v25, v23, v1.h[3], v1.h[2], \sz // -> t14a
+        smull_smlsl     v4,  v5,  v21, v27, v1.h[4], v1.h[5], \sz // -> t10a
+        sqrshrn_sz      v23, v6,  v7,  #12, \sz                   // t9a
+        sqrshrn_sz      v25, v2,  v3,  #12, \sz                   // t14a
+        smull_smlal     v6,  v7,  v21, v27, v1.h[5], v1.h[4], \sz // -> t13a
+        smull_smlsl     v2,  v3,  v29, v19, v1.h[6], v1.h[7], \sz // -> t11a
+        sqrshrn_sz      v21, v4,  v5,  #12, \sz                   // t10a
+        sqrshrn_sz      v27, v6,  v7,  #12, \sz                   // t13a
+        smull_smlal     v4,  v5,  v29, v19, v1.h[7], v1.h[6], \sz // -> t12a
+        sqrshrn_sz      v19, v2,  v3,  #12, \sz                   // t11a
+        sqrshrn_sz      v29, v4,  v5,  #12, \sz                   // t12a
+
+        sqsub           v2\sz,   v17\sz,  v23\sz  // t9
+        sqadd           v17\sz,  v17\sz,  v23\sz  // t8
+        sqsub           v3\sz,   v31\sz,  v25\sz  // t14
+        sqadd           v31\sz,  v31\sz,  v25\sz  // t15
+        sqsub           v23\sz,  v19\sz,  v21\sz  // t10
+        sqadd           v19\sz,  v19\sz,  v21\sz  // t11
+        sqadd           v25\sz,  v29\sz,  v27\sz  // t12
+        sqsub           v29\sz,  v29\sz,  v27\sz  // t13
+
+        smull_smlsl     v4,  v5,  v3,  v2,  v0.h[2], v0.h[3], \sz // -> t9a
+        smull_smlal     v6,  v7,  v3,  v2,  v0.h[3], v0.h[2], \sz // -> t14a
+        sqrshrn_sz      v21, v4,  v5,  #12, \sz                   // t9a
+        sqrshrn_sz      v27, v6,  v7,  #12, \sz                   // t14a
+
+        smull_smlsl     v4,  v5,  v29, v23, v0.h[2], v0.h[3], \sz // -> t13a
+        smull_smlal     v6,  v7,  v29, v23, v0.h[3], v0.h[2], \sz // -> t10a
+        sqrshrn_sz      v29, v4,  v5,  #12, \sz                   // t13a
+        neg             v6.4s,   v6.4s
+.ifc \sz, .8h
+        neg             v7.4s,   v7.4s
+.endif
+        sqrshrn_sz      v23, v6,  v7,  #12, \sz                   // t10a
+
+        sqsub           v2\sz,   v17\sz,  v19\sz  // t11a
+        sqadd           v17\sz,  v17\sz,  v19\sz  // t8a
+        sqsub           v3\sz,   v31\sz,  v25\sz  // t12a
+        sqadd           v31\sz,  v31\sz,  v25\sz  // t15a
+        sqadd           v19\sz,  v21\sz,  v23\sz  // t9
+        sqsub           v21\sz,  v21\sz,  v23\sz  // t10
+        sqsub           v25\sz,  v27\sz,  v29\sz  // t13
+        sqadd           v27\sz,  v27\sz,  v29\sz  // t14
+
+        smull_smlsl     v4,  v5,  v3,  v2,  v0.h[0], v0.h[0], \sz // -> t11
+        smull_smlal     v6,  v7,  v3,  v2,  v0.h[0], v0.h[0], \sz // -> t12
+        smull_smlsl     v2,  v3,  v25, v21, v0.h[0], v0.h[0], \sz // -> t10a
+
+        sqrshrn_sz      v4,  v4,  v5,  #12, \sz   // t11
+        sqrshrn_sz      v5,  v6,  v7,  #12, \sz   // t12
+        smull_smlal     v6,  v7,  v25, v21, v0.h[0], v0.h[0], \sz // -> t13a
+        sqrshrn_sz      v2,  v2,  v3,  #12, \sz   // t10a
+        sqrshrn_sz      v3,  v6,  v7,  #12, \sz   // t13a
+
+        sqadd           v6\sz,   v16\sz,  v31\sz  // out0
+        sqsub           v31\sz,  v16\sz,  v31\sz  // out15
+        mov             v16\szb, v6\szb
+        sqadd           v23\sz,  v30\sz,  v17\sz  // out7
+        sqsub           v7\sz,   v30\sz,  v17\sz  // out8
+        sqadd           v17\sz,  v18\sz,  v27\sz  // out1
+        sqsub           v30\sz,  v18\sz,  v27\sz  // out14
+        sqadd           v18\sz,  v20\sz,  v3\sz   // out2
+        sqsub           v29\sz,  v20\sz,  v3\sz   // out13
+        sqadd           v3\sz,   v28\sz,  v19\sz  // out6
+        sqsub           v25\sz,  v28\sz,  v19\sz  // out9
+        sqadd           v19\sz,  v22\sz,  v5\sz   // out3
+        sqsub           v28\sz,  v22\sz,  v5\sz   // out12
+        sqadd           v20\sz,  v24\sz,  v4\sz   // out4
+        sqsub           v27\sz,  v24\sz,  v4\sz   // out11
+        sqadd           v21\sz,  v26\sz,  v2\sz   // out5
+        sqsub           v26\sz,  v26\sz,  v2\sz   // out10
+        mov             v24\szb, v7\szb
+        mov             v22\szb, v3\szb
+.endm
+
+function inv_dct_8h_x16_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.8h, v1.8h}, [x16]
+        idct_16         .8h, .16b
+        ret
+endfunc
+
+function inv_dct_4h_x16_neon, export=1
+        movrel          x16, idct_coeffs
+        ld1             {v0.8h, v1.8h}, [x16]
+        idct_16         .4h, .8b
+        ret
+endfunc
+
+.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, sz, szb
+        movrel          x16, iadst16_coeffs
+        ld1             {v0.8h, v1.8h}, [x16]
+        movrel          x16, idct_coeffs
+
+        smull_smlal     v2,  v3,  v31, v16, v0.h[0], v0.h[1], \sz // -> t0
+        smull_smlsl     v4,  v5,  v31, v16, v0.h[1], v0.h[0], \sz // -> t1
+        smull_smlal     v6,  v7,  v29, v18, v0.h[2], v0.h[3], \sz // -> t2
+        sqrshrn_sz      v16, v2,  v3,  #12, \sz   // t0
+        sqrshrn_sz      v31, v4,  v5,  #12, \sz   // t1
+        smull_smlsl     v2,  v3,  v29, v18, v0.h[3], v0.h[2], \sz // -> t3
+        smull_smlal     v4,  v5,  v27, v20, v0.h[4], v0.h[5], \sz // -> t4
+        sqrshrn_sz      v18, v6,  v7,  #12, \sz   // t2
+        sqrshrn_sz      v29, v2,  v3,  #12, \sz   // t3
+        smull_smlsl     v6,  v7,  v27, v20, v0.h[5], v0.h[4], \sz // -> t5
+        smull_smlal     v2,  v3,  v25, v22, v0.h[6], v0.h[7], \sz // -> t6
+        sqrshrn_sz      v20, v4,  v5,  #12, \sz   // t4
+        sqrshrn_sz      v27, v6,  v7,  #12, \sz   // t5
+        smull_smlsl     v4,  v5,  v25, v22, v0.h[7], v0.h[6], \sz // -> t7
+        smull_smlal     v6,  v7,  v23, v24, v1.h[0], v1.h[1], \sz // -> t8
+        sqrshrn_sz      v22, v2,  v3,  #12, \sz   // t6
+        sqrshrn_sz      v25, v4,  v5,  #12, \sz   // t7
+        smull_smlsl     v2,  v3,  v23, v24, v1.h[1], v1.h[0], \sz // -> t9
+        smull_smlal     v4,  v5,  v21, v26, v1.h[2], v1.h[3], \sz // -> t10
+        sqrshrn_sz      v23, v6,  v7,  #12, \sz   // t8
+        sqrshrn_sz      v24, v2,  v3,  #12, \sz   // t9
+        smull_smlsl     v6,  v7,  v21, v26, v1.h[3], v1.h[2], \sz // -> t11
+        smull_smlal     v2,  v3,  v19, v28, v1.h[4], v1.h[5], \sz // -> t12
+        sqrshrn_sz      v21, v4,  v5,  #12, \sz   // t10
+        sqrshrn_sz      v26, v6,  v7,  #12, \sz   // t11
+        smull_smlsl     v4,  v5,  v19, v28, v1.h[5], v1.h[4], \sz // -> t13
+        smull_smlal     v6,  v7,  v17, v30, v1.h[6], v1.h[7], \sz // -> t14
+        sqrshrn_sz      v19, v2,  v3,  #12, \sz   // t12
+        sqrshrn_sz      v28, v4,  v5,  #12, \sz   // t13
+        smull_smlsl     v2,  v3,  v17, v30, v1.h[7], v1.h[6], \sz // -> t15
+        sqrshrn_sz      v17, v6,  v7,  #12, \sz   // t14
+        sqrshrn_sz      v30, v2,  v3,  #12, \sz   // t15
+
+        ld1             {v0.8h}, [x16]
+
+        sqsub           v2\sz,   v16\sz,  v23\sz // t8a
+        sqadd           v16\sz,  v16\sz,  v23\sz // t0a
+        sqsub           v3\sz,   v31\sz,  v24\sz // t9a
+        sqadd           v31\sz,  v31\sz,  v24\sz // t1a
+        sqadd           v23\sz,  v18\sz,  v21\sz // t2a
+        sqsub           v18\sz,  v18\sz,  v21\sz // t10a
+        sqadd           v24\sz,  v29\sz,  v26\sz // t3a
+        sqsub           v29\sz,  v29\sz,  v26\sz // t11a
+        sqadd           v21\sz,  v20\sz,  v19\sz // t4a
+        sqsub           v20\sz,  v20\sz,  v19\sz // t12a
+        sqadd           v26\sz,  v27\sz,  v28\sz // t5a
+        sqsub           v27\sz,  v27\sz,  v28\sz // t13a
+        sqadd           v19\sz,  v22\sz,  v17\sz // t6a
+        sqsub           v22\sz,  v22\sz,  v17\sz // t14a
+        sqadd           v28\sz,  v25\sz,  v30\sz // t7a
+        sqsub           v25\sz,  v25\sz,  v30\sz // t15a
+
+        smull_smlal     v4,  v5,  v2,  v3,  v0.h[5], v0.h[4], \sz // -> t8
+        smull_smlsl     v6,  v7,  v2,  v3,  v0.h[4], v0.h[5], \sz // -> t9
+        smull_smlal     v2,  v3,  v18, v29, v0.h[7], v0.h[6], \sz // -> t10
+        sqrshrn_sz      v17, v4,  v5,  #12, \sz  // t8
+        sqrshrn_sz      v30, v6,  v7,  #12, \sz  // t9
+        smull_smlsl     v4,  v5,  v18, v29, v0.h[6], v0.h[7], \sz // -> t11
+        smull_smlsl     v6,  v7,  v27, v20, v0.h[5], v0.h[4], \sz // -> t12
+        sqrshrn_sz      v18, v2,  v3,  #12, \sz  // t10
+        sqrshrn_sz      v29, v4,  v5,  #12, \sz  // t11
+        smull_smlal     v2,  v3,  v27, v20, v0.h[4], v0.h[5], \sz // -> t13
+        smull_smlsl     v4,  v5,  v25, v22, v0.h[7], v0.h[6], \sz // -> t14
+        sqrshrn_sz      v27, v6,  v7,  #12, \sz  // t12
+        sqrshrn_sz      v20, v2,  v3,  #12, \sz  // t13
+        smull_smlal     v6,  v7,  v25, v22, v0.h[6], v0.h[7], \sz // -> t15
+        sqrshrn_sz      v25, v4,  v5,  #12, \sz  // t14
+        sqrshrn_sz      v22, v6,  v7,  #12, \sz  // t15
+
+        sqsub           v2\sz,   v16\sz,  v21\sz // t4
+        sqadd           v16\sz,  v16\sz,  v21\sz // t0
+        sqsub           v3\sz,   v31\sz,  v26\sz // t5
+        sqadd           v31\sz,  v31\sz,  v26\sz // t1
+        sqadd           v21\sz,  v23\sz,  v19\sz // t2
+        sqsub           v23\sz,  v23\sz,  v19\sz // t6
+        sqadd           v26\sz,  v24\sz,  v28\sz // t3
+        sqsub           v24\sz,  v24\sz,  v28\sz // t7
+        sqadd           v19\sz,  v17\sz,  v27\sz // t8a
+        sqsub           v17\sz,  v17\sz,  v27\sz // t12a
+        sqadd           v28\sz,  v30\sz,  v20\sz // t9a
+        sqsub           v30\sz,  v30\sz,  v20\sz // t13a
+        sqadd           v27\sz,  v18\sz,  v25\sz // t10a
+        sqsub           v18\sz,  v18\sz,  v25\sz // t14a
+        sqadd           v20\sz,  v29\sz,  v22\sz // t11a
+        sqsub           v29\sz,  v29\sz,  v22\sz // t15a
+
+        smull_smlal     v4,  v5,  v2,  v3,  v0.h[3], v0.h[2], \sz // -> t4a
+        smull_smlsl     v6,  v7,  v2,  v3,  v0.h[2], v0.h[3], \sz // -> t5a
+        smull_smlsl     v2,  v3,  v24, v23, v0.h[3], v0.h[2], \sz // -> t6a
+        sqrshrn_sz      v22, v4,  v5,  #12, \sz // t4a
+        sqrshrn_sz      v25, v6,  v7,  #12, \sz // t5a
+        smull_smlal     v4,  v5,  v24, v23, v0.h[2], v0.h[3], \sz // -> t7a
+        smull_smlal     v6,  v7,  v17, v30, v0.h[3], v0.h[2], \sz // -> t12
+        sqrshrn_sz      v24, v2,  v3,  #12, \sz // t6a
+        sqrshrn_sz      v23, v4,  v5,  #12, \sz // t7a
+        smull_smlsl     v2,  v3,  v17, v30, v0.h[2], v0.h[3], \sz // -> t13
+        smull_smlsl     v4,  v5,  v29, v18, v0.h[3], v0.h[2], \sz // -> t14
+        sqrshrn_sz      v17, v6,  v7,  #12, \sz // t12
+        smull_smlal     v6,  v7,  v29, v18, v0.h[2], v0.h[3], \sz // -> t15
+        sqrshrn_sz      v29, v2,  v3,  #12, \sz // t13
+        sqrshrn_sz      v30, v4,  v5,  #12, \sz // t14
+        sqrshrn_sz      v18, v6,  v7,  #12, \sz // t15
+
+        sqsub           v2\sz,   v16\sz,  v21\sz // t2a
+.ifc \o0, v16
+        sqadd           \o0\sz,  v16\sz,  v21\sz // out0
+        sqsub           v21\sz,  v31\sz,  v26\sz // t3a
+        sqadd           \o15\sz, v31\sz,  v26\sz // out15
+.else
+        sqadd           v4\sz,   v16\sz,  v21\sz // out0
+        sqsub           v21\sz,  v31\sz,  v26\sz // t3a
+        sqadd           \o15\sz, v31\sz,  v26\sz // out15
+        mov             \o0\szb, v4\szb
+.endif
+        sqneg           \o15\sz, \o15\sz         // out15
+
+        sqsub           v3\sz,   v29\sz,  v18\sz // t15a
+        sqadd           \o13\sz, v29\sz,  v18\sz // out13
+        sqadd           \o2\sz,  v17\sz,  v30\sz // out2
+        sqsub           v26\sz,  v17\sz,  v30\sz // t14a
+        sqneg           \o13\sz, \o13\sz         // out13
+
+        sqadd           \o1\sz,  v19\sz,  v27\sz // out1
+        sqsub           v27\sz,  v19\sz,  v27\sz // t10
+        sqadd           \o14\sz, v28\sz,  v20\sz // out14
+        sqsub           v20\sz,  v28\sz,  v20\sz // t11
+        sqneg           \o1\sz,  \o1\sz          // out1
+
+        sqadd           \o3\sz,  v22\sz,  v24\sz // out3
+        sqsub           v22\sz,  v22\sz,  v24\sz // t6
+        sqadd           \o12\sz, v25\sz,  v23\sz // out12
+        sqsub           v23\sz,  v25\sz,  v23\sz // t7
+        sqneg           \o3\sz,  \o3\sz          // out3
+
+        smull_smlsl     v24, v25, v2,  v21, v0.h[0], v0.h[0], \sz // -> out8 (v24 or v23)
+        smull_smlal     v4,  v5,  v2,  v21, v0.h[0], v0.h[0], \sz // -> out7 (v23 or v24)
+        smull_smlal     v6,  v7,  v26, v3,  v0.h[0], v0.h[0], \sz // -> out5 (v21 or v26)
+
+        sqrshrn_sz      v24, v24, v25, #12, \sz // out8
+        sqrshrn_sz      v4,  v4,  v5,  #12, \sz // out7
+        sqrshrn_sz      v5,  v6,  v7,  #12, \sz // out5
+        smull_smlsl     v6,  v7,  v26, v3,  v0.h[0], v0.h[0], \sz // -> out10 (v26 or v21)
+        smull_smlal     v2,  v3,  v22, v23, v0.h[0], v0.h[0], \sz // -> out4 (v20 or v27)
+        sqrshrn_sz      v26, v6,  v7,  #12, \sz // out10
+
+        smull_smlsl     v6,  v7,  v22, v23, v0.h[0], v0.h[0], \sz // -> out11 (v27 or v20)
+        smull_smlal     v22, v23, v27, v20, v0.h[0], v0.h[0], \sz // -> out6 (v22 or v25)
+        smull_smlsl     v21, v25, v27, v20, v0.h[0], v0.h[0], \sz // -> out9 (v25 or v22)
+
+        sqrshrn_sz      \o4, v2,  v3,  #12, \sz // out4
+        sqrshrn_sz      v6,  v6,  v7,  #12, \sz // out11
+        sqrshrn_sz      v7,  v21, v25, #12, \sz // out9
+        sqrshrn_sz      \o6, v22, v23, #12, \sz // out6
+
+.ifc \o8, v23
+        mov             \o8\szb,  v24\szb
+        mov             \o10\szb, v26\szb
+.endif
+
+        sqneg           \o7\sz,  v4\sz // out7
+        sqneg           \o5\sz,  v5\sz // out5
+        sqneg           \o11\sz, v6\sz // out11
+        sqneg           \o9\sz,  v7\sz // out9
+.endm
+
+function inv_adst_8h_x16_neon, export=1
+        iadst_16        v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .8h, .16b
+        ret
+endfunc
+
+function inv_flipadst_8h_x16_neon, export=1
+        iadst_16        v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .8h, .16b
+        ret
+endfunc
+
+function inv_adst_4h_x16_neon, export=1
+        iadst_16        v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .4h, .8b
+        ret
+endfunc
+
+function inv_flipadst_4h_x16_neon, export=1
+        iadst_16        v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .4h, .8b
+        ret
+endfunc
+
+function inv_identity_8h_x16_neon, export=1
+        mov             w16, #2*(5793-4096)*8
+        dup             v0.4h,   w16
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        sqrdmulh        v2.8h,      v\i\().8h,  v0.h[0]
+        sqadd           v\i\().8h,  v\i\().8h,  v\i\().8h
+        sqadd           v\i\().8h,  v\i\().8h,  v2.8h
+.endr
+        ret
+endfunc
+
+function inv_identity_4h_x16_neon, export=1
+        mov             w16, #2*(5793-4096)*8
+        dup             v0.4h,   w16
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        sqrdmulh        v2.4h,      v\i\().4h,  v0.h[0]
+        sqadd           v\i\().4h,  v\i\().4h,  v\i\().4h
+        sqadd           v\i\().4h,  v\i\().4h,  v2.4h
+.endr
+        ret
+endfunc
+
+.macro identity_8x16_shift2 c
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        sqrdmulh        v2.8h,   \i,      \c
+        sshr            v2.8h,   v2.8h,   #1
+        srhadd          \i,      \i,      v2.8h
+.endr
+.endm
+
+.macro identity_8x16_shift1 c
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        sqrdmulh        v2.8h,   \i,      \c
+        srshr           v2.8h,   v2.8h,   #1
+        sqadd           \i,      \i,      v2.8h
+.endr
+.endm
+
+.macro identity_8x8_shift1 c
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        sqrdmulh        v2.8h,   \i,      \c
+        srshr           v2.8h,   v2.8h,   #1
+        sqadd           \i,      \i,      v2.8h
+.endr
+.endm
+
+.macro identity_8x8 c
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        sqrdmulh        v2.8h,   \i,      \c
+        sqadd           \i,      \i,      \i
+        sqadd           \i,      \i,      v2.8h
+.endr
+.endm
+
+.macro def_horz_16 scale=0, identity=0, shift=2, suffix
+function inv_txfm_horz\suffix\()_16x8_neon
+        AARCH64_VALID_CALL_TARGET
+        mov             x14, x30
+        movi            v7.8h,  #0
+.if \identity
+        mov             w16, #2*(5793-4096)*8
+        dup             v0.4h,   w16
+.elseif \scale
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+.endif
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        ld1             {\i}, [x7]
+        st1             {v7.8h}, [x7], x8
+.endr
+.if \scale
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+        scale_input     .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31
+.endif
+.if \identity
+        identity_8x16_shift2 v0.h[0]
+.else
+        blr             x4
+.endif
+.if \shift > 0
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        srshr           \i,  \i,  #\shift
+.endr
+.endif
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+        transpose_8x8h  v24, v25, v26, v27, v28, v29, v30, v31, v4, v5
+
+.irp i, v16.8h, v24.8h, v17.8h, v25.8h, v18.8h, v26.8h, v19.8h, v27.8h, v20.8h, v28.8h, v21.8h, v29.8h, v22.8h, v30.8h, v23.8h, v31.8h
+        st1             {\i}, [x6], #16
+.endr
+
+        ret             x14
+endfunc
+.endm
+
+def_horz_16 scale=0, identity=0, shift=2
+def_horz_16 scale=1, identity=0, shift=1, suffix=_scale
+def_horz_16 scale=0, identity=1, shift=0, suffix=_identity
+
+function inv_txfm_add_vert_8x16_neon
+        mov             x14, x30
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        ld1             {v\i\().8h}, [x7], x8
+.endr
+        blr             x5
+        load_add_store_8x16 x6, x7
+        ret             x14
+endfunc
+
+function inv_txfm_add_16x16_neon
+        mov             x15, x30
+        sub             sp,  sp,  #512
+.irp i, 0, 8
+        add             x6,  sp,  #(\i*16*2)
+.if \i == 8
+        cmp             w3,  w13
+        b.lt            1f
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #16*2
+        blr             x9
+.endr
+        b               2f
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+2:
+.irp i, 0, 8
+        add             x6,  x0,  #(\i)
+        add             x7,  sp,  #(\i*2)
+        mov             x8,  #32
+        bl              inv_txfm_add_vert_8x16_neon
+.endr
+
+        add             sp,  sp,  #512
+        ret             x15
+endfunc
+
+.macro def_fn_16x16 txfm1, txfm2, eob_half
+function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_8bpc_neon, export=1
+.ifc \txfm1\()_\txfm2, dct_dct
+        idct_dc         16,  16,  2
+.endif
+.ifc \txfm1, identity
+        adr             x9,  inv_txfm_horz_identity_16x8_neon
+.else
+        adr             x9,  inv_txfm_horz_16x8_neon
+        adr             x4,  inv_\txfm1\()_8h_x16_neon
+.endif
+        adr             x5,  inv_\txfm2\()_8h_x16_neon
+        mov             x13, #\eob_half
+        b               inv_txfm_add_16x16_neon
+endfunc
+.endm
+
+def_fn_16x16 dct, dct, 36
+def_fn_16x16 identity, identity, 36
+def_fn_16x16 dct, adst, 36
+def_fn_16x16 dct, flipadst, 36
+def_fn_16x16 dct, identity, 8
+def_fn_16x16 adst, dct, 36
+def_fn_16x16 adst, adst, 36
+def_fn_16x16 adst, flipadst, 36
+def_fn_16x16 flipadst, dct, 36
+def_fn_16x16 flipadst, adst, 36
+def_fn_16x16 flipadst, flipadst, 36
+def_fn_16x16 identity, dct, 8
+
+.macro def_fn_416_base variant
+function inv_txfm_\variant\()add_16x4_neon
+        mov             x15, x30
+        movi            v4.8h,  #0
+
+.ifc \variant, identity_
+.irp i, v16.4h, v17.4h, v18.4h, v19.4h
+        ld1             {\i},    [x2]
+        st1             {v4.4h}, [x2], #8
+.endr
+.irp i, v16.d, v17.d, v18.d, v19.d
+        ld1             {\i}[1], [x2]
+        st1             {v4.4h}, [x2], #8
+.endr
+        mov             w16, #2*(5793-4096)*8
+        dup             v0.4h,   w16
+.irp i, v20.4h, v21.4h, v22.4h, v23.4h
+        ld1             {\i},    [x2]
+        st1             {v4.4h}, [x2], #8
+.endr
+.irp i, v20.d, v21.d, v22.d, v23.d
+        ld1             {\i}[1], [x2]
+        st1             {v4.4h}, [x2], #8
+.endr
+
+        identity_8x16_shift1 v0.h[0]
+.else
+.irp i, v16.4h, v17.4h, v18.4h, v19.4h, v20.4h, v21.4h, v22.4h, v23.4h, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h
+        ld1             {\i},    [x2]
+        st1             {v4.4h}, [x2], #8
+.endr
+
+        blr             x4
+
+        ins             v16.d[1], v20.d[0]
+        ins             v17.d[1], v21.d[0]
+        ins             v18.d[1], v22.d[0]
+        ins             v19.d[1], v23.d[0]
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h
+        srshr           \i,  \i,  #1
+.endr
+.endif
+        transpose_4x8h  v16, v17, v18, v19, v2,  v3,  v4,  v5
+        blr             x5
+        mov             x6,  x0
+        load_add_store_8x4 x6, x7
+
+.ifc \variant, identity_
+        mov             v16.16b, v20.16b
+        mov             v17.16b, v21.16b
+        mov             v18.16b, v22.16b
+        mov             v19.16b, v23.16b
+.else
+        ins             v24.d[1], v28.d[0]
+        ins             v25.d[1], v29.d[0]
+        ins             v26.d[1], v30.d[0]
+        ins             v27.d[1], v31.d[0]
+        srshr           v16.8h,  v24.8h,  #1
+        srshr           v17.8h,  v25.8h,  #1
+        srshr           v18.8h,  v26.8h,  #1
+        srshr           v19.8h,  v27.8h,  #1
+.endif
+        transpose_4x8h  v16, v17, v18, v19, v2,  v3,  v4,  v5
+        blr             x5
+        add             x6,  x0,  #8
+        load_add_store_8x4 x6, x7
+
+        ret             x15
+endfunc
+
+function inv_txfm_\variant\()add_4x16_neon
+        mov             x15, x30
+        movi            v2.8h,   #0
+
+        mov             x11, #32
+        cmp             w3,  w13
+        b.lt            1f
+
+        add             x6,  x2,  #16
+.ifc \variant, identity_
+.irp i, v24.8h, v25.8h, v26.8h, v27.8h
+        ld1             {\i},    [x6]
+        st1             {v2.8h}, [x6], x11
+.endr
+        mov             w16, #(5793-4096)*8
+        dup             v0.4h,   w16
+        identity_8x4_shift1 v24, v25, v26, v27, v0.h[0]
+.else
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h
+        ld1             {\i},    [x6]
+        st1             {v2.8h}, [x6], x11
+.endr
+        blr             x4
+        srshr           v24.8h,  v16.8h,  #1
+        srshr           v25.8h,  v17.8h,  #1
+        srshr           v26.8h,  v18.8h,  #1
+        srshr           v27.8h,  v19.8h,  #1
+.endif
+        transpose_4x8h  v24, v25, v26, v27, v4,  v5,  v6,  v7
+        ins             v28.d[0], v24.d[1]
+        ins             v29.d[0], v25.d[1]
+        ins             v30.d[0], v26.d[1]
+        ins             v31.d[0], v27.d[1]
+
+        b               2f
+1:
+.irp i, v24.4h, v25.4h, v26.4h, v27.4h, v28.4h, v29.4h, v30.4h, v31.4h
+        movi            \i,  #0
+.endr
+2:
+        movi            v2.8h,   #0
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h
+        ld1             {\i},    [x2]
+        st1             {v2.8h}, [x2], x11
+.endr
+.ifc \variant, identity_
+        mov             w16, #(5793-4096)*8
+        dup             v0.4h,   w16
+        identity_8x4_shift1 v16, v17, v18, v19, v0.h[0]
+.else
+        blr             x4
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h
+        srshr           \i,  \i,  #1
+.endr
+.endif
+        transpose_4x8h  v16, v17, v18, v19, v4,  v5,  v6,  v7
+        ins             v20.d[0], v16.d[1]
+        ins             v21.d[0], v17.d[1]
+        ins             v22.d[0], v18.d[1]
+        ins             v23.d[0], v19.d[1]
+
+        blr             x5
+
+        load_add_store_4x16 x0, x6
+
+        ret             x15
+endfunc
+.endm
+
+def_fn_416_base
+def_fn_416_base identity_
+
+.macro def_fn_416 w, h, txfm1, txfm2, eob_half
+function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1
+.ifc \txfm1\()_\txfm2, dct_dct
+        idct_dc         \w,  \h,  1
+.endif
+.if \w == 4
+        adr             x4,  inv_\txfm1\()_8h_x\w\()_neon
+        adr             x5,  inv_\txfm2\()_4h_x\h\()_neon
+        mov             w13, #\eob_half
+.else
+        adr             x4,  inv_\txfm1\()_4h_x\w\()_neon
+        adr             x5,  inv_\txfm2\()_8h_x\h\()_neon
+.endif
+.ifc \txfm1, identity
+        b               inv_txfm_identity_add_\w\()x\h\()_neon
+.else
+        b               inv_txfm_add_\w\()x\h\()_neon
+.endif
+endfunc
+.endm
+
+.macro def_fns_416 w, h
+def_fn_416 \w, \h, dct, dct, 29
+def_fn_416 \w, \h, identity, identity, 29
+def_fn_416 \w, \h, dct, adst, 29
+def_fn_416 \w, \h, dct, flipadst, 29
+def_fn_416 \w, \h, dct, identity, 8
+def_fn_416 \w, \h, adst, dct, 29
+def_fn_416 \w, \h, adst, adst, 29
+def_fn_416 \w, \h, adst, flipadst, 29
+def_fn_416 \w, \h, flipadst, dct, 29
+def_fn_416 \w, \h, flipadst, adst, 29
+def_fn_416 \w, \h, flipadst, flipadst, 29
+def_fn_416 \w, \h, identity, dct, 32
+def_fn_416 \w, \h, adst, identity, 8
+def_fn_416 \w, \h, flipadst, identity, 8
+def_fn_416 \w, \h, identity, adst, 32
+def_fn_416 \w, \h, identity, flipadst, 32
+.endm
+
+def_fns_416 4, 16
+def_fns_416 16, 4
+
+
+.macro def_fn_816_base variant
+function inv_txfm_\variant\()add_16x8_neon
+        mov             x15, x30
+        movi            v4.8h,  #0
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        ld1             {\i},    [x2]
+        st1             {v4.8h}, [x2], #16
+.endr
+
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+        scale_input     .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31
+.ifc \variant, identity_
+        mov             w16, #2*(5793-4096)*8
+        dup             v0.4h,   w16
+        identity_8x16_shift1 v0.h[0]
+.else
+        blr             x4
+
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        srshr           \i,  \i,  #1
+.endr
+.endif
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
+
+        blr             x5
+
+        mov             x6,  x0
+        load_add_store_8x8 x6, x7
+
+.ifc \variant, identity_
+        mov             v16.16b, v24.16b
+        mov             v17.16b, v25.16b
+        mov             v18.16b, v26.16b
+        mov             v19.16b, v27.16b
+        mov             v20.16b, v28.16b
+        mov             v21.16b, v29.16b
+        mov             v22.16b, v30.16b
+        mov             v23.16b, v31.16b
+.else
+        srshr           v16.8h,  v24.8h,  #1
+        srshr           v17.8h,  v25.8h,  #1
+        srshr           v18.8h,  v26.8h,  #1
+        srshr           v19.8h,  v27.8h,  #1
+        srshr           v20.8h,  v28.8h,  #1
+        srshr           v21.8h,  v29.8h,  #1
+        srshr           v22.8h,  v30.8h,  #1
+        srshr           v23.8h,  v31.8h,  #1
+.endif
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
+
+        blr             x5
+
+        add             x0,  x0,  #8
+        load_add_store_8x8 x0, x7
+
+        ret             x15
+endfunc
+
+function inv_txfm_\variant\()add_8x16_neon
+        mov             x15, x30
+        movi            v4.8h,   #0
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+        mov             x11, #32
+
+        cmp             w3,  w13
+        b.lt            1f
+
+        add             x6,  x2,  #16
+.ifc \variant, identity_
+.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        ld1             {\i},    [x6]
+        st1             {v4.8h}, [x6], x11
+.endr
+        scale_input     .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31
+        // The identity shl #1 and downshift srshr #1 cancel out
+.else
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        ld1             {\i},    [x6]
+        st1             {v4.8h}, [x6], x11
+.endr
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+        blr             x4
+
+        srshr           v24.8h,  v16.8h,  #1
+        srshr           v25.8h,  v17.8h,  #1
+        srshr           v26.8h,  v18.8h,  #1
+        srshr           v27.8h,  v19.8h,  #1
+        srshr           v28.8h,  v20.8h,  #1
+        srshr           v29.8h,  v21.8h,  #1
+        srshr           v30.8h,  v22.8h,  #1
+        srshr           v31.8h,  v23.8h,  #1
+.endif
+        transpose_8x8h  v24, v25, v26, v27, v28, v29, v30, v31, v2, v3
+
+        b               2f
+
+1:
+.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        movi            \i,  #0
+.endr
+
+2:
+        movi            v4.8h,   #0
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        ld1             {\i},    [x2]
+        st1             {v4.8h}, [x2], x11
+.endr
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+.ifc \variant, identity_
+        // The identity shl #1 and downshift srshr #1 cancel out
+.else
+        blr             x4
+
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        srshr           \i,  \i,  #1
+.endr
+.endif
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
+
+        blr             x5
+
+        load_add_store_8x16 x0, x6
+
+        ret             x15
+endfunc
+.endm
+
+def_fn_816_base
+def_fn_816_base identity_
+
+.macro def_fn_816 w, h, txfm1, txfm2, eob_half
+function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1
+.ifc \txfm1\()_\txfm2, dct_dct
+        idct_dc         \w,  \h,  1
+.endif
+        adr             x4,  inv_\txfm1\()_8h_x\w\()_neon
+        adr             x5,  inv_\txfm2\()_8h_x\h\()_neon
+.if \w == 8
+        mov             x13, #\eob_half
+.endif
+.ifc \txfm1, identity
+        b               inv_txfm_identity_add_\w\()x\h\()_neon
+.else
+        b               inv_txfm_add_\w\()x\h\()_neon
+.endif
+endfunc
+.endm
+
+.macro def_fns_816 w, h
+def_fn_816 \w, \h, dct, dct, 43
+def_fn_816 \w, \h, identity, identity, 43
+def_fn_816 \w, \h, dct, adst, 43
+def_fn_816 \w, \h, dct, flipadst, 43
+def_fn_816 \w, \h, dct, identity, 8
+def_fn_816 \w, \h, adst, dct, 43
+def_fn_816 \w, \h, adst, adst, 43
+def_fn_816 \w, \h, adst, flipadst, 43
+def_fn_816 \w, \h, flipadst, dct, 43
+def_fn_816 \w, \h, flipadst, adst, 43
+def_fn_816 \w, \h, flipadst, flipadst, 43
+def_fn_816 \w, \h, identity, dct, 64
+def_fn_816 \w, \h, adst, identity, 8
+def_fn_816 \w, \h, flipadst, identity, 8
+def_fn_816 \w, \h, identity, adst, 64
+def_fn_816 \w, \h, identity, flipadst, 64
+.endm
+
+def_fns_816 8, 16
+def_fns_816 16, 8
+
+function inv_dct32_odd_8h_x16_neon, export=1
+        movrel          x16, idct_coeffs, 2*16
+        ld1             {v0.8h, v1.8h}, [x16]
+        sub             x16, x16, #2*16
+
+        smull_smlsl     v2,  v3,  v16, v31, v0.h[0], v0.h[1], .8h // -> t16a
+        smull_smlal     v4,  v5,  v16, v31, v0.h[1], v0.h[0], .8h // -> t31a
+        smull_smlsl     v6,  v7,  v24, v23, v0.h[2], v0.h[3], .8h // -> t17a
+        sqrshrn_sz      v16, v2,  v3,  #12, .8h                   // t16a
+        sqrshrn_sz      v31, v4,  v5,  #12, .8h                   // t31a
+        smull_smlal     v2,  v3,  v24, v23, v0.h[3], v0.h[2], .8h // -> t30a
+        smull_smlsl     v4,  v5,  v20, v27, v0.h[4], v0.h[5], .8h // -> t18a
+        sqrshrn_sz      v24, v6,  v7,  #12, .8h                   // t17a
+        sqrshrn_sz      v23, v2,  v3,  #12, .8h                   // t30a
+        smull_smlal     v6,  v7,  v20, v27, v0.h[5], v0.h[4], .8h // -> t29a
+        smull_smlsl     v2,  v3,  v28, v19, v0.h[6], v0.h[7], .8h // -> t19a
+        sqrshrn_sz      v20, v4,  v5,  #12, .8h                   // t18a
+        sqrshrn_sz      v27, v6,  v7,  #12, .8h                   // t29a
+        smull_smlal     v4,  v5,  v28, v19, v0.h[7], v0.h[6], .8h // -> t28a
+        smull_smlsl     v6,  v7,  v18, v29, v1.h[0], v1.h[1], .8h // -> t20a
+        sqrshrn_sz      v28, v2,  v3,  #12, .8h                   // t19a
+        sqrshrn_sz      v19, v4,  v5,  #12, .8h                   // t28a
+        smull_smlal     v2,  v3,  v18, v29, v1.h[1], v1.h[0], .8h // -> t27a
+        smull_smlsl     v4,  v5,  v26, v21, v1.h[2], v1.h[3], .8h // -> t21a
+        sqrshrn_sz      v18, v6,  v7,  #12, .8h                   // t20a
+        sqrshrn_sz      v29, v2,  v3,  #12, .8h                   // t27a
+        smull_smlal     v6,  v7,  v26, v21, v1.h[3], v1.h[2], .8h // -> t26a
+        smull_smlsl     v2,  v3,  v22, v25, v1.h[4], v1.h[5], .8h // -> t22a
+        sqrshrn_sz      v26, v4,  v5,  #12, .8h                   // t21a
+        sqrshrn_sz      v21, v6,  v7,  #12, .8h                   // t26a
+        smull_smlal     v4,  v5,  v22, v25, v1.h[5], v1.h[4], .8h // -> t25a
+        smull_smlsl     v6,  v7,  v30, v17, v1.h[6], v1.h[7], .8h // -> t23a
+        sqrshrn_sz      v22, v2,  v3,  #12, .8h                   // t22a
+        sqrshrn_sz      v25, v4,  v5,  #12, .8h                   // t25a
+        smull_smlal     v2,  v3,  v30, v17, v1.h[7], v1.h[6], .8h // -> t24a
+        sqrshrn_sz      v30, v6,  v7,  #12, .8h                   // t23a
+        sqrshrn_sz      v17, v2,  v3,  #12, .8h                   // t24a
+
+        ld1             {v0.8h}, [x16]
+
+        sqsub           v2.8h,   v16.8h,  v24.8h // t17
+        sqadd           v16.8h,  v16.8h,  v24.8h // t16
+        sqsub           v3.8h,   v31.8h,  v23.8h // t30
+        sqadd           v31.8h,  v31.8h,  v23.8h // t31
+        sqsub           v24.8h,  v28.8h,  v20.8h // t18
+        sqadd           v28.8h,  v28.8h,  v20.8h // t19
+        sqadd           v23.8h,  v18.8h,  v26.8h // t20
+        sqsub           v18.8h,  v18.8h,  v26.8h // t21
+        sqsub           v20.8h,  v30.8h,  v22.8h // t22
+        sqadd           v30.8h,  v30.8h,  v22.8h // t23
+        sqadd           v26.8h,  v17.8h,  v25.8h // t24
+        sqsub           v17.8h,  v17.8h,  v25.8h // t25
+        sqsub           v22.8h,  v29.8h,  v21.8h // t26
+        sqadd           v29.8h,  v29.8h,  v21.8h // t27
+        sqadd           v25.8h,  v19.8h,  v27.8h // t28
+        sqsub           v19.8h,  v19.8h,  v27.8h // t29
+
+        smull_smlsl     v4,  v5,  v3,  v2,  v0.h[4], v0.h[5], .8h // -> t17a
+        smull_smlal     v6,  v7,  v3,  v2,  v0.h[5], v0.h[4], .8h // -> t30a
+        smull_smlal     v2,  v3,  v19, v24, v0.h[5], v0.h[4], .8h // -> t18a
+        sqrshrn_sz      v21, v4,  v5,  #12, .8h                   // t17a
+        sqrshrn_sz      v27, v6,  v7,  #12, .8h                   // t30a
+        neg             v2.4s,   v2.4s                            // -> t18a
+        neg             v3.4s,   v3.4s                            // -> t18a
+        smull_smlsl     v4,  v5,  v19, v24, v0.h[4], v0.h[5], .8h // -> t29a
+        smull_smlsl     v6,  v7,  v22, v18, v0.h[6], v0.h[7], .8h // -> t21a
+        sqrshrn_sz      v19, v2,  v3,  #12, .8h                   // t18a
+        sqrshrn_sz      v24, v4,  v5,  #12, .8h                   // t29a
+        smull_smlal     v2,  v3,  v22, v18, v0.h[7], v0.h[6], .8h // -> t26a
+        smull_smlal     v4,  v5,  v17, v20, v0.h[7], v0.h[6], .8h // -> t22a
+        sqrshrn_sz      v22, v6,  v7,  #12, .8h                   // t21a
+        sqrshrn_sz      v18, v2,  v3,  #12, .8h                   // t26a
+        neg             v4.4s,   v4.4s                            // -> t22a
+        neg             v5.4s,   v5.4s                            // -> t22a
+        smull_smlsl     v6,  v7,  v17, v20, v0.h[6], v0.h[7], .8h // -> t25a
+        sqrshrn_sz      v17, v4,  v5,  #12, .8h                   // t22a
+        sqrshrn_sz      v20, v6,  v7,  #12, .8h                   // t25a
+
+        sqsub           v2.8h,   v27.8h,  v24.8h // t29
+        sqadd           v27.8h,  v27.8h,  v24.8h // t30
+        sqsub           v3.8h,   v21.8h,  v19.8h // t18
+        sqadd           v21.8h,  v21.8h,  v19.8h // t17
+        sqsub           v24.8h,  v16.8h,  v28.8h // t19a
+        sqadd           v16.8h,  v16.8h,  v28.8h // t16a
+        sqsub           v19.8h,  v30.8h,  v23.8h // t20a
+        sqadd           v30.8h,  v30.8h,  v23.8h // t23a
+        sqsub           v28.8h,  v17.8h,  v22.8h // t21
+        sqadd           v17.8h,  v17.8h,  v22.8h // t22
+        sqadd           v23.8h,  v26.8h,  v29.8h // t24a
+        sqsub           v26.8h,  v26.8h,  v29.8h // t27a
+        sqadd           v22.8h,  v20.8h,  v18.8h // t25
+        sqsub           v20.8h,  v20.8h,  v18.8h // t26
+        sqsub           v29.8h,  v31.8h,  v25.8h // t28a
+        sqadd           v31.8h,  v31.8h,  v25.8h // t31a
+
+        smull_smlsl     v4,  v5,  v2,  v3,  v0.h[2], v0.h[3], .8h // -> t18a
+        smull_smlal     v6,  v7,  v2,  v3,  v0.h[3], v0.h[2], .8h // -> t29a
+        smull_smlsl     v2,  v3,  v29, v24, v0.h[2], v0.h[3], .8h // -> t19
+        sqrshrn_sz      v18, v4,  v5,  #12, .8h                   // t18a
+        sqrshrn_sz      v25, v6,  v7,  #12, .8h                   // t29a
+        smull_smlal     v4,  v5,  v29, v24, v0.h[3], v0.h[2], .8h // -> t28
+        smull_smlal     v6,  v7,  v26, v19, v0.h[3], v0.h[2], .8h // -> t20
+        sqrshrn_sz      v29, v2,  v3,  #12, .8h                   // t19
+        sqrshrn_sz      v24, v4,  v5,  #12, .8h                   // t28
+        neg             v6.4s,   v6.4s                            // -> t20
+        neg             v7.4s,   v7.4s                            // -> t20
+        smull_smlsl     v2,  v3,  v26, v19, v0.h[2], v0.h[3], .8h // -> t27
+        smull_smlal     v4,  v5,  v20, v28, v0.h[3], v0.h[2], .8h // -> t21a
+        sqrshrn_sz      v26, v6,  v7,  #12, .8h                   // t20
+        sqrshrn_sz      v19, v2,  v3,  #12, .8h                   // t27
+        neg             v4.4s,   v4.4s                            // -> t21a
+        neg             v5.4s,   v5.4s                            // -> t21a
+        smull_smlsl     v6,  v7,  v20, v28, v0.h[2], v0.h[3], .8h // -> t26a
+        sqrshrn_sz      v20, v4,  v5,  #12, .8h                   // t21a
+        sqrshrn_sz      v28, v6,  v7,  #12, .8h                   // t26a
+
+        sqsub           v2.8h,   v16.8h,  v30.8h // t23
+        sqadd           v16.8h,  v16.8h,  v30.8h // t16 = out16
+        sqsub           v3.8h,   v31.8h,  v23.8h // t24
+        sqadd           v31.8h,  v31.8h,  v23.8h // t31 = out31
+        sqsub           v23.8h,  v21.8h,  v17.8h // t22a
+        sqadd           v17.8h,  v21.8h,  v17.8h // t17a = out17
+        sqadd           v30.8h,  v27.8h,  v22.8h // t30a = out30
+        sqsub           v21.8h,  v27.8h,  v22.8h // t25a
+        sqsub           v27.8h,  v18.8h,  v20.8h // t21
+        sqadd           v18.8h,  v18.8h,  v20.8h // t18 = out18
+        sqadd           v4.8h,   v29.8h,  v26.8h // t19a = out19
+        sqsub           v26.8h,  v29.8h,  v26.8h // t20a
+        sqadd           v29.8h,  v25.8h,  v28.8h // t29 = out29
+        sqsub           v25.8h,  v25.8h,  v28.8h // t26
+        sqadd           v28.8h,  v24.8h,  v19.8h // t28a = out28
+        sqsub           v24.8h,  v24.8h,  v19.8h // t27a
+        mov             v19.16b, v4.16b          // out19
+
+        smull_smlsl     v4,  v5,  v24, v26, v0.h[0], v0.h[0], .8h // -> t20
+        smull_smlal     v6,  v7,  v24, v26, v0.h[0], v0.h[0], .8h // -> t27
+        sqrshrn_sz      v20, v4,  v5,  #12, .8h   // t20
+        sqrshrn_sz      v22, v6,  v7,  #12, .8h   // t27
+
+        smull_smlal     v4,  v5,  v25, v27, v0.h[0], v0.h[0], .8h // -> t26a
+        smull_smlsl     v6,  v7,  v25, v27, v0.h[0], v0.h[0], .8h // -> t21a
+        mov             v27.16b,  v22.16b         // t27
+        sqrshrn_sz      v26, v4,  v5,  #12, .8h   // t26a
+
+        smull_smlsl     v24, v25, v21, v23, v0.h[0], v0.h[0], .8h // -> t22
+        smull_smlal     v4,  v5,  v21, v23, v0.h[0], v0.h[0], .8h // -> t25
+        sqrshrn_sz      v21, v6,  v7,  #12, .8h   // t21a
+        sqrshrn_sz      v22, v24, v25, #12, .8h   // t22
+        sqrshrn_sz      v25, v4,  v5,  #12, .8h   // t25
+
+        smull_smlsl     v4,  v5,  v3,  v2,  v0.h[0], v0.h[0], .8h // -> t23a
+        smull_smlal     v6,  v7,  v3,  v2,  v0.h[0], v0.h[0], .8h // -> t24a
+        sqrshrn_sz      v23, v4,  v5,  #12, .8h   // t23a
+        sqrshrn_sz      v24, v6,  v7,  #12, .8h   // t24a
+
+        ret
+endfunc
+
+.macro def_horz_32 scale=0, shift=2, suffix
+function inv_txfm_horz\suffix\()_dct_32x8_neon
+        mov             x14, x30
+        movi            v7.8h,  #0
+        lsl             x8,  x8,  #1
+.if \scale
+        mov             w16, #2896*8
+        dup             v0.4h,   w16
+.endif
+
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        ld1             {\i}, [x7]
+        st1             {v7.8h}, [x7], x8
+.endr
+        sub             x7,  x7,  x8, lsl #4
+        add             x7,  x7,  x8, lsr #1
+.if \scale
+        scale_input     .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+        scale_input     .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31
+.endif
+        bl              inv_dct_8h_x16_neon
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+        transpose_8x8h  v24, v25, v26, v27, v28, v29, v30, v31, v4, v5
+
+.macro store1 r0, r1
+        st1             {\r0}, [x6], #16
+        st1             {\r1}, [x6], #16
+        add             x6,  x6,  #32
+.endm
+        store1          v16.8h,  v24.8h
+        store1          v17.8h,  v25.8h
+        store1          v18.8h,  v26.8h
+        store1          v19.8h,  v27.8h
+        store1          v20.8h,  v28.8h
+        store1          v21.8h,  v29.8h
+        store1          v22.8h,  v30.8h
+        store1          v23.8h,  v31.8h
+.purgem store1
+        sub             x6,  x6,  #64*8
+
+        movi            v7.8h,  #0
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        ld1             {\i}, [x7]
+        st1             {v7.8h}, [x7], x8
+.endr
+.if \scale
+        // This relies on the fact that the idct also leaves the right coeff in v0.h[1]
+        scale_input     .8h, v0.h[1], v16, v17, v18, v19, v20, v21, v22, v23
+        scale_input     .8h, v0.h[1], v24, v25, v26, v27, v28, v29, v30, v31
+.endif
+        bl              inv_dct32_odd_8h_x16_neon
+        transpose_8x8h  v31, v30, v29, v28, v27, v26, v25, v24, v4, v5
+        transpose_8x8h  v23, v22, v21, v20, v19, v18, v17, v16, v4, v5
+.macro store2 r0, r1, shift
+        ld1             {v4.8h, v5.8h}, [x6]
+        sqsub           v7.8h,   v4.8h,   \r0
+        sqsub           v6.8h,   v5.8h,   \r1
+        sqadd           v4.8h,   v4.8h,   \r0
+        sqadd           v5.8h,   v5.8h,   \r1
+        rev64           v6.8h,   v6.8h
+        rev64           v7.8h,   v7.8h
+        srshr           v4.8h,   v4.8h,   #\shift
+        srshr           v5.8h,   v5.8h,   #\shift
+        srshr           v6.8h,   v6.8h,   #\shift
+        srshr           v7.8h,   v7.8h,   #\shift
+        ext             v6.16b,  v6.16b,  v6.16b,  #8
+        st1             {v4.8h, v5.8h}, [x6], #32
+        ext             v7.16b,  v7.16b,  v7.16b,  #8
+        st1             {v6.8h, v7.8h}, [x6], #32
+.endm
+
+        store2          v31.8h,  v23.8h, \shift
+        store2          v30.8h,  v22.8h, \shift
+        store2          v29.8h,  v21.8h, \shift
+        store2          v28.8h,  v20.8h, \shift
+        store2          v27.8h,  v19.8h, \shift
+        store2          v26.8h,  v18.8h, \shift
+        store2          v25.8h,  v17.8h, \shift
+        store2          v24.8h,  v16.8h, \shift
+.purgem store2
+        ret             x14
+endfunc
+.endm
+
+def_horz_32 scale=0, shift=2
+def_horz_32 scale=1, shift=1, suffix=_scale
+
+function inv_txfm_add_vert_dct_8x32_neon
+        mov             x14, x30
+        lsl             x8,  x8,  #1
+
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        ld1             {v\i\().8h}, [x7], x8
+.endr
+        sub             x7,  x7,  x8, lsl #4
+
+        bl              inv_dct_8h_x16_neon
+
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        st1             {v\i\().8h}, [x7], x8
+.endr
+        sub             x7,  x7,  x8, lsl #4
+        add             x7,  x7,  x8, lsr #1
+
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+        ld1             {v\i\().8h}, [x7], x8
+.endr
+        sub             x7,  x7,  x8, lsl #4
+        sub             x7,  x7,  x8, lsr #1
+        bl              inv_dct32_odd_8h_x16_neon
+
+        neg             x9,  x8
+        mov             x10, x6
+.macro combine r0, r1, r2, r3, op, stride
+        ld1             {v5.8h}, [x7],    \stride
+        ld1             {v2.8b}, [x10],   x1
+        ld1             {v6.8h}, [x7],    \stride
+        ld1             {v3.8b}, [x10],   x1
+        \op             v5.8h,   v5.8h,   \r0
+        ld1             {v7.8h}, [x7],    \stride
+        ld1             {v4.8b}, [x10],   x1
+        srshr           v5.8h,   v5.8h,   #4
+        \op             v6.8h,   v6.8h,   \r1
+        uaddw           v5.8h,   v5.8h,   v2.8b
+        srshr           v6.8h,   v6.8h,   #4
+        \op             v7.8h,   v7.8h,   \r2
+        sqxtun          v2.8b,   v5.8h
+        ld1             {v5.8h}, [x7],    \stride
+        uaddw           v6.8h,   v6.8h,   v3.8b
+        srshr           v7.8h,   v7.8h,   #4
+        \op             v5.8h,   v5.8h,   \r3
+        st1             {v2.8b}, [x6],    x1
+        ld1             {v2.8b}, [x10],   x1
+        sqxtun          v3.8b,   v6.8h
+        uaddw           v7.8h,   v7.8h,   v4.8b
+        srshr           v5.8h,   v5.8h,   #4
+        st1             {v3.8b}, [x6],    x1
+        sqxtun          v4.8b,   v7.8h
+        uaddw           v5.8h,   v5.8h,   v2.8b
+        st1             {v4.8b}, [x6],    x1
+        sqxtun          v2.8b,   v5.8h
+        st1             {v2.8b}, [x6],    x1
+.endm
+        combine         v31.8h, v30.8h, v29.8h, v28.8h, sqadd, x8
+        combine         v27.8h, v26.8h, v25.8h, v24.8h, sqadd, x8
+        combine         v23.8h, v22.8h, v21.8h, v20.8h, sqadd, x8
+        combine         v19.8h, v18.8h, v17.8h, v16.8h, sqadd, x8
+        sub             x7,  x7,  x8
+        combine         v16.8h, v17.8h, v18.8h, v19.8h, sqsub, x9
+        combine         v20.8h, v21.8h, v22.8h, v23.8h, sqsub, x9
+        combine         v24.8h, v25.8h, v26.8h, v27.8h, sqsub, x9
+        combine         v28.8h, v29.8h, v30.8h, v31.8h, sqsub, x9
+.purgem combine
+
+        ret             x14
+endfunc
+
+const eob_32x32
+        .short 36, 136, 300, 1024
+endconst
+
+const eob_16x32
+        .short 36, 151, 279, 512
+endconst
+
+const eob_16x32_shortside
+        .short 36, 512
+endconst
+
+const eob_8x32
+        .short 43, 107, 171, 256
+endconst
+
+function inv_txfm_add_identity_identity_32x32_8bpc_neon, export=1
+        movi            v0.8h,  #0
+        movrel          x13, eob_32x32
+
+        mov             x8,  #2*32
+1:
+        mov             w9,  #0
+        movrel          x12, eob_32x32
+2:
+        add             w9,  w9,  #8
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        ld1             {v\i\().8h}, [x2]
+        st1             {v0.8h}, [x2], x8
+.endr
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+
+        load_add_store_8x8 x0, x7, shiftbits=2
+        ldrh            w11, [x12], #2
+        sub             x0,  x0,  x1, lsl #3
+        add             x0,  x0,  #8
+        cmp             w3,  w11
+        b.ge            2b
+
+        ldrh            w11, [x13], #2
+        cmp             w3,  w11
+        b.lt            9f
+
+        sub             x0,  x0,  w9, uxtw
+        add             x0,  x0,  x1, lsl #3
+        msub            x2,  x8,  x9,  x2
+        add             x2,  x2,  #2*8
+        b               1b
+9:
+        ret
+endfunc
+
+.macro shift_8_regs op, shift
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        \op             \i,  \i,  #\shift
+.endr
+.endm
+
+.macro def_identity_1632 w, h, wshort, hshort
+function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1
+        mov             w16, #2896*8
+        mov             w17, #2*(5793-4096)*8
+        dup             v1.4h,   w16
+        movi            v0.8h,   #0
+        mov             v1.h[1], w17
+        movrel          x13, eob_16x32\hshort
+
+        mov             x8,  #2*\h
+1:
+        mov             w9,  #0
+        movrel          x12, eob_16x32\wshort
+2:
+        add             w9,  w9,  #8
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        ld1             {\i}, [x2]
+        st1             {v0.8h}, [x2], x8
+.endr
+        scale_input     .8h, v1.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+
+.if \w == 16
+        // 16x32
+        identity_8x8_shift1 v1.h[1]
+.else
+        // 32x16
+        shift_8_regs    sqshl, 1
+        identity_8x8    v1.h[1]
+.endif
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+
+.if \w == 16
+        load_add_store_8x8 x0, x7, shiftbits=2
+.else
+        load_add_store_8x8 x0, x7, shiftbits=4
+.endif
+        ldrh            w11, [x12], #2
+        sub             x0,  x0,  x1, lsl #3
+        add             x0,  x0,  #8
+        cmp             w3,  w11
+        b.ge            2b
+
+        ldrh            w11, [x13], #2
+        cmp             w3,  w11
+        b.lt            9f
+
+        sub             x0,  x0,  w9, uxtw
+        add             x0,  x0,  x1, lsl #3
+        msub            x2,  x8,  x9,  x2
+        add             x2,  x2,  #2*8
+        b               1b
+9:
+        ret
+endfunc
+.endm
+
+def_identity_1632 16, 32, _shortside,
+def_identity_1632 32, 16, , _shortside
+
+.macro def_identity_832 w, h
+function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1
+        movi            v0.8h,  #0
+        movrel          x13, eob_8x32
+
+        mov             w8,  #2*\h
+1:
+        ldrh            w12, [x13], #2
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+        ld1             {\i}, [x2]
+        st1             {v0.8h}, [x2], x8
+.endr
+
+.if \w == 8
+        // 8x32
+        shift_8_regs    srshr, 1
+.endif
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+
+        cmp             w3,  w12
+.if \w == 8
+        load_add_store_8x8 x0, x7, shiftbits=2
+.else
+        load_add_store_8x8 x0, x7, shiftbits=3
+.endif
+
+        b.lt            9f
+.if \w == 8
+        sub             x2,  x2,  x8, lsl #3
+        add             x2,  x2,  #2*8
+.else
+        sub             x0,  x0,  x1, lsl #3
+        add             x0,  x0,  #8
+.endif
+        b               1b
+
+9:
+        ret
+endfunc
+.endm
+
+def_identity_832 8, 32
+def_identity_832 32, 8
+
+function inv_txfm_add_dct_dct_32x32_8bpc_neon, export=1
+        idct_dc         32,  32,  2
+
+        mov             x15, x30
+        sub             sp,  sp,  #2048
+        movrel          x13, eob_32x32
+        ldrh            w12, [x13], #2
+
+.irp i, 0, 8, 16, 24
+        add             x6,  sp,  #(\i*32*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_horz_dct_32x8_neon
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #4
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8, 16, 24
+        add             x6,  x0,  #(\i)
+        add             x7,  sp,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_add_vert_dct_8x32_neon
+.endr
+
+        add             sp,  sp,  #2048
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_16x32_8bpc_neon, export=1
+        idct_dc         16,  32,  1
+
+        mov             x15, x30
+        sub             sp,  sp,  #1024
+        movrel          x13, eob_16x32
+        ldrh            w12, [x13], #2
+        adr             x4,  inv_dct_8h_x16_neon
+
+.irp i, 0, 8, 16, 24
+        add             x6,  sp,  #(\i*16*2)
+        add             x7,  x2,  #(\i*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endif
+        mov             x8,  #2*32
+        bl              inv_txfm_horz_scale_16x8_neon
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #8
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8
+        add             x6,  x0,  #(\i)
+        add             x7,  sp,  #(\i*2)
+        mov             x8,  #16*2
+        bl              inv_txfm_add_vert_dct_8x32_neon
+.endr
+
+        add             sp,  sp,  #1024
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_32x16_8bpc_neon, export=1
+        idct_dc         32,  16,  1
+
+        mov             x15, x30
+        sub             sp,  sp,  #1024
+
+        adr             x5,  inv_dct_8h_x16_neon
+
+.irp i, 0, 8
+        add             x6,  sp,  #(\i*32*2)
+        add             x7,  x2,  #(\i*2)
+.if \i > 0
+        mov             w8,  #(16 - \i)
+        cmp             w3,  #36
+        b.lt            1f
+.endif
+        mov             x8,  #2*16
+        bl              inv_txfm_horz_scale_dct_32x8_neon
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #4
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8, 16, 24
+        add             x6,  x0,  #(\i)
+        add             x7,  sp,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_add_vert_8x16_neon
+.endr
+
+        add             sp,  sp,  #1024
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_8x32_8bpc_neon, export=1
+        idct_dc         8,   32, 2
+
+        mov             x15, x30
+        sub             sp,  sp,  #512
+
+        movrel          x13, eob_8x32
+
+        movi            v28.8h,  #0
+        mov             x8,  #2*32
+        mov             w9,  #32
+        mov             x6,  sp
+1:
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        ld1             {v\i\().8h}, [x2]
+        st1             {v28.8h}, [x2], x8
+.endr
+        ldrh            w12, [x13], #2
+        sub             x2,  x2,  x8, lsl #3
+        sub             w9,  w9,  #8
+        add             x2,  x2,  #2*8
+
+        bl              inv_dct_8h_x8_neon
+
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        srshr           v\i\().8h,  v\i\().8h,  #2
+.endr
+
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v24, v25
+
+        st1             {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64
+        cmp             w3,  w12
+        st1             {v20.8h, v21.8h, v22.8h, v23.8h}, [x6], #64
+
+        b.ge            1b
+        cbz             w9,  3f
+
+        movi            v29.8h,  #0
+        movi            v30.8h,  #0
+        movi            v31.8h,  #0
+2:
+        subs            w9,  w9,  #8
+.rept 2
+        st1             {v28.8h,v29.8h,v30.8h,v31.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+        mov             x6,  x0
+        mov             x7,  sp
+        mov             x8,  #8*2
+        bl              inv_txfm_add_vert_dct_8x32_neon
+
+        add             sp,  sp,  #512
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_32x8_8bpc_neon, export=1
+        idct_dc         32,  8,   2
+
+        mov             x15, x30
+        sub             sp,  sp,  #512
+
+        mov             x6,  sp
+        mov             x7,  x2
+        mov             x8,  #8*2
+        bl              inv_txfm_horz_dct_32x8_neon
+
+        mov             x8,  #2*32
+        mov             w9,  #0
+1:
+        add             x6,  x0,  x9
+        add             x7,  sp,  x9, lsl #1 // #(\i*2)
+
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        ld1             {v\i\().8h}, [x7], x8
+.endr
+        add             w9,  w9,  #8
+
+        bl              inv_dct_8h_x8_neon
+
+        cmp             w9,  #32
+
+        load_add_store_8x8 x6, x7
+
+        b.lt            1b
+
+        add             sp,  sp,  #512
+        ret             x15
+endfunc
+
+function inv_dct64_step1_neon
+        // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a
+        // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a
+        // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a
+        // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a
+
+        ld1             {v0.8h, v1.8h}, [x17], #32
+
+        sqrdmulh        v23.8h,  v16.8h,  v0.h[1]   // t63a
+        sqrdmulh        v16.8h,  v16.8h,  v0.h[0]   // t32a
+        sqrdmulh        v22.8h,  v17.8h,  v0.h[2]   // t62a
+        sqrdmulh        v17.8h,  v17.8h,  v0.h[3]   // t33a
+        sqrdmulh        v21.8h,  v18.8h,  v0.h[5]   // t61a
+        sqrdmulh        v18.8h,  v18.8h,  v0.h[4]   // t34a
+        sqrdmulh        v20.8h,  v19.8h,  v0.h[6]   // t60a
+        sqrdmulh        v19.8h,  v19.8h,  v0.h[7]   // t35a
+
+        sqadd           v24.8h,  v16.8h,  v17.8h    // t32
+        sqsub           v25.8h,  v16.8h,  v17.8h    // t33
+        sqsub           v26.8h,  v19.8h,  v18.8h    // t34
+        sqadd           v27.8h,  v19.8h,  v18.8h    // t35
+        sqadd           v28.8h,  v20.8h,  v21.8h    // t60
+        sqsub           v29.8h,  v20.8h,  v21.8h    // t61
+        sqsub           v30.8h,  v23.8h,  v22.8h    // t62
+        sqadd           v31.8h,  v23.8h,  v22.8h    // t63
+
+        smull_smlal     v2,  v3,  v29, v26, v1.h[0], v1.h[1], .8h // -> t34a
+        smull_smlsl     v4,  v5,  v29, v26, v1.h[1], v1.h[0], .8h // -> t61a
+        neg             v2.4s,   v2.4s              // t34a
+        neg             v3.4s,   v3.4s              // t34a
+        smull_smlsl     v6,  v7,  v30, v25, v1.h[1], v1.h[0], .8h // -> t33a
+        sqrshrn_sz      v26, v2,  v3,  #12, .8h     // t34a
+        smull_smlal     v2,  v3,  v30, v25, v1.h[0], v1.h[1], .8h // -> t62a
+        sqrshrn_sz      v29, v4,  v5,  #12, .8h     // t61a
+        sqrshrn_sz      v25, v6,  v7,  #12, .8h     // t33a
+        sqrshrn_sz      v30, v2,  v3,  #12, .8h     // t62a
+
+        sqadd           v16.8h,  v24.8h,  v27.8h    // t32a
+        sqsub           v19.8h,  v24.8h,  v27.8h    // t35a
+        sqadd           v17.8h,  v25.8h,  v26.8h    // t33
+        sqsub           v18.8h,  v25.8h,  v26.8h    // t34
+        sqsub           v20.8h,  v31.8h,  v28.8h    // t60a
+        sqadd           v23.8h,  v31.8h,  v28.8h    // t63a
+        sqsub           v21.8h,  v30.8h,  v29.8h    // t61
+        sqadd           v22.8h,  v30.8h,  v29.8h    // t62
+
+        smull_smlal     v2,  v3,  v21, v18, v1.h[2], v1.h[3], .8h // -> t61a
+        smull_smlsl     v4,  v5,  v21, v18, v1.h[3], v1.h[2], .8h // -> t34a
+        smull_smlal     v6,  v7,  v20, v19, v1.h[2], v1.h[3], .8h // -> t60
+        sqrshrn_sz      v21, v2,  v3,  #12, .8h     // t61a
+        sqrshrn_sz      v18, v4,  v5,  #12, .8h     // t34a
+        smull_smlsl     v2,  v3,  v20, v19, v1.h[3], v1.h[2], .8h // -> t35
+        sqrshrn_sz      v20, v6,  v7,  #12, .8h     // t60
+        sqrshrn_sz      v19, v2,  v3,  #12, .8h     // t35
+
+        st1             {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64
+        st1             {v20.8h, v21.8h, v22.8h, v23.8h}, [x6], #64
+
+        ret
+endfunc
+
+function inv_dct64_step2_neon
+        movrel          x16, idct_coeffs
+        ld1             {v0.4h}, [x16]
+1:
+        // t32a/33/34a/35/60/61a/62/63a
+        // t56a/57/58a/59/36/37a/38/39a
+        // t40a/41/42a/43/52/53a/54/55a
+        // t48a/49/50a/51/44/45a/46/47a
+        ldr             q16, [x6, #2*8*0]  // t32a
+        ldr             q17, [x9, #2*8*8]  // t39a
+        ldr             q18, [x9, #2*8*0]  // t63a
+        ldr             q19, [x6, #2*8*8]  // t56a
+        ldr             q20, [x6, #2*8*16] // t40a
+        ldr             q21, [x9, #2*8*24] // t47a
+        ldr             q22, [x9, #2*8*16] // t55a
+        ldr             q23, [x6, #2*8*24] // t48a
+
+        sqadd           v24.8h,  v16.8h, v17.8h // t32
+        sqsub           v25.8h,  v16.8h, v17.8h // t39
+        sqadd           v26.8h,  v18.8h, v19.8h // t63
+        sqsub           v27.8h,  v18.8h, v19.8h // t56
+        sqsub           v28.8h,  v21.8h, v20.8h // t40
+        sqadd           v29.8h,  v21.8h, v20.8h // t47
+        sqadd           v30.8h,  v23.8h, v22.8h // t48
+        sqsub           v31.8h,  v23.8h, v22.8h // t55
+
+        smull_smlal     v2,  v3,  v27, v25, v0.h[3], v0.h[2], .8h // -> t56a
+        smull_smlsl     v4,  v5,  v27, v25, v0.h[2], v0.h[3], .8h // -> t39a
+        smull_smlal     v6,  v7,  v31, v28, v0.h[3], v0.h[2], .8h // -> t40a
+        sqrshrn_sz      v25, v2,  v3,  #12, .8h     // t56a
+        sqrshrn_sz      v27, v4,  v5,  #12, .8h     // t39a
+        neg             v6.4s,   v6.4s              // t40a
+        neg             v7.4s,   v7.4s              // t40a
+        smull_smlsl     v2,  v3,  v31, v28, v0.h[2], v0.h[3], .8h // -> t55a
+        sqrshrn_sz      v31, v6,  v7,  #12, .8h     // t40a
+        sqrshrn_sz      v28, v2,  v3,  #12, .8h     // t55a
+
+        sqadd           v16.8h,  v24.8h,  v29.8h    // t32a
+        sqsub           v19.8h,  v24.8h,  v29.8h    // t47a
+        sqadd           v17.8h,  v27.8h,  v31.8h    // t39
+        sqsub           v18.8h,  v27.8h,  v31.8h    // t40
+        sqsub           v20.8h,  v26.8h,  v30.8h    // t48a
+        sqadd           v23.8h,  v26.8h,  v30.8h    // t63a
+        sqsub           v21.8h,  v25.8h,  v28.8h    // t55
+        sqadd           v22.8h,  v25.8h,  v28.8h    // t56
+
+        smull_smlsl     v2,  v3,  v21, v18, v0.h[0], v0.h[0], .8h // -> t40a
+        smull_smlal     v4,  v5,  v21, v18, v0.h[0], v0.h[0], .8h // -> t55a
+        smull_smlsl     v6,  v7,  v20, v19, v0.h[0], v0.h[0], .8h // -> t47
+        sqrshrn_sz      v18, v2,  v3,  #12, .8h     // t40a
+        sqrshrn_sz      v21, v4,  v5,  #12, .8h     // t55a
+        smull_smlal     v2,  v3,  v20, v19, v0.h[0], v0.h[0], .8h // -> t48
+        sqrshrn_sz      v19, v6,  v7,  #12, .8h     // t47
+        sqrshrn_sz      v20, v2,  v3,  #12, .8h     // t48
+
+        str             q16, [x6, #2*8*0]  // t32a
+        str             q17, [x9, #2*8*0]  // t39
+        str             q18, [x6, #2*8*8]  // t40a
+        str             q19, [x9, #2*8*8]  // t47
+        str             q20, [x6, #2*8*16] // t48
+        str             q21, [x9, #2*8*16] // t55a
+        str             q22, [x6, #2*8*24] // t56
+        str             q23, [x9, #2*8*24] // t63a
+
+        add             x6,  x6,  #2*8
+        sub             x9,  x9,  #2*8
+        cmp             x6,  x9
+        b.lt            1b
+        ret
+endfunc
+
+.macro load8 src, strd, zero, clear
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h
+.if \clear
+        ld1             {\i}, [\src]
+        st1             {\zero}, [\src], \strd
+.else
+        ld1             {\i}, [\src], \strd
+.endif
+.endr
+.endm
+
+.macro store16 dst
+.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        st1             {\i}, [\dst], #16
+.endr
+.endm
+
+.macro clear_upper8
+.irp i, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h
+        movi            \i,  #0
+.endr
+.endm
+
+.macro movi_if reg, val, cond
+.if \cond
+        movi            \reg, \val
+.endif
+.endm
+
+.macro movdup_if reg, gpr, val, cond
+.if \cond
+        mov             \gpr, \val
+        dup             \reg, \gpr
+.endif
+.endm
+
+.macro st1_if regs, dst, cond
+.if \cond
+        st1             \regs, \dst
+.endif
+.endm
+
+.macro str_if reg, dst, cond
+.if \cond
+        str             \reg, \dst
+.endif
+.endm
+
+.macro stroff_if reg, dst, dstoff, cond
+.if \cond
+        str             \reg, \dst, \dstoff
+.endif
+.endm
+
+.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7
+.if \cond
+        scale_input     .8h, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7
+.endif
+.endm
+
+.macro def_dct64_func suffix, clear=0, scale=0
+function inv_txfm_dct\suffix\()_8h_x64_neon, export=1
+        mov             x14, x30
+        mov             x6,  sp
+        lsl             x8,  x8,  #2
+
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        load8           x7,  x8,  v7.8h, \clear
+        clear_upper8
+        sub             x7,  x7,  x8, lsl #3
+        add             x7,  x7,  x8, lsr #1
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+
+        bl              inv_dct_8h_x16_neon
+
+        store16         x6
+
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        load8           x7,  x8,  v7.8h, \clear
+        clear_upper8
+        sub             x7,  x7,  x8, lsl #3
+        lsr             x8,  x8,  #1
+        sub             x7,  x7,  x8, lsr #1
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23
+
+        bl              inv_dct32_odd_8h_x16_neon
+
+        add             x10, x6,  #16*15
+        sub             x6,  x6,  #16*16
+
+        mov             x9,  #-16
+
+.macro store_addsub r0, r1, r2, r3
+        ld1             {v2.8h}, [x6], #16
+        ld1             {v3.8h}, [x6], #16
+        sqadd           v6.8h,  v2.8h,  \r0
+        sqsub           \r0,    v2.8h,  \r0
+        ld1             {v4.8h}, [x6], #16
+        sqadd           v7.8h,  v3.8h,  \r1
+        sqsub           \r1,    v3.8h,  \r1
+        ld1             {v5.8h}, [x6], #16
+        sqadd           v2.8h,  v4.8h,  \r2
+        sub             x6,  x6,  #16*4
+        sqsub           \r2,    v4.8h,  \r2
+        st1             {v6.8h}, [x6], #16
+        st1             {\r0},   [x10], x9
+        sqadd           v3.8h,  v5.8h,  \r3
+        sqsub           \r3,    v5.8h,  \r3
+        st1             {v7.8h}, [x6], #16
+        st1             {\r1},   [x10], x9
+        st1             {v2.8h}, [x6], #16
+        st1             {\r2},   [x10], x9
+        st1             {v3.8h}, [x6], #16
+        st1             {\r3},   [x10], x9
+.endm
+        store_addsub    v31.8h, v30.8h, v29.8h, v28.8h
+        store_addsub    v27.8h, v26.8h, v25.8h, v24.8h
+        store_addsub    v23.8h, v22.8h, v21.8h, v20.8h
+        store_addsub    v19.8h, v18.8h, v17.8h, v16.8h
+.purgem store_addsub
+
+        add             x6,  x6,  #2*8*16
+
+        movrel          x17, idct64_coeffs
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        add             x9,  x7,  x8, lsl #4 // offset 16
+        add             x10, x7,  x8, lsl #3 // offset 8
+        sub             x9,  x9,  x8         // offset 15
+        sub             x11, x10, x8         // offset 7
+        ld1             {v16.8h}, [x7]  // in1  (offset 0)
+        ld1             {v17.8h}, [x9]  // in31 (offset 15)
+        ld1             {v18.8h}, [x10] // in17 (offset 8)
+        ld1             {v19.8h}, [x11] // in15 (offset 7)
+        st1_if          {v7.8h}, [x7],  \clear
+        st1_if          {v7.8h}, [x9],  \clear
+        st1_if          {v7.8h}, [x10], \clear
+        st1_if          {v7.8h}, [x11], \clear
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19
+        bl              inv_dct64_step1_neon
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        add             x7,  x7,  x8, lsl #2 // offset 4
+        sub             x9,  x9,  x8, lsl #2 // offset 11
+        sub             x10, x7,  x8         // offset 3
+        add             x11, x9,  x8         // offset 12
+        ld1             {v16.8h}, [x10] // in7  (offset 3)
+        ld1             {v17.8h}, [x11] // in25 (offset 12)
+        ld1             {v18.8h}, [x9]  // in23 (offset 11)
+        ld1             {v19.8h}, [x7]  // in9  (offset 4)
+        st1_if          {v7.8h}, [x7],  \clear
+        st1_if          {v7.8h}, [x9],  \clear
+        st1_if          {v7.8h}, [x10], \clear
+        st1_if          {v7.8h}, [x11], \clear
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19
+        bl              inv_dct64_step1_neon
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        sub             x10, x10, x8, lsl #1 // offset 1
+        sub             x9,  x9,  x8, lsl #1 // offset 9
+        add             x7,  x7,  x8         // offset 5
+        add             x11, x11, x8         // offset 13
+        ldr             q16, [x10, x8] // in5  (offset 2)
+        ldr             q17, [x11]     // in27 (offset 13)
+        ldr             q18, [x9,  x8] // in21 (offset 10)
+        ldr             q19, [x7]      // in11 (offset 5)
+        stroff_if       q7,  [x10, x8], \clear
+        str_if          q7,  [x11],     \clear
+        stroff_if       q7,  [x9,  x8], \clear
+        str_if          q7,  [x7],      \clear
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19
+        bl              inv_dct64_step1_neon
+        movdup_if       v0.4h, w16, #2896*8, \scale
+        movi_if         v7.8h,  #0, \clear
+        ldr             q16, [x10]     // in3  (offset 1)
+        ldr             q17, [x11, x8] // in29 (offset 14)
+        ldr             q18, [x9]      // in19 (offset 9)
+        ldr             q19, [x7,  x8] // in13 (offset 6)
+        str_if          q7,  [x10],     \clear
+        stroff_if       q7,  [x11, x8], \clear
+        str_if          q7,  [x9],      \clear
+        stroff_if       q7,  [x7,  x8], \clear
+        scale_if        \scale, v0.h[0], v16, v17, v18, v19
+        bl              inv_dct64_step1_neon
+
+        sub             x6,  x6,  #2*8*32
+        add             x9,  x6,  #2*8*7
+
+        bl              inv_dct64_step2_neon
+
+        ret             x14
+endfunc
+.endm
+
+def_dct64_func
+def_dct64_func _clear, clear=1
+def_dct64_func _clear_scale, clear=1, scale=1
+
+
+function inv_txfm_horz_dct_64x8_neon
+        mov             x14, x30
+
+        mov             x7,  sp
+        add             x8,  sp,  #2*8*(64 - 4)
+        add             x9,  x6,  #2*56
+        mov             x10, #2*64
+        mov             x11, #-2*8*4
+
+        dup             v7.8h,  w12
+1:
+        ld1             {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64
+        ld1             {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11
+        ld1             {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64
+        ld1             {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11
+        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v4, v5
+        transpose_8x8h  v31, v30, v29, v28, v27, v26, v25, v24, v4, v5
+
+.macro store_addsub src0, src1, src2, src3
+        sqsub           v1.8h,   \src0,   \src1
+        sqadd           v0.8h,   \src0,   \src1
+        sqsub           v3.8h,   \src2,   \src3
+        srshl           v1.8h,   v1.8h,   v7.8h
+        sqadd           v2.8h,   \src2,   \src3
+        srshl           v0.8h,   v0.8h,   v7.8h
+        srshl           v3.8h,   v3.8h,   v7.8h
+        rev64           v1.8h,   v1.8h
+        srshl           v2.8h,   v2.8h,   v7.8h
+        rev64           v3.8h,   v3.8h
+        ext             v1.16b,  v1.16b,  v1.16b,  #8
+        st1             {v0.8h},  [x6], x10
+        ext             v3.16b,  v3.16b,  v3.16b,  #8
+        st1             {v1.8h},  [x9], x10
+        st1             {v2.8h},  [x6], x10
+        st1             {v3.8h},  [x9], x10
+.endm
+        store_addsub    v16.8h,  v31.8h,  v17.8h,  v30.8h
+        store_addsub    v18.8h,  v29.8h,  v19.8h,  v28.8h
+        store_addsub    v20.8h,  v27.8h,  v21.8h,  v26.8h
+        store_addsub    v22.8h,  v25.8h,  v23.8h,  v24.8h
+.purgem store_addsub
+        sub             x6,  x6,  x10, lsl #3
+        sub             x9,  x9,  x10, lsl #3
+        add             x6,  x6,  #16
+        sub             x9,  x9,  #16
+
+        cmp             x7,  x8
+        b.lt            1b
+        ret             x14
+endfunc
+
+function inv_txfm_add_vert_dct_8x64_neon
+        mov             x14, x30
+        lsl             x8,  x8,  #1
+
+        mov             x7,  sp
+        add             x8,  sp,  #2*8*(64 - 4)
+        add             x9,  x6,  x1, lsl #6
+        sub             x9,  x9,  x1
+        neg             x10, x1
+        mov             x11, #-2*8*4
+
+1:
+        ld1             {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64
+        ld1             {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11
+        ld1             {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64
+        ld1             {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11
+
+.macro add_dest_addsub src0, src1, src2, src3
+        ld1             {v0.8b}, [x6], x1
+        ld1             {v1.8b}, [x9], x10
+        sqadd           v4.8h,   \src0,   \src1
+        ld1             {v2.8b}, [x6]
+        sqsub           v5.8h,   \src0,   \src1
+        ld1             {v3.8b}, [x9]
+        sqadd           v6.8h,   \src2,   \src3
+        sqsub           v7.8h,   \src2,   \src3
+        sub             x6,  x6,  x1
+        sub             x9,  x9,  x10
+        srshr           v4.8h,   v4.8h,   #4
+        srshr           v5.8h,   v5.8h,   #4
+        srshr           v6.8h,   v6.8h,   #4
+        uaddw           v4.8h,   v4.8h,   v0.8b
+        srshr           v7.8h,   v7.8h,   #4
+        uaddw           v5.8h,   v5.8h,   v1.8b
+        uaddw           v6.8h,   v6.8h,   v2.8b
+        sqxtun          v0.8b,   v4.8h
+        uaddw           v7.8h,   v7.8h,   v3.8b
+        sqxtun          v1.8b,   v5.8h
+        st1             {v0.8b}, [x6], x1
+        sqxtun          v2.8b,   v6.8h
+        st1             {v1.8b}, [x9], x10
+        sqxtun          v3.8b,   v7.8h
+        st1             {v2.8b}, [x6], x1
+        st1             {v3.8b}, [x9], x10
+.endm
+        add_dest_addsub v16.8h,  v31.8h,  v17.8h,  v30.8h
+        add_dest_addsub v18.8h,  v29.8h,  v19.8h,  v28.8h
+        add_dest_addsub v20.8h,  v27.8h,  v21.8h,  v26.8h
+        add_dest_addsub v22.8h,  v25.8h,  v23.8h,  v24.8h
+.purgem add_dest_addsub
+        cmp             x7,  x8
+        b.lt            1b
+
+        ret             x14
+endfunc
+
+function inv_txfm_add_dct_dct_64x64_8bpc_neon, export=1
+        idct_dc         64,  64,  2
+
+        mov             x15, x30
+
+        sub_sp          64*32*2+64*8*2
+        add             x5,  sp, #64*8*2
+
+        movrel          x13, eob_32x32
+
+.irp i, 0, 8, 16, 24
+        add             x6,  x5,  #(\i*64*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #32*2
+        mov             x12, #-2 // shift
+        bl              inv_txfm_dct_clear_8h_x64_neon
+        add             x6,  x5,  #(\i*64*2)
+        bl              inv_txfm_horz_dct_64x8_neon
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #2
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8, 16, 24, 32, 40, 48, 56
+        add             x7,  x5,  #(\i*2)
+        mov             x8,  #64*2
+        bl              inv_txfm_dct_8h_x64_neon
+        add             x6,  x0,  #(\i)
+        bl              inv_txfm_add_vert_dct_8x64_neon
+.endr
+
+        add             sp,  x5,  #64*32*2
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_64x32_8bpc_neon, export=1
+        idct_dc         64,  32,  1
+
+        mov             x15, x30
+
+        sub_sp          64*32*2+64*8*2
+        add             x5,  sp, #64*8*2
+
+        movrel          x13, eob_32x32
+
+.irp i, 0, 8, 16, 24
+        add             x6,  x5,  #(\i*64*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #32*2
+        mov             x12, #-1 // shift
+        bl              inv_txfm_dct_clear_scale_8h_x64_neon
+        add             x6,  x5,  #(\i*64*2)
+        bl              inv_txfm_horz_dct_64x8_neon
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #2
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8, 16, 24, 32, 40, 48, 56
+        add             x6,  x0,  #(\i)
+        add             x7,  x5,  #(\i*2)
+        mov             x8,  #64*2
+        bl              inv_txfm_add_vert_dct_8x32_neon
+.endr
+
+        add             sp,  x5,  #64*32*2
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_32x64_8bpc_neon, export=1
+        idct_dc         32,  64,  1
+
+        mov             x15, x30
+
+        sub_sp          32*32*2+64*8*2
+        add             x5,  sp, #64*8*2
+
+        movrel          x13, eob_32x32
+        ldrh            w12, [x13], #2
+
+.irp i, 0, 8, 16, 24
+        add             x6,  x5,  #(\i*32*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_horz_scale_dct_32x8_neon
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #4
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8, 16, 24
+        add             x7,  x5,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_dct_8h_x64_neon
+        add             x6,  x0,  #(\i)
+        bl              inv_txfm_add_vert_dct_8x64_neon
+.endr
+
+        add             sp,  x5,  #32*32*2
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_64x16_8bpc_neon, export=1
+        idct_dc         64,  16,  2
+
+        mov             x15, x30
+
+        sub_sp          64*16*2+64*8*2
+        add             x4,  sp, #64*8*2
+
+        movrel          x13, eob_16x32
+
+.irp i, 0, 8
+        add             x6,  x4,  #(\i*64*2)
+.if \i > 0
+        mov             w8,  #(16 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #16*2
+        mov             x12, #-2 // shift
+        bl              inv_txfm_dct_clear_8h_x64_neon
+        add             x6,  x4,  #(\i*64*2)
+        bl              inv_txfm_horz_dct_64x8_neon
+.if \i < 8
+        ldrh            w12, [x13], #2
+.endif
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #2
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+        adr             x5,  inv_dct_8h_x16_neon
+.irp i, 0, 8, 16, 24, 32, 40, 48, 56
+        add             x6,  x0,  #(\i)
+        add             x7,  x4,  #(\i*2)
+        mov             x8,  #64*2
+        bl              inv_txfm_add_vert_8x16_neon
+.endr
+
+        add             sp,  x4,  #64*16*2
+        ret             x15
+endfunc
+
+function inv_txfm_add_dct_dct_16x64_8bpc_neon, export=1
+        idct_dc         16,  64,  2
+
+        mov             x15, x30
+
+        sub_sp          16*32*2+64*8*2
+        add             x5,  sp, #64*8*2
+
+        movrel          x13, eob_16x32
+        ldrh            w12, [x13], #2
+
+        adr             x4,  inv_dct_8h_x16_neon
+.irp i, 0, 8, 16, 24
+        add             x6,  x5,  #(\i*16*2)
+.if \i > 0
+        mov             w8,  #(32 - \i)
+        cmp             w3,  w12
+        b.lt            1f
+.if \i < 24
+        ldrh            w12, [x13], #2
+.endif
+.endif
+        add             x7,  x2,  #(\i*2)
+        mov             x8,  #32*2
+        bl              inv_txfm_horz_16x8_neon
+.endr
+        b               3f
+
+1:
+        movi            v4.8h,  #0
+        movi            v5.8h,  #0
+        movi            v6.8h,  #0
+        movi            v7.8h,  #0
+2:
+        subs            w8,  w8,  #8
+.rept 4
+        st1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64
+.endr
+        b.gt            2b
+
+3:
+.irp i, 0, 8
+        add             x7,  x5,  #(\i*2)
+        mov             x8,  #16*2
+        bl              inv_txfm_dct_8h_x64_neon
+        add             x6,  x0,  #(\i)
+        bl              inv_txfm_add_vert_dct_8x64_neon
+.endr
+
+        add             sp,  x5,  #16*32*2
+        ret             x15
+endfunc