path: root/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h

/*
 *  Copyright (c) 2022 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef VPX_VPX_DSP_ARM_FDCT_NEON_H_
#define VPX_VPX_DSP_ARM_FDCT_NEON_H_

#include <arm_neon.h>

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulh_s16 operation on half vector
// can be slightly less accurate, adequate for pass1
static INLINE void butterfly_one_coeff_s16_fast_half(const int16x4_t a,
                                                     const int16x4_t b,
                                                     const tran_coef_t constant,
                                                     int16x4_t *add,
                                                     int16x4_t *sub) {
  int16x4_t c = vdup_n_s16(2 * constant);
  *add = vqrdmulh_s16(vadd_s16(a, b), c);
  *sub = vqrdmulh_s16(vsub_s16(a, b), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulh_s16 operation on full vector
// can be slightly less accurate, adequate for pass1
static INLINE void butterfly_one_coeff_s16_fast(const int16x8_t a,
                                                const int16x8_t b,
                                                const tran_coef_t constant,
                                                int16x8_t *add,
                                                int16x8_t *sub) {
  int16x8_t c = vdupq_n_s16(2 * constant);
  *add = vqrdmulhq_s16(vaddq_s16(a, b), c);
  *sub = vqrdmulhq_s16(vsubq_s16(a, b), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes 16-bit input values,
// returns full 32-bit values, high/low
static INLINE void butterfly_one_coeff_s16_s32_fast(
    const int16x8_t a, const int16x8_t b, const tran_coef_t constant,
    int32x4_t *add_lo, int32x4_t *add_hi, int32x4_t *sub_lo,
    int32x4_t *sub_hi) {
  int32x4_t c = vdupq_n_s32(constant << 17);
  const int16x4_t a_lo = vget_low_s16(a);
  const int16x4_t a_hi = vget_high_s16(a);
  const int16x4_t b_lo = vget_low_s16(b);
  const int16x4_t b_hi = vget_high_s16(b);
  *add_lo = vqrdmulhq_s32(vaddl_s16(a_lo, b_lo), c);
  *add_hi = vqrdmulhq_s32(vaddl_s16(a_hi, b_hi), c);
  *sub_lo = vqrdmulhq_s32(vsubl_s16(a_lo, b_lo), c);
  *sub_hi = vqrdmulhq_s32(vsubl_s16(a_hi, b_hi), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes 16-bit input values,
// returns full 32-bit values, high/low
static INLINE void butterfly_one_coeff_s16_s32_fast_narrow(
    const int16x8_t a, const int16x8_t b, const tran_coef_t constant,
    int16x8_t *add, int16x8_t *sub) {
  int32x4_t add_lo, add_hi, sub_lo, sub_hi;
  butterfly_one_coeff_s16_s32_fast(a, b, constant, &add_lo, &add_hi, &sub_lo,
                                   &sub_hi);
  *add = vcombine_s16(vmovn_s32(add_lo), vmovn_s32(add_hi));
  *sub = vcombine_s16(vmovn_s32(sub_lo), vmovn_s32(sub_hi));
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes 16-bit input values,
// returns full 32-bit values, high/low
static INLINE void butterfly_one_coeff_s16_s32_fast_half(
    const int16x4_t a, const int16x4_t b, const tran_coef_t constant,
    int32x4_t *add, int32x4_t *sub) {
  int32x4_t c = vdupq_n_s32(constant << 17);
  *add = vqrdmulhq_s32(vaddl_s16(a, b), c);
  *sub = vqrdmulhq_s32(vsubl_s16(a, b), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on half vector
// more accurate does 32-bit processing, takes 16-bit input values,
// returns narrowed down 16-bit values
static INLINE void butterfly_one_coeff_s16_s32_fast_narrow_half(
    const int16x4_t a, const int16x4_t b, const tran_coef_t constant,
    int16x4_t *add, int16x4_t *sub) {
  int32x4_t add32, sub32;
  butterfly_one_coeff_s16_s32_fast_half(a, b, constant, &add32, &sub32);
  *add = vmovn_s32(add32);
  *sub = vmovn_s32(sub32);
}

// fdct_round_shift((a +/- b) * c)
// Original Variant that performs normal implementation on full vector
// fully accurate does 32-bit processing, takes 16-bit values
static INLINE void butterfly_one_coeff_s16_s32(
    const int16x8_t a, const int16x8_t b, const tran_coef_t constant,
    int32x4_t *add_lo, int32x4_t *add_hi, int32x4_t *sub_lo,
    int32x4_t *sub_hi) {
  const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), constant);
  const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), constant);
  const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), constant);
  const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), constant);
  const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), constant);
  const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), constant);
  *add_lo = vrshrq_n_s32(sum0, DCT_CONST_BITS);
  *add_hi = vrshrq_n_s32(sum1, DCT_CONST_BITS);
  *sub_lo = vrshrq_n_s32(diff0, DCT_CONST_BITS);
  *sub_hi = vrshrq_n_s32(diff1, DCT_CONST_BITS);
}

// fdct_round_shift((a +/- b) * c)
// Original Variant that performs normal implementation on full vector
// fully accurate does 32-bit processing, takes 16-bit values
// returns narrowed down 16-bit values
static INLINE void butterfly_one_coeff_s16_s32_narrow(
    const int16x8_t a, const int16x8_t b, const tran_coef_t constant,
    int16x8_t *add, int16x8_t *sub) {
  int32x4_t add32_lo, add32_hi, sub32_lo, sub32_hi;
  butterfly_one_coeff_s16_s32(a, b, constant, &add32_lo, &add32_hi, &sub32_lo,
                              &sub32_hi);
  *add = vcombine_s16(vmovn_s32(add32_lo), vmovn_s32(add32_hi));
  *sub = vcombine_s16(vmovn_s32(sub32_lo), vmovn_s32(sub32_hi));
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values,
// high/low
static INLINE void butterfly_one_coeff_s32_noround(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant, int32x4_t *add_lo,
    int32x4_t *add_hi, int32x4_t *sub_lo, int32x4_t *sub_hi) {
  const int32x4_t a1 = vmulq_n_s32(a_lo, constant);
  const int32x4_t a2 = vmulq_n_s32(a_hi, constant);
  const int32x4_t a3 = vmulq_n_s32(a_lo, constant);
  const int32x4_t a4 = vmulq_n_s32(a_hi, constant);
  *add_lo = vmlaq_n_s32(a1, b_lo, constant);
  *add_hi = vmlaq_n_s32(a2, b_hi, constant);
  *sub_lo = vmlsq_n_s32(a3, b_lo, constant);
  *sub_hi = vmlsq_n_s32(a4, b_hi, constant);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values,
// high/low
static INLINE void butterfly_one_coeff_s32_fast_half(const int32x4_t a,
                                                     const int32x4_t b,
                                                     const tran_coef_t constant,
                                                     int32x4_t *add,
                                                     int32x4_t *sub) {
  const int32x4_t c = vdupq_n_s32(constant << 17);
  *add = vqrdmulhq_s32(vaddq_s32(a, b), c);
  *sub = vqrdmulhq_s32(vsubq_s32(a, b), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs fast vqrdmulhq_s32 operation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values,
// high/low
static INLINE void butterfly_one_coeff_s32_fast(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant, int32x4_t *add_lo,
    int32x4_t *add_hi, int32x4_t *sub_lo, int32x4_t *sub_hi) {
  const int32x4_t c = vdupq_n_s32(constant << 17);
  *add_lo = vqrdmulhq_s32(vaddq_s32(a_lo, b_lo), c);
  *add_hi = vqrdmulhq_s32(vaddq_s32(a_hi, b_hi), c);
  *sub_lo = vqrdmulhq_s32(vsubq_s32(a_lo, b_lo), c);
  *sub_hi = vqrdmulhq_s32(vsubq_s32(a_hi, b_hi), c);
}

// fdct_round_shift((a +/- b) * c)
// Variant that performs normal implementation on full vector
// more accurate does 64-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_one_coeff_s32_s64_narrow(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant, int32x4_t *add_lo,
    int32x4_t *add_hi, int32x4_t *sub_lo, int32x4_t *sub_hi) {
  // ac holds the following values:
  // ac: vget_low_s32(a_lo) * c, vget_high_s32(a_lo) * c,
  //     vget_low_s32(a_hi) * c, vget_high_s32(a_hi) * c
  int64x2_t ac[4];
  int64x2_t sum[4];
  int64x2_t diff[4];

  ac[0] = vmull_n_s32(vget_low_s32(a_lo), constant);
  ac[1] = vmull_n_s32(vget_high_s32(a_lo), constant);
  ac[2] = vmull_n_s32(vget_low_s32(a_hi), constant);
  ac[3] = vmull_n_s32(vget_high_s32(a_hi), constant);

  sum[0] = vmlal_n_s32(ac[0], vget_low_s32(b_lo), constant);
  sum[1] = vmlal_n_s32(ac[1], vget_high_s32(b_lo), constant);
  sum[2] = vmlal_n_s32(ac[2], vget_low_s32(b_hi), constant);
  sum[3] = vmlal_n_s32(ac[3], vget_high_s32(b_hi), constant);
  *add_lo = vcombine_s32(vrshrn_n_s64(sum[0], DCT_CONST_BITS),
                         vrshrn_n_s64(sum[1], DCT_CONST_BITS));
  *add_hi = vcombine_s32(vrshrn_n_s64(sum[2], DCT_CONST_BITS),
                         vrshrn_n_s64(sum[3], DCT_CONST_BITS));

  diff[0] = vmlsl_n_s32(ac[0], vget_low_s32(b_lo), constant);
  diff[1] = vmlsl_n_s32(ac[1], vget_high_s32(b_lo), constant);
  diff[2] = vmlsl_n_s32(ac[2], vget_low_s32(b_hi), constant);
  diff[3] = vmlsl_n_s32(ac[3], vget_high_s32(b_hi), constant);
  *sub_lo = vcombine_s32(vrshrn_n_s64(diff[0], DCT_CONST_BITS),
                         vrshrn_n_s64(diff[1], DCT_CONST_BITS));
  *sub_hi = vcombine_s32(vrshrn_n_s64(diff[2], DCT_CONST_BITS),
                         vrshrn_n_s64(diff[3], DCT_CONST_BITS));
}

// fdct_round_shift(a * c1 +/- b * c2)
// Variant that performs normal implementation on half vector
// more accurate does 64-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_s32_s64_narrow_half(
    const int32x4_t a, const int32x4_t b, const tran_coef_t constant1,
    const tran_coef_t constant2, int32x4_t *add, int32x4_t *sub) {
  const int32x2_t a_lo = vget_low_s32(a);
  const int32x2_t a_hi = vget_high_s32(a);
  const int32x2_t b_lo = vget_low_s32(b);
  const int32x2_t b_hi = vget_high_s32(b);

  const int64x2_t axc0_64_lo = vmull_n_s32(a_lo, constant1);
  const int64x2_t axc0_64_hi = vmull_n_s32(a_hi, constant1);
  const int64x2_t axc1_64_lo = vmull_n_s32(a_lo, constant2);
  const int64x2_t axc1_64_hi = vmull_n_s32(a_hi, constant2);

  const int64x2_t sum_lo = vmlal_n_s32(axc0_64_lo, b_lo, constant2);
  const int64x2_t sum_hi = vmlal_n_s32(axc0_64_hi, b_hi, constant2);
  const int64x2_t diff_lo = vmlsl_n_s32(axc1_64_lo, b_lo, constant1);
  const int64x2_t diff_hi = vmlsl_n_s32(axc1_64_hi, b_hi, constant1);

  *add = vcombine_s32(vrshrn_n_s64(sum_lo, DCT_CONST_BITS),
                      vrshrn_n_s64(sum_hi, DCT_CONST_BITS));
  *sub = vcombine_s32(vrshrn_n_s64(diff_lo, DCT_CONST_BITS),
                      vrshrn_n_s64(diff_hi, DCT_CONST_BITS));
}

// fdct_round_shift(a * c1 +/- b * c2)
// Variant that performs normal implementation on full vector
// more accurate does 64-bit processing, takes and returns 64-bit values
// returns results without rounding
static INLINE void butterfly_two_coeff_s32_s64_noround(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant1,
    const tran_coef_t constant2, int64x2_t *add_lo /*[2]*/,
    int64x2_t *add_hi /*[2]*/, int64x2_t *sub_lo /*[2]*/,
    int64x2_t *sub_hi /*[2]*/) {
  // ac1/ac2 hold the following values:
  // ac1: vget_low_s32(a_lo) * c1, vget_high_s32(a_lo) * c1,
  //      vget_low_s32(a_hi) * c1, vget_high_s32(a_hi) * c1
  // ac2: vget_low_s32(a_lo) * c2, vget_high_s32(a_lo) * c2,
  //      vget_low_s32(a_hi) * c2, vget_high_s32(a_hi) * c2
  int64x2_t ac1[4];
  int64x2_t ac2[4];

  ac1[0] = vmull_n_s32(vget_low_s32(a_lo), constant1);
  ac1[1] = vmull_n_s32(vget_high_s32(a_lo), constant1);
  ac1[2] = vmull_n_s32(vget_low_s32(a_hi), constant1);
  ac1[3] = vmull_n_s32(vget_high_s32(a_hi), constant1);
  ac2[0] = vmull_n_s32(vget_low_s32(a_lo), constant2);
  ac2[1] = vmull_n_s32(vget_high_s32(a_lo), constant2);
  ac2[2] = vmull_n_s32(vget_low_s32(a_hi), constant2);
  ac2[3] = vmull_n_s32(vget_high_s32(a_hi), constant2);

  add_lo[0] = vmlal_n_s32(ac1[0], vget_low_s32(b_lo), constant2);
  add_lo[1] = vmlal_n_s32(ac1[1], vget_high_s32(b_lo), constant2);
  add_hi[0] = vmlal_n_s32(ac1[2], vget_low_s32(b_hi), constant2);
  add_hi[1] = vmlal_n_s32(ac1[3], vget_high_s32(b_hi), constant2);

  sub_lo[0] = vmlsl_n_s32(ac2[0], vget_low_s32(b_lo), constant1);
  sub_lo[1] = vmlsl_n_s32(ac2[1], vget_high_s32(b_lo), constant1);
  sub_hi[0] = vmlsl_n_s32(ac2[2], vget_low_s32(b_hi), constant1);
  sub_hi[1] = vmlsl_n_s32(ac2[3], vget_high_s32(b_hi), constant1);
}

// fdct_round_shift(a * c1 +/- b * c2)
// Variant that performs normal implementation on full vector
// more accurate does 64-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_s32_s64_narrow(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant1,
    const tran_coef_t constant2, int32x4_t *add_lo, int32x4_t *add_hi,
    int32x4_t *sub_lo, int32x4_t *sub_hi) {
  // ac1/ac2 hold the following values:
  // ac1: vget_low_s32(a_lo) * c1, vget_high_s32(a_lo) * c1,
  //      vget_low_s32(a_hi) * c1, vget_high_s32(a_hi) * c1
  // ac2: vget_low_s32(a_lo) * c2, vget_high_s32(a_lo) * c2,
  //      vget_low_s32(a_hi) * c2, vget_high_s32(a_hi) * c2
  int64x2_t ac1[4];
  int64x2_t ac2[4];
  int64x2_t sum[4];
  int64x2_t diff[4];

  ac1[0] = vmull_n_s32(vget_low_s32(a_lo), constant1);
  ac1[1] = vmull_n_s32(vget_high_s32(a_lo), constant1);
  ac1[2] = vmull_n_s32(vget_low_s32(a_hi), constant1);
  ac1[3] = vmull_n_s32(vget_high_s32(a_hi), constant1);
  ac2[0] = vmull_n_s32(vget_low_s32(a_lo), constant2);
  ac2[1] = vmull_n_s32(vget_high_s32(a_lo), constant2);
  ac2[2] = vmull_n_s32(vget_low_s32(a_hi), constant2);
  ac2[3] = vmull_n_s32(vget_high_s32(a_hi), constant2);

  sum[0] = vmlal_n_s32(ac1[0], vget_low_s32(b_lo), constant2);
  sum[1] = vmlal_n_s32(ac1[1], vget_high_s32(b_lo), constant2);
  sum[2] = vmlal_n_s32(ac1[2], vget_low_s32(b_hi), constant2);
  sum[3] = vmlal_n_s32(ac1[3], vget_high_s32(b_hi), constant2);
  *add_lo = vcombine_s32(vrshrn_n_s64(sum[0], DCT_CONST_BITS),
                         vrshrn_n_s64(sum[1], DCT_CONST_BITS));
  *add_hi = vcombine_s32(vrshrn_n_s64(sum[2], DCT_CONST_BITS),
                         vrshrn_n_s64(sum[3], DCT_CONST_BITS));

  diff[0] = vmlsl_n_s32(ac2[0], vget_low_s32(b_lo), constant1);
  diff[1] = vmlsl_n_s32(ac2[1], vget_high_s32(b_lo), constant1);
  diff[2] = vmlsl_n_s32(ac2[2], vget_low_s32(b_hi), constant1);
  diff[3] = vmlsl_n_s32(ac2[3], vget_high_s32(b_hi), constant1);
  *sub_lo = vcombine_s32(vrshrn_n_s64(diff[0], DCT_CONST_BITS),
                         vrshrn_n_s64(diff[1], DCT_CONST_BITS));
  *sub_hi = vcombine_s32(vrshrn_n_s64(diff[2], DCT_CONST_BITS),
                         vrshrn_n_s64(diff[3], DCT_CONST_BITS));
}

// fdct_round_shift(a * c1 +/- b * c2)
// Original Variant that performs normal implementation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_s16_s32_noround(
    const int16x4_t a_lo, const int16x4_t a_hi, const int16x4_t b_lo,
    const int16x4_t b_hi, const tran_coef_t constant1,
    const tran_coef_t constant2, int32x4_t *add_lo, int32x4_t *add_hi,
    int32x4_t *sub_lo, int32x4_t *sub_hi) {
  const int32x4_t a1 = vmull_n_s16(a_lo, constant1);
  const int32x4_t a2 = vmull_n_s16(a_hi, constant1);
  const int32x4_t a3 = vmull_n_s16(a_lo, constant2);
  const int32x4_t a4 = vmull_n_s16(a_hi, constant2);
  *add_lo = vmlal_n_s16(a1, b_lo, constant2);
  *add_hi = vmlal_n_s16(a2, b_hi, constant2);
  *sub_lo = vmlsl_n_s16(a3, b_lo, constant1);
  *sub_hi = vmlsl_n_s16(a4, b_hi, constant1);
}

// fdct_round_shift(a * c1 +/- b * c2)
// Original Variant that performs normal implementation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_s32_noround(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant1,
    const tran_coef_t constant2, int32x4_t *add_lo, int32x4_t *add_hi,
    int32x4_t *sub_lo, int32x4_t *sub_hi) {
  const int32x4_t a1 = vmulq_n_s32(a_lo, constant1);
  const int32x4_t a2 = vmulq_n_s32(a_hi, constant1);
  const int32x4_t a3 = vmulq_n_s32(a_lo, constant2);
  const int32x4_t a4 = vmulq_n_s32(a_hi, constant2);
  *add_lo = vmlaq_n_s32(a1, b_lo, constant2);
  *add_hi = vmlaq_n_s32(a2, b_hi, constant2);
  *sub_lo = vmlsq_n_s32(a3, b_lo, constant1);
  *sub_hi = vmlsq_n_s32(a4, b_hi, constant1);
}

// fdct_round_shift(a * c1 +/- b * c2)
// Variant that performs normal implementation on half vector
// more accurate does 32-bit processing, takes and returns 16-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_half(const int16x4_t a,
                                            const int16x4_t b,
                                            const tran_coef_t constant1,
                                            const tran_coef_t constant2,
                                            int16x4_t *add, int16x4_t *sub) {
  const int32x4_t a1 = vmull_n_s16(a, constant1);
  const int32x4_t a2 = vmull_n_s16(a, constant2);
  const int32x4_t sum = vmlal_n_s16(a1, b, constant2);
  const int32x4_t diff = vmlsl_n_s16(a2, b, constant1);
  *add = vqrshrn_n_s32(sum, DCT_CONST_BITS);
  *sub = vqrshrn_n_s32(diff, DCT_CONST_BITS);
}

// fdct_round_shift(a * c1 +/- b * c2)
// Original Variant that performs normal implementation on full vector
// more accurate does 32-bit processing, takes and returns 16-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b,
                                       const tran_coef_t constant1,
                                       const tran_coef_t constant2,
                                       int16x8_t *add, int16x8_t *sub) {
  const int32x4_t a1 = vmull_n_s16(vget_low_s16(a), constant1);
  const int32x4_t a2 = vmull_n_s16(vget_high_s16(a), constant1);
  const int32x4_t a3 = vmull_n_s16(vget_low_s16(a), constant2);
  const int32x4_t a4 = vmull_n_s16(vget_high_s16(a), constant2);
  const int32x4_t sum0 = vmlal_n_s16(a1, vget_low_s16(b), constant2);
  const int32x4_t sum1 = vmlal_n_s16(a2, vget_high_s16(b), constant2);
  const int32x4_t diff0 = vmlsl_n_s16(a3, vget_low_s16(b), constant1);
  const int32x4_t diff1 = vmlsl_n_s16(a4, vget_high_s16(b), constant1);
  const int16x4_t rounded0 = vqrshrn_n_s32(sum0, DCT_CONST_BITS);
  const int16x4_t rounded1 = vqrshrn_n_s32(sum1, DCT_CONST_BITS);
  const int16x4_t rounded2 = vqrshrn_n_s32(diff0, DCT_CONST_BITS);
  const int16x4_t rounded3 = vqrshrn_n_s32(diff1, DCT_CONST_BITS);
  *add = vcombine_s16(rounded0, rounded1);
  *sub = vcombine_s16(rounded2, rounded3);
}

// fdct_round_shift(a * c1 +/- b * c2)
// Original Variant that performs normal implementation on full vector
// more accurate does 32-bit processing, takes and returns 32-bit values
// returns narrowed results
static INLINE void butterfly_two_coeff_s32(
    const int32x4_t a_lo, const int32x4_t a_hi, const int32x4_t b_lo,
    const int32x4_t b_hi, const tran_coef_t constant1,
    const tran_coef_t constant2, int32x4_t *add_lo, int32x4_t *add_hi,
    int32x4_t *sub_lo, int32x4_t *sub_hi) {
  const int32x4_t a1 = vmulq_n_s32(a_lo, constant1);
  const int32x4_t a2 = vmulq_n_s32(a_hi, constant1);
  const int32x4_t a3 = vmulq_n_s32(a_lo, constant2);
  const int32x4_t a4 = vmulq_n_s32(a_hi, constant2);
  const int32x4_t sum0 = vmlaq_n_s32(a1, b_lo, constant2);
  const int32x4_t sum1 = vmlaq_n_s32(a2, b_hi, constant2);
  const int32x4_t diff0 = vmlsq_n_s32(a3, b_lo, constant1);
  const int32x4_t diff1 = vmlsq_n_s32(a4, b_hi, constant1);
  *add_lo = vrshrq_n_s32(sum0, DCT_CONST_BITS);
  *add_hi = vrshrq_n_s32(sum1, DCT_CONST_BITS);
  *sub_lo = vrshrq_n_s32(diff0, DCT_CONST_BITS);
  *sub_hi = vrshrq_n_s32(diff1, DCT_CONST_BITS);
}

// Add 1 if positive, 2 if negative, and shift by 2.
// In practice, add 1, then add the sign bit, then shift without rounding.
static INLINE int16x8_t add_round_shift_s16(const int16x8_t a) {
  const int16x8_t one = vdupq_n_s16(1);
  const uint16x8_t a_u16 = vreinterpretq_u16_s16(a);
  const uint16x8_t a_sign_u16 = vshrq_n_u16(a_u16, 15);
  const int16x8_t a_sign_s16 = vreinterpretq_s16_u16(a_sign_u16);
  return vshrq_n_s16(vaddq_s16(vaddq_s16(a, a_sign_s16), one), 2);
}

// Add 1 if positive, 2 if negative, and shift by 2.
// In practice, add 1, then add the sign bit, then shift and round,
// return narrowed results
static INLINE int16x8_t add_round_shift_s32_narrow(const int32x4_t a_lo,
                                                   const int32x4_t a_hi) {
  const int32x4_t one = vdupq_n_s32(1);
  const uint32x4_t a_lo_u32 = vreinterpretq_u32_s32(a_lo);
  const uint32x4_t a_lo_sign_u32 = vshrq_n_u32(a_lo_u32, 31);
  const int32x4_t a_lo_sign_s32 = vreinterpretq_s32_u32(a_lo_sign_u32);
  const int16x4_t b_lo =
      vshrn_n_s32(vqaddq_s32(vqaddq_s32(a_lo, a_lo_sign_s32), one), 2);
  const uint32x4_t a_hi_u32 = vreinterpretq_u32_s32(a_hi);
  const uint32x4_t a_hi_sign_u32 = vshrq_n_u32(a_hi_u32, 31);
  const int32x4_t a_hi_sign_s32 = vreinterpretq_s32_u32(a_hi_sign_u32);
  const int16x4_t b_hi =
      vshrn_n_s32(vqaddq_s32(vqaddq_s32(a_hi, a_hi_sign_s32), one), 2);
  return vcombine_s16(b_lo, b_hi);
}

// Add 1 if negative, and shift by 1.
// In practice, add the sign bit, then shift and round
static INLINE int32x4_t add_round_shift_half_s32(const int32x4_t a) {
  const uint32x4_t a_u32 = vreinterpretq_u32_s32(a);
  const uint32x4_t a_sign_u32 = vshrq_n_u32(a_u32, 31);
  const int32x4_t a_sign_s32 = vreinterpretq_s32_u32(a_sign_u32);
  return vshrq_n_s32(vaddq_s32(a, a_sign_s32), 1);
}

// Add 1 if positive, 2 if negative, and shift by 2.
// In practice, add 1, then add the sign bit, then shift without rounding.
static INLINE int32x4_t add_round_shift_s32(const int32x4_t a) {
  const int32x4_t one = vdupq_n_s32(1);
  const uint32x4_t a_u32 = vreinterpretq_u32_s32(a);
  const uint32x4_t a_sign_u32 = vshrq_n_u32(a_u32, 31);
  const int32x4_t a_sign_s32 = vreinterpretq_s32_u32(a_sign_u32);
  return vshrq_n_s32(vaddq_s32(vaddq_s32(a, a_sign_s32), one), 2);
}

// Add 2 if positive, 1 if negative, and shift by 2.
// In practice, subtract the sign bit, then shift with rounding.
static INLINE int16x8_t sub_round_shift_s16(const int16x8_t a) {
  const uint16x8_t a_u16 = vreinterpretq_u16_s16(a);
  const uint16x8_t a_sign_u16 = vshrq_n_u16(a_u16, 15);
  const int16x8_t a_sign_s16 = vreinterpretq_s16_u16(a_sign_u16);
  return vrshrq_n_s16(vsubq_s16(a, a_sign_s16), 2);
}

// Add 2 if positive, 1 if negative, and shift by 2.
// In practice, subtract the sign bit, then shift with rounding.
static INLINE int32x4_t sub_round_shift_s32(const int32x4_t a) {
  const uint32x4_t a_u32 = vreinterpretq_u32_s32(a);
  const uint32x4_t a_sign_u32 = vshrq_n_u32(a_u32, 31);
  const int32x4_t a_sign_s32 = vreinterpretq_s32_u32(a_sign_u32);
  return vrshrq_n_s32(vsubq_s32(a, a_sign_s32), 2);
}

static INLINE int32x4_t add_s64_round_narrow(const int64x2_t *a /*[2]*/,
                                             const int64x2_t *b /*[2]*/) {
  int64x2_t result[2];
  result[0] = vaddq_s64(a[0], b[0]);
  result[1] = vaddq_s64(a[1], b[1]);
  return vcombine_s32(vrshrn_n_s64(result[0], DCT_CONST_BITS),
                      vrshrn_n_s64(result[1], DCT_CONST_BITS));
}

static INLINE int32x4_t sub_s64_round_narrow(const int64x2_t *a /*[2]*/,
                                             const int64x2_t *b /*[2]*/) {
  int64x2_t result[2];
  result[0] = vsubq_s64(a[0], b[0]);
  result[1] = vsubq_s64(a[1], b[1]);
  return vcombine_s32(vrshrn_n_s64(result[0], DCT_CONST_BITS),
                      vrshrn_n_s64(result[1], DCT_CONST_BITS));
}

static INLINE int32x4_t add_s32_s64_narrow(const int32x4_t a,
                                           const int32x4_t b) {
  int64x2_t a64[2], b64[2], result[2];
  a64[0] = vmovl_s32(vget_low_s32(a));
  a64[1] = vmovl_s32(vget_high_s32(a));
  b64[0] = vmovl_s32(vget_low_s32(b));
  b64[1] = vmovl_s32(vget_high_s32(b));
  result[0] = vaddq_s64(a64[0], b64[0]);
  result[1] = vaddq_s64(a64[1], b64[1]);
  return vcombine_s32(vmovn_s64(result[0]), vmovn_s64(result[1]));
}

static INLINE int32x4_t sub_s32_s64_narrow(const int32x4_t a,
                                           const int32x4_t b) {
  int64x2_t a64[2], b64[2], result[2];
  a64[0] = vmovl_s32(vget_low_s32(a));
  a64[1] = vmovl_s32(vget_high_s32(a));
  b64[0] = vmovl_s32(vget_low_s32(b));
  b64[1] = vmovl_s32(vget_high_s32(b));
  result[0] = vsubq_s64(a64[0], b64[0]);
  result[1] = vsubq_s64(a64[1], b64[1]);
  return vcombine_s32(vmovn_s64(result[0]), vmovn_s64(result[1]));
}

#endif  // VPX_VPX_DSP_ARM_FDCT_NEON_H_