diff options
Diffstat (limited to '')
| -rw-r--r-- | third_party/aom/aom_dsp/arm/highbd_obmc_variance_neon.c | 369 |
1 files changed, 369 insertions, 0 deletions
diff --git a/third_party/aom/aom_dsp/arm/highbd_obmc_variance_neon.c b/third_party/aom/aom_dsp/arm/highbd_obmc_variance_neon.c new file mode 100644 index 0000000000..d59224619b --- /dev/null +++ b/third_party/aom/aom_dsp/arm/highbd_obmc_variance_neon.c @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2023, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> + +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" + +#include "aom/aom_integer.h" +#include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/sum_neon.h" + +static INLINE void highbd_obmc_variance_8x1_s16_neon(uint16x8_t pre, + const int32_t *wsrc, + const int32_t *mask, + uint32x4_t *sse, + int32x4_t *sum) { + int16x8_t pre_s16 = vreinterpretq_s16_u16(pre); + int32x4_t wsrc_lo = vld1q_s32(&wsrc[0]); + int32x4_t wsrc_hi = vld1q_s32(&wsrc[4]); + + int32x4_t mask_lo = vld1q_s32(&mask[0]); + int32x4_t mask_hi = vld1q_s32(&mask[4]); + + int16x8_t mask_s16 = vcombine_s16(vmovn_s32(mask_lo), vmovn_s32(mask_hi)); + + int32x4_t diff_lo = vmull_s16(vget_low_s16(pre_s16), vget_low_s16(mask_s16)); + int32x4_t diff_hi = + vmull_s16(vget_high_s16(pre_s16), vget_high_s16(mask_s16)); + + diff_lo = vsubq_s32(wsrc_lo, diff_lo); + diff_hi = vsubq_s32(wsrc_hi, diff_hi); + + // ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away + // from zero, however vrshrq_n_s32 rounds to nearest with ties rounded up. + // This difference only affects the bit patterns at the rounding breakpoints + // exactly, so we can add -1 to all negative numbers to move the breakpoint + // one value across and into the correct rounding region. + diff_lo = vsraq_n_s32(diff_lo, diff_lo, 31); + diff_hi = vsraq_n_s32(diff_hi, diff_hi, 31); + int32x4_t round_lo = vrshrq_n_s32(diff_lo, 12); + int32x4_t round_hi = vrshrq_n_s32(diff_hi, 12); + + *sum = vaddq_s32(*sum, round_lo); + *sum = vaddq_s32(*sum, round_hi); + *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_lo), + vreinterpretq_u32_s32(round_lo)); + *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_hi), + vreinterpretq_u32_s32(round_hi)); +} + +// For 12-bit data, we can only accumulate up to 256 elements in the unsigned +// 32-bit elements (4095*4095*256 = 4292870400) before we have to accumulate +// into 64-bit elements. Therefore blocks of size 32x64, 64x32, 64x64, 64x128, +// 128x64, 128x128 are processed in a different helper function. +static INLINE void highbd_obmc_variance_xlarge_neon( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int width, int h, int h_limit, uint64_t *sse, + int64_t *sum) { + uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); + int32x4_t sum_s32 = vdupq_n_s32(0); + uint64x2_t sse_u64 = vdupq_n_u64(0); + + // 'h_limit' is the number of 'w'-width rows we can process before our 32-bit + // accumulator overflows. After hitting this limit we accumulate into 64-bit + // elements. + int h_tmp = h > h_limit ? h_limit : h; + + do { + uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; + int j = 0; + + do { + int i = 0; + + do { + uint16x8_t pre0 = vld1q_u16(pre_ptr + i); + highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32[0], + &sum_s32); + + uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8); + highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32[1], + &sum_s32); + + i += 16; + wsrc += 16; + mask += 16; + } while (i < width); + + pre_ptr += pre_stride; + j++; + } while (j < h_tmp); + + sse_u64 = vpadalq_u32(sse_u64, sse_u32[0]); + sse_u64 = vpadalq_u32(sse_u64, sse_u32[1]); + h -= h_tmp; + } while (h != 0); + + *sse = horizontal_add_u64x2(sse_u64); + *sum = horizontal_long_add_s32x4(sum_s32); +} + +static INLINE void highbd_obmc_variance_xlarge_neon_128xh( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 128, h, 16, sse, + sum); +} + +static INLINE void highbd_obmc_variance_xlarge_neon_64xh( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 64, h, 32, sse, + sum); +} + +static INLINE void highbd_obmc_variance_xlarge_neon_32xh( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 32, h, 64, sse, + sum); +} + +static INLINE void highbd_obmc_variance_large_neon( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int width, int h, uint64_t *sse, int64_t *sum) { + uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); + uint32x4_t sse_u32 = vdupq_n_u32(0); + int32x4_t sum_s32 = vdupq_n_s32(0); + + do { + int i = 0; + do { + uint16x8_t pre0 = vld1q_u16(pre_ptr + i); + highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32, &sum_s32); + + uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8); + highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32, + &sum_s32); + + i += 16; + wsrc += 16; + mask += 16; + } while (i < width); + + pre_ptr += pre_stride; + } while (--h != 0); + + *sse = horizontal_long_add_u32x4(sse_u32); + *sum = horizontal_long_add_s32x4(sum_s32); +} + +static INLINE void highbd_obmc_variance_neon_128xh( + const uint8_t *pre, int pre_stride, const int32_t *wsrc, + const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 128, h, sse, + sum); +} + +static INLINE void highbd_obmc_variance_neon_64xh(const uint8_t *pre, + int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int h, + uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 64, h, sse, sum); +} + +static INLINE void highbd_obmc_variance_neon_32xh(const uint8_t *pre, + int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int h, + uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 32, h, sse, sum); +} + +static INLINE void highbd_obmc_variance_neon_16xh(const uint8_t *pre, + int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int h, + uint64_t *sse, int64_t *sum) { + highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 16, h, sse, sum); +} + +static INLINE void highbd_obmc_variance_neon_8xh(const uint8_t *pre8, + int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int h, + uint64_t *sse, int64_t *sum) { + uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + uint32x4_t sse_u32 = vdupq_n_u32(0); + int32x4_t sum_s32 = vdupq_n_s32(0); + + do { + uint16x8_t pre_u16 = vld1q_u16(pre); + + highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32); + + pre += pre_stride; + wsrc += 8; + mask += 8; + } while (--h != 0); + + *sse = horizontal_long_add_u32x4(sse_u32); + *sum = horizontal_long_add_s32x4(sum_s32); +} + +static INLINE void highbd_obmc_variance_neon_4xh(const uint8_t *pre8, + int pre_stride, + const int32_t *wsrc, + const int32_t *mask, int h, + uint64_t *sse, int64_t *sum) { + assert(h % 2 == 0); + uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + uint32x4_t sse_u32 = vdupq_n_u32(0); + int32x4_t sum_s32 = vdupq_n_s32(0); + + do { + uint16x8_t pre_u16 = load_unaligned_u16_4x2(pre, pre_stride); + + highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32); + + pre += 2 * pre_stride; + wsrc += 8; + mask += 8; + h -= 2; + } while (h != 0); + + *sse = horizontal_long_add_u32x4(sse_u32); + *sum = horizontal_long_add_s32x4(sum_s32); +} + +static INLINE void highbd_8_obmc_variance_cast(int64_t sum64, uint64_t sse64, + int *sum, unsigned int *sse) { + *sum = (int)sum64; + *sse = (unsigned int)sse64; +} + +static INLINE void highbd_10_obmc_variance_cast(int64_t sum64, uint64_t sse64, + int *sum, unsigned int *sse) { + *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); +} + +static INLINE void highbd_12_obmc_variance_cast(int64_t sum64, uint64_t sse64, + int *sum, unsigned int *sse) { + *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); +} + +#define HIGHBD_OBMC_VARIANCE_WXH_NEON(w, h, bitdepth) \ + unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t sum64; \ + uint64_t sse64; \ + highbd_obmc_variance_neon_##w##xh(pre, pre_stride, wsrc, mask, h, &sse64, \ + &sum64); \ + highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \ + } + +#define HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(w, h, bitdepth) \ + unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + int sum; \ + int64_t sum64; \ + uint64_t sse64; \ + highbd_obmc_variance_xlarge_neon_##w##xh(pre, pre_stride, wsrc, mask, h, \ + &sse64, &sum64); \ + highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \ + return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \ + } + +// 8-bit +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 8) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 8) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 8) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 8) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 8) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 8) +HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 8) + +// 10-bit +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 10) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 10) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 10) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 10) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 10) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 10) +HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 10) + +// 12-bit +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 12) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 12) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 12) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 12) +HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 12) +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(32, 64, 12) + +HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 12) +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 32, 12) +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 64, 12) +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 128, 12) + +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 64, 12) +HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 128, 12) |