summaryrefslogtreecommitdiffstats
path: root/media/libvpx/libvpx/vpx_dsp/ppc
diff options
context:
space:
mode:
Diffstat (limited to 'media/libvpx/libvpx/vpx_dsp/ppc')
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/bitdepth_conversion_vsx.h47
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/deblock_vsx.c374
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/fdct32x32_vsx.c553
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/hadamard_vsx.c119
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/intrapred_vsx.c767
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.c1828
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.h48
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c301
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/sad_vsx.c261
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/subtract_vsx.c117
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/transpose_vsx.h133
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/txfm_common_vsx.h90
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/types_vsx.h108
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/variance_vsx.c271
-rw-r--r--media/libvpx/libvpx/vpx_dsp/ppc/vpx_convolve_vsx.c408
15 files changed, 5425 insertions, 0 deletions
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/bitdepth_conversion_vsx.h b/media/libvpx/libvpx/vpx_dsp/ppc/bitdepth_conversion_vsx.h
new file mode 100644
index 0000000000..7ac873f9fc
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/bitdepth_conversion_vsx.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
+#define VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
+
+#include "./vpx_config.h"
+#include "vpx/vpx_integer.h"
+#include "vpx_dsp/vpx_dsp_common.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+// Load 8 16 bit values. If the source is 32 bits then pack down with
+// saturation.
+static INLINE int16x8_t load_tran_low(int32_t c, const tran_low_t *s) {
+#if CONFIG_VP9_HIGHBITDEPTH
+ int32x4_t u = vec_vsx_ld(c, s);
+ int32x4_t v = vec_vsx_ld(c, s + 4);
+ return vec_packs(u, v);
+#else
+ return vec_vsx_ld(c, s);
+#endif
+}
+
+// Store 8 16 bit values. If the destination is 32 bits then sign extend the
+// values by multiplying by 1.
+static INLINE void store_tran_low(int16x8_t v, int32_t c, tran_low_t *s) {
+#if CONFIG_VP9_HIGHBITDEPTH
+ const int16x8_t one = vec_splat_s16(1);
+ const int32x4_t even = vec_mule(v, one);
+ const int32x4_t odd = vec_mulo(v, one);
+ const int32x4_t high = vec_mergeh(even, odd);
+ const int32x4_t low = vec_mergel(even, odd);
+ vec_vsx_st(high, c, s);
+ vec_vsx_st(low, c, s + 4);
+#else
+ vec_vsx_st(v, c, s);
+#endif
+}
+
+#endif // VPX_VPX_DSP_PPC_BITDEPTH_CONVERSION_VSX_H_
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/deblock_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/deblock_vsx.c
new file mode 100644
index 0000000000..2129911696
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/deblock_vsx.c
@@ -0,0 +1,374 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+extern const int16_t vpx_rv[];
+
+static const uint8x16_t load_merge = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A,
+ 0x0C, 0x0E, 0x18, 0x19, 0x1A, 0x1B,
+ 0x1C, 0x1D, 0x1E, 0x1F };
+
+static const uint8x16_t st8_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x18, 0x19, 0x1A, 0x1B,
+ 0x1C, 0x1D, 0x1E, 0x1F };
+
+static INLINE uint8x16_t apply_filter(uint8x16_t ctx[4], uint8x16_t v,
+ uint8x16_t filter) {
+ const uint8x16_t k1 = vec_avg(ctx[0], ctx[1]);
+ const uint8x16_t k2 = vec_avg(ctx[3], ctx[2]);
+ const uint8x16_t k3 = vec_avg(k1, k2);
+ const uint8x16_t f_a = vec_max(vec_absd(v, ctx[0]), vec_absd(v, ctx[1]));
+ const uint8x16_t f_b = vec_max(vec_absd(v, ctx[2]), vec_absd(v, ctx[3]));
+ const bool8x16_t mask = vec_cmplt(vec_max(f_a, f_b), filter);
+ return vec_sel(v, vec_avg(k3, v), mask);
+}
+
+static INLINE void vert_ctx(uint8x16_t ctx[4], int col, uint8_t *src,
+ int stride) {
+ ctx[0] = vec_vsx_ld(col - 2 * stride, src);
+ ctx[1] = vec_vsx_ld(col - stride, src);
+ ctx[2] = vec_vsx_ld(col + stride, src);
+ ctx[3] = vec_vsx_ld(col + 2 * stride, src);
+}
+
+static INLINE void horz_ctx(uint8x16_t ctx[4], uint8x16_t left_ctx,
+ uint8x16_t v, uint8x16_t right_ctx) {
+ static const uint8x16_t l2_perm = { 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
+ 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
+ 0x1A, 0x1B, 0x1C, 0x1D };
+
+ static const uint8x16_t l1_perm = { 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14,
+ 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A,
+ 0x1B, 0x1C, 0x1D, 0x1E };
+
+ static const uint8x16_t r1_perm = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+ 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
+ 0x0D, 0x0E, 0x0F, 0x10 };
+
+ static const uint8x16_t r2_perm = { 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
+ 0x0E, 0x0F, 0x10, 0x11 };
+ ctx[0] = vec_perm(left_ctx, v, l2_perm);
+ ctx[1] = vec_perm(left_ctx, v, l1_perm);
+ ctx[2] = vec_perm(v, right_ctx, r1_perm);
+ ctx[3] = vec_perm(v, right_ctx, r2_perm);
+}
+void vpx_post_proc_down_and_across_mb_row_vsx(unsigned char *src_ptr,
+ unsigned char *dst_ptr,
+ int src_pixels_per_line,
+ int dst_pixels_per_line, int cols,
+ unsigned char *f, int size) {
+ int row, col;
+ uint8x16_t ctx[4], out, v, left_ctx;
+
+ for (row = 0; row < size; row++) {
+ for (col = 0; col < cols - 8; col += 16) {
+ const uint8x16_t filter = vec_vsx_ld(col, f);
+ v = vec_vsx_ld(col, src_ptr);
+ vert_ctx(ctx, col, src_ptr, src_pixels_per_line);
+ vec_vsx_st(apply_filter(ctx, v, filter), col, dst_ptr);
+ }
+
+ if (col != cols) {
+ const uint8x16_t filter = vec_vsx_ld(col, f);
+ v = vec_vsx_ld(col, src_ptr);
+ vert_ctx(ctx, col, src_ptr, src_pixels_per_line);
+ out = apply_filter(ctx, v, filter);
+ vec_vsx_st(vec_perm(out, v, st8_perm), col, dst_ptr);
+ }
+
+ /* now post_proc_across */
+ left_ctx = vec_splats(dst_ptr[0]);
+ v = vec_vsx_ld(0, dst_ptr);
+ for (col = 0; col < cols - 8; col += 16) {
+ const uint8x16_t filter = vec_vsx_ld(col, f);
+ const uint8x16_t right_ctx = (col + 16 == cols)
+ ? vec_splats(dst_ptr[cols - 1])
+ : vec_vsx_ld(col, dst_ptr + 16);
+ horz_ctx(ctx, left_ctx, v, right_ctx);
+ vec_vsx_st(apply_filter(ctx, v, filter), col, dst_ptr);
+ left_ctx = v;
+ v = right_ctx;
+ }
+
+ if (col != cols) {
+ const uint8x16_t filter = vec_vsx_ld(col, f);
+ const uint8x16_t right_ctx = vec_splats(dst_ptr[cols - 1]);
+ horz_ctx(ctx, left_ctx, v, right_ctx);
+ out = apply_filter(ctx, v, filter);
+ vec_vsx_st(vec_perm(out, v, st8_perm), col, dst_ptr);
+ }
+
+ src_ptr += src_pixels_per_line;
+ dst_ptr += dst_pixels_per_line;
+ }
+}
+
+// C: s[c + 7]
+static INLINE int16x8_t next7l_s16(uint8x16_t c) {
+ static const uint8x16_t next7_perm = {
+ 0x07, 0x10, 0x08, 0x11, 0x09, 0x12, 0x0A, 0x13,
+ 0x0B, 0x14, 0x0C, 0x15, 0x0D, 0x16, 0x0E, 0x17,
+ };
+ return (int16x8_t)vec_perm(c, vec_zeros_u8, next7_perm);
+}
+
+// Slide across window and add.
+static INLINE int16x8_t slide_sum_s16(int16x8_t x) {
+ // x = A B C D E F G H
+ //
+ // 0 A B C D E F G
+ const int16x8_t sum1 = vec_add(x, vec_slo(x, vec_splats((int8_t)(2 << 3))));
+ // 0 0 A B C D E F
+ const int16x8_t sum2 = vec_add(vec_slo(x, vec_splats((int8_t)(4 << 3))),
+ // 0 0 0 A B C D E
+ vec_slo(x, vec_splats((int8_t)(6 << 3))));
+ // 0 0 0 0 A B C D
+ const int16x8_t sum3 = vec_add(vec_slo(x, vec_splats((int8_t)(8 << 3))),
+ // 0 0 0 0 0 A B C
+ vec_slo(x, vec_splats((int8_t)(10 << 3))));
+ // 0 0 0 0 0 0 A B
+ const int16x8_t sum4 = vec_add(vec_slo(x, vec_splats((int8_t)(12 << 3))),
+ // 0 0 0 0 0 0 0 A
+ vec_slo(x, vec_splats((int8_t)(14 << 3))));
+ return vec_add(vec_add(sum1, sum2), vec_add(sum3, sum4));
+}
+
+// Slide across window and add.
+static INLINE int32x4_t slide_sumsq_s32(int32x4_t xsq_even, int32x4_t xsq_odd) {
+ // 0 A C E
+ // + 0 B D F
+ int32x4_t sumsq_1 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(4 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(4 << 3))));
+ // 0 0 A C
+ // + 0 0 B D
+ int32x4_t sumsq_2 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(8 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(8 << 3))));
+ // 0 0 0 A
+ // + 0 0 0 B
+ int32x4_t sumsq_3 = vec_add(vec_slo(xsq_even, vec_splats((int8_t)(12 << 3))),
+ vec_slo(xsq_odd, vec_splats((int8_t)(12 << 3))));
+ sumsq_1 = vec_add(sumsq_1, xsq_even);
+ sumsq_2 = vec_add(sumsq_2, sumsq_3);
+ return vec_add(sumsq_1, sumsq_2);
+}
+
+// C: (b + sum + val) >> 4
+static INLINE int16x8_t filter_s16(int16x8_t b, int16x8_t sum, int16x8_t val) {
+ return vec_sra(vec_add(vec_add(b, sum), val), vec_splats((uint16_t)4));
+}
+
+// C: sumsq * 15 - sum * sum
+static INLINE bool16x8_t mask_s16(int32x4_t sumsq_even, int32x4_t sumsq_odd,
+ int16x8_t sum, int32x4_t lim) {
+ static const uint8x16_t mask_merge = { 0x00, 0x01, 0x10, 0x11, 0x04, 0x05,
+ 0x14, 0x15, 0x08, 0x09, 0x18, 0x19,
+ 0x0C, 0x0D, 0x1C, 0x1D };
+ const int32x4_t sumsq_odd_scaled =
+ vec_mul(sumsq_odd, vec_splats((int32_t)15));
+ const int32x4_t sumsq_even_scaled =
+ vec_mul(sumsq_even, vec_splats((int32_t)15));
+ const int32x4_t thres_odd = vec_sub(sumsq_odd_scaled, vec_mulo(sum, sum));
+ const int32x4_t thres_even = vec_sub(sumsq_even_scaled, vec_mule(sum, sum));
+
+ const bool32x4_t mask_odd = vec_cmplt(thres_odd, lim);
+ const bool32x4_t mask_even = vec_cmplt(thres_even, lim);
+ return vec_perm((bool16x8_t)mask_even, (bool16x8_t)mask_odd, mask_merge);
+}
+
+void vpx_mbpost_proc_across_ip_vsx(unsigned char *src, int pitch, int rows,
+ int cols, int flimit) {
+ int row, col;
+ const int32x4_t lim = vec_splats(flimit);
+
+ // 8 columns are processed at a time.
+ assert(cols % 8 == 0);
+
+ for (row = 0; row < rows; row++) {
+ // The sum is signed and requires at most 13 bits.
+ // (8 bits + sign) * 15 (4 bits)
+ int16x8_t sum;
+ // The sum of squares requires at most 20 bits.
+ // (16 bits + sign) * 15 (4 bits)
+ int32x4_t sumsq_even, sumsq_odd;
+
+ // Fill left context with first col.
+ int16x8_t left_ctx = vec_splats((int16_t)src[0]);
+ int16_t s = src[0] * 9;
+ int32_t ssq = src[0] * src[0] * 9 + 16;
+
+ // Fill the next 6 columns of the sliding window with cols 2 to 7.
+ for (col = 1; col <= 6; ++col) {
+ s += src[col];
+ ssq += src[col] * src[col];
+ }
+ // Set this sum to every element in the window.
+ sum = vec_splats(s);
+ sumsq_even = vec_splats(ssq);
+ sumsq_odd = vec_splats(ssq);
+
+ for (col = 0; col < cols; col += 8) {
+ bool16x8_t mask;
+ int16x8_t filtered, masked;
+ uint8x16_t out;
+
+ const uint8x16_t val = vec_vsx_ld(0, src + col);
+ const int16x8_t val_high = unpack_to_s16_h(val);
+
+ // C: s[c + 7]
+ const int16x8_t right_ctx = (col + 8 == cols)
+ ? vec_splats((int16_t)src[col + 7])
+ : next7l_s16(val);
+
+ // C: x = s[c + 7] - s[c - 8];
+ const int16x8_t x = vec_sub(right_ctx, left_ctx);
+ const int32x4_t xsq_even =
+ vec_sub(vec_mule(right_ctx, right_ctx), vec_mule(left_ctx, left_ctx));
+ const int32x4_t xsq_odd =
+ vec_sub(vec_mulo(right_ctx, right_ctx), vec_mulo(left_ctx, left_ctx));
+
+ const int32x4_t sumsq_tmp = slide_sumsq_s32(xsq_even, xsq_odd);
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_even = vec_add(sumsq_even, sumsq_tmp);
+ // B D F G
+ // A C E G
+ // 0 B D F
+ // 0 A C E
+ // 0 0 B D
+ // 0 0 A C
+ // 0 0 0 B
+ // 0 0 0 A
+ sumsq_odd = vec_add(sumsq_odd, vec_add(sumsq_tmp, xsq_odd));
+
+ sum = vec_add(sum, slide_sum_s16(x));
+
+ // C: (8 + sum + s[c]) >> 4
+ filtered = filter_s16(vec_splats((int16_t)8), sum, val_high);
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
+ masked = vec_sel(val_high, filtered, mask);
+
+ out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, src + col), load_merge);
+ vec_vsx_st(out, 0, src + col);
+
+ // Update window sum and square sum
+ sum = vec_splat(sum, 7);
+ sumsq_even = vec_splat(sumsq_odd, 3);
+ sumsq_odd = vec_splat(sumsq_odd, 3);
+
+ // C: s[c - 8] (for next iteration)
+ left_ctx = val_high;
+ }
+ src += pitch;
+ }
+}
+
+void vpx_mbpost_proc_down_vsx(uint8_t *dst, int pitch, int rows, int cols,
+ int flimit) {
+ int col, row, i;
+ int16x8_t window[16];
+ const int32x4_t lim = vec_splats(flimit);
+
+ // 8 columns are processed at a time.
+ assert(cols % 8 == 0);
+ // If rows is less than 8 the bottom border extension fails.
+ assert(rows >= 8);
+
+ for (col = 0; col < cols; col += 8) {
+ // The sum is signed and requires at most 13 bits.
+ // (8 bits + sign) * 15 (4 bits)
+ int16x8_t r1, sum;
+ // The sum of squares requires at most 20 bits.
+ // (16 bits + sign) * 15 (4 bits)
+ int32x4_t sumsq_even, sumsq_odd;
+
+ r1 = unpack_to_s16_h(vec_vsx_ld(0, dst));
+ // Fill sliding window with first row.
+ for (i = 0; i <= 8; i++) {
+ window[i] = r1;
+ }
+ // First 9 rows of the sliding window are the same.
+ // sum = r1 * 9
+ sum = vec_mladd(r1, vec_splats((int16_t)9), vec_zeros_s16);
+
+ // sumsq = r1 * r1 * 9
+ sumsq_even = vec_mule(sum, r1);
+ sumsq_odd = vec_mulo(sum, r1);
+
+ // Fill the next 6 rows of the sliding window with rows 2 to 7.
+ for (i = 1; i <= 6; ++i) {
+ const int16x8_t next_row = unpack_to_s16_h(vec_vsx_ld(i * pitch, dst));
+ window[i + 8] = next_row;
+ sum = vec_add(sum, next_row);
+ sumsq_odd = vec_add(sumsq_odd, vec_mulo(next_row, next_row));
+ sumsq_even = vec_add(sumsq_even, vec_mule(next_row, next_row));
+ }
+
+ for (row = 0; row < rows; row++) {
+ int32x4_t d15_even, d15_odd, d0_even, d0_odd;
+ bool16x8_t mask;
+ int16x8_t filtered, masked;
+ uint8x16_t out;
+
+ const int16x8_t rv = vec_vsx_ld(0, vpx_rv + (row & 127));
+
+ // Move the sliding window
+ if (row + 7 < rows) {
+ window[15] = unpack_to_s16_h(vec_vsx_ld((row + 7) * pitch, dst));
+ } else {
+ window[15] = window[14];
+ }
+
+ // C: sum += s[7 * pitch] - s[-8 * pitch];
+ sum = vec_add(sum, vec_sub(window[15], window[0]));
+
+ // C: sumsq += s[7 * pitch] * s[7 * pitch] - s[-8 * pitch] * s[-8 *
+ // pitch];
+ // Optimization Note: Caching a squared-window for odd and even is
+ // slower than just repeating the multiplies.
+ d15_odd = vec_mulo(window[15], window[15]);
+ d15_even = vec_mule(window[15], window[15]);
+ d0_odd = vec_mulo(window[0], window[0]);
+ d0_even = vec_mule(window[0], window[0]);
+ sumsq_odd = vec_add(sumsq_odd, vec_sub(d15_odd, d0_odd));
+ sumsq_even = vec_add(sumsq_even, vec_sub(d15_even, d0_even));
+
+ // C: (vpx_rv[(r & 127) + (c & 7)] + sum + s[0]) >> 4
+ filtered = filter_s16(rv, sum, window[8]);
+
+ // C: sumsq * 15 - sum * sum
+ mask = mask_s16(sumsq_even, sumsq_odd, sum, lim);
+ masked = vec_sel(window[8], filtered, mask);
+
+ // TODO(ltrudeau) If cols % 16 == 0, we could just process 16 per
+ // iteration
+ out = vec_perm((uint8x16_t)masked, vec_vsx_ld(0, dst + row * pitch),
+ load_merge);
+ vec_vsx_st(out, 0, dst + row * pitch);
+
+ // Optimization Note: Turns out that the following loop is faster than
+ // using pointers to manage the sliding window.
+ for (i = 1; i < 16; i++) {
+ window[i - 1] = window[i];
+ }
+ }
+ dst += 8;
+ }
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/fdct32x32_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/fdct32x32_vsx.c
new file mode 100644
index 0000000000..328b0e3130
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/fdct32x32_vsx.c
@@ -0,0 +1,553 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "./vpx_config.h"
+#include "./vpx_dsp_rtcd.h"
+
+#include "vpx_dsp/ppc/transpose_vsx.h"
+#include "vpx_dsp/ppc/txfm_common_vsx.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+// Returns ((a +/- b) * cospi16 + (2 << 13)) >> 14.
+static INLINE void single_butterfly(int16x8_t a, int16x8_t b, int16x8_t *add,
+ int16x8_t *sub) {
+ // Since a + b can overflow 16 bits, the multiplication is distributed
+ // (a * c +/- b * c).
+ const int32x4_t ac_e = vec_mule(a, cospi16_v);
+ const int32x4_t ac_o = vec_mulo(a, cospi16_v);
+ const int32x4_t bc_e = vec_mule(b, cospi16_v);
+ const int32x4_t bc_o = vec_mulo(b, cospi16_v);
+
+ // Reuse the same multiplies for sum and difference.
+ const int32x4_t sum_e = vec_add(ac_e, bc_e);
+ const int32x4_t sum_o = vec_add(ac_o, bc_o);
+ const int32x4_t diff_e = vec_sub(ac_e, bc_e);
+ const int32x4_t diff_o = vec_sub(ac_o, bc_o);
+
+ // Add rounding offset
+ const int32x4_t rsum_o = vec_add(sum_o, vec_dct_const_rounding);
+ const int32x4_t rsum_e = vec_add(sum_e, vec_dct_const_rounding);
+ const int32x4_t rdiff_o = vec_add(diff_o, vec_dct_const_rounding);
+ const int32x4_t rdiff_e = vec_add(diff_e, vec_dct_const_rounding);
+
+ const int32x4_t ssum_o = vec_sra(rsum_o, vec_dct_const_bits);
+ const int32x4_t ssum_e = vec_sra(rsum_e, vec_dct_const_bits);
+ const int32x4_t sdiff_o = vec_sra(rdiff_o, vec_dct_const_bits);
+ const int32x4_t sdiff_e = vec_sra(rdiff_e, vec_dct_const_bits);
+
+ // There's no pack operation for even and odd, so we need to permute.
+ *add = (int16x8_t)vec_perm(ssum_e, ssum_o, vec_perm_odd_even_pack);
+ *sub = (int16x8_t)vec_perm(sdiff_e, sdiff_o, vec_perm_odd_even_pack);
+}
+
+// Returns (a * c1 +/- b * c2 + (2 << 13)) >> 14
+static INLINE void double_butterfly(int16x8_t a, int16x8_t c1, int16x8_t b,
+ int16x8_t c2, int16x8_t *add,
+ int16x8_t *sub) {
+ const int32x4_t ac1_o = vec_mulo(a, c1);
+ const int32x4_t ac1_e = vec_mule(a, c1);
+ const int32x4_t ac2_o = vec_mulo(a, c2);
+ const int32x4_t ac2_e = vec_mule(a, c2);
+
+ const int32x4_t bc1_o = vec_mulo(b, c1);
+ const int32x4_t bc1_e = vec_mule(b, c1);
+ const int32x4_t bc2_o = vec_mulo(b, c2);
+ const int32x4_t bc2_e = vec_mule(b, c2);
+
+ const int32x4_t sum_o = vec_add(ac1_o, bc2_o);
+ const int32x4_t sum_e = vec_add(ac1_e, bc2_e);
+ const int32x4_t diff_o = vec_sub(ac2_o, bc1_o);
+ const int32x4_t diff_e = vec_sub(ac2_e, bc1_e);
+
+ // Add rounding offset
+ const int32x4_t rsum_o = vec_add(sum_o, vec_dct_const_rounding);
+ const int32x4_t rsum_e = vec_add(sum_e, vec_dct_const_rounding);
+ const int32x4_t rdiff_o = vec_add(diff_o, vec_dct_const_rounding);
+ const int32x4_t rdiff_e = vec_add(diff_e, vec_dct_const_rounding);
+
+ const int32x4_t ssum_o = vec_sra(rsum_o, vec_dct_const_bits);
+ const int32x4_t ssum_e = vec_sra(rsum_e, vec_dct_const_bits);
+ const int32x4_t sdiff_o = vec_sra(rdiff_o, vec_dct_const_bits);
+ const int32x4_t sdiff_e = vec_sra(rdiff_e, vec_dct_const_bits);
+
+ // There's no pack operation for even and odd, so we need to permute.
+ *add = (int16x8_t)vec_perm(ssum_e, ssum_o, vec_perm_odd_even_pack);
+ *sub = (int16x8_t)vec_perm(sdiff_e, sdiff_o, vec_perm_odd_even_pack);
+}
+
+// While other architecture combine the load and the stage 1 operations, Power9
+// benchmarking show no benefit in such an approach.
+static INLINE void load(const int16_t *a, int stride, int16x8_t *b) {
+ // Tried out different combinations of load and shift instructions, this is
+ // the fastest one.
+ {
+ const int16x8_t l0 = vec_vsx_ld(0, a);
+ const int16x8_t l1 = vec_vsx_ld(0, a + stride);
+ const int16x8_t l2 = vec_vsx_ld(0, a + 2 * stride);
+ const int16x8_t l3 = vec_vsx_ld(0, a + 3 * stride);
+ const int16x8_t l4 = vec_vsx_ld(0, a + 4 * stride);
+ const int16x8_t l5 = vec_vsx_ld(0, a + 5 * stride);
+ const int16x8_t l6 = vec_vsx_ld(0, a + 6 * stride);
+ const int16x8_t l7 = vec_vsx_ld(0, a + 7 * stride);
+
+ const int16x8_t l8 = vec_vsx_ld(0, a + 8 * stride);
+ const int16x8_t l9 = vec_vsx_ld(0, a + 9 * stride);
+ const int16x8_t l10 = vec_vsx_ld(0, a + 10 * stride);
+ const int16x8_t l11 = vec_vsx_ld(0, a + 11 * stride);
+ const int16x8_t l12 = vec_vsx_ld(0, a + 12 * stride);
+ const int16x8_t l13 = vec_vsx_ld(0, a + 13 * stride);
+ const int16x8_t l14 = vec_vsx_ld(0, a + 14 * stride);
+ const int16x8_t l15 = vec_vsx_ld(0, a + 15 * stride);
+
+ b[0] = vec_sl(l0, vec_dct_scale_log2);
+ b[1] = vec_sl(l1, vec_dct_scale_log2);
+ b[2] = vec_sl(l2, vec_dct_scale_log2);
+ b[3] = vec_sl(l3, vec_dct_scale_log2);
+ b[4] = vec_sl(l4, vec_dct_scale_log2);
+ b[5] = vec_sl(l5, vec_dct_scale_log2);
+ b[6] = vec_sl(l6, vec_dct_scale_log2);
+ b[7] = vec_sl(l7, vec_dct_scale_log2);
+
+ b[8] = vec_sl(l8, vec_dct_scale_log2);
+ b[9] = vec_sl(l9, vec_dct_scale_log2);
+ b[10] = vec_sl(l10, vec_dct_scale_log2);
+ b[11] = vec_sl(l11, vec_dct_scale_log2);
+ b[12] = vec_sl(l12, vec_dct_scale_log2);
+ b[13] = vec_sl(l13, vec_dct_scale_log2);
+ b[14] = vec_sl(l14, vec_dct_scale_log2);
+ b[15] = vec_sl(l15, vec_dct_scale_log2);
+ }
+ {
+ const int16x8_t l16 = vec_vsx_ld(0, a + 16 * stride);
+ const int16x8_t l17 = vec_vsx_ld(0, a + 17 * stride);
+ const int16x8_t l18 = vec_vsx_ld(0, a + 18 * stride);
+ const int16x8_t l19 = vec_vsx_ld(0, a + 19 * stride);
+ const int16x8_t l20 = vec_vsx_ld(0, a + 20 * stride);
+ const int16x8_t l21 = vec_vsx_ld(0, a + 21 * stride);
+ const int16x8_t l22 = vec_vsx_ld(0, a + 22 * stride);
+ const int16x8_t l23 = vec_vsx_ld(0, a + 23 * stride);
+
+ const int16x8_t l24 = vec_vsx_ld(0, a + 24 * stride);
+ const int16x8_t l25 = vec_vsx_ld(0, a + 25 * stride);
+ const int16x8_t l26 = vec_vsx_ld(0, a + 26 * stride);
+ const int16x8_t l27 = vec_vsx_ld(0, a + 27 * stride);
+ const int16x8_t l28 = vec_vsx_ld(0, a + 28 * stride);
+ const int16x8_t l29 = vec_vsx_ld(0, a + 29 * stride);
+ const int16x8_t l30 = vec_vsx_ld(0, a + 30 * stride);
+ const int16x8_t l31 = vec_vsx_ld(0, a + 31 * stride);
+
+ b[16] = vec_sl(l16, vec_dct_scale_log2);
+ b[17] = vec_sl(l17, vec_dct_scale_log2);
+ b[18] = vec_sl(l18, vec_dct_scale_log2);
+ b[19] = vec_sl(l19, vec_dct_scale_log2);
+ b[20] = vec_sl(l20, vec_dct_scale_log2);
+ b[21] = vec_sl(l21, vec_dct_scale_log2);
+ b[22] = vec_sl(l22, vec_dct_scale_log2);
+ b[23] = vec_sl(l23, vec_dct_scale_log2);
+
+ b[24] = vec_sl(l24, vec_dct_scale_log2);
+ b[25] = vec_sl(l25, vec_dct_scale_log2);
+ b[26] = vec_sl(l26, vec_dct_scale_log2);
+ b[27] = vec_sl(l27, vec_dct_scale_log2);
+ b[28] = vec_sl(l28, vec_dct_scale_log2);
+ b[29] = vec_sl(l29, vec_dct_scale_log2);
+ b[30] = vec_sl(l30, vec_dct_scale_log2);
+ b[31] = vec_sl(l31, vec_dct_scale_log2);
+ }
+}
+
+static INLINE void store(tran_low_t *a, const int16x8_t *b) {
+ vec_vsx_st(b[0], 0, a);
+ vec_vsx_st(b[8], 0, a + 8);
+ vec_vsx_st(b[16], 0, a + 16);
+ vec_vsx_st(b[24], 0, a + 24);
+
+ vec_vsx_st(b[1], 0, a + 32);
+ vec_vsx_st(b[9], 0, a + 40);
+ vec_vsx_st(b[17], 0, a + 48);
+ vec_vsx_st(b[25], 0, a + 56);
+
+ vec_vsx_st(b[2], 0, a + 64);
+ vec_vsx_st(b[10], 0, a + 72);
+ vec_vsx_st(b[18], 0, a + 80);
+ vec_vsx_st(b[26], 0, a + 88);
+
+ vec_vsx_st(b[3], 0, a + 96);
+ vec_vsx_st(b[11], 0, a + 104);
+ vec_vsx_st(b[19], 0, a + 112);
+ vec_vsx_st(b[27], 0, a + 120);
+
+ vec_vsx_st(b[4], 0, a + 128);
+ vec_vsx_st(b[12], 0, a + 136);
+ vec_vsx_st(b[20], 0, a + 144);
+ vec_vsx_st(b[28], 0, a + 152);
+
+ vec_vsx_st(b[5], 0, a + 160);
+ vec_vsx_st(b[13], 0, a + 168);
+ vec_vsx_st(b[21], 0, a + 176);
+ vec_vsx_st(b[29], 0, a + 184);
+
+ vec_vsx_st(b[6], 0, a + 192);
+ vec_vsx_st(b[14], 0, a + 200);
+ vec_vsx_st(b[22], 0, a + 208);
+ vec_vsx_st(b[30], 0, a + 216);
+
+ vec_vsx_st(b[7], 0, a + 224);
+ vec_vsx_st(b[15], 0, a + 232);
+ vec_vsx_st(b[23], 0, a + 240);
+ vec_vsx_st(b[31], 0, a + 248);
+}
+
+// Returns 1 if negative 0 if positive
+static INLINE int16x8_t vec_sign_s16(int16x8_t a) {
+ return vec_sr(a, vec_shift_sign_s16);
+}
+
+// Add 2 if positive, 1 if negative, and shift by 2.
+static INLINE int16x8_t sub_round_shift(const int16x8_t a) {
+ const int16x8_t sign = vec_sign_s16(a);
+ return vec_sra(vec_sub(vec_add(a, vec_twos_s16), sign), vec_dct_scale_log2);
+}
+
+// Add 1 if positive, 2 if negative, and shift by 2.
+// In practice, add 1, then add the sign bit, then shift without rounding.
+static INLINE int16x8_t add_round_shift_s16(const int16x8_t a) {
+ const int16x8_t sign = vec_sign_s16(a);
+ return vec_sra(vec_add(vec_add(a, vec_ones_s16), sign), vec_dct_scale_log2);
+}
+
+static void fdct32_vsx(const int16x8_t *in, int16x8_t *out, int pass) {
+ int16x8_t temp0[32]; // Hold stages: 1, 4, 7
+ int16x8_t temp1[32]; // Hold stages: 2, 5
+ int16x8_t temp2[32]; // Hold stages: 3, 6
+ int i;
+
+ // Stage 1
+ // Unrolling this loops actually slows down Power9 benchmarks
+ for (i = 0; i < 16; i++) {
+ temp0[i] = vec_add(in[i], in[31 - i]);
+ // pass through to stage 3.
+ temp1[i + 16] = vec_sub(in[15 - i], in[i + 16]);
+ }
+
+ // Stage 2
+ // Unrolling this loops actually slows down Power9 benchmarks
+ for (i = 0; i < 8; i++) {
+ temp1[i] = vec_add(temp0[i], temp0[15 - i]);
+ temp1[i + 8] = vec_sub(temp0[7 - i], temp0[i + 8]);
+ }
+
+ // Apply butterflies (in place) on pass through to stage 3.
+ single_butterfly(temp1[27], temp1[20], &temp1[27], &temp1[20]);
+ single_butterfly(temp1[26], temp1[21], &temp1[26], &temp1[21]);
+ single_butterfly(temp1[25], temp1[22], &temp1[25], &temp1[22]);
+ single_butterfly(temp1[24], temp1[23], &temp1[24], &temp1[23]);
+
+ // dump the magnitude by 4, hence the intermediate values are within
+ // the range of 16 bits.
+ if (pass) {
+ temp1[0] = add_round_shift_s16(temp1[0]);
+ temp1[1] = add_round_shift_s16(temp1[1]);
+ temp1[2] = add_round_shift_s16(temp1[2]);
+ temp1[3] = add_round_shift_s16(temp1[3]);
+ temp1[4] = add_round_shift_s16(temp1[4]);
+ temp1[5] = add_round_shift_s16(temp1[5]);
+ temp1[6] = add_round_shift_s16(temp1[6]);
+ temp1[7] = add_round_shift_s16(temp1[7]);
+ temp1[8] = add_round_shift_s16(temp1[8]);
+ temp1[9] = add_round_shift_s16(temp1[9]);
+ temp1[10] = add_round_shift_s16(temp1[10]);
+ temp1[11] = add_round_shift_s16(temp1[11]);
+ temp1[12] = add_round_shift_s16(temp1[12]);
+ temp1[13] = add_round_shift_s16(temp1[13]);
+ temp1[14] = add_round_shift_s16(temp1[14]);
+ temp1[15] = add_round_shift_s16(temp1[15]);
+
+ temp1[16] = add_round_shift_s16(temp1[16]);
+ temp1[17] = add_round_shift_s16(temp1[17]);
+ temp1[18] = add_round_shift_s16(temp1[18]);
+ temp1[19] = add_round_shift_s16(temp1[19]);
+ temp1[20] = add_round_shift_s16(temp1[20]);
+ temp1[21] = add_round_shift_s16(temp1[21]);
+ temp1[22] = add_round_shift_s16(temp1[22]);
+ temp1[23] = add_round_shift_s16(temp1[23]);
+ temp1[24] = add_round_shift_s16(temp1[24]);
+ temp1[25] = add_round_shift_s16(temp1[25]);
+ temp1[26] = add_round_shift_s16(temp1[26]);
+ temp1[27] = add_round_shift_s16(temp1[27]);
+ temp1[28] = add_round_shift_s16(temp1[28]);
+ temp1[29] = add_round_shift_s16(temp1[29]);
+ temp1[30] = add_round_shift_s16(temp1[30]);
+ temp1[31] = add_round_shift_s16(temp1[31]);
+ }
+
+ // Stage 3
+ temp2[0] = vec_add(temp1[0], temp1[7]);
+ temp2[1] = vec_add(temp1[1], temp1[6]);
+ temp2[2] = vec_add(temp1[2], temp1[5]);
+ temp2[3] = vec_add(temp1[3], temp1[4]);
+ temp2[5] = vec_sub(temp1[2], temp1[5]);
+ temp2[6] = vec_sub(temp1[1], temp1[6]);
+ temp2[8] = temp1[8];
+ temp2[9] = temp1[9];
+
+ single_butterfly(temp1[13], temp1[10], &temp2[13], &temp2[10]);
+ single_butterfly(temp1[12], temp1[11], &temp2[12], &temp2[11]);
+ temp2[14] = temp1[14];
+ temp2[15] = temp1[15];
+
+ temp2[18] = vec_add(temp1[18], temp1[21]);
+ temp2[19] = vec_add(temp1[19], temp1[20]);
+
+ temp2[20] = vec_sub(temp1[19], temp1[20]);
+ temp2[21] = vec_sub(temp1[18], temp1[21]);
+
+ temp2[26] = vec_sub(temp1[29], temp1[26]);
+ temp2[27] = vec_sub(temp1[28], temp1[27]);
+
+ temp2[28] = vec_add(temp1[28], temp1[27]);
+ temp2[29] = vec_add(temp1[29], temp1[26]);
+
+ // Pass through Stage 4
+ temp0[7] = vec_sub(temp1[0], temp1[7]);
+ temp0[4] = vec_sub(temp1[3], temp1[4]);
+ temp0[16] = vec_add(temp1[16], temp1[23]);
+ temp0[17] = vec_add(temp1[17], temp1[22]);
+ temp0[22] = vec_sub(temp1[17], temp1[22]);
+ temp0[23] = vec_sub(temp1[16], temp1[23]);
+ temp0[24] = vec_sub(temp1[31], temp1[24]);
+ temp0[25] = vec_sub(temp1[30], temp1[25]);
+ temp0[30] = vec_add(temp1[30], temp1[25]);
+ temp0[31] = vec_add(temp1[31], temp1[24]);
+
+ // Stage 4
+ temp0[0] = vec_add(temp2[0], temp2[3]);
+ temp0[1] = vec_add(temp2[1], temp2[2]);
+ temp0[2] = vec_sub(temp2[1], temp2[2]);
+ temp0[3] = vec_sub(temp2[0], temp2[3]);
+ single_butterfly(temp2[6], temp2[5], &temp0[6], &temp0[5]);
+
+ temp0[9] = vec_add(temp2[9], temp2[10]);
+ temp0[10] = vec_sub(temp2[9], temp2[10]);
+ temp0[13] = vec_sub(temp2[14], temp2[13]);
+ temp0[14] = vec_add(temp2[14], temp2[13]);
+
+ double_butterfly(temp2[29], cospi8_v, temp2[18], cospi24_v, &temp0[29],
+ &temp0[18]);
+ double_butterfly(temp2[28], cospi8_v, temp2[19], cospi24_v, &temp0[28],
+ &temp0[19]);
+ double_butterfly(temp2[27], cospi24_v, temp2[20], cospi8m_v, &temp0[27],
+ &temp0[20]);
+ double_butterfly(temp2[26], cospi24_v, temp2[21], cospi8m_v, &temp0[26],
+ &temp0[21]);
+
+ // Pass through Stage 5
+ temp1[8] = vec_add(temp2[8], temp2[11]);
+ temp1[11] = vec_sub(temp2[8], temp2[11]);
+ temp1[12] = vec_sub(temp2[15], temp2[12]);
+ temp1[15] = vec_add(temp2[15], temp2[12]);
+
+ // Stage 5
+ // 0 and 1 pass through to 0 and 16 at the end
+ single_butterfly(temp0[0], temp0[1], &out[0], &out[16]);
+
+ // 2 and 3 pass through to 8 and 24 at the end
+ double_butterfly(temp0[3], cospi8_v, temp0[2], cospi24_v, &out[8], &out[24]);
+
+ temp1[4] = vec_add(temp0[4], temp0[5]);
+ temp1[5] = vec_sub(temp0[4], temp0[5]);
+ temp1[6] = vec_sub(temp0[7], temp0[6]);
+ temp1[7] = vec_add(temp0[7], temp0[6]);
+
+ double_butterfly(temp0[14], cospi8_v, temp0[9], cospi24_v, &temp1[14],
+ &temp1[9]);
+ double_butterfly(temp0[13], cospi24_v, temp0[10], cospi8m_v, &temp1[13],
+ &temp1[10]);
+
+ temp1[17] = vec_add(temp0[17], temp0[18]);
+ temp1[18] = vec_sub(temp0[17], temp0[18]);
+
+ temp1[21] = vec_sub(temp0[22], temp0[21]);
+ temp1[22] = vec_add(temp0[22], temp0[21]);
+
+ temp1[25] = vec_add(temp0[25], temp0[26]);
+ temp1[26] = vec_sub(temp0[25], temp0[26]);
+
+ temp1[29] = vec_sub(temp0[30], temp0[29]);
+ temp1[30] = vec_add(temp0[30], temp0[29]);
+
+ // Pass through Stage 6
+ temp2[16] = vec_add(temp0[16], temp0[19]);
+ temp2[19] = vec_sub(temp0[16], temp0[19]);
+ temp2[20] = vec_sub(temp0[23], temp0[20]);
+ temp2[23] = vec_add(temp0[23], temp0[20]);
+ temp2[24] = vec_add(temp0[24], temp0[27]);
+ temp2[27] = vec_sub(temp0[24], temp0[27]);
+ temp2[28] = vec_sub(temp0[31], temp0[28]);
+ temp2[31] = vec_add(temp0[31], temp0[28]);
+
+ // Stage 6
+ // 4 and 7 pass through to 4 and 28 at the end
+ double_butterfly(temp1[7], cospi4_v, temp1[4], cospi28_v, &out[4], &out[28]);
+ // 5 and 6 pass through to 20 and 12 at the end
+ double_butterfly(temp1[6], cospi20_v, temp1[5], cospi12_v, &out[20],
+ &out[12]);
+ temp2[8] = vec_add(temp1[8], temp1[9]);
+ temp2[9] = vec_sub(temp1[8], temp1[9]);
+ temp2[10] = vec_sub(temp1[11], temp1[10]);
+ temp2[11] = vec_add(temp1[11], temp1[10]);
+ temp2[12] = vec_add(temp1[12], temp1[13]);
+ temp2[13] = vec_sub(temp1[12], temp1[13]);
+ temp2[14] = vec_sub(temp1[15], temp1[14]);
+ temp2[15] = vec_add(temp1[15], temp1[14]);
+
+ double_butterfly(temp1[30], cospi4_v, temp1[17], cospi28_v, &temp2[30],
+ &temp2[17]);
+ double_butterfly(temp1[29], cospi28_v, temp1[18], cospi4m_v, &temp2[29],
+ &temp2[18]);
+ double_butterfly(temp1[26], cospi20_v, temp1[21], cospi12_v, &temp2[26],
+ &temp2[21]);
+ double_butterfly(temp1[25], cospi12_v, temp1[22], cospi20m_v, &temp2[25],
+ &temp2[22]);
+
+ // Stage 7
+ double_butterfly(temp2[15], cospi2_v, temp2[8], cospi30_v, &out[2], &out[30]);
+ double_butterfly(temp2[14], cospi18_v, temp2[9], cospi14_v, &out[18],
+ &out[14]);
+ double_butterfly(temp2[13], cospi10_v, temp2[10], cospi22_v, &out[10],
+ &out[22]);
+ double_butterfly(temp2[12], cospi26_v, temp2[11], cospi6_v, &out[26],
+ &out[6]);
+
+ temp0[16] = vec_add(temp2[16], temp2[17]);
+ temp0[17] = vec_sub(temp2[16], temp2[17]);
+ temp0[18] = vec_sub(temp2[19], temp2[18]);
+ temp0[19] = vec_add(temp2[19], temp2[18]);
+ temp0[20] = vec_add(temp2[20], temp2[21]);
+ temp0[21] = vec_sub(temp2[20], temp2[21]);
+ temp0[22] = vec_sub(temp2[23], temp2[22]);
+ temp0[23] = vec_add(temp2[23], temp2[22]);
+ temp0[24] = vec_add(temp2[24], temp2[25]);
+ temp0[25] = vec_sub(temp2[24], temp2[25]);
+ temp0[26] = vec_sub(temp2[27], temp2[26]);
+ temp0[27] = vec_add(temp2[27], temp2[26]);
+ temp0[28] = vec_add(temp2[28], temp2[29]);
+ temp0[29] = vec_sub(temp2[28], temp2[29]);
+ temp0[30] = vec_sub(temp2[31], temp2[30]);
+ temp0[31] = vec_add(temp2[31], temp2[30]);
+
+ // Final stage --- outputs indices are bit-reversed.
+ double_butterfly(temp0[31], cospi1_v, temp0[16], cospi31_v, &out[1],
+ &out[31]);
+ double_butterfly(temp0[30], cospi17_v, temp0[17], cospi15_v, &out[17],
+ &out[15]);
+ double_butterfly(temp0[29], cospi9_v, temp0[18], cospi23_v, &out[9],
+ &out[23]);
+ double_butterfly(temp0[28], cospi25_v, temp0[19], cospi7_v, &out[25],
+ &out[7]);
+ double_butterfly(temp0[27], cospi5_v, temp0[20], cospi27_v, &out[5],
+ &out[27]);
+ double_butterfly(temp0[26], cospi21_v, temp0[21], cospi11_v, &out[21],
+ &out[11]);
+ double_butterfly(temp0[25], cospi13_v, temp0[22], cospi19_v, &out[13],
+ &out[19]);
+ double_butterfly(temp0[24], cospi29_v, temp0[23], cospi3_v, &out[29],
+ &out[3]);
+
+ if (pass == 0) {
+ for (i = 0; i < 32; i++) {
+ out[i] = sub_round_shift(out[i]);
+ }
+ }
+}
+
+void vpx_fdct32x32_rd_vsx(const int16_t *input, tran_low_t *out, int stride) {
+ int16x8_t temp0[32];
+ int16x8_t temp1[32];
+ int16x8_t temp2[32];
+ int16x8_t temp3[32];
+ int16x8_t temp4[32];
+ int16x8_t temp5[32];
+ int16x8_t temp6[32];
+
+ // Process in 8x32 columns.
+ load(input, stride, temp0);
+ fdct32_vsx(temp0, temp1, 0);
+
+ load(input + 8, stride, temp0);
+ fdct32_vsx(temp0, temp2, 0);
+
+ load(input + 16, stride, temp0);
+ fdct32_vsx(temp0, temp3, 0);
+
+ load(input + 24, stride, temp0);
+ fdct32_vsx(temp0, temp4, 0);
+
+ // Generate the top row by munging the first set of 8 from each one
+ // together.
+ transpose_8x8(&temp1[0], &temp0[0]);
+ transpose_8x8(&temp2[0], &temp0[8]);
+ transpose_8x8(&temp3[0], &temp0[16]);
+ transpose_8x8(&temp4[0], &temp0[24]);
+
+ fdct32_vsx(temp0, temp5, 1);
+
+ transpose_8x8(&temp5[0], &temp6[0]);
+ transpose_8x8(&temp5[8], &temp6[8]);
+ transpose_8x8(&temp5[16], &temp6[16]);
+ transpose_8x8(&temp5[24], &temp6[24]);
+
+ store(out, temp6);
+
+ // Second row of 8x32.
+ transpose_8x8(&temp1[8], &temp0[0]);
+ transpose_8x8(&temp2[8], &temp0[8]);
+ transpose_8x8(&temp3[8], &temp0[16]);
+ transpose_8x8(&temp4[8], &temp0[24]);
+
+ fdct32_vsx(temp0, temp5, 1);
+
+ transpose_8x8(&temp5[0], &temp6[0]);
+ transpose_8x8(&temp5[8], &temp6[8]);
+ transpose_8x8(&temp5[16], &temp6[16]);
+ transpose_8x8(&temp5[24], &temp6[24]);
+
+ store(out + 8 * 32, temp6);
+
+ // Third row of 8x32
+ transpose_8x8(&temp1[16], &temp0[0]);
+ transpose_8x8(&temp2[16], &temp0[8]);
+ transpose_8x8(&temp3[16], &temp0[16]);
+ transpose_8x8(&temp4[16], &temp0[24]);
+
+ fdct32_vsx(temp0, temp5, 1);
+
+ transpose_8x8(&temp5[0], &temp6[0]);
+ transpose_8x8(&temp5[8], &temp6[8]);
+ transpose_8x8(&temp5[16], &temp6[16]);
+ transpose_8x8(&temp5[24], &temp6[24]);
+
+ store(out + 16 * 32, temp6);
+
+ // Final row of 8x32.
+ transpose_8x8(&temp1[24], &temp0[0]);
+ transpose_8x8(&temp2[24], &temp0[8]);
+ transpose_8x8(&temp3[24], &temp0[16]);
+ transpose_8x8(&temp4[24], &temp0[24]);
+
+ fdct32_vsx(temp0, temp5, 1);
+
+ transpose_8x8(&temp5[0], &temp6[0]);
+ transpose_8x8(&temp5[8], &temp6[8]);
+ transpose_8x8(&temp5[16], &temp6[16]);
+ transpose_8x8(&temp5[24], &temp6[24]);
+
+ store(out + 24 * 32, temp6);
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/hadamard_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/hadamard_vsx.c
new file mode 100644
index 0000000000..e279b30478
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/hadamard_vsx.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+#include "vpx_dsp/ppc/transpose_vsx.h"
+#include "vpx_dsp/ppc/bitdepth_conversion_vsx.h"
+
+static void vpx_hadamard_s16_8x8_one_pass(int16x8_t v[8]) {
+ const int16x8_t b0 = vec_add(v[0], v[1]);
+ const int16x8_t b1 = vec_sub(v[0], v[1]);
+ const int16x8_t b2 = vec_add(v[2], v[3]);
+ const int16x8_t b3 = vec_sub(v[2], v[3]);
+ const int16x8_t b4 = vec_add(v[4], v[5]);
+ const int16x8_t b5 = vec_sub(v[4], v[5]);
+ const int16x8_t b6 = vec_add(v[6], v[7]);
+ const int16x8_t b7 = vec_sub(v[6], v[7]);
+
+ const int16x8_t c0 = vec_add(b0, b2);
+ const int16x8_t c1 = vec_add(b1, b3);
+ const int16x8_t c2 = vec_sub(b0, b2);
+ const int16x8_t c3 = vec_sub(b1, b3);
+ const int16x8_t c4 = vec_add(b4, b6);
+ const int16x8_t c5 = vec_add(b5, b7);
+ const int16x8_t c6 = vec_sub(b4, b6);
+ const int16x8_t c7 = vec_sub(b5, b7);
+
+ v[0] = vec_add(c0, c4);
+ v[1] = vec_sub(c2, c6);
+ v[2] = vec_sub(c0, c4);
+ v[3] = vec_add(c2, c6);
+ v[4] = vec_add(c3, c7);
+ v[5] = vec_sub(c3, c7);
+ v[6] = vec_sub(c1, c5);
+ v[7] = vec_add(c1, c5);
+}
+
+void vpx_hadamard_8x8_vsx(const int16_t *src_diff, ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ int16x8_t v[8];
+
+ v[0] = vec_vsx_ld(0, src_diff);
+ v[1] = vec_vsx_ld(0, src_diff + src_stride);
+ v[2] = vec_vsx_ld(0, src_diff + (2 * src_stride));
+ v[3] = vec_vsx_ld(0, src_diff + (3 * src_stride));
+ v[4] = vec_vsx_ld(0, src_diff + (4 * src_stride));
+ v[5] = vec_vsx_ld(0, src_diff + (5 * src_stride));
+ v[6] = vec_vsx_ld(0, src_diff + (6 * src_stride));
+ v[7] = vec_vsx_ld(0, src_diff + (7 * src_stride));
+
+ vpx_hadamard_s16_8x8_one_pass(v);
+
+ vpx_transpose_s16_8x8(v);
+
+ vpx_hadamard_s16_8x8_one_pass(v);
+
+ store_tran_low(v[0], 0, coeff);
+ store_tran_low(v[1], 0, coeff + 8);
+ store_tran_low(v[2], 0, coeff + 16);
+ store_tran_low(v[3], 0, coeff + 24);
+ store_tran_low(v[4], 0, coeff + 32);
+ store_tran_low(v[5], 0, coeff + 40);
+ store_tran_low(v[6], 0, coeff + 48);
+ store_tran_low(v[7], 0, coeff + 56);
+}
+
+void vpx_hadamard_16x16_vsx(const int16_t *src_diff, ptrdiff_t src_stride,
+ tran_low_t *coeff) {
+ int i;
+ const uint16x8_t ones = vec_splat_u16(1);
+
+ /* Rearrange 16x16 to 8x32 and remove stride.
+ * Top left first. */
+ vpx_hadamard_8x8_vsx(src_diff, src_stride, coeff);
+ /* Top right. */
+ vpx_hadamard_8x8_vsx(src_diff + 8 + 0 * src_stride, src_stride, coeff + 64);
+ /* Bottom left. */
+ vpx_hadamard_8x8_vsx(src_diff + 0 + 8 * src_stride, src_stride, coeff + 128);
+ /* Bottom right. */
+ vpx_hadamard_8x8_vsx(src_diff + 8 + 8 * src_stride, src_stride, coeff + 192);
+
+ /* Overlay the 8x8 blocks and combine. */
+ for (i = 0; i < 64; i += 8) {
+ const int16x8_t a0 = load_tran_low(0, coeff);
+ const int16x8_t a1 = load_tran_low(0, coeff + 64);
+ const int16x8_t a2 = load_tran_low(0, coeff + 128);
+ const int16x8_t a3 = load_tran_low(0, coeff + 192);
+
+ /* Prevent the result from escaping int16_t. */
+ const int16x8_t b0 = vec_sra(a0, ones);
+ const int16x8_t b1 = vec_sra(a1, ones);
+ const int16x8_t b2 = vec_sra(a2, ones);
+ const int16x8_t b3 = vec_sra(a3, ones);
+
+ const int16x8_t c0 = vec_add(b0, b1);
+ const int16x8_t c2 = vec_add(b2, b3);
+ const int16x8_t c1 = vec_sub(b0, b1);
+ const int16x8_t c3 = vec_sub(b2, b3);
+
+ const int16x8_t d0 = vec_add(c0, c2);
+ const int16x8_t d1 = vec_add(c1, c3);
+ const int16x8_t d2 = vec_sub(c0, c2);
+ const int16x8_t d3 = vec_sub(c1, c3);
+
+ store_tran_low(d0, 0, coeff);
+ store_tran_low(d1, 0, coeff + 64);
+ store_tran_low(d2, 0, coeff + 128);
+ store_tran_low(d3, 0, coeff + 192);
+
+ coeff += 8;
+ }
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/intrapred_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/intrapred_vsx.c
new file mode 100644
index 0000000000..a4c8322ff2
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/intrapred_vsx.c
@@ -0,0 +1,767 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+void vpx_v_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d = vec_vsx_ld(0, above);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 16; i++, dst += stride) {
+ vec_vsx_st(d, 0, dst);
+ }
+}
+
+void vpx_v_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d0 = vec_vsx_ld(0, above);
+ const uint8x16_t d1 = vec_vsx_ld(16, above);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 32; i++, dst += stride) {
+ vec_vsx_st(d0, 0, dst);
+ vec_vsx_st(d1, 16, dst);
+ }
+}
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+static const uint32x4_t mask4 = { 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
+
+void vpx_h_predictor_4x4_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d = vec_vsx_ld(0, left);
+ const uint8x16_t v0 = vec_splat(d, 0);
+ const uint8x16_t v1 = vec_splat(d, 1);
+ const uint8x16_t v2 = vec_splat(d, 2);
+ const uint8x16_t v3 = vec_splat(d, 3);
+
+ (void)above;
+
+ vec_vsx_st(vec_sel(v0, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+ vec_vsx_st(vec_sel(v1, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+ vec_vsx_st(vec_sel(v2, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+ vec_vsx_st(vec_sel(v3, vec_vsx_ld(0, dst), (uint8x16_t)mask4), 0, dst);
+}
+
+void vpx_h_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d = vec_vsx_ld(0, left);
+ const uint8x16_t v0 = vec_splat(d, 0);
+ const uint8x16_t v1 = vec_splat(d, 1);
+ const uint8x16_t v2 = vec_splat(d, 2);
+ const uint8x16_t v3 = vec_splat(d, 3);
+
+ const uint8x16_t v4 = vec_splat(d, 4);
+ const uint8x16_t v5 = vec_splat(d, 5);
+ const uint8x16_t v6 = vec_splat(d, 6);
+ const uint8x16_t v7 = vec_splat(d, 7);
+
+ (void)above;
+
+ vec_vsx_st(xxpermdi(v0, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v1, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v2, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v3, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v4, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v5, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v6, vec_vsx_ld(0, dst), 1), 0, dst);
+ dst += stride;
+ vec_vsx_st(xxpermdi(v7, vec_vsx_ld(0, dst), 1), 0, dst);
+}
+#endif
+
+void vpx_h_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d = vec_vsx_ld(0, left);
+ const uint8x16_t v0 = vec_splat(d, 0);
+ const uint8x16_t v1 = vec_splat(d, 1);
+ const uint8x16_t v2 = vec_splat(d, 2);
+ const uint8x16_t v3 = vec_splat(d, 3);
+
+ const uint8x16_t v4 = vec_splat(d, 4);
+ const uint8x16_t v5 = vec_splat(d, 5);
+ const uint8x16_t v6 = vec_splat(d, 6);
+ const uint8x16_t v7 = vec_splat(d, 7);
+
+ const uint8x16_t v8 = vec_splat(d, 8);
+ const uint8x16_t v9 = vec_splat(d, 9);
+ const uint8x16_t v10 = vec_splat(d, 10);
+ const uint8x16_t v11 = vec_splat(d, 11);
+
+ const uint8x16_t v12 = vec_splat(d, 12);
+ const uint8x16_t v13 = vec_splat(d, 13);
+ const uint8x16_t v14 = vec_splat(d, 14);
+ const uint8x16_t v15 = vec_splat(d, 15);
+
+ (void)above;
+
+ vec_vsx_st(v0, 0, dst);
+ dst += stride;
+ vec_vsx_st(v1, 0, dst);
+ dst += stride;
+ vec_vsx_st(v2, 0, dst);
+ dst += stride;
+ vec_vsx_st(v3, 0, dst);
+ dst += stride;
+ vec_vsx_st(v4, 0, dst);
+ dst += stride;
+ vec_vsx_st(v5, 0, dst);
+ dst += stride;
+ vec_vsx_st(v6, 0, dst);
+ dst += stride;
+ vec_vsx_st(v7, 0, dst);
+ dst += stride;
+ vec_vsx_st(v8, 0, dst);
+ dst += stride;
+ vec_vsx_st(v9, 0, dst);
+ dst += stride;
+ vec_vsx_st(v10, 0, dst);
+ dst += stride;
+ vec_vsx_st(v11, 0, dst);
+ dst += stride;
+ vec_vsx_st(v12, 0, dst);
+ dst += stride;
+ vec_vsx_st(v13, 0, dst);
+ dst += stride;
+ vec_vsx_st(v14, 0, dst);
+ dst += stride;
+ vec_vsx_st(v15, 0, dst);
+}
+
+#define H_PREDICTOR_32(v) \
+ vec_vsx_st(v, 0, dst); \
+ vec_vsx_st(v, 16, dst); \
+ dst += stride
+
+void vpx_h_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t d0 = vec_vsx_ld(0, left);
+ const uint8x16_t d1 = vec_vsx_ld(16, left);
+
+ const uint8x16_t v0_0 = vec_splat(d0, 0);
+ const uint8x16_t v1_0 = vec_splat(d0, 1);
+ const uint8x16_t v2_0 = vec_splat(d0, 2);
+ const uint8x16_t v3_0 = vec_splat(d0, 3);
+ const uint8x16_t v4_0 = vec_splat(d0, 4);
+ const uint8x16_t v5_0 = vec_splat(d0, 5);
+ const uint8x16_t v6_0 = vec_splat(d0, 6);
+ const uint8x16_t v7_0 = vec_splat(d0, 7);
+ const uint8x16_t v8_0 = vec_splat(d0, 8);
+ const uint8x16_t v9_0 = vec_splat(d0, 9);
+ const uint8x16_t v10_0 = vec_splat(d0, 10);
+ const uint8x16_t v11_0 = vec_splat(d0, 11);
+ const uint8x16_t v12_0 = vec_splat(d0, 12);
+ const uint8x16_t v13_0 = vec_splat(d0, 13);
+ const uint8x16_t v14_0 = vec_splat(d0, 14);
+ const uint8x16_t v15_0 = vec_splat(d0, 15);
+
+ const uint8x16_t v0_1 = vec_splat(d1, 0);
+ const uint8x16_t v1_1 = vec_splat(d1, 1);
+ const uint8x16_t v2_1 = vec_splat(d1, 2);
+ const uint8x16_t v3_1 = vec_splat(d1, 3);
+ const uint8x16_t v4_1 = vec_splat(d1, 4);
+ const uint8x16_t v5_1 = vec_splat(d1, 5);
+ const uint8x16_t v6_1 = vec_splat(d1, 6);
+ const uint8x16_t v7_1 = vec_splat(d1, 7);
+ const uint8x16_t v8_1 = vec_splat(d1, 8);
+ const uint8x16_t v9_1 = vec_splat(d1, 9);
+ const uint8x16_t v10_1 = vec_splat(d1, 10);
+ const uint8x16_t v11_1 = vec_splat(d1, 11);
+ const uint8x16_t v12_1 = vec_splat(d1, 12);
+ const uint8x16_t v13_1 = vec_splat(d1, 13);
+ const uint8x16_t v14_1 = vec_splat(d1, 14);
+ const uint8x16_t v15_1 = vec_splat(d1, 15);
+
+ (void)above;
+
+ H_PREDICTOR_32(v0_0);
+ H_PREDICTOR_32(v1_0);
+ H_PREDICTOR_32(v2_0);
+ H_PREDICTOR_32(v3_0);
+
+ H_PREDICTOR_32(v4_0);
+ H_PREDICTOR_32(v5_0);
+ H_PREDICTOR_32(v6_0);
+ H_PREDICTOR_32(v7_0);
+
+ H_PREDICTOR_32(v8_0);
+ H_PREDICTOR_32(v9_0);
+ H_PREDICTOR_32(v10_0);
+ H_PREDICTOR_32(v11_0);
+
+ H_PREDICTOR_32(v12_0);
+ H_PREDICTOR_32(v13_0);
+ H_PREDICTOR_32(v14_0);
+ H_PREDICTOR_32(v15_0);
+
+ H_PREDICTOR_32(v0_1);
+ H_PREDICTOR_32(v1_1);
+ H_PREDICTOR_32(v2_1);
+ H_PREDICTOR_32(v3_1);
+
+ H_PREDICTOR_32(v4_1);
+ H_PREDICTOR_32(v5_1);
+ H_PREDICTOR_32(v6_1);
+ H_PREDICTOR_32(v7_1);
+
+ H_PREDICTOR_32(v8_1);
+ H_PREDICTOR_32(v9_1);
+ H_PREDICTOR_32(v10_1);
+ H_PREDICTOR_32(v11_1);
+
+ H_PREDICTOR_32(v12_1);
+ H_PREDICTOR_32(v13_1);
+ H_PREDICTOR_32(v14_1);
+ H_PREDICTOR_32(v15_1);
+}
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+void vpx_tm_predictor_4x4_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
+ const int16x8_t l = unpack_to_s16_h(vec_vsx_ld(0, left));
+ const int16x8_t a = unpack_to_s16_h(vec_vsx_ld(0, above));
+ int16x8_t tmp, val;
+ uint8x16_t d;
+
+ d = vec_vsx_ld(0, dst);
+ tmp = unpack_to_s16_l(d);
+ val = vec_sub(vec_add(vec_splat(l, 0), a), tl);
+ vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+
+ d = vec_vsx_ld(0, dst);
+ tmp = unpack_to_s16_l(d);
+ val = vec_sub(vec_add(vec_splat(l, 1), a), tl);
+ vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+
+ d = vec_vsx_ld(0, dst);
+ tmp = unpack_to_s16_l(d);
+ val = vec_sub(vec_add(vec_splat(l, 2), a), tl);
+ vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
+ dst += stride;
+
+ d = vec_vsx_ld(0, dst);
+ tmp = unpack_to_s16_l(d);
+ val = vec_sub(vec_add(vec_splat(l, 3), a), tl);
+ vec_vsx_st(vec_sel(vec_packsu(val, tmp), d, (uint8x16_t)mask4), 0, dst);
+}
+
+void vpx_tm_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
+ const int16x8_t l = unpack_to_s16_h(vec_vsx_ld(0, left));
+ const int16x8_t a = unpack_to_s16_h(vec_vsx_ld(0, above));
+ int16x8_t tmp, val;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 0), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 1), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 2), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 3), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 4), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 5), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 6), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+ dst += stride;
+
+ tmp = unpack_to_s16_l(vec_vsx_ld(0, dst));
+ val = vec_sub(vec_add(vec_splat(l, 7), a), tl);
+ vec_vsx_st(vec_packsu(val, tmp), 0, dst);
+}
+#endif
+
+static void tm_predictor_16x8(uint8_t *dst, const ptrdiff_t stride, int16x8_t l,
+ int16x8_t ah, int16x8_t al, int16x8_t tl) {
+ int16x8_t vh, vl, ls;
+
+ ls = vec_splat(l, 0);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 1);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 2);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 3);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 4);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 5);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 6);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ dst += stride;
+
+ ls = vec_splat(l, 7);
+ vh = vec_sub(vec_add(ls, ah), tl);
+ vl = vec_sub(vec_add(ls, al), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+}
+
+void vpx_tm_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
+ const uint8x16_t l = vec_vsx_ld(0, left);
+ const int16x8_t lh = unpack_to_s16_h(l);
+ const int16x8_t ll = unpack_to_s16_l(l);
+ const uint8x16_t a = vec_vsx_ld(0, above);
+ const int16x8_t ah = unpack_to_s16_h(a);
+ const int16x8_t al = unpack_to_s16_l(a);
+
+ tm_predictor_16x8(dst, stride, lh, ah, al, tl);
+
+ dst += stride * 8;
+
+ tm_predictor_16x8(dst, stride, ll, ah, al, tl);
+}
+
+static INLINE void tm_predictor_32x1(uint8_t *dst, const int16x8_t ls,
+ const int16x8_t a0h, const int16x8_t a0l,
+ const int16x8_t a1h, const int16x8_t a1l,
+ const int16x8_t tl) {
+ int16x8_t vh, vl;
+
+ vh = vec_sub(vec_add(ls, a0h), tl);
+ vl = vec_sub(vec_add(ls, a0l), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 0, dst);
+ vh = vec_sub(vec_add(ls, a1h), tl);
+ vl = vec_sub(vec_add(ls, a1l), tl);
+ vec_vsx_st(vec_packsu(vh, vl), 16, dst);
+}
+
+static void tm_predictor_32x8(uint8_t *dst, const ptrdiff_t stride,
+ const int16x8_t l, const uint8x16_t a0,
+ const uint8x16_t a1, const int16x8_t tl) {
+ const int16x8_t a0h = unpack_to_s16_h(a0);
+ const int16x8_t a0l = unpack_to_s16_l(a0);
+ const int16x8_t a1h = unpack_to_s16_h(a1);
+ const int16x8_t a1l = unpack_to_s16_l(a1);
+
+ tm_predictor_32x1(dst, vec_splat(l, 0), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 1), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 2), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 3), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 4), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 5), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 6), a0h, a0l, a1h, a1l, tl);
+ dst += stride;
+
+ tm_predictor_32x1(dst, vec_splat(l, 7), a0h, a0l, a1h, a1l, tl);
+}
+
+void vpx_tm_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const int16x8_t tl = unpack_to_s16_h(vec_splat(vec_vsx_ld(-1, above), 0));
+ const uint8x16_t l0 = vec_vsx_ld(0, left);
+ const uint8x16_t l1 = vec_vsx_ld(16, left);
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t a1 = vec_vsx_ld(16, above);
+
+ tm_predictor_32x8(dst, stride, unpack_to_s16_h(l0), a0, a1, tl);
+ dst += stride * 8;
+
+ tm_predictor_32x8(dst, stride, unpack_to_s16_l(l0), a0, a1, tl);
+ dst += stride * 8;
+
+ tm_predictor_32x8(dst, stride, unpack_to_s16_h(l1), a0, a1, tl);
+ dst += stride * 8;
+
+ tm_predictor_32x8(dst, stride, unpack_to_s16_l(l1), a0, a1, tl);
+}
+
+static INLINE void dc_fill_predictor_8x8(uint8_t *dst, const ptrdiff_t stride,
+ const uint8x16_t val) {
+ int i;
+
+ for (i = 0; i < 8; i++, dst += stride) {
+ const uint8x16_t d = vec_vsx_ld(0, dst);
+ vec_vsx_st(xxpermdi(val, d, 1), 0, dst);
+ }
+}
+
+static INLINE void dc_fill_predictor_16x16(uint8_t *dst, const ptrdiff_t stride,
+ const uint8x16_t val) {
+ int i;
+
+ for (i = 0; i < 16; i++, dst += stride) {
+ vec_vsx_st(val, 0, dst);
+ }
+}
+
+void vpx_dc_128_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t v128 = vec_sl(vec_splat_u8(1), vec_splat_u8(7));
+ (void)above;
+ (void)left;
+
+ dc_fill_predictor_16x16(dst, stride, v128);
+}
+
+static INLINE void dc_fill_predictor_32x32(uint8_t *dst, const ptrdiff_t stride,
+ const uint8x16_t val) {
+ int i;
+
+ for (i = 0; i < 32; i++, dst += stride) {
+ vec_vsx_st(val, 0, dst);
+ vec_vsx_st(val, 16, dst);
+ }
+}
+
+void vpx_dc_128_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t v128 = vec_sl(vec_splat_u8(1), vec_splat_u8(7));
+ (void)above;
+ (void)left;
+
+ dc_fill_predictor_32x32(dst, stride, v128);
+}
+
+static uint8x16_t avg16(const uint8_t *values) {
+ const int32x4_t sum4s =
+ (int32x4_t)vec_sum4s(vec_vsx_ld(0, values), vec_splat_u32(0));
+ const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, vec_splat_s32(8));
+ const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(4));
+
+ return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
+ 3);
+}
+
+void vpx_dc_left_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above,
+ const uint8_t *left) {
+ (void)above;
+
+ dc_fill_predictor_16x16(dst, stride, avg16(left));
+}
+
+void vpx_dc_top_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ (void)left;
+
+ dc_fill_predictor_16x16(dst, stride, avg16(above));
+}
+
+static uint8x16_t avg32(const uint8_t *values) {
+ const uint8x16_t v0 = vec_vsx_ld(0, values);
+ const uint8x16_t v1 = vec_vsx_ld(16, values);
+ const int32x4_t v16 = vec_sl(vec_splat_s32(1), vec_splat_u32(4));
+ const int32x4_t sum4s =
+ (int32x4_t)vec_sum4s(v0, vec_sum4s(v1, vec_splat_u32(0)));
+ const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v16);
+ const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(5));
+
+ return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
+ 3);
+}
+
+void vpx_dc_left_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above,
+ const uint8_t *left) {
+ (void)above;
+
+ dc_fill_predictor_32x32(dst, stride, avg32(left));
+}
+
+void vpx_dc_top_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ (void)left;
+
+ dc_fill_predictor_32x32(dst, stride, avg32(above));
+}
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+static uint8x16_t dc_avg8(const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t l0 = vec_vsx_ld(0, left);
+ const int32x4_t sum4s =
+ (int32x4_t)vec_sum4s(l0, vec_sum4s(a0, vec_splat_u32(0)));
+ const int32x4_t sum4s8 = xxpermdi(sum4s, vec_splat_s32(0), 1);
+ const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s8, vec_splat_s32(8));
+ const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(4));
+
+ return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
+ 3);
+}
+#endif
+
+static uint8x16_t dc_avg16(const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t l0 = vec_vsx_ld(0, left);
+ const int32x4_t v16 = vec_sl(vec_splat_s32(1), vec_splat_u32(4));
+ const int32x4_t sum4s =
+ (int32x4_t)vec_sum4s(l0, vec_sum4s(a0, vec_splat_u32(0)));
+ const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v16);
+ const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(5));
+
+ return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
+ 3);
+}
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+void vpx_dc_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ dc_fill_predictor_8x8(dst, stride, dc_avg8(above, left));
+}
+#endif
+
+void vpx_dc_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ dc_fill_predictor_16x16(dst, stride, dc_avg16(above, left));
+}
+
+static uint8x16_t dc_avg32(const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t a1 = vec_vsx_ld(16, above);
+ const uint8x16_t l0 = vec_vsx_ld(0, left);
+ const uint8x16_t l1 = vec_vsx_ld(16, left);
+ const int32x4_t v32 = vec_sl(vec_splat_s32(1), vec_splat_u32(5));
+ const uint32x4_t a_sum = vec_sum4s(a0, vec_sum4s(a1, vec_splat_u32(0)));
+ const int32x4_t sum4s = (int32x4_t)vec_sum4s(l0, vec_sum4s(l1, a_sum));
+ const uint32x4_t sum = (uint32x4_t)vec_sums(sum4s, v32);
+ const uint32x4_t avg = (uint32x4_t)vec_sr(sum, vec_splat_u32(6));
+
+ return vec_splat(vec_pack(vec_pack(avg, vec_splat_u32(0)), vec_splat_u16(0)),
+ 3);
+}
+
+void vpx_dc_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ dc_fill_predictor_32x32(dst, stride, dc_avg32(above, left));
+}
+
+static uint8x16_t avg3(const uint8x16_t a, const uint8x16_t b,
+ const uint8x16_t c) {
+ const uint8x16_t ac =
+ vec_adds(vec_and(a, c), vec_sr(vec_xor(a, c), vec_splat_u8(1)));
+
+ return vec_avg(ac, b);
+}
+
+// Workaround vec_sld/vec_xxsldi/vec_lsdoi being missing or broken.
+static const uint8x16_t sl1 = { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8,
+ 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, 0x10 };
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+void vpx_d45_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t af = vec_vsx_ld(0, above);
+ const uint8x16_t above_right = vec_splat(af, 7);
+ const uint8x16_t a = xxpermdi(af, above_right, 1);
+ const uint8x16_t b = vec_perm(a, above_right, sl1);
+ const uint8x16_t c = vec_perm(b, above_right, sl1);
+ uint8x16_t row = avg3(a, b, c);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 8; i++) {
+ const uint8x16_t d = vec_vsx_ld(0, dst);
+ vec_vsx_st(xxpermdi(row, d, 1), 0, dst);
+ dst += stride;
+ row = vec_perm(row, above_right, sl1);
+ }
+}
+#endif
+
+void vpx_d45_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a = vec_vsx_ld(0, above);
+ const uint8x16_t above_right = vec_splat(a, 15);
+ const uint8x16_t b = vec_perm(a, above_right, sl1);
+ const uint8x16_t c = vec_perm(b, above_right, sl1);
+ uint8x16_t row = avg3(a, b, c);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 16; i++) {
+ vec_vsx_st(row, 0, dst);
+ dst += stride;
+ row = vec_perm(row, above_right, sl1);
+ }
+}
+
+void vpx_d45_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t a1 = vec_vsx_ld(16, above);
+ const uint8x16_t above_right = vec_splat(a1, 15);
+ const uint8x16_t b0 = vec_perm(a0, a1, sl1);
+ const uint8x16_t b1 = vec_perm(a1, above_right, sl1);
+ const uint8x16_t c0 = vec_perm(b0, b1, sl1);
+ const uint8x16_t c1 = vec_perm(b1, above_right, sl1);
+ uint8x16_t row0 = avg3(a0, b0, c0);
+ uint8x16_t row1 = avg3(a1, b1, c1);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 32; i++) {
+ vec_vsx_st(row0, 0, dst);
+ vec_vsx_st(row1, 16, dst);
+ dst += stride;
+ row0 = vec_perm(row0, row1, sl1);
+ row1 = vec_perm(row1, above_right, sl1);
+ }
+}
+
+// TODO(crbug.com/webm/1522): Fix test failures.
+#if 0
+void vpx_d63_predictor_8x8_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t af = vec_vsx_ld(0, above);
+ const uint8x16_t above_right = vec_splat(af, 9);
+ const uint8x16_t a = xxpermdi(af, above_right, 1);
+ const uint8x16_t b = vec_perm(a, above_right, sl1);
+ const uint8x16_t c = vec_perm(b, above_right, sl1);
+ uint8x16_t row0 = vec_avg(a, b);
+ uint8x16_t row1 = avg3(a, b, c);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 4; i++) {
+ const uint8x16_t d0 = vec_vsx_ld(0, dst);
+ const uint8x16_t d1 = vec_vsx_ld(0, dst + stride);
+ vec_vsx_st(xxpermdi(row0, d0, 1), 0, dst);
+ vec_vsx_st(xxpermdi(row1, d1, 1), 0, dst + stride);
+ dst += stride * 2;
+ row0 = vec_perm(row0, above_right, sl1);
+ row1 = vec_perm(row1, above_right, sl1);
+ }
+}
+#endif
+
+void vpx_d63_predictor_16x16_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t a1 = vec_vsx_ld(16, above);
+ const uint8x16_t above_right = vec_splat(a1, 0);
+ const uint8x16_t b = vec_perm(a0, above_right, sl1);
+ const uint8x16_t c = vec_perm(b, above_right, sl1);
+ uint8x16_t row0 = vec_avg(a0, b);
+ uint8x16_t row1 = avg3(a0, b, c);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 8; i++) {
+ vec_vsx_st(row0, 0, dst);
+ vec_vsx_st(row1, 0, dst + stride);
+ dst += stride * 2;
+ row0 = vec_perm(row0, above_right, sl1);
+ row1 = vec_perm(row1, above_right, sl1);
+ }
+}
+
+void vpx_d63_predictor_32x32_vsx(uint8_t *dst, ptrdiff_t stride,
+ const uint8_t *above, const uint8_t *left) {
+ const uint8x16_t a0 = vec_vsx_ld(0, above);
+ const uint8x16_t a1 = vec_vsx_ld(16, above);
+ const uint8x16_t a2 = vec_vsx_ld(32, above);
+ const uint8x16_t above_right = vec_splat(a2, 0);
+ const uint8x16_t b0 = vec_perm(a0, a1, sl1);
+ const uint8x16_t b1 = vec_perm(a1, above_right, sl1);
+ const uint8x16_t c0 = vec_perm(b0, b1, sl1);
+ const uint8x16_t c1 = vec_perm(b1, above_right, sl1);
+ uint8x16_t row0_0 = vec_avg(a0, b0);
+ uint8x16_t row0_1 = vec_avg(a1, b1);
+ uint8x16_t row1_0 = avg3(a0, b0, c0);
+ uint8x16_t row1_1 = avg3(a1, b1, c1);
+ int i;
+ (void)left;
+
+ for (i = 0; i < 16; i++) {
+ vec_vsx_st(row0_0, 0, dst);
+ vec_vsx_st(row0_1, 16, dst);
+ vec_vsx_st(row1_0, 0, dst + stride);
+ vec_vsx_st(row1_1, 16, dst + stride);
+ dst += stride * 2;
+ row0_0 = vec_perm(row0_0, row0_1, sl1);
+ row0_1 = vec_perm(row0_1, above_right, sl1);
+ row1_0 = vec_perm(row1_0, row1_1, sl1);
+ row1_1 = vec_perm(row1_1, above_right, sl1);
+ }
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.c
new file mode 100644
index 0000000000..e99412ecab
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.c
@@ -0,0 +1,1828 @@
+/*
+ * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "vpx_dsp/ppc/bitdepth_conversion_vsx.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+#include "vpx_dsp/ppc/inv_txfm_vsx.h"
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/inv_txfm.h"
+
+static const int16x8_t cospi1_v = { 16364, 16364, 16364, 16364,
+ 16364, 16364, 16364, 16364 };
+static const int16x8_t cospi1m_v = { -16364, -16364, -16364, -16364,
+ -16364, -16364, -16364, -16364 };
+static const int16x8_t cospi2_v = { 16305, 16305, 16305, 16305,
+ 16305, 16305, 16305, 16305 };
+static const int16x8_t cospi2m_v = { -16305, -16305, -16305, -16305,
+ -16305, -16305, -16305, -16305 };
+static const int16x8_t cospi3_v = { 16207, 16207, 16207, 16207,
+ 16207, 16207, 16207, 16207 };
+static const int16x8_t cospi4_v = { 16069, 16069, 16069, 16069,
+ 16069, 16069, 16069, 16069 };
+static const int16x8_t cospi4m_v = { -16069, -16069, -16069, -16069,
+ -16069, -16069, -16069, -16069 };
+static const int16x8_t cospi5_v = { 15893, 15893, 15893, 15893,
+ 15893, 15893, 15893, 15893 };
+static const int16x8_t cospi5m_v = { -15893, -15893, -15893, -15893,
+ -15893, -15893, -15893, -15893 };
+static const int16x8_t cospi6_v = { 15679, 15679, 15679, 15679,
+ 15679, 15679, 15679, 15679 };
+static const int16x8_t cospi7_v = { 15426, 15426, 15426, 15426,
+ 15426, 15426, 15426, 15426 };
+static const int16x8_t cospi8_v = { 15137, 15137, 15137, 15137,
+ 15137, 15137, 15137, 15137 };
+static const int16x8_t cospi8m_v = { -15137, -15137, -15137, -15137,
+ -15137, -15137, -15137, -15137 };
+static const int16x8_t cospi9_v = { 14811, 14811, 14811, 14811,
+ 14811, 14811, 14811, 14811 };
+static const int16x8_t cospi9m_v = { -14811, -14811, -14811, -14811,
+ -14811, -14811, -14811, -14811 };
+static const int16x8_t cospi10_v = { 14449, 14449, 14449, 14449,
+ 14449, 14449, 14449, 14449 };
+static const int16x8_t cospi10m_v = { -14449, -14449, -14449, -14449,
+ -14449, -14449, -14449, -14449 };
+static const int16x8_t cospi11_v = { 14053, 14053, 14053, 14053,
+ 14053, 14053, 14053, 14053 };
+static const int16x8_t cospi12_v = { 13623, 13623, 13623, 13623,
+ 13623, 13623, 13623, 13623 };
+static const int16x8_t cospi12m_v = { -13623, -13623, -13623, -13623,
+ -13623, -13623, -13623, -13623 };
+static const int16x8_t cospi13_v = { 13160, 13160, 13160, 13160,
+ 13160, 13160, 13160, 13160 };
+static const int16x8_t cospi13m_v = { -13160, -13160, -13160, -13160,
+ -13160, -13160, -13160, -13160 };
+static const int16x8_t cospi14_v = { 12665, 12665, 12665, 12665,
+ 12665, 12665, 12665, 12665 };
+static const int16x8_t cospi15_v = { 12140, 12140, 12140, 12140,
+ 12140, 12140, 12140, 12140 };
+static const int16x8_t cospi16_v = { 11585, 11585, 11585, 11585,
+ 11585, 11585, 11585, 11585 };
+static const int16x8_t cospi16m_v = { -11585, -11585, -11585, -11585,
+ -11585, -11585, -11585, -11585 };
+static const int16x8_t cospi17_v = { 11003, 11003, 11003, 11003,
+ 11003, 11003, 11003, 11003 };
+static const int16x8_t cospi17m_v = { -11003, -11003, -11003, -11003,
+ -11003, -11003, -11003, -11003 };
+static const int16x8_t cospi18_v = { 10394, 10394, 10394, 10394,
+ 10394, 10394, 10394, 10394 };
+static const int16x8_t cospi18m_v = { -10394, -10394, -10394, -10394,
+ -10394, -10394, -10394, -10394 };
+static const int16x8_t cospi19_v = { 9760, 9760, 9760, 9760,
+ 9760, 9760, 9760, 9760 };
+static const int16x8_t cospi20_v = { 9102, 9102, 9102, 9102,
+ 9102, 9102, 9102, 9102 };
+static const int16x8_t cospi20m_v = { -9102, -9102, -9102, -9102,
+ -9102, -9102, -9102, -9102 };
+static const int16x8_t cospi21_v = { 8423, 8423, 8423, 8423,
+ 8423, 8423, 8423, 8423 };
+static const int16x8_t cospi21m_v = { -8423, -8423, -8423, -8423,
+ -8423, -8423, -8423, -8423 };
+static const int16x8_t cospi22_v = { 7723, 7723, 7723, 7723,
+ 7723, 7723, 7723, 7723 };
+static const int16x8_t cospi23_v = { 7005, 7005, 7005, 7005,
+ 7005, 7005, 7005, 7005 };
+static const int16x8_t cospi24_v = { 6270, 6270, 6270, 6270,
+ 6270, 6270, 6270, 6270 };
+static const int16x8_t cospi24m_v = { -6270, -6270, -6270, -6270,
+ -6270, -6270, -6270, -6270 };
+static const int16x8_t cospi25_v = { 5520, 5520, 5520, 5520,
+ 5520, 5520, 5520, 5520 };
+static const int16x8_t cospi25m_v = { -5520, -5520, -5520, -5520,
+ -5520, -5520, -5520, -5520 };
+static const int16x8_t cospi26_v = { 4756, 4756, 4756, 4756,
+ 4756, 4756, 4756, 4756 };
+static const int16x8_t cospi26m_v = { -4756, -4756, -4756, -4756,
+ -4756, -4756, -4756, -4756 };
+static const int16x8_t cospi27_v = { 3981, 3981, 3981, 3981,
+ 3981, 3981, 3981, 3981 };
+static const int16x8_t cospi28_v = { 3196, 3196, 3196, 3196,
+ 3196, 3196, 3196, 3196 };
+static const int16x8_t cospi28m_v = { -3196, -3196, -3196, -3196,
+ -3196, -3196, -3196, -3196 };
+static const int16x8_t cospi29_v = { 2404, 2404, 2404, 2404,
+ 2404, 2404, 2404, 2404 };
+static const int16x8_t cospi29m_v = { -2404, -2404, -2404, -2404,
+ -2404, -2404, -2404, -2404 };
+static const int16x8_t cospi30_v = { 1606, 1606, 1606, 1606,
+ 1606, 1606, 1606, 1606 };
+static const int16x8_t cospi31_v = { 804, 804, 804, 804, 804, 804, 804, 804 };
+
+static const int16x8_t sinpi_1_9_v = { 5283, 5283, 5283, 5283,
+ 5283, 5283, 5283, 5283 };
+static const int16x8_t sinpi_2_9_v = { 9929, 9929, 9929, 9929,
+ 9929, 9929, 9929, 9929 };
+static const int16x8_t sinpi_3_9_v = { 13377, 13377, 13377, 13377,
+ 13377, 13377, 13377, 13377 };
+static const int16x8_t sinpi_4_9_v = { 15212, 15212, 15212, 15212,
+ 15212, 15212, 15212, 15212 };
+
+static uint8x16_t tr8_mask0 = {
+ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17
+};
+
+static uint8x16_t tr8_mask1 = {
+ 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF,
+ 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F
+};
+
+#define ROUND_SHIFT_INIT \
+ const int32x4_t shift = vec_sl(vec_splat_s32(1), vec_splat_u32(13)); \
+ const uint32x4_t shift14 = vec_splat_u32(14);
+
+#define DCT_CONST_ROUND_SHIFT(vec) vec = vec_sra(vec_add(vec, shift), shift14);
+
+#define PIXEL_ADD_INIT \
+ int16x8_t add8 = vec_splat_s16(8); \
+ uint16x8_t shift4 = vec_splat_u16(4);
+
+#define PIXEL_ADD4(out, in) out = vec_sra(vec_add(in, add8), shift4);
+
+#define IDCT4(in0, in1, out0, out1) \
+ t0 = vec_add(in0, in1); \
+ t1 = vec_sub(in0, in1); \
+ tmp16_0 = vec_mergeh(t0, t1); \
+ temp1 = vec_sra(vec_add(vec_mule(tmp16_0, cospi16_v), shift), shift14); \
+ temp2 = vec_sra(vec_add(vec_mulo(tmp16_0, cospi16_v), shift), shift14); \
+ \
+ tmp16_0 = vec_mergel(in0, in1); \
+ temp3 = vec_sub(vec_mule(tmp16_0, cospi24_v), vec_mulo(tmp16_0, cospi8_v)); \
+ DCT_CONST_ROUND_SHIFT(temp3); \
+ temp4 = vec_add(vec_mule(tmp16_0, cospi8_v), vec_mulo(tmp16_0, cospi24_v)); \
+ DCT_CONST_ROUND_SHIFT(temp4); \
+ \
+ step0 = vec_packs(temp1, temp2); \
+ step1 = vec_packs(temp4, temp3); \
+ out0 = vec_add(step0, step1); \
+ out1 = vec_sub(step0, step1); \
+ out1 = vec_perm(out1, out1, mask0);
+
+#define PACK_STORE(v0, v1) \
+ tmp16_0 = vec_add(vec_perm(d_u0, d_u1, tr8_mask0), v0); \
+ tmp16_1 = vec_add(vec_perm(d_u2, d_u3, tr8_mask0), v1); \
+ output_v = vec_packsu(tmp16_0, tmp16_1); \
+ \
+ vec_vsx_st(output_v, 0, tmp_dest); \
+ for (i = 0; i < 4; i++) \
+ for (j = 0; j < 4; j++) dest[j * stride + i] = tmp_dest[j * 4 + i];
+
+void vpx_round_store4x4_vsx(int16x8_t *in, int16x8_t *out, uint8_t *dest,
+ int stride) {
+ int i, j;
+ uint8x16_t dest0 = vec_vsx_ld(0, dest);
+ uint8x16_t dest1 = vec_vsx_ld(stride, dest);
+ uint8x16_t dest2 = vec_vsx_ld(2 * stride, dest);
+ uint8x16_t dest3 = vec_vsx_ld(3 * stride, dest);
+ uint8x16_t zerov = vec_splat_u8(0);
+ int16x8_t d_u0 = (int16x8_t)vec_mergeh(dest0, zerov);
+ int16x8_t d_u1 = (int16x8_t)vec_mergeh(dest1, zerov);
+ int16x8_t d_u2 = (int16x8_t)vec_mergeh(dest2, zerov);
+ int16x8_t d_u3 = (int16x8_t)vec_mergeh(dest3, zerov);
+ int16x8_t tmp16_0, tmp16_1;
+ uint8x16_t output_v;
+ uint8_t tmp_dest[16];
+ PIXEL_ADD_INIT;
+
+ PIXEL_ADD4(out[0], in[0]);
+ PIXEL_ADD4(out[1], in[1]);
+
+ PACK_STORE(out[0], out[1]);
+}
+
+void vpx_idct4_vsx(int16x8_t *in, int16x8_t *out) {
+ int32x4_t temp1, temp2, temp3, temp4;
+ int16x8_t step0, step1, tmp16_0;
+ uint8x16_t mask0 = { 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF,
+ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 };
+ int16x8_t t0 = vec_mergeh(in[0], in[1]);
+ int16x8_t t1 = vec_mergel(in[0], in[1]);
+ ROUND_SHIFT_INIT
+
+ in[0] = vec_mergeh(t0, t1);
+ in[1] = vec_mergel(t0, t1);
+
+ IDCT4(in[0], in[1], out[0], out[1]);
+}
+
+void vpx_idct4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ int16x8_t in[2], out[2];
+
+ in[0] = load_tran_low(0, input);
+ in[1] = load_tran_low(8 * sizeof(*input), input);
+ // Rows
+ vpx_idct4_vsx(in, out);
+
+ // Columns
+ vpx_idct4_vsx(out, in);
+
+ vpx_round_store4x4_vsx(in, out, dest, stride);
+}
+
+#define TRANSPOSE8x8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \
+ out3, out4, out5, out6, out7) \
+ out0 = vec_mergeh(in0, in1); \
+ out1 = vec_mergel(in0, in1); \
+ out2 = vec_mergeh(in2, in3); \
+ out3 = vec_mergel(in2, in3); \
+ out4 = vec_mergeh(in4, in5); \
+ out5 = vec_mergel(in4, in5); \
+ out6 = vec_mergeh(in6, in7); \
+ out7 = vec_mergel(in6, in7); \
+ in0 = (int16x8_t)vec_mergeh((int32x4_t)out0, (int32x4_t)out2); \
+ in1 = (int16x8_t)vec_mergel((int32x4_t)out0, (int32x4_t)out2); \
+ in2 = (int16x8_t)vec_mergeh((int32x4_t)out1, (int32x4_t)out3); \
+ in3 = (int16x8_t)vec_mergel((int32x4_t)out1, (int32x4_t)out3); \
+ in4 = (int16x8_t)vec_mergeh((int32x4_t)out4, (int32x4_t)out6); \
+ in5 = (int16x8_t)vec_mergel((int32x4_t)out4, (int32x4_t)out6); \
+ in6 = (int16x8_t)vec_mergeh((int32x4_t)out5, (int32x4_t)out7); \
+ in7 = (int16x8_t)vec_mergel((int32x4_t)out5, (int32x4_t)out7); \
+ out0 = vec_perm(in0, in4, tr8_mask0); \
+ out1 = vec_perm(in0, in4, tr8_mask1); \
+ out2 = vec_perm(in1, in5, tr8_mask0); \
+ out3 = vec_perm(in1, in5, tr8_mask1); \
+ out4 = vec_perm(in2, in6, tr8_mask0); \
+ out5 = vec_perm(in2, in6, tr8_mask1); \
+ out6 = vec_perm(in3, in7, tr8_mask0); \
+ out7 = vec_perm(in3, in7, tr8_mask1);
+
+/* for the: temp1 = step[x] * cospi_q - step[y] * cospi_z
+ * temp2 = step[x] * cospi_z + step[y] * cospi_q */
+#define STEP8_0(inpt0, inpt1, outpt0, outpt1, cospi0, cospi1) \
+ tmp16_0 = vec_mergeh(inpt0, inpt1); \
+ tmp16_1 = vec_mergel(inpt0, inpt1); \
+ temp10 = vec_sub(vec_mule(tmp16_0, cospi0), vec_mulo(tmp16_0, cospi1)); \
+ temp11 = vec_sub(vec_mule(tmp16_1, cospi0), vec_mulo(tmp16_1, cospi1)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt0 = vec_packs(temp10, temp11); \
+ temp10 = vec_add(vec_mule(tmp16_0, cospi1), vec_mulo(tmp16_0, cospi0)); \
+ temp11 = vec_add(vec_mule(tmp16_1, cospi1), vec_mulo(tmp16_1, cospi0)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt1 = vec_packs(temp10, temp11);
+
+#define STEP8_1(inpt0, inpt1, outpt0, outpt1, cospi) \
+ tmp16_2 = vec_sub(inpt0, inpt1); \
+ tmp16_3 = vec_add(inpt0, inpt1); \
+ tmp16_0 = vec_mergeh(tmp16_2, tmp16_3); \
+ tmp16_1 = vec_mergel(tmp16_2, tmp16_3); \
+ temp10 = vec_mule(tmp16_0, cospi); \
+ temp11 = vec_mule(tmp16_1, cospi); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt0 = vec_packs(temp10, temp11); \
+ temp10 = vec_mulo(tmp16_0, cospi); \
+ temp11 = vec_mulo(tmp16_1, cospi); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt1 = vec_packs(temp10, temp11);
+
+#define IDCT8(in0, in1, in2, in3, in4, in5, in6, in7) \
+ /* stage 1 */ \
+ step0 = in0; \
+ step2 = in4; \
+ step1 = in2; \
+ step3 = in6; \
+ \
+ STEP8_0(in1, in7, step4, step7, cospi28_v, cospi4_v); \
+ STEP8_0(in5, in3, step5, step6, cospi12_v, cospi20_v); \
+ \
+ /* stage 2 */ \
+ STEP8_1(step0, step2, in1, in0, cospi16_v); \
+ STEP8_0(step1, step3, in2, in3, cospi24_v, cospi8_v); \
+ in4 = vec_add(step4, step5); \
+ in5 = vec_sub(step4, step5); \
+ in6 = vec_sub(step7, step6); \
+ in7 = vec_add(step6, step7); \
+ \
+ /* stage 3 */ \
+ step0 = vec_add(in0, in3); \
+ step1 = vec_add(in1, in2); \
+ step2 = vec_sub(in1, in2); \
+ step3 = vec_sub(in0, in3); \
+ step4 = in4; \
+ STEP8_1(in6, in5, step5, step6, cospi16_v); \
+ step7 = in7; \
+ \
+ /* stage 4 */ \
+ in0 = vec_add(step0, step7); \
+ in1 = vec_add(step1, step6); \
+ in2 = vec_add(step2, step5); \
+ in3 = vec_add(step3, step4); \
+ in4 = vec_sub(step3, step4); \
+ in5 = vec_sub(step2, step5); \
+ in6 = vec_sub(step1, step6); \
+ in7 = vec_sub(step0, step7);
+
+#define PIXEL_ADD(in, out, add, shiftx) \
+ out = vec_add(vec_sra(vec_add(in, add), shiftx), out);
+
+void vpx_idct8_vsx(int16x8_t *in, int16x8_t *out) {
+ int16x8_t step0, step1, step2, step3, step4, step5, step6, step7;
+ int16x8_t tmp16_0, tmp16_1, tmp16_2, tmp16_3;
+ int32x4_t temp10, temp11;
+ ROUND_SHIFT_INIT;
+
+ TRANSPOSE8x8(in[0], in[1], in[2], in[3], in[4], in[5], in[6], in[7], out[0],
+ out[1], out[2], out[3], out[4], out[5], out[6], out[7]);
+
+ IDCT8(out[0], out[1], out[2], out[3], out[4], out[5], out[6], out[7]);
+}
+
+void vpx_round_store8x8_vsx(int16x8_t *in, uint8_t *dest, int stride) {
+ uint8x16_t zerov = vec_splat_u8(0);
+ uint8x16_t dest0 = vec_vsx_ld(0, dest);
+ uint8x16_t dest1 = vec_vsx_ld(stride, dest);
+ uint8x16_t dest2 = vec_vsx_ld(2 * stride, dest);
+ uint8x16_t dest3 = vec_vsx_ld(3 * stride, dest);
+ uint8x16_t dest4 = vec_vsx_ld(4 * stride, dest);
+ uint8x16_t dest5 = vec_vsx_ld(5 * stride, dest);
+ uint8x16_t dest6 = vec_vsx_ld(6 * stride, dest);
+ uint8x16_t dest7 = vec_vsx_ld(7 * stride, dest);
+ int16x8_t d_u0 = (int16x8_t)vec_mergeh(dest0, zerov);
+ int16x8_t d_u1 = (int16x8_t)vec_mergeh(dest1, zerov);
+ int16x8_t d_u2 = (int16x8_t)vec_mergeh(dest2, zerov);
+ int16x8_t d_u3 = (int16x8_t)vec_mergeh(dest3, zerov);
+ int16x8_t d_u4 = (int16x8_t)vec_mergeh(dest4, zerov);
+ int16x8_t d_u5 = (int16x8_t)vec_mergeh(dest5, zerov);
+ int16x8_t d_u6 = (int16x8_t)vec_mergeh(dest6, zerov);
+ int16x8_t d_u7 = (int16x8_t)vec_mergeh(dest7, zerov);
+ int16x8_t add = vec_sl(vec_splat_s16(8), vec_splat_u16(1));
+ uint16x8_t shift5 = vec_splat_u16(5);
+ uint8x16_t output0, output1, output2, output3;
+
+ PIXEL_ADD(in[0], d_u0, add, shift5);
+ PIXEL_ADD(in[1], d_u1, add, shift5);
+ PIXEL_ADD(in[2], d_u2, add, shift5);
+ PIXEL_ADD(in[3], d_u3, add, shift5);
+ PIXEL_ADD(in[4], d_u4, add, shift5);
+ PIXEL_ADD(in[5], d_u5, add, shift5);
+ PIXEL_ADD(in[6], d_u6, add, shift5);
+ PIXEL_ADD(in[7], d_u7, add, shift5);
+ output0 = vec_packsu(d_u0, d_u1);
+ output1 = vec_packsu(d_u2, d_u3);
+ output2 = vec_packsu(d_u4, d_u5);
+ output3 = vec_packsu(d_u6, d_u7);
+
+ vec_vsx_st(xxpermdi(output0, dest0, 1), 0, dest);
+ vec_vsx_st(xxpermdi(output0, dest1, 3), stride, dest);
+ vec_vsx_st(xxpermdi(output1, dest2, 1), 2 * stride, dest);
+ vec_vsx_st(xxpermdi(output1, dest3, 3), 3 * stride, dest);
+ vec_vsx_st(xxpermdi(output2, dest4, 1), 4 * stride, dest);
+ vec_vsx_st(xxpermdi(output2, dest5, 3), 5 * stride, dest);
+ vec_vsx_st(xxpermdi(output3, dest6, 1), 6 * stride, dest);
+ vec_vsx_st(xxpermdi(output3, dest7, 3), 7 * stride, dest);
+}
+
+void vpx_idct8x8_64_add_vsx(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ int16x8_t src[8], tmp[8];
+
+ src[0] = load_tran_low(0, input);
+ src[1] = load_tran_low(8 * sizeof(*input), input);
+ src[2] = load_tran_low(16 * sizeof(*input), input);
+ src[3] = load_tran_low(24 * sizeof(*input), input);
+ src[4] = load_tran_low(32 * sizeof(*input), input);
+ src[5] = load_tran_low(40 * sizeof(*input), input);
+ src[6] = load_tran_low(48 * sizeof(*input), input);
+ src[7] = load_tran_low(56 * sizeof(*input), input);
+
+ vpx_idct8_vsx(src, tmp);
+ vpx_idct8_vsx(tmp, src);
+
+ vpx_round_store8x8_vsx(src, dest, stride);
+}
+
+#define STEP16_1(inpt0, inpt1, outpt0, outpt1, cospi) \
+ tmp16_0 = vec_mergeh(inpt0, inpt1); \
+ tmp16_1 = vec_mergel(inpt0, inpt1); \
+ temp10 = vec_mule(tmp16_0, cospi); \
+ temp11 = vec_mule(tmp16_1, cospi); \
+ temp20 = vec_mulo(tmp16_0, cospi); \
+ temp21 = vec_mulo(tmp16_1, cospi); \
+ temp30 = vec_sub(temp10, temp20); \
+ temp10 = vec_add(temp10, temp20); \
+ temp20 = vec_sub(temp11, temp21); \
+ temp21 = vec_add(temp11, temp21); \
+ DCT_CONST_ROUND_SHIFT(temp30); \
+ DCT_CONST_ROUND_SHIFT(temp20); \
+ outpt0 = vec_packs(temp30, temp20); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp21); \
+ outpt1 = vec_packs(temp10, temp21);
+
+#define IDCT16(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, inA, inB, \
+ inC, inD, inE, inF, out0, out1, out2, out3, out4, out5, out6, \
+ out7, out8, out9, outA, outB, outC, outD, outE, outF) \
+ /* stage 1 */ \
+ /* out0 = in0; */ \
+ out1 = in8; \
+ out2 = in4; \
+ out3 = inC; \
+ out4 = in2; \
+ out5 = inA; \
+ out6 = in6; \
+ out7 = inE; \
+ out8 = in1; \
+ out9 = in9; \
+ outA = in5; \
+ outB = inD; \
+ outC = in3; \
+ outD = inB; \
+ outE = in7; \
+ outF = inF; \
+ \
+ /* stage 2 */ \
+ /* in0 = out0; */ \
+ in1 = out1; \
+ in2 = out2; \
+ in3 = out3; \
+ in4 = out4; \
+ in5 = out5; \
+ in6 = out6; \
+ in7 = out7; \
+ \
+ STEP8_0(out8, outF, in8, inF, cospi30_v, cospi2_v); \
+ STEP8_0(out9, outE, in9, inE, cospi14_v, cospi18_v); \
+ STEP8_0(outA, outD, inA, inD, cospi22_v, cospi10_v); \
+ STEP8_0(outB, outC, inB, inC, cospi6_v, cospi26_v); \
+ \
+ /* stage 3 */ \
+ out0 = in0; \
+ out1 = in1; \
+ out2 = in2; \
+ out3 = in3; \
+ \
+ STEP8_0(in4, in7, out4, out7, cospi28_v, cospi4_v); \
+ STEP8_0(in5, in6, out5, out6, cospi12_v, cospi20_v); \
+ \
+ out8 = vec_add(in8, in9); \
+ out9 = vec_sub(in8, in9); \
+ outA = vec_sub(inB, inA); \
+ outB = vec_add(inA, inB); \
+ outC = vec_add(inC, inD); \
+ outD = vec_sub(inC, inD); \
+ outE = vec_sub(inF, inE); \
+ outF = vec_add(inE, inF); \
+ \
+ /* stage 4 */ \
+ STEP16_1(out0, out1, in1, in0, cospi16_v); \
+ STEP8_0(out2, out3, in2, in3, cospi24_v, cospi8_v); \
+ in4 = vec_add(out4, out5); \
+ in5 = vec_sub(out4, out5); \
+ in6 = vec_sub(out7, out6); \
+ in7 = vec_add(out6, out7); \
+ \
+ in8 = out8; \
+ inF = outF; \
+ tmp16_0 = vec_mergeh(out9, outE); \
+ tmp16_1 = vec_mergel(out9, outE); \
+ temp10 = vec_sub(vec_mulo(tmp16_0, cospi24_v), vec_mule(tmp16_0, cospi8_v)); \
+ temp11 = vec_sub(vec_mulo(tmp16_1, cospi24_v), vec_mule(tmp16_1, cospi8_v)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ in9 = vec_packs(temp10, temp11); \
+ temp10 = vec_add(vec_mule(tmp16_0, cospi24_v), vec_mulo(tmp16_0, cospi8_v)); \
+ temp11 = vec_add(vec_mule(tmp16_1, cospi24_v), vec_mulo(tmp16_1, cospi8_v)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ inE = vec_packs(temp10, temp11); \
+ \
+ tmp16_0 = vec_mergeh(outA, outD); \
+ tmp16_1 = vec_mergel(outA, outD); \
+ temp10 = \
+ vec_sub(vec_mule(tmp16_0, cospi24m_v), vec_mulo(tmp16_0, cospi8_v)); \
+ temp11 = \
+ vec_sub(vec_mule(tmp16_1, cospi24m_v), vec_mulo(tmp16_1, cospi8_v)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ inA = vec_packs(temp10, temp11); \
+ temp10 = vec_sub(vec_mulo(tmp16_0, cospi24_v), vec_mule(tmp16_0, cospi8_v)); \
+ temp11 = vec_sub(vec_mulo(tmp16_1, cospi24_v), vec_mule(tmp16_1, cospi8_v)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ inD = vec_packs(temp10, temp11); \
+ \
+ inB = outB; \
+ inC = outC; \
+ \
+ /* stage 5 */ \
+ out0 = vec_add(in0, in3); \
+ out1 = vec_add(in1, in2); \
+ out2 = vec_sub(in1, in2); \
+ out3 = vec_sub(in0, in3); \
+ out4 = in4; \
+ STEP16_1(in6, in5, out5, out6, cospi16_v); \
+ out7 = in7; \
+ \
+ out8 = vec_add(in8, inB); \
+ out9 = vec_add(in9, inA); \
+ outA = vec_sub(in9, inA); \
+ outB = vec_sub(in8, inB); \
+ outC = vec_sub(inF, inC); \
+ outD = vec_sub(inE, inD); \
+ outE = vec_add(inD, inE); \
+ outF = vec_add(inC, inF); \
+ \
+ /* stage 6 */ \
+ in0 = vec_add(out0, out7); \
+ in1 = vec_add(out1, out6); \
+ in2 = vec_add(out2, out5); \
+ in3 = vec_add(out3, out4); \
+ in4 = vec_sub(out3, out4); \
+ in5 = vec_sub(out2, out5); \
+ in6 = vec_sub(out1, out6); \
+ in7 = vec_sub(out0, out7); \
+ in8 = out8; \
+ in9 = out9; \
+ STEP16_1(outD, outA, inA, inD, cospi16_v); \
+ STEP16_1(outC, outB, inB, inC, cospi16_v); \
+ inE = outE; \
+ inF = outF; \
+ \
+ /* stage 7 */ \
+ out0 = vec_add(in0, inF); \
+ out1 = vec_add(in1, inE); \
+ out2 = vec_add(in2, inD); \
+ out3 = vec_add(in3, inC); \
+ out4 = vec_add(in4, inB); \
+ out5 = vec_add(in5, inA); \
+ out6 = vec_add(in6, in9); \
+ out7 = vec_add(in7, in8); \
+ out8 = vec_sub(in7, in8); \
+ out9 = vec_sub(in6, in9); \
+ outA = vec_sub(in5, inA); \
+ outB = vec_sub(in4, inB); \
+ outC = vec_sub(in3, inC); \
+ outD = vec_sub(in2, inD); \
+ outE = vec_sub(in1, inE); \
+ outF = vec_sub(in0, inF);
+
+#define PIXEL_ADD_STORE16(in0, in1, dst, offset) \
+ d_uh = (int16x8_t)vec_mergeh(dst, zerov); \
+ d_ul = (int16x8_t)vec_mergel(dst, zerov); \
+ PIXEL_ADD(in0, d_uh, add, shift6); \
+ PIXEL_ADD(in1, d_ul, add, shift6); \
+ vec_vsx_st(vec_packsu(d_uh, d_ul), offset, dest);
+
+static void half_idct16x8_vsx(int16x8_t *src) {
+ int16x8_t tmp0[8], tmp1[8];
+ int32x4_t temp10, temp11, temp20, temp21, temp30;
+ int16x8_t tmp16_0, tmp16_1;
+ ROUND_SHIFT_INIT;
+
+ TRANSPOSE8x8(src[0], src[2], src[4], src[6], src[8], src[10], src[12],
+ src[14], tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5],
+ tmp0[6], tmp0[7]);
+ TRANSPOSE8x8(src[1], src[3], src[5], src[7], src[9], src[11], src[13],
+ src[15], tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5],
+ tmp1[6], tmp1[7]);
+ IDCT16(tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5], tmp0[6], tmp0[7],
+ tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5], tmp1[6], tmp1[7],
+ src[0], src[2], src[4], src[6], src[8], src[10], src[12], src[14],
+ src[1], src[3], src[5], src[7], src[9], src[11], src[13], src[15]);
+}
+
+void vpx_idct16_vsx(int16x8_t *src0, int16x8_t *src1) {
+ int16x8_t tmp0[8], tmp1[8], tmp2[8], tmp3[8];
+ int32x4_t temp10, temp11, temp20, temp21, temp30;
+ int16x8_t tmp16_0, tmp16_1;
+ ROUND_SHIFT_INIT;
+
+ TRANSPOSE8x8(src0[0], src0[2], src0[4], src0[6], src0[8], src0[10], src0[12],
+ src0[14], tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5],
+ tmp0[6], tmp0[7]);
+ TRANSPOSE8x8(src0[1], src0[3], src0[5], src0[7], src0[9], src0[11], src0[13],
+ src0[15], tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5],
+ tmp1[6], tmp1[7]);
+ TRANSPOSE8x8(src1[0], src1[2], src1[4], src1[6], src1[8], src1[10], src1[12],
+ src1[14], tmp2[0], tmp2[1], tmp2[2], tmp2[3], tmp2[4], tmp2[5],
+ tmp2[6], tmp2[7]);
+ TRANSPOSE8x8(src1[1], src1[3], src1[5], src1[7], src1[9], src1[11], src1[13],
+ src1[15], tmp3[0], tmp3[1], tmp3[2], tmp3[3], tmp3[4], tmp3[5],
+ tmp3[6], tmp3[7]);
+
+ IDCT16(tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5], tmp0[6], tmp0[7],
+ tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5], tmp1[6], tmp1[7],
+ src0[0], src0[2], src0[4], src0[6], src0[8], src0[10], src0[12],
+ src0[14], src1[0], src1[2], src1[4], src1[6], src1[8], src1[10],
+ src1[12], src1[14]);
+
+ IDCT16(tmp2[0], tmp2[1], tmp2[2], tmp2[3], tmp2[4], tmp2[5], tmp2[6], tmp2[7],
+ tmp3[0], tmp3[1], tmp3[2], tmp3[3], tmp3[4], tmp3[5], tmp3[6], tmp3[7],
+ src0[1], src0[3], src0[5], src0[7], src0[9], src0[11], src0[13],
+ src0[15], src1[1], src1[3], src1[5], src1[7], src1[9], src1[11],
+ src1[13], src1[15]);
+}
+
+void vpx_round_store16x16_vsx(int16x8_t *src0, int16x8_t *src1, uint8_t *dest,
+ int stride) {
+ uint8x16_t destv[16];
+ int16x8_t d_uh, d_ul;
+ uint8x16_t zerov = vec_splat_u8(0);
+ uint16x8_t shift6 = vec_splat_u16(6);
+ int16x8_t add = vec_sl(vec_splat_s16(8), vec_splat_u16(2));
+
+ // load dest
+ LOAD_INPUT16(vec_vsx_ld, dest, 0, stride, destv);
+
+ PIXEL_ADD_STORE16(src0[0], src0[1], destv[0], 0);
+ PIXEL_ADD_STORE16(src0[2], src0[3], destv[1], stride);
+ PIXEL_ADD_STORE16(src0[4], src0[5], destv[2], 2 * stride);
+ PIXEL_ADD_STORE16(src0[6], src0[7], destv[3], 3 * stride);
+ PIXEL_ADD_STORE16(src0[8], src0[9], destv[4], 4 * stride);
+ PIXEL_ADD_STORE16(src0[10], src0[11], destv[5], 5 * stride);
+ PIXEL_ADD_STORE16(src0[12], src0[13], destv[6], 6 * stride);
+ PIXEL_ADD_STORE16(src0[14], src0[15], destv[7], 7 * stride);
+
+ PIXEL_ADD_STORE16(src1[0], src1[1], destv[8], 8 * stride);
+ PIXEL_ADD_STORE16(src1[2], src1[3], destv[9], 9 * stride);
+ PIXEL_ADD_STORE16(src1[4], src1[5], destv[10], 10 * stride);
+ PIXEL_ADD_STORE16(src1[6], src1[7], destv[11], 11 * stride);
+ PIXEL_ADD_STORE16(src1[8], src1[9], destv[12], 12 * stride);
+ PIXEL_ADD_STORE16(src1[10], src1[11], destv[13], 13 * stride);
+ PIXEL_ADD_STORE16(src1[12], src1[13], destv[14], 14 * stride);
+ PIXEL_ADD_STORE16(src1[14], src1[15], destv[15], 15 * stride);
+}
+void vpx_idct16x16_256_add_vsx(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ int16x8_t src0[16], src1[16];
+ int16x8_t tmp0[8], tmp1[8], tmp2[8], tmp3[8];
+ int32x4_t temp10, temp11, temp20, temp21, temp30;
+ int16x8_t tmp16_0, tmp16_1;
+ ROUND_SHIFT_INIT;
+
+ LOAD_INPUT16(load_tran_low, input, 0, 8 * sizeof(*input), src0);
+ LOAD_INPUT16(load_tran_low, input, 8 * 8 * 2 * sizeof(*input),
+ 8 * sizeof(*input), src1);
+
+ // transform rows
+ // transform the upper half of 16x16 matrix
+ half_idct16x8_vsx(src0);
+ TRANSPOSE8x8(src0[0], src0[2], src0[4], src0[6], src0[8], src0[10], src0[12],
+ src0[14], tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5],
+ tmp0[6], tmp0[7]);
+ TRANSPOSE8x8(src0[1], src0[3], src0[5], src0[7], src0[9], src0[11], src0[13],
+ src0[15], tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5],
+ tmp1[6], tmp1[7]);
+
+ // transform the lower half of 16x16 matrix
+ half_idct16x8_vsx(src1);
+ TRANSPOSE8x8(src1[0], src1[2], src1[4], src1[6], src1[8], src1[10], src1[12],
+ src1[14], tmp2[0], tmp2[1], tmp2[2], tmp2[3], tmp2[4], tmp2[5],
+ tmp2[6], tmp2[7]);
+ TRANSPOSE8x8(src1[1], src1[3], src1[5], src1[7], src1[9], src1[11], src1[13],
+ src1[15], tmp3[0], tmp3[1], tmp3[2], tmp3[3], tmp3[4], tmp3[5],
+ tmp3[6], tmp3[7]);
+
+ // transform columns
+ // left half first
+ IDCT16(tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5], tmp0[6], tmp0[7],
+ tmp2[0], tmp2[1], tmp2[2], tmp2[3], tmp2[4], tmp2[5], tmp2[6], tmp2[7],
+ src0[0], src0[2], src0[4], src0[6], src0[8], src0[10], src0[12],
+ src0[14], src1[0], src1[2], src1[4], src1[6], src1[8], src1[10],
+ src1[12], src1[14]);
+ // right half
+ IDCT16(tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5], tmp1[6], tmp1[7],
+ tmp3[0], tmp3[1], tmp3[2], tmp3[3], tmp3[4], tmp3[5], tmp3[6], tmp3[7],
+ src0[1], src0[3], src0[5], src0[7], src0[9], src0[11], src0[13],
+ src0[15], src1[1], src1[3], src1[5], src1[7], src1[9], src1[11],
+ src1[13], src1[15]);
+
+ vpx_round_store16x16_vsx(src0, src1, dest, stride);
+}
+
+#define LOAD_8x32(load, in00, in01, in02, in03, in10, in11, in12, in13, in20, \
+ in21, in22, in23, in30, in31, in32, in33, in40, in41, in42, \
+ in43, in50, in51, in52, in53, in60, in61, in62, in63, in70, \
+ in71, in72, in73, offset) \
+ /* load the first row from the 8x32 block*/ \
+ in00 = load(offset, input); \
+ in01 = load(offset + 16, input); \
+ in02 = load(offset + 2 * 16, input); \
+ in03 = load(offset + 3 * 16, input); \
+ \
+ in10 = load(offset + 4 * 16, input); \
+ in11 = load(offset + 5 * 16, input); \
+ in12 = load(offset + 6 * 16, input); \
+ in13 = load(offset + 7 * 16, input); \
+ \
+ in20 = load(offset + 8 * 16, input); \
+ in21 = load(offset + 9 * 16, input); \
+ in22 = load(offset + 10 * 16, input); \
+ in23 = load(offset + 11 * 16, input); \
+ \
+ in30 = load(offset + 12 * 16, input); \
+ in31 = load(offset + 13 * 16, input); \
+ in32 = load(offset + 14 * 16, input); \
+ in33 = load(offset + 15 * 16, input); \
+ \
+ in40 = load(offset + 16 * 16, input); \
+ in41 = load(offset + 17 * 16, input); \
+ in42 = load(offset + 18 * 16, input); \
+ in43 = load(offset + 19 * 16, input); \
+ \
+ in50 = load(offset + 20 * 16, input); \
+ in51 = load(offset + 21 * 16, input); \
+ in52 = load(offset + 22 * 16, input); \
+ in53 = load(offset + 23 * 16, input); \
+ \
+ in60 = load(offset + 24 * 16, input); \
+ in61 = load(offset + 25 * 16, input); \
+ in62 = load(offset + 26 * 16, input); \
+ in63 = load(offset + 27 * 16, input); \
+ \
+ /* load the last row from the 8x32 block*/ \
+ in70 = load(offset + 28 * 16, input); \
+ in71 = load(offset + 29 * 16, input); \
+ in72 = load(offset + 30 * 16, input); \
+ in73 = load(offset + 31 * 16, input);
+
+/* for the: temp1 = -step[x] * cospi_q + step[y] * cospi_z
+ * temp2 = step[x] * cospi_z + step[y] * cospi_q */
+#define STEP32(inpt0, inpt1, outpt0, outpt1, cospi0, cospi1) \
+ tmp16_0 = vec_mergeh(inpt0, inpt1); \
+ tmp16_1 = vec_mergel(inpt0, inpt1); \
+ temp10 = vec_sub(vec_mulo(tmp16_0, cospi1), vec_mule(tmp16_0, cospi0)); \
+ temp11 = vec_sub(vec_mulo(tmp16_1, cospi1), vec_mule(tmp16_1, cospi0)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt0 = vec_packs(temp10, temp11); \
+ temp10 = vec_add(vec_mule(tmp16_0, cospi1), vec_mulo(tmp16_0, cospi0)); \
+ temp11 = vec_add(vec_mule(tmp16_1, cospi1), vec_mulo(tmp16_1, cospi0)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt1 = vec_packs(temp10, temp11);
+
+/* for the: temp1 = -step[x] * cospi_q - step[y] * cospi_z
+ * temp2 = -step[x] * cospi_z + step[y] * cospi_q */
+#define STEP32_1(inpt0, inpt1, outpt0, outpt1, cospi0, cospi1, cospi1m) \
+ tmp16_0 = vec_mergeh(inpt0, inpt1); \
+ tmp16_1 = vec_mergel(inpt0, inpt1); \
+ temp10 = vec_sub(vec_mulo(tmp16_0, cospi1m), vec_mule(tmp16_0, cospi0)); \
+ temp11 = vec_sub(vec_mulo(tmp16_1, cospi1m), vec_mule(tmp16_1, cospi0)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt0 = vec_packs(temp10, temp11); \
+ temp10 = vec_sub(vec_mulo(tmp16_0, cospi0), vec_mule(tmp16_0, cospi1)); \
+ temp11 = vec_sub(vec_mulo(tmp16_1, cospi0), vec_mule(tmp16_1, cospi1)); \
+ DCT_CONST_ROUND_SHIFT(temp10); \
+ DCT_CONST_ROUND_SHIFT(temp11); \
+ outpt1 = vec_packs(temp10, temp11);
+
+#define IDCT32(in0, in1, in2, in3, out) \
+ \
+ /* stage 1 */ \
+ /* out[0][0] = in[0][0]; */ \
+ out[0][1] = in2[0]; \
+ out[0][2] = in1[0]; \
+ out[0][3] = in3[0]; \
+ out[0][4] = in0[4]; \
+ out[0][5] = in2[4]; \
+ out[0][6] = in1[4]; \
+ out[0][7] = in3[4]; \
+ out[1][0] = in0[2]; \
+ out[1][1] = in2[2]; \
+ out[1][2] = in1[2]; \
+ out[1][3] = in3[2]; \
+ out[1][4] = in0[6]; \
+ out[1][5] = in2[6]; \
+ out[1][6] = in1[6]; \
+ out[1][7] = in3[6]; \
+ \
+ STEP8_0(in0[1], in3[7], out[2][0], out[3][7], cospi31_v, cospi1_v); \
+ STEP8_0(in2[1], in1[7], out[2][1], out[3][6], cospi15_v, cospi17_v); \
+ STEP8_0(in1[1], in2[7], out[2][2], out[3][5], cospi23_v, cospi9_v); \
+ STEP8_0(in3[1], in0[7], out[2][3], out[3][4], cospi7_v, cospi25_v); \
+ STEP8_0(in0[5], in3[3], out[2][4], out[3][3], cospi27_v, cospi5_v); \
+ STEP8_0(in2[5], in1[3], out[2][5], out[3][2], cospi11_v, cospi21_v); \
+ STEP8_0(in1[5], in2[3], out[2][6], out[3][1], cospi19_v, cospi13_v); \
+ STEP8_0(in3[5], in0[3], out[2][7], out[3][0], cospi3_v, cospi29_v); \
+ \
+ /* stage 2 */ \
+ /* in0[0] = out[0][0]; */ \
+ in0[1] = out[0][1]; \
+ in0[2] = out[0][2]; \
+ in0[3] = out[0][3]; \
+ in0[4] = out[0][4]; \
+ in0[5] = out[0][5]; \
+ in0[6] = out[0][6]; \
+ in0[7] = out[0][7]; \
+ \
+ STEP8_0(out[1][0], out[1][7], in1[0], in1[7], cospi30_v, cospi2_v); \
+ STEP8_0(out[1][1], out[1][6], in1[1], in1[6], cospi14_v, cospi18_v); \
+ STEP8_0(out[1][2], out[1][5], in1[2], in1[5], cospi22_v, cospi10_v); \
+ STEP8_0(out[1][3], out[1][4], in1[3], in1[4], cospi6_v, cospi26_v); \
+ \
+ in2[0] = vec_add(out[2][0], out[2][1]); \
+ in2[1] = vec_sub(out[2][0], out[2][1]); \
+ in2[2] = vec_sub(out[2][3], out[2][2]); \
+ in2[3] = vec_add(out[2][3], out[2][2]); \
+ in2[4] = vec_add(out[2][4], out[2][5]); \
+ in2[5] = vec_sub(out[2][4], out[2][5]); \
+ in2[6] = vec_sub(out[2][7], out[2][6]); \
+ in2[7] = vec_add(out[2][7], out[2][6]); \
+ in3[0] = vec_add(out[3][0], out[3][1]); \
+ in3[1] = vec_sub(out[3][0], out[3][1]); \
+ in3[2] = vec_sub(out[3][3], out[3][2]); \
+ in3[3] = vec_add(out[3][3], out[3][2]); \
+ in3[4] = vec_add(out[3][4], out[3][5]); \
+ in3[5] = vec_sub(out[3][4], out[3][5]); \
+ in3[6] = vec_sub(out[3][7], out[3][6]); \
+ in3[7] = vec_add(out[3][6], out[3][7]); \
+ \
+ /* stage 3 */ \
+ out[0][0] = in0[0]; \
+ out[0][1] = in0[1]; \
+ out[0][2] = in0[2]; \
+ out[0][3] = in0[3]; \
+ \
+ STEP8_0(in0[4], in0[7], out[0][4], out[0][7], cospi28_v, cospi4_v); \
+ STEP8_0(in0[5], in0[6], out[0][5], out[0][6], cospi12_v, cospi20_v); \
+ \
+ out[1][0] = vec_add(in1[0], in1[1]); \
+ out[1][1] = vec_sub(in1[0], in1[1]); \
+ out[1][2] = vec_sub(in1[3], in1[2]); \
+ out[1][3] = vec_add(in1[2], in1[3]); \
+ out[1][4] = vec_add(in1[4], in1[5]); \
+ out[1][5] = vec_sub(in1[4], in1[5]); \
+ out[1][6] = vec_sub(in1[7], in1[6]); \
+ out[1][7] = vec_add(in1[6], in1[7]); \
+ \
+ out[2][0] = in2[0]; \
+ out[3][7] = in3[7]; \
+ STEP32(in2[1], in3[6], out[2][1], out[3][6], cospi4_v, cospi28_v); \
+ STEP32_1(in2[2], in3[5], out[2][2], out[3][5], cospi28_v, cospi4_v, \
+ cospi4m_v); \
+ out[2][3] = in2[3]; \
+ out[2][4] = in2[4]; \
+ STEP32(in2[5], in3[2], out[2][5], out[3][2], cospi20_v, cospi12_v); \
+ STEP32_1(in2[6], in3[1], out[2][6], out[3][1], cospi12_v, cospi20_v, \
+ cospi20m_v); \
+ out[2][7] = in2[7]; \
+ out[3][0] = in3[0]; \
+ out[3][3] = in3[3]; \
+ out[3][4] = in3[4]; \
+ \
+ /* stage 4 */ \
+ STEP16_1(out[0][0], out[0][1], in0[1], in0[0], cospi16_v); \
+ STEP8_0(out[0][2], out[0][3], in0[2], in0[3], cospi24_v, cospi8_v); \
+ in0[4] = vec_add(out[0][4], out[0][5]); \
+ in0[5] = vec_sub(out[0][4], out[0][5]); \
+ in0[6] = vec_sub(out[0][7], out[0][6]); \
+ in0[7] = vec_add(out[0][7], out[0][6]); \
+ \
+ in1[0] = out[1][0]; \
+ in1[7] = out[1][7]; \
+ STEP32(out[1][1], out[1][6], in1[1], in1[6], cospi8_v, cospi24_v); \
+ STEP32_1(out[1][2], out[1][5], in1[2], in1[5], cospi24_v, cospi8_v, \
+ cospi8m_v); \
+ in1[3] = out[1][3]; \
+ in1[4] = out[1][4]; \
+ \
+ in2[0] = vec_add(out[2][0], out[2][3]); \
+ in2[1] = vec_add(out[2][1], out[2][2]); \
+ in2[2] = vec_sub(out[2][1], out[2][2]); \
+ in2[3] = vec_sub(out[2][0], out[2][3]); \
+ in2[4] = vec_sub(out[2][7], out[2][4]); \
+ in2[5] = vec_sub(out[2][6], out[2][5]); \
+ in2[6] = vec_add(out[2][5], out[2][6]); \
+ in2[7] = vec_add(out[2][4], out[2][7]); \
+ \
+ in3[0] = vec_add(out[3][0], out[3][3]); \
+ in3[1] = vec_add(out[3][1], out[3][2]); \
+ in3[2] = vec_sub(out[3][1], out[3][2]); \
+ in3[3] = vec_sub(out[3][0], out[3][3]); \
+ in3[4] = vec_sub(out[3][7], out[3][4]); \
+ in3[5] = vec_sub(out[3][6], out[3][5]); \
+ in3[6] = vec_add(out[3][5], out[3][6]); \
+ in3[7] = vec_add(out[3][4], out[3][7]); \
+ \
+ /* stage 5 */ \
+ out[0][0] = vec_add(in0[0], in0[3]); \
+ out[0][1] = vec_add(in0[1], in0[2]); \
+ out[0][2] = vec_sub(in0[1], in0[2]); \
+ out[0][3] = vec_sub(in0[0], in0[3]); \
+ out[0][4] = in0[4]; \
+ STEP16_1(in0[6], in0[5], out[0][5], out[0][6], cospi16_v); \
+ out[0][7] = in0[7]; \
+ \
+ out[1][0] = vec_add(in1[0], in1[3]); \
+ out[1][1] = vec_add(in1[1], in1[2]); \
+ out[1][2] = vec_sub(in1[1], in1[2]); \
+ out[1][3] = vec_sub(in1[0], in1[3]); \
+ out[1][4] = vec_sub(in1[7], in1[4]); \
+ out[1][5] = vec_sub(in1[6], in1[5]); \
+ out[1][6] = vec_add(in1[5], in1[6]); \
+ out[1][7] = vec_add(in1[4], in1[7]); \
+ \
+ out[2][0] = in2[0]; \
+ out[2][1] = in2[1]; \
+ STEP32(in2[2], in3[5], out[2][2], out[3][5], cospi8_v, cospi24_v); \
+ STEP32(in2[3], in3[4], out[2][3], out[3][4], cospi8_v, cospi24_v); \
+ STEP32_1(in2[4], in3[3], out[2][4], out[3][3], cospi24_v, cospi8_v, \
+ cospi8m_v); \
+ STEP32_1(in2[5], in3[2], out[2][5], out[3][2], cospi24_v, cospi8_v, \
+ cospi8m_v); \
+ out[2][6] = in2[6]; \
+ out[2][7] = in2[7]; \
+ out[3][0] = in3[0]; \
+ out[3][1] = in3[1]; \
+ out[3][6] = in3[6]; \
+ out[3][7] = in3[7]; \
+ \
+ /* stage 6 */ \
+ in0[0] = vec_add(out[0][0], out[0][7]); \
+ in0[1] = vec_add(out[0][1], out[0][6]); \
+ in0[2] = vec_add(out[0][2], out[0][5]); \
+ in0[3] = vec_add(out[0][3], out[0][4]); \
+ in0[4] = vec_sub(out[0][3], out[0][4]); \
+ in0[5] = vec_sub(out[0][2], out[0][5]); \
+ in0[6] = vec_sub(out[0][1], out[0][6]); \
+ in0[7] = vec_sub(out[0][0], out[0][7]); \
+ in1[0] = out[1][0]; \
+ in1[1] = out[1][1]; \
+ STEP16_1(out[1][5], out[1][2], in1[2], in1[5], cospi16_v); \
+ STEP16_1(out[1][4], out[1][3], in1[3], in1[4], cospi16_v); \
+ in1[6] = out[1][6]; \
+ in1[7] = out[1][7]; \
+ \
+ in2[0] = vec_add(out[2][0], out[2][7]); \
+ in2[1] = vec_add(out[2][1], out[2][6]); \
+ in2[2] = vec_add(out[2][2], out[2][5]); \
+ in2[3] = vec_add(out[2][3], out[2][4]); \
+ in2[4] = vec_sub(out[2][3], out[2][4]); \
+ in2[5] = vec_sub(out[2][2], out[2][5]); \
+ in2[6] = vec_sub(out[2][1], out[2][6]); \
+ in2[7] = vec_sub(out[2][0], out[2][7]); \
+ \
+ in3[0] = vec_sub(out[3][7], out[3][0]); \
+ in3[1] = vec_sub(out[3][6], out[3][1]); \
+ in3[2] = vec_sub(out[3][5], out[3][2]); \
+ in3[3] = vec_sub(out[3][4], out[3][3]); \
+ in3[4] = vec_add(out[3][4], out[3][3]); \
+ in3[5] = vec_add(out[3][5], out[3][2]); \
+ in3[6] = vec_add(out[3][6], out[3][1]); \
+ in3[7] = vec_add(out[3][7], out[3][0]); \
+ \
+ /* stage 7 */ \
+ out[0][0] = vec_add(in0[0], in1[7]); \
+ out[0][1] = vec_add(in0[1], in1[6]); \
+ out[0][2] = vec_add(in0[2], in1[5]); \
+ out[0][3] = vec_add(in0[3], in1[4]); \
+ out[0][4] = vec_add(in0[4], in1[3]); \
+ out[0][5] = vec_add(in0[5], in1[2]); \
+ out[0][6] = vec_add(in0[6], in1[1]); \
+ out[0][7] = vec_add(in0[7], in1[0]); \
+ out[1][0] = vec_sub(in0[7], in1[0]); \
+ out[1][1] = vec_sub(in0[6], in1[1]); \
+ out[1][2] = vec_sub(in0[5], in1[2]); \
+ out[1][3] = vec_sub(in0[4], in1[3]); \
+ out[1][4] = vec_sub(in0[3], in1[4]); \
+ out[1][5] = vec_sub(in0[2], in1[5]); \
+ out[1][6] = vec_sub(in0[1], in1[6]); \
+ out[1][7] = vec_sub(in0[0], in1[7]); \
+ \
+ out[2][0] = in2[0]; \
+ out[2][1] = in2[1]; \
+ out[2][2] = in2[2]; \
+ out[2][3] = in2[3]; \
+ STEP16_1(in3[3], in2[4], out[2][4], out[3][3], cospi16_v); \
+ STEP16_1(in3[2], in2[5], out[2][5], out[3][2], cospi16_v); \
+ STEP16_1(in3[1], in2[6], out[2][6], out[3][1], cospi16_v); \
+ STEP16_1(in3[0], in2[7], out[2][7], out[3][0], cospi16_v); \
+ out[3][4] = in3[4]; \
+ out[3][5] = in3[5]; \
+ out[3][6] = in3[6]; \
+ out[3][7] = in3[7]; \
+ \
+ /* final */ \
+ in0[0] = vec_add(out[0][0], out[3][7]); \
+ in0[1] = vec_add(out[0][1], out[3][6]); \
+ in0[2] = vec_add(out[0][2], out[3][5]); \
+ in0[3] = vec_add(out[0][3], out[3][4]); \
+ in0[4] = vec_add(out[0][4], out[3][3]); \
+ in0[5] = vec_add(out[0][5], out[3][2]); \
+ in0[6] = vec_add(out[0][6], out[3][1]); \
+ in0[7] = vec_add(out[0][7], out[3][0]); \
+ in1[0] = vec_add(out[1][0], out[2][7]); \
+ in1[1] = vec_add(out[1][1], out[2][6]); \
+ in1[2] = vec_add(out[1][2], out[2][5]); \
+ in1[3] = vec_add(out[1][3], out[2][4]); \
+ in1[4] = vec_add(out[1][4], out[2][3]); \
+ in1[5] = vec_add(out[1][5], out[2][2]); \
+ in1[6] = vec_add(out[1][6], out[2][1]); \
+ in1[7] = vec_add(out[1][7], out[2][0]); \
+ in2[0] = vec_sub(out[1][7], out[2][0]); \
+ in2[1] = vec_sub(out[1][6], out[2][1]); \
+ in2[2] = vec_sub(out[1][5], out[2][2]); \
+ in2[3] = vec_sub(out[1][4], out[2][3]); \
+ in2[4] = vec_sub(out[1][3], out[2][4]); \
+ in2[5] = vec_sub(out[1][2], out[2][5]); \
+ in2[6] = vec_sub(out[1][1], out[2][6]); \
+ in2[7] = vec_sub(out[1][0], out[2][7]); \
+ in3[0] = vec_sub(out[0][7], out[3][0]); \
+ in3[1] = vec_sub(out[0][6], out[3][1]); \
+ in3[2] = vec_sub(out[0][5], out[3][2]); \
+ in3[3] = vec_sub(out[0][4], out[3][3]); \
+ in3[4] = vec_sub(out[0][3], out[3][4]); \
+ in3[5] = vec_sub(out[0][2], out[3][5]); \
+ in3[6] = vec_sub(out[0][1], out[3][6]); \
+ in3[7] = vec_sub(out[0][0], out[3][7]);
+
+// NOT A FULL TRANSPOSE! Transposes just each 8x8 block in each row,
+// does not transpose rows
+#define TRANSPOSE_8x32(in, out) \
+ /* transpose 4 of 8x8 blocks */ \
+ TRANSPOSE8x8(in[0][0], in[0][1], in[0][2], in[0][3], in[0][4], in[0][5], \
+ in[0][6], in[0][7], out[0][0], out[0][1], out[0][2], out[0][3], \
+ out[0][4], out[0][5], out[0][6], out[0][7]); \
+ TRANSPOSE8x8(in[1][0], in[1][1], in[1][2], in[1][3], in[1][4], in[1][5], \
+ in[1][6], in[1][7], out[1][0], out[1][1], out[1][2], out[1][3], \
+ out[1][4], out[1][5], out[1][6], out[1][7]); \
+ TRANSPOSE8x8(in[2][0], in[2][1], in[2][2], in[2][3], in[2][4], in[2][5], \
+ in[2][6], in[2][7], out[2][0], out[2][1], out[2][2], out[2][3], \
+ out[2][4], out[2][5], out[2][6], out[2][7]); \
+ TRANSPOSE8x8(in[3][0], in[3][1], in[3][2], in[3][3], in[3][4], in[3][5], \
+ in[3][6], in[3][7], out[3][0], out[3][1], out[3][2], out[3][3], \
+ out[3][4], out[3][5], out[3][6], out[3][7]);
+
+#define PIXEL_ADD_STORE32(in0, in1, in2, in3, step) \
+ dst = vec_vsx_ld((step)*stride, dest); \
+ d_uh = (int16x8_t)vec_mergeh(dst, zerov); \
+ d_ul = (int16x8_t)vec_mergel(dst, zerov); \
+ PIXEL_ADD(in0, d_uh, add, shift6); \
+ PIXEL_ADD(in1, d_ul, add, shift6); \
+ vec_vsx_st(vec_packsu(d_uh, d_ul), (step)*stride, dest); \
+ dst = vec_vsx_ld((step)*stride + 16, dest); \
+ d_uh = (int16x8_t)vec_mergeh(dst, zerov); \
+ d_ul = (int16x8_t)vec_mergel(dst, zerov); \
+ PIXEL_ADD(in2, d_uh, add, shift6); \
+ PIXEL_ADD(in3, d_ul, add, shift6); \
+ vec_vsx_st(vec_packsu(d_uh, d_ul), (step)*stride + 16, dest);
+
+#define ADD_STORE_BLOCK(in, offset) \
+ PIXEL_ADD_STORE32(in[0][0], in[1][0], in[2][0], in[3][0], (offset) + 0); \
+ PIXEL_ADD_STORE32(in[0][1], in[1][1], in[2][1], in[3][1], (offset) + 1); \
+ PIXEL_ADD_STORE32(in[0][2], in[1][2], in[2][2], in[3][2], (offset) + 2); \
+ PIXEL_ADD_STORE32(in[0][3], in[1][3], in[2][3], in[3][3], (offset) + 3); \
+ PIXEL_ADD_STORE32(in[0][4], in[1][4], in[2][4], in[3][4], (offset) + 4); \
+ PIXEL_ADD_STORE32(in[0][5], in[1][5], in[2][5], in[3][5], (offset) + 5); \
+ PIXEL_ADD_STORE32(in[0][6], in[1][6], in[2][6], in[3][6], (offset) + 6); \
+ PIXEL_ADD_STORE32(in[0][7], in[1][7], in[2][7], in[3][7], (offset) + 7);
+
+void vpx_idct32x32_1024_add_vsx(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ int16x8_t src0[4][8], src1[4][8], src2[4][8], src3[4][8], tmp[4][8];
+ int16x8_t tmp16_0, tmp16_1;
+ int32x4_t temp10, temp11, temp20, temp21, temp30;
+ uint8x16_t dst;
+ int16x8_t d_uh, d_ul;
+ int16x8_t add = vec_sl(vec_splat_s16(8), vec_splat_u16(2));
+ uint16x8_t shift6 = vec_splat_u16(6);
+ uint8x16_t zerov = vec_splat_u8(0);
+
+ ROUND_SHIFT_INIT;
+
+ LOAD_8x32(load_tran_low, src0[0][0], src0[1][0], src0[2][0], src0[3][0],
+ src0[0][1], src0[1][1], src0[2][1], src0[3][1], src0[0][2],
+ src0[1][2], src0[2][2], src0[3][2], src0[0][3], src0[1][3],
+ src0[2][3], src0[3][3], src0[0][4], src0[1][4], src0[2][4],
+ src0[3][4], src0[0][5], src0[1][5], src0[2][5], src0[3][5],
+ src0[0][6], src0[1][6], src0[2][6], src0[3][6], src0[0][7],
+ src0[1][7], src0[2][7], src0[3][7], 0);
+ // Rows
+ // transpose the first row of 8x8 blocks
+ TRANSPOSE_8x32(src0, tmp);
+ // transform the 32x8 column
+ IDCT32(tmp[0], tmp[1], tmp[2], tmp[3], src0);
+ TRANSPOSE_8x32(tmp, src0);
+
+ LOAD_8x32(load_tran_low, src1[0][0], src1[1][0], src1[2][0], src1[3][0],
+ src1[0][1], src1[1][1], src1[2][1], src1[3][1], src1[0][2],
+ src1[1][2], src1[2][2], src1[3][2], src1[0][3], src1[1][3],
+ src1[2][3], src1[3][3], src1[0][4], src1[1][4], src1[2][4],
+ src1[3][4], src1[0][5], src1[1][5], src1[2][5], src1[3][5],
+ src1[0][6], src1[1][6], src1[2][6], src1[3][6], src1[0][7],
+ src1[1][7], src1[2][7], src1[3][7], 512);
+ TRANSPOSE_8x32(src1, tmp);
+ IDCT32(tmp[0], tmp[1], tmp[2], tmp[3], src1);
+ TRANSPOSE_8x32(tmp, src1);
+
+ LOAD_8x32(load_tran_low, src2[0][0], src2[1][0], src2[2][0], src2[3][0],
+ src2[0][1], src2[1][1], src2[2][1], src2[3][1], src2[0][2],
+ src2[1][2], src2[2][2], src2[3][2], src2[0][3], src2[1][3],
+ src2[2][3], src2[3][3], src2[0][4], src2[1][4], src2[2][4],
+ src2[3][4], src2[0][5], src2[1][5], src2[2][5], src2[3][5],
+ src2[0][6], src2[1][6], src2[2][6], src2[3][6], src2[0][7],
+ src2[1][7], src2[2][7], src2[3][7], 1024);
+ TRANSPOSE_8x32(src2, tmp);
+ IDCT32(tmp[0], tmp[1], tmp[2], tmp[3], src2);
+ TRANSPOSE_8x32(tmp, src2);
+
+ LOAD_8x32(load_tran_low, src3[0][0], src3[1][0], src3[2][0], src3[3][0],
+ src3[0][1], src3[1][1], src3[2][1], src3[3][1], src3[0][2],
+ src3[1][2], src3[2][2], src3[3][2], src3[0][3], src3[1][3],
+ src3[2][3], src3[3][3], src3[0][4], src3[1][4], src3[2][4],
+ src3[3][4], src3[0][5], src3[1][5], src3[2][5], src3[3][5],
+ src3[0][6], src3[1][6], src3[2][6], src3[3][6], src3[0][7],
+ src3[1][7], src3[2][7], src3[3][7], 1536);
+ TRANSPOSE_8x32(src3, tmp);
+ IDCT32(tmp[0], tmp[1], tmp[2], tmp[3], src3);
+ TRANSPOSE_8x32(tmp, src3);
+
+ // Columns
+ IDCT32(src0[0], src1[0], src2[0], src3[0], tmp);
+ IDCT32(src0[1], src1[1], src2[1], src3[1], tmp);
+ IDCT32(src0[2], src1[2], src2[2], src3[2], tmp);
+ IDCT32(src0[3], src1[3], src2[3], src3[3], tmp);
+
+ ADD_STORE_BLOCK(src0, 0);
+ ADD_STORE_BLOCK(src1, 8);
+ ADD_STORE_BLOCK(src2, 16);
+ ADD_STORE_BLOCK(src3, 24);
+}
+
+#define TRANSFORM_COLS \
+ v32_a = vec_add(v32_a, v32_c); \
+ v32_d = vec_sub(v32_d, v32_b); \
+ v32_e = vec_sub(v32_a, v32_d); \
+ v32_e = vec_sra(v32_e, one); \
+ v32_b = vec_sub(v32_e, v32_b); \
+ v32_c = vec_sub(v32_e, v32_c); \
+ v32_a = vec_sub(v32_a, v32_b); \
+ v32_d = vec_add(v32_d, v32_c); \
+ v_a = vec_packs(v32_a, v32_b); \
+ v_c = vec_packs(v32_c, v32_d);
+
+#define TRANSPOSE_WHT \
+ tmp_a = vec_mergeh(v_a, v_c); \
+ tmp_c = vec_mergel(v_a, v_c); \
+ v_a = vec_mergeh(tmp_a, tmp_c); \
+ v_c = vec_mergel(tmp_a, tmp_c);
+
+void vpx_iwht4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest,
+ int stride) {
+ int16x8_t v_a = load_tran_low(0, input);
+ int16x8_t v_c = load_tran_low(8 * sizeof(*input), input);
+ int16x8_t tmp_a, tmp_c;
+ uint16x8_t two = vec_splat_u16(2);
+ uint32x4_t one = vec_splat_u32(1);
+ int16x8_t tmp16_0, tmp16_1;
+ int32x4_t v32_a, v32_c, v32_d, v32_b, v32_e;
+ uint8x16_t dest0 = vec_vsx_ld(0, dest);
+ uint8x16_t dest1 = vec_vsx_ld(stride, dest);
+ uint8x16_t dest2 = vec_vsx_ld(2 * stride, dest);
+ uint8x16_t dest3 = vec_vsx_ld(3 * stride, dest);
+ int16x8_t d_u0 = (int16x8_t)unpack_to_u16_h(dest0);
+ int16x8_t d_u1 = (int16x8_t)unpack_to_u16_h(dest1);
+ int16x8_t d_u2 = (int16x8_t)unpack_to_u16_h(dest2);
+ int16x8_t d_u3 = (int16x8_t)unpack_to_u16_h(dest3);
+ uint8x16_t output_v;
+ uint8_t tmp_dest[16];
+ int i, j;
+
+ v_a = vec_sra(v_a, two);
+ v_c = vec_sra(v_c, two);
+
+ TRANSPOSE_WHT;
+
+ v32_a = vec_unpackh(v_a);
+ v32_c = vec_unpackl(v_a);
+
+ v32_d = vec_unpackh(v_c);
+ v32_b = vec_unpackl(v_c);
+
+ TRANSFORM_COLS;
+
+ TRANSPOSE_WHT;
+
+ v32_a = vec_unpackh(v_a);
+ v32_c = vec_unpackl(v_a);
+ v32_d = vec_unpackh(v_c);
+ v32_b = vec_unpackl(v_c);
+
+ TRANSFORM_COLS;
+
+ PACK_STORE(v_a, v_c);
+}
+
+void vp9_iadst4_vsx(int16x8_t *in, int16x8_t *out) {
+ int16x8_t sinpi_1_3_v, sinpi_4_2_v, sinpi_2_3_v, sinpi_1_4_v, sinpi_12_n3_v;
+ int32x4_t v_v[5], u_v[4];
+ int32x4_t zerov = vec_splat_s32(0);
+ int16x8_t tmp0, tmp1;
+ int16x8_t zero16v = vec_splat_s16(0);
+ uint32x4_t shift16 = vec_sl(vec_splat_u32(8), vec_splat_u32(1));
+ ROUND_SHIFT_INIT;
+
+ sinpi_1_3_v = vec_mergel(sinpi_1_9_v, sinpi_3_9_v);
+ sinpi_4_2_v = vec_mergel(sinpi_4_9_v, sinpi_2_9_v);
+ sinpi_2_3_v = vec_mergel(sinpi_2_9_v, sinpi_3_9_v);
+ sinpi_1_4_v = vec_mergel(sinpi_1_9_v, sinpi_4_9_v);
+ sinpi_12_n3_v = vec_mergel(vec_add(sinpi_1_9_v, sinpi_2_9_v),
+ vec_sub(zero16v, sinpi_3_9_v));
+
+ tmp0 = (int16x8_t)vec_mergeh((int32x4_t)in[0], (int32x4_t)in[1]);
+ tmp1 = (int16x8_t)vec_mergel((int32x4_t)in[0], (int32x4_t)in[1]);
+ in[0] = (int16x8_t)vec_mergeh((int32x4_t)tmp0, (int32x4_t)tmp1);
+ in[1] = (int16x8_t)vec_mergel((int32x4_t)tmp0, (int32x4_t)tmp1);
+
+ v_v[0] = vec_msum(in[0], sinpi_1_3_v, zerov);
+ v_v[1] = vec_msum(in[1], sinpi_4_2_v, zerov);
+ v_v[2] = vec_msum(in[0], sinpi_2_3_v, zerov);
+ v_v[3] = vec_msum(in[1], sinpi_1_4_v, zerov);
+ v_v[4] = vec_msum(in[0], sinpi_12_n3_v, zerov);
+
+ in[0] = vec_sub(in[0], in[1]);
+ in[1] = (int16x8_t)vec_sra((int32x4_t)in[1], shift16);
+ in[0] = vec_add(in[0], in[1]);
+ in[0] = (int16x8_t)vec_sl((int32x4_t)in[0], shift16);
+
+ u_v[0] = vec_add(v_v[0], v_v[1]);
+ u_v[1] = vec_sub(v_v[2], v_v[3]);
+ u_v[2] = vec_msum(in[0], sinpi_1_3_v, zerov);
+ u_v[3] = vec_sub(v_v[1], v_v[3]);
+ u_v[3] = vec_add(u_v[3], v_v[4]);
+
+ DCT_CONST_ROUND_SHIFT(u_v[0]);
+ DCT_CONST_ROUND_SHIFT(u_v[1]);
+ DCT_CONST_ROUND_SHIFT(u_v[2]);
+ DCT_CONST_ROUND_SHIFT(u_v[3]);
+
+ out[0] = vec_packs(u_v[0], u_v[1]);
+ out[1] = vec_packs(u_v[2], u_v[3]);
+}
+
+#define MSUM_ROUND_SHIFT(a, b, cospi) \
+ b = vec_msums(a, cospi, zerov); \
+ DCT_CONST_ROUND_SHIFT(b);
+
+#define IADST_WRAPLOW(in0, in1, tmp0, tmp1, out, cospi) \
+ MSUM_ROUND_SHIFT(in0, tmp0, cospi); \
+ MSUM_ROUND_SHIFT(in1, tmp1, cospi); \
+ out = vec_packs(tmp0, tmp1);
+
+void vp9_iadst8_vsx(int16x8_t *in, int16x8_t *out) {
+ int32x4_t tmp0[16], tmp1[16];
+
+ int32x4_t zerov = vec_splat_s32(0);
+ int16x8_t zero16v = vec_splat_s16(0);
+ int16x8_t cospi_p02_p30_v = vec_mergel(cospi2_v, cospi30_v);
+ int16x8_t cospi_p30_m02_v = vec_mergel(cospi30_v, cospi2m_v);
+ int16x8_t cospi_p10_p22_v = vec_mergel(cospi10_v, cospi22_v);
+ int16x8_t cospi_p22_m10_v = vec_mergel(cospi22_v, cospi10m_v);
+ int16x8_t cospi_p18_p14_v = vec_mergel(cospi18_v, cospi14_v);
+ int16x8_t cospi_p14_m18_v = vec_mergel(cospi14_v, cospi18m_v);
+ int16x8_t cospi_p26_p06_v = vec_mergel(cospi26_v, cospi6_v);
+ int16x8_t cospi_p06_m26_v = vec_mergel(cospi6_v, cospi26m_v);
+ int16x8_t cospi_p08_p24_v = vec_mergel(cospi8_v, cospi24_v);
+ int16x8_t cospi_p24_m08_v = vec_mergel(cospi24_v, cospi8m_v);
+ int16x8_t cospi_m24_p08_v = vec_mergel(cospi24m_v, cospi8_v);
+ int16x8_t cospi_p16_m16_v = vec_mergel(cospi16_v, cospi16m_v);
+ ROUND_SHIFT_INIT;
+
+ TRANSPOSE8x8(in[0], in[1], in[2], in[3], in[4], in[5], in[6], in[7], out[0],
+ out[1], out[2], out[3], out[4], out[5], out[6], out[7]);
+
+ // stage 1
+ // interleave and multiply/add into 32-bit integer
+ in[0] = vec_mergeh(out[7], out[0]);
+ in[1] = vec_mergel(out[7], out[0]);
+ in[2] = vec_mergeh(out[5], out[2]);
+ in[3] = vec_mergel(out[5], out[2]);
+ in[4] = vec_mergeh(out[3], out[4]);
+ in[5] = vec_mergel(out[3], out[4]);
+ in[6] = vec_mergeh(out[1], out[6]);
+ in[7] = vec_mergel(out[1], out[6]);
+
+ tmp1[0] = vec_msum(in[0], cospi_p02_p30_v, zerov);
+ tmp1[1] = vec_msum(in[1], cospi_p02_p30_v, zerov);
+ tmp1[2] = vec_msum(in[0], cospi_p30_m02_v, zerov);
+ tmp1[3] = vec_msum(in[1], cospi_p30_m02_v, zerov);
+ tmp1[4] = vec_msum(in[2], cospi_p10_p22_v, zerov);
+ tmp1[5] = vec_msum(in[3], cospi_p10_p22_v, zerov);
+ tmp1[6] = vec_msum(in[2], cospi_p22_m10_v, zerov);
+ tmp1[7] = vec_msum(in[3], cospi_p22_m10_v, zerov);
+ tmp1[8] = vec_msum(in[4], cospi_p18_p14_v, zerov);
+ tmp1[9] = vec_msum(in[5], cospi_p18_p14_v, zerov);
+ tmp1[10] = vec_msum(in[4], cospi_p14_m18_v, zerov);
+ tmp1[11] = vec_msum(in[5], cospi_p14_m18_v, zerov);
+ tmp1[12] = vec_msum(in[6], cospi_p26_p06_v, zerov);
+ tmp1[13] = vec_msum(in[7], cospi_p26_p06_v, zerov);
+ tmp1[14] = vec_msum(in[6], cospi_p06_m26_v, zerov);
+ tmp1[15] = vec_msum(in[7], cospi_p06_m26_v, zerov);
+
+ tmp0[0] = vec_add(tmp1[0], tmp1[8]);
+ tmp0[1] = vec_add(tmp1[1], tmp1[9]);
+ tmp0[2] = vec_add(tmp1[2], tmp1[10]);
+ tmp0[3] = vec_add(tmp1[3], tmp1[11]);
+ tmp0[4] = vec_add(tmp1[4], tmp1[12]);
+ tmp0[5] = vec_add(tmp1[5], tmp1[13]);
+ tmp0[6] = vec_add(tmp1[6], tmp1[14]);
+ tmp0[7] = vec_add(tmp1[7], tmp1[15]);
+ tmp0[8] = vec_sub(tmp1[0], tmp1[8]);
+ tmp0[9] = vec_sub(tmp1[1], tmp1[9]);
+ tmp0[10] = vec_sub(tmp1[2], tmp1[10]);
+ tmp0[11] = vec_sub(tmp1[3], tmp1[11]);
+ tmp0[12] = vec_sub(tmp1[4], tmp1[12]);
+ tmp0[13] = vec_sub(tmp1[5], tmp1[13]);
+ tmp0[14] = vec_sub(tmp1[6], tmp1[14]);
+ tmp0[15] = vec_sub(tmp1[7], tmp1[15]);
+
+ // shift and rounding
+ DCT_CONST_ROUND_SHIFT(tmp0[0]);
+ DCT_CONST_ROUND_SHIFT(tmp0[1]);
+ DCT_CONST_ROUND_SHIFT(tmp0[2]);
+ DCT_CONST_ROUND_SHIFT(tmp0[3]);
+ DCT_CONST_ROUND_SHIFT(tmp0[4]);
+ DCT_CONST_ROUND_SHIFT(tmp0[5]);
+ DCT_CONST_ROUND_SHIFT(tmp0[6]);
+ DCT_CONST_ROUND_SHIFT(tmp0[7]);
+ DCT_CONST_ROUND_SHIFT(tmp0[8]);
+ DCT_CONST_ROUND_SHIFT(tmp0[9]);
+ DCT_CONST_ROUND_SHIFT(tmp0[10]);
+ DCT_CONST_ROUND_SHIFT(tmp0[11]);
+ DCT_CONST_ROUND_SHIFT(tmp0[12]);
+ DCT_CONST_ROUND_SHIFT(tmp0[13]);
+ DCT_CONST_ROUND_SHIFT(tmp0[14]);
+ DCT_CONST_ROUND_SHIFT(tmp0[15]);
+
+ // back to 16-bit
+ out[0] = vec_packs(tmp0[0], tmp0[1]);
+ out[1] = vec_packs(tmp0[2], tmp0[3]);
+ out[2] = vec_packs(tmp0[4], tmp0[5]);
+ out[3] = vec_packs(tmp0[6], tmp0[7]);
+ out[4] = vec_packs(tmp0[8], tmp0[9]);
+ out[5] = vec_packs(tmp0[10], tmp0[11]);
+ out[6] = vec_packs(tmp0[12], tmp0[13]);
+ out[7] = vec_packs(tmp0[14], tmp0[15]);
+
+ // stage 2
+ in[0] = vec_add(out[0], out[2]);
+ in[1] = vec_add(out[1], out[3]);
+ in[2] = vec_sub(out[0], out[2]);
+ in[3] = vec_sub(out[1], out[3]);
+ in[4] = vec_mergeh(out[4], out[5]);
+ in[5] = vec_mergel(out[4], out[5]);
+ in[6] = vec_mergeh(out[6], out[7]);
+ in[7] = vec_mergel(out[6], out[7]);
+
+ tmp1[0] = vec_msum(in[4], cospi_p08_p24_v, zerov);
+ tmp1[1] = vec_msum(in[5], cospi_p08_p24_v, zerov);
+ tmp1[2] = vec_msum(in[4], cospi_p24_m08_v, zerov);
+ tmp1[3] = vec_msum(in[5], cospi_p24_m08_v, zerov);
+ tmp1[4] = vec_msum(in[6], cospi_m24_p08_v, zerov);
+ tmp1[5] = vec_msum(in[7], cospi_m24_p08_v, zerov);
+ tmp1[6] = vec_msum(in[6], cospi_p08_p24_v, zerov);
+ tmp1[7] = vec_msum(in[7], cospi_p08_p24_v, zerov);
+
+ tmp0[0] = vec_add(tmp1[0], tmp1[4]);
+ tmp0[1] = vec_add(tmp1[1], tmp1[5]);
+ tmp0[2] = vec_add(tmp1[2], tmp1[6]);
+ tmp0[3] = vec_add(tmp1[3], tmp1[7]);
+ tmp0[4] = vec_sub(tmp1[0], tmp1[4]);
+ tmp0[5] = vec_sub(tmp1[1], tmp1[5]);
+ tmp0[6] = vec_sub(tmp1[2], tmp1[6]);
+ tmp0[7] = vec_sub(tmp1[3], tmp1[7]);
+
+ DCT_CONST_ROUND_SHIFT(tmp0[0]);
+ DCT_CONST_ROUND_SHIFT(tmp0[1]);
+ DCT_CONST_ROUND_SHIFT(tmp0[2]);
+ DCT_CONST_ROUND_SHIFT(tmp0[3]);
+ DCT_CONST_ROUND_SHIFT(tmp0[4]);
+ DCT_CONST_ROUND_SHIFT(tmp0[5]);
+ DCT_CONST_ROUND_SHIFT(tmp0[6]);
+ DCT_CONST_ROUND_SHIFT(tmp0[7]);
+
+ in[4] = vec_packs(tmp0[0], tmp0[1]);
+ in[5] = vec_packs(tmp0[2], tmp0[3]);
+ in[6] = vec_packs(tmp0[4], tmp0[5]);
+ in[7] = vec_packs(tmp0[6], tmp0[7]);
+
+ // stage 3
+ out[0] = vec_mergeh(in[2], in[3]);
+ out[1] = vec_mergel(in[2], in[3]);
+ out[2] = vec_mergeh(in[6], in[7]);
+ out[3] = vec_mergel(in[6], in[7]);
+
+ IADST_WRAPLOW(out[0], out[1], tmp0[0], tmp0[1], in[2], cospi16_v);
+ IADST_WRAPLOW(out[0], out[1], tmp0[0], tmp0[1], in[3], cospi_p16_m16_v);
+ IADST_WRAPLOW(out[2], out[3], tmp0[0], tmp0[1], in[6], cospi16_v);
+ IADST_WRAPLOW(out[2], out[3], tmp0[0], tmp0[1], in[7], cospi_p16_m16_v);
+
+ out[0] = in[0];
+ out[2] = in[6];
+ out[4] = in[3];
+ out[6] = in[5];
+
+ out[1] = vec_sub(zero16v, in[4]);
+ out[3] = vec_sub(zero16v, in[2]);
+ out[5] = vec_sub(zero16v, in[7]);
+ out[7] = vec_sub(zero16v, in[1]);
+}
+
+static void iadst16x8_vsx(int16x8_t *in, int16x8_t *out) {
+ int32x4_t tmp0[32], tmp1[32];
+ int16x8_t tmp16_0[8];
+ int16x8_t cospi_p01_p31 = vec_mergel(cospi1_v, cospi31_v);
+ int16x8_t cospi_p31_m01 = vec_mergel(cospi31_v, cospi1m_v);
+ int16x8_t cospi_p05_p27 = vec_mergel(cospi5_v, cospi27_v);
+ int16x8_t cospi_p27_m05 = vec_mergel(cospi27_v, cospi5m_v);
+ int16x8_t cospi_p09_p23 = vec_mergel(cospi9_v, cospi23_v);
+ int16x8_t cospi_p23_m09 = vec_mergel(cospi23_v, cospi9m_v);
+ int16x8_t cospi_p13_p19 = vec_mergel(cospi13_v, cospi19_v);
+ int16x8_t cospi_p19_m13 = vec_mergel(cospi19_v, cospi13m_v);
+ int16x8_t cospi_p17_p15 = vec_mergel(cospi17_v, cospi15_v);
+ int16x8_t cospi_p15_m17 = vec_mergel(cospi15_v, cospi17m_v);
+ int16x8_t cospi_p21_p11 = vec_mergel(cospi21_v, cospi11_v);
+ int16x8_t cospi_p11_m21 = vec_mergel(cospi11_v, cospi21m_v);
+ int16x8_t cospi_p25_p07 = vec_mergel(cospi25_v, cospi7_v);
+ int16x8_t cospi_p07_m25 = vec_mergel(cospi7_v, cospi25m_v);
+ int16x8_t cospi_p29_p03 = vec_mergel(cospi29_v, cospi3_v);
+ int16x8_t cospi_p03_m29 = vec_mergel(cospi3_v, cospi29m_v);
+ int16x8_t cospi_p04_p28 = vec_mergel(cospi4_v, cospi28_v);
+ int16x8_t cospi_p28_m04 = vec_mergel(cospi28_v, cospi4m_v);
+ int16x8_t cospi_p20_p12 = vec_mergel(cospi20_v, cospi12_v);
+ int16x8_t cospi_p12_m20 = vec_mergel(cospi12_v, cospi20m_v);
+ int16x8_t cospi_m28_p04 = vec_mergel(cospi28m_v, cospi4_v);
+ int16x8_t cospi_m12_p20 = vec_mergel(cospi12m_v, cospi20_v);
+ int16x8_t cospi_p08_p24 = vec_mergel(cospi8_v, cospi24_v);
+ int16x8_t cospi_p24_m08 = vec_mergel(cospi24_v, cospi8m_v);
+ int16x8_t cospi_m24_p08 = vec_mergel(cospi24m_v, cospi8_v);
+ int32x4_t zerov = vec_splat_s32(0);
+ ROUND_SHIFT_INIT;
+
+ tmp16_0[0] = vec_mergeh(in[15], in[0]);
+ tmp16_0[1] = vec_mergel(in[15], in[0]);
+ tmp16_0[2] = vec_mergeh(in[13], in[2]);
+ tmp16_0[3] = vec_mergel(in[13], in[2]);
+ tmp16_0[4] = vec_mergeh(in[11], in[4]);
+ tmp16_0[5] = vec_mergel(in[11], in[4]);
+ tmp16_0[6] = vec_mergeh(in[9], in[6]);
+ tmp16_0[7] = vec_mergel(in[9], in[6]);
+ tmp16_0[8] = vec_mergeh(in[7], in[8]);
+ tmp16_0[9] = vec_mergel(in[7], in[8]);
+ tmp16_0[10] = vec_mergeh(in[5], in[10]);
+ tmp16_0[11] = vec_mergel(in[5], in[10]);
+ tmp16_0[12] = vec_mergeh(in[3], in[12]);
+ tmp16_0[13] = vec_mergel(in[3], in[12]);
+ tmp16_0[14] = vec_mergeh(in[1], in[14]);
+ tmp16_0[15] = vec_mergel(in[1], in[14]);
+
+ tmp0[0] = vec_msum(tmp16_0[0], cospi_p01_p31, zerov);
+ tmp0[1] = vec_msum(tmp16_0[1], cospi_p01_p31, zerov);
+ tmp0[2] = vec_msum(tmp16_0[0], cospi_p31_m01, zerov);
+ tmp0[3] = vec_msum(tmp16_0[1], cospi_p31_m01, zerov);
+ tmp0[4] = vec_msum(tmp16_0[2], cospi_p05_p27, zerov);
+ tmp0[5] = vec_msum(tmp16_0[3], cospi_p05_p27, zerov);
+ tmp0[6] = vec_msum(tmp16_0[2], cospi_p27_m05, zerov);
+ tmp0[7] = vec_msum(tmp16_0[3], cospi_p27_m05, zerov);
+ tmp0[8] = vec_msum(tmp16_0[4], cospi_p09_p23, zerov);
+ tmp0[9] = vec_msum(tmp16_0[5], cospi_p09_p23, zerov);
+ tmp0[10] = vec_msum(tmp16_0[4], cospi_p23_m09, zerov);
+ tmp0[11] = vec_msum(tmp16_0[5], cospi_p23_m09, zerov);
+ tmp0[12] = vec_msum(tmp16_0[6], cospi_p13_p19, zerov);
+ tmp0[13] = vec_msum(tmp16_0[7], cospi_p13_p19, zerov);
+ tmp0[14] = vec_msum(tmp16_0[6], cospi_p19_m13, zerov);
+ tmp0[15] = vec_msum(tmp16_0[7], cospi_p19_m13, zerov);
+ tmp0[16] = vec_msum(tmp16_0[8], cospi_p17_p15, zerov);
+ tmp0[17] = vec_msum(tmp16_0[9], cospi_p17_p15, zerov);
+ tmp0[18] = vec_msum(tmp16_0[8], cospi_p15_m17, zerov);
+ tmp0[19] = vec_msum(tmp16_0[9], cospi_p15_m17, zerov);
+ tmp0[20] = vec_msum(tmp16_0[10], cospi_p21_p11, zerov);
+ tmp0[21] = vec_msum(tmp16_0[11], cospi_p21_p11, zerov);
+ tmp0[22] = vec_msum(tmp16_0[10], cospi_p11_m21, zerov);
+ tmp0[23] = vec_msum(tmp16_0[11], cospi_p11_m21, zerov);
+ tmp0[24] = vec_msum(tmp16_0[12], cospi_p25_p07, zerov);
+ tmp0[25] = vec_msum(tmp16_0[13], cospi_p25_p07, zerov);
+ tmp0[26] = vec_msum(tmp16_0[12], cospi_p07_m25, zerov);
+ tmp0[27] = vec_msum(tmp16_0[13], cospi_p07_m25, zerov);
+ tmp0[28] = vec_msum(tmp16_0[14], cospi_p29_p03, zerov);
+ tmp0[29] = vec_msum(tmp16_0[15], cospi_p29_p03, zerov);
+ tmp0[30] = vec_msum(tmp16_0[14], cospi_p03_m29, zerov);
+ tmp0[31] = vec_msum(tmp16_0[15], cospi_p03_m29, zerov);
+
+ tmp1[0] = vec_add(tmp0[0], tmp0[16]);
+ tmp1[1] = vec_add(tmp0[1], tmp0[17]);
+ tmp1[2] = vec_add(tmp0[2], tmp0[18]);
+ tmp1[3] = vec_add(tmp0[3], tmp0[19]);
+ tmp1[4] = vec_add(tmp0[4], tmp0[20]);
+ tmp1[5] = vec_add(tmp0[5], tmp0[21]);
+ tmp1[6] = vec_add(tmp0[6], tmp0[22]);
+ tmp1[7] = vec_add(tmp0[7], tmp0[23]);
+ tmp1[8] = vec_add(tmp0[8], tmp0[24]);
+ tmp1[9] = vec_add(tmp0[9], tmp0[25]);
+ tmp1[10] = vec_add(tmp0[10], tmp0[26]);
+ tmp1[11] = vec_add(tmp0[11], tmp0[27]);
+ tmp1[12] = vec_add(tmp0[12], tmp0[28]);
+ tmp1[13] = vec_add(tmp0[13], tmp0[29]);
+ tmp1[14] = vec_add(tmp0[14], tmp0[30]);
+ tmp1[15] = vec_add(tmp0[15], tmp0[31]);
+ tmp1[16] = vec_sub(tmp0[0], tmp0[16]);
+ tmp1[17] = vec_sub(tmp0[1], tmp0[17]);
+ tmp1[18] = vec_sub(tmp0[2], tmp0[18]);
+ tmp1[19] = vec_sub(tmp0[3], tmp0[19]);
+ tmp1[20] = vec_sub(tmp0[4], tmp0[20]);
+ tmp1[21] = vec_sub(tmp0[5], tmp0[21]);
+ tmp1[22] = vec_sub(tmp0[6], tmp0[22]);
+ tmp1[23] = vec_sub(tmp0[7], tmp0[23]);
+ tmp1[24] = vec_sub(tmp0[8], tmp0[24]);
+ tmp1[25] = vec_sub(tmp0[9], tmp0[25]);
+ tmp1[26] = vec_sub(tmp0[10], tmp0[26]);
+ tmp1[27] = vec_sub(tmp0[11], tmp0[27]);
+ tmp1[28] = vec_sub(tmp0[12], tmp0[28]);
+ tmp1[29] = vec_sub(tmp0[13], tmp0[29]);
+ tmp1[30] = vec_sub(tmp0[14], tmp0[30]);
+ tmp1[31] = vec_sub(tmp0[15], tmp0[31]);
+
+ DCT_CONST_ROUND_SHIFT(tmp1[0]);
+ DCT_CONST_ROUND_SHIFT(tmp1[1]);
+ DCT_CONST_ROUND_SHIFT(tmp1[2]);
+ DCT_CONST_ROUND_SHIFT(tmp1[3]);
+ DCT_CONST_ROUND_SHIFT(tmp1[4]);
+ DCT_CONST_ROUND_SHIFT(tmp1[5]);
+ DCT_CONST_ROUND_SHIFT(tmp1[6]);
+ DCT_CONST_ROUND_SHIFT(tmp1[7]);
+ DCT_CONST_ROUND_SHIFT(tmp1[8]);
+ DCT_CONST_ROUND_SHIFT(tmp1[9]);
+ DCT_CONST_ROUND_SHIFT(tmp1[10]);
+ DCT_CONST_ROUND_SHIFT(tmp1[11]);
+ DCT_CONST_ROUND_SHIFT(tmp1[12]);
+ DCT_CONST_ROUND_SHIFT(tmp1[13]);
+ DCT_CONST_ROUND_SHIFT(tmp1[14]);
+ DCT_CONST_ROUND_SHIFT(tmp1[15]);
+ DCT_CONST_ROUND_SHIFT(tmp1[16]);
+ DCT_CONST_ROUND_SHIFT(tmp1[17]);
+ DCT_CONST_ROUND_SHIFT(tmp1[18]);
+ DCT_CONST_ROUND_SHIFT(tmp1[19]);
+ DCT_CONST_ROUND_SHIFT(tmp1[20]);
+ DCT_CONST_ROUND_SHIFT(tmp1[21]);
+ DCT_CONST_ROUND_SHIFT(tmp1[22]);
+ DCT_CONST_ROUND_SHIFT(tmp1[23]);
+ DCT_CONST_ROUND_SHIFT(tmp1[24]);
+ DCT_CONST_ROUND_SHIFT(tmp1[25]);
+ DCT_CONST_ROUND_SHIFT(tmp1[26]);
+ DCT_CONST_ROUND_SHIFT(tmp1[27]);
+ DCT_CONST_ROUND_SHIFT(tmp1[28]);
+ DCT_CONST_ROUND_SHIFT(tmp1[29]);
+ DCT_CONST_ROUND_SHIFT(tmp1[30]);
+ DCT_CONST_ROUND_SHIFT(tmp1[31]);
+
+ in[0] = vec_packs(tmp1[0], tmp1[1]);
+ in[1] = vec_packs(tmp1[2], tmp1[3]);
+ in[2] = vec_packs(tmp1[4], tmp1[5]);
+ in[3] = vec_packs(tmp1[6], tmp1[7]);
+ in[4] = vec_packs(tmp1[8], tmp1[9]);
+ in[5] = vec_packs(tmp1[10], tmp1[11]);
+ in[6] = vec_packs(tmp1[12], tmp1[13]);
+ in[7] = vec_packs(tmp1[14], tmp1[15]);
+ in[8] = vec_packs(tmp1[16], tmp1[17]);
+ in[9] = vec_packs(tmp1[18], tmp1[19]);
+ in[10] = vec_packs(tmp1[20], tmp1[21]);
+ in[11] = vec_packs(tmp1[22], tmp1[23]);
+ in[12] = vec_packs(tmp1[24], tmp1[25]);
+ in[13] = vec_packs(tmp1[26], tmp1[27]);
+ in[14] = vec_packs(tmp1[28], tmp1[29]);
+ in[15] = vec_packs(tmp1[30], tmp1[31]);
+
+ // stage 2
+ tmp16_0[0] = vec_mergeh(in[8], in[9]);
+ tmp16_0[1] = vec_mergel(in[8], in[9]);
+ tmp16_0[2] = vec_mergeh(in[10], in[11]);
+ tmp16_0[3] = vec_mergel(in[10], in[11]);
+ tmp16_0[4] = vec_mergeh(in[12], in[13]);
+ tmp16_0[5] = vec_mergel(in[12], in[13]);
+ tmp16_0[6] = vec_mergeh(in[14], in[15]);
+ tmp16_0[7] = vec_mergel(in[14], in[15]);
+
+ tmp0[0] = vec_msum(tmp16_0[0], cospi_p04_p28, zerov);
+ tmp0[1] = vec_msum(tmp16_0[1], cospi_p04_p28, zerov);
+ tmp0[2] = vec_msum(tmp16_0[0], cospi_p28_m04, zerov);
+ tmp0[3] = vec_msum(tmp16_0[1], cospi_p28_m04, zerov);
+ tmp0[4] = vec_msum(tmp16_0[2], cospi_p20_p12, zerov);
+ tmp0[5] = vec_msum(tmp16_0[3], cospi_p20_p12, zerov);
+ tmp0[6] = vec_msum(tmp16_0[2], cospi_p12_m20, zerov);
+ tmp0[7] = vec_msum(tmp16_0[3], cospi_p12_m20, zerov);
+ tmp0[8] = vec_msum(tmp16_0[4], cospi_m28_p04, zerov);
+ tmp0[9] = vec_msum(tmp16_0[5], cospi_m28_p04, zerov);
+ tmp0[10] = vec_msum(tmp16_0[4], cospi_p04_p28, zerov);
+ tmp0[11] = vec_msum(tmp16_0[5], cospi_p04_p28, zerov);
+ tmp0[12] = vec_msum(tmp16_0[6], cospi_m12_p20, zerov);
+ tmp0[13] = vec_msum(tmp16_0[7], cospi_m12_p20, zerov);
+ tmp0[14] = vec_msum(tmp16_0[6], cospi_p20_p12, zerov);
+ tmp0[15] = vec_msum(tmp16_0[7], cospi_p20_p12, zerov);
+
+ tmp1[0] = vec_add(tmp0[0], tmp0[8]);
+ tmp1[1] = vec_add(tmp0[1], tmp0[9]);
+ tmp1[2] = vec_add(tmp0[2], tmp0[10]);
+ tmp1[3] = vec_add(tmp0[3], tmp0[11]);
+ tmp1[4] = vec_add(tmp0[4], tmp0[12]);
+ tmp1[5] = vec_add(tmp0[5], tmp0[13]);
+ tmp1[6] = vec_add(tmp0[6], tmp0[14]);
+ tmp1[7] = vec_add(tmp0[7], tmp0[15]);
+ tmp1[8] = vec_sub(tmp0[0], tmp0[8]);
+ tmp1[9] = vec_sub(tmp0[1], tmp0[9]);
+ tmp1[10] = vec_sub(tmp0[2], tmp0[10]);
+ tmp1[11] = vec_sub(tmp0[3], tmp0[11]);
+ tmp1[12] = vec_sub(tmp0[4], tmp0[12]);
+ tmp1[13] = vec_sub(tmp0[5], tmp0[13]);
+ tmp1[14] = vec_sub(tmp0[6], tmp0[14]);
+ tmp1[15] = vec_sub(tmp0[7], tmp0[15]);
+
+ DCT_CONST_ROUND_SHIFT(tmp1[0]);
+ DCT_CONST_ROUND_SHIFT(tmp1[1]);
+ DCT_CONST_ROUND_SHIFT(tmp1[2]);
+ DCT_CONST_ROUND_SHIFT(tmp1[3]);
+ DCT_CONST_ROUND_SHIFT(tmp1[4]);
+ DCT_CONST_ROUND_SHIFT(tmp1[5]);
+ DCT_CONST_ROUND_SHIFT(tmp1[6]);
+ DCT_CONST_ROUND_SHIFT(tmp1[7]);
+ DCT_CONST_ROUND_SHIFT(tmp1[8]);
+ DCT_CONST_ROUND_SHIFT(tmp1[9]);
+ DCT_CONST_ROUND_SHIFT(tmp1[10]);
+ DCT_CONST_ROUND_SHIFT(tmp1[11]);
+ DCT_CONST_ROUND_SHIFT(tmp1[12]);
+ DCT_CONST_ROUND_SHIFT(tmp1[13]);
+ DCT_CONST_ROUND_SHIFT(tmp1[14]);
+ DCT_CONST_ROUND_SHIFT(tmp1[15]);
+
+ tmp16_0[0] = vec_add(in[0], in[4]);
+ tmp16_0[1] = vec_add(in[1], in[5]);
+ tmp16_0[2] = vec_add(in[2], in[6]);
+ tmp16_0[3] = vec_add(in[3], in[7]);
+ tmp16_0[4] = vec_sub(in[0], in[4]);
+ tmp16_0[5] = vec_sub(in[1], in[5]);
+ tmp16_0[6] = vec_sub(in[2], in[6]);
+ tmp16_0[7] = vec_sub(in[3], in[7]);
+ tmp16_0[8] = vec_packs(tmp1[0], tmp1[1]);
+ tmp16_0[9] = vec_packs(tmp1[2], tmp1[3]);
+ tmp16_0[10] = vec_packs(tmp1[4], tmp1[5]);
+ tmp16_0[11] = vec_packs(tmp1[6], tmp1[7]);
+ tmp16_0[12] = vec_packs(tmp1[8], tmp1[9]);
+ tmp16_0[13] = vec_packs(tmp1[10], tmp1[11]);
+ tmp16_0[14] = vec_packs(tmp1[12], tmp1[13]);
+ tmp16_0[15] = vec_packs(tmp1[14], tmp1[15]);
+
+ // stage 3
+ in[0] = vec_mergeh(tmp16_0[4], tmp16_0[5]);
+ in[1] = vec_mergel(tmp16_0[4], tmp16_0[5]);
+ in[2] = vec_mergeh(tmp16_0[6], tmp16_0[7]);
+ in[3] = vec_mergel(tmp16_0[6], tmp16_0[7]);
+ in[4] = vec_mergeh(tmp16_0[12], tmp16_0[13]);
+ in[5] = vec_mergel(tmp16_0[12], tmp16_0[13]);
+ in[6] = vec_mergeh(tmp16_0[14], tmp16_0[15]);
+ in[7] = vec_mergel(tmp16_0[14], tmp16_0[15]);
+
+ tmp0[0] = vec_msum(in[0], cospi_p08_p24, zerov);
+ tmp0[1] = vec_msum(in[1], cospi_p08_p24, zerov);
+ tmp0[2] = vec_msum(in[0], cospi_p24_m08, zerov);
+ tmp0[3] = vec_msum(in[1], cospi_p24_m08, zerov);
+ tmp0[4] = vec_msum(in[2], cospi_m24_p08, zerov);
+ tmp0[5] = vec_msum(in[3], cospi_m24_p08, zerov);
+ tmp0[6] = vec_msum(in[2], cospi_p08_p24, zerov);
+ tmp0[7] = vec_msum(in[3], cospi_p08_p24, zerov);
+ tmp0[8] = vec_msum(in[4], cospi_p08_p24, zerov);
+ tmp0[9] = vec_msum(in[5], cospi_p08_p24, zerov);
+ tmp0[10] = vec_msum(in[4], cospi_p24_m08, zerov);
+ tmp0[11] = vec_msum(in[5], cospi_p24_m08, zerov);
+ tmp0[12] = vec_msum(in[6], cospi_m24_p08, zerov);
+ tmp0[13] = vec_msum(in[7], cospi_m24_p08, zerov);
+ tmp0[14] = vec_msum(in[6], cospi_p08_p24, zerov);
+ tmp0[15] = vec_msum(in[7], cospi_p08_p24, zerov);
+
+ tmp1[0] = vec_add(tmp0[0], tmp0[4]);
+ tmp1[1] = vec_add(tmp0[1], tmp0[5]);
+ tmp1[2] = vec_add(tmp0[2], tmp0[6]);
+ tmp1[3] = vec_add(tmp0[3], tmp0[7]);
+ tmp1[4] = vec_sub(tmp0[0], tmp0[4]);
+ tmp1[5] = vec_sub(tmp0[1], tmp0[5]);
+ tmp1[6] = vec_sub(tmp0[2], tmp0[6]);
+ tmp1[7] = vec_sub(tmp0[3], tmp0[7]);
+ tmp1[8] = vec_add(tmp0[8], tmp0[12]);
+ tmp1[9] = vec_add(tmp0[9], tmp0[13]);
+ tmp1[10] = vec_add(tmp0[10], tmp0[14]);
+ tmp1[11] = vec_add(tmp0[11], tmp0[15]);
+ tmp1[12] = vec_sub(tmp0[8], tmp0[12]);
+ tmp1[13] = vec_sub(tmp0[9], tmp0[13]);
+ tmp1[14] = vec_sub(tmp0[10], tmp0[14]);
+ tmp1[15] = vec_sub(tmp0[11], tmp0[15]);
+
+ DCT_CONST_ROUND_SHIFT(tmp1[0]);
+ DCT_CONST_ROUND_SHIFT(tmp1[1]);
+ DCT_CONST_ROUND_SHIFT(tmp1[2]);
+ DCT_CONST_ROUND_SHIFT(tmp1[3]);
+ DCT_CONST_ROUND_SHIFT(tmp1[4]);
+ DCT_CONST_ROUND_SHIFT(tmp1[5]);
+ DCT_CONST_ROUND_SHIFT(tmp1[6]);
+ DCT_CONST_ROUND_SHIFT(tmp1[7]);
+ DCT_CONST_ROUND_SHIFT(tmp1[8]);
+ DCT_CONST_ROUND_SHIFT(tmp1[9]);
+ DCT_CONST_ROUND_SHIFT(tmp1[10]);
+ DCT_CONST_ROUND_SHIFT(tmp1[11]);
+ DCT_CONST_ROUND_SHIFT(tmp1[12]);
+ DCT_CONST_ROUND_SHIFT(tmp1[13]);
+ DCT_CONST_ROUND_SHIFT(tmp1[14]);
+ DCT_CONST_ROUND_SHIFT(tmp1[15]);
+
+ in[0] = vec_add(tmp16_0[0], tmp16_0[2]);
+ in[1] = vec_add(tmp16_0[1], tmp16_0[3]);
+ in[2] = vec_sub(tmp16_0[0], tmp16_0[2]);
+ in[3] = vec_sub(tmp16_0[1], tmp16_0[3]);
+ in[4] = vec_packs(tmp1[0], tmp1[1]);
+ in[5] = vec_packs(tmp1[2], tmp1[3]);
+ in[6] = vec_packs(tmp1[4], tmp1[5]);
+ in[7] = vec_packs(tmp1[6], tmp1[7]);
+ in[8] = vec_add(tmp16_0[8], tmp16_0[10]);
+ in[9] = vec_add(tmp16_0[9], tmp16_0[11]);
+ in[10] = vec_sub(tmp16_0[8], tmp16_0[10]);
+ in[11] = vec_sub(tmp16_0[9], tmp16_0[11]);
+ in[12] = vec_packs(tmp1[8], tmp1[9]);
+ in[13] = vec_packs(tmp1[10], tmp1[11]);
+ in[14] = vec_packs(tmp1[12], tmp1[13]);
+ in[15] = vec_packs(tmp1[14], tmp1[15]);
+
+ // stage 4
+ out[0] = vec_mergeh(in[2], in[3]);
+ out[1] = vec_mergel(in[2], in[3]);
+ out[2] = vec_mergeh(in[6], in[7]);
+ out[3] = vec_mergel(in[6], in[7]);
+ out[4] = vec_mergeh(in[10], in[11]);
+ out[5] = vec_mergel(in[10], in[11]);
+ out[6] = vec_mergeh(in[14], in[15]);
+ out[7] = vec_mergel(in[14], in[15]);
+}
+
+void vpx_iadst16_vsx(int16x8_t *src0, int16x8_t *src1) {
+ int16x8_t tmp0[16], tmp1[16], tmp2[8];
+ int32x4_t tmp3, tmp4;
+ int16x8_t zero16v = vec_splat_s16(0);
+ int32x4_t zerov = vec_splat_s32(0);
+ int16x8_t cospi_p16_m16 = vec_mergel(cospi16_v, cospi16m_v);
+ int16x8_t cospi_m16_p16 = vec_mergel(cospi16m_v, cospi16_v);
+ ROUND_SHIFT_INIT;
+
+ TRANSPOSE8x8(src0[0], src0[2], src0[4], src0[6], src0[8], src0[10], src0[12],
+ src0[14], tmp0[0], tmp0[1], tmp0[2], tmp0[3], tmp0[4], tmp0[5],
+ tmp0[6], tmp0[7]);
+ TRANSPOSE8x8(src1[0], src1[2], src1[4], src1[6], src1[8], src1[10], src1[12],
+ src1[14], tmp1[0], tmp1[1], tmp1[2], tmp1[3], tmp1[4], tmp1[5],
+ tmp1[6], tmp1[7]);
+ TRANSPOSE8x8(src0[1], src0[3], src0[5], src0[7], src0[9], src0[11], src0[13],
+ src0[15], tmp0[8], tmp0[9], tmp0[10], tmp0[11], tmp0[12],
+ tmp0[13], tmp0[14], tmp0[15]);
+ TRANSPOSE8x8(src1[1], src1[3], src1[5], src1[7], src1[9], src1[11], src1[13],
+ src1[15], tmp1[8], tmp1[9], tmp1[10], tmp1[11], tmp1[12],
+ tmp1[13], tmp1[14], tmp1[15]);
+
+ iadst16x8_vsx(tmp0, tmp2);
+ IADST_WRAPLOW(tmp2[0], tmp2[1], tmp3, tmp4, src0[14], cospi16m_v);
+ IADST_WRAPLOW(tmp2[0], tmp2[1], tmp3, tmp4, src1[0], cospi_p16_m16);
+ IADST_WRAPLOW(tmp2[2], tmp2[3], tmp3, tmp4, src0[8], cospi16_v);
+ IADST_WRAPLOW(tmp2[2], tmp2[3], tmp3, tmp4, src1[6], cospi_m16_p16);
+ IADST_WRAPLOW(tmp2[4], tmp2[5], tmp3, tmp4, src0[12], cospi16_v);
+ IADST_WRAPLOW(tmp2[4], tmp2[5], tmp3, tmp4, src1[2], cospi_m16_p16);
+ IADST_WRAPLOW(tmp2[6], tmp2[7], tmp3, tmp4, src0[10], cospi16m_v);
+ IADST_WRAPLOW(tmp2[6], tmp2[7], tmp3, tmp4, src1[4], cospi_p16_m16);
+
+ src0[0] = tmp0[0];
+ src0[2] = vec_sub(zero16v, tmp0[8]);
+ src0[4] = tmp0[12];
+ src0[6] = vec_sub(zero16v, tmp0[4]);
+ src1[8] = tmp0[5];
+ src1[10] = vec_sub(zero16v, tmp0[13]);
+ src1[12] = tmp0[9];
+ src1[14] = vec_sub(zero16v, tmp0[1]);
+
+ iadst16x8_vsx(tmp1, tmp2);
+ IADST_WRAPLOW(tmp2[0], tmp2[1], tmp3, tmp4, src0[15], cospi16m_v);
+ IADST_WRAPLOW(tmp2[0], tmp2[1], tmp3, tmp4, src1[1], cospi_p16_m16);
+ IADST_WRAPLOW(tmp2[2], tmp2[3], tmp3, tmp4, src0[9], cospi16_v);
+ IADST_WRAPLOW(tmp2[2], tmp2[3], tmp3, tmp4, src1[7], cospi_m16_p16);
+ IADST_WRAPLOW(tmp2[4], tmp2[5], tmp3, tmp4, src0[13], cospi16_v);
+ IADST_WRAPLOW(tmp2[4], tmp2[5], tmp3, tmp4, src1[3], cospi_m16_p16);
+ IADST_WRAPLOW(tmp2[6], tmp2[7], tmp3, tmp4, src0[11], cospi16m_v);
+ IADST_WRAPLOW(tmp2[6], tmp2[7], tmp3, tmp4, src1[5], cospi_p16_m16);
+
+ src0[1] = tmp1[0];
+ src0[3] = vec_sub(zero16v, tmp1[8]);
+ src0[5] = tmp1[12];
+ src0[7] = vec_sub(zero16v, tmp1[4]);
+ src1[9] = tmp1[5];
+ src1[11] = vec_sub(zero16v, tmp1[13]);
+ src1[13] = tmp1[9];
+ src1[15] = vec_sub(zero16v, tmp1[1]);
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.h b/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.h
new file mode 100644
index 0000000000..7031742c1c
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/inv_txfm_vsx.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
+#define VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
+
+#include "vpx_dsp/ppc/types_vsx.h"
+
+void vpx_round_store4x4_vsx(int16x8_t *in, int16x8_t *out, uint8_t *dest,
+ int stride);
+void vpx_idct4_vsx(int16x8_t *in, int16x8_t *out);
+void vp9_iadst4_vsx(int16x8_t *in, int16x8_t *out);
+
+void vpx_round_store8x8_vsx(int16x8_t *in, uint8_t *dest, int stride);
+void vpx_idct8_vsx(int16x8_t *in, int16x8_t *out);
+void vp9_iadst8_vsx(int16x8_t *in, int16x8_t *out);
+
+#define LOAD_INPUT16(load, source, offset, step, in) \
+ in[0] = load(offset, source); \
+ in[1] = load((step) + (offset), source); \
+ in[2] = load(2 * (step) + (offset), source); \
+ in[3] = load(3 * (step) + (offset), source); \
+ in[4] = load(4 * (step) + (offset), source); \
+ in[5] = load(5 * (step) + (offset), source); \
+ in[6] = load(6 * (step) + (offset), source); \
+ in[7] = load(7 * (step) + (offset), source); \
+ in[8] = load(8 * (step) + (offset), source); \
+ in[9] = load(9 * (step) + (offset), source); \
+ in[10] = load(10 * (step) + (offset), source); \
+ in[11] = load(11 * (step) + (offset), source); \
+ in[12] = load(12 * (step) + (offset), source); \
+ in[13] = load(13 * (step) + (offset), source); \
+ in[14] = load(14 * (step) + (offset), source); \
+ in[15] = load(15 * (step) + (offset), source);
+
+void vpx_round_store16x16_vsx(int16x8_t *src0, int16x8_t *src1, uint8_t *dest,
+ int stride);
+void vpx_idct16_vsx(int16x8_t *src0, int16x8_t *src1);
+void vpx_iadst16_vsx(int16x8_t *src0, int16x8_t *src1);
+
+#endif // VPX_VPX_DSP_PPC_INV_TXFM_VSX_H_
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c
new file mode 100644
index 0000000000..ab71f6e235
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c
@@ -0,0 +1,301 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+// Negate 16-bit integers in a when the corresponding signed 16-bit
+// integer in b is negative.
+static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) {
+ const int16x8_t mask = vec_sra(b, vec_shift_sign_s16);
+ return vec_xor(vec_add(a, mask), mask);
+}
+
+// Sets the value of a 32-bit integers to 1 when the corresponding value in a is
+// negative.
+static INLINE int32x4_t vec_is_neg(int32x4_t a) {
+ return vec_sr(a, vec_shift_sign_s32);
+}
+
+// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit
+// integers, and return the high 16 bits of the intermediate integers.
+// (a * b) >> 16
+static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) {
+ // madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right
+ // shift.
+ return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16);
+}
+
+// Quantization function used for 4x4, 8x8 and 16x16 blocks.
+static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs,
+ int16x8_t round, int16x8_t quant,
+ int16x8_t quant_shift, bool16x8_t mask) {
+ const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
+ int16x8_t qcoeff = vec_mulhi(rounded, quant);
+ qcoeff = vec_add(qcoeff, rounded);
+ qcoeff = vec_mulhi(qcoeff, quant_shift);
+ qcoeff = vec_sign(qcoeff, coeff);
+ return vec_and(qcoeff, mask);
+}
+
+// Quantization function used for 32x32 blocks.
+static INLINE int16x8_t quantize_coeff_32(int16x8_t coeff, int16x8_t coeff_abs,
+ int16x8_t round, int16x8_t quant,
+ int16x8_t quant_shift,
+ bool16x8_t mask) {
+ const int16x8_t rounded = vec_vaddshs(coeff_abs, round);
+ int16x8_t qcoeff = vec_mulhi(rounded, quant);
+ qcoeff = vec_add(qcoeff, rounded);
+ // 32x32 blocks require an extra multiplication by 2, this compensates for the
+ // extra right shift added in vec_mulhi, as such vec_madds can be used
+ // directly instead of vec_mulhi (((a * b) >> 15) >> 1) << 1 == (a * b >> 15)
+ qcoeff = vec_madds(qcoeff, quant_shift, vec_zeros_s16);
+ qcoeff = vec_sign(qcoeff, coeff);
+ return vec_and(qcoeff, mask);
+}
+
+// DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32
+// blocks are twice as big as for other block sizes. As such, using
+// vec_mladd results in overflow.
+static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff,
+ int16x8_t dequant) {
+ int32x4_t dqcoeffe = vec_mule(qcoeff, dequant);
+ int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant);
+ // Add 1 if negative to round towards zero because the C uses division.
+ dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe));
+ dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo));
+ dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32);
+ dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32);
+ return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack);
+}
+
+static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff,
+ const int16_t *iscan_ptr, int index) {
+ int16x8_t scan = vec_vsx_ld(index, iscan_ptr);
+ bool16x8_t zero_coeff = vec_cmpeq(qcoeff, vec_zeros_s16);
+ return vec_andc(scan, zero_coeff);
+}
+
+// Compare packed 16-bit integers across a, and return the maximum value in
+// every element. Returns a vector containing the biggest value across vector a.
+static INLINE int16x8_t vec_max_across(int16x8_t a) {
+ a = vec_max(a, vec_perm(a, a, vec_perm64));
+ a = vec_max(a, vec_perm(a, a, vec_perm32));
+ return vec_max(a, vec_perm(a, a, vec_perm16));
+}
+
+void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *zbin_ptr, const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
+ uint16_t *eob_ptr, const int16_t *scan_ptr,
+ const int16_t *iscan_ptr) {
+ int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob;
+ bool16x8_t zero_mask0, zero_mask1;
+
+ // First set of 8 coeff starts with DC + 7 AC
+ int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
+ int16x8_t round = vec_vsx_ld(0, round_ptr);
+ int16x8_t quant = vec_vsx_ld(0, quant_ptr);
+ int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
+ int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
+
+ int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
+ int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
+
+ int16x8_t coeff0_abs = vec_abs(coeff0);
+ int16x8_t coeff1_abs = vec_abs(coeff1);
+
+ zero_mask0 = vec_cmpge(coeff0_abs, zbin);
+ zbin = vec_splat(zbin, 1);
+ zero_mask1 = vec_cmpge(coeff1_abs, zbin);
+
+ (void)scan_ptr;
+
+ qcoeff0 =
+ quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0);
+ vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
+ round = vec_splat(round, 1);
+ quant = vec_splat(quant, 1);
+ quant_shift = vec_splat(quant_shift, 1);
+ qcoeff1 =
+ quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift, zero_mask1);
+ vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
+
+ dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
+ vec_vsx_st(dqcoeff0, 0, dqcoeff_ptr);
+ dequant = vec_splat(dequant, 1);
+ dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
+ vec_vsx_st(dqcoeff1, 16, dqcoeff_ptr);
+
+ eob = vec_max(nonzero_scanindex(qcoeff0, iscan_ptr, 0),
+ nonzero_scanindex(qcoeff1, iscan_ptr, 16));
+
+ if (n_coeffs > 16) {
+ int index = 16;
+ int off0 = 32;
+ int off1 = 48;
+ int off2 = 64;
+ do {
+ int16x8_t coeff2, coeff2_abs, qcoeff2, dqcoeff2, eob2;
+ bool16x8_t zero_mask2;
+ coeff0 = vec_vsx_ld(off0, coeff_ptr);
+ coeff1 = vec_vsx_ld(off1, coeff_ptr);
+ coeff2 = vec_vsx_ld(off2, coeff_ptr);
+ coeff0_abs = vec_abs(coeff0);
+ coeff1_abs = vec_abs(coeff1);
+ coeff2_abs = vec_abs(coeff2);
+ zero_mask0 = vec_cmpge(coeff0_abs, zbin);
+ zero_mask1 = vec_cmpge(coeff1_abs, zbin);
+ zero_mask2 = vec_cmpge(coeff2_abs, zbin);
+ qcoeff0 = quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift,
+ zero_mask0);
+ qcoeff1 = quantize_coeff(coeff1, coeff1_abs, round, quant, quant_shift,
+ zero_mask1);
+ qcoeff2 = quantize_coeff(coeff2, coeff2_abs, round, quant, quant_shift,
+ zero_mask2);
+ vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
+ vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
+ vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
+
+ dqcoeff0 = vec_mladd(qcoeff0, dequant, vec_zeros_s16);
+ dqcoeff1 = vec_mladd(qcoeff1, dequant, vec_zeros_s16);
+ dqcoeff2 = vec_mladd(qcoeff2, dequant, vec_zeros_s16);
+
+ vec_vsx_st(dqcoeff0, off0, dqcoeff_ptr);
+ vec_vsx_st(dqcoeff1, off1, dqcoeff_ptr);
+ vec_vsx_st(dqcoeff2, off2, dqcoeff_ptr);
+
+ eob = vec_max(eob, nonzero_scanindex(qcoeff0, iscan_ptr, off0));
+ eob2 = vec_max(nonzero_scanindex(qcoeff1, iscan_ptr, off1),
+ nonzero_scanindex(qcoeff2, iscan_ptr, off2));
+ eob = vec_max(eob, eob2);
+
+ index += 24;
+ off0 += 48;
+ off1 += 48;
+ off2 += 48;
+ } while (index < n_coeffs);
+ }
+
+ eob = vec_max_across(eob);
+ *eob_ptr = eob[0];
+}
+
+void vpx_quantize_b_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *zbin_ptr, const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan_ptr,
+ const int16_t *iscan_ptr) {
+ // In stage 1, we quantize 16 coeffs (DC + 15 AC)
+ // In stage 2, we loop 42 times and quantize 24 coeffs per iteration
+ // (32 * 32 - 16) / 24 = 42
+ int num_itr = 42;
+ // Offsets are in bytes, 16 coeffs = 32 bytes
+ int off0 = 32;
+ int off1 = 48;
+ int off2 = 64;
+
+ int16x8_t qcoeff0, qcoeff1, eob;
+ bool16x8_t zero_mask0, zero_mask1;
+
+ int16x8_t zbin = vec_vsx_ld(0, zbin_ptr);
+ int16x8_t round = vec_vsx_ld(0, round_ptr);
+ int16x8_t quant = vec_vsx_ld(0, quant_ptr);
+ int16x8_t dequant = vec_vsx_ld(0, dequant_ptr);
+ int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr);
+
+ int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr);
+ int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr);
+
+ int16x8_t coeff0_abs = vec_abs(coeff0);
+ int16x8_t coeff1_abs = vec_abs(coeff1);
+
+ (void)scan_ptr;
+ (void)n_coeffs;
+
+ // 32x32 quantization requires that zbin and round be divided by 2
+ zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16);
+ round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16);
+
+ zero_mask0 = vec_cmpge(coeff0_abs, zbin);
+ zbin = vec_splat(zbin, 1); // remove DC from zbin
+ zero_mask1 = vec_cmpge(coeff1_abs, zbin);
+
+ qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
+ zero_mask0);
+ round = vec_splat(round, 1); // remove DC from round
+ quant = vec_splat(quant, 1); // remove DC from quant
+ quant_shift = vec_splat(quant_shift, 1); // remove DC from quant_shift
+ qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
+ zero_mask1);
+
+ vec_vsx_st(qcoeff0, 0, qcoeff_ptr);
+ vec_vsx_st(qcoeff1, 16, qcoeff_ptr);
+
+ vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), 0, dqcoeff_ptr);
+ dequant = vec_splat(dequant, 1); // remove DC from dequant
+ vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), 16, dqcoeff_ptr);
+
+ eob = vec_max(nonzero_scanindex(qcoeff0, iscan_ptr, 0),
+ nonzero_scanindex(qcoeff1, iscan_ptr, 16));
+
+ do {
+ int16x8_t coeff2, coeff2_abs, qcoeff2, eob2;
+ bool16x8_t zero_mask2;
+
+ coeff0 = vec_vsx_ld(off0, coeff_ptr);
+ coeff1 = vec_vsx_ld(off1, coeff_ptr);
+ coeff2 = vec_vsx_ld(off2, coeff_ptr);
+
+ coeff0_abs = vec_abs(coeff0);
+ coeff1_abs = vec_abs(coeff1);
+ coeff2_abs = vec_abs(coeff2);
+
+ zero_mask0 = vec_cmpge(coeff0_abs, zbin);
+ zero_mask1 = vec_cmpge(coeff1_abs, zbin);
+ zero_mask2 = vec_cmpge(coeff2_abs, zbin);
+
+ qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift,
+ zero_mask0);
+ qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift,
+ zero_mask1);
+ qcoeff2 = quantize_coeff_32(coeff2, coeff2_abs, round, quant, quant_shift,
+ zero_mask2);
+
+ vec_vsx_st(qcoeff0, off0, qcoeff_ptr);
+ vec_vsx_st(qcoeff1, off1, qcoeff_ptr);
+ vec_vsx_st(qcoeff2, off2, qcoeff_ptr);
+
+ vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), off0, dqcoeff_ptr);
+ vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), off1, dqcoeff_ptr);
+ vec_vsx_st(dequantize_coeff_32(qcoeff2, dequant), off2, dqcoeff_ptr);
+
+ eob = vec_max(eob, nonzero_scanindex(qcoeff0, iscan_ptr, off0));
+ eob2 = vec_max(nonzero_scanindex(qcoeff1, iscan_ptr, off1),
+ nonzero_scanindex(qcoeff2, iscan_ptr, off2));
+ eob = vec_max(eob, eob2);
+
+ // 24 int16_t is 48 bytes
+ off0 += 48;
+ off1 += 48;
+ off2 += 48;
+ num_itr--;
+ } while (num_itr != 0);
+
+ eob = vec_max_across(eob);
+ *eob_ptr = eob[0];
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/sad_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/sad_vsx.c
new file mode 100644
index 0000000000..a08ae12413
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/sad_vsx.c
@@ -0,0 +1,261 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stdlib.h>
+
+#include "./vpx_dsp_rtcd.h"
+
+#include "vpx_dsp/ppc/types_vsx.h"
+
+#include "vpx/vpx_integer.h"
+#include "vpx_ports/mem.h"
+
+#define PROCESS16(offset) \
+ v_a = vec_vsx_ld(offset, a); \
+ v_b = vec_vsx_ld(offset, b); \
+ v_abs = vec_absd(v_a, v_b); \
+ v_sad = vec_sum4s(v_abs, v_sad);
+
+#define SAD8(height) \
+ unsigned int vpx_sad8x##height##_vsx(const uint8_t *a, int a_stride, \
+ const uint8_t *b, int b_stride) { \
+ int y = 0; \
+ uint8x16_t v_a, v_b, v_abs; \
+ uint32x4_t v_sad = vec_zeros_u32; \
+ \
+ do { \
+ PROCESS16(0) \
+ \
+ a += a_stride; \
+ b += b_stride; \
+ y++; \
+ } while (y < height); \
+ \
+ return v_sad[1] + v_sad[0]; \
+ }
+
+#define SAD16(height) \
+ unsigned int vpx_sad16x##height##_vsx(const uint8_t *a, int a_stride, \
+ const uint8_t *b, int b_stride) { \
+ int y = 0; \
+ uint8x16_t v_a, v_b, v_abs; \
+ uint32x4_t v_sad = vec_zeros_u32; \
+ \
+ do { \
+ PROCESS16(0); \
+ \
+ a += a_stride; \
+ b += b_stride; \
+ y++; \
+ } while (y < height); \
+ \
+ return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
+ }
+
+#define SAD32(height) \
+ unsigned int vpx_sad32x##height##_vsx(const uint8_t *a, int a_stride, \
+ const uint8_t *b, int b_stride) { \
+ int y = 0; \
+ uint8x16_t v_a, v_b, v_abs; \
+ uint32x4_t v_sad = vec_zeros_u32; \
+ \
+ do { \
+ PROCESS16(0); \
+ PROCESS16(16); \
+ \
+ a += a_stride; \
+ b += b_stride; \
+ y++; \
+ } while (y < height); \
+ \
+ return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
+ }
+
+#define SAD64(height) \
+ unsigned int vpx_sad64x##height##_vsx(const uint8_t *a, int a_stride, \
+ const uint8_t *b, int b_stride) { \
+ int y = 0; \
+ uint8x16_t v_a, v_b, v_abs; \
+ uint32x4_t v_sad = vec_zeros_u32; \
+ \
+ do { \
+ PROCESS16(0); \
+ PROCESS16(16); \
+ PROCESS16(32); \
+ PROCESS16(48); \
+ \
+ a += a_stride; \
+ b += b_stride; \
+ y++; \
+ } while (y < height); \
+ \
+ return v_sad[3] + v_sad[2] + v_sad[1] + v_sad[0]; \
+ }
+
+SAD8(4);
+SAD8(8);
+SAD8(16);
+SAD16(8);
+SAD16(16);
+SAD16(32);
+SAD32(16);
+SAD32(32);
+SAD32(64);
+SAD64(32);
+SAD64(64);
+
+#define SAD16AVG(height) \
+ unsigned int vpx_sad16x##height##_avg_vsx( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred) { \
+ DECLARE_ALIGNED(16, uint8_t, comp_pred[16 * (height)]); \
+ vpx_comp_avg_pred_vsx(comp_pred, second_pred, 16, height, ref, \
+ ref_stride); \
+ \
+ return vpx_sad16x##height##_vsx(src, src_stride, comp_pred, 16); \
+ }
+
+#define SAD32AVG(height) \
+ unsigned int vpx_sad32x##height##_avg_vsx( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred) { \
+ DECLARE_ALIGNED(32, uint8_t, comp_pred[32 * (height)]); \
+ vpx_comp_avg_pred_vsx(comp_pred, second_pred, 32, height, ref, \
+ ref_stride); \
+ \
+ return vpx_sad32x##height##_vsx(src, src_stride, comp_pred, 32); \
+ }
+
+#define SAD64AVG(height) \
+ unsigned int vpx_sad64x##height##_avg_vsx( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
+ const uint8_t *second_pred) { \
+ DECLARE_ALIGNED(64, uint8_t, comp_pred[64 * (height)]); \
+ vpx_comp_avg_pred_vsx(comp_pred, second_pred, 64, height, ref, \
+ ref_stride); \
+ return vpx_sad64x##height##_vsx(src, src_stride, comp_pred, 64); \
+ }
+
+SAD16AVG(8);
+SAD16AVG(16);
+SAD16AVG(32);
+SAD32AVG(16);
+SAD32AVG(32);
+SAD32AVG(64);
+SAD64AVG(32);
+SAD64AVG(64);
+
+#define PROCESS16_4D(offset, ref, v_h, v_l) \
+ v_b = vec_vsx_ld(offset, ref); \
+ v_bh = unpack_to_s16_h(v_b); \
+ v_bl = unpack_to_s16_l(v_b); \
+ v_subh = vec_sub(v_h, v_bh); \
+ v_subl = vec_sub(v_l, v_bl); \
+ v_absh = vec_abs(v_subh); \
+ v_absl = vec_abs(v_subl); \
+ v_sad = vec_sum4s(v_absh, v_sad); \
+ v_sad = vec_sum4s(v_absl, v_sad);
+
+#define UNPACK_SRC(offset, srcv_h, srcv_l) \
+ v_a = vec_vsx_ld(offset, src); \
+ srcv_h = unpack_to_s16_h(v_a); \
+ srcv_l = unpack_to_s16_l(v_a);
+
+#define SAD16_4D(height) \
+ void vpx_sad16x##height##x4d_vsx(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ int i; \
+ int y; \
+ unsigned int sad[4]; \
+ uint8x16_t v_a, v_b; \
+ int16x8_t v_ah, v_al, v_bh, v_bl, v_absh, v_absl, v_subh, v_subl; \
+ \
+ for (i = 0; i < 4; i++) sad_array[i] = 0; \
+ \
+ for (y = 0; y < height; y++) { \
+ UNPACK_SRC(y *src_stride, v_ah, v_al); \
+ for (i = 0; i < 4; i++) { \
+ int32x4_t v_sad = vec_splat_s32(0); \
+ PROCESS16_4D(y *ref_stride, ref_array[i], v_ah, v_al); \
+ \
+ vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
+ sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
+ } \
+ } \
+ }
+
+#define SAD32_4D(height) \
+ void vpx_sad32x##height##x4d_vsx(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ int i; \
+ int y; \
+ unsigned int sad[4]; \
+ uint8x16_t v_a, v_b; \
+ int16x8_t v_ah1, v_al1, v_ah2, v_al2, v_bh, v_bl; \
+ int16x8_t v_absh, v_absl, v_subh, v_subl; \
+ \
+ for (i = 0; i < 4; i++) sad_array[i] = 0; \
+ \
+ for (y = 0; y < height; y++) { \
+ UNPACK_SRC(y *src_stride, v_ah1, v_al1); \
+ UNPACK_SRC(y *src_stride + 16, v_ah2, v_al2); \
+ for (i = 0; i < 4; i++) { \
+ int32x4_t v_sad = vec_splat_s32(0); \
+ PROCESS16_4D(y *ref_stride, ref_array[i], v_ah1, v_al1); \
+ PROCESS16_4D(y *ref_stride + 16, ref_array[i], v_ah2, v_al2); \
+ \
+ vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
+ sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
+ } \
+ } \
+ }
+
+#define SAD64_4D(height) \
+ void vpx_sad64x##height##x4d_vsx(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ int i; \
+ int y; \
+ unsigned int sad[4]; \
+ uint8x16_t v_a, v_b; \
+ int16x8_t v_ah1, v_al1, v_ah2, v_al2, v_bh, v_bl; \
+ int16x8_t v_ah3, v_al3, v_ah4, v_al4; \
+ int16x8_t v_absh, v_absl, v_subh, v_subl; \
+ \
+ for (i = 0; i < 4; i++) sad_array[i] = 0; \
+ \
+ for (y = 0; y < height; y++) { \
+ UNPACK_SRC(y *src_stride, v_ah1, v_al1); \
+ UNPACK_SRC(y *src_stride + 16, v_ah2, v_al2); \
+ UNPACK_SRC(y *src_stride + 32, v_ah3, v_al3); \
+ UNPACK_SRC(y *src_stride + 48, v_ah4, v_al4); \
+ for (i = 0; i < 4; i++) { \
+ int32x4_t v_sad = vec_splat_s32(0); \
+ PROCESS16_4D(y *ref_stride, ref_array[i], v_ah1, v_al1); \
+ PROCESS16_4D(y *ref_stride + 16, ref_array[i], v_ah2, v_al2); \
+ PROCESS16_4D(y *ref_stride + 32, ref_array[i], v_ah3, v_al3); \
+ PROCESS16_4D(y *ref_stride + 48, ref_array[i], v_ah4, v_al4); \
+ \
+ vec_vsx_st((uint32x4_t)v_sad, 0, sad); \
+ sad_array[i] += (sad[3] + sad[2] + sad[1] + sad[0]); \
+ } \
+ } \
+ }
+
+SAD16_4D(8);
+SAD16_4D(16);
+SAD16_4D(32);
+SAD32_4D(16);
+SAD32_4D(32);
+SAD32_4D(64);
+SAD64_4D(32);
+SAD64_4D(64);
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/subtract_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/subtract_vsx.c
new file mode 100644
index 0000000000..76ad302da6
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/subtract_vsx.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+
+#include "./vpx_config.h"
+#include "./vpx_dsp_rtcd.h"
+#include "vpx/vpx_integer.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+static VPX_FORCE_INLINE void subtract_block4x4(
+ int16_t *diff, ptrdiff_t diff_stride, const uint8_t *src,
+ ptrdiff_t src_stride, const uint8_t *pred, ptrdiff_t pred_stride) {
+ int16_t *diff1 = diff + 2 * diff_stride;
+ const uint8_t *src1 = src + 2 * src_stride;
+ const uint8_t *pred1 = pred + 2 * pred_stride;
+
+ const int16x8_t d0 = vec_vsx_ld(0, diff);
+ const int16x8_t d1 = vec_vsx_ld(0, diff + diff_stride);
+ const int16x8_t d2 = vec_vsx_ld(0, diff1);
+ const int16x8_t d3 = vec_vsx_ld(0, diff1 + diff_stride);
+
+ const uint8x16_t s0 = read4x2(src, (int)src_stride);
+ const uint8x16_t p0 = read4x2(pred, (int)pred_stride);
+ const uint8x16_t s1 = read4x2(src1, (int)src_stride);
+ const uint8x16_t p1 = read4x2(pred1, (int)pred_stride);
+
+ const int16x8_t da = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
+ const int16x8_t db = vec_sub(unpack_to_s16_h(s1), unpack_to_s16_h(p1));
+
+ vec_vsx_st(xxpermdi(da, d0, 1), 0, diff);
+ vec_vsx_st(xxpermdi(da, d1, 3), 0, diff + diff_stride);
+ vec_vsx_st(xxpermdi(db, d2, 1), 0, diff1);
+ vec_vsx_st(xxpermdi(db, d3, 3), 0, diff1 + diff_stride);
+}
+
+void vpx_subtract_block_vsx(int rows, int cols, int16_t *diff,
+ ptrdiff_t diff_stride, const uint8_t *src,
+ ptrdiff_t src_stride, const uint8_t *pred,
+ ptrdiff_t pred_stride) {
+ int r = rows, c;
+
+ switch (cols) {
+ case 64:
+ case 32:
+ do {
+ for (c = 0; c < cols; c += 32) {
+ const uint8x16_t s0 = vec_vsx_ld(0, src + c);
+ const uint8x16_t s1 = vec_vsx_ld(16, src + c);
+ const uint8x16_t p0 = vec_vsx_ld(0, pred + c);
+ const uint8x16_t p1 = vec_vsx_ld(16, pred + c);
+ const int16x8_t d0l =
+ vec_sub(unpack_to_s16_l(s0), unpack_to_s16_l(p0));
+ const int16x8_t d0h =
+ vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
+ const int16x8_t d1l =
+ vec_sub(unpack_to_s16_l(s1), unpack_to_s16_l(p1));
+ const int16x8_t d1h =
+ vec_sub(unpack_to_s16_h(s1), unpack_to_s16_h(p1));
+ vec_vsx_st(d0h, 0, diff + c);
+ vec_vsx_st(d0l, 16, diff + c);
+ vec_vsx_st(d1h, 0, diff + c + 16);
+ vec_vsx_st(d1l, 16, diff + c + 16);
+ }
+ diff += diff_stride;
+ pred += pred_stride;
+ src += src_stride;
+ } while (--r);
+ break;
+ case 16:
+ do {
+ const uint8x16_t s0 = vec_vsx_ld(0, src);
+ const uint8x16_t p0 = vec_vsx_ld(0, pred);
+ const int16x8_t d0l = vec_sub(unpack_to_s16_l(s0), unpack_to_s16_l(p0));
+ const int16x8_t d0h = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
+ vec_vsx_st(d0h, 0, diff);
+ vec_vsx_st(d0l, 16, diff);
+ diff += diff_stride;
+ pred += pred_stride;
+ src += src_stride;
+ } while (--r);
+ break;
+ case 8:
+ do {
+ const uint8x16_t s0 = vec_vsx_ld(0, src);
+ const uint8x16_t p0 = vec_vsx_ld(0, pred);
+ const int16x8_t d0h = vec_sub(unpack_to_s16_h(s0), unpack_to_s16_h(p0));
+ vec_vsx_st(d0h, 0, diff);
+ diff += diff_stride;
+ pred += pred_stride;
+ src += src_stride;
+ } while (--r);
+ break;
+ case 4:
+ subtract_block4x4(diff, diff_stride, src, src_stride, pred, pred_stride);
+ if (r > 4) {
+ diff += 4 * diff_stride;
+ pred += 4 * pred_stride;
+ src += 4 * src_stride;
+
+ subtract_block4x4(diff, diff_stride,
+
+ src, src_stride,
+
+ pred, pred_stride);
+ }
+ break;
+ default: assert(0); // unreachable
+ }
+}
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/transpose_vsx.h b/media/libvpx/libvpx/vpx_dsp/ppc/transpose_vsx.h
new file mode 100644
index 0000000000..4883b734ad
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/transpose_vsx.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
+#define VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
+
+#include "./vpx_config.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+static INLINE void vpx_transpose_s16_8x8(int16x8_t v[8]) {
+ // d = vec_mergeh(a,b):
+ // The even elements of the result are obtained left-to-right,
+ // from the high elements of a.
+ // The odd elements of the result are obtained left-to-right,
+ // from the high elements of b.
+ //
+ // d = vec_mergel(a,b):
+ // The even elements of the result are obtained left-to-right,
+ // from the low elements of a.
+ // The odd elements of the result are obtained left-to-right,
+ // from the low elements of b.
+
+ // Example, starting with:
+ // v[0]: 00 01 02 03 04 05 06 07
+ // v[1]: 10 11 12 13 14 15 16 17
+ // v[2]: 20 21 22 23 24 25 26 27
+ // v[3]: 30 31 32 33 34 35 36 37
+ // v[4]: 40 41 42 43 44 45 46 47
+ // v[5]: 50 51 52 53 54 55 56 57
+ // v[6]: 60 61 62 63 64 65 66 67
+ // v[7]: 70 71 72 73 74 75 76 77
+
+ int16x8_t b0, b1, b2, b3, b4, b5, b6, b7;
+ int16x8_t c0, c1, c2, c3, c4, c5, c6, c7;
+
+ b0 = vec_mergeh(v[0], v[4]);
+ b1 = vec_mergel(v[0], v[4]);
+ b2 = vec_mergeh(v[1], v[5]);
+ b3 = vec_mergel(v[1], v[5]);
+ b4 = vec_mergeh(v[2], v[6]);
+ b5 = vec_mergel(v[2], v[6]);
+ b6 = vec_mergeh(v[3], v[7]);
+ b7 = vec_mergel(v[3], v[7]);
+
+ // After first merge operation
+ // b0: 00 40 01 41 02 42 03 43
+ // b1: 04 44 05 45 06 46 07 47
+ // b2: 10 50 11 51 12 52 13 53
+ // b3: 14 54 15 55 16 56 17 57
+ // b4: 20 60 21 61 22 62 23 63
+ // b5: 24 64 25 65 26 66 27 67
+ // b6: 30 70 31 71 32 62 33 73
+ // b7: 34 74 35 75 36 76 37 77
+
+ c0 = vec_mergeh(b0, b4);
+ c1 = vec_mergel(b0, b4);
+ c2 = vec_mergeh(b1, b5);
+ c3 = vec_mergel(b1, b5);
+ c4 = vec_mergeh(b2, b6);
+ c5 = vec_mergel(b2, b6);
+ c6 = vec_mergeh(b3, b7);
+ c7 = vec_mergel(b3, b7);
+
+ // After second merge operation
+ // c0: 00 20 40 60 01 21 41 61
+ // c1: 02 22 42 62 03 23 43 63
+ // c2: 04 24 44 64 05 25 45 65
+ // c3: 06 26 46 66 07 27 47 67
+ // c4: 10 30 50 70 11 31 51 71
+ // c5: 12 32 52 72 13 33 53 73
+ // c6: 14 34 54 74 15 35 55 75
+ // c7: 16 36 56 76 17 37 57 77
+
+ v[0] = vec_mergeh(c0, c4);
+ v[1] = vec_mergel(c0, c4);
+ v[2] = vec_mergeh(c1, c5);
+ v[3] = vec_mergel(c1, c5);
+ v[4] = vec_mergeh(c2, c6);
+ v[5] = vec_mergel(c2, c6);
+ v[6] = vec_mergeh(c3, c7);
+ v[7] = vec_mergel(c3, c7);
+
+ // After last merge operation
+ // v[0]: 00 10 20 30 40 50 60 70
+ // v[1]: 01 11 21 31 41 51 61 71
+ // v[2]: 02 12 22 32 42 52 62 72
+ // v[3]: 03 13 23 33 43 53 63 73
+ // v[4]: 04 14 24 34 44 54 64 74
+ // v[5]: 05 15 25 35 45 55 65 75
+ // v[6]: 06 16 26 36 46 56 66 76
+ // v[7]: 07 17 27 37 47 57 67 77
+}
+
+static INLINE void transpose_8x8(const int16x8_t *a, int16x8_t *b) {
+ // Stage 1
+ const int16x8_t s1_0 = vec_mergeh(a[0], a[4]);
+ const int16x8_t s1_1 = vec_mergel(a[0], a[4]);
+ const int16x8_t s1_2 = vec_mergeh(a[1], a[5]);
+ const int16x8_t s1_3 = vec_mergel(a[1], a[5]);
+ const int16x8_t s1_4 = vec_mergeh(a[2], a[6]);
+ const int16x8_t s1_5 = vec_mergel(a[2], a[6]);
+ const int16x8_t s1_6 = vec_mergeh(a[3], a[7]);
+ const int16x8_t s1_7 = vec_mergel(a[3], a[7]);
+
+ // Stage 2
+ const int16x8_t s2_0 = vec_mergeh(s1_0, s1_4);
+ const int16x8_t s2_1 = vec_mergel(s1_0, s1_4);
+ const int16x8_t s2_2 = vec_mergeh(s1_1, s1_5);
+ const int16x8_t s2_3 = vec_mergel(s1_1, s1_5);
+ const int16x8_t s2_4 = vec_mergeh(s1_2, s1_6);
+ const int16x8_t s2_5 = vec_mergel(s1_2, s1_6);
+ const int16x8_t s2_6 = vec_mergeh(s1_3, s1_7);
+ const int16x8_t s2_7 = vec_mergel(s1_3, s1_7);
+
+ // Stage 2
+ b[0] = vec_mergeh(s2_0, s2_4);
+ b[1] = vec_mergel(s2_0, s2_4);
+ b[2] = vec_mergeh(s2_1, s2_5);
+ b[3] = vec_mergel(s2_1, s2_5);
+ b[4] = vec_mergeh(s2_2, s2_6);
+ b[5] = vec_mergel(s2_2, s2_6);
+ b[6] = vec_mergeh(s2_3, s2_7);
+ b[7] = vec_mergel(s2_3, s2_7);
+}
+
+#endif // VPX_VPX_DSP_PPC_TRANSPOSE_VSX_H_
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/txfm_common_vsx.h b/media/libvpx/libvpx/vpx_dsp/ppc/txfm_common_vsx.h
new file mode 100644
index 0000000000..2907a1fe40
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/txfm_common_vsx.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
+#define VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
+
+#include "vpx_dsp/ppc/types_vsx.h"
+
+static const int32x4_t vec_dct_const_rounding = { 8192, 8192, 8192, 8192 };
+
+static const uint32x4_t vec_dct_const_bits = { 14, 14, 14, 14 };
+
+static const uint16x8_t vec_dct_scale_log2 = { 2, 2, 2, 2, 2, 2, 2, 2 };
+
+static const int16x8_t cospi1_v = { 16364, 16364, 16364, 16364,
+ 16364, 16364, 16364, 16364 };
+static const int16x8_t cospi2_v = { 16305, 16305, 16305, 16305,
+ 16305, 16305, 16305, 16305 };
+static const int16x8_t cospi3_v = { 16207, 16207, 16207, 16207,
+ 16207, 16207, 16207, 16207 };
+static const int16x8_t cospi4_v = { 16069, 16069, 16069, 16069,
+ 16069, 16069, 16069, 16069 };
+static const int16x8_t cospi4m_v = { -16069, -16069, -16069, -16069,
+ -16069, -16069, -16069, -16069 };
+static const int16x8_t cospi5_v = { 15893, 15893, 15893, 15893,
+ 15893, 15893, 15893, 15893 };
+static const int16x8_t cospi6_v = { 15679, 15679, 15679, 15679,
+ 15679, 15679, 15679, 15679 };
+static const int16x8_t cospi7_v = { 15426, 15426, 15426, 15426,
+ 15426, 15426, 15426, 15426 };
+static const int16x8_t cospi8_v = { 15137, 15137, 15137, 15137,
+ 15137, 15137, 15137, 15137 };
+static const int16x8_t cospi8m_v = { -15137, -15137, -15137, -15137,
+ -15137, -15137, -15137, -15137 };
+static const int16x8_t cospi9_v = { 14811, 14811, 14811, 14811,
+ 14811, 14811, 14811, 14811 };
+static const int16x8_t cospi10_v = { 14449, 14449, 14449, 14449,
+ 14449, 14449, 14449, 14449 };
+static const int16x8_t cospi11_v = { 14053, 14053, 14053, 14053,
+ 14053, 14053, 14053, 14053 };
+static const int16x8_t cospi12_v = { 13623, 13623, 13623, 13623,
+ 13623, 13623, 13623, 13623 };
+static const int16x8_t cospi13_v = { 13160, 13160, 13160, 13160,
+ 13160, 13160, 13160, 13160 };
+static const int16x8_t cospi14_v = { 12665, 12665, 12665, 12665,
+ 12665, 12665, 12665, 12665 };
+static const int16x8_t cospi15_v = { 12140, 12140, 12140, 12140,
+ 12140, 12140, 12140, 12140 };
+static const int16x8_t cospi16_v = { 11585, 11585, 11585, 11585,
+ 11585, 11585, 11585, 11585 };
+static const int16x8_t cospi17_v = { 11003, 11003, 11003, 11003,
+ 11003, 11003, 11003, 11003 };
+static const int16x8_t cospi18_v = { 10394, 10394, 10394, 10394,
+ 10394, 10394, 10394, 10394 };
+static const int16x8_t cospi19_v = { 9760, 9760, 9760, 9760,
+ 9760, 9760, 9760, 9760 };
+static const int16x8_t cospi20_v = { 9102, 9102, 9102, 9102,
+ 9102, 9102, 9102, 9102 };
+static const int16x8_t cospi20m_v = { -9102, -9102, -9102, -9102,
+ -9102, -9102, -9102, -9102 };
+static const int16x8_t cospi21_v = { 8423, 8423, 8423, 8423,
+ 8423, 8423, 8423, 8423 };
+static const int16x8_t cospi22_v = { 7723, 7723, 7723, 7723,
+ 7723, 7723, 7723, 7723 };
+static const int16x8_t cospi23_v = { 7005, 7005, 7005, 7005,
+ 7005, 7005, 7005, 7005 };
+static const int16x8_t cospi24_v = { 6270, 6270, 6270, 6270,
+ 6270, 6270, 6270, 6270 };
+static const int16x8_t cospi25_v = { 5520, 5520, 5520, 5520,
+ 5520, 5520, 5520, 5520 };
+static const int16x8_t cospi26_v = { 4756, 4756, 4756, 4756,
+ 4756, 4756, 4756, 4756 };
+static const int16x8_t cospi27_v = { 3981, 3981, 3981, 3981,
+ 3981, 3981, 3981, 3981 };
+static const int16x8_t cospi28_v = { 3196, 3196, 3196, 3196,
+ 3196, 3196, 3196, 3196 };
+static const int16x8_t cospi29_v = { 2404, 2404, 2404, 2404,
+ 2404, 2404, 2404, 2404 };
+static const int16x8_t cospi30_v = { 1606, 1606, 1606, 1606,
+ 1606, 1606, 1606, 1606 };
+static const int16x8_t cospi31_v = { 804, 804, 804, 804, 804, 804, 804, 804 };
+
+#endif // VPX_VPX_DSP_PPC_TXFM_COMMON_VSX_H_
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/types_vsx.h b/media/libvpx/libvpx/vpx_dsp/ppc/types_vsx.h
new file mode 100644
index 0000000000..b891169245
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/types_vsx.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VPX_VPX_DSP_PPC_TYPES_VSX_H_
+#define VPX_VPX_DSP_PPC_TYPES_VSX_H_
+
+#include <altivec.h>
+
+typedef vector signed char int8x16_t;
+typedef vector unsigned char uint8x16_t;
+typedef vector signed short int16x8_t;
+typedef vector unsigned short uint16x8_t;
+typedef vector signed int int32x4_t;
+typedef vector unsigned int uint32x4_t;
+typedef vector bool char bool8x16_t;
+typedef vector bool short bool16x8_t;
+typedef vector bool int bool32x4_t;
+
+#if defined(__clang__) && __clang_major__ < 6
+static const uint8x16_t xxpermdi0_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x10, 0x11, 0x12, 0x13,
+ 0x14, 0x15, 0x16, 0x17 };
+static const uint8x16_t xxpermdi1_perm = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x18, 0x19, 0x1A, 0x1B,
+ 0x1C, 0x1D, 0x1E, 0x1F };
+static const uint8x16_t xxpermdi2_perm = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
+ 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13,
+ 0x14, 0x15, 0x16, 0x17 };
+static const uint8x16_t xxpermdi3_perm = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
+ 0x0E, 0x0F, 0x18, 0x19, 0x1A, 0x1B,
+ 0x1C, 0x1D, 0x1E, 0x1F };
+#define xxpermdi(a, b, c) vec_perm(a, b, xxpermdi##c##_perm)
+#elif defined(__GNUC__) && \
+ (__GNUC__ > 6 || (__GNUC__ == 6 && __GNUC_MINOR__ >= 3))
+#define xxpermdi(a, b, c) vec_xxpermdi(a, b, c)
+#endif
+
+#ifdef WORDS_BIGENDIAN
+#define unpack_to_u16_h(v) \
+ (uint16x8_t) vec_mergeh(vec_splat_u8(0), (uint8x16_t)v)
+#define unpack_to_u16_l(v) \
+ (uint16x8_t) vec_mergel(vec_splat_u8(0), (uint8x16_t)v)
+#define unpack_to_s16_h(v) \
+ (int16x8_t) vec_mergeh(vec_splat_u8(0), (uint8x16_t)v)
+#define unpack_to_s16_l(v) \
+ (int16x8_t) vec_mergel(vec_splat_u8(0), (uint8x16_t)v)
+#ifndef xxpermdi
+#define xxpermdi(a, b, c) vec_xxpermdi(a, b, c)
+#endif
+#else
+#define unpack_to_u16_h(v) \
+ (uint16x8_t) vec_mergeh((uint8x16_t)v, vec_splat_u8(0))
+#define unpack_to_u16_l(v) \
+ (uint16x8_t) vec_mergel((uint8x16_t)v, vec_splat_u8(0))
+#define unpack_to_s16_h(v) \
+ (int16x8_t) vec_mergeh((uint8x16_t)v, vec_splat_u8(0))
+#define unpack_to_s16_l(v) \
+ (int16x8_t) vec_mergel((uint8x16_t)v, vec_splat_u8(0))
+#ifndef xxpermdi
+#define xxpermdi(a, b, c) vec_xxpermdi(b, a, (((c) >> 1) | ((c)&1) << 1) ^ 3)
+#endif
+#endif
+
+static INLINE uint8x16_t read4x2(const uint8_t *a, int stride) {
+ const uint32x4_t a0 = (uint32x4_t)vec_vsx_ld(0, a);
+ const uint32x4_t a1 = (uint32x4_t)vec_vsx_ld(0, a + stride);
+
+ return (uint8x16_t)vec_mergeh(a0, a1);
+}
+
+#ifndef __POWER9_VECTOR__
+#define vec_absd(a, b) vec_sub(vec_max(a, b), vec_min(a, b))
+#endif
+
+static const uint8x16_t vec_zeros_u8 = { 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int16x8_t vec_zeros_s16 = { 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int16x8_t vec_ones_s16 = { 1, 1, 1, 1, 1, 1, 1, 1 };
+static const int16x8_t vec_twos_s16 = { 2, 2, 2, 2, 2, 2, 2, 2 };
+static const uint16x8_t vec_ones_u16 = { 1, 1, 1, 1, 1, 1, 1, 1 };
+static const uint32x4_t vec_ones_u32 = { 1, 1, 1, 1 };
+static const int32x4_t vec_zeros_s32 = { 0, 0, 0, 0 };
+static const uint32x4_t vec_zeros_u32 = { 0, 0, 0, 0 };
+static const uint16x8_t vec_shift_sign_s16 = { 15, 15, 15, 15, 15, 15, 15, 15 };
+static const uint32x4_t vec_shift_sign_s32 = { 31, 31, 31, 31 };
+static const uint8x16_t vec_perm64 = { 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D,
+ 0x0E, 0x0F, 0x00, 0x01, 0x02, 0x03,
+ 0x04, 0x05, 0x06, 0x07 };
+static const uint8x16_t vec_perm32 = { 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+ 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x00, 0x01, 0x02, 0x03 };
+static const uint8x16_t vec_perm16 = { 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0A, 0x0B, 0x0E, 0x0D,
+ 0x0E, 0x0F, 0x00, 0x01 };
+
+static const uint8x16_t vec_perm_odd_even_pack = { 0x00, 0x01, 0x10, 0x11,
+ 0x04, 0x05, 0x14, 0x15,
+ 0x08, 0x09, 0x18, 0x19,
+ 0x0C, 0x0D, 0x1C, 0x1D };
+
+#endif // VPX_VPX_DSP_PPC_TYPES_VSX_H_
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/variance_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/variance_vsx.c
new file mode 100644
index 0000000000..be9614a358
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/variance_vsx.c
@@ -0,0 +1,271 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+
+#include "./vpx_config.h"
+#include "./vpx_dsp_rtcd.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+
+uint32_t vpx_get4x4sse_cs_vsx(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride) {
+ int distortion;
+
+ const int16x8_t a0 = unpack_to_s16_h(read4x2(src_ptr, src_stride));
+ const int16x8_t a1 =
+ unpack_to_s16_h(read4x2(src_ptr + src_stride * 2, src_stride));
+ const int16x8_t b0 = unpack_to_s16_h(read4x2(ref_ptr, ref_stride));
+ const int16x8_t b1 =
+ unpack_to_s16_h(read4x2(ref_ptr + ref_stride * 2, ref_stride));
+ const int16x8_t d0 = vec_sub(a0, b0);
+ const int16x8_t d1 = vec_sub(a1, b1);
+ const int32x4_t ds = vec_msum(d1, d1, vec_msum(d0, d0, vec_splat_s32(0)));
+ const int32x4_t d = vec_splat(vec_sums(ds, vec_splat_s32(0)), 3);
+
+ vec_ste(d, 0, &distortion);
+
+ return distortion;
+}
+
+// TODO(lu_zero): Unroll
+uint32_t vpx_get_mb_ss_vsx(const int16_t *src_ptr) {
+ unsigned int i, sum = 0;
+ int32x4_t s = vec_splat_s32(0);
+
+ for (i = 0; i < 256; i += 8) {
+ const int16x8_t v = vec_vsx_ld(0, src_ptr + i);
+ s = vec_msum(v, v, s);
+ }
+
+ s = vec_splat(vec_sums(s, vec_splat_s32(0)), 3);
+
+ vec_ste((uint32x4_t)s, 0, &sum);
+
+ return sum;
+}
+
+void vpx_comp_avg_pred_vsx(uint8_t *comp_pred, const uint8_t *pred, int width,
+ int height, const uint8_t *ref, int ref_stride) {
+ int i, j;
+ /* comp_pred and pred must be 16 byte aligned. */
+ assert(((intptr_t)comp_pred & 0xf) == 0);
+ assert(((intptr_t)pred & 0xf) == 0);
+ if (width >= 16) {
+ for (i = 0; i < height; ++i) {
+ for (j = 0; j < width; j += 16) {
+ const uint8x16_t v = vec_avg(vec_vsx_ld(j, pred), vec_vsx_ld(j, ref));
+ vec_vsx_st(v, j, comp_pred);
+ }
+ comp_pred += width;
+ pred += width;
+ ref += ref_stride;
+ }
+ } else if (width == 8) {
+ // Process 2 lines at time
+ for (i = 0; i < height / 2; ++i) {
+ const uint8x16_t r0 = vec_vsx_ld(0, ref);
+ const uint8x16_t r1 = vec_vsx_ld(0, ref + ref_stride);
+ const uint8x16_t r = xxpermdi(r0, r1, 0);
+ const uint8x16_t v = vec_avg(vec_vsx_ld(0, pred), r);
+ vec_vsx_st(v, 0, comp_pred);
+ comp_pred += 16; // width * 2;
+ pred += 16; // width * 2;
+ ref += ref_stride * 2;
+ }
+ } else {
+ assert(width == 4);
+ // process 4 lines at time
+ for (i = 0; i < height / 4; ++i) {
+ const uint32x4_t r0 = (uint32x4_t)vec_vsx_ld(0, ref);
+ const uint32x4_t r1 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride);
+ const uint32x4_t r2 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride * 2);
+ const uint32x4_t r3 = (uint32x4_t)vec_vsx_ld(0, ref + ref_stride * 3);
+ const uint8x16_t r =
+ (uint8x16_t)xxpermdi(vec_mergeh(r0, r1), vec_mergeh(r2, r3), 0);
+ const uint8x16_t v = vec_avg(vec_vsx_ld(0, pred), r);
+ vec_vsx_st(v, 0, comp_pred);
+ comp_pred += 16; // width * 4;
+ pred += 16; // width * 4;
+ ref += ref_stride * 4;
+ }
+ }
+}
+
+static INLINE void variance_inner_32(const uint8_t *src_ptr,
+ const uint8_t *ref_ptr,
+ int32x4_t *sum_squared, int32x4_t *sum) {
+ int32x4_t s = *sum;
+ int32x4_t ss = *sum_squared;
+
+ const uint8x16_t va0 = vec_vsx_ld(0, src_ptr);
+ const uint8x16_t vb0 = vec_vsx_ld(0, ref_ptr);
+ const uint8x16_t va1 = vec_vsx_ld(16, src_ptr);
+ const uint8x16_t vb1 = vec_vsx_ld(16, ref_ptr);
+
+ const int16x8_t a0 = unpack_to_s16_h(va0);
+ const int16x8_t b0 = unpack_to_s16_h(vb0);
+ const int16x8_t a1 = unpack_to_s16_l(va0);
+ const int16x8_t b1 = unpack_to_s16_l(vb0);
+ const int16x8_t a2 = unpack_to_s16_h(va1);
+ const int16x8_t b2 = unpack_to_s16_h(vb1);
+ const int16x8_t a3 = unpack_to_s16_l(va1);
+ const int16x8_t b3 = unpack_to_s16_l(vb1);
+ const int16x8_t d0 = vec_sub(a0, b0);
+ const int16x8_t d1 = vec_sub(a1, b1);
+ const int16x8_t d2 = vec_sub(a2, b2);
+ const int16x8_t d3 = vec_sub(a3, b3);
+
+ s = vec_sum4s(d0, s);
+ ss = vec_msum(d0, d0, ss);
+ s = vec_sum4s(d1, s);
+ ss = vec_msum(d1, d1, ss);
+ s = vec_sum4s(d2, s);
+ ss = vec_msum(d2, d2, ss);
+ s = vec_sum4s(d3, s);
+ ss = vec_msum(d3, d3, ss);
+ *sum = s;
+ *sum_squared = ss;
+}
+
+static INLINE void variance(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride, int w,
+ int h, uint32_t *sse, int *sum) {
+ int i;
+
+ int32x4_t s = vec_splat_s32(0);
+ int32x4_t ss = vec_splat_s32(0);
+
+ switch (w) {
+ case 4:
+ for (i = 0; i < h / 2; ++i) {
+ const int16x8_t a0 = unpack_to_s16_h(read4x2(src_ptr, src_stride));
+ const int16x8_t b0 = unpack_to_s16_h(read4x2(ref_ptr, ref_stride));
+ const int16x8_t d = vec_sub(a0, b0);
+ s = vec_sum4s(d, s);
+ ss = vec_msum(d, d, ss);
+ src_ptr += src_stride * 2;
+ ref_ptr += ref_stride * 2;
+ }
+ break;
+ case 8:
+ for (i = 0; i < h; ++i) {
+ const int16x8_t a0 = unpack_to_s16_h(vec_vsx_ld(0, src_ptr));
+ const int16x8_t b0 = unpack_to_s16_h(vec_vsx_ld(0, ref_ptr));
+ const int16x8_t d = vec_sub(a0, b0);
+
+ s = vec_sum4s(d, s);
+ ss = vec_msum(d, d, ss);
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ }
+ break;
+ case 16:
+ for (i = 0; i < h; ++i) {
+ const uint8x16_t va = vec_vsx_ld(0, src_ptr);
+ const uint8x16_t vb = vec_vsx_ld(0, ref_ptr);
+ const int16x8_t a0 = unpack_to_s16_h(va);
+ const int16x8_t b0 = unpack_to_s16_h(vb);
+ const int16x8_t a1 = unpack_to_s16_l(va);
+ const int16x8_t b1 = unpack_to_s16_l(vb);
+ const int16x8_t d0 = vec_sub(a0, b0);
+ const int16x8_t d1 = vec_sub(a1, b1);
+
+ s = vec_sum4s(d0, s);
+ ss = vec_msum(d0, d0, ss);
+ s = vec_sum4s(d1, s);
+ ss = vec_msum(d1, d1, ss);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ }
+ break;
+ case 32:
+ for (i = 0; i < h; ++i) {
+ variance_inner_32(src_ptr, ref_ptr, &ss, &s);
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ }
+ break;
+ case 64:
+ for (i = 0; i < h; ++i) {
+ variance_inner_32(src_ptr, ref_ptr, &ss, &s);
+ variance_inner_32(src_ptr + 32, ref_ptr + 32, &ss, &s);
+
+ src_ptr += src_stride;
+ ref_ptr += ref_stride;
+ }
+ break;
+ }
+
+ s = vec_splat(vec_sums(s, vec_splat_s32(0)), 3);
+
+ vec_ste(s, 0, sum);
+
+ ss = vec_splat(vec_sums(ss, vec_splat_s32(0)), 3);
+
+ vec_ste((uint32x4_t)ss, 0, sse);
+}
+
+/* Identical to the variance call except it takes an additional parameter, sum,
+ * and returns that value using pass-by-reference instead of returning
+ * sse - sum^2 / w*h
+ */
+#define GET_VAR(W, H) \
+ void vpx_get##W##x##H##var_vsx(const uint8_t *src_ptr, int src_stride, \
+ const uint8_t *ref_ptr, int ref_stride, \
+ uint32_t *sse, int *sum) { \
+ variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, sum); \
+ }
+
+/* Identical to the variance call except it does not calculate the
+ * sse - sum^2 / w*h and returns sse in addtion to modifying the passed in
+ * variable.
+ */
+#define MSE(W, H) \
+ uint32_t vpx_mse##W##x##H##_vsx(const uint8_t *src_ptr, int src_stride, \
+ const uint8_t *ref_ptr, int ref_stride, \
+ uint32_t *sse) { \
+ int sum; \
+ variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, &sum); \
+ return *sse; \
+ }
+
+#define VAR(W, H) \
+ uint32_t vpx_variance##W##x##H##_vsx(const uint8_t *src_ptr, int src_stride, \
+ const uint8_t *ref_ptr, int ref_stride, \
+ uint32_t *sse) { \
+ int sum; \
+ variance(src_ptr, src_stride, ref_ptr, ref_stride, W, H, sse, &sum); \
+ return *sse - (uint32_t)(((int64_t)sum * sum) / ((W) * (H))); \
+ }
+
+#define VARIANCES(W, H) VAR(W, H)
+
+VARIANCES(64, 64)
+VARIANCES(64, 32)
+VARIANCES(32, 64)
+VARIANCES(32, 32)
+VARIANCES(32, 16)
+VARIANCES(16, 32)
+VARIANCES(16, 16)
+VARIANCES(16, 8)
+VARIANCES(8, 16)
+VARIANCES(8, 8)
+VARIANCES(8, 4)
+VARIANCES(4, 8)
+VARIANCES(4, 4)
+
+GET_VAR(16, 16)
+GET_VAR(8, 8)
+
+MSE(16, 16)
+MSE(16, 8)
+MSE(8, 16)
+MSE(8, 8)
diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/vpx_convolve_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/vpx_convolve_vsx.c
new file mode 100644
index 0000000000..2dc66055cc
--- /dev/null
+++ b/media/libvpx/libvpx/vpx_dsp/ppc/vpx_convolve_vsx.c
@@ -0,0 +1,408 @@
+/*
+ * Copyright (c) 2017 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include <assert.h>
+#include <string.h>
+
+#include "./vpx_dsp_rtcd.h"
+#include "vpx/vpx_integer.h"
+#include "vpx_dsp/ppc/types_vsx.h"
+#include "vpx_dsp/vpx_filter.h"
+
+// TODO(lu_zero): unroll
+static VPX_FORCE_INLINE void copy_w16(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static VPX_FORCE_INLINE void copy_w32(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
+ vec_vsx_st(vec_vsx_ld(16, src), 16, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static VPX_FORCE_INLINE void copy_w64(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ vec_vsx_st(vec_vsx_ld(0, src), 0, dst);
+ vec_vsx_st(vec_vsx_ld(16, src), 16, dst);
+ vec_vsx_st(vec_vsx_ld(32, src), 32, dst);
+ vec_vsx_st(vec_vsx_ld(48, src), 48, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+void vpx_convolve_copy_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4, int x_step_q4,
+ int y0_q4, int32_t y_step_q4, int32_t w, int32_t h) {
+ (void)filter;
+ (void)x0_q4;
+ (void)x_step_q4;
+ (void)y0_q4;
+ (void)y_step_q4;
+
+ switch (w) {
+ case 16: {
+ copy_w16(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ case 32: {
+ copy_w32(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ case 64: {
+ copy_w64(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ default: {
+ int i;
+ for (i = h; i--;) {
+ memcpy(dst, src, w);
+ src += src_stride;
+ dst += dst_stride;
+ }
+ break;
+ }
+ }
+}
+
+static VPX_FORCE_INLINE void avg_w16(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ const uint8x16_t v = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
+ vec_vsx_st(v, 0, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static VPX_FORCE_INLINE void avg_w32(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ const uint8x16_t v0 = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
+ const uint8x16_t v1 = vec_avg(vec_vsx_ld(16, src), vec_vsx_ld(16, dst));
+ vec_vsx_st(v0, 0, dst);
+ vec_vsx_st(v1, 16, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static VPX_FORCE_INLINE void avg_w64(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ int32_t h) {
+ int i;
+
+ for (i = h; i--;) {
+ const uint8x16_t v0 = vec_avg(vec_vsx_ld(0, src), vec_vsx_ld(0, dst));
+ const uint8x16_t v1 = vec_avg(vec_vsx_ld(16, src), vec_vsx_ld(16, dst));
+ const uint8x16_t v2 = vec_avg(vec_vsx_ld(32, src), vec_vsx_ld(32, dst));
+ const uint8x16_t v3 = vec_avg(vec_vsx_ld(48, src), vec_vsx_ld(48, dst));
+ vec_vsx_st(v0, 0, dst);
+ vec_vsx_st(v1, 16, dst);
+ vec_vsx_st(v2, 32, dst);
+ vec_vsx_st(v3, 48, dst);
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+void vpx_convolve_avg_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4, int x_step_q4,
+ int y0_q4, int32_t y_step_q4, int32_t w, int32_t h) {
+ switch (w) {
+ case 16: {
+ avg_w16(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ case 32: {
+ avg_w32(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ case 64: {
+ avg_w64(src, src_stride, dst, dst_stride, h);
+ break;
+ }
+ default: {
+ vpx_convolve_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4,
+ x_step_q4, y0_q4, y_step_q4, w, h);
+ break;
+ }
+ }
+}
+
+static VPX_FORCE_INLINE void convolve_line(uint8_t *dst, const int16x8_t s,
+ const int16x8_t f) {
+ const int32x4_t sum = vec_msum(s, f, vec_splat_s32(0));
+ const int32x4_t bias =
+ vec_sl(vec_splat_s32(1), vec_splat_u32(FILTER_BITS - 1));
+ const int32x4_t avg = vec_sr(vec_sums(sum, bias), vec_splat_u32(FILTER_BITS));
+ const uint8x16_t v = vec_splat(
+ vec_packsu(vec_pack(avg, vec_splat_s32(0)), vec_splat_s16(0)), 3);
+ vec_ste(v, 0, dst);
+}
+
+static VPX_FORCE_INLINE void convolve_line_h(uint8_t *dst,
+ const uint8_t *const src_x,
+ const int16_t *const x_filter) {
+ const int16x8_t s = unpack_to_s16_h(vec_vsx_ld(0, src_x));
+ const int16x8_t f = vec_vsx_ld(0, x_filter);
+
+ convolve_line(dst, s, f);
+}
+
+// TODO(lu_zero): Implement 8x8 and bigger block special cases
+static VPX_FORCE_INLINE void convolve_horiz(const uint8_t *src,
+ ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride,
+ const InterpKernel *x_filters,
+ int x0_q4, int x_step_q4, int w,
+ int h) {
+ int x, y;
+ src -= SUBPEL_TAPS / 2 - 1;
+
+ for (y = 0; y < h; ++y) {
+ int x_q4 = x0_q4;
+ for (x = 0; x < w; ++x) {
+ convolve_line_h(dst + x, &src[x_q4 >> SUBPEL_BITS],
+ x_filters[x_q4 & SUBPEL_MASK]);
+ x_q4 += x_step_q4;
+ }
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static VPX_FORCE_INLINE void convolve_avg_horiz(
+ const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, const InterpKernel *x_filters, int x0_q4,
+ int x_step_q4, int w, int h) {
+ int x, y;
+ src -= SUBPEL_TAPS / 2 - 1;
+
+ for (y = 0; y < h; ++y) {
+ int x_q4 = x0_q4;
+ for (x = 0; x < w; ++x) {
+ uint8_t v;
+ convolve_line_h(&v, &src[x_q4 >> SUBPEL_BITS],
+ x_filters[x_q4 & SUBPEL_MASK]);
+ dst[x] = ROUND_POWER_OF_TWO(dst[x] + v, 1);
+ x_q4 += x_step_q4;
+ }
+ src += src_stride;
+ dst += dst_stride;
+ }
+}
+
+static uint8x16_t transpose_line_u8_8x8(uint8x16_t a, uint8x16_t b,
+ uint8x16_t c, uint8x16_t d,
+ uint8x16_t e, uint8x16_t f,
+ uint8x16_t g, uint8x16_t h) {
+ uint16x8_t ab = (uint16x8_t)vec_mergeh(a, b);
+ uint16x8_t cd = (uint16x8_t)vec_mergeh(c, d);
+ uint16x8_t ef = (uint16x8_t)vec_mergeh(e, f);
+ uint16x8_t gh = (uint16x8_t)vec_mergeh(g, h);
+
+ uint32x4_t abcd = (uint32x4_t)vec_mergeh(ab, cd);
+ uint32x4_t efgh = (uint32x4_t)vec_mergeh(ef, gh);
+
+ return (uint8x16_t)vec_mergeh(abcd, efgh);
+}
+
+static VPX_FORCE_INLINE void convolve_line_v(uint8_t *dst,
+ const uint8_t *const src_y,
+ ptrdiff_t src_stride,
+ const int16_t *const y_filter) {
+ uint8x16_t s0 = vec_vsx_ld(0, src_y + 0 * src_stride);
+ uint8x16_t s1 = vec_vsx_ld(0, src_y + 1 * src_stride);
+ uint8x16_t s2 = vec_vsx_ld(0, src_y + 2 * src_stride);
+ uint8x16_t s3 = vec_vsx_ld(0, src_y + 3 * src_stride);
+ uint8x16_t s4 = vec_vsx_ld(0, src_y + 4 * src_stride);
+ uint8x16_t s5 = vec_vsx_ld(0, src_y + 5 * src_stride);
+ uint8x16_t s6 = vec_vsx_ld(0, src_y + 6 * src_stride);
+ uint8x16_t s7 = vec_vsx_ld(0, src_y + 7 * src_stride);
+ const int16x8_t f = vec_vsx_ld(0, y_filter);
+ uint8_t buf[16];
+ const uint8x16_t s = transpose_line_u8_8x8(s0, s1, s2, s3, s4, s5, s6, s7);
+
+ vec_vsx_st(s, 0, buf);
+
+ convolve_line(dst, unpack_to_s16_h(s), f);
+}
+
+static VPX_FORCE_INLINE void convolve_vert(const uint8_t *src,
+ ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride,
+ const InterpKernel *y_filters,
+ int y0_q4, int y_step_q4, int w,
+ int h) {
+ int x, y;
+ src -= src_stride * (SUBPEL_TAPS / 2 - 1);
+
+ for (x = 0; x < w; ++x) {
+ int y_q4 = y0_q4;
+ for (y = 0; y < h; ++y) {
+ convolve_line_v(dst + y * dst_stride,
+ &src[(y_q4 >> SUBPEL_BITS) * src_stride], src_stride,
+ y_filters[y_q4 & SUBPEL_MASK]);
+ y_q4 += y_step_q4;
+ }
+ ++src;
+ ++dst;
+ }
+}
+
+static VPX_FORCE_INLINE void convolve_avg_vert(
+ const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, const InterpKernel *y_filters, int y0_q4,
+ int y_step_q4, int w, int h) {
+ int x, y;
+ src -= src_stride * (SUBPEL_TAPS / 2 - 1);
+
+ for (x = 0; x < w; ++x) {
+ int y_q4 = y0_q4;
+ for (y = 0; y < h; ++y) {
+ uint8_t v;
+ convolve_line_v(&v, &src[(y_q4 >> SUBPEL_BITS) * src_stride], src_stride,
+ y_filters[y_q4 & SUBPEL_MASK]);
+ dst[y * dst_stride] = ROUND_POWER_OF_TWO(dst[y * dst_stride] + v, 1);
+ y_q4 += y_step_q4;
+ }
+ ++src;
+ ++dst;
+ }
+}
+
+static VPX_FORCE_INLINE void convolve(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *const filter,
+ int x0_q4, int x_step_q4, int y0_q4,
+ int y_step_q4, int w, int h) {
+ // Note: Fixed size intermediate buffer, temp, places limits on parameters.
+ // 2d filtering proceeds in 2 steps:
+ // (1) Interpolate horizontally into an intermediate buffer, temp.
+ // (2) Interpolate temp vertically to derive the sub-pixel result.
+ // Deriving the maximum number of rows in the temp buffer (135):
+ // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
+ // --Largest block size is 64x64 pixels.
+ // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
+ // original frame (in 1/16th pixel units).
+ // --Must round-up because block may be located at sub-pixel position.
+ // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
+ // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
+ DECLARE_ALIGNED(16, uint8_t, temp[64 * 135]);
+ const int intermediate_height =
+ (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
+
+ assert(w <= 64);
+ assert(h <= 64);
+ assert(y_step_q4 <= 32);
+ assert(x_step_q4 <= 32);
+
+ convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp, 64,
+ filter, x0_q4, x_step_q4, w, intermediate_height);
+ convolve_vert(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst, dst_stride, filter,
+ y0_q4, y_step_q4, w, h);
+}
+
+void vpx_convolve8_horiz_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4,
+ int x_step_q4, int y0_q4, int y_step_q4, int w,
+ int h) {
+ (void)y0_q4;
+ (void)y_step_q4;
+
+ convolve_horiz(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, w,
+ h);
+}
+
+void vpx_convolve8_avg_horiz_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4,
+ int x_step_q4, int y0_q4, int y_step_q4, int w,
+ int h) {
+ (void)y0_q4;
+ (void)y_step_q4;
+
+ convolve_avg_horiz(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4,
+ w, h);
+}
+
+void vpx_convolve8_vert_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4,
+ int x_step_q4, int y0_q4, int y_step_q4, int w,
+ int h) {
+ (void)x0_q4;
+ (void)x_step_q4;
+
+ convolve_vert(src, src_stride, dst, dst_stride, filter, y0_q4, y_step_q4, w,
+ h);
+}
+
+void vpx_convolve8_avg_vert_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4,
+ int x_step_q4, int y0_q4, int y_step_q4, int w,
+ int h) {
+ (void)x0_q4;
+ (void)x_step_q4;
+
+ convolve_avg_vert(src, src_stride, dst, dst_stride, filter, y0_q4, y_step_q4,
+ w, h);
+}
+
+void vpx_convolve8_vsx(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
+ ptrdiff_t dst_stride, const InterpKernel *filter,
+ int x0_q4, int x_step_q4, int y0_q4, int y_step_q4,
+ int w, int h) {
+ convolve(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4,
+ y_step_q4, w, h);
+}
+
+void vpx_convolve8_avg_vsx(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const InterpKernel *filter, int x0_q4, int x_step_q4,
+ int y0_q4, int y_step_q4, int w, int h) {
+ // Fixed size intermediate buffer places limits on parameters.
+ DECLARE_ALIGNED(16, uint8_t, temp[64 * 64]);
+ assert(w <= 64);
+ assert(h <= 64);
+
+ vpx_convolve8_vsx(src, src_stride, temp, 64, filter, x0_q4, x_step_q4, y0_q4,
+ y_step_q4, w, h);
+ vpx_convolve_avg_vsx(temp, 64, dst, dst_stride, NULL, 0, 0, 0, 0, w, h);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-07 06:40:51 +0000