diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_processing/util')
12 files changed, 1043 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.cc b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.cc new file mode 100644 index 0000000000..0e1570114a --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/denoiser_filter.h" + +#include "modules/video_processing/util/denoiser_filter_c.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/arch.h" +#include "system_wrappers/include/cpu_features_wrapper.h" + +#if defined(WEBRTC_ARCH_X86_FAMILY) +#include "modules/video_processing/util/denoiser_filter_sse2.h" +#elif defined(WEBRTC_HAS_NEON) +#include "modules/video_processing/util/denoiser_filter_neon.h" +#endif + +namespace webrtc { + +const int kMotionMagnitudeThreshold = 8 * 3; +const int kSumDiffThreshold = 96; +const int kSumDiffThresholdHigh = 448; + +std::unique_ptr<DenoiserFilter> DenoiserFilter::Create( + bool runtime_cpu_detection, + CpuType* cpu_type) { + std::unique_ptr<DenoiserFilter> filter; + + if (cpu_type != nullptr) + *cpu_type = CPU_NOT_NEON; + if (runtime_cpu_detection) { +// If we know the minimum architecture at compile time, avoid CPU detection. +#if defined(WEBRTC_ARCH_X86_FAMILY) +#if defined(__SSE2__) + filter.reset(new DenoiserFilterSSE2()); +#else + // x86 CPU detection required. + if (GetCPUInfo(kSSE2)) { + filter.reset(new DenoiserFilterSSE2()); + } else { + filter.reset(new DenoiserFilterC()); + } +#endif +#elif defined(WEBRTC_HAS_NEON) + filter.reset(new DenoiserFilterNEON()); + if (cpu_type != nullptr) + *cpu_type = CPU_NEON; +#else + filter.reset(new DenoiserFilterC()); +#endif + } else { + filter.reset(new DenoiserFilterC()); + } + + RTC_DCHECK(filter.get() != nullptr); + return filter; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.h b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.h new file mode 100644 index 0000000000..1d574f4a4f --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_H_ + +#include <stdint.h> + +#include <memory> + +namespace webrtc { + +extern const int kMotionMagnitudeThreshold; +extern const int kSumDiffThreshold; +extern const int kSumDiffThresholdHigh; + +enum DenoiserDecision { COPY_BLOCK, FILTER_BLOCK }; +enum CpuType { CPU_NEON, CPU_NOT_NEON }; + +class DenoiserFilter { + public: + static std::unique_ptr<DenoiserFilter> Create(bool runtime_cpu_detection, + CpuType* cpu_type); + + virtual ~DenoiserFilter() {} + virtual uint32_t Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + unsigned int* sse) = 0; + virtual DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_H_ diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.cc b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.cc new file mode 100644 index 0000000000..55c0ea7b35 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/denoiser_filter_c.h" + +#include <stdlib.h> +#include <string.h> + +namespace webrtc { + +uint32_t DenoiserFilterC::Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + uint32_t* sse) { + int sum = 0; + *sse = 0; + a_stride <<= 1; + b_stride <<= 1; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 16; j++) { + const int diff = a[j] - b[j]; + sum += diff; + *sse += diff * diff; + } + + a += a_stride; + b += b_stride; + } + return *sse - ((static_cast<int64_t>(sum) * sum) >> 7); +} + +DenoiserDecision DenoiserFilterC::MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) { + int sum_diff_thresh = 0; + int sum_diff = 0; + int adj_val[3] = {3, 4, 6}; + int shift_inc1 = 0; + int shift_inc2 = 1; + int col_sum[16] = {0}; + if (motion_magnitude <= kMotionMagnitudeThreshold) { + if (increase_denoising) { + shift_inc1 = 1; + shift_inc2 = 2; + } + adj_val[0] += shift_inc2; + adj_val[1] += shift_inc2; + adj_val[2] += shift_inc2; + } + + for (int r = 0; r < 16; ++r) { + for (int c = 0; c < 16; ++c) { + int diff = 0; + int adjustment = 0; + int absdiff = 0; + + diff = mc_running_avg_y[c] - sig[c]; + absdiff = abs(diff); + + // When `diff` <= |3 + shift_inc1|, use pixel value from + // last denoised raw. + if (absdiff <= 3 + shift_inc1) { + running_avg_y[c] = mc_running_avg_y[c]; + col_sum[c] += diff; + } else { + if (absdiff >= 4 + shift_inc1 && absdiff <= 7) + adjustment = adj_val[0]; + else if (absdiff >= 8 && absdiff <= 15) + adjustment = adj_val[1]; + else + adjustment = adj_val[2]; + + if (diff > 0) { + if ((sig[c] + adjustment) > 255) + running_avg_y[c] = 255; + else + running_avg_y[c] = sig[c] + adjustment; + + col_sum[c] += adjustment; + } else { + if ((sig[c] - adjustment) < 0) + running_avg_y[c] = 0; + else + running_avg_y[c] = sig[c] - adjustment; + + col_sum[c] -= adjustment; + } + } + } + + // Update pointers for next iteration. + sig += sig_stride; + mc_running_avg_y += mc_avg_y_stride; + running_avg_y += avg_y_stride; + } + + for (int c = 0; c < 16; ++c) { + if (col_sum[c] >= 128) { + col_sum[c] = 127; + } + sum_diff += col_sum[c]; + } + + sum_diff_thresh = + increase_denoising ? kSumDiffThresholdHigh : kSumDiffThreshold; + if (abs(sum_diff) > sum_diff_thresh) + return COPY_BLOCK; + + return FILTER_BLOCK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.h b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.h new file mode 100644 index 0000000000..5633c171f0 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_c.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_C_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_C_H_ + +#include <stdint.h> + +#include "modules/video_processing/util/denoiser_filter.h" + +namespace webrtc { + +class DenoiserFilterC : public DenoiserFilter { + public: + DenoiserFilterC() {} + uint32_t Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + unsigned int* sse) override; + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) override; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_C_H_ diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.cc b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.cc new file mode 100644 index 0000000000..e1e6ed4f18 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/denoiser_filter_neon.h" + +#include <arm_neon.h> + +namespace webrtc { + +const int kSumDiffThresholdHighNeon = 600; + +static int HorizontalAddS16x8(const int16x8_t v_16x8) { + const int32x4_t a = vpaddlq_s16(v_16x8); + const int64x2_t b = vpaddlq_s32(a); + const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), + vreinterpret_s32_s64(vget_high_s64(b))); + return vget_lane_s32(c, 0); +} + +static int HorizontalAddS32x4(const int32x4_t v_32x4) { + const int64x2_t b = vpaddlq_s32(v_32x4); + const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)), + vreinterpret_s32_s64(vget_high_s64(b))); + return vget_lane_s32(c, 0); +} + +static void VarianceNeonW8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + int w, + int h, + uint32_t* sse, + int64_t* sum) { + int16x8_t v_sum = vdupq_n_s16(0); + int32x4_t v_sse_lo = vdupq_n_s32(0); + int32x4_t v_sse_hi = vdupq_n_s32(0); + + for (int i = 0; i < h; ++i) { + for (int j = 0; j < w; j += 8) { + const uint8x8_t v_a = vld1_u8(&a[j]); + const uint8x8_t v_b = vld1_u8(&b[j]); + const uint16x8_t v_diff = vsubl_u8(v_a, v_b); + const int16x8_t sv_diff = vreinterpretq_s16_u16(v_diff); + v_sum = vaddq_s16(v_sum, sv_diff); + v_sse_lo = + vmlal_s16(v_sse_lo, vget_low_s16(sv_diff), vget_low_s16(sv_diff)); + v_sse_hi = + vmlal_s16(v_sse_hi, vget_high_s16(sv_diff), vget_high_s16(sv_diff)); + } + a += a_stride; + b += b_stride; + } + + *sum = HorizontalAddS16x8(v_sum); + *sse = + static_cast<uint32_t>(HorizontalAddS32x4(vaddq_s32(v_sse_lo, v_sse_hi))); +} + +uint32_t DenoiserFilterNEON::Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + uint32_t* sse) { + int64_t sum = 0; + VarianceNeonW8(a, a_stride << 1, b, b_stride << 1, 16, 8, sse, &sum); + return *sse - ((sum * sum) >> 7); +} + +DenoiserDecision DenoiserFilterNEON::MbDenoise(const uint8_t* mc_running_avg_y, + int mc_running_avg_y_stride, + uint8_t* running_avg_y, + int running_avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) { + // If motion_magnitude is small, making the denoiser more aggressive by + // increasing the adjustment for each level, level1 adjustment is + // increased, the deltas stay the same. + int shift_inc = + (increase_denoising && motion_magnitude <= kMotionMagnitudeThreshold) ? 1 + : 0; + int sum_diff_thresh = 0; + const uint8x16_t v_level1_adjustment = vmovq_n_u8( + (motion_magnitude <= kMotionMagnitudeThreshold) ? 4 + shift_inc : 3); + const uint8x16_t v_delta_level_1_and_2 = vdupq_n_u8(1); + const uint8x16_t v_delta_level_2_and_3 = vdupq_n_u8(2); + const uint8x16_t v_level1_threshold = vmovq_n_u8(4 + shift_inc); + const uint8x16_t v_level2_threshold = vdupq_n_u8(8); + const uint8x16_t v_level3_threshold = vdupq_n_u8(16); + int64x2_t v_sum_diff_total = vdupq_n_s64(0); + + // Go over lines. + for (int r = 0; r < 16; ++r) { + // Load inputs. + const uint8x16_t v_sig = vld1q_u8(sig); + const uint8x16_t v_mc_running_avg_y = vld1q_u8(mc_running_avg_y); + + // Calculate absolute difference and sign masks. + const uint8x16_t v_abs_diff = vabdq_u8(v_sig, v_mc_running_avg_y); + const uint8x16_t v_diff_pos_mask = vcltq_u8(v_sig, v_mc_running_avg_y); + const uint8x16_t v_diff_neg_mask = vcgtq_u8(v_sig, v_mc_running_avg_y); + + // Figure out which level that put us in. + const uint8x16_t v_level1_mask = vcleq_u8(v_level1_threshold, v_abs_diff); + const uint8x16_t v_level2_mask = vcleq_u8(v_level2_threshold, v_abs_diff); + const uint8x16_t v_level3_mask = vcleq_u8(v_level3_threshold, v_abs_diff); + + // Calculate absolute adjustments for level 1, 2 and 3. + const uint8x16_t v_level2_adjustment = + vandq_u8(v_level2_mask, v_delta_level_1_and_2); + const uint8x16_t v_level3_adjustment = + vandq_u8(v_level3_mask, v_delta_level_2_and_3); + const uint8x16_t v_level1and2_adjustment = + vaddq_u8(v_level1_adjustment, v_level2_adjustment); + const uint8x16_t v_level1and2and3_adjustment = + vaddq_u8(v_level1and2_adjustment, v_level3_adjustment); + + // Figure adjustment absolute value by selecting between the absolute + // difference if in level0 or the value for level 1, 2 and 3. + const uint8x16_t v_abs_adjustment = + vbslq_u8(v_level1_mask, v_level1and2and3_adjustment, v_abs_diff); + + // Calculate positive and negative adjustments. Apply them to the signal + // and accumulate them. Adjustments are less than eight and the maximum + // sum of them (7 * 16) can fit in a signed char. + const uint8x16_t v_pos_adjustment = + vandq_u8(v_diff_pos_mask, v_abs_adjustment); + const uint8x16_t v_neg_adjustment = + vandq_u8(v_diff_neg_mask, v_abs_adjustment); + + uint8x16_t v_running_avg_y = vqaddq_u8(v_sig, v_pos_adjustment); + v_running_avg_y = vqsubq_u8(v_running_avg_y, v_neg_adjustment); + + // Store results. + vst1q_u8(running_avg_y, v_running_avg_y); + + // Sum all the accumulators to have the sum of all pixel differences + // for this macroblock. + { + const int8x16_t v_sum_diff = + vqsubq_s8(vreinterpretq_s8_u8(v_pos_adjustment), + vreinterpretq_s8_u8(v_neg_adjustment)); + const int16x8_t fe_dc_ba_98_76_54_32_10 = vpaddlq_s8(v_sum_diff); + const int32x4_t fedc_ba98_7654_3210 = + vpaddlq_s16(fe_dc_ba_98_76_54_32_10); + const int64x2_t fedcba98_76543210 = vpaddlq_s32(fedc_ba98_7654_3210); + + v_sum_diff_total = vqaddq_s64(v_sum_diff_total, fedcba98_76543210); + } + + // Update pointers for next iteration. + sig += sig_stride; + mc_running_avg_y += mc_running_avg_y_stride; + running_avg_y += running_avg_y_stride; + } + + // Too much adjustments => copy block. + int64x1_t x = vqadd_s64(vget_high_s64(v_sum_diff_total), + vget_low_s64(v_sum_diff_total)); + int sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0); + sum_diff_thresh = + increase_denoising ? kSumDiffThresholdHighNeon : kSumDiffThreshold; + if (sum_diff > sum_diff_thresh) + return COPY_BLOCK; + + // Tell above level that block was filtered. + running_avg_y -= running_avg_y_stride * 16; + sig -= sig_stride * 16; + + return FILTER_BLOCK; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.h b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.h new file mode 100644 index 0000000000..4d9f271e5a --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_neon.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_NEON_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_NEON_H_ + +#include "modules/video_processing/util/denoiser_filter.h" + +namespace webrtc { + +class DenoiserFilterNEON : public DenoiserFilter { + public: + DenoiserFilterNEON() {} + uint32_t Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + unsigned int* sse) override; + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) override; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_NEON_H_ diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.cc b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.cc new file mode 100644 index 0000000000..5ca5f0cf34 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.cc @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/denoiser_filter_sse2.h" + +#include <emmintrin.h> +#include <stdlib.h> +#include <string.h> + +namespace webrtc { + +static void Get8x8varSse2(const uint8_t* src, + int src_stride, + const uint8_t* ref, + int ref_stride, + unsigned int* sse, + int* sum) { + const __m128i zero = _mm_setzero_si128(); + __m128i vsum = _mm_setzero_si128(); + __m128i vsse = _mm_setzero_si128(); + + for (int i = 0; i < 8; i += 2) { + const __m128i src0 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i*)(src + i * src_stride)), zero); + const __m128i ref0 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i*)(ref + i * ref_stride)), zero); + const __m128i diff0 = _mm_sub_epi16(src0, ref0); + + const __m128i src1 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i*)(src + (i + 1) * src_stride)), zero); + const __m128i ref1 = _mm_unpacklo_epi8( + _mm_loadl_epi64((const __m128i*)(ref + (i + 1) * ref_stride)), zero); + const __m128i diff1 = _mm_sub_epi16(src1, ref1); + + vsum = _mm_add_epi16(vsum, diff0); + vsum = _mm_add_epi16(vsum, diff1); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff0, diff0)); + vsse = _mm_add_epi32(vsse, _mm_madd_epi16(diff1, diff1)); + } + + // sum + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 8)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 4)); + vsum = _mm_add_epi16(vsum, _mm_srli_si128(vsum, 2)); + *sum = static_cast<int16_t>(_mm_extract_epi16(vsum, 0)); + + // sse + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 8)); + vsse = _mm_add_epi32(vsse, _mm_srli_si128(vsse, 4)); + *sse = _mm_cvtsi128_si32(vsse); +} + +static void VarianceSSE2(const unsigned char* src, + int src_stride, + const unsigned char* ref, + int ref_stride, + int w, + int h, + uint32_t* sse, + int64_t* sum, + int block_size) { + *sse = 0; + *sum = 0; + + for (int i = 0; i < h; i += block_size) { + for (int j = 0; j < w; j += block_size) { + uint32_t sse0 = 0; + int32_t sum0 = 0; + + Get8x8varSse2(src + src_stride * i + j, src_stride, + ref + ref_stride * i + j, ref_stride, &sse0, &sum0); + *sse += sse0; + *sum += sum0; + } + } +} + +// Compute the sum of all pixel differences of this MB. +static uint32_t AbsSumDiff16x1(__m128i acc_diff) { + const __m128i k_1 = _mm_set1_epi16(1); + const __m128i acc_diff_lo = + _mm_srai_epi16(_mm_unpacklo_epi8(acc_diff, acc_diff), 8); + const __m128i acc_diff_hi = + _mm_srai_epi16(_mm_unpackhi_epi8(acc_diff, acc_diff), 8); + const __m128i acc_diff_16 = _mm_add_epi16(acc_diff_lo, acc_diff_hi); + const __m128i hg_fe_dc_ba = _mm_madd_epi16(acc_diff_16, k_1); + const __m128i hgfe_dcba = + _mm_add_epi32(hg_fe_dc_ba, _mm_srli_si128(hg_fe_dc_ba, 8)); + const __m128i hgfedcba = + _mm_add_epi32(hgfe_dcba, _mm_srli_si128(hgfe_dcba, 4)); + unsigned int sum_diff = abs(_mm_cvtsi128_si32(hgfedcba)); + + return sum_diff; +} + +uint32_t DenoiserFilterSSE2::Variance16x8(const uint8_t* src, + int src_stride, + const uint8_t* ref, + int ref_stride, + uint32_t* sse) { + int64_t sum = 0; + VarianceSSE2(src, src_stride << 1, ref, ref_stride << 1, 16, 8, sse, &sum, 8); + return *sse - ((sum * sum) >> 7); +} + +DenoiserDecision DenoiserFilterSSE2::MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) { + DenoiserDecision decision = FILTER_BLOCK; + unsigned int sum_diff_thresh = 0; + int shift_inc = + (increase_denoising && motion_magnitude <= kMotionMagnitudeThreshold) ? 1 + : 0; + __m128i acc_diff = _mm_setzero_si128(); + const __m128i k_0 = _mm_setzero_si128(); + const __m128i k_4 = _mm_set1_epi8(4 + shift_inc); + const __m128i k_8 = _mm_set1_epi8(8); + const __m128i k_16 = _mm_set1_epi8(16); + // Modify each level's adjustment according to motion_magnitude. + const __m128i l3 = _mm_set1_epi8( + (motion_magnitude <= kMotionMagnitudeThreshold) ? 7 + shift_inc : 6); + // Difference between level 3 and level 2 is 2. + const __m128i l32 = _mm_set1_epi8(2); + // Difference between level 2 and level 1 is 1. + const __m128i l21 = _mm_set1_epi8(1); + + for (int r = 0; r < 16; ++r) { + // Calculate differences. + const __m128i v_sig = + _mm_loadu_si128(reinterpret_cast<const __m128i*>(&sig[0])); + const __m128i v_mc_running_avg_y = + _mm_loadu_si128(reinterpret_cast<const __m128i*>(&mc_running_avg_y[0])); + __m128i v_running_avg_y; + const __m128i pdiff = _mm_subs_epu8(v_mc_running_avg_y, v_sig); + const __m128i ndiff = _mm_subs_epu8(v_sig, v_mc_running_avg_y); + // Obtain the sign. FF if diff is negative. + const __m128i diff_sign = _mm_cmpeq_epi8(pdiff, k_0); + // Clamp absolute difference to 16 to be used to get mask. Doing this + // allows us to use _mm_cmpgt_epi8, which operates on signed byte. + const __m128i clamped_absdiff = + _mm_min_epu8(_mm_or_si128(pdiff, ndiff), k_16); + // Get masks for l2 l1 and l0 adjustments. + const __m128i mask2 = _mm_cmpgt_epi8(k_16, clamped_absdiff); + const __m128i mask1 = _mm_cmpgt_epi8(k_8, clamped_absdiff); + const __m128i mask0 = _mm_cmpgt_epi8(k_4, clamped_absdiff); + // Get adjustments for l2, l1, and l0. + __m128i adj2 = _mm_and_si128(mask2, l32); + const __m128i adj1 = _mm_and_si128(mask1, l21); + const __m128i adj0 = _mm_and_si128(mask0, clamped_absdiff); + __m128i adj, padj, nadj; + + // Combine the adjustments and get absolute adjustments. + adj2 = _mm_add_epi8(adj2, adj1); + adj = _mm_sub_epi8(l3, adj2); + adj = _mm_andnot_si128(mask0, adj); + adj = _mm_or_si128(adj, adj0); + + // Restore the sign and get positive and negative adjustments. + padj = _mm_andnot_si128(diff_sign, adj); + nadj = _mm_and_si128(diff_sign, adj); + + // Calculate filtered value. + v_running_avg_y = _mm_adds_epu8(v_sig, padj); + v_running_avg_y = _mm_subs_epu8(v_running_avg_y, nadj); + _mm_storeu_si128(reinterpret_cast<__m128i*>(running_avg_y), + v_running_avg_y); + + // Adjustments <=7, and each element in acc_diff can fit in signed + // char. + acc_diff = _mm_adds_epi8(acc_diff, padj); + acc_diff = _mm_subs_epi8(acc_diff, nadj); + + // Update pointers for next iteration. + sig += sig_stride; + mc_running_avg_y += mc_avg_y_stride; + running_avg_y += avg_y_stride; + } + + // Compute the sum of all pixel differences of this MB. + unsigned int abs_sum_diff = AbsSumDiff16x1(acc_diff); + sum_diff_thresh = + increase_denoising ? kSumDiffThresholdHigh : kSumDiffThreshold; + if (abs_sum_diff > sum_diff_thresh) + decision = COPY_BLOCK; + return decision; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.h b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.h new file mode 100644 index 0000000000..8fe4b905ae --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/denoiser_filter_sse2.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_SSE2_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_SSE2_H_ + +#include <stdint.h> + +#include "modules/video_processing/util/denoiser_filter.h" + +namespace webrtc { + +class DenoiserFilterSSE2 : public DenoiserFilter { + public: + DenoiserFilterSSE2() {} + uint32_t Variance16x8(const uint8_t* a, + int a_stride, + const uint8_t* b, + int b_stride, + unsigned int* sse) override; + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, + int mc_avg_y_stride, + uint8_t* running_avg_y, + int avg_y_stride, + const uint8_t* sig, + int sig_stride, + uint8_t motion_magnitude, + int increase_denoising) override; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_SSE2_H_ diff --git a/third_party/libwebrtc/modules/video_processing/util/noise_estimation.cc b/third_party/libwebrtc/modules/video_processing/util/noise_estimation.cc new file mode 100644 index 0000000000..c72f764901 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/noise_estimation.cc @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/noise_estimation.h" +#if DISPLAYNEON +#include <android/log.h> +#endif + +namespace webrtc { + +void NoiseEstimation::Init(int width, int height, CpuType cpu_type) { + int mb_cols = width >> 4; + int mb_rows = height >> 4; + consec_low_var_.reset(new uint32_t[mb_cols * mb_rows]()); + width_ = width; + height_ = height; + mb_cols_ = width_ >> 4; + mb_rows_ = height_ >> 4; + cpu_type_ = cpu_type; +} + +void NoiseEstimation::GetNoise(int mb_index, uint32_t var, uint32_t luma) { + consec_low_var_[mb_index]++; + num_static_block_++; + if (consec_low_var_[mb_index] >= kConsecLowVarFrame && + (luma >> 6) < kAverageLumaMax && (luma >> 6) > kAverageLumaMin) { + // Normalized var by the average luma value, this gives more weight to + // darker blocks. + int nor_var = var / (luma >> 10); + noise_var_ += + nor_var > kBlockSelectionVarMax ? kBlockSelectionVarMax : nor_var; + num_noisy_block_++; + } +} + +void NoiseEstimation::ResetConsecLowVar(int mb_index) { + consec_low_var_[mb_index] = 0; +} + +void NoiseEstimation::UpdateNoiseLevel() { + // TODO(jackychen): Tune a threshold for numb_noisy_block > T to make the + // condition more reasonable. + // No enough samples implies the motion of the camera or too many moving + // objects in the frame. + if (num_static_block_ < + (0.65 * mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL) || + !num_noisy_block_) { +#if DISPLAY + printf("Not enough samples. %d \n", num_static_block_); +#elif DISPLAYNEON + __android_log_print(ANDROID_LOG_DEBUG, "DISPLAY", + "Not enough samples. %d \n", num_static_block_); +#endif + noise_var_ = 0; + noise_var_accum_ = 0; + num_noisy_block_ = 0; + num_static_block_ = 0; + return; + } else { +#if DISPLAY + printf("%d %d fraction = %.3f\n", num_static_block_, + mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL, + percent_static_block_); +#elif DISPLAYNEON + __android_log_print(ANDROID_LOG_DEBUG, "DISPLAY", "%d %d fraction = %.3f\n", + num_static_block_, + mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL, + percent_static_block_); +#endif + // Normalized by the number of noisy blocks. + noise_var_ /= num_noisy_block_; + // Get the percentage of static blocks. + percent_static_block_ = static_cast<double>(num_static_block_) / + (mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL); + num_noisy_block_ = 0; + num_static_block_ = 0; + } + // For the first frame just update the value with current noise_var_, + // otherwise, use the averaging window. + if (noise_var_accum_ == 0) { + noise_var_accum_ = noise_var_; + } else { + noise_var_accum_ = (noise_var_accum_ * 15 + noise_var_) / 16; + } +#if DISPLAY + printf("noise_var_accum_ = %.1f, noise_var_ = %d.\n", noise_var_accum_, + noise_var_); +#elif DISPLAYNEON + __android_log_print(ANDROID_LOG_DEBUG, "DISPLAY", + "noise_var_accum_ = %.1f, noise_var_ = %d.\n", + noise_var_accum_, noise_var_); +#endif + // Reset noise_var_ for the next frame. + noise_var_ = 0; +} + +uint8_t NoiseEstimation::GetNoiseLevel() { + int noise_thr = cpu_type_ ? kNoiseThreshold : kNoiseThresholdNeon; + UpdateNoiseLevel(); + if (noise_var_accum_ > noise_thr) { + return 1; + } + return 0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/noise_estimation.h b/third_party/libwebrtc/modules/video_processing/util/noise_estimation.h new file mode 100644 index 0000000000..4c5f10f1d8 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/noise_estimation.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_ + +#include <cstdint> +#include <memory> + +#include "modules/video_processing/util/denoiser_filter.h" + +namespace webrtc { + +#define DISPLAY 0 // Rectangle diagnostics +#define DISPLAYNEON 0 // Rectangle diagnostics on NEON + +const int kNoiseThreshold = 150; +const int kNoiseThresholdNeon = 70; +const int kConsecLowVarFrame = 6; +const int kAverageLumaMin = 20; +const int kAverageLumaMax = 220; +const int kBlockSelectionVarMax = kNoiseThreshold << 1; + +// TODO(jackychen): To test different sampling strategy. +// Collect noise data every NOISE_SUBSAMPLE_INTERVAL blocks. +#define NOISE_SUBSAMPLE_INTERVAL 41 + +class NoiseEstimation { + public: + void Init(int width, int height, CpuType cpu_type); + // Collect noise data from one qualified block. + void GetNoise(int mb_index, uint32_t var, uint32_t luma); + // Reset the counter for consecutive low-var blocks. + void ResetConsecLowVar(int mb_index); + // Update noise level for current frame. + void UpdateNoiseLevel(); + // 0: low noise, 1: high noise + uint8_t GetNoiseLevel(); + + private: + int width_; + int height_; + int mb_rows_; + int mb_cols_; + int num_noisy_block_; + int num_static_block_; + CpuType cpu_type_; + uint32_t noise_var_; + double noise_var_accum_; + double percent_static_block_; + std::unique_ptr<uint32_t[]> consec_low_var_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_ diff --git a/third_party/libwebrtc/modules/video_processing/util/skin_detection.cc b/third_party/libwebrtc/modules/video_processing/util/skin_detection.cc new file mode 100644 index 0000000000..76399d6e73 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/skin_detection.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_processing/util/skin_detection.h" + +namespace webrtc { + +// Fixed-point skin color model parameters. +static const int skin_mean[5][2] = {{7463, 9614}, + {6400, 10240}, + {7040, 10240}, + {8320, 9280}, + {6800, 9614}}; +static const int skin_inv_cov[4] = {4107, 1663, 1663, 2157}; // q16 +static const int skin_threshold[6] = {1570636, 1400000, 800000, + 800000, 800000, 800000}; // q18 + +// Thresholds on luminance. +static const int y_low = 40; +static const int y_high = 220; + +// Evaluates the Mahalanobis distance measure for the input CbCr values. +static int EvaluateSkinColorDifference(int cb, int cr, int idx) { + const int cb_q6 = cb << 6; + const int cr_q6 = cr << 6; + const int cb_diff_q12 = + (cb_q6 - skin_mean[idx][0]) * (cb_q6 - skin_mean[idx][0]); + const int cbcr_diff_q12 = + (cb_q6 - skin_mean[idx][0]) * (cr_q6 - skin_mean[idx][1]); + const int cr_diff_q12 = + (cr_q6 - skin_mean[idx][1]) * (cr_q6 - skin_mean[idx][1]); + const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10; + const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10; + const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10; + const int skin_diff = + skin_inv_cov[0] * cb_diff_q2 + skin_inv_cov[1] * cbcr_diff_q2 + + skin_inv_cov[2] * cbcr_diff_q2 + skin_inv_cov[3] * cr_diff_q2; + return skin_diff; +} + +static int SkinPixel(const uint8_t y, const uint8_t cb, const uint8_t cr) { + if (y < y_low || y > y_high) { + return 0; + } else { + if (MODEL_MODE == 0) { + return (EvaluateSkinColorDifference(cb, cr, 0) < skin_threshold[0]); + } else { + // Exit on grey. + if (cb == 128 && cr == 128) + return 0; + // Exit on very strong cb. + if (cb > 150 && cr < 110) + return 0; + // Exit on (another) low luminance threshold if either color is high. + if (y < 50 && (cb > 140 || cr > 140)) + return 0; + for (int i = 0; i < 5; i++) { + int diff = EvaluateSkinColorDifference(cb, cr, i); + if (diff < skin_threshold[i + 1]) { + return 1; + } else if (diff > (skin_threshold[i + 1] << 3)) { + // Exit if difference is much large than the threshold. + return 0; + } + } + return 0; + } + } +} + +bool MbHasSkinColor(const uint8_t* y_src, + const uint8_t* u_src, + const uint8_t* v_src, + const int stride_y, + const int stride_u, + const int stride_v, + const int mb_row, + const int mb_col) { + const uint8_t* y = y_src + ((mb_row << 4) + 8) * stride_y + (mb_col << 4) + 8; + const uint8_t* u = u_src + ((mb_row << 3) + 4) * stride_u + (mb_col << 3) + 4; + const uint8_t* v = v_src + ((mb_row << 3) + 4) * stride_v + (mb_col << 3) + 4; + // Use 2x2 average of center pixel to compute skin area. + uint8_t y_avg = (*y + *(y + 1) + *(y + stride_y) + *(y + stride_y + 1)) >> 2; + uint8_t u_avg = (*u + *(u + 1) + *(u + stride_u) + *(u + stride_u + 1)) >> 2; + uint8_t v_avg = (*v + *(v + 1) + *(v + stride_v) + *(v + stride_v + 1)) >> 2; + return SkinPixel(y_avg, u_avg, v_avg) == 1; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_processing/util/skin_detection.h b/third_party/libwebrtc/modules/video_processing/util/skin_detection.h new file mode 100644 index 0000000000..7f2e17aa87 --- /dev/null +++ b/third_party/libwebrtc/modules/video_processing/util/skin_detection.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_PROCESSING_UTIL_SKIN_DETECTION_H_ +#define MODULES_VIDEO_PROCESSING_UTIL_SKIN_DETECTION_H_ + +namespace webrtc { + +#define MODEL_MODE 0 + +typedef unsigned char uint8_t; +bool MbHasSkinColor(const uint8_t* y_src, + const uint8_t* u_src, + const uint8_t* v_src, + int stride_y, + int stride_u, + int stride_v, + int mb_row, + int mb_col); + +} // namespace webrtc + +#endif // MODULES_VIDEO_PROCESSING_UTIL_SKIN_DETECTION_H_ |