diff options
Diffstat (limited to 'third_party/libwebrtc/modules/audio_mixer')
23 files changed, 2738 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/audio_mixer/BUILD.gn b/third_party/libwebrtc/modules/audio_mixer/BUILD.gn new file mode 100644 index 0000000000..fb038bf677 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/BUILD.gn @@ -0,0 +1,145 @@ +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +group("audio_mixer") { + deps = [ + ":audio_frame_manipulator", + ":audio_mixer_impl", + ] +} + +rtc_library("audio_mixer_impl") { + visibility = [ "*" ] + sources = [ + "audio_mixer_impl.cc", + "audio_mixer_impl.h", + "default_output_rate_calculator.cc", + "default_output_rate_calculator.h", + "frame_combiner.cc", + "frame_combiner.h", + "output_rate_calculator.h", + ] + + public = [ + "audio_mixer_impl.h", + "default_output_rate_calculator.h", # For creating a mixer with limiter + # disabled. + "frame_combiner.h", + ] + + configs += [ "../audio_processing:apm_debug_dump" ] + + deps = [ + ":audio_frame_manipulator", + "../../api:array_view", + "../../api:rtp_packet_info", + "../../api:scoped_refptr", + "../../api/audio:audio_frame_api", + "../../api/audio:audio_mixer_api", + "../../audio/utility:audio_frame_operations", + "../../common_audio", + "../../rtc_base:checks", + "../../rtc_base:event_tracer", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:race_checker", + "../../rtc_base:refcount", + "../../rtc_base:safe_conversions", + "../../rtc_base/synchronization:mutex", + "../../system_wrappers", + "../../system_wrappers:metrics", + "../audio_processing:api", + "../audio_processing:apm_logging", + "../audio_processing:audio_frame_view", + "../audio_processing/agc2:fixed_digital", + ] +} + +rtc_library("audio_frame_manipulator") { + visibility = [ + ":*", + "../../modules:*", + ] + + sources = [ + "audio_frame_manipulator.cc", + "audio_frame_manipulator.h", + ] + + deps = [ + "../../api/audio:audio_frame_api", + "../../audio/utility:audio_frame_operations", + "../../rtc_base:checks", + ] +} + +if (rtc_include_tests) { + rtc_library("audio_mixer_test_utils") { + testonly = true + + sources = [ + "gain_change_calculator.cc", + "gain_change_calculator.h", + "sine_wave_generator.cc", + "sine_wave_generator.h", + ] + + deps = [ + ":audio_frame_manipulator", + ":audio_mixer_impl", + "../../api:array_view", + "../../api/audio:audio_frame_api", + "../../rtc_base:checks", + "../../rtc_base:safe_conversions", + ] + } + + rtc_library("audio_mixer_unittests") { + testonly = true + + sources = [ + "audio_frame_manipulator_unittest.cc", + "audio_mixer_impl_unittest.cc", + "frame_combiner_unittest.cc", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + deps = [ + ":audio_frame_manipulator", + ":audio_mixer_impl", + ":audio_mixer_test_utils", + "../../api:array_view", + "../../api:rtp_packet_info", + "../../api/audio:audio_mixer_api", + "../../api/units:timestamp", + "../../audio/utility:audio_frame_operations", + "../../rtc_base:checks", + "../../rtc_base:stringutils", + "../../rtc_base:task_queue_for_test", + "../../system_wrappers:metrics", + "../../test:test_support", + ] + } + + if (!build_with_chromium) { + rtc_executable("audio_mixer_test") { + testonly = true + sources = [ "audio_mixer_test.cc" ] + + deps = [ + ":audio_mixer_impl", + "../../api/audio:audio_mixer_api", + "../../common_audio", + "../../rtc_base:stringutils", + "//third_party/abseil-cpp/absl/flags:flag", + "//third_party/abseil-cpp/absl/flags:parse", + ] + } + } +} diff --git a/third_party/libwebrtc/modules/audio_mixer/DEPS b/third_party/libwebrtc/modules/audio_mixer/DEPS new file mode 100644 index 0000000000..46f29bccf8 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/DEPS @@ -0,0 +1,13 @@ +include_rules = [ + "+audio/utility/audio_frame_operations.h", + "+audio/utility/channel_mixer.h", + "+call", + "+common_audio", + "+modules/audio_coding", + "+modules/audio_device", + "+modules/audio_processing", + "+modules/pacing", + "+modules/rtp_rtcp", + "+modules/utility", + "+system_wrappers", +] diff --git a/third_party/libwebrtc/modules/audio_mixer/OWNERS b/third_party/libwebrtc/modules/audio_mixer/OWNERS new file mode 100644 index 0000000000..5edc304ab3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/OWNERS @@ -0,0 +1,2 @@ +alessiob@webrtc.org +henrik.lundin@webrtc.org diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.cc b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.cc new file mode 100644 index 0000000000..3100271cfb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_frame_manipulator.h" + +#include "audio/utility/audio_frame_operations.h" +#include "audio/utility/channel_mixer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame) { + if (audio_frame.muted()) { + return 0; + } + + uint32_t energy = 0; + const int16_t* frame_data = audio_frame.data(); + for (size_t position = 0; + position < audio_frame.samples_per_channel_ * audio_frame.num_channels_; + position++) { + // TODO(aleloi): This can overflow. Convert to floats. + energy += frame_data[position] * frame_data[position]; + } + return energy; +} + +void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame) { + RTC_DCHECK(audio_frame); + RTC_DCHECK_GE(start_gain, 0.0f); + RTC_DCHECK_GE(target_gain, 0.0f); + if (start_gain == target_gain || audio_frame->muted()) { + return; + } + + size_t samples = audio_frame->samples_per_channel_; + RTC_DCHECK_LT(0, samples); + float increment = (target_gain - start_gain) / samples; + float gain = start_gain; + int16_t* frame_data = audio_frame->mutable_data(); + for (size_t i = 0; i < samples; ++i) { + // If the audio is interleaved of several channels, we want to + // apply the same gain change to the ith sample of every channel. + for (size_t ch = 0; ch < audio_frame->num_channels_; ++ch) { + frame_data[audio_frame->num_channels_ * i + ch] *= gain; + } + gain += increment; + } +} + +void RemixFrame(size_t target_number_of_channels, AudioFrame* frame) { + RTC_DCHECK_GE(target_number_of_channels, 1); + // TODO(bugs.webrtc.org/10783): take channel layout into account as well. + if (frame->num_channels() == target_number_of_channels) { + return; + } + + // Use legacy components for the most simple cases (mono <-> stereo) to ensure + // that native WebRTC clients are not affected when support for multi-channel + // audio is added to Chrome. + // TODO(bugs.webrtc.org/10783): utilize channel mixer for mono/stereo as well. + if (target_number_of_channels < 3 && frame->num_channels() < 3) { + if (frame->num_channels() > target_number_of_channels) { + AudioFrameOperations::DownmixChannels(target_number_of_channels, frame); + } else { + AudioFrameOperations::UpmixChannels(target_number_of_channels, frame); + } + } else { + // Use generic channel mixer when the number of channels for input our + // output is larger than two. E.g. stereo -> 5.1 channel up-mixing. + // TODO(bugs.webrtc.org/10783): ensure that actual channel layouts are used + // instead of guessing based on number of channels. + const ChannelLayout output_layout( + GuessChannelLayout(target_number_of_channels)); + ChannelMixer mixer(GuessChannelLayout(frame->num_channels()), + output_layout); + mixer.Transform(frame); + RTC_DCHECK_EQ(frame->channel_layout(), output_layout); + } + RTC_DCHECK_EQ(frame->num_channels(), target_number_of_channels) + << "Wrong number of channels, " << frame->num_channels() << " vs " + << target_number_of_channels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.h b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.h new file mode 100644 index 0000000000..ab3633d266 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ +#define MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "api/audio/audio_frame.h" + +namespace webrtc { + +// Updates the audioFrame's energy (based on its samples). +uint32_t AudioMixerCalculateEnergy(const AudioFrame& audio_frame); + +// Ramps up or down the provided audio frame. Ramp(0, 1, frame) will +// linearly increase the samples in the frame from 0 to full volume. +void Ramp(float start_gain, float target_gain, AudioFrame* audio_frame); + +// Downmixes or upmixes a frame between stereo and mono. +void RemixFrame(size_t target_number_of_channels, AudioFrame* frame); + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_AUDIO_FRAME_MANIPULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build new file mode 100644 index 0000000000..edfac56a3a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_gn/moz.build @@ -0,0 +1,236 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["RTC_ENABLE_WIN_WGC"] = True + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "arm": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_frame_manipulator_gn") diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc new file mode 100644 index 0000000000..cfb3f2c230 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_frame_manipulator_unittest.cc @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_frame_manipulator.h" + +#include <algorithm> + +#include "test/gtest.h" + +namespace webrtc { +namespace { + +void FillFrameWithConstants(size_t samples_per_channel, + size_t number_of_channels, + int16_t value, + AudioFrame* frame) { + frame->num_channels_ = number_of_channels; + frame->samples_per_channel_ = samples_per_channel; + int16_t* frame_data = frame->mutable_data(); + std::fill(frame_data, frame_data + samples_per_channel * number_of_channels, + value); +} +} // namespace + +TEST(AudioFrameManipulator, CompareForwardRampWithExpectedResultStereo) { + constexpr int kSamplesPerChannel = 5; + constexpr int kNumberOfChannels = 2; + + // Create a frame with values 5, 5, 5, ... and channels & samples as above. + AudioFrame frame; + FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame); + + Ramp(0.0f, 1.0f, &frame); + + const int total_samples = kSamplesPerChannel * kNumberOfChannels; + const int16_t expected_result[total_samples] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}; + const int16_t* frame_data = frame.data(); + EXPECT_TRUE( + std::equal(frame_data, frame_data + total_samples, expected_result)); +} + +TEST(AudioFrameManipulator, CompareBackwardRampWithExpectedResultMono) { + constexpr int kSamplesPerChannel = 5; + constexpr int kNumberOfChannels = 1; + + // Create a frame with values 5, 5, 5, ... and channels & samples as above. + AudioFrame frame; + FillFrameWithConstants(kSamplesPerChannel, kNumberOfChannels, 5, &frame); + + Ramp(1.0f, 0.0f, &frame); + + const int total_samples = kSamplesPerChannel * kNumberOfChannels; + const int16_t expected_result[total_samples] = {5, 4, 3, 2, 1}; + const int16_t* frame_data = frame.data(); + EXPECT_TRUE( + std::equal(frame_data, frame_data + total_samples, expected_result)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.cc b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.cc new file mode 100644 index 0000000000..faa2b1e1ee --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_mixer_impl.h" + +#include <stdint.h> + +#include <algorithm> +#include <iterator> +#include <type_traits> +#include <utility> + +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "modules/audio_mixer/default_output_rate_calculator.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +struct AudioMixerImpl::SourceStatus { + explicit SourceStatus(Source* audio_source) : audio_source(audio_source) {} + Source* audio_source = nullptr; + + // A frame that will be passed to audio_source->GetAudioFrameWithInfo. + AudioFrame audio_frame; +}; + +namespace { + +std::vector<std::unique_ptr<AudioMixerImpl::SourceStatus>>::const_iterator +FindSourceInList( + AudioMixerImpl::Source const* audio_source, + std::vector<std::unique_ptr<AudioMixerImpl::SourceStatus>> const* + audio_source_list) { + return std::find_if( + audio_source_list->begin(), audio_source_list->end(), + [audio_source](const std::unique_ptr<AudioMixerImpl::SourceStatus>& p) { + return p->audio_source == audio_source; + }); +} +} // namespace + +struct AudioMixerImpl::HelperContainers { + void resize(size_t size) { + audio_to_mix.resize(size); + preferred_rates.resize(size); + } + + std::vector<AudioFrame*> audio_to_mix; + std::vector<int> preferred_rates; +}; + +AudioMixerImpl::AudioMixerImpl( + std::unique_ptr<OutputRateCalculator> output_rate_calculator, + bool use_limiter) + : output_rate_calculator_(std::move(output_rate_calculator)), + audio_source_list_(), + helper_containers_(std::make_unique<HelperContainers>()), + frame_combiner_(use_limiter) {} + +AudioMixerImpl::~AudioMixerImpl() {} + +rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create() { + return Create(std::unique_ptr<DefaultOutputRateCalculator>( + new DefaultOutputRateCalculator()), + /*use_limiter=*/true); +} + +rtc::scoped_refptr<AudioMixerImpl> AudioMixerImpl::Create( + std::unique_ptr<OutputRateCalculator> output_rate_calculator, + bool use_limiter) { + return rtc::make_ref_counted<AudioMixerImpl>( + std::move(output_rate_calculator), use_limiter); +} + +void AudioMixerImpl::Mix(size_t number_of_channels, + AudioFrame* audio_frame_for_mixing) { + TRACE_EVENT0("webrtc", "AudioMixerImpl::Mix"); + RTC_DCHECK(number_of_channels >= 1); + MutexLock lock(&mutex_); + + size_t number_of_streams = audio_source_list_.size(); + + std::transform(audio_source_list_.begin(), audio_source_list_.end(), + helper_containers_->preferred_rates.begin(), + [&](std::unique_ptr<SourceStatus>& a) { + return a->audio_source->PreferredSampleRate(); + }); + + int output_frequency = output_rate_calculator_->CalculateOutputRateFromRange( + rtc::ArrayView<const int>(helper_containers_->preferred_rates.data(), + number_of_streams)); + + frame_combiner_.Combine(GetAudioFromSources(output_frequency), + number_of_channels, output_frequency, + number_of_streams, audio_frame_for_mixing); +} + +bool AudioMixerImpl::AddSource(Source* audio_source) { + RTC_DCHECK(audio_source); + MutexLock lock(&mutex_); + RTC_DCHECK(FindSourceInList(audio_source, &audio_source_list_) == + audio_source_list_.end()) + << "Source already added to mixer"; + audio_source_list_.emplace_back(new SourceStatus(audio_source)); + helper_containers_->resize(audio_source_list_.size()); + UpdateSourceCountStats(); + return true; +} + +void AudioMixerImpl::RemoveSource(Source* audio_source) { + RTC_DCHECK(audio_source); + MutexLock lock(&mutex_); + const auto iter = FindSourceInList(audio_source, &audio_source_list_); + RTC_DCHECK(iter != audio_source_list_.end()) << "Source not present in mixer"; + audio_source_list_.erase(iter); +} + +rtc::ArrayView<AudioFrame* const> AudioMixerImpl::GetAudioFromSources( + int output_frequency) { + int audio_to_mix_count = 0; + for (auto& source_and_status : audio_source_list_) { + const auto audio_frame_info = + source_and_status->audio_source->GetAudioFrameWithInfo( + output_frequency, &source_and_status->audio_frame); + switch (audio_frame_info) { + case Source::AudioFrameInfo::kError: + RTC_LOG_F(LS_WARNING) + << "failed to GetAudioFrameWithInfo() from source"; + break; + case Source::AudioFrameInfo::kMuted: + break; + case Source::AudioFrameInfo::kNormal: + helper_containers_->audio_to_mix[audio_to_mix_count++] = + &source_and_status->audio_frame; + } + } + return rtc::ArrayView<AudioFrame* const>( + helper_containers_->audio_to_mix.data(), audio_to_mix_count); +} + +void AudioMixerImpl::UpdateSourceCountStats() { + size_t current_source_count = audio_source_list_.size(); + // Log to the histogram whenever the maximum number of sources increases. + if (current_source_count > max_source_count_ever_) { + RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.AudioMixer.NewHighestSourceCount", + current_source_count, 1, 20, 20); + max_source_count_ever_ = current_source_count; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.h b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.h new file mode 100644 index 0000000000..8319487018 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ +#define MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ + +#include <stddef.h> + +#include <memory> +#include <vector> + +#include "api/array_view.h" +#include "api/audio/audio_frame.h" +#include "api/audio/audio_mixer.h" +#include "api/scoped_refptr.h" +#include "modules/audio_mixer/frame_combiner.h" +#include "modules/audio_mixer/output_rate_calculator.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class AudioMixerImpl : public AudioMixer { + public: + struct SourceStatus; + + // AudioProcessing only accepts 10 ms frames. + static const int kFrameDurationInMs = 10; + + static rtc::scoped_refptr<AudioMixerImpl> Create(); + + static rtc::scoped_refptr<AudioMixerImpl> Create( + std::unique_ptr<OutputRateCalculator> output_rate_calculator, + bool use_limiter); + + ~AudioMixerImpl() override; + + AudioMixerImpl(const AudioMixerImpl&) = delete; + AudioMixerImpl& operator=(const AudioMixerImpl&) = delete; + + // AudioMixer functions + bool AddSource(Source* audio_source) override; + void RemoveSource(Source* audio_source) override; + + void Mix(size_t number_of_channels, + AudioFrame* audio_frame_for_mixing) override + RTC_LOCKS_EXCLUDED(mutex_); + + protected: + AudioMixerImpl(std::unique_ptr<OutputRateCalculator> output_rate_calculator, + bool use_limiter); + + private: + struct HelperContainers; + + void UpdateSourceCountStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Fetches audio frames to mix from sources. + rtc::ArrayView<AudioFrame* const> GetAudioFromSources(int output_frequency) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // The critical section lock guards audio source insertion and + // removal, which can be done from any thread. The race checker + // checks that mixing is done sequentially. + mutable Mutex mutex_; + + std::unique_ptr<OutputRateCalculator> output_rate_calculator_; + + // List of all audio sources. + std::vector<std::unique_ptr<SourceStatus>> audio_source_list_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr<HelperContainers> helper_containers_ + RTC_GUARDED_BY(mutex_); + + // Component that handles actual adding of audio frames. + FrameCombiner frame_combiner_; + + // The highest source count this mixer has ever had. Used for UMA stats. + size_t max_source_count_ever_ = 0; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_AUDIO_MIXER_IMPL_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build new file mode 100644 index 0000000000..7108d9fbe1 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_gn/moz.build @@ -0,0 +1,239 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_APM_DEBUG_DUMP"] = "1" +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl.cc", + "/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.cc", + "/third_party/libwebrtc/modules/audio_mixer/frame_combiner.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["RTC_ENABLE_WIN_WGC"] = True + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "arm": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["_GNU_SOURCE"] = True + +Library("audio_mixer_impl_gn") diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc new file mode 100644 index 0000000000..2044cb9b90 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -0,0 +1,530 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/audio_mixer_impl.h" + +#include <string.h> + +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/audio/audio_mixer.h" +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/units/timestamp.h" +#include "modules/audio_mixer/default_output_rate_calculator.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/task_queue_for_test.h" +#include "system_wrappers/include/metrics.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::Exactly; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::UnorderedElementsAre; + +namespace webrtc { + +namespace { + +constexpr int kDefaultSampleRateHz = 48000; +const char kSourceCountHistogramName[] = + "WebRTC.Audio.AudioMixer.NewHighestSourceCount"; + +// Utility function that resets the frame member variables with +// sensible defaults. +void ResetFrame(AudioFrame* frame) { + frame->sample_rate_hz_ = kDefaultSampleRateHz; + frame->num_channels_ = 1; + + // Frame duration 10ms. + frame->samples_per_channel_ = kDefaultSampleRateHz / 100; + frame->vad_activity_ = AudioFrame::kVadActive; + frame->speech_type_ = AudioFrame::kNormalSpeech; +} + +std::string ProduceDebugText(int sample_rate_hz, + int number_of_channels, + int number_of_sources) { + rtc::StringBuilder ss; + ss << "Sample rate: " << sample_rate_hz << " "; + ss << "Number of channels: " << number_of_channels << " "; + ss << "Number of sources: " << number_of_sources; + return ss.Release(); +} + +AudioFrame frame_for_mixing; + +} // namespace + +class MockMixerAudioSource : public ::testing::NiceMock<AudioMixer::Source> { + public: + MockMixerAudioSource() + : fake_audio_frame_info_(AudioMixer::Source::AudioFrameInfo::kNormal) { + ON_CALL(*this, GetAudioFrameWithInfo(_, _)) + .WillByDefault( + Invoke(this, &MockMixerAudioSource::FakeAudioFrameWithInfo)); + ON_CALL(*this, PreferredSampleRate()) + .WillByDefault(Return(kDefaultSampleRateHz)); + } + + MOCK_METHOD(AudioFrameInfo, + GetAudioFrameWithInfo, + (int sample_rate_hz, AudioFrame* audio_frame), + (override)); + + MOCK_METHOD(int, PreferredSampleRate, (), (const, override)); + MOCK_METHOD(int, Ssrc, (), (const, override)); + + AudioFrame* fake_frame() { return &fake_frame_; } + AudioFrameInfo fake_info() { return fake_audio_frame_info_; } + void set_fake_info(const AudioFrameInfo audio_frame_info) { + fake_audio_frame_info_ = audio_frame_info; + } + + void set_packet_infos(const RtpPacketInfos& packet_infos) { + packet_infos_ = packet_infos; + } + + private: + AudioFrameInfo FakeAudioFrameWithInfo(int sample_rate_hz, + AudioFrame* audio_frame) { + audio_frame->CopyFrom(fake_frame_); + audio_frame->sample_rate_hz_ = sample_rate_hz; + audio_frame->samples_per_channel_ = + rtc::CheckedDivExact(sample_rate_hz, 100); + audio_frame->packet_infos_ = packet_infos_; + return fake_info(); + } + + AudioFrame fake_frame_; + AudioFrameInfo fake_audio_frame_info_; + RtpPacketInfos packet_infos_; +}; + +class CustomRateCalculator : public OutputRateCalculator { + public: + explicit CustomRateCalculator(int rate) : rate_(rate) {} + int CalculateOutputRateFromRange( + rtc::ArrayView<const int> preferred_rates) override { + return rate_; + } + + private: + const int rate_; +}; + +void MixMonoAtGivenNativeRate(int native_sample_rate, + AudioFrame* mix_frame, + rtc::scoped_refptr<AudioMixer> mixer, + MockMixerAudioSource* audio_source) { + ON_CALL(*audio_source, PreferredSampleRate()) + .WillByDefault(Return(native_sample_rate)); + audio_source->fake_frame()->sample_rate_hz_ = native_sample_rate; + audio_source->fake_frame()->samples_per_channel_ = native_sample_rate / 100; + + mixer->Mix(1, mix_frame); +} + +TEST(AudioMixer, UpdatesSourceCountHistogram) { + metrics::Reset(); + constexpr int kAudioSourcesGroup1 = 5; + constexpr int kAudioSourcesGroup2 = 3; + + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participants[kAudioSourcesGroup1 + kAudioSourcesGroup2]; + + // Add the sources in group 1. + for (int i = 0; i < kAudioSourcesGroup1; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_EQ(i + 1, metrics::NumSamples(kSourceCountHistogramName)); + EXPECT_EQ(1, metrics::NumEvents(kSourceCountHistogramName, i + 1)); + } + // Remove the sources again. + for (int i = 0; i < kAudioSourcesGroup1; ++i) { + mixer->RemoveSource(&participants[i]); + } + // Add the first group again. This should not add anything new to the + // histogram. + for (int i = 0; i < kAudioSourcesGroup1; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_EQ(kAudioSourcesGroup1, + metrics::NumSamples(kSourceCountHistogramName)); + EXPECT_EQ(1, metrics::NumEvents(kSourceCountHistogramName, i + 1)); + } + // Add the second group. This adds to the histogram again. + for (int i = kAudioSourcesGroup1; + i < kAudioSourcesGroup1 + kAudioSourcesGroup2; ++i) { + EXPECT_TRUE(mixer->AddSource(&participants[i])); + EXPECT_EQ(i + 1, metrics::NumSamples(kSourceCountHistogramName)); + EXPECT_EQ(1, metrics::NumEvents(kSourceCountHistogramName, i + 1)); + } +} + +TEST(AudioMixer, FrameNotModifiedForSingleParticipant) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + + ResetFrame(participant.fake_frame()); + const size_t n_samples = participant.fake_frame()->samples_per_channel_; + + // Modify the frame so that it's not zero. + int16_t* fake_frame_data = participant.fake_frame()->mutable_data(); + for (size_t j = 0; j < n_samples; ++j) { + fake_frame_data[j] = static_cast<int16_t>(j); + } + + EXPECT_TRUE(mixer->AddSource(&participant)); + EXPECT_CALL(participant, GetAudioFrameWithInfo(_, _)).Times(Exactly(2)); + + AudioFrame audio_frame; + // Two mix iteration to compare after the ramp-up step. + for (int i = 0; i < 2; ++i) { + mixer->Mix(1, // number of channels + &audio_frame); + } + + EXPECT_EQ(0, memcmp(participant.fake_frame()->data(), audio_frame.data(), + n_samples)); +} + +TEST(AudioMixer, SourceAtNativeRateShouldNeverResample) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource audio_source; + ResetFrame(audio_source.fake_frame()); + + mixer->AddSource(&audio_source); + + for (auto frequency : {8000, 16000, 32000, 48000}) { + EXPECT_CALL(audio_source, GetAudioFrameWithInfo(frequency, _)) + .Times(Exactly(1)); + + MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer, + &audio_source); + } +} + +TEST(AudioMixer, MixerShouldMixAtNativeSourceRate) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource audio_source; + ResetFrame(audio_source.fake_frame()); + + mixer->AddSource(&audio_source); + + for (auto frequency : {8000, 16000, 32000, 48000}) { + MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer, + &audio_source); + + EXPECT_EQ(frequency, frame_for_mixing.sample_rate_hz_); + } +} + +TEST(AudioMixer, MixerShouldAlwaysMixAtNativeRate) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + ResetFrame(participant.fake_frame()); + mixer->AddSource(&participant); + + const int needed_frequency = 44100; + ON_CALL(participant, PreferredSampleRate()) + .WillByDefault(Return(needed_frequency)); + + // We expect mixing frequency to be native and >= needed_frequency. + const int expected_mix_frequency = 48000; + EXPECT_CALL(participant, GetAudioFrameWithInfo(expected_mix_frequency, _)) + .Times(Exactly(1)); + participant.fake_frame()->sample_rate_hz_ = expected_mix_frequency; + participant.fake_frame()->samples_per_channel_ = expected_mix_frequency / 100; + + mixer->Mix(1, &frame_for_mixing); + + EXPECT_EQ(48000, frame_for_mixing.sample_rate_hz_); +} + +// Check that the mixing rate is always >= participants preferred rate. +TEST(AudioMixer, ShouldNotCauseQualityLossForMultipleSources) { + const auto mixer = AudioMixerImpl::Create(); + + std::vector<MockMixerAudioSource> audio_sources(2); + const std::vector<int> source_sample_rates = {8000, 16000}; + for (int i = 0; i < 2; ++i) { + auto& source = audio_sources[i]; + ResetFrame(source.fake_frame()); + mixer->AddSource(&source); + const auto sample_rate = source_sample_rates[i]; + EXPECT_CALL(source, PreferredSampleRate()).WillOnce(Return(sample_rate)); + + EXPECT_CALL(source, GetAudioFrameWithInfo(::testing::Ge(sample_rate), _)); + } + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, ParticipantNumberOfChannels) { + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource participant; + ResetFrame(participant.fake_frame()); + + EXPECT_TRUE(mixer->AddSource(&participant)); + for (size_t number_of_channels : {1, 2}) { + EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + mixer->Mix(number_of_channels, &frame_for_mixing); + EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_); + } +} + +// This test checks that the initialization and participant addition +// can be done on a different thread. +TEST(AudioMixer, ConstructFromOtherThread) { + TaskQueueForTest init_queue("init"); + rtc::scoped_refptr<AudioMixer> mixer; + init_queue.SendTask([&mixer]() { mixer = AudioMixerImpl::Create(); }); + + MockMixerAudioSource participant; + EXPECT_CALL(participant, PreferredSampleRate()) + .WillRepeatedly(Return(kDefaultSampleRateHz)); + + ResetFrame(participant.fake_frame()); + + TaskQueueForTest participant_queue("participant"); + participant_queue.SendTask( + [&mixer, &participant]() { mixer->AddSource(&participant); }); + + EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) + .Times(Exactly(1)); + + // Do one mixer iteration + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, MixingRateShouldBeDecidedByRateCalculator) { + constexpr int kOutputRate = 22000; + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>( + new CustomRateCalculator(kOutputRate)), + true); + MockMixerAudioSource audio_source; + mixer->AddSource(&audio_source); + ResetFrame(audio_source.fake_frame()); + + EXPECT_CALL(audio_source, GetAudioFrameWithInfo(kOutputRate, _)) + .Times(Exactly(1)); + + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, ZeroSourceRateShouldBeDecidedByRateCalculator) { + constexpr int kOutputRate = 8000; + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>( + new CustomRateCalculator(kOutputRate)), + true); + + mixer->Mix(1, &frame_for_mixing); + + EXPECT_EQ(kOutputRate, frame_for_mixing.sample_rate_hz_); +} + +TEST(AudioMixer, NoLimiterBasicApiCalls) { + const auto mixer = AudioMixerImpl::Create( + std::unique_ptr<OutputRateCalculator>(new DefaultOutputRateCalculator()), + false); + mixer->Mix(1, &frame_for_mixing); +} + +TEST(AudioMixer, AnyRateIsPossibleWithNoLimiter) { + // No APM limiter means no AudioProcessing::NativeRate restriction + // on mixing rate. The rate has to be divisible by 100 since we use + // 10 ms frames, though. + for (const auto rate : {8000, 20000, 24000, 32000, 44100}) { + for (const size_t number_of_channels : {1, 2}) { + for (const auto number_of_sources : {0, 1, 2, 3, 4}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_sources, number_of_sources)); + const auto mixer = + AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>( + new CustomRateCalculator(rate)), + false); + + std::vector<MockMixerAudioSource> sources(number_of_sources); + for (auto& source : sources) { + ResetFrame(source.fake_frame()); + mixer->AddSource(&source); + } + + mixer->Mix(number_of_channels, &frame_for_mixing); + EXPECT_EQ(rate, frame_for_mixing.sample_rate_hz_); + EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_); + } + } + } +} + +TEST(AudioMixer, MultipleChannelsOneParticipant) { + // Set up a participant with a 6-channel frame, and make sure a 6-channel + // frame with the right sample values comes out from the mixer. There are 2 + // Mix calls because of ramp-up. + constexpr size_t kNumberOfChannels = 6; + MockMixerAudioSource source; + ResetFrame(source.fake_frame()); + const auto mixer = AudioMixerImpl::Create(); + mixer->AddSource(&source); + mixer->Mix(1, &frame_for_mixing); + auto* frame = source.fake_frame(); + frame->num_channels_ = kNumberOfChannels; + std::fill(frame->mutable_data(), + frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0); + for (size_t i = 0; i < kNumberOfChannels; ++i) { + frame->mutable_data()[100 * frame->num_channels_ + i] = 1000 * i; + } + + mixer->Mix(kNumberOfChannels, &frame_for_mixing); + + EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels); + for (size_t i = 0; i < kNumberOfChannels; ++i) { + EXPECT_EQ(frame_for_mixing.data()[100 * frame_for_mixing.num_channels_ + i], + static_cast<int16_t>(1000 * i)); + } +} + +TEST(AudioMixer, MultipleChannelsManyParticipants) { + // Sets up 2 participants. One has a 6-channel frame. Make sure a 6-channel + // frame with the right sample values comes out from the mixer. There are 2 + // Mix calls because of ramp-up. + constexpr size_t kNumberOfChannels = 6; + MockMixerAudioSource source; + const auto mixer = AudioMixerImpl::Create(); + mixer->AddSource(&source); + ResetFrame(source.fake_frame()); + mixer->Mix(1, &frame_for_mixing); + auto* frame = source.fake_frame(); + frame->num_channels_ = kNumberOfChannels; + std::fill(frame->mutable_data(), + frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0); + for (size_t i = 0; i < kNumberOfChannels; ++i) { + frame->mutable_data()[100 * frame->num_channels_ + i] = 1000 * i; + } + MockMixerAudioSource other_source; + ResetFrame(other_source.fake_frame()); + mixer->AddSource(&other_source); + + mixer->Mix(kNumberOfChannels, &frame_for_mixing); + + EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels); + for (size_t i = 0; i < kNumberOfChannels; ++i) { + EXPECT_EQ(frame_for_mixing.data()[100 * frame_for_mixing.num_channels_ + i], + static_cast<int16_t>(1000 * i)); + } +} + +TEST(AudioMixer, ShouldIncludeRtpPacketInfoFromAllMixedSources) { + const uint32_t kSsrc0 = 10; + const uint32_t kSsrc1 = 11; + const uint32_t kSsrc2 = 12; + const uint32_t kCsrc0 = 20; + const uint32_t kCsrc1 = 21; + const uint32_t kCsrc2 = 22; + const uint32_t kCsrc3 = 23; + const int kAudioLevel0 = 10; + const int kAudioLevel1 = 40; + const absl::optional<uint32_t> kAudioLevel2 = absl::nullopt; + const uint32_t kRtpTimestamp0 = 300; + const uint32_t kRtpTimestamp1 = 400; + const Timestamp kReceiveTime0 = Timestamp::Millis(10); + const Timestamp kReceiveTime1 = Timestamp::Millis(20); + + RtpPacketInfo p0(kSsrc0, {kCsrc0, kCsrc1}, kRtpTimestamp0, kReceiveTime0); + p0.set_audio_level(kAudioLevel0); + RtpPacketInfo p1(kSsrc1, {kCsrc2}, kRtpTimestamp1, kReceiveTime1); + p1.set_audio_level(kAudioLevel1); + RtpPacketInfo p2(kSsrc2, {kCsrc3}, kRtpTimestamp1, kReceiveTime1); + p2.set_audio_level(kAudioLevel2); + + const auto mixer = AudioMixerImpl::Create(); + + MockMixerAudioSource source; + source.set_packet_infos(RtpPacketInfos({p0})); + mixer->AddSource(&source); + ResetFrame(source.fake_frame()); + mixer->Mix(1, &frame_for_mixing); + + MockMixerAudioSource other_source; + other_source.set_packet_infos(RtpPacketInfos({p1, p2})); + ResetFrame(other_source.fake_frame()); + mixer->AddSource(&other_source); + + mixer->Mix(/*number_of_channels=*/1, &frame_for_mixing); + + EXPECT_THAT(frame_for_mixing.packet_infos_, UnorderedElementsAre(p0, p1, p2)); +} + +class HighOutputRateCalculator : public OutputRateCalculator { + public: + static const int kDefaultFrequency = 76000; + int CalculateOutputRateFromRange( + rtc::ArrayView<const int> preferred_sample_rates) override { + return kDefaultFrequency; + } + ~HighOutputRateCalculator() override {} +}; +const int HighOutputRateCalculator::kDefaultFrequency; + +TEST(AudioMixerDeathTest, MultipleChannelsAndHighRate) { + constexpr size_t kSamplesPerChannel = + HighOutputRateCalculator::kDefaultFrequency / 100; + // As many channels as an AudioFrame can fit: + constexpr size_t kNumberOfChannels = + AudioFrame::kMaxDataSizeSamples / kSamplesPerChannel; + MockMixerAudioSource source; + const auto mixer = AudioMixerImpl::Create( + std::make_unique<HighOutputRateCalculator>(), true); + mixer->AddSource(&source); + ResetFrame(source.fake_frame()); + mixer->Mix(1, &frame_for_mixing); + auto* frame = source.fake_frame(); + frame->num_channels_ = kNumberOfChannels; + frame->sample_rate_hz_ = HighOutputRateCalculator::kDefaultFrequency; + frame->samples_per_channel_ = kSamplesPerChannel; + + std::fill(frame->mutable_data(), + frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0); + MockMixerAudioSource other_source; + ResetFrame(other_source.fake_frame()); + auto* other_frame = other_source.fake_frame(); + other_frame->num_channels_ = kNumberOfChannels; + other_frame->sample_rate_hz_ = HighOutputRateCalculator::kDefaultFrequency; + other_frame->samples_per_channel_ = kSamplesPerChannel; + mixer->AddSource(&other_source); + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + EXPECT_DEATH(mixer->Mix(kNumberOfChannels, &frame_for_mixing), ""); +#elif !RTC_DCHECK_IS_ON + mixer->Mix(kNumberOfChannels, &frame_for_mixing); + EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels); + EXPECT_EQ(frame_for_mixing.sample_rate_hz_, + HighOutputRateCalculator::kDefaultFrequency); +#endif +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/audio_mixer_test.cc b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_test.cc new file mode 100644 index 0000000000..3ee28a7937 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/audio_mixer_test.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio/audio_mixer.h" + +#include <cstring> +#include <iostream> +#include <vector> + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "common_audio/wav_file.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_mixer/default_output_rate_calculator.h" +#include "rtc_base/strings/string_builder.h" + +ABSL_FLAG(int, + sampling_rate, + 16000, + "Rate at which to mix (all input streams must have this rate)"); + +ABSL_FLAG(bool, + stereo, + false, + "Enable stereo (interleaved). Inputs need not be as this parameter."); + +ABSL_FLAG(bool, limiter, true, "Enable limiter."); +ABSL_FLAG(std::string, + output_file, + "mixed_file.wav", + "File in which to store the mixed result."); +ABSL_FLAG(std::string, input_file_1, "", "First input. Default none."); +ABSL_FLAG(std::string, input_file_2, "", "Second input. Default none."); +ABSL_FLAG(std::string, input_file_3, "", "Third input. Default none."); +ABSL_FLAG(std::string, input_file_4, "", "Fourth input. Default none."); + +namespace webrtc { +namespace test { + +class FilePlayingSource : public AudioMixer::Source { + public: + explicit FilePlayingSource(absl::string_view filename) + : wav_reader_(new WavReader(filename)), + sample_rate_hz_(wav_reader_->sample_rate()), + samples_per_channel_(sample_rate_hz_ / 100), + number_of_channels_(wav_reader_->num_channels()) {} + + AudioFrameInfo GetAudioFrameWithInfo(int target_rate_hz, + AudioFrame* frame) override { + frame->samples_per_channel_ = samples_per_channel_; + frame->num_channels_ = number_of_channels_; + frame->sample_rate_hz_ = target_rate_hz; + + RTC_CHECK_EQ(target_rate_hz, sample_rate_hz_); + + const size_t num_to_read = number_of_channels_ * samples_per_channel_; + const size_t num_read = + wav_reader_->ReadSamples(num_to_read, frame->mutable_data()); + + file_has_ended_ = num_to_read != num_read; + if (file_has_ended_) { + frame->Mute(); + } + return file_has_ended_ ? AudioFrameInfo::kMuted : AudioFrameInfo::kNormal; + } + + int Ssrc() const override { return 0; } + + int PreferredSampleRate() const override { return sample_rate_hz_; } + + bool FileHasEnded() const { return file_has_ended_; } + + std::string ToString() const { + rtc::StringBuilder ss; + ss << "{rate: " << sample_rate_hz_ << ", channels: " << number_of_channels_ + << ", samples_tot: " << wav_reader_->num_samples() << "}"; + return ss.Release(); + } + + private: + std::unique_ptr<WavReader> wav_reader_; + int sample_rate_hz_; + int samples_per_channel_; + int number_of_channels_; + bool file_has_ended_ = false; +}; +} // namespace test +} // namespace webrtc + +namespace { + +const std::vector<std::string> parse_input_files() { + std::vector<std::string> result; + for (auto& x : + {absl::GetFlag(FLAGS_input_file_1), absl::GetFlag(FLAGS_input_file_2), + absl::GetFlag(FLAGS_input_file_3), absl::GetFlag(FLAGS_input_file_4)}) { + if (!x.empty()) { + result.push_back(x); + } + } + return result; +} +} // namespace + +int main(int argc, char* argv[]) { + absl::ParseCommandLine(argc, argv); + + rtc::scoped_refptr<webrtc::AudioMixerImpl> mixer( + webrtc::AudioMixerImpl::Create( + std::unique_ptr<webrtc::OutputRateCalculator>( + new webrtc::DefaultOutputRateCalculator()), + absl::GetFlag(FLAGS_limiter))); + + const std::vector<std::string> input_files = parse_input_files(); + std::vector<webrtc::test::FilePlayingSource> sources; + const int num_channels = absl::GetFlag(FLAGS_stereo) ? 2 : 1; + sources.reserve(input_files.size()); + for (const auto& input_file : input_files) { + sources.emplace_back(input_file); + } + + for (auto& source : sources) { + auto error = mixer->AddSource(&source); + RTC_CHECK(error); + } + + if (sources.empty()) { + std::cout << "Need at least one source!\n"; + return 1; + } + + const size_t sample_rate = sources[0].PreferredSampleRate(); + for (const auto& source : sources) { + RTC_CHECK_EQ(sample_rate, source.PreferredSampleRate()); + } + + // Print stats. + std::cout << "Limiting is: " << (absl::GetFlag(FLAGS_limiter) ? "on" : "off") + << "\n" + "Channels: " + << num_channels + << "\n" + "Rate: " + << sample_rate + << "\n" + "Number of input streams: " + << input_files.size() << "\n"; + for (const auto& source : sources) { + std::cout << "\t" << source.ToString() << "\n"; + } + std::cout << "Now mixing\n...\n"; + + webrtc::WavWriter wav_writer(absl::GetFlag(FLAGS_output_file), sample_rate, + num_channels); + + webrtc::AudioFrame frame; + + bool all_streams_finished = false; + while (!all_streams_finished) { + mixer->Mix(num_channels, &frame); + RTC_CHECK_EQ(sample_rate / 100, frame.samples_per_channel_); + RTC_CHECK_EQ(sample_rate, frame.sample_rate_hz_); + RTC_CHECK_EQ(num_channels, frame.num_channels_); + wav_writer.WriteSamples(frame.data(), + num_channels * frame.samples_per_channel_); + + all_streams_finished = + std::all_of(sources.begin(), sources.end(), + [](const webrtc::test::FilePlayingSource& source) { + return source.FileHasEnded(); + }); + } + + std::cout << "Done!\n" << std::endl; +} diff --git a/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.cc b/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.cc new file mode 100644 index 0000000000..5f24b653a3 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/default_output_rate_calculator.h" + +#include <algorithm> +#include <iterator> + +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +int DefaultOutputRateCalculator::CalculateOutputRateFromRange( + rtc::ArrayView<const int> preferred_sample_rates) { + if (preferred_sample_rates.empty()) { + return DefaultOutputRateCalculator::kDefaultFrequency; + } + using NativeRate = AudioProcessing::NativeRate; + const int maximal_frequency = *std::max_element( + preferred_sample_rates.cbegin(), preferred_sample_rates.cend()); + + RTC_DCHECK_LE(NativeRate::kSampleRate8kHz, maximal_frequency); + RTC_DCHECK_GE(NativeRate::kSampleRate48kHz, maximal_frequency); + + static constexpr NativeRate native_rates[] = { + NativeRate::kSampleRate8kHz, NativeRate::kSampleRate16kHz, + NativeRate::kSampleRate32kHz, NativeRate::kSampleRate48kHz}; + const auto* rounded_up_index = std::lower_bound( + std::begin(native_rates), std::end(native_rates), maximal_frequency); + RTC_DCHECK(rounded_up_index != std::end(native_rates)); + return *rounded_up_index; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.h b/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.h new file mode 100644 index 0000000000..02a3b5c37b --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/default_output_rate_calculator.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ + +#include <vector> + +#include "api/array_view.h" +#include "modules/audio_mixer/output_rate_calculator.h" + +namespace webrtc { + +class DefaultOutputRateCalculator : public OutputRateCalculator { + public: + static const int kDefaultFrequency = 48000; + + // Produces the least native rate greater or equal to the preferred + // sample rates. A native rate is one in + // AudioProcessing::NativeRate. If `preferred_sample_rates` is + // empty, returns `kDefaultFrequency`. + int CalculateOutputRateFromRange( + rtc::ArrayView<const int> preferred_sample_rates) override; + ~DefaultOutputRateCalculator() override {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_DEFAULT_OUTPUT_RATE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/frame_combiner.cc b/third_party/libwebrtc/modules/audio_mixer/frame_combiner.cc new file mode 100644 index 0000000000..96c62f6b0d --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/frame_combiner.cc @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/frame_combiner.h" + +#include <algorithm> +#include <array> +#include <cstdint> +#include <iterator> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "common_audio/include/audio_util.h" +#include "modules/audio_mixer/audio_frame_manipulator.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_processing/include/audio_frame_view.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "modules/audio_processing/logging/apm_data_dumper.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +using MixingBuffer = + std::array<std::array<float, FrameCombiner::kMaximumChannelSize>, + FrameCombiner::kMaximumNumberOfChannels>; + +void SetAudioFrameFields(rtc::ArrayView<const AudioFrame* const> mix_list, + size_t number_of_channels, + int sample_rate, + size_t number_of_streams, + AudioFrame* audio_frame_for_mixing) { + const size_t samples_per_channel = static_cast<size_t>( + (sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000); + + // TODO(minyue): Issue bugs.webrtc.org/3390. + // Audio frame timestamp. The 'timestamp_' field is set to dummy + // value '0', because it is only supported in the one channel case and + // is then updated in the helper functions. + audio_frame_for_mixing->UpdateFrame( + 0, nullptr, samples_per_channel, sample_rate, AudioFrame::kUndefined, + AudioFrame::kVadUnknown, number_of_channels); + + if (mix_list.empty()) { + audio_frame_for_mixing->elapsed_time_ms_ = -1; + } else { + audio_frame_for_mixing->timestamp_ = mix_list[0]->timestamp_; + audio_frame_for_mixing->elapsed_time_ms_ = mix_list[0]->elapsed_time_ms_; + audio_frame_for_mixing->ntp_time_ms_ = mix_list[0]->ntp_time_ms_; + std::vector<RtpPacketInfo> packet_infos; + for (const auto& frame : mix_list) { + audio_frame_for_mixing->timestamp_ = + std::min(audio_frame_for_mixing->timestamp_, frame->timestamp_); + audio_frame_for_mixing->ntp_time_ms_ = + std::min(audio_frame_for_mixing->ntp_time_ms_, frame->ntp_time_ms_); + audio_frame_for_mixing->elapsed_time_ms_ = std::max( + audio_frame_for_mixing->elapsed_time_ms_, frame->elapsed_time_ms_); + packet_infos.insert(packet_infos.end(), frame->packet_infos_.begin(), + frame->packet_infos_.end()); + } + audio_frame_for_mixing->packet_infos_ = + RtpPacketInfos(std::move(packet_infos)); + } +} + +void MixFewFramesWithNoLimiter(rtc::ArrayView<const AudioFrame* const> mix_list, + AudioFrame* audio_frame_for_mixing) { + if (mix_list.empty()) { + audio_frame_for_mixing->Mute(); + return; + } + RTC_DCHECK_LE(mix_list.size(), 1); + std::copy(mix_list[0]->data(), + mix_list[0]->data() + + mix_list[0]->num_channels_ * mix_list[0]->samples_per_channel_, + audio_frame_for_mixing->mutable_data()); +} + +void MixToFloatFrame(rtc::ArrayView<const AudioFrame* const> mix_list, + size_t samples_per_channel, + size_t number_of_channels, + MixingBuffer* mixing_buffer) { + RTC_DCHECK_LE(samples_per_channel, FrameCombiner::kMaximumChannelSize); + RTC_DCHECK_LE(number_of_channels, FrameCombiner::kMaximumNumberOfChannels); + // Clear the mixing buffer. + *mixing_buffer = {}; + + // Convert to FloatS16 and mix. + for (size_t i = 0; i < mix_list.size(); ++i) { + const AudioFrame* const frame = mix_list[i]; + const int16_t* const frame_data = frame->data(); + for (size_t j = 0; j < std::min(number_of_channels, + FrameCombiner::kMaximumNumberOfChannels); + ++j) { + for (size_t k = 0; k < std::min(samples_per_channel, + FrameCombiner::kMaximumChannelSize); + ++k) { + (*mixing_buffer)[j][k] += frame_data[number_of_channels * k + j]; + } + } + } +} + +void RunLimiter(AudioFrameView<float> mixing_buffer_view, Limiter* limiter) { + const size_t sample_rate = mixing_buffer_view.samples_per_channel() * 1000 / + AudioMixerImpl::kFrameDurationInMs; + // TODO(alessiob): Avoid calling SetSampleRate every time. + limiter->SetSampleRate(sample_rate); + limiter->Process(mixing_buffer_view); +} + +// Both interleaves and rounds. +void InterleaveToAudioFrame(AudioFrameView<const float> mixing_buffer_view, + AudioFrame* audio_frame_for_mixing) { + const size_t number_of_channels = mixing_buffer_view.num_channels(); + const size_t samples_per_channel = mixing_buffer_view.samples_per_channel(); + int16_t* const mixing_data = audio_frame_for_mixing->mutable_data(); + // Put data in the result frame. + for (size_t i = 0; i < number_of_channels; ++i) { + for (size_t j = 0; j < samples_per_channel; ++j) { + mixing_data[number_of_channels * j + i] = + FloatS16ToS16(mixing_buffer_view.channel(i)[j]); + } + } +} +} // namespace + +constexpr size_t FrameCombiner::kMaximumNumberOfChannels; +constexpr size_t FrameCombiner::kMaximumChannelSize; + +FrameCombiner::FrameCombiner(bool use_limiter) + : data_dumper_(new ApmDataDumper(0)), + mixing_buffer_( + std::make_unique<std::array<std::array<float, kMaximumChannelSize>, + kMaximumNumberOfChannels>>()), + limiter_(static_cast<size_t>(48000), data_dumper_.get(), "AudioMixer"), + use_limiter_(use_limiter) { + static_assert(kMaximumChannelSize * kMaximumNumberOfChannels <= + AudioFrame::kMaxDataSizeSamples, + ""); +} + +FrameCombiner::~FrameCombiner() = default; + +void FrameCombiner::Combine(rtc::ArrayView<AudioFrame* const> mix_list, + size_t number_of_channels, + int sample_rate, + size_t number_of_streams, + AudioFrame* audio_frame_for_mixing) { + RTC_DCHECK(audio_frame_for_mixing); + + SetAudioFrameFields(mix_list, number_of_channels, sample_rate, + number_of_streams, audio_frame_for_mixing); + + const size_t samples_per_channel = static_cast<size_t>( + (sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000); + + for (const auto* frame : mix_list) { + RTC_DCHECK_EQ(samples_per_channel, frame->samples_per_channel_); + RTC_DCHECK_EQ(sample_rate, frame->sample_rate_hz_); + } + + // The 'num_channels_' field of frames in 'mix_list' could be + // different from 'number_of_channels'. + for (auto* frame : mix_list) { + RemixFrame(number_of_channels, frame); + } + + if (number_of_streams <= 1) { + MixFewFramesWithNoLimiter(mix_list, audio_frame_for_mixing); + return; + } + + MixToFloatFrame(mix_list, samples_per_channel, number_of_channels, + mixing_buffer_.get()); + + const size_t output_number_of_channels = + std::min(number_of_channels, kMaximumNumberOfChannels); + const size_t output_samples_per_channel = + std::min(samples_per_channel, kMaximumChannelSize); + + // Put float data in an AudioFrameView. + std::array<float*, kMaximumNumberOfChannels> channel_pointers{}; + for (size_t i = 0; i < output_number_of_channels; ++i) { + channel_pointers[i] = &(*mixing_buffer_.get())[i][0]; + } + AudioFrameView<float> mixing_buffer_view(&channel_pointers[0], + output_number_of_channels, + output_samples_per_channel); + + if (use_limiter_) { + RunLimiter(mixing_buffer_view, &limiter_); + } + + InterleaveToAudioFrame(mixing_buffer_view, audio_frame_for_mixing); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/frame_combiner.h b/third_party/libwebrtc/modules/audio_mixer/frame_combiner.h new file mode 100644 index 0000000000..6185b29f8a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/frame_combiner.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ +#define MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ + +#include <memory> +#include <vector> + +#include "api/array_view.h" +#include "api/audio/audio_frame.h" +#include "modules/audio_processing/agc2/limiter.h" + +namespace webrtc { +class ApmDataDumper; + +class FrameCombiner { + public: + explicit FrameCombiner(bool use_limiter); + ~FrameCombiner(); + + // Combine several frames into one. Assumes sample_rate, + // samples_per_channel of the input frames match the parameters. The + // parameters 'number_of_channels' and 'sample_rate' are needed + // because 'mix_list' can be empty. The parameter + // 'number_of_streams' is used for determining whether to pass the + // data through a limiter. + void Combine(rtc::ArrayView<AudioFrame* const> mix_list, + size_t number_of_channels, + int sample_rate, + size_t number_of_streams, + AudioFrame* audio_frame_for_mixing); + + // Stereo, 48 kHz, 10 ms. + static constexpr size_t kMaximumNumberOfChannels = 8; + static constexpr size_t kMaximumChannelSize = 48 * 10; + + using MixingBuffer = std::array<std::array<float, kMaximumChannelSize>, + kMaximumNumberOfChannels>; + + private: + std::unique_ptr<ApmDataDumper> data_dumper_; + std::unique_ptr<MixingBuffer> mixing_buffer_; + Limiter limiter_; + const bool use_limiter_; +}; +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_FRAME_COMBINER_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/frame_combiner_unittest.cc b/third_party/libwebrtc/modules/audio_mixer/frame_combiner_unittest.cc new file mode 100644 index 0000000000..6c64d0852a --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/frame_combiner_unittest.cc @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/frame_combiner.h" + +#include <cstdint> +#include <initializer_list> +#include <numeric> +#include <string> +#include <type_traits> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/units/timestamp.h" +#include "audio/utility/audio_frame_operations.h" +#include "modules/audio_mixer/gain_change_calculator.h" +#include "modules/audio_mixer/sine_wave_generator.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +using ::testing::ElementsAreArray; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAreArray; + +struct FrameCombinerConfig { + bool use_limiter; + int sample_rate_hz; + int number_of_channels; + float wave_frequency; +}; + +std::string ProduceDebugText(int sample_rate_hz, + int number_of_channels, + int number_of_sources) { + rtc::StringBuilder ss; + ss << "Sample rate: " << sample_rate_hz << " ,"; + ss << "number of channels: " << number_of_channels << " ,"; + ss << "number of sources: " << number_of_sources; + return ss.Release(); +} + +std::string ProduceDebugText(const FrameCombinerConfig& config) { + rtc::StringBuilder ss; + ss << "Sample rate: " << config.sample_rate_hz << " ,"; + ss << "number of channels: " << config.number_of_channels << " ,"; + ss << "limiter active: " << (config.use_limiter ? "on" : "off") << " ,"; + ss << "wave frequency: " << config.wave_frequency << " ,"; + return ss.Release(); +} + +AudioFrame frame1; +AudioFrame frame2; + +void SetUpFrames(int sample_rate_hz, int number_of_channels) { + RtpPacketInfo packet_info1(/*ssrc=*/1001, /*csrcs=*/{}, + /*rtp_timestamp=*/1000, + /*receive_time=*/Timestamp::Millis(1)); + RtpPacketInfo packet_info2(/*ssrc=*/4004, /*csrcs=*/{}, + /*rtp_timestamp=*/1234, + /*receive_time=*/Timestamp::Millis(2)); + RtpPacketInfo packet_info3(/*ssrc=*/7007, /*csrcs=*/{}, + /*rtp_timestamp=*/1333, + /*receive_time=*/Timestamp::Millis(2)); + + frame1.packet_infos_ = RtpPacketInfos({packet_info1}); + frame2.packet_infos_ = RtpPacketInfos({packet_info2, packet_info3}); + + for (auto* frame : {&frame1, &frame2}) { + frame->UpdateFrame(0, nullptr, rtc::CheckedDivExact(sample_rate_hz, 100), + sample_rate_hz, AudioFrame::kNormalSpeech, + AudioFrame::kVadActive, number_of_channels); + } +} +} // namespace + +// The limiter requires sample rate divisible by 2000. +TEST(FrameCombiner, BasicApiCallsLimiter) { + FrameCombiner combiner(true); + for (const int rate : {8000, 18000, 34000, 48000}) { + for (const int number_of_channels : {1, 2, 4, 8}) { + const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + for (const int number_of_frames : {0, 1, 2}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector<AudioFrame*> frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + AudioFrame audio_frame_for_mixing; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + } + } + } +} + +// The RtpPacketInfos field of the mixed packet should contain the union of the +// RtpPacketInfos from the frames that were actually mixed. +TEST(FrameCombiner, ContainsAllRtpPacketInfos) { + static constexpr int kSampleRateHz = 48000; + static constexpr int kNumChannels = 1; + FrameCombiner combiner(true); + const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; + SetUpFrames(kSampleRateHz, kNumChannels); + + for (const int number_of_frames : {0, 1, 2}) { + SCOPED_TRACE( + ProduceDebugText(kSampleRateHz, kNumChannels, number_of_frames)); + const std::vector<AudioFrame*> frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + + std::vector<RtpPacketInfo> packet_infos; + for (const auto& frame : frames_to_combine) { + packet_infos.insert(packet_infos.end(), frame->packet_infos_.begin(), + frame->packet_infos_.end()); + } + + AudioFrame audio_frame_for_mixing; + combiner.Combine(frames_to_combine, kNumChannels, kSampleRateHz, + frames_to_combine.size(), &audio_frame_for_mixing); + EXPECT_THAT(audio_frame_for_mixing.packet_infos_, + UnorderedElementsAreArray(packet_infos)); + } +} + +// There are DCHECKs in place to check for invalid parameters. +TEST(FrameCombinerDeathTest, DebugBuildCrashesWithManyChannels) { + FrameCombiner combiner(true); + for (const int rate : {8000, 18000, 34000, 48000}) { + for (const int number_of_channels : {10, 20, 21}) { + if (static_cast<size_t>(rate / 100 * number_of_channels) > + AudioFrame::kMaxDataSizeSamples) { + continue; + } + const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + const int number_of_frames = 2; + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector<AudioFrame*> frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + AudioFrame audio_frame_for_mixing; +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + EXPECT_DEATH( + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing), + ""); +#elif !RTC_DCHECK_IS_ON + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); +#endif + } + } +} + +TEST(FrameCombinerDeathTest, DebugBuildCrashesWithHighRate) { + FrameCombiner combiner(true); + for (const int rate : {50000, 96000, 128000, 196000}) { + for (const int number_of_channels : {1, 2, 3}) { + if (static_cast<size_t>(rate / 100 * number_of_channels) > + AudioFrame::kMaxDataSizeSamples) { + continue; + } + const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + const int number_of_frames = 2; + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector<AudioFrame*> frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + AudioFrame audio_frame_for_mixing; +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + EXPECT_DEATH( + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing), + ""); +#elif !RTC_DCHECK_IS_ON + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); +#endif + } + } +} + +// With no limiter, the rate has to be divisible by 100 since we use +// 10 ms frames. +TEST(FrameCombiner, BasicApiCallsNoLimiter) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2, 4, 8}) { + const std::vector<AudioFrame*> all_frames = {&frame1, &frame2}; + SetUpFrames(rate, number_of_channels); + + for (const int number_of_frames : {0, 1, 2}) { + SCOPED_TRACE( + ProduceDebugText(rate, number_of_channels, number_of_frames)); + const std::vector<AudioFrame*> frames_to_combine( + all_frames.begin(), all_frames.begin() + number_of_frames); + AudioFrame audio_frame_for_mixing; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + } + } + } +} + +TEST(FrameCombiner, CombiningZeroFramesShouldProduceSilence) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2}) { + SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 0)); + + AudioFrame audio_frame_for_mixing; + + const std::vector<AudioFrame*> frames_to_combine; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + const int16_t* audio_frame_for_mixing_data = + audio_frame_for_mixing.data(); + const std::vector<int16_t> mixed_data( + audio_frame_for_mixing_data, + audio_frame_for_mixing_data + number_of_channels * rate / 100); + + const std::vector<int16_t> expected(number_of_channels * rate / 100, 0); + EXPECT_EQ(mixed_data, expected); + EXPECT_THAT(audio_frame_for_mixing.packet_infos_, IsEmpty()); + } + } +} + +TEST(FrameCombiner, CombiningOneFrameShouldNotChangeFrame) { + FrameCombiner combiner(false); + for (const int rate : {8000, 10000, 11000, 32000, 44100}) { + for (const int number_of_channels : {1, 2, 4, 8, 10}) { + SCOPED_TRACE(ProduceDebugText(rate, number_of_channels, 1)); + + AudioFrame audio_frame_for_mixing; + + SetUpFrames(rate, number_of_channels); + int16_t* frame1_data = frame1.mutable_data(); + std::iota(frame1_data, frame1_data + number_of_channels * rate / 100, 0); + const std::vector<AudioFrame*> frames_to_combine = {&frame1}; + combiner.Combine(frames_to_combine, number_of_channels, rate, + frames_to_combine.size(), &audio_frame_for_mixing); + + const int16_t* audio_frame_for_mixing_data = + audio_frame_for_mixing.data(); + const std::vector<int16_t> mixed_data( + audio_frame_for_mixing_data, + audio_frame_for_mixing_data + number_of_channels * rate / 100); + + std::vector<int16_t> expected(number_of_channels * rate / 100); + std::iota(expected.begin(), expected.end(), 0); + EXPECT_EQ(mixed_data, expected); + EXPECT_THAT(audio_frame_for_mixing.packet_infos_, + ElementsAreArray(frame1.packet_infos_)); + } + } +} + +// Send a sine wave through the FrameCombiner, and check that the +// difference between input and output varies smoothly. Also check +// that it is inside reasonable bounds. This is to catch issues like +// chromium:695993 and chromium:816875. +TEST(FrameCombiner, GainCurveIsSmoothForAlternatingNumberOfStreams) { + // Rates are divisible by 2000 when limiter is active. + std::vector<FrameCombinerConfig> configs = { + {false, 30100, 2, 50.f}, {false, 16500, 1, 3200.f}, + {true, 8000, 1, 3200.f}, {true, 16000, 1, 50.f}, + {true, 18000, 8, 3200.f}, {true, 10000, 2, 50.f}, + }; + + for (const auto& config : configs) { + SCOPED_TRACE(ProduceDebugText(config)); + + FrameCombiner combiner(config.use_limiter); + + constexpr int16_t wave_amplitude = 30000; + SineWaveGenerator wave_generator(config.wave_frequency, wave_amplitude); + + GainChangeCalculator change_calculator; + float cumulative_change = 0.f; + + constexpr size_t iterations = 100; + + for (size_t i = 0; i < iterations; ++i) { + SetUpFrames(config.sample_rate_hz, config.number_of_channels); + wave_generator.GenerateNextFrame(&frame1); + AudioFrameOperations::Mute(&frame2); + + std::vector<AudioFrame*> frames_to_combine = {&frame1}; + if (i % 2 == 0) { + frames_to_combine.push_back(&frame2); + } + const size_t number_of_samples = + frame1.samples_per_channel_ * config.number_of_channels; + + // Ensures limiter is on if 'use_limiter'. + constexpr size_t number_of_streams = 2; + AudioFrame audio_frame_for_mixing; + combiner.Combine(frames_to_combine, config.number_of_channels, + config.sample_rate_hz, number_of_streams, + &audio_frame_for_mixing); + cumulative_change += change_calculator.CalculateGainChange( + rtc::ArrayView<const int16_t>(frame1.data(), number_of_samples), + rtc::ArrayView<const int16_t>(audio_frame_for_mixing.data(), + number_of_samples)); + } + + // Check that the gain doesn't vary too much. + EXPECT_LT(cumulative_change, 10); + + // Check that the latest gain is within reasonable bounds. It + // should be slightly less that 1. + EXPECT_LT(0.9f, change_calculator.LatestGain()); + EXPECT_LT(change_calculator.LatestGain(), 1.01f); + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/g3doc/index.md b/third_party/libwebrtc/modules/audio_mixer/g3doc/index.md new file mode 100644 index 0000000000..6b48378fcb --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/g3doc/index.md @@ -0,0 +1,54 @@ +<!-- go/cmark --> +<!--* freshness: {owner: 'alessiob' reviewed: '2021-04-21'} *--> + +# The WebRTC Audio Mixer Module + +The WebRTC audio mixer module is responsible for mixing multiple incoming audio +streams (sources) into a single audio stream (mix). It works with 10 ms frames, +it supports sample rates up to 48 kHz and up to 8 audio channels. The API is +defined in +[`api/audio/audio_mixer.h`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/audio/audio_mixer.h) +and it includes the definition of +[`AudioMixer::Source`](https://source.chromium.org/search?q=symbol:AudioMixer::Source%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h), +which describes an incoming audio stream, and the definition of +[`AudioMixer`](https://source.chromium.org/search?q=symbol:AudioMixer%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h), +which operates on a collection of +[`AudioMixer::Source`](https://source.chromium.org/search?q=symbol:AudioMixer::Source%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h) +objects to produce a mix. + +## AudioMixer::Source + +A source has different characteristic (e.g., sample rate, number of channels, +muted state) and it is identified by an SSRC[^1]. +[`AudioMixer::Source::GetAudioFrameWithInfo()`](https://source.chromium.org/search?q=symbol:AudioMixer::Source::GetAudioFrameWithInfo%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h) +is used to retrieve the next 10 ms chunk of audio to be mixed. + +[^1]: A synchronization source (SSRC) is the source of a stream of RTP packets, + identified by a 32-bit numeric SSRC identifier carried in the RTP header + so as not to be dependent upon the network address (see + [RFC 3550](https://tools.ietf.org/html/rfc3550#section-3)). + +## AudioMixer + +The interface allows to add and remove sources and the +[`AudioMixer::Mix()`](https://source.chromium.org/search?q=symbol:AudioMixer::Mix%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h) +method allows to generates a mix with the desired number of channels. + +## WebRTC implementation + +The interface is implemented in different parts of WebRTC: + +* [`AudioMixer::Source`](https://source.chromium.org/search?q=symbol:AudioMixer::Source%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h): + [`audio/audio_receive_stream.h`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/audio/audio_receive_stream.h) +* [`AudioMixer`](https://source.chromium.org/search?q=symbol:AudioMixer%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h): + [`modules/audio_mixer/audio_mixer_impl.h`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/audio_mixer/audio_mixer_impl.h) + +[`AudioMixer`](https://source.chromium.org/search?q=symbol:AudioMixer%20file:third_party%2Fwebrtc%2Fapi%2Faudio%2Faudio_mixer.h) +is thread-safe. The output sample rate of the generated mix is automatically +assigned depending on the sample rate of the sources; whereas the number of +output channels is defined by the caller[^2]. Samples from the non-muted sources +are summed up and then a limiter is used to apply soft-clipping when needed. + +[^2]: [`audio/utility/channel_mixer.h`](https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/audio/utility/channel_mixer.h) + is used to mix channels in the non-trivial cases - i.e., if the number of + channels for a source or the mix is greater than 3. diff --git a/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.cc b/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.cc new file mode 100644 index 0000000000..dbd0945239 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/gain_change_calculator.h" + +#include <math.h> + +#include <cstdlib> +#include <vector> + +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { +constexpr int16_t kReliabilityThreshold = 100; +} // namespace + +float GainChangeCalculator::CalculateGainChange( + rtc::ArrayView<const int16_t> in, + rtc::ArrayView<const int16_t> out) { + RTC_DCHECK_EQ(in.size(), out.size()); + + std::vector<float> gain(in.size()); + CalculateGain(in, out, gain); + return CalculateDifferences(gain); +} + +float GainChangeCalculator::LatestGain() const { + return last_reliable_gain_; +} + +void GainChangeCalculator::CalculateGain(rtc::ArrayView<const int16_t> in, + rtc::ArrayView<const int16_t> out, + rtc::ArrayView<float> gain) { + RTC_DCHECK_EQ(in.size(), out.size()); + RTC_DCHECK_EQ(in.size(), gain.size()); + + for (size_t i = 0; i < in.size(); ++i) { + if (std::abs(in[i]) >= kReliabilityThreshold) { + last_reliable_gain_ = out[i] / static_cast<float>(in[i]); + } + gain[i] = last_reliable_gain_; + } +} + +float GainChangeCalculator::CalculateDifferences( + rtc::ArrayView<const float> values) { + float res = 0; + for (float f : values) { + res += fabs(f - last_value_); + last_value_ = f; + } + return res; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.h b/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.h new file mode 100644 index 0000000000..3dde9be61e --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/gain_change_calculator.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ + +#include <stdint.h> + +#include "api/array_view.h" + +namespace webrtc { + +class GainChangeCalculator { + public: + // The 'out' signal is assumed to be produced from 'in' by applying + // a smoothly varying gain. This method computes variations of the + // gain and handles special cases when the samples are small. + float CalculateGainChange(rtc::ArrayView<const int16_t> in, + rtc::ArrayView<const int16_t> out); + + float LatestGain() const; + + private: + void CalculateGain(rtc::ArrayView<const int16_t> in, + rtc::ArrayView<const int16_t> out, + rtc::ArrayView<float> gain); + + float CalculateDifferences(rtc::ArrayView<const float> values); + float last_value_ = 0.f; + float last_reliable_gain_ = 1.0f; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_GAIN_CHANGE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/output_rate_calculator.h b/third_party/libwebrtc/modules/audio_mixer/output_rate_calculator.h new file mode 100644 index 0000000000..46b65a8b57 --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/output_rate_calculator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ +#define MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ + +#include <vector> + +#include "api/array_view.h" + +namespace webrtc { + +// Decides the sample rate of a mixing iteration given the preferred +// sample rates of the sources. +class OutputRateCalculator { + public: + virtual int CalculateOutputRateFromRange( + rtc::ArrayView<const int> preferred_sample_rates) = 0; + + virtual ~OutputRateCalculator() {} +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_OUTPUT_RATE_CALCULATOR_H_ diff --git a/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.cc b/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.cc new file mode 100644 index 0000000000..591fe14e8c --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_mixer/sine_wave_generator.h" + +#include <math.h> +#include <stddef.h> + +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +namespace { +constexpr float kPi = 3.14159265f; +} // namespace + +void SineWaveGenerator::GenerateNextFrame(AudioFrame* frame) { + RTC_DCHECK(frame); + int16_t* frame_data = frame->mutable_data(); + for (size_t i = 0; i < frame->samples_per_channel_; ++i) { + for (size_t ch = 0; ch < frame->num_channels_; ++ch) { + frame_data[frame->num_channels_ * i + ch] = + rtc::saturated_cast<int16_t>(amplitude_ * sinf(phase_)); + } + phase_ += wave_frequency_hz_ * 2 * kPi / frame->sample_rate_hz_; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.h b/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.h new file mode 100644 index 0000000000..ec0fcd24bd --- /dev/null +++ b/third_party/libwebrtc/modules/audio_mixer/sine_wave_generator.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ +#define MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ + +#include <stdint.h> + +#include "api/audio/audio_frame.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +class SineWaveGenerator { + public: + SineWaveGenerator(float wave_frequency_hz, int16_t amplitude) + : wave_frequency_hz_(wave_frequency_hz), amplitude_(amplitude) { + RTC_DCHECK_GT(wave_frequency_hz, 0); + } + + // Produces appropriate output based on frame->num_channels_, + // frame->sample_rate_hz_. + void GenerateNextFrame(AudioFrame* frame); + + private: + float phase_ = 0.f; + const float wave_frequency_hz_; + const int16_t amplitude_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_MIXER_SINE_WAVE_GENERATOR_H_ |