/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include #include #include #include #include #include "absl/strings/string_view.h" #include "api/array_view.h" #include "api/numerics/samples_stats_counter.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/metrics/metric.h" #include "modules/audio_processing/audio_processing_impl.h" #include "modules/audio_processing/test/audio_processing_builder_for_testing.h" #include "modules/audio_processing/test/test_utils.h" #include "rtc_base/event.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/platform_thread.h" #include "rtc_base/random.h" #include "system_wrappers/include/clock.h" #include "test/gtest.h" namespace webrtc { namespace { using ::webrtc::test::GetGlobalMetricsLogger; using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Metric; using ::webrtc::test::Unit; class CallSimulator; // Type of the render thread APM API call to use in the test. enum class ProcessorType { kRender, kCapture }; // Variant of APM processing settings to use in the test. enum class SettingsType { kDefaultApmDesktop, kDefaultApmMobile, kAllSubmodulesTurnedOff, kDefaultApmDesktopWithoutDelayAgnostic, kDefaultApmDesktopWithoutExtendedFilter }; // Variables related to the audio data and formats. struct AudioFrameData { explicit AudioFrameData(size_t max_frame_size) { // Set up the two-dimensional arrays needed for the APM API calls. input_framechannels.resize(2 * max_frame_size); input_frame.resize(2); input_frame[0] = &input_framechannels[0]; input_frame[1] = &input_framechannels[max_frame_size]; output_frame_channels.resize(2 * max_frame_size); output_frame.resize(2); output_frame[0] = &output_frame_channels[0]; output_frame[1] = &output_frame_channels[max_frame_size]; } std::vector output_frame_channels; std::vector output_frame; std::vector input_framechannels; std::vector input_frame; StreamConfig input_stream_config; StreamConfig output_stream_config; }; // The configuration for the test. struct SimulationConfig { SimulationConfig(int sample_rate_hz, SettingsType simulation_settings) : sample_rate_hz(sample_rate_hz), simulation_settings(simulation_settings) {} static std::vector GenerateSimulationConfigs() { std::vector simulation_configs; #ifndef WEBRTC_ANDROID const SettingsType desktop_settings[] = { SettingsType::kDefaultApmDesktop, SettingsType::kAllSubmodulesTurnedOff, SettingsType::kDefaultApmDesktopWithoutDelayAgnostic, SettingsType::kDefaultApmDesktopWithoutExtendedFilter}; const int desktop_sample_rates[] = {8000, 16000, 32000, 48000}; for (auto sample_rate : desktop_sample_rates) { for (auto settings : desktop_settings) { simulation_configs.push_back(SimulationConfig(sample_rate, settings)); } } #endif const SettingsType mobile_settings[] = {SettingsType::kDefaultApmMobile}; const int mobile_sample_rates[] = {8000, 16000}; for (auto sample_rate : mobile_sample_rates) { for (auto settings : mobile_settings) { simulation_configs.push_back(SimulationConfig(sample_rate, settings)); } } return simulation_configs; } std::string SettingsDescription() const { std::string description; switch (simulation_settings) { case SettingsType::kDefaultApmMobile: description = "DefaultApmMobile"; break; case SettingsType::kDefaultApmDesktop: description = "DefaultApmDesktop"; break; case SettingsType::kAllSubmodulesTurnedOff: description = "AllSubmodulesOff"; break; case SettingsType::kDefaultApmDesktopWithoutDelayAgnostic: description = "DefaultApmDesktopWithoutDelayAgnostic"; break; case SettingsType::kDefaultApmDesktopWithoutExtendedFilter: description = "DefaultApmDesktopWithoutExtendedFilter"; break; } return description; } int sample_rate_hz = 16000; SettingsType simulation_settings = SettingsType::kDefaultApmDesktop; }; // Handler for the frame counters. class FrameCounters { public: void IncreaseRenderCounter() { render_count_.fetch_add(1); } void IncreaseCaptureCounter() { capture_count_.fetch_add(1); } int CaptureMinusRenderCounters() const { // The return value will be approximate, but that's good enough since // by the time we return the value, it's not guaranteed to be correct // anyway. return capture_count_.load(std::memory_order_acquire) - render_count_.load(std::memory_order_acquire); } int RenderMinusCaptureCounters() const { return -CaptureMinusRenderCounters(); } bool BothCountersExceedeThreshold(int threshold) const { // TODO(tommi): We could use an event to signal this so that we don't need // to be polling from the main thread and possibly steal cycles. const int capture_count = capture_count_.load(std::memory_order_acquire); const int render_count = render_count_.load(std::memory_order_acquire); return (render_count > threshold && capture_count > threshold); } private: std::atomic render_count_{0}; std::atomic capture_count_{0}; }; // Class that represents a flag that can only be raised. class LockedFlag { public: bool get_flag() const { return flag_.load(std::memory_order_acquire); } void set_flag() { if (!get_flag()) { // read-only operation to avoid affecting the cache-line. int zero = 0; flag_.compare_exchange_strong(zero, 1); } } private: std::atomic flag_{0}; }; // Parent class for the thread processors. class TimedThreadApiProcessor { public: TimedThreadApiProcessor(ProcessorType processor_type, Random* rand_gen, FrameCounters* shared_counters_state, LockedFlag* capture_call_checker, CallSimulator* test_framework, const SimulationConfig* simulation_config, AudioProcessing* apm, int num_durations_to_store, float input_level, int num_channels) : rand_gen_(rand_gen), frame_counters_(shared_counters_state), capture_call_checker_(capture_call_checker), test_(test_framework), simulation_config_(simulation_config), apm_(apm), frame_data_(kMaxFrameSize), clock_(webrtc::Clock::GetRealTimeClock()), num_durations_to_store_(num_durations_to_store), api_call_durations_(num_durations_to_store_ - kNumInitializationFrames), samples_count_(0), input_level_(input_level), processor_type_(processor_type), num_channels_(num_channels) {} // Implements the callback functionality for the threads. bool Process(); // Method for printing out the simulation statistics. void print_processor_statistics(absl::string_view processor_name) const { const std::string modifier = "_api_call_duration"; const std::string sample_rate_name = "_" + std::to_string(simulation_config_->sample_rate_hz) + "Hz"; GetGlobalMetricsLogger()->LogMetric( "apm_timing" + sample_rate_name, processor_name, api_call_durations_, Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); } void AddDuration(int64_t duration) { if (samples_count_ >= kNumInitializationFrames && samples_count_ < num_durations_to_store_) { api_call_durations_.AddSample(duration); } samples_count_++; } private: static const int kMaxCallDifference = 10; static const int kMaxFrameSize = 480; static const int kNumInitializationFrames = 5; int ProcessCapture() { // Set the stream delay. apm_->set_stream_delay_ms(30); // Call and time the specified capture side API processing method. const int64_t start_time = clock_->TimeInMicroseconds(); const int result = apm_->ProcessStream( &frame_data_.input_frame[0], frame_data_.input_stream_config, frame_data_.output_stream_config, &frame_data_.output_frame[0]); const int64_t end_time = clock_->TimeInMicroseconds(); frame_counters_->IncreaseCaptureCounter(); AddDuration(end_time - start_time); if (first_process_call_) { // Flag that the capture side has been called at least once // (needed to ensure that a capture call has been done // before the first render call is performed (implicitly // required by the APM API). capture_call_checker_->set_flag(); first_process_call_ = false; } return result; } bool ReadyToProcessCapture() { return (frame_counters_->CaptureMinusRenderCounters() <= kMaxCallDifference); } int ProcessRender() { // Call and time the specified render side API processing method. const int64_t start_time = clock_->TimeInMicroseconds(); const int result = apm_->ProcessReverseStream( &frame_data_.input_frame[0], frame_data_.input_stream_config, frame_data_.output_stream_config, &frame_data_.output_frame[0]); const int64_t end_time = clock_->TimeInMicroseconds(); frame_counters_->IncreaseRenderCounter(); AddDuration(end_time - start_time); return result; } bool ReadyToProcessRender() { // Do not process until at least one capture call has been done. // (implicitly required by the APM API). if (first_process_call_ && !capture_call_checker_->get_flag()) { return false; } // Ensure that the number of render and capture calls do not differ too // much. if (frame_counters_->RenderMinusCaptureCounters() > kMaxCallDifference) { return false; } first_process_call_ = false; return true; } void PrepareFrame() { // Lambda function for populating a float multichannel audio frame // with random data. auto populate_audio_frame = [](float amplitude, size_t num_channels, size_t samples_per_channel, Random* rand_gen, float** frame) { for (size_t ch = 0; ch < num_channels; ch++) { for (size_t k = 0; k < samples_per_channel; k++) { // Store random float number with a value between +-amplitude. frame[ch][k] = amplitude * (2 * rand_gen->Rand() - 1); } } }; // Prepare the audio input data and metadata. frame_data_.input_stream_config.set_sample_rate_hz( simulation_config_->sample_rate_hz); frame_data_.input_stream_config.set_num_channels(num_channels_); populate_audio_frame(input_level_, num_channels_, (simulation_config_->sample_rate_hz * AudioProcessing::kChunkSizeMs / 1000), rand_gen_, &frame_data_.input_frame[0]); // Prepare the float audio output data and metadata. frame_data_.output_stream_config.set_sample_rate_hz( simulation_config_->sample_rate_hz); frame_data_.output_stream_config.set_num_channels(1); } bool ReadyToProcess() { switch (processor_type_) { case ProcessorType::kRender: return ReadyToProcessRender(); case ProcessorType::kCapture: return ReadyToProcessCapture(); } // Should not be reached, but the return statement is needed for the code to // build successfully on Android. RTC_DCHECK_NOTREACHED(); return false; } Random* rand_gen_ = nullptr; FrameCounters* frame_counters_ = nullptr; LockedFlag* capture_call_checker_ = nullptr; CallSimulator* test_ = nullptr; const SimulationConfig* const simulation_config_ = nullptr; AudioProcessing* apm_ = nullptr; AudioFrameData frame_data_; webrtc::Clock* clock_; const size_t num_durations_to_store_; SamplesStatsCounter api_call_durations_; size_t samples_count_ = 0; const float input_level_; bool first_process_call_ = true; const ProcessorType processor_type_; const int num_channels_ = 1; }; // Class for managing the test simulation. class CallSimulator : public ::testing::TestWithParam { public: CallSimulator() : rand_gen_(42U), simulation_config_(static_cast(GetParam())) {} // Run the call simulation with a timeout. bool Run() { StartThreads(); bool result = test_complete_.Wait(kTestTimeout); StopThreads(); render_thread_state_->print_processor_statistics( simulation_config_.SettingsDescription() + "_render"); capture_thread_state_->print_processor_statistics( simulation_config_.SettingsDescription() + "_capture"); return result; } // Tests whether all the required render and capture side calls have been // done. bool MaybeEndTest() { if (frame_counters_.BothCountersExceedeThreshold(kMinNumFramesToProcess)) { test_complete_.Set(); return true; } return false; } private: static const float kCaptureInputFloatLevel; static const float kRenderInputFloatLevel; static const int kMinNumFramesToProcess = 150; static constexpr TimeDelta kTestTimeout = TimeDelta::Millis(3 * 10 * kMinNumFramesToProcess); // Stop all running threads. void StopThreads() { render_thread_.Finalize(); capture_thread_.Finalize(); } // Simulator and APM setup. void SetUp() override { // Lambda function for setting the default APM runtime settings for desktop. auto set_default_desktop_apm_runtime_settings = [](AudioProcessing* apm) { AudioProcessing::Config apm_config = apm->GetConfig(); apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = false; apm_config.noise_suppression.enabled = true; apm_config.gain_controller1.enabled = true; apm_config.gain_controller1.mode = AudioProcessing::Config::GainController1::kAdaptiveDigital; apm->ApplyConfig(apm_config); }; // Lambda function for setting the default APM runtime settings for mobile. auto set_default_mobile_apm_runtime_settings = [](AudioProcessing* apm) { AudioProcessing::Config apm_config = apm->GetConfig(); apm_config.echo_canceller.enabled = true; apm_config.echo_canceller.mobile_mode = true; apm_config.noise_suppression.enabled = true; apm_config.gain_controller1.mode = AudioProcessing::Config::GainController1::kAdaptiveDigital; apm->ApplyConfig(apm_config); }; // Lambda function for turning off all of the APM runtime settings // submodules. auto turn_off_default_apm_runtime_settings = [](AudioProcessing* apm) { AudioProcessing::Config apm_config = apm->GetConfig(); apm_config.echo_canceller.enabled = false; apm_config.gain_controller1.enabled = false; apm_config.noise_suppression.enabled = false; apm->ApplyConfig(apm_config); }; int num_capture_channels = 1; switch (simulation_config_.simulation_settings) { case SettingsType::kDefaultApmMobile: { apm_ = AudioProcessingBuilderForTesting().Create(); ASSERT_TRUE(!!apm_); set_default_mobile_apm_runtime_settings(apm_.get()); break; } case SettingsType::kDefaultApmDesktop: { apm_ = AudioProcessingBuilderForTesting().Create(); ASSERT_TRUE(!!apm_); set_default_desktop_apm_runtime_settings(apm_.get()); break; } case SettingsType::kAllSubmodulesTurnedOff: { apm_ = AudioProcessingBuilderForTesting().Create(); ASSERT_TRUE(!!apm_); turn_off_default_apm_runtime_settings(apm_.get()); break; } case SettingsType::kDefaultApmDesktopWithoutDelayAgnostic: { apm_ = AudioProcessingBuilderForTesting().Create(); ASSERT_TRUE(!!apm_); set_default_desktop_apm_runtime_settings(apm_.get()); break; } case SettingsType::kDefaultApmDesktopWithoutExtendedFilter: { apm_ = AudioProcessingBuilderForTesting().Create(); ASSERT_TRUE(!!apm_); set_default_desktop_apm_runtime_settings(apm_.get()); break; } } render_thread_state_.reset(new TimedThreadApiProcessor( ProcessorType::kRender, &rand_gen_, &frame_counters_, &capture_call_checker_, this, &simulation_config_, apm_.get(), kMinNumFramesToProcess, kRenderInputFloatLevel, 1)); capture_thread_state_.reset(new TimedThreadApiProcessor( ProcessorType::kCapture, &rand_gen_, &frame_counters_, &capture_call_checker_, this, &simulation_config_, apm_.get(), kMinNumFramesToProcess, kCaptureInputFloatLevel, num_capture_channels)); } // Start the threads used in the test. void StartThreads() { const auto attributes = rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime); render_thread_ = rtc::PlatformThread::SpawnJoinable( [this] { while (render_thread_state_->Process()) { } }, "render", attributes); capture_thread_ = rtc::PlatformThread::SpawnJoinable( [this] { while (capture_thread_state_->Process()) { } }, "capture", attributes); } // Event handler for the test. rtc::Event test_complete_; // Thread related variables. Random rand_gen_; rtc::scoped_refptr apm_; const SimulationConfig simulation_config_; FrameCounters frame_counters_; LockedFlag capture_call_checker_; std::unique_ptr render_thread_state_; std::unique_ptr capture_thread_state_; rtc::PlatformThread render_thread_; rtc::PlatformThread capture_thread_; }; // Implements the callback functionality for the threads. bool TimedThreadApiProcessor::Process() { PrepareFrame(); // Wait in a spinlock manner until it is ok to start processing. // Note that SleepMs is not applicable since it only allows sleeping // on a millisecond basis which is too long. // TODO(tommi): This loop may affect the performance of the test that it's // meant to measure. See if we could use events instead to signal readiness. while (!ReadyToProcess()) { } int result = AudioProcessing::kNoError; switch (processor_type_) { case ProcessorType::kRender: result = ProcessRender(); break; case ProcessorType::kCapture: result = ProcessCapture(); break; } EXPECT_EQ(result, AudioProcessing::kNoError); return !test_->MaybeEndTest(); } const float CallSimulator::kRenderInputFloatLevel = 0.5f; const float CallSimulator::kCaptureInputFloatLevel = 0.03125f; } // anonymous namespace // TODO(peah): Reactivate once issue 7712 has been resolved. TEST_P(CallSimulator, DISABLED_ApiCallDurationTest) { // Run test and verify that it did not time out. EXPECT_TRUE(Run()); } INSTANTIATE_TEST_SUITE_P( AudioProcessingPerformanceTest, CallSimulator, ::testing::ValuesIn(SimulationConfig::GenerateSimulationConfigs())); } // namespace webrtc