/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "voice_engine/transmit_mixer.h" #include #include "audio/utility/audio_frame_operations.h" #include "rtc_base/format_macros.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "system_wrappers/include/event_wrapper.h" #include "voice_engine/channel.h" #include "voice_engine/channel_manager.h" #include "voice_engine/utility.h" namespace webrtc { namespace voe { // TODO(solenberg): The thread safety in this class is dubious. int32_t TransmitMixer::Create(TransmitMixer*& mixer) { mixer = new TransmitMixer(); if (mixer == NULL) { RTC_DLOG(LS_ERROR) << "TransmitMixer::Create() unable to allocate memory for mixer"; return -1; } return 0; } void TransmitMixer::Destroy(TransmitMixer*& mixer) { if (mixer) { delete mixer; mixer = NULL; } } TransmitMixer::~TransmitMixer() = default; void TransmitMixer::SetEngineInformation(ChannelManager* channelManager) { _channelManagerPtr = channelManager; } int32_t TransmitMixer::SetAudioProcessingModule(AudioProcessing* audioProcessingModule) { audioproc_ = audioProcessingModule; return 0; } void TransmitMixer::GetSendCodecInfo(int* max_sample_rate, size_t* max_channels) { *max_sample_rate = 8000; *max_channels = 1; for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); it.Increment()) { Channel* channel = it.GetChannel(); if (channel->Sending()) { const auto props = channel->GetEncoderProps(); RTC_CHECK(props); *max_sample_rate = std::max(*max_sample_rate, props->sample_rate_hz); *max_channels = std::max(*max_channels, props->num_channels); } } } int32_t TransmitMixer::PrepareDemux(const void* audioSamples, size_t nSamples, size_t nChannels, uint32_t samplesPerSec, uint16_t totalDelayMS, int32_t clockDrift, uint16_t currentMicLevel, bool keyPressed) { // --- Resample input audio and create/store the initial audio frame GenerateAudioFrame(static_cast(audioSamples), nSamples, nChannels, samplesPerSec); // --- Near-end audio processing. ProcessAudio(totalDelayMS, clockDrift, currentMicLevel, keyPressed); if (swap_stereo_channels_ && stereo_codec_) // Only bother swapping if we're using a stereo codec. AudioFrameOperations::SwapStereoChannels(&_audioFrame); // --- Annoying typing detection (utilizes the APM/VAD decision) #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION TypingDetection(keyPressed); #endif // --- Measure audio level of speech after all processing. double sample_duration = static_cast(nSamples) / samplesPerSec; _audioLevel.ComputeLevel(_audioFrame, sample_duration); return 0; } void TransmitMixer::ProcessAndEncodeAudio() { RTC_DCHECK_GT(_audioFrame.samples_per_channel_, 0); for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid(); it.Increment()) { Channel* const channel = it.GetChannel(); if (channel->Sending()) { channel->ProcessAndEncodeAudio(_audioFrame); } } } uint32_t TransmitMixer::CaptureLevel() const { return _captureLevel; } int32_t TransmitMixer::StopSend() { _audioLevel.Clear(); return 0; } int8_t TransmitMixer::AudioLevel() const { // Speech + file level [0,9] return _audioLevel.Level(); } int16_t TransmitMixer::AudioLevelFullRange() const { // Speech + file level [0,32767] return _audioLevel.LevelFullRange(); } double TransmitMixer::GetTotalInputEnergy() const { return _audioLevel.TotalEnergy(); } double TransmitMixer::GetTotalInputDuration() const { return _audioLevel.TotalDuration(); } void TransmitMixer::GenerateAudioFrame(const int16_t* audio, size_t samples_per_channel, size_t num_channels, int sample_rate_hz) { int codec_rate; size_t num_codec_channels; GetSendCodecInfo(&codec_rate, &num_codec_channels); stereo_codec_ = num_codec_channels == 2; // We want to process at the lowest rate possible without losing information. // Choose the lowest native rate at least equal to the input and codec rates. const int min_processing_rate = std::min(sample_rate_hz, codec_rate); for (size_t i = 0; i < AudioProcessing::kNumNativeSampleRates; ++i) { _audioFrame.sample_rate_hz_ = AudioProcessing::kNativeSampleRatesHz[i]; if (_audioFrame.sample_rate_hz_ >= min_processing_rate) { break; } } _audioFrame.num_channels_ = std::min(num_channels, num_codec_channels); RemixAndResample(audio, samples_per_channel, num_channels, sample_rate_hz, &resampler_, &_audioFrame); } void TransmitMixer::ProcessAudio(int delay_ms, int clock_drift, int current_mic_level, bool key_pressed) { if (audioproc_->set_stream_delay_ms(delay_ms) != 0) { // Silently ignore this failure to avoid flooding the logs. } GainControl* agc = audioproc_->gain_control(); if (agc->set_stream_analog_level(current_mic_level) != 0) { RTC_DLOG(LS_ERROR) << "set_stream_analog_level failed: current_mic_level = " << current_mic_level; assert(false); } EchoCancellation* aec = audioproc_->echo_cancellation(); if (aec->is_drift_compensation_enabled()) { aec->set_stream_drift_samples(clock_drift); } audioproc_->set_stream_key_pressed(key_pressed); int err = audioproc_->ProcessStream(&_audioFrame); if (err != 0) { RTC_DLOG(LS_ERROR) << "ProcessStream() error: " << err; assert(false); } // Store new capture level. Only updated when analog AGC is enabled. _captureLevel = agc->stream_analog_level(); } #if WEBRTC_VOICE_ENGINE_TYPING_DETECTION void TransmitMixer::TypingDetection(bool key_pressed) { // We let the VAD determine if we're using this feature or not. if (_audioFrame.vad_activity_ == AudioFrame::kVadUnknown) { return; } bool vad_active = _audioFrame.vad_activity_ == AudioFrame::kVadActive; bool typing_detected = typing_detection_.Process(key_pressed, vad_active); rtc::CritScope cs(&lock_); typing_noise_detected_ = typing_detected; } #endif void TransmitMixer::EnableStereoChannelSwapping(bool enable) { swap_stereo_channels_ = enable; } bool TransmitMixer::IsStereoChannelSwappingEnabled() { return swap_stereo_channels_; } bool TransmitMixer::typing_noise_detected() const { rtc::CritScope cs(&lock_); return typing_noise_detected_; } } // namespace voe } // namespace webrtc