diff options
Diffstat (limited to 'third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc')
-rw-r--r-- | third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc | 140 |
1 files changed, 60 insertions, 80 deletions
diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc index e5c8bf6c08..6a76096b49 100644 --- a/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/third_party/libwebrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -20,6 +20,7 @@ #include <vector> #include "api/audio_codecs/audio_decoder.h" +#include "api/neteq/neteq_controller.h" #include "api/neteq/tick_timer.h" #include "common_audio/signal_processing/include/signal_processing_library.h" #include "modules/audio_coding/codecs/cng/webrtc_cng.h" @@ -36,7 +37,6 @@ #include "modules/audio_coding/neteq/normal.h" #include "modules/audio_coding/neteq/packet.h" #include "modules/audio_coding/neteq/packet_buffer.h" -#include "modules/audio_coding/neteq/post_decode_vad.h" #include "modules/audio_coding/neteq/preemptive_expand.h" #include "modules/audio_coding/neteq/red_payload_splitter.h" #include "modules/audio_coding/neteq/statistics_calculator.h" @@ -50,6 +50,7 @@ #include "rtc_base/strings/audio_format_to_string.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { @@ -70,49 +71,26 @@ std::unique_ptr<NetEqController> CreateNetEqController( return controller_factory.CreateNetEqController(config); } -void SetAudioFrameActivityAndType(bool vad_enabled, - NetEqImpl::OutputType type, - AudioFrame::VADActivity last_vad_activity, - AudioFrame* audio_frame) { +AudioFrame::SpeechType ToSpeechType(NetEqImpl::OutputType type) { switch (type) { case NetEqImpl::OutputType::kNormalSpeech: { - audio_frame->speech_type_ = AudioFrame::kNormalSpeech; - audio_frame->vad_activity_ = AudioFrame::kVadActive; - break; - } - case NetEqImpl::OutputType::kVadPassive: { - // This should only be reached if the VAD is enabled. - RTC_DCHECK(vad_enabled); - audio_frame->speech_type_ = AudioFrame::kNormalSpeech; - audio_frame->vad_activity_ = AudioFrame::kVadPassive; - break; + return AudioFrame::kNormalSpeech; } case NetEqImpl::OutputType::kCNG: { - audio_frame->speech_type_ = AudioFrame::kCNG; - audio_frame->vad_activity_ = AudioFrame::kVadPassive; - break; + return AudioFrame::kCNG; } case NetEqImpl::OutputType::kPLC: { - audio_frame->speech_type_ = AudioFrame::kPLC; - audio_frame->vad_activity_ = last_vad_activity; - break; + return AudioFrame::kPLC; } case NetEqImpl::OutputType::kPLCCNG: { - audio_frame->speech_type_ = AudioFrame::kPLCCNG; - audio_frame->vad_activity_ = AudioFrame::kVadPassive; - break; + return AudioFrame::kPLCCNG; } case NetEqImpl::OutputType::kCodecPLC: { - audio_frame->speech_type_ = AudioFrame::kCodecPLC; - audio_frame->vad_activity_ = last_vad_activity; - break; + return AudioFrame::kCodecPLC; } default: RTC_DCHECK_NOTREACHED(); - } - if (!vad_enabled) { - // Always set kVadUnknown when receive VAD is inactive. - audio_frame->vad_activity_ = AudioFrame::kVadUnknown; + return AudioFrame::kUndefined; } } @@ -169,11 +147,12 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, packet_buffer_(std::move(deps.packet_buffer)), red_payload_splitter_(std::move(deps.red_payload_splitter)), timestamp_scaler_(std::move(deps.timestamp_scaler)), - vad_(new PostDecodeVad()), expand_factory_(std::move(deps.expand_factory)), accelerate_factory_(std::move(deps.accelerate_factory)), preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), stats_(std::move(deps.stats)), + enable_fec_delay_adaptation_( + !field_trial::IsDisabled("WebRTC-Audio-NetEqFecDelayAdaptation")), controller_(std::move(deps.neteq_controller)), last_mode_(Mode::kNormal), decoded_buffer_length_(kMaxFrameSize), @@ -211,10 +190,6 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, if (create_components) { SetSampleRateAndChannels(fs, 1); // Default is 1 channel. } - RTC_DCHECK(!vad_->enabled()); - if (config.enable_post_decode_vad) { - vad_->Enable(); - } } NetEqImpl::~NetEqImpl() = default; @@ -252,9 +227,7 @@ int NetEqImpl::GetAudio(AudioFrame* audio_frame, audio_frame->sample_rate_hz_, rtc::dchecked_cast<int>(audio_frame->samples_per_channel_ * 100)); RTC_DCHECK_EQ(*muted, audio_frame->muted()); - SetAudioFrameActivityAndType(vad_->enabled(), LastOutputType(), - last_vad_activity_, audio_frame); - last_vad_activity_ = audio_frame->vad_activity_; + audio_frame->speech_type_ = ToSpeechType(LastOutputType()); last_output_sample_rate_hz_ = audio_frame->sample_rate_hz_; RTC_DCHECK(last_output_sample_rate_hz_ == 8000 || last_output_sample_rate_hz_ == 16000 || @@ -398,18 +371,6 @@ NetEqOperationsAndState NetEqImpl::GetOperationsAndState() const { return result; } -void NetEqImpl::EnableVad() { - MutexLock lock(&mutex_); - RTC_DCHECK(vad_.get()); - vad_->Enable(); -} - -void NetEqImpl::DisableVad() { - MutexLock lock(&mutex_); - RTC_DCHECK(vad_.get()); - vad_->Disable(); -} - absl::optional<uint32_t> NetEqImpl::GetPlayoutTimestamp() const { MutexLock lock(&mutex_); if (first_packet_ || last_mode_ == Mode::kRfc3389Cng || @@ -695,6 +656,7 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, packet_buffer_->Flush(); buffer_flush_occured = true; } + NetEqController::PacketArrivedInfo info = ToPacketArrivedInfo(packet); int return_val = packet_buffer_->InsertPacket(std::move(packet)); if (return_val == PacketBuffer::kFlushed) { buffer_flush_occured = true; @@ -702,6 +664,15 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, // An error occurred. return kOtherError; } + if (enable_fec_delay_adaptation_) { + info.buffer_flush = buffer_flush_occured; + const bool should_update_stats = !new_codec_ && !buffer_flush_occured; + auto relative_delay = + controller_->PacketArrived(fs_hz_, should_update_stats, info); + if (relative_delay) { + stats_->RelativePacketArrivalDelay(relative_delay.value()); + } + } } if (buffer_flush_occured) { @@ -752,24 +723,26 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, } } - const DecoderDatabase::DecoderInfo* dec_info = - decoder_database_->GetDecoderInfo(main_payload_type); - RTC_DCHECK(dec_info); // Already checked that the payload type is known. - - NetEqController::PacketArrivedInfo info; - info.is_cng_or_dtmf = dec_info->IsComfortNoise() || dec_info->IsDtmf(); - info.packet_length_samples = - number_of_primary_packets * decoder_frame_length_; - info.main_timestamp = main_timestamp; - info.main_sequence_number = main_sequence_number; - info.is_dtx = is_dtx; - info.buffer_flush = buffer_flush_occured; - - const bool should_update_stats = !new_codec_; - auto relative_delay = - controller_->PacketArrived(fs_hz_, should_update_stats, info); - if (relative_delay) { - stats_->RelativePacketArrivalDelay(relative_delay.value()); + if (!enable_fec_delay_adaptation_) { + const DecoderDatabase::DecoderInfo* dec_info = + decoder_database_->GetDecoderInfo(main_payload_type); + RTC_DCHECK(dec_info); // Already checked that the payload type is known. + + NetEqController::PacketArrivedInfo info; + info.is_cng_or_dtmf = dec_info->IsComfortNoise() || dec_info->IsDtmf(); + info.packet_length_samples = + number_of_primary_packets * decoder_frame_length_; + info.main_timestamp = main_timestamp; + info.main_sequence_number = main_sequence_number; + info.is_dtx = is_dtx; + info.buffer_flush = buffer_flush_occured; + + const bool should_update_stats = !new_codec_; + auto relative_delay = + controller_->PacketArrived(fs_hz_, should_update_stats, info); + if (relative_delay) { + stats_->RelativePacketArrivalDelay(relative_delay.value()); + } } return 0; } @@ -858,11 +831,8 @@ int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, last_decoded_type_ = speech_type; } - RTC_DCHECK(vad_.get()); bool sid_frame_available = (operation == Operation::kRfc3389Cng && !packet_list.empty()); - vad_->Update(decoded_buffer_.get(), static_cast<size_t>(length), speech_type, - sid_frame_available, fs_hz_); // This is the criterion that we did decode some data through the speech // decoder, and the operation resulted in comfort noise. @@ -1012,7 +982,7 @@ int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, (last_mode_ == Mode::kPreemptiveExpandFail) || (last_mode_ == Mode::kRfc3389Cng) || (last_mode_ == Mode::kCodecInternalCng)) { - background_noise_->Update(*sync_buffer_, *vad_.get()); + background_noise_->Update(*sync_buffer_); } if (operation == Operation::kDtmf) { @@ -2088,10 +2058,6 @@ void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) { if (cng_decoder) cng_decoder->Reset(); - // Reinit post-decode VAD with new sample rate. - RTC_DCHECK(vad_.get()); // Cannot be NULL here. - vad_->Init(); - // Delete algorithm buffer and create a new one. algorithm_buffer_.reset(new AudioMultiVector(channels)); @@ -2132,7 +2098,6 @@ void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) { } NetEqImpl::OutputType NetEqImpl::LastOutputType() { - RTC_DCHECK(vad_.get()); RTC_DCHECK(expand_.get()); if (last_mode_ == Mode::kCodecInternalCng || last_mode_ == Mode::kRfc3389Cng) { @@ -2142,12 +2107,27 @@ NetEqImpl::OutputType NetEqImpl::LastOutputType() { return OutputType::kPLCCNG; } else if (last_mode_ == Mode::kExpand) { return OutputType::kPLC; - } else if (vad_->running() && !vad_->active_speech()) { - return OutputType::kVadPassive; } else if (last_mode_ == Mode::kCodecPlc) { return OutputType::kCodecPLC; } else { return OutputType::kNormalSpeech; } } + +NetEqController::PacketArrivedInfo NetEqImpl::ToPacketArrivedInfo( + const Packet& packet) const { + const DecoderDatabase::DecoderInfo* dec_info = + decoder_database_->GetDecoderInfo(packet.payload_type); + + NetEqController::PacketArrivedInfo info; + info.is_cng_or_dtmf = + dec_info && (dec_info->IsComfortNoise() || dec_info->IsDtmf()); + info.packet_length_samples = + packet.frame ? packet.frame->Duration() : decoder_frame_length_; + info.main_timestamp = packet.timestamp; + info.main_sequence_number = packet.sequence_number; + info.is_dtx = packet.frame && packet.frame->IsDtxPacket(); + return info; +} + } // namespace webrtc |