diff options
Diffstat (limited to 'third_party/libwebrtc/audio/voip')
| -rw-r--r-- | third_party/libwebrtc/audio/voip/BUILD.gn | 103 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_channel.cc | 173 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_channel.h | 131 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_egress.cc | 182 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_egress.h | 158 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_ingress.cc | 296 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/audio_ingress.h | 145 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/BUILD.gn | 101 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc | 357 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc | 327 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc | 238 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/mock_task_queue.h | 55 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc | 193 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/voip_core.cc | 500 | ||||
| -rw-r--r-- | third_party/libwebrtc/audio/voip/voip_core.h | 174 |
15 files changed, 3133 insertions, 0 deletions
diff --git a/third_party/libwebrtc/audio/voip/BUILD.gn b/third_party/libwebrtc/audio/voip/BUILD.gn new file mode 100644 index 0000000000..e807e2276b --- /dev/null +++ b/third_party/libwebrtc/audio/voip/BUILD.gn @@ -0,0 +1,103 @@ +# Copyright(c) 2020 The WebRTC project authors.All Rights Reserved. +# +# Use of this source code is governed by a BSD - style license +# that can be found in the LICENSE file in the root of the source +# tree.An additional intellectual property rights grant can be found +# in the file PATENTS.All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_library("voip_core") { + sources = [ + "voip_core.cc", + "voip_core.h", + ] + deps = [ + ":audio_channel", + "..:audio", + "../../api:scoped_refptr", + "../../api/audio_codecs:audio_codecs_api", + "../../api/task_queue", + "../../api/voip:voip_api", + "../../modules/audio_device:audio_device_api", + "../../modules/audio_mixer:audio_mixer_impl", + "../../modules/audio_processing:api", + "../../rtc_base:criticalsection", + "../../rtc_base:logging", + "../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("audio_channel") { + sources = [ + "audio_channel.cc", + "audio_channel.h", + ] + deps = [ + ":audio_egress", + ":audio_ingress", + "../../api:transport_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/task_queue", + "../../api/voip:voip_api", + "../../modules/audio_device:audio_device_api", + "../../modules/rtp_rtcp", + "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../rtc_base:criticalsection", + "../../rtc_base:logging", + "../../rtc_base:refcount", + ] +} + +rtc_library("audio_ingress") { + sources = [ + "audio_ingress.cc", + "audio_ingress.h", + ] + deps = [ + "..:audio", + "../../api:array_view", + "../../api:rtp_headers", + "../../api:scoped_refptr", + "../../api:transport_api", + "../../api/audio:audio_mixer_api", + "../../api/audio_codecs:audio_codecs_api", + "../../api/voip:voip_api", + "../../modules/audio_coding", + "../../modules/rtp_rtcp", + "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../rtc_base:criticalsection", + "../../rtc_base:logging", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_minmax", + "../../rtc_base:timeutils", + "../../rtc_base/synchronization:mutex", + "../utility:audio_frame_operations", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("audio_egress") { + sources = [ + "audio_egress.cc", + "audio_egress.h", + ] + deps = [ + "..:audio", + "../../api:sequence_checker", + "../../api/audio_codecs:audio_codecs_api", + "../../api/task_queue", + "../../call:audio_sender_interface", + "../../modules/audio_coding", + "../../modules/rtp_rtcp", + "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../rtc_base:logging", + "../../rtc_base:rtc_task_queue", + "../../rtc_base:timeutils", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:no_unique_address", + "../utility:audio_frame_operations", + ] +} diff --git a/third_party/libwebrtc/audio/voip/audio_channel.cc b/third_party/libwebrtc/audio/voip/audio_channel.cc new file mode 100644 index 0000000000..a70e33ec38 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_channel.cc @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_channel.h" + +#include <utility> +#include <vector> + +#include "api/audio_codecs/audio_format.h" +#include "api/task_queue/task_queue_factory.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { + +constexpr int kRtcpReportIntervalMs = 5000; + +} // namespace + +AudioChannel::AudioChannel( + Transport* transport, + uint32_t local_ssrc, + TaskQueueFactory* task_queue_factory, + AudioMixer* audio_mixer, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) + : audio_mixer_(audio_mixer) { + RTC_DCHECK(task_queue_factory); + RTC_DCHECK(audio_mixer); + + Clock* clock = Clock::GetRealTimeClock(); + receive_statistics_ = ReceiveStatistics::Create(clock); + + RtpRtcpInterface::Configuration rtp_config; + rtp_config.clock = clock; + rtp_config.audio = true; + rtp_config.receive_statistics = receive_statistics_.get(); + rtp_config.rtcp_report_interval_ms = kRtcpReportIntervalMs; + rtp_config.outgoing_transport = transport; + rtp_config.local_media_ssrc = local_ssrc; + + rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(rtp_config); + + rtp_rtcp_->SetSendingMediaStatus(false); + rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); + + ingress_ = std::make_unique<AudioIngress>(rtp_rtcp_.get(), clock, + receive_statistics_.get(), + std::move(decoder_factory)); + egress_ = + std::make_unique<AudioEgress>(rtp_rtcp_.get(), clock, task_queue_factory); + + // Set the instance of audio ingress to be part of audio mixer for ADM to + // fetch audio samples to play. + audio_mixer_->AddSource(ingress_.get()); +} + +AudioChannel::~AudioChannel() { + if (egress_->IsSending()) { + StopSend(); + } + if (ingress_->IsPlaying()) { + StopPlay(); + } + + audio_mixer_->RemoveSource(ingress_.get()); + + // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `egress_` + // here. + egress_.reset(); + ingress_.reset(); +} + +bool AudioChannel::StartSend() { + // If encoder has not been set, return false. + if (!egress_->StartSend()) { + return false; + } + + // Start sending with RTP stack if it has not been sending yet. + if (!rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(true); + } + return true; +} + +void AudioChannel::StopSend() { + egress_->StopSend(); + + // Deactivate RTP stack when both sending and receiving are stopped. + // SetSendingStatus(false) triggers the transmission of RTCP BYE + // message to remote endpoint. + if (!ingress_->IsPlaying() && rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(false); + } +} + +bool AudioChannel::StartPlay() { + // If decoders have not been set, return false. + if (!ingress_->StartPlay()) { + return false; + } + + // If RTP stack is not sending then start sending as in recv-only mode, RTCP + // receiver report is expected. + if (!rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(true); + } + return true; +} + +void AudioChannel::StopPlay() { + ingress_->StopPlay(); + + // Deactivate RTP stack only when both sending and receiving are stopped. + if (!rtp_rtcp_->SendingMedia() && rtp_rtcp_->Sending()) { + rtp_rtcp_->SetSendingStatus(false); + } +} + +IngressStatistics AudioChannel::GetIngressStatistics() { + IngressStatistics ingress_stats; + NetworkStatistics stats = ingress_->GetNetworkStatistics(); + ingress_stats.neteq_stats.total_samples_received = stats.totalSamplesReceived; + ingress_stats.neteq_stats.concealed_samples = stats.concealedSamples; + ingress_stats.neteq_stats.concealment_events = stats.concealmentEvents; + ingress_stats.neteq_stats.jitter_buffer_delay_ms = stats.jitterBufferDelayMs; + ingress_stats.neteq_stats.jitter_buffer_emitted_count = + stats.jitterBufferEmittedCount; + ingress_stats.neteq_stats.jitter_buffer_target_delay_ms = + stats.jitterBufferTargetDelayMs; + ingress_stats.neteq_stats.inserted_samples_for_deceleration = + stats.insertedSamplesForDeceleration; + ingress_stats.neteq_stats.removed_samples_for_acceleration = + stats.removedSamplesForAcceleration; + ingress_stats.neteq_stats.silent_concealed_samples = + stats.silentConcealedSamples; + ingress_stats.neteq_stats.fec_packets_received = stats.fecPacketsReceived; + ingress_stats.neteq_stats.fec_packets_discarded = stats.fecPacketsDiscarded; + ingress_stats.neteq_stats.delayed_packet_outage_samples = + stats.delayedPacketOutageSamples; + ingress_stats.neteq_stats.relative_packet_arrival_delay_ms = + stats.relativePacketArrivalDelayMs; + ingress_stats.neteq_stats.interruption_count = stats.interruptionCount; + ingress_stats.neteq_stats.total_interruption_duration_ms = + stats.totalInterruptionDurationMs; + ingress_stats.total_duration = ingress_->GetOutputTotalDuration(); + return ingress_stats; +} + +ChannelStatistics AudioChannel::GetChannelStatistics() { + ChannelStatistics channel_stat = ingress_->GetChannelStatistics(); + + StreamDataCounters rtp_stats, rtx_stats; + rtp_rtcp_->GetSendStreamDataCounters(&rtp_stats, &rtx_stats); + channel_stat.bytes_sent = + rtp_stats.transmitted.payload_bytes + rtx_stats.transmitted.payload_bytes; + channel_stat.packets_sent = + rtp_stats.transmitted.packets + rtx_stats.transmitted.packets; + + return channel_stat; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/audio_channel.h b/third_party/libwebrtc/audio/voip/audio_channel.h new file mode 100644 index 0000000000..7338d9faab --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_channel.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_AUDIO_CHANNEL_H_ +#define AUDIO_VOIP_AUDIO_CHANNEL_H_ + +#include <map> +#include <memory> +#include <queue> +#include <utility> + +#include "api/task_queue/task_queue_factory.h" +#include "api/voip/voip_base.h" +#include "api/voip/voip_statistics.h" +#include "audio/voip/audio_egress.h" +#include "audio/voip/audio_ingress.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "rtc_base/ref_count.h" + +namespace webrtc { + +// AudioChannel represents a single media session and provides APIs over +// AudioIngress and AudioEgress. Note that a single RTP stack is shared with +// these two classes as it has both sending and receiving capabilities. +class AudioChannel : public rtc::RefCountInterface { + public: + AudioChannel(Transport* transport, + uint32_t local_ssrc, + TaskQueueFactory* task_queue_factory, + AudioMixer* audio_mixer, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory); + ~AudioChannel() override; + + // Set and get ChannelId that this audio channel belongs for debugging and + // logging purpose. + void SetId(ChannelId id) { id_ = id; } + ChannelId GetId() const { return id_; } + + // APIs to start/stop audio channel on each direction. + // StartSend/StartPlay returns false if encoder/decoders + // have not been set, respectively. + bool StartSend(); + void StopSend(); + bool StartPlay(); + void StopPlay(); + + // APIs relayed to AudioEgress. + bool IsSendingMedia() const { return egress_->IsSending(); } + AudioSender* GetAudioSender() { return egress_.get(); } + void SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, + std::unique_ptr<AudioEncoder> encoder) { + egress_->SetEncoder(payload_type, encoder_format, std::move(encoder)); + } + absl::optional<SdpAudioFormat> GetEncoderFormat() const { + return egress_->GetEncoderFormat(); + } + void RegisterTelephoneEventType(int rtp_payload_type, int sample_rate_hz) { + egress_->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz); + } + bool SendTelephoneEvent(int dtmf_event, int duration_ms) { + return egress_->SendTelephoneEvent(dtmf_event, duration_ms); + } + void SetMute(bool enable) { egress_->SetMute(enable); } + + // APIs relayed to AudioIngress. + bool IsPlaying() const { return ingress_->IsPlaying(); } + void ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) { + ingress_->ReceivedRTPPacket(rtp_packet); + } + void ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet) { + ingress_->ReceivedRTCPPacket(rtcp_packet); + } + void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) { + ingress_->SetReceiveCodecs(codecs); + } + IngressStatistics GetIngressStatistics(); + ChannelStatistics GetChannelStatistics(); + + // See comments on the methods used from AudioEgress and AudioIngress. + // Conversion to double is following what is done in + // DoubleAudioLevelFromIntAudioLevel method in rtc_stats_collector.cc to be + // consistent. + double GetInputAudioLevel() const { + return egress_->GetInputAudioLevel() / 32767.0; + } + double GetInputTotalEnergy() const { return egress_->GetInputTotalEnergy(); } + double GetInputTotalDuration() const { + return egress_->GetInputTotalDuration(); + } + double GetOutputAudioLevel() const { + return ingress_->GetOutputAudioLevel() / 32767.0; + } + double GetOutputTotalEnergy() const { + return ingress_->GetOutputTotalEnergy(); + } + double GetOutputTotalDuration() const { + return ingress_->GetOutputTotalDuration(); + } + + // Internal API for testing purpose. + void SendRTCPReportForTesting(RTCPPacketType type) { + int32_t result = rtp_rtcp_->SendRTCP(type); + RTC_DCHECK(result == 0); + } + + private: + // ChannelId that this audio channel belongs for logging purpose. + ChannelId id_; + + // Synchronization is handled internally by AudioMixer. + AudioMixer* audio_mixer_; + + // Listed in order for safe destruction of AudioChannel object. + // Synchronization for these are handled internally. + std::unique_ptr<ReceiveStatistics> receive_statistics_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; + std::unique_ptr<AudioIngress> ingress_; + std::unique_ptr<AudioEgress> egress_; +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_AUDIO_CHANNEL_H_ diff --git a/third_party/libwebrtc/audio/voip/audio_egress.cc b/third_party/libwebrtc/audio/voip/audio_egress.cc new file mode 100644 index 0000000000..1162824c9e --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_egress.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_egress.h" + +#include <utility> +#include <vector> + +#include "rtc_base/logging.h" + +namespace webrtc { + +AudioEgress::AudioEgress(RtpRtcpInterface* rtp_rtcp, + Clock* clock, + TaskQueueFactory* task_queue_factory) + : rtp_rtcp_(rtp_rtcp), + rtp_sender_audio_(clock, rtp_rtcp_->RtpSender()), + audio_coding_(AudioCodingModule::Create(AudioCodingModule::Config())), + encoder_queue_(task_queue_factory->CreateTaskQueue( + "AudioEncoder", + TaskQueueFactory::Priority::NORMAL)) { + audio_coding_->RegisterTransportCallback(this); +} + +AudioEgress::~AudioEgress() { + audio_coding_->RegisterTransportCallback(nullptr); +} + +bool AudioEgress::IsSending() const { + return rtp_rtcp_->SendingMedia(); +} + +void AudioEgress::SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, + std::unique_ptr<AudioEncoder> encoder) { + RTC_DCHECK_GE(payload_type, 0); + RTC_DCHECK_LE(payload_type, 127); + + SetEncoderFormat(encoder_format); + + // The RTP/RTCP module needs to know the RTP timestamp rate (i.e. clockrate) + // as well as some other things, so we collect this info and send it along. + rtp_rtcp_->RegisterSendPayloadFrequency(payload_type, + encoder->RtpTimestampRateHz()); + rtp_sender_audio_.RegisterAudioPayload("audio", payload_type, + encoder->RtpTimestampRateHz(), + encoder->NumChannels(), 0); + + audio_coding_->SetEncoder(std::move(encoder)); +} + +bool AudioEgress::StartSend() { + if (!GetEncoderFormat()) { + RTC_DLOG(LS_WARNING) << "Send codec has not been set yet"; + return false; + } + rtp_rtcp_->SetSendingMediaStatus(true); + return true; +} + +void AudioEgress::StopSend() { + rtp_rtcp_->SetSendingMediaStatus(false); +} + +void AudioEgress::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { + RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); + RTC_DCHECK_LE(audio_frame->num_channels_, 8); + + encoder_queue_.PostTask( + [this, audio_frame = std::move(audio_frame)]() mutable { + RTC_DCHECK_RUN_ON(&encoder_queue_); + if (!rtp_rtcp_->SendingMedia()) { + return; + } + + double duration_seconds = + static_cast<double>(audio_frame->samples_per_channel_) / + audio_frame->sample_rate_hz_; + + input_audio_level_.ComputeLevel(*audio_frame, duration_seconds); + + AudioFrameOperations::Mute(audio_frame.get(), + encoder_context_.previously_muted_, + encoder_context_.mute_); + encoder_context_.previously_muted_ = encoder_context_.mute_; + + audio_frame->timestamp_ = encoder_context_.frame_rtp_timestamp_; + + // This call will trigger AudioPacketizationCallback::SendData if + // encoding is done and payload is ready for packetization and + // transmission. Otherwise, it will return without invoking the + // callback. + if (audio_coding_->Add10MsData(*audio_frame) < 0) { + RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed."; + return; + } + + encoder_context_.frame_rtp_timestamp_ += + rtc::dchecked_cast<uint32_t>(audio_frame->samples_per_channel_); + }); +} + +int32_t AudioEgress::SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + + rtc::ArrayView<const uint8_t> payload(payload_data, payload_size); + + // Currently we don't get a capture time from downstream modules (ADM, + // AudioTransportImpl). + // TODO(natim@webrtc.org): Integrate once it's ready. + constexpr uint32_t kUndefinedCaptureTime = -1; + + // Push data from ACM to RTP/RTCP-module to deliver audio frame for + // packetization. + if (!rtp_rtcp_->OnSendingRtpFrame(timestamp, kUndefinedCaptureTime, + payload_type, + /*force_sender_report=*/false)) { + return -1; + } + + const uint32_t rtp_timestamp = timestamp + rtp_rtcp_->StartTimestamp(); + + // This call will trigger Transport::SendPacket() from the RTP/RTCP module. + if (!rtp_sender_audio_.SendAudio(frame_type, payload_type, rtp_timestamp, + payload.data(), payload.size())) { + RTC_DLOG(LS_ERROR) + << "AudioEgress::SendData() failed to send data to RTP/RTCP module"; + return -1; + } + + return 0; +} + +void AudioEgress::RegisterTelephoneEventType(int rtp_payload_type, + int sample_rate_hz) { + RTC_DCHECK_GE(rtp_payload_type, 0); + RTC_DCHECK_LE(rtp_payload_type, 127); + + rtp_rtcp_->RegisterSendPayloadFrequency(rtp_payload_type, sample_rate_hz); + rtp_sender_audio_.RegisterAudioPayload("telephone-event", rtp_payload_type, + sample_rate_hz, 0, 0); +} + +bool AudioEgress::SendTelephoneEvent(int dtmf_event, int duration_ms) { + RTC_DCHECK_GE(dtmf_event, 0); + RTC_DCHECK_LE(dtmf_event, 255); + RTC_DCHECK_GE(duration_ms, 0); + RTC_DCHECK_LE(duration_ms, 65535); + + if (!IsSending()) { + return false; + } + + constexpr int kTelephoneEventAttenuationdB = 10; + + if (rtp_sender_audio_.SendTelephoneEvent(dtmf_event, duration_ms, + kTelephoneEventAttenuationdB) != 0) { + RTC_DLOG(LS_ERROR) << "SendTelephoneEvent() failed to send event"; + return false; + } + return true; +} + +void AudioEgress::SetMute(bool mute) { + encoder_queue_.PostTask([this, mute] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + encoder_context_.mute_ = mute; + }); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/audio_egress.h b/third_party/libwebrtc/audio/voip/audio_egress.h new file mode 100644 index 0000000000..989e5bda59 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_egress.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_AUDIO_EGRESS_H_ +#define AUDIO_VOIP_AUDIO_EGRESS_H_ + +#include <memory> +#include <string> + +#include "api/audio_codecs/audio_format.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_factory.h" +#include "audio/audio_level.h" +#include "audio/utility/audio_frame_operations.h" +#include "call/audio_sender.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/rtp_rtcp/include/report_block_data.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" +#include "modules/rtp_rtcp/source/rtp_sender_audio.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +// AudioEgress receives input samples from AudioDeviceModule via +// AudioTransportImpl through AudioSender interface. Once it encodes the sample +// via selected encoder through AudioPacketizationCallback interface, the +// encoded payload will be packetized by the RTP stack, resulting in ready to +// send RTP packet to remote endpoint. +// +// TaskQueue is used to encode and send RTP asynchrounously as some OS platform +// uses the same thread for both audio input and output sample deliveries which +// can affect audio quality. +// +// Note that this class is originally based on ChannelSend in +// audio/channel_send.cc with non-audio related logic trimmed as aimed for +// smaller footprint. +class AudioEgress : public AudioSender, public AudioPacketizationCallback { + public: + AudioEgress(RtpRtcpInterface* rtp_rtcp, + Clock* clock, + TaskQueueFactory* task_queue_factory); + ~AudioEgress() override; + + // Set the encoder format and payload type for AudioCodingModule. + // It's possible to change the encoder type during its active usage. + // `payload_type` must be the type that is negotiated with peer through + // offer/answer. + void SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, + std::unique_ptr<AudioEncoder> encoder); + + // Start or stop sending operation of AudioEgress. This will start/stop + // the RTP stack also causes encoder queue thread to start/stop + // processing input audio samples. StartSend will return false if + // a send codec has not been set. + bool StartSend(); + void StopSend(); + + // Query the state of the RTP stack. This returns true if StartSend() + // called and false if StopSend() is called. + bool IsSending() const; + + // Enable or disable Mute state. + void SetMute(bool mute); + + // Retrieve current encoder format info. This returns encoder format set + // by SetEncoder() and if encoder is not set, this will return nullopt. + absl::optional<SdpAudioFormat> GetEncoderFormat() const { + MutexLock lock(&lock_); + return encoder_format_; + } + + // Register the payload type and sample rate for DTMF (RFC 4733) payload. + void RegisterTelephoneEventType(int rtp_payload_type, int sample_rate_hz); + + // Send DTMF named event as specified by + // https://tools.ietf.org/html/rfc4733#section-3.2 + // `duration_ms` specifies the duration of DTMF packets that will be emitted + // in place of real RTP packets instead. + // This will return true when requested dtmf event is successfully scheduled + // otherwise false when the dtmf queue reached maximum of 20 events. + bool SendTelephoneEvent(int dtmf_event, int duration_ms); + + // See comments on LevelFullRange, TotalEnergy, TotalDuration from + // audio/audio_level.h. + int GetInputAudioLevel() const { return input_audio_level_.LevelFullRange(); } + double GetInputTotalEnergy() const { + return input_audio_level_.TotalEnergy(); + } + double GetInputTotalDuration() const { + return input_audio_level_.TotalDuration(); + } + + // Implementation of AudioSender interface. + void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; + + // Implementation of AudioPacketizationCallback interface. + int32_t SendData(AudioFrameType frame_type, + uint8_t payload_type, + uint32_t timestamp, + const uint8_t* payload_data, + size_t payload_size) override; + + private: + void SetEncoderFormat(const SdpAudioFormat& encoder_format) { + MutexLock lock(&lock_); + encoder_format_ = encoder_format; + } + + mutable Mutex lock_; + + // Current encoder format selected by caller. + absl::optional<SdpAudioFormat> encoder_format_ RTC_GUARDED_BY(lock_); + + // Synchronization is handled internally by RtpRtcp. + RtpRtcpInterface* const rtp_rtcp_; + + // Synchronization is handled internally by RTPSenderAudio. + RTPSenderAudio rtp_sender_audio_; + + // Synchronization is handled internally by AudioCodingModule. + const std::unique_ptr<AudioCodingModule> audio_coding_; + + // Synchronization is handled internally by voe::AudioLevel. + voe::AudioLevel input_audio_level_; + + // Struct that holds all variables used by encoder task queue. + struct EncoderContext { + // Offset used to mark rtp timestamp in sample rate unit in + // newly received audio frame from AudioTransport. + uint32_t frame_rtp_timestamp_ = 0; + + // Flag to track mute state from caller. `previously_muted_` is used to + // track previous state as part of input to AudioFrameOperations::Mute + // to implement fading effect when (un)mute is invoked. + bool mute_ = false; + bool previously_muted_ = false; + }; + + EncoderContext encoder_context_ RTC_GUARDED_BY(encoder_queue_); + + // Defined last to ensure that there are no running tasks when the other + // members are destroyed. + rtc::TaskQueue encoder_queue_; +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_AUDIO_EGRESS_H_ diff --git a/third_party/libwebrtc/audio/voip/audio_ingress.cc b/third_party/libwebrtc/audio/voip/audio_ingress.cc new file mode 100644 index 0000000000..9492a51a21 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_ingress.cc @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_ingress.h" + +#include <algorithm> +#include <utility> +#include <vector> + +#include "api/audio_codecs/audio_format.h" +#include "audio/utility/audio_frame_operations.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "modules/rtp_rtcp/source/rtcp_packet/common_header.h" +#include "modules/rtp_rtcp/source/rtcp_packet/receiver_report.h" +#include "modules/rtp_rtcp/source/rtcp_packet/sender_report.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_minmax.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +namespace { + +AudioCodingModule::Config CreateAcmConfig( + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) { + AudioCodingModule::Config acm_config; + acm_config.neteq_config.enable_muted_state = true; + acm_config.decoder_factory = decoder_factory; + return acm_config; +} + +} // namespace + +AudioIngress::AudioIngress( + RtpRtcpInterface* rtp_rtcp, + Clock* clock, + ReceiveStatistics* receive_statistics, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) + : playing_(false), + remote_ssrc_(0), + first_rtp_timestamp_(-1), + rtp_receive_statistics_(receive_statistics), + rtp_rtcp_(rtp_rtcp), + acm_receiver_(CreateAcmConfig(decoder_factory)), + ntp_estimator_(clock) {} + +AudioIngress::~AudioIngress() = default; + +AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo( + int sampling_rate, + AudioFrame* audio_frame) { + audio_frame->sample_rate_hz_ = sampling_rate; + + // Get 10ms raw PCM data from the ACM. + bool muted = false; + if (acm_receiver_.GetAudio(sampling_rate, audio_frame, &muted) == -1) { + RTC_DLOG(LS_ERROR) << "GetAudio() failed!"; + // In all likelihood, the audio in this frame is garbage. We return an + // error so that the audio mixer module doesn't add it to the mix. As + // a result, it won't be played out and the actions skipped here are + // irrelevant. + return AudioMixer::Source::AudioFrameInfo::kError; + } + + if (muted) { + AudioFrameOperations::Mute(audio_frame); + } + + // Measure audio level. + constexpr double kAudioSampleDurationSeconds = 0.01; + output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds); + + // If caller invoked StopPlay(), then mute the frame. + if (!playing_) { + AudioFrameOperations::Mute(audio_frame); + muted = true; + } + + // Set first rtp timestamp with first audio frame with valid timestamp. + if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) { + first_rtp_timestamp_ = audio_frame->timestamp_; + } + + if (first_rtp_timestamp_ >= 0) { + // Compute elapsed and NTP times. + int64_t unwrap_timestamp; + { + MutexLock lock(&lock_); + unwrap_timestamp = + timestamp_wrap_handler_.Unwrap(audio_frame->timestamp_); + audio_frame->ntp_time_ms_ = + ntp_estimator_.Estimate(audio_frame->timestamp_); + } + // For clock rate, default to the playout sampling rate if we haven't + // received any packets yet. + absl::optional<std::pair<int, SdpAudioFormat>> decoder = + acm_receiver_.LastDecoder(); + int clock_rate = decoder ? decoder->second.clockrate_hz + : acm_receiver_.last_output_sample_rate_hz(); + RTC_DCHECK_GT(clock_rate, 0); + audio_frame->elapsed_time_ms_ = + (unwrap_timestamp - first_rtp_timestamp_) / (clock_rate / 1000); + } + + return muted ? AudioMixer::Source::AudioFrameInfo::kMuted + : AudioMixer::Source::AudioFrameInfo::kNormal; +} + +bool AudioIngress::StartPlay() { + { + MutexLock lock(&lock_); + if (receive_codec_info_.empty()) { + RTC_DLOG(LS_WARNING) << "Receive codecs have not been set yet"; + return false; + } + } + playing_ = true; + return true; +} + +void AudioIngress::SetReceiveCodecs( + const std::map<int, SdpAudioFormat>& codecs) { + { + MutexLock lock(&lock_); + for (const auto& kv : codecs) { + receive_codec_info_[kv.first] = kv.second.clockrate_hz; + } + } + acm_receiver_.SetCodecs(codecs); +} + +void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) { + RtpPacketReceived rtp_packet_received; + rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size()); + + // Set payload type's sampling rate before we feed it into ReceiveStatistics. + { + MutexLock lock(&lock_); + const auto& it = + receive_codec_info_.find(rtp_packet_received.PayloadType()); + // If sampling rate info is not available in our received codec set, it + // would mean that remote media endpoint is sending incorrect payload id + // which can't be processed correctly especially on payload type id in + // dynamic range. + if (it == receive_codec_info_.end()) { + RTC_DLOG(LS_WARNING) << "Unexpected payload id received: " + << rtp_packet_received.PayloadType(); + return; + } + rtp_packet_received.set_payload_type_frequency(it->second); + } + + // Track current remote SSRC. + if (rtp_packet_received.Ssrc() != remote_ssrc_) { + rtp_rtcp_->SetRemoteSSRC(rtp_packet_received.Ssrc()); + remote_ssrc_.store(rtp_packet_received.Ssrc()); + } + + rtp_receive_statistics_->OnRtpPacket(rtp_packet_received); + + RTPHeader header; + rtp_packet_received.GetHeader(&header); + + size_t packet_length = rtp_packet_received.size(); + if (packet_length < header.headerLength || + (packet_length - header.headerLength) < header.paddingLength) { + RTC_DLOG(LS_ERROR) << "Packet length(" << packet_length << ") header(" + << header.headerLength << ") padding(" + << header.paddingLength << ")"; + return; + } + + const uint8_t* payload = rtp_packet_received.data() + header.headerLength; + size_t payload_length = packet_length - header.headerLength; + size_t payload_data_length = payload_length - header.paddingLength; + auto data_view = rtc::ArrayView<const uint8_t>(payload, payload_data_length); + + // Push the incoming payload (parsed and ready for decoding) into the ACM. + if (acm_receiver_.InsertPacket(header, data_view) != 0) { + RTC_DLOG(LS_ERROR) << "AudioIngress::ReceivedRTPPacket() unable to " + "push data to the ACM"; + } +} + +void AudioIngress::ReceivedRTCPPacket( + rtc::ArrayView<const uint8_t> rtcp_packet) { + rtcp::CommonHeader rtcp_header; + if (rtcp_header.Parse(rtcp_packet.data(), rtcp_packet.size()) && + (rtcp_header.type() == rtcp::SenderReport::kPacketType || + rtcp_header.type() == rtcp::ReceiverReport::kPacketType)) { + RTC_DCHECK_GE(rtcp_packet.size(), 8); + + uint32_t sender_ssrc = + ByteReader<uint32_t>::ReadBigEndian(rtcp_packet.data() + 4); + + // If we don't have remote ssrc at this point, it's likely that remote + // endpoint is receive-only or it could have restarted the media. + if (sender_ssrc != remote_ssrc_) { + rtp_rtcp_->SetRemoteSSRC(sender_ssrc); + remote_ssrc_.store(sender_ssrc); + } + } + + // Deliver RTCP packet to RTP/RTCP module for parsing and processing. + rtp_rtcp_->IncomingRtcpPacket(rtcp_packet.data(), rtcp_packet.size()); + + int64_t rtt = 0; + if (rtp_rtcp_->RTT(remote_ssrc_, &rtt, nullptr, nullptr, nullptr) != 0) { + // Waiting for valid RTT. + return; + } + + uint32_t ntp_secs = 0, ntp_frac = 0, rtp_timestamp = 0; + if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr, + &rtp_timestamp) != 0) { + // Waiting for RTCP. + return; + } + + { + MutexLock lock(&lock_); + ntp_estimator_.UpdateRtcpTimestamp( + TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp); + } +} + +ChannelStatistics AudioIngress::GetChannelStatistics() { + ChannelStatistics channel_stats; + + // Get clockrate for current decoder ahead of jitter calculation. + uint32_t clockrate_hz = 0; + absl::optional<std::pair<int, SdpAudioFormat>> decoder = + acm_receiver_.LastDecoder(); + if (decoder) { + clockrate_hz = decoder->second.clockrate_hz; + } + + StreamStatistician* statistician = + rtp_receive_statistics_->GetStatistician(remote_ssrc_); + if (statistician) { + RtpReceiveStats stats = statistician->GetStats(); + channel_stats.packets_lost = stats.packets_lost; + channel_stats.packets_received = stats.packet_counter.packets; + channel_stats.bytes_received = stats.packet_counter.payload_bytes; + channel_stats.remote_ssrc = remote_ssrc_; + if (clockrate_hz > 0) { + channel_stats.jitter = static_cast<double>(stats.jitter) / clockrate_hz; + } + } + + // Get RTCP report using remote SSRC. + const std::vector<ReportBlockData>& report_data = + rtp_rtcp_->GetLatestReportBlockData(); + for (const ReportBlockData& block_data : report_data) { + const RTCPReportBlock& rtcp_report = block_data.report_block(); + if (rtp_rtcp_->SSRC() != rtcp_report.source_ssrc || + remote_ssrc_ != rtcp_report.sender_ssrc) { + continue; + } + RemoteRtcpStatistics remote_stat; + remote_stat.packets_lost = rtcp_report.packets_lost; + remote_stat.fraction_lost = + static_cast<double>(rtcp_report.fraction_lost) / (1 << 8); + if (clockrate_hz > 0) { + remote_stat.jitter = + static_cast<double>(rtcp_report.jitter) / clockrate_hz; + } + if (block_data.has_rtt()) { + remote_stat.round_trip_time = + static_cast<double>(block_data.last_rtt_ms()) / + rtc::kNumMillisecsPerSec; + } + remote_stat.last_report_received_timestamp_ms = + block_data.report_block_timestamp_utc_us() / + rtc::kNumMicrosecsPerMillisec; + channel_stats.remote_rtcp = remote_stat; + + // Receive only channel won't send any RTP packets. + if (!channel_stats.remote_ssrc.has_value()) { + channel_stats.remote_ssrc = remote_ssrc_; + } + break; + } + + return channel_stats; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/audio_ingress.h b/third_party/libwebrtc/audio/voip/audio_ingress.h new file mode 100644 index 0000000000..11bde7ce28 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/audio_ingress.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_AUDIO_INGRESS_H_ +#define AUDIO_VOIP_AUDIO_INGRESS_H_ + +#include <algorithm> +#include <atomic> +#include <map> +#include <memory> +#include <utility> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/audio/audio_mixer.h" +#include "api/rtp_headers.h" +#include "api/scoped_refptr.h" +#include "api/voip/voip_statistics.h" +#include "audio/audio_level.h" +#include "modules/audio_coding/acm2/acm_receiver.h" +#include "modules/audio_coding/include/audio_coding_module.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// AudioIngress handles incoming RTP/RTCP packets from the remote +// media endpoint. Received RTP packets are injected into AcmReceiver and +// when audio output thread requests for audio samples to play through system +// output such as speaker device, AudioIngress provides the samples via its +// implementation on AudioMixer::Source interface. +// +// Note that this class is originally based on ChannelReceive in +// audio/channel_receive.cc with non-audio related logic trimmed as aimed for +// smaller footprint. +class AudioIngress : public AudioMixer::Source { + public: + AudioIngress(RtpRtcpInterface* rtp_rtcp, + Clock* clock, + ReceiveStatistics* receive_statistics, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory); + ~AudioIngress() override; + + // Start or stop receiving operation of AudioIngress. + bool StartPlay(); + void StopPlay() { + playing_ = false; + output_audio_level_.ResetLevelFullRange(); + } + + // Query the state of the AudioIngress. + bool IsPlaying() const { return playing_; } + + // Set the decoder formats and payload type for AcmReceiver where the + // key type (int) of the map is the payload type of SdpAudioFormat. + void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs); + + // APIs to handle received RTP/RTCP packets from caller. + void ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet); + void ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet); + + // See comments on LevelFullRange, TotalEnergy, TotalDuration from + // audio/audio_level.h. + int GetOutputAudioLevel() const { + return output_audio_level_.LevelFullRange(); + } + double GetOutputTotalEnergy() { return output_audio_level_.TotalEnergy(); } + double GetOutputTotalDuration() { + return output_audio_level_.TotalDuration(); + } + + NetworkStatistics GetNetworkStatistics() const { + NetworkStatistics stats; + acm_receiver_.GetNetworkStatistics(&stats, + /*get_and_clear_legacy_stats=*/false); + return stats; + } + + ChannelStatistics GetChannelStatistics(); + + // Implementation of AudioMixer::Source interface. + AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( + int sampling_rate, + AudioFrame* audio_frame) override; + int Ssrc() const override { + return rtc::dchecked_cast<int>(remote_ssrc_.load()); + } + int PreferredSampleRate() const override { + // If we haven't received any RTP packet from remote and thus + // last_packet_sampling_rate is not available then use NetEq's sampling + // rate as that would be what would be used for audio output sample. + return std::max(acm_receiver_.last_packet_sample_rate_hz().value_or(0), + acm_receiver_.last_output_sample_rate_hz()); + } + + private: + // Indicates AudioIngress status as caller invokes Start/StopPlaying. + // If not playing, incoming RTP data processing is skipped, thus + // producing no data to output device. + std::atomic<bool> playing_; + + // Currently active remote ssrc from remote media endpoint. + std::atomic<uint32_t> remote_ssrc_; + + // The first rtp timestamp of the output audio frame that is used to + // calculate elasped time for subsequent audio frames. + std::atomic<int64_t> first_rtp_timestamp_; + + // Synchronizaton is handled internally by ReceiveStatistics. + ReceiveStatistics* const rtp_receive_statistics_; + + // Synchronizaton is handled internally by RtpRtcpInterface. + RtpRtcpInterface* const rtp_rtcp_; + + // Synchronizaton is handled internally by acm2::AcmReceiver. + acm2::AcmReceiver acm_receiver_; + + // Synchronizaton is handled internally by voe::AudioLevel. + voe::AudioLevel output_audio_level_; + + Mutex lock_; + + RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(lock_); + + // For receiving RTP statistics, this tracks the sampling rate value + // per payload type set when caller set via SetReceiveCodecs. + std::map<int, int> receive_codec_info_ RTC_GUARDED_BY(lock_); + + RtpTimestampUnwrapper timestamp_wrap_handler_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_AUDIO_INGRESS_H_ diff --git a/third_party/libwebrtc/audio/voip/test/BUILD.gn b/third_party/libwebrtc/audio/voip/test/BUILD.gn new file mode 100644 index 0000000000..e89f2b001a --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/BUILD.gn @@ -0,0 +1,101 @@ +# Copyright(c) 2020 The WebRTC project authors.All Rights Reserved. +# +# Use of this source code is governed by a BSD - style license +# that can be found in the LICENSE file in the root of the source +# tree.An additional intellectual property rights grant can be found +# in the file PATENTS.All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +if (rtc_include_tests) { + rtc_source_set("mock_task_queue") { + testonly = true + visibility = [ "*" ] + sources = [ "mock_task_queue.h" ] + deps = [ + "../../../api/task_queue:task_queue", + "../../../api/task_queue/test:mock_task_queue_base", + "../../../test:test_support", + ] + } + + if (!build_with_chromium) { + rtc_library("voip_core_unittests") { + testonly = true + sources = [ "voip_core_unittest.cc" ] + deps = [ + "..:voip_core", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../modules/audio_device:mock_audio_device", + "../../../modules/audio_processing:mocks", + "../../../test:audio_codec_mocks", + "../../../test:mock_transport", + "../../../test:run_loop", + "../../../test:test_support", + ] + } + } + + rtc_library("audio_channel_unittests") { + testonly = true + sources = [ "audio_channel_unittest.cc" ] + deps = [ + ":mock_task_queue", + "..:audio_channel", + "../../../api:transport_api", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/task_queue:task_queue", + "../../../modules/audio_mixer:audio_mixer_impl", + "../../../modules/audio_mixer:audio_mixer_test_utils", + "../../../modules/rtp_rtcp:rtp_rtcp", + "../../../modules/rtp_rtcp:rtp_rtcp_format", + "../../../rtc_base:logging", + "../../../test:mock_transport", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/functional:any_invocable" ] + } + + rtc_library("audio_ingress_unittests") { + testonly = true + sources = [ "audio_ingress_unittest.cc" ] + deps = [ + "..:audio_egress", + "..:audio_ingress", + "../../../api:transport_api", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../modules/audio_mixer:audio_mixer_test_utils", + "../../../modules/rtp_rtcp:rtp_rtcp", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_event", + "../../../test:mock_transport", + "../../../test:run_loop", + "../../../test:test_support", + ] + } + + rtc_library("audio_egress_unittests") { + testonly = true + sources = [ "audio_egress_unittest.cc" ] + deps = [ + "..:audio_egress", + "../../../api:transport_api", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../modules/audio_mixer:audio_mixer_test_utils", + "../../../modules/rtp_rtcp:rtp_rtcp", + "../../../modules/rtp_rtcp:rtp_rtcp_format", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_event", + "../../../test:mock_transport", + "../../../test:run_loop", + "../../../test:test_support", + ] + } +} diff --git a/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc new file mode 100644 index 0000000000..8955810429 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_channel.h" + +#include "absl/functional/any_invocable.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/call/transport.h" +#include "api/task_queue/task_queue_factory.h" +#include "audio/voip/test/mock_task_queue.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_mixer/sine_wave_generator.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "rtc_base/logging.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_transport.h" + +namespace webrtc { +namespace { + +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::Unused; + +constexpr uint64_t kStartTime = 123456789; +constexpr uint32_t kLocalSsrc = 0xdeadc0de; +constexpr int16_t kAudioLevel = 3004; // used for sine wave level +constexpr int kPcmuPayload = 0; + +class AudioChannelTest : public ::testing::Test { + public: + const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1}; + + AudioChannelTest() + : fake_clock_(kStartTime), wave_generator_(1000.0, kAudioLevel) { + task_queue_factory_ = std::make_unique<MockTaskQueueFactory>(&task_queue_); + audio_mixer_ = AudioMixerImpl::Create(); + encoder_factory_ = CreateBuiltinAudioEncoderFactory(); + decoder_factory_ = CreateBuiltinAudioDecoderFactory(); + + // By default, run the queued task immediately. + ON_CALL(task_queue_, PostTask) + .WillByDefault( + [](absl::AnyInvocable<void() &&> task) { std::move(task)(); }); + } + + void SetUp() override { audio_channel_ = CreateAudioChannel(kLocalSsrc); } + + void TearDown() override { audio_channel_ = nullptr; } + + rtc::scoped_refptr<AudioChannel> CreateAudioChannel(uint32_t ssrc) { + // Use same audio mixer here for simplicity sake as we are not checking + // audio activity of RTP in our testcases. If we need to do test on audio + // signal activity then we need to assign audio mixer for each channel. + // Also this uses the same transport object for different audio channel to + // simplify network routing logic. + rtc::scoped_refptr<AudioChannel> audio_channel = + rtc::make_ref_counted<AudioChannel>( + &transport_, ssrc, task_queue_factory_.get(), audio_mixer_.get(), + decoder_factory_); + audio_channel->SetEncoder(kPcmuPayload, kPcmuFormat, + encoder_factory_->MakeAudioEncoder( + kPcmuPayload, kPcmuFormat, absl::nullopt)); + audio_channel->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}}); + audio_channel->StartSend(); + audio_channel->StartPlay(); + return audio_channel; + } + + std::unique_ptr<AudioFrame> GetAudioFrame(int order) { + auto frame = std::make_unique<AudioFrame>(); + frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz; + frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms. + frame->num_channels_ = kPcmuFormat.num_channels; + frame->timestamp_ = frame->samples_per_channel_ * order; + wave_generator_.GenerateNextFrame(frame.get()); + return frame; + } + + SimulatedClock fake_clock_; + SineWaveGenerator wave_generator_; + NiceMock<MockTransport> transport_; + NiceMock<MockTaskQueue> task_queue_; + std::unique_ptr<TaskQueueFactory> task_queue_factory_; + rtc::scoped_refptr<AudioMixer> audio_mixer_; + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; + rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_; + rtc::scoped_refptr<AudioChannel> audio_channel_; +}; + +// Validate RTP packet generation by feeding audio frames with sine wave. +// Resulted RTP packet is looped back into AudioChannel and gets decoded into +// audio frame to see if it has some signal to indicate its validity. +TEST_F(AudioChannelTest, PlayRtpByLocalLoop) { + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + audio_channel_->ReceivedRTPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp)); + + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + AudioFrame empty_frame, audio_frame; + empty_frame.Mute(); + empty_frame.mutable_data(); // This will zero out the data. + audio_frame.CopyFrom(empty_frame); + audio_mixer_->Mix(/*number_of_channels*/ 1, &audio_frame); + + // We expect now audio frame to pick up something. + EXPECT_NE(memcmp(empty_frame.data(), audio_frame.data(), + AudioFrame::kMaxDataSizeBytes), + 0); +} + +// Validate assigned local SSRC is resulted in RTP packet. +TEST_F(AudioChannelTest, VerifyLocalSsrcAsAssigned) { + RtpPacketReceived rtp; + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + rtp.Parse(packet, length); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp)); + + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + EXPECT_EQ(rtp.Ssrc(), kLocalSsrc); +} + +// Check metrics after processing an RTP packet. +TEST_F(AudioChannelTest, TestIngressStatistics) { + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + audio_channel_->ReceivedRTPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp)); + + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + absl::optional<IngressStatistics> ingress_stats = + audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 160ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + // To extract the jitter buffer length in millisecond, jitter_buffer_delay_ms + // needs to be divided by jitter_buffer_emitted_count (number of samples). + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.02); + + // Now without any RTP pending in jitter buffer pull more. + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Send another RTP packet to intentionally break PLC. + audio_sender->SendAudioData(GetAudioFrame(2)); + audio_sender->SendAudioData(GetAudioFrame(3)); + + ingress_stats = audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 320ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.04); + + // Pull the last RTP packet. + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + ingress_stats = audio_channel_->GetIngressStatistics(); + EXPECT_TRUE(ingress_stats); + EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 480ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL); + EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL); + EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 3200ULL); + EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 320ULL); + EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL); + EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0); + EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0); + EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.06); +} + +// Check ChannelStatistics metric after processing RTP and RTCP packets. +TEST_F(AudioChannelTest, TestChannelStatistics) { + auto loop_rtp = [&](const uint8_t* packet, size_t length, Unused) { + audio_channel_->ReceivedRTPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + auto loop_rtcp = [&](const uint8_t* packet, size_t length) { + audio_channel_->ReceivedRTCPPacket( + rtc::ArrayView<const uint8_t>(packet, length)); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp)); + EXPECT_CALL(transport_, SendRtcp).WillRepeatedly(Invoke(loop_rtcp)); + + // Simulate microphone giving audio frame (10 ms). This will trigger tranport + // to send RTP as handled in loop_rtp above. + auto audio_sender = audio_channel_->GetAudioSender(); + audio_sender->SendAudioData(GetAudioFrame(0)); + audio_sender->SendAudioData(GetAudioFrame(1)); + + // Simulate speaker requesting audio frame (10 ms). This will trigger VoIP + // engine to fetch audio samples from RTP packets stored in jitter buffer. + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Force sending RTCP SR report in order to have remote_rtcp field available + // in channel statistics. This will trigger tranport to send RTCP as handled + // in loop_rtcp above. + audio_channel_->SendRTCPReportForTesting(kRtcpSr); + + absl::optional<ChannelStatistics> channel_stats = + audio_channel_->GetChannelStatistics(); + EXPECT_TRUE(channel_stats); + + EXPECT_EQ(channel_stats->packets_sent, 1ULL); + EXPECT_EQ(channel_stats->bytes_sent, 160ULL); + + EXPECT_EQ(channel_stats->packets_received, 1ULL); + EXPECT_EQ(channel_stats->bytes_received, 160ULL); + EXPECT_EQ(channel_stats->jitter, 0); + EXPECT_EQ(channel_stats->packets_lost, 0); + EXPECT_EQ(channel_stats->remote_ssrc.value(), kLocalSsrc); + + EXPECT_TRUE(channel_stats->remote_rtcp.has_value()); + + EXPECT_EQ(channel_stats->remote_rtcp->jitter, 0); + EXPECT_EQ(channel_stats->remote_rtcp->packets_lost, 0); + EXPECT_EQ(channel_stats->remote_rtcp->fraction_lost, 0); + EXPECT_GT(channel_stats->remote_rtcp->last_report_received_timestamp_ms, 0); + EXPECT_FALSE(channel_stats->remote_rtcp->round_trip_time.has_value()); +} + +// Check ChannelStatistics RTT metric after processing RTP and RTCP packets +// using three audio channels where each represents media endpoint. +// +// 1) AC1 <- RTP/RTCP -> AC2 +// 2) AC1 <- RTP/RTCP -> AC3 +// +// During step 1), AC1 should be able to check RTT from AC2's SSRC. +// During step 2), AC1 should be able to check RTT from AC3's SSRC. +TEST_F(AudioChannelTest, RttIsAvailableAfterChangeOfRemoteSsrc) { + // Create AC2 and AC3. + constexpr uint32_t kAc2Ssrc = 0xdeadbeef; + constexpr uint32_t kAc3Ssrc = 0xdeafbeef; + + auto ac_2 = CreateAudioChannel(kAc2Ssrc); + auto ac_3 = CreateAudioChannel(kAc3Ssrc); + + auto send_recv_rtp = [&](rtc::scoped_refptr<AudioChannel> rtp_sender, + rtc::scoped_refptr<AudioChannel> rtp_receiver) { + // Setup routing logic via transport_. + auto route_rtp = [&](const uint8_t* packet, size_t length, Unused) { + rtp_receiver->ReceivedRTPPacket(rtc::MakeArrayView(packet, length)); + return true; + }; + ON_CALL(transport_, SendRtp).WillByDefault(route_rtp); + + // This will trigger route_rtp callback via transport_. + rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(0)); + rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(1)); + + // Process received RTP in receiver. + AudioFrame audio_frame; + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame); + + // Revert to default to avoid using reference in route_rtp lambda. + ON_CALL(transport_, SendRtp).WillByDefault(Return(true)); + }; + + auto send_recv_rtcp = [&](rtc::scoped_refptr<AudioChannel> rtcp_sender, + rtc::scoped_refptr<AudioChannel> rtcp_receiver) { + // Setup routing logic via transport_. + auto route_rtcp = [&](const uint8_t* packet, size_t length) { + rtcp_receiver->ReceivedRTCPPacket(rtc::MakeArrayView(packet, length)); + return true; + }; + ON_CALL(transport_, SendRtcp).WillByDefault(route_rtcp); + + // This will trigger route_rtcp callback via transport_. + rtcp_sender->SendRTCPReportForTesting(kRtcpSr); + + // Revert to default to avoid using reference in route_rtcp lambda. + ON_CALL(transport_, SendRtcp).WillByDefault(Return(true)); + }; + + // AC1 <-- RTP/RTCP --> AC2 + send_recv_rtp(audio_channel_, ac_2); + send_recv_rtp(ac_2, audio_channel_); + send_recv_rtcp(audio_channel_, ac_2); + send_recv_rtcp(ac_2, audio_channel_); + + absl::optional<ChannelStatistics> channel_stats = + audio_channel_->GetChannelStatistics(); + ASSERT_TRUE(channel_stats); + EXPECT_EQ(channel_stats->remote_ssrc, kAc2Ssrc); + ASSERT_TRUE(channel_stats->remote_rtcp); + EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0); + + // AC1 <-- RTP/RTCP --> AC3 + send_recv_rtp(audio_channel_, ac_3); + send_recv_rtp(ac_3, audio_channel_); + send_recv_rtcp(audio_channel_, ac_3); + send_recv_rtcp(ac_3, audio_channel_); + + channel_stats = audio_channel_->GetChannelStatistics(); + ASSERT_TRUE(channel_stats); + EXPECT_EQ(channel_stats->remote_ssrc, kAc3Ssrc); + ASSERT_TRUE(channel_stats->remote_rtcp); + EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc new file mode 100644 index 0000000000..34c5585347 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_egress.h" + +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/call/transport.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "modules/audio_mixer/sine_wave_generator.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_transport.h" +#include "test/run_loop.h" + +namespace webrtc { +namespace { + +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Unused; + +std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpStack(Clock* clock, + Transport* transport, + uint32_t remote_ssrc) { + RtpRtcpInterface::Configuration rtp_config; + rtp_config.clock = clock; + rtp_config.audio = true; + rtp_config.rtcp_report_interval_ms = 5000; + rtp_config.outgoing_transport = transport; + rtp_config.local_media_ssrc = remote_ssrc; + auto rtp_rtcp = ModuleRtpRtcpImpl2::Create(rtp_config); + rtp_rtcp->SetSendingMediaStatus(false); + rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); + return rtp_rtcp; +} + +constexpr int16_t kAudioLevel = 3004; // Used for sine wave level. + +// AudioEgressTest configures audio egress by using Rtp Stack, fake clock, +// and task queue factory. Encoder factory is needed to create codec and +// configure the RTP stack in audio egress. +class AudioEgressTest : public ::testing::Test { + public: + static constexpr uint16_t kSeqNum = 12345; + static constexpr uint64_t kStartTime = 123456789; + static constexpr uint32_t kRemoteSsrc = 0xDEADBEEF; + const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1}; + + AudioEgressTest() + : fake_clock_(kStartTime), wave_generator_(1000.0, kAudioLevel) { + task_queue_factory_ = CreateDefaultTaskQueueFactory(); + encoder_factory_ = CreateBuiltinAudioEncoderFactory(); + } + + // Prepare test on audio egress by using PCMu codec with specific + // sequence number and its status to be running. + void SetUp() override { + rtp_rtcp_ = CreateRtpStack(&fake_clock_, &transport_, kRemoteSsrc); + egress_ = std::make_unique<AudioEgress>(rtp_rtcp_.get(), &fake_clock_, + task_queue_factory_.get()); + constexpr int kPcmuPayload = 0; + egress_->SetEncoder(kPcmuPayload, kPcmuFormat, + encoder_factory_->MakeAudioEncoder( + kPcmuPayload, kPcmuFormat, absl::nullopt)); + egress_->StartSend(); + rtp_rtcp_->SetSequenceNumber(kSeqNum); + rtp_rtcp_->SetSendingStatus(true); + } + + // Make sure we have shut down rtp stack and reset egress for each test. + void TearDown() override { + egress_->StopSend(); + rtp_rtcp_->SetSendingStatus(false); + egress_.reset(); + rtp_rtcp_.reset(); + } + + // Create an audio frame prepared for pcmu encoding. Timestamp is + // increased per RTP specification which is the number of samples it contains. + // Wave generator writes sine wave which has expected high level set + // by kAudioLevel. + std::unique_ptr<AudioFrame> GetAudioFrame(int order) { + auto frame = std::make_unique<AudioFrame>(); + frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz; + frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms. + frame->num_channels_ = kPcmuFormat.num_channels; + frame->timestamp_ = frame->samples_per_channel_ * order; + wave_generator_.GenerateNextFrame(frame.get()); + return frame; + } + + test::RunLoop run_loop_; + // SimulatedClock doesn't directly affect this testcase as the the + // AudioFrame's timestamp is driven by GetAudioFrame. + SimulatedClock fake_clock_; + NiceMock<MockTransport> transport_; + SineWaveGenerator wave_generator_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; + std::unique_ptr<TaskQueueFactory> task_queue_factory_; + rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_; + std::unique_ptr<AudioEgress> egress_; +}; + +TEST_F(AudioEgressTest, SendingStatusAfterStartAndStop) { + EXPECT_TRUE(egress_->IsSending()); + egress_->StopSend(); + EXPECT_FALSE(egress_->IsSending()); +} + +TEST_F(AudioEgressTest, ProcessAudioWithMute) { + constexpr int kExpected = 10; + rtc::Event event; + int rtp_count = 0; + RtpPacketReceived rtp; + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + rtp.Parse(packet, length); + if (++rtp_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + egress_->SetMute(true); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(TimeDelta::Seconds(1)); + EXPECT_EQ(rtp_count, kExpected); + + // we expect on pcmu payload to result in 255 for silenced payload + RTPHeader header; + rtp.GetHeader(&header); + size_t packet_length = rtp.size(); + size_t payload_length = packet_length - header.headerLength; + size_t payload_data_length = payload_length - header.paddingLength; + const uint8_t* payload = rtp.data() + header.headerLength; + for (size_t i = 0; i < payload_data_length; ++i) { + EXPECT_EQ(*payload++, 255); + } +} + +TEST_F(AudioEgressTest, ProcessAudioWithSineWave) { + constexpr int kExpected = 10; + rtc::Event event; + int rtp_count = 0; + RtpPacketReceived rtp; + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + rtp.Parse(packet, length); + if (++rtp_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(TimeDelta::Seconds(1)); + EXPECT_EQ(rtp_count, kExpected); + + // we expect on pcmu to result in < 255 for payload with sine wave + RTPHeader header; + rtp.GetHeader(&header); + size_t packet_length = rtp.size(); + size_t payload_length = packet_length - header.headerLength; + size_t payload_data_length = payload_length - header.paddingLength; + const uint8_t* payload = rtp.data() + header.headerLength; + for (size_t i = 0; i < payload_data_length; ++i) { + EXPECT_NE(*payload++, 255); + } +} + +TEST_F(AudioEgressTest, SkipAudioEncodingAfterStopSend) { + constexpr int kExpected = 10; + rtc::Event event; + int rtp_count = 0; + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + if (++rtp_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(TimeDelta::Seconds(1)); + EXPECT_EQ(rtp_count, kExpected); + + // Now stop send and yet feed more data. + egress_->StopSend(); + + // It should be safe to exit the test case while encoder_queue_ has + // outstanding data to process. We are making sure that this doesn't + // result in crahses or sanitizer errors due to remaining data. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } +} + +TEST_F(AudioEgressTest, ChangeEncoderFromPcmuToOpus) { + absl::optional<SdpAudioFormat> pcmu = egress_->GetEncoderFormat(); + EXPECT_TRUE(pcmu); + EXPECT_EQ(pcmu->clockrate_hz, kPcmuFormat.clockrate_hz); + EXPECT_EQ(pcmu->num_channels, kPcmuFormat.num_channels); + + constexpr int kOpusPayload = 120; + const SdpAudioFormat kOpusFormat = {"opus", 48000, 2}; + + egress_->SetEncoder(kOpusPayload, kOpusFormat, + encoder_factory_->MakeAudioEncoder( + kOpusPayload, kOpusFormat, absl::nullopt)); + + absl::optional<SdpAudioFormat> opus = egress_->GetEncoderFormat(); + EXPECT_TRUE(opus); + EXPECT_EQ(opus->clockrate_hz, kOpusFormat.clockrate_hz); + EXPECT_EQ(opus->num_channels, kOpusFormat.num_channels); +} + +TEST_F(AudioEgressTest, SendDTMF) { + constexpr int kExpected = 7; + constexpr int kPayloadType = 100; + constexpr int kDurationMs = 100; + constexpr int kSampleRate = 8000; + constexpr int kEvent = 3; + + egress_->RegisterTelephoneEventType(kPayloadType, kSampleRate); + // 100 ms duration will produce total 7 DTMF + // 1 @ 20 ms, 2 @ 40 ms, 3 @ 60 ms, 4 @ 80 ms + // 5, 6, 7 @ 100 ms (last one sends 3 dtmf) + egress_->SendTelephoneEvent(kEvent, kDurationMs); + + rtc::Event event; + int dtmf_count = 0; + auto is_dtmf = [&](RtpPacketReceived& rtp) { + return (rtp.PayloadType() == kPayloadType && + rtp.SequenceNumber() == kSeqNum + dtmf_count && + rtp.padding_size() == 0 && rtp.Marker() == (dtmf_count == 0) && + rtp.Ssrc() == kRemoteSsrc); + }; + + // It's possible that we may have actual audio RTP packets along with + // DTMF packtets. We are only interested in the exact number of DTMF + // packets rtp stack is emitting. + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + RtpPacketReceived rtp; + rtp.Parse(packet, length); + if (is_dtmf(rtp) && ++dtmf_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(TimeDelta::Seconds(1)); + EXPECT_EQ(dtmf_count, kExpected); +} + +TEST_F(AudioEgressTest, TestAudioInputLevelAndEnergyDuration) { + // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to + // get audio level from input source. + constexpr int kExpected = 6; + rtc::Event event; + int rtp_count = 0; + auto rtp_sent = [&](const uint8_t* packet, size_t length, Unused) { + if (++rtp_count == kExpected) { + event.Set(); + } + return true; + }; + + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent)); + + // Two 10 ms audio frames will result in rtp packet with ptime 20. + for (size_t i = 0; i < kExpected * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); + EXPECT_EQ(rtp_count, kExpected); + + constexpr double kExpectedEnergy = 0.00016809565587789564; + constexpr double kExpectedDuration = 0.11999999999999998; + + EXPECT_EQ(egress_->GetInputAudioLevel(), kAudioLevel); + EXPECT_DOUBLE_EQ(egress_->GetInputTotalEnergy(), kExpectedEnergy); + EXPECT_DOUBLE_EQ(egress_->GetInputTotalDuration(), kExpectedDuration); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc new file mode 100644 index 0000000000..3c309dbf82 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/audio_ingress.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/call/transport.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "audio/voip/audio_egress.h" +#include "modules/audio_mixer/sine_wave_generator.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_transport.h" +#include "test/run_loop.h" + +namespace webrtc { +namespace { + +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Unused; + +constexpr int16_t kAudioLevel = 3004; // Used for sine wave level. + +class AudioIngressTest : public ::testing::Test { + public: + const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1}; + + AudioIngressTest() + : fake_clock_(123456789), wave_generator_(1000.0, kAudioLevel) { + receive_statistics_ = ReceiveStatistics::Create(&fake_clock_); + + RtpRtcpInterface::Configuration rtp_config; + rtp_config.clock = &fake_clock_; + rtp_config.audio = true; + rtp_config.receive_statistics = receive_statistics_.get(); + rtp_config.rtcp_report_interval_ms = 5000; + rtp_config.outgoing_transport = &transport_; + rtp_config.local_media_ssrc = 0xdeadc0de; + rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(rtp_config); + + rtp_rtcp_->SetSendingMediaStatus(false); + rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound); + + task_queue_factory_ = CreateDefaultTaskQueueFactory(); + encoder_factory_ = CreateBuiltinAudioEncoderFactory(); + decoder_factory_ = CreateBuiltinAudioDecoderFactory(); + } + + void SetUp() override { + constexpr int kPcmuPayload = 0; + ingress_ = std::make_unique<AudioIngress>(rtp_rtcp_.get(), &fake_clock_, + receive_statistics_.get(), + decoder_factory_); + ingress_->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}}); + + egress_ = std::make_unique<AudioEgress>(rtp_rtcp_.get(), &fake_clock_, + task_queue_factory_.get()); + egress_->SetEncoder(kPcmuPayload, kPcmuFormat, + encoder_factory_->MakeAudioEncoder( + kPcmuPayload, kPcmuFormat, absl::nullopt)); + egress_->StartSend(); + ingress_->StartPlay(); + rtp_rtcp_->SetSendingStatus(true); + } + + void TearDown() override { + rtp_rtcp_->SetSendingStatus(false); + ingress_->StopPlay(); + egress_->StopSend(); + egress_.reset(); + ingress_.reset(); + } + + std::unique_ptr<AudioFrame> GetAudioFrame(int order) { + auto frame = std::make_unique<AudioFrame>(); + frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz; + frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms. + frame->num_channels_ = kPcmuFormat.num_channels; + frame->timestamp_ = frame->samples_per_channel_ * order; + wave_generator_.GenerateNextFrame(frame.get()); + return frame; + } + + test::RunLoop run_loop_; + SimulatedClock fake_clock_; + SineWaveGenerator wave_generator_; + NiceMock<MockTransport> transport_; + std::unique_ptr<ReceiveStatistics> receive_statistics_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; + rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_; + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; + std::unique_ptr<TaskQueueFactory> task_queue_factory_; + std::unique_ptr<AudioIngress> ingress_; + std::unique_ptr<AudioEgress> egress_; +}; + +TEST_F(AudioIngressTest, PlayingAfterStartAndStop) { + EXPECT_EQ(ingress_->IsPlaying(), true); + ingress_->StopPlay(); + EXPECT_EQ(ingress_->IsPlaying(), false); +} + +TEST_F(AudioIngressTest, GetAudioFrameAfterRtpReceived) { + rtc::Event event; + auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { + ingress_->ReceivedRTPPacket(rtc::ArrayView<const uint8_t>(packet, length)); + event.Set(); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); + egress_->SendAudioData(GetAudioFrame(0)); + egress_->SendAudioData(GetAudioFrame(1)); + event.Wait(TimeDelta::Seconds(1)); + + AudioFrame audio_frame; + EXPECT_EQ( + ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), + AudioMixer::Source::AudioFrameInfo::kNormal); + EXPECT_FALSE(audio_frame.muted()); + EXPECT_EQ(audio_frame.num_channels_, 1u); + EXPECT_EQ(audio_frame.samples_per_channel_, + static_cast<size_t>(kPcmuFormat.clockrate_hz / 100)); + EXPECT_EQ(audio_frame.sample_rate_hz_, kPcmuFormat.clockrate_hz); + EXPECT_NE(audio_frame.timestamp_, 0u); + EXPECT_EQ(audio_frame.elapsed_time_ms_, 0); +} + +TEST_F(AudioIngressTest, TestSpeechOutputLevelAndEnergyDuration) { + // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to + // get audio level from output source. + constexpr int kNumRtp = 6; + int rtp_count = 0; + rtc::Event event; + auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { + ingress_->ReceivedRTPPacket(rtc::ArrayView<const uint8_t>(packet, length)); + if (++rtp_count == kNumRtp) { + event.Set(); + } + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); + for (int i = 0; i < kNumRtp * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); + + for (int i = 0; i < kNumRtp * 2; ++i) { + AudioFrame audio_frame; + EXPECT_EQ( + ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), + AudioMixer::Source::AudioFrameInfo::kNormal); + } + EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel); + + constexpr double kExpectedEnergy = 0.00016809565587789564; + constexpr double kExpectedDuration = 0.11999999999999998; + + EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalEnergy(), kExpectedEnergy); + EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalDuration(), kExpectedDuration); +} + +TEST_F(AudioIngressTest, PreferredSampleRate) { + rtc::Event event; + auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { + ingress_->ReceivedRTPPacket(rtc::ArrayView<const uint8_t>(packet, length)); + event.Set(); + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); + egress_->SendAudioData(GetAudioFrame(0)); + egress_->SendAudioData(GetAudioFrame(1)); + event.Wait(TimeDelta::Seconds(1)); + + AudioFrame audio_frame; + EXPECT_EQ( + ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), + AudioMixer::Source::AudioFrameInfo::kNormal); + EXPECT_EQ(ingress_->PreferredSampleRate(), kPcmuFormat.clockrate_hz); +} + +// This test highlights the case where caller invokes StopPlay() which then +// AudioIngress should play silence frame afterwards. +TEST_F(AudioIngressTest, GetMutedAudioFrameAfterRtpReceivedAndStopPlay) { + // StopPlay before we start sending RTP packet with sine wave. + ingress_->StopPlay(); + + // Send 6 RTP packets to generate more than 100 ms audio sample to get + // valid speech level. + constexpr int kNumRtp = 6; + int rtp_count = 0; + rtc::Event event; + auto handle_rtp = [&](const uint8_t* packet, size_t length, Unused) { + ingress_->ReceivedRTPPacket(rtc::ArrayView<const uint8_t>(packet, length)); + if (++rtp_count == kNumRtp) { + event.Set(); + } + return true; + }; + EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp)); + for (int i = 0; i < kNumRtp * 2; i++) { + egress_->SendAudioData(GetAudioFrame(i)); + fake_clock_.AdvanceTimeMilliseconds(10); + } + event.Wait(/*give_up_after=*/TimeDelta::Seconds(1)); + + for (int i = 0; i < kNumRtp * 2; ++i) { + AudioFrame audio_frame; + EXPECT_EQ( + ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame), + AudioMixer::Source::AudioFrameInfo::kMuted); + const int16_t* audio_data = audio_frame.data(); + size_t length = + audio_frame.samples_per_channel_ * audio_frame.num_channels_; + for (size_t j = 0; j < length; ++j) { + EXPECT_EQ(audio_data[j], 0); + } + } + + // Now we should still see valid speech output level as StopPlay won't affect + // the measurement. + EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/test/mock_task_queue.h b/third_party/libwebrtc/audio/voip/test/mock_task_queue.h new file mode 100644 index 0000000000..547b0d3f75 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/mock_task_queue.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ +#define AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ + +#include <memory> + +#include "api/task_queue/task_queue_factory.h" +#include "api/task_queue/test/mock_task_queue_base.h" +#include "test/gmock.h" + +namespace webrtc { + +// MockTaskQueue enables immediate task run from global TaskQueueBase. +// It's necessary for some tests depending on TaskQueueBase internally. +class MockTaskQueue : public MockTaskQueueBase { + public: + MockTaskQueue() : current_(this) {} + + // Delete is deliberately defined as no-op as MockTaskQueue is expected to + // hold onto current global TaskQueueBase throughout the testing. + void Delete() override {} + + private: + CurrentTaskQueueSetter current_; +}; + +class MockTaskQueueFactory : public TaskQueueFactory { + public: + explicit MockTaskQueueFactory(MockTaskQueue* task_queue) + : task_queue_(task_queue) {} + + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue( + absl::string_view name, + Priority priority) const override { + // Default MockTaskQueue::Delete is no-op, therefore it's safe to pass the + // raw pointer. + return std::unique_ptr<TaskQueueBase, TaskQueueDeleter>(task_queue_); + } + + private: + MockTaskQueue* task_queue_; +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_ diff --git a/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc b/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc new file mode 100644 index 0000000000..b432506b12 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/voip_core.h" + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "modules/audio_device/include/mock_audio_device.h" +#include "modules/audio_processing/include/mock_audio_processing.h" +#include "test/gtest.h" +#include "test/mock_transport.h" +#include "test/run_loop.h" + +namespace webrtc { +namespace { + +using ::testing::NiceMock; +using ::testing::Return; + +constexpr int kPcmuPayload = 0; +constexpr int kPcmuSampleRateHz = 8000; +constexpr int kDtmfEventDurationMs = 1000; +constexpr DtmfEvent kDtmfEventCode = DtmfEvent::kDigitZero; + +class VoipCoreTest : public ::testing::Test { + public: + const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1}; + + VoipCoreTest() { audio_device_ = test::MockAudioDeviceModule::CreateNice(); } + + void SetUp() override { + auto encoder_factory = CreateBuiltinAudioEncoderFactory(); + auto decoder_factory = CreateBuiltinAudioDecoderFactory(); + rtc::scoped_refptr<AudioProcessing> audio_processing = + rtc::make_ref_counted<NiceMock<test::MockAudioProcessing>>(); + + voip_core_ = std::make_unique<VoipCore>( + std::move(encoder_factory), std::move(decoder_factory), + CreateDefaultTaskQueueFactory(), audio_device_, + std::move(audio_processing)); + } + + test::RunLoop run_loop_; + std::unique_ptr<VoipCore> voip_core_; + NiceMock<MockTransport> transport_; + rtc::scoped_refptr<test::MockAudioDeviceModule> audio_device_; +}; + +// Validate expected API calls that involves with VoipCore. Some verification is +// involved with checking mock audio device. +TEST_F(VoipCoreTest, BasicVoipCoreOperation) { + // Program mock as non-operational and ready to start. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(false)); + EXPECT_CALL(*audio_device_, Playing()).WillOnce(Return(false)); + EXPECT_CALL(*audio_device_, InitRecording()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, InitPlayout()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, StartRecording()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, StartPlayout()).WillOnce(Return(0)); + + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kOk); + + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kOk); + + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kOk); + + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kOk); + + // Program mock as operational that is ready to be stopped. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true)); + EXPECT_CALL(*audio_device_, Playing()).WillOnce(Return(true)); + EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, StopPlayout()).WillOnce(Return(0)); + + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, ExpectFailToUseReleasedChannelId) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + // Release right after creation. + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); + + // Now use released channel. + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kInvalidArgument); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kInvalidArgument); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kInvalidArgument); + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kInvalidArgument); +} + +TEST_F(VoipCoreTest, SendDtmfEventWithoutRegistering) { + // Program mock as non-operational and ready to start send. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(false)); + EXPECT_CALL(*audio_device_, InitRecording()).WillOnce(Return(0)); + EXPECT_CALL(*audio_device_, StartRecording()).WillOnce(Return(0)); + + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk); + // Send Dtmf event without registering beforehand, thus payload + // type is not set and kFailedPrecondition is expected. + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kFailedPrecondition); + + // Program mock as sending and is ready to be stopped. + EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true)); + EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0)); + + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, SendDtmfEventWithoutStartSend) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload, + kPcmuSampleRateHz), + VoipResult::kOk); + + // Send Dtmf event without calling StartSend beforehand, thus + // Dtmf events cannot be sent and kFailedPrecondition is expected. + EXPECT_EQ( + voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs), + VoipResult::kFailedPrecondition); + + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, StartSendAndPlayoutWithoutSettingCodec) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + // Call StartSend and StartPlayout without setting send/receive + // codec. Code should see that codecs aren't set and return false. + EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kFailedPrecondition); + EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kFailedPrecondition); + + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +TEST_F(VoipCoreTest, StopSendAndPlayoutWithoutStarting) { + auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de); + + EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat), + VoipResult::kOk); + EXPECT_EQ( + voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}), + VoipResult::kOk); + + // Call StopSend and StopPlayout without starting them in + // the first place. Should see that it is already in the + // stopped state and return true. + EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk); + EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk); + + EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/voip_core.cc b/third_party/libwebrtc/audio/voip/voip_core.cc new file mode 100644 index 0000000000..8df1c594aa --- /dev/null +++ b/third_party/libwebrtc/audio/voip/voip_core.cc @@ -0,0 +1,500 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "audio/voip/voip_core.h" + +#include <algorithm> +#include <memory> +#include <utility> + +#include "api/audio_codecs/audio_format.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +namespace { + +// For Windows, use specific enum type to initialize default audio device as +// defined in AudioDeviceModule::WindowsDeviceType. +#if defined(WEBRTC_WIN) +constexpr AudioDeviceModule::WindowsDeviceType kAudioDeviceId = + AudioDeviceModule::WindowsDeviceType::kDefaultCommunicationDevice; +#else +constexpr uint16_t kAudioDeviceId = 0; +#endif // defined(WEBRTC_WIN) + +// Maximum value range limit on ChannelId. This can be increased without any +// side effect and only set at this moderate value for better readability for +// logging. +static constexpr int kMaxChannelId = 100000; + +} // namespace + +VoipCore::VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, + std::unique_ptr<TaskQueueFactory> task_queue_factory, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module, + rtc::scoped_refptr<AudioProcessing> audio_processing) { + encoder_factory_ = std::move(encoder_factory); + decoder_factory_ = std::move(decoder_factory); + task_queue_factory_ = std::move(task_queue_factory); + audio_device_module_ = std::move(audio_device_module); + audio_processing_ = std::move(audio_processing); + audio_mixer_ = AudioMixerImpl::Create(); + + // AudioTransportImpl depends on audio mixer and audio processing instances. + audio_transport_ = std::make_unique<AudioTransportImpl>( + audio_mixer_.get(), audio_processing_.get(), nullptr); +} + +bool VoipCore::InitializeIfNeeded() { + // `audio_device_module_` internally owns a lock and the whole logic here + // needs to be executed atomically once using another lock in VoipCore. + // Further changes in this method will need to make sure that no deadlock is + // introduced in the future. + MutexLock lock(&lock_); + + if (initialized_) { + return true; + } + + // Initialize ADM. + if (audio_device_module_->Init() != 0) { + RTC_LOG(LS_ERROR) << "Failed to initialize the ADM."; + return false; + } + + // Note that failures on initializing default recording/speaker devices are + // not considered to be fatal here. In certain case, caller may not care about + // recording device functioning (e.g webinar where only speaker is available). + // It's also possible that there are other audio devices available that may + // work. + + // Initialize default speaker device. + if (audio_device_module_->SetPlayoutDevice(kAudioDeviceId) != 0) { + RTC_LOG(LS_WARNING) << "Unable to set playout device."; + } + if (audio_device_module_->InitSpeaker() != 0) { + RTC_LOG(LS_WARNING) << "Unable to access speaker."; + } + + // Initialize default recording device. + if (audio_device_module_->SetRecordingDevice(kAudioDeviceId) != 0) { + RTC_LOG(LS_WARNING) << "Unable to set recording device."; + } + if (audio_device_module_->InitMicrophone() != 0) { + RTC_LOG(LS_WARNING) << "Unable to access microphone."; + } + + // Set number of channels on speaker device. + bool available = false; + if (audio_device_module_->StereoPlayoutIsAvailable(&available) != 0) { + RTC_LOG(LS_WARNING) << "Unable to query stereo playout."; + } + if (audio_device_module_->SetStereoPlayout(available) != 0) { + RTC_LOG(LS_WARNING) << "Unable to set mono/stereo playout mode."; + } + + // Set number of channels on recording device. + available = false; + if (audio_device_module_->StereoRecordingIsAvailable(&available) != 0) { + RTC_LOG(LS_WARNING) << "Unable to query stereo recording."; + } + if (audio_device_module_->SetStereoRecording(available) != 0) { + RTC_LOG(LS_WARNING) << "Unable to set stereo recording mode."; + } + + if (audio_device_module_->RegisterAudioCallback(audio_transport_.get()) != + 0) { + RTC_LOG(LS_WARNING) << "Unable to register audio callback."; + } + + initialized_ = true; + + return true; +} + +ChannelId VoipCore::CreateChannel(Transport* transport, + absl::optional<uint32_t> local_ssrc) { + ChannelId channel_id; + + // Set local ssrc to random if not set by caller. + if (!local_ssrc) { + Random random(rtc::TimeMicros()); + local_ssrc = random.Rand<uint32_t>(); + } + + rtc::scoped_refptr<AudioChannel> channel = + rtc::make_ref_counted<AudioChannel>(transport, local_ssrc.value(), + task_queue_factory_.get(), + audio_mixer_.get(), decoder_factory_); + + { + MutexLock lock(&lock_); + + channel_id = static_cast<ChannelId>(next_channel_id_); + channels_[channel_id] = channel; + next_channel_id_++; + if (next_channel_id_ >= kMaxChannelId) { + next_channel_id_ = 0; + } + } + + // Set ChannelId in audio channel for logging/debugging purpose. + channel->SetId(channel_id); + + return channel_id; +} + +VoipResult VoipCore::ReleaseChannel(ChannelId channel_id) { + // Destroy channel outside of the lock. + rtc::scoped_refptr<AudioChannel> channel; + + bool no_channels_after_release = false; + + { + MutexLock lock(&lock_); + + auto iter = channels_.find(channel_id); + if (iter != channels_.end()) { + channel = std::move(iter->second); + channels_.erase(iter); + } + + no_channels_after_release = channels_.empty(); + } + + VoipResult status_code = VoipResult::kOk; + if (!channel) { + RTC_LOG(LS_WARNING) << "Channel " << channel_id << " not found"; + status_code = VoipResult::kInvalidArgument; + } + + if (no_channels_after_release) { + // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `channel` + // here. + channel = nullptr; + + // Make sure to stop playout on ADM if it is playing. + if (audio_device_module_->Playing()) { + if (audio_device_module_->StopPlayout() != 0) { + RTC_LOG(LS_WARNING) << "StopPlayout failed"; + status_code = VoipResult::kInternal; + } + } + } + + return status_code; +} + +rtc::scoped_refptr<AudioChannel> VoipCore::GetChannel(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel; + { + MutexLock lock(&lock_); + auto iter = channels_.find(channel_id); + if (iter != channels_.end()) { + channel = iter->second; + } + } + if (!channel) { + RTC_LOG(LS_ERROR) << "Channel " << channel_id << " not found"; + } + return channel; +} + +bool VoipCore::UpdateAudioTransportWithSenders() { + std::vector<AudioSender*> audio_senders; + + // Gather a list of audio channel that are currently sending along with + // highest sampling rate and channel numbers to configure into audio + // transport. + int max_sampling_rate = 8000; + size_t max_num_channels = 1; + { + MutexLock lock(&lock_); + // Reserve to prevent run time vector re-allocation. + audio_senders.reserve(channels_.size()); + for (auto kv : channels_) { + rtc::scoped_refptr<AudioChannel>& channel = kv.second; + if (channel->IsSendingMedia()) { + auto encoder_format = channel->GetEncoderFormat(); + if (!encoder_format) { + RTC_LOG(LS_ERROR) + << "channel " << channel->GetId() << " encoder is not set"; + continue; + } + audio_senders.push_back(channel->GetAudioSender()); + max_sampling_rate = + std::max(max_sampling_rate, encoder_format->clockrate_hz); + max_num_channels = + std::max(max_num_channels, encoder_format->num_channels); + } + } + } + + audio_transport_->UpdateAudioSenders(audio_senders, max_sampling_rate, + max_num_channels); + + // Depending on availability of senders, turn on or off ADM recording. + if (!audio_senders.empty()) { + // Initialize audio device module and default device if needed. + if (!InitializeIfNeeded()) { + return false; + } + + if (!audio_device_module_->Recording()) { + if (audio_device_module_->InitRecording() != 0) { + RTC_LOG(LS_ERROR) << "InitRecording failed"; + return false; + } + if (audio_device_module_->StartRecording() != 0) { + RTC_LOG(LS_ERROR) << "StartRecording failed"; + return false; + } + } + } else { + if (audio_device_module_->Recording() && + audio_device_module_->StopRecording() != 0) { + RTC_LOG(LS_ERROR) << "StopRecording failed"; + return false; + } + } + return true; +} + +VoipResult VoipCore::StartSend(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + if (!channel->StartSend()) { + return VoipResult::kFailedPrecondition; + } + + return UpdateAudioTransportWithSenders() ? VoipResult::kOk + : VoipResult::kInternal; +} + +VoipResult VoipCore::StopSend(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->StopSend(); + + return UpdateAudioTransportWithSenders() ? VoipResult::kOk + : VoipResult::kInternal; +} + +VoipResult VoipCore::StartPlayout(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + if (channel->IsPlaying()) { + return VoipResult::kOk; + } + + if (!channel->StartPlay()) { + return VoipResult::kFailedPrecondition; + } + + // Initialize audio device module and default device if needed. + if (!InitializeIfNeeded()) { + return VoipResult::kInternal; + } + + if (!audio_device_module_->Playing()) { + if (audio_device_module_->InitPlayout() != 0) { + RTC_LOG(LS_ERROR) << "InitPlayout failed"; + return VoipResult::kInternal; + } + if (audio_device_module_->StartPlayout() != 0) { + RTC_LOG(LS_ERROR) << "StartPlayout failed"; + return VoipResult::kInternal; + } + } + + return VoipResult::kOk; +} + +VoipResult VoipCore::StopPlayout(ChannelId channel_id) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->StopPlay(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::ReceivedRTPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtp_packet) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->ReceivedRTPPacket(rtp_packet); + + return VoipResult::kOk; +} + +VoipResult VoipCore::ReceivedRTCPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtcp_packet) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->ReceivedRTCPPacket(rtcp_packet); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SetSendCodec(ChannelId channel_id, + int payload_type, + const SdpAudioFormat& encoder_format) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + auto encoder = encoder_factory_->MakeAudioEncoder( + payload_type, encoder_format, absl::nullopt); + channel->SetEncoder(payload_type, encoder_format, std::move(encoder)); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SetReceiveCodecs( + ChannelId channel_id, + const std::map<int, SdpAudioFormat>& decoder_specs) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->SetReceiveCodecs(decoder_specs); + + return VoipResult::kOk; +} + +VoipResult VoipCore::RegisterTelephoneEventType(ChannelId channel_id, + int rtp_payload_type, + int sample_rate_hz) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SendDtmfEvent(ChannelId channel_id, + DtmfEvent dtmf_event, + int duration_ms) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + return (channel->SendTelephoneEvent(static_cast<int>(dtmf_event), duration_ms) + ? VoipResult::kOk + : VoipResult::kFailedPrecondition); +} + +VoipResult VoipCore::GetIngressStatistics(ChannelId channel_id, + IngressStatistics& ingress_stats) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + ingress_stats = channel->GetIngressStatistics(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetChannelStatistics(ChannelId channel_id, + ChannelStatistics& channel_stats) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel_stats = channel->GetChannelStatistics(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::SetInputMuted(ChannelId channel_id, bool enable) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + channel->SetMute(enable); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetInputVolumeInfo(ChannelId channel_id, + VolumeInfo& input_volume) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + input_volume.audio_level = channel->GetInputAudioLevel(); + input_volume.total_energy = channel->GetInputTotalEnergy(); + input_volume.total_duration = channel->GetInputTotalDuration(); + + return VoipResult::kOk; +} + +VoipResult VoipCore::GetOutputVolumeInfo(ChannelId channel_id, + VolumeInfo& output_volume) { + rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id); + + if (!channel) { + return VoipResult::kInvalidArgument; + } + + output_volume.audio_level = channel->GetOutputAudioLevel(); + output_volume.total_energy = channel->GetOutputTotalEnergy(); + output_volume.total_duration = channel->GetOutputTotalDuration(); + + return VoipResult::kOk; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/audio/voip/voip_core.h b/third_party/libwebrtc/audio/voip/voip_core.h new file mode 100644 index 0000000000..6c3aec6fa2 --- /dev/null +++ b/third_party/libwebrtc/audio/voip/voip_core.h @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AUDIO_VOIP_VOIP_CORE_H_ +#define AUDIO_VOIP_VOIP_CORE_H_ + +#include <map> +#include <memory> +#include <queue> +#include <unordered_map> +#include <vector> + +#include "api/audio_codecs/audio_decoder_factory.h" +#include "api/audio_codecs/audio_encoder_factory.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/voip/voip_base.h" +#include "api/voip/voip_codec.h" +#include "api/voip/voip_dtmf.h" +#include "api/voip/voip_engine.h" +#include "api/voip/voip_network.h" +#include "api/voip/voip_statistics.h" +#include "api/voip/voip_volume_control.h" +#include "audio/audio_transport_impl.h" +#include "audio/voip/audio_channel.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_mixer/audio_mixer_impl.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// VoipCore is the implementatino of VoIP APIs listed in api/voip directory. +// It manages a vector of AudioChannel objects where each is mapped with a +// ChannelId (int) type. ChannelId is the primary key to locate a specific +// AudioChannel object to operate requested VoIP API from the caller. +// +// This class receives required audio components from caller at construction and +// owns the life cycle of them to orchestrate the proper destruction sequence. +class VoipCore : public VoipEngine, + public VoipBase, + public VoipNetwork, + public VoipCodec, + public VoipDtmf, + public VoipStatistics, + public VoipVolumeControl { + public: + // Construct VoipCore with provided arguments. + VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory, + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory, + std::unique_ptr<TaskQueueFactory> task_queue_factory, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module, + rtc::scoped_refptr<AudioProcessing> audio_processing); + ~VoipCore() override = default; + + // Implements VoipEngine interfaces. + VoipBase& Base() override { return *this; } + VoipNetwork& Network() override { return *this; } + VoipCodec& Codec() override { return *this; } + VoipDtmf& Dtmf() override { return *this; } + VoipStatistics& Statistics() override { return *this; } + VoipVolumeControl& VolumeControl() override { return *this; } + + // Implements VoipBase interfaces. + ChannelId CreateChannel(Transport* transport, + absl::optional<uint32_t> local_ssrc) override; + VoipResult ReleaseChannel(ChannelId channel_id) override; + VoipResult StartSend(ChannelId channel_id) override; + VoipResult StopSend(ChannelId channel_id) override; + VoipResult StartPlayout(ChannelId channel_id) override; + VoipResult StopPlayout(ChannelId channel_id) override; + + // Implements VoipNetwork interfaces. + VoipResult ReceivedRTPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtp_packet) override; + VoipResult ReceivedRTCPPacket( + ChannelId channel_id, + rtc::ArrayView<const uint8_t> rtcp_packet) override; + + // Implements VoipCodec interfaces. + VoipResult SetSendCodec(ChannelId channel_id, + int payload_type, + const SdpAudioFormat& encoder_format) override; + VoipResult SetReceiveCodecs( + ChannelId channel_id, + const std::map<int, SdpAudioFormat>& decoder_specs) override; + + // Implements VoipDtmf interfaces. + VoipResult RegisterTelephoneEventType(ChannelId channel_id, + int rtp_payload_type, + int sample_rate_hz) override; + VoipResult SendDtmfEvent(ChannelId channel_id, + DtmfEvent dtmf_event, + int duration_ms) override; + + // Implements VoipStatistics interfaces. + VoipResult GetIngressStatistics(ChannelId channel_id, + IngressStatistics& ingress_stats) override; + VoipResult GetChannelStatistics(ChannelId channe_id, + ChannelStatistics& channel_stats) override; + + // Implements VoipVolumeControl interfaces. + VoipResult SetInputMuted(ChannelId channel_id, bool enable) override; + VoipResult GetInputVolumeInfo(ChannelId channel_id, + VolumeInfo& volume_info) override; + VoipResult GetOutputVolumeInfo(ChannelId channel_id, + VolumeInfo& volume_info) override; + + private: + // Initialize ADM and default audio device if needed. + // Returns true if ADM is successfully initialized or already in such state + // (e.g called more than once). Returns false when ADM fails to initialize + // which would presumably render further processing useless. Note that such + // failure won't necessarily succeed in next initialization attempt as it + // would mean changing the ADM implementation. From Android N and onwards, the + // mobile app may not be able to gain microphone access when in background + // mode. Therefore it would be better to delay the logic as late as possible. + bool InitializeIfNeeded(); + + // Fetches the corresponding AudioChannel assigned with given `channel`. + // Returns nullptr if not found. + rtc::scoped_refptr<AudioChannel> GetChannel(ChannelId channel_id); + + // Updates AudioTransportImpl with a new set of actively sending AudioSender + // (AudioEgress). This needs to be invoked whenever StartSend/StopSend is + // involved by caller. Returns false when the selected audio device fails to + // initialize where it can't expect to deliver any audio input sample. + bool UpdateAudioTransportWithSenders(); + + // Synchronization for these are handled internally. + rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_; + rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_; + std::unique_ptr<TaskQueueFactory> task_queue_factory_; + + // Synchronization is handled internally by AudioProcessing. + // Must be placed before `audio_device_module_` for proper destruction. + rtc::scoped_refptr<AudioProcessing> audio_processing_; + + // Synchronization is handled internally by AudioMixer. + // Must be placed before `audio_device_module_` for proper destruction. + rtc::scoped_refptr<AudioMixer> audio_mixer_; + + // Synchronization is handled internally by AudioTransportImpl. + // Must be placed before `audio_device_module_` for proper destruction. + std::unique_ptr<AudioTransportImpl> audio_transport_; + + // Synchronization is handled internally by AudioDeviceModule. + rtc::scoped_refptr<AudioDeviceModule> audio_device_module_; + + Mutex lock_; + + // Member to track a next ChannelId for new AudioChannel. + int next_channel_id_ RTC_GUARDED_BY(lock_) = 0; + + // Container to track currently active AudioChannel objects mapped by + // ChannelId. + std::unordered_map<ChannelId, rtc::scoped_refptr<AudioChannel>> channels_ + RTC_GUARDED_BY(lock_); + + // Boolean flag to ensure initialization only occurs once. + bool initialized_ RTC_GUARDED_BY(lock_) = false; +}; + +} // namespace webrtc + +#endif // AUDIO_VOIP_VOIP_CORE_H_ |