summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/audio/voip
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /third_party/libwebrtc/audio/voip
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/audio/voip')
-rw-r--r--third_party/libwebrtc/audio/voip/BUILD.gn103
-rw-r--r--third_party/libwebrtc/audio/voip/audio_channel.cc173
-rw-r--r--third_party/libwebrtc/audio/voip/audio_channel.h131
-rw-r--r--third_party/libwebrtc/audio/voip/audio_egress.cc184
-rw-r--r--third_party/libwebrtc/audio/voip/audio_egress.h158
-rw-r--r--third_party/libwebrtc/audio/voip/audio_ingress.cc290
-rw-r--r--third_party/libwebrtc/audio/voip/audio_ingress.h145
-rw-r--r--third_party/libwebrtc/audio/voip/test/BUILD.gn107
-rw-r--r--third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc355
-rw-r--r--third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc326
-rw-r--r--third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc240
-rw-r--r--third_party/libwebrtc/audio/voip/test/mock_task_queue.h55
-rw-r--r--third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc193
-rw-r--r--third_party/libwebrtc/audio/voip/voip_core.cc500
-rw-r--r--third_party/libwebrtc/audio/voip/voip_core.h174
15 files changed, 3134 insertions, 0 deletions
diff --git a/third_party/libwebrtc/audio/voip/BUILD.gn b/third_party/libwebrtc/audio/voip/BUILD.gn
new file mode 100644
index 0000000000..e807e2276b
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/BUILD.gn
@@ -0,0 +1,103 @@
+# Copyright(c) 2020 The WebRTC project authors.All Rights Reserved.
+#
+# Use of this source code is governed by a BSD - style license
+# that can be found in the LICENSE file in the root of the source
+# tree.An additional intellectual property rights grant can be found
+# in the file PATENTS.All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+
+rtc_library("voip_core") {
+ sources = [
+ "voip_core.cc",
+ "voip_core.h",
+ ]
+ deps = [
+ ":audio_channel",
+ "..:audio",
+ "../../api:scoped_refptr",
+ "../../api/audio_codecs:audio_codecs_api",
+ "../../api/task_queue",
+ "../../api/voip:voip_api",
+ "../../modules/audio_device:audio_device_api",
+ "../../modules/audio_mixer:audio_mixer_impl",
+ "../../modules/audio_processing:api",
+ "../../rtc_base:criticalsection",
+ "../../rtc_base:logging",
+ "../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("audio_channel") {
+ sources = [
+ "audio_channel.cc",
+ "audio_channel.h",
+ ]
+ deps = [
+ ":audio_egress",
+ ":audio_ingress",
+ "../../api:transport_api",
+ "../../api/audio_codecs:audio_codecs_api",
+ "../../api/task_queue",
+ "../../api/voip:voip_api",
+ "../../modules/audio_device:audio_device_api",
+ "../../modules/rtp_rtcp",
+ "../../modules/rtp_rtcp:rtp_rtcp_format",
+ "../../rtc_base:criticalsection",
+ "../../rtc_base:logging",
+ "../../rtc_base:refcount",
+ ]
+}
+
+rtc_library("audio_ingress") {
+ sources = [
+ "audio_ingress.cc",
+ "audio_ingress.h",
+ ]
+ deps = [
+ "..:audio",
+ "../../api:array_view",
+ "../../api:rtp_headers",
+ "../../api:scoped_refptr",
+ "../../api:transport_api",
+ "../../api/audio:audio_mixer_api",
+ "../../api/audio_codecs:audio_codecs_api",
+ "../../api/voip:voip_api",
+ "../../modules/audio_coding",
+ "../../modules/rtp_rtcp",
+ "../../modules/rtp_rtcp:rtp_rtcp_format",
+ "../../rtc_base:criticalsection",
+ "../../rtc_base:logging",
+ "../../rtc_base:rtc_numerics",
+ "../../rtc_base:safe_minmax",
+ "../../rtc_base:timeutils",
+ "../../rtc_base/synchronization:mutex",
+ "../utility:audio_frame_operations",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("audio_egress") {
+ sources = [
+ "audio_egress.cc",
+ "audio_egress.h",
+ ]
+ deps = [
+ "..:audio",
+ "../../api:sequence_checker",
+ "../../api/audio_codecs:audio_codecs_api",
+ "../../api/task_queue",
+ "../../call:audio_sender_interface",
+ "../../modules/audio_coding",
+ "../../modules/rtp_rtcp",
+ "../../modules/rtp_rtcp:rtp_rtcp_format",
+ "../../rtc_base:logging",
+ "../../rtc_base:rtc_task_queue",
+ "../../rtc_base:timeutils",
+ "../../rtc_base/synchronization:mutex",
+ "../../rtc_base/system:no_unique_address",
+ "../utility:audio_frame_operations",
+ ]
+}
diff --git a/third_party/libwebrtc/audio/voip/audio_channel.cc b/third_party/libwebrtc/audio/voip/audio_channel.cc
new file mode 100644
index 0000000000..a70e33ec38
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_channel.cc
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_channel.h"
+
+#include <utility>
+#include <vector>
+
+#include "api/audio_codecs/audio_format.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "modules/rtp_rtcp/include/receive_statistics.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+namespace {
+
+constexpr int kRtcpReportIntervalMs = 5000;
+
+} // namespace
+
+AudioChannel::AudioChannel(
+ Transport* transport,
+ uint32_t local_ssrc,
+ TaskQueueFactory* task_queue_factory,
+ AudioMixer* audio_mixer,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
+ : audio_mixer_(audio_mixer) {
+ RTC_DCHECK(task_queue_factory);
+ RTC_DCHECK(audio_mixer);
+
+ Clock* clock = Clock::GetRealTimeClock();
+ receive_statistics_ = ReceiveStatistics::Create(clock);
+
+ RtpRtcpInterface::Configuration rtp_config;
+ rtp_config.clock = clock;
+ rtp_config.audio = true;
+ rtp_config.receive_statistics = receive_statistics_.get();
+ rtp_config.rtcp_report_interval_ms = kRtcpReportIntervalMs;
+ rtp_config.outgoing_transport = transport;
+ rtp_config.local_media_ssrc = local_ssrc;
+
+ rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(rtp_config);
+
+ rtp_rtcp_->SetSendingMediaStatus(false);
+ rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound);
+
+ ingress_ = std::make_unique<AudioIngress>(rtp_rtcp_.get(), clock,
+ receive_statistics_.get(),
+ std::move(decoder_factory));
+ egress_ =
+ std::make_unique<AudioEgress>(rtp_rtcp_.get(), clock, task_queue_factory);
+
+ // Set the instance of audio ingress to be part of audio mixer for ADM to
+ // fetch audio samples to play.
+ audio_mixer_->AddSource(ingress_.get());
+}
+
+AudioChannel::~AudioChannel() {
+ if (egress_->IsSending()) {
+ StopSend();
+ }
+ if (ingress_->IsPlaying()) {
+ StopPlay();
+ }
+
+ audio_mixer_->RemoveSource(ingress_.get());
+
+ // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `egress_`
+ // here.
+ egress_.reset();
+ ingress_.reset();
+}
+
+bool AudioChannel::StartSend() {
+ // If encoder has not been set, return false.
+ if (!egress_->StartSend()) {
+ return false;
+ }
+
+ // Start sending with RTP stack if it has not been sending yet.
+ if (!rtp_rtcp_->Sending()) {
+ rtp_rtcp_->SetSendingStatus(true);
+ }
+ return true;
+}
+
+void AudioChannel::StopSend() {
+ egress_->StopSend();
+
+ // Deactivate RTP stack when both sending and receiving are stopped.
+ // SetSendingStatus(false) triggers the transmission of RTCP BYE
+ // message to remote endpoint.
+ if (!ingress_->IsPlaying() && rtp_rtcp_->Sending()) {
+ rtp_rtcp_->SetSendingStatus(false);
+ }
+}
+
+bool AudioChannel::StartPlay() {
+ // If decoders have not been set, return false.
+ if (!ingress_->StartPlay()) {
+ return false;
+ }
+
+ // If RTP stack is not sending then start sending as in recv-only mode, RTCP
+ // receiver report is expected.
+ if (!rtp_rtcp_->Sending()) {
+ rtp_rtcp_->SetSendingStatus(true);
+ }
+ return true;
+}
+
+void AudioChannel::StopPlay() {
+ ingress_->StopPlay();
+
+ // Deactivate RTP stack only when both sending and receiving are stopped.
+ if (!rtp_rtcp_->SendingMedia() && rtp_rtcp_->Sending()) {
+ rtp_rtcp_->SetSendingStatus(false);
+ }
+}
+
+IngressStatistics AudioChannel::GetIngressStatistics() {
+ IngressStatistics ingress_stats;
+ NetworkStatistics stats = ingress_->GetNetworkStatistics();
+ ingress_stats.neteq_stats.total_samples_received = stats.totalSamplesReceived;
+ ingress_stats.neteq_stats.concealed_samples = stats.concealedSamples;
+ ingress_stats.neteq_stats.concealment_events = stats.concealmentEvents;
+ ingress_stats.neteq_stats.jitter_buffer_delay_ms = stats.jitterBufferDelayMs;
+ ingress_stats.neteq_stats.jitter_buffer_emitted_count =
+ stats.jitterBufferEmittedCount;
+ ingress_stats.neteq_stats.jitter_buffer_target_delay_ms =
+ stats.jitterBufferTargetDelayMs;
+ ingress_stats.neteq_stats.inserted_samples_for_deceleration =
+ stats.insertedSamplesForDeceleration;
+ ingress_stats.neteq_stats.removed_samples_for_acceleration =
+ stats.removedSamplesForAcceleration;
+ ingress_stats.neteq_stats.silent_concealed_samples =
+ stats.silentConcealedSamples;
+ ingress_stats.neteq_stats.fec_packets_received = stats.fecPacketsReceived;
+ ingress_stats.neteq_stats.fec_packets_discarded = stats.fecPacketsDiscarded;
+ ingress_stats.neteq_stats.delayed_packet_outage_samples =
+ stats.delayedPacketOutageSamples;
+ ingress_stats.neteq_stats.relative_packet_arrival_delay_ms =
+ stats.relativePacketArrivalDelayMs;
+ ingress_stats.neteq_stats.interruption_count = stats.interruptionCount;
+ ingress_stats.neteq_stats.total_interruption_duration_ms =
+ stats.totalInterruptionDurationMs;
+ ingress_stats.total_duration = ingress_->GetOutputTotalDuration();
+ return ingress_stats;
+}
+
+ChannelStatistics AudioChannel::GetChannelStatistics() {
+ ChannelStatistics channel_stat = ingress_->GetChannelStatistics();
+
+ StreamDataCounters rtp_stats, rtx_stats;
+ rtp_rtcp_->GetSendStreamDataCounters(&rtp_stats, &rtx_stats);
+ channel_stat.bytes_sent =
+ rtp_stats.transmitted.payload_bytes + rtx_stats.transmitted.payload_bytes;
+ channel_stat.packets_sent =
+ rtp_stats.transmitted.packets + rtx_stats.transmitted.packets;
+
+ return channel_stat;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/audio_channel.h b/third_party/libwebrtc/audio/voip/audio_channel.h
new file mode 100644
index 0000000000..7338d9faab
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_channel.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_VOIP_AUDIO_CHANNEL_H_
+#define AUDIO_VOIP_AUDIO_CHANNEL_H_
+
+#include <map>
+#include <memory>
+#include <queue>
+#include <utility>
+
+#include "api/task_queue/task_queue_factory.h"
+#include "api/voip/voip_base.h"
+#include "api/voip/voip_statistics.h"
+#include "audio/voip/audio_egress.h"
+#include "audio/voip/audio_ingress.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
+#include "rtc_base/ref_count.h"
+
+namespace webrtc {
+
+// AudioChannel represents a single media session and provides APIs over
+// AudioIngress and AudioEgress. Note that a single RTP stack is shared with
+// these two classes as it has both sending and receiving capabilities.
+class AudioChannel : public rtc::RefCountInterface {
+ public:
+ AudioChannel(Transport* transport,
+ uint32_t local_ssrc,
+ TaskQueueFactory* task_queue_factory,
+ AudioMixer* audio_mixer,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory);
+ ~AudioChannel() override;
+
+ // Set and get ChannelId that this audio channel belongs for debugging and
+ // logging purpose.
+ void SetId(ChannelId id) { id_ = id; }
+ ChannelId GetId() const { return id_; }
+
+ // APIs to start/stop audio channel on each direction.
+ // StartSend/StartPlay returns false if encoder/decoders
+ // have not been set, respectively.
+ bool StartSend();
+ void StopSend();
+ bool StartPlay();
+ void StopPlay();
+
+ // APIs relayed to AudioEgress.
+ bool IsSendingMedia() const { return egress_->IsSending(); }
+ AudioSender* GetAudioSender() { return egress_.get(); }
+ void SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
+ std::unique_ptr<AudioEncoder> encoder) {
+ egress_->SetEncoder(payload_type, encoder_format, std::move(encoder));
+ }
+ absl::optional<SdpAudioFormat> GetEncoderFormat() const {
+ return egress_->GetEncoderFormat();
+ }
+ void RegisterTelephoneEventType(int rtp_payload_type, int sample_rate_hz) {
+ egress_->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz);
+ }
+ bool SendTelephoneEvent(int dtmf_event, int duration_ms) {
+ return egress_->SendTelephoneEvent(dtmf_event, duration_ms);
+ }
+ void SetMute(bool enable) { egress_->SetMute(enable); }
+
+ // APIs relayed to AudioIngress.
+ bool IsPlaying() const { return ingress_->IsPlaying(); }
+ void ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) {
+ ingress_->ReceivedRTPPacket(rtp_packet);
+ }
+ void ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet) {
+ ingress_->ReceivedRTCPPacket(rtcp_packet);
+ }
+ void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) {
+ ingress_->SetReceiveCodecs(codecs);
+ }
+ IngressStatistics GetIngressStatistics();
+ ChannelStatistics GetChannelStatistics();
+
+ // See comments on the methods used from AudioEgress and AudioIngress.
+ // Conversion to double is following what is done in
+ // DoubleAudioLevelFromIntAudioLevel method in rtc_stats_collector.cc to be
+ // consistent.
+ double GetInputAudioLevel() const {
+ return egress_->GetInputAudioLevel() / 32767.0;
+ }
+ double GetInputTotalEnergy() const { return egress_->GetInputTotalEnergy(); }
+ double GetInputTotalDuration() const {
+ return egress_->GetInputTotalDuration();
+ }
+ double GetOutputAudioLevel() const {
+ return ingress_->GetOutputAudioLevel() / 32767.0;
+ }
+ double GetOutputTotalEnergy() const {
+ return ingress_->GetOutputTotalEnergy();
+ }
+ double GetOutputTotalDuration() const {
+ return ingress_->GetOutputTotalDuration();
+ }
+
+ // Internal API for testing purpose.
+ void SendRTCPReportForTesting(RTCPPacketType type) {
+ int32_t result = rtp_rtcp_->SendRTCP(type);
+ RTC_DCHECK(result == 0);
+ }
+
+ private:
+ // ChannelId that this audio channel belongs for logging purpose.
+ ChannelId id_;
+
+ // Synchronization is handled internally by AudioMixer.
+ AudioMixer* audio_mixer_;
+
+ // Listed in order for safe destruction of AudioChannel object.
+ // Synchronization for these are handled internally.
+ std::unique_ptr<ReceiveStatistics> receive_statistics_;
+ std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
+ std::unique_ptr<AudioIngress> ingress_;
+ std::unique_ptr<AudioEgress> egress_;
+};
+
+} // namespace webrtc
+
+#endif // AUDIO_VOIP_AUDIO_CHANNEL_H_
diff --git a/third_party/libwebrtc/audio/voip/audio_egress.cc b/third_party/libwebrtc/audio/voip/audio_egress.cc
new file mode 100644
index 0000000000..95a1a3351e
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_egress.cc
@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_egress.h"
+
+#include <utility>
+#include <vector>
+
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+AudioEgress::AudioEgress(RtpRtcpInterface* rtp_rtcp,
+ Clock* clock,
+ TaskQueueFactory* task_queue_factory)
+ : rtp_rtcp_(rtp_rtcp),
+ rtp_sender_audio_(clock, rtp_rtcp_->RtpSender()),
+ audio_coding_(AudioCodingModule::Create()),
+ encoder_queue_(task_queue_factory->CreateTaskQueue(
+ "AudioEncoder",
+ TaskQueueFactory::Priority::NORMAL)) {
+ audio_coding_->RegisterTransportCallback(this);
+}
+
+AudioEgress::~AudioEgress() {
+ audio_coding_->RegisterTransportCallback(nullptr);
+}
+
+bool AudioEgress::IsSending() const {
+ return rtp_rtcp_->SendingMedia();
+}
+
+void AudioEgress::SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
+ std::unique_ptr<AudioEncoder> encoder) {
+ RTC_DCHECK_GE(payload_type, 0);
+ RTC_DCHECK_LE(payload_type, 127);
+
+ SetEncoderFormat(encoder_format);
+
+ // The RTP/RTCP module needs to know the RTP timestamp rate (i.e. clockrate)
+ // as well as some other things, so we collect this info and send it along.
+ rtp_rtcp_->RegisterSendPayloadFrequency(payload_type,
+ encoder->RtpTimestampRateHz());
+ rtp_sender_audio_.RegisterAudioPayload("audio", payload_type,
+ encoder->RtpTimestampRateHz(),
+ encoder->NumChannels(), 0);
+
+ audio_coding_->SetEncoder(std::move(encoder));
+}
+
+bool AudioEgress::StartSend() {
+ if (!GetEncoderFormat()) {
+ RTC_DLOG(LS_WARNING) << "Send codec has not been set yet";
+ return false;
+ }
+ rtp_rtcp_->SetSendingMediaStatus(true);
+ return true;
+}
+
+void AudioEgress::StopSend() {
+ rtp_rtcp_->SetSendingMediaStatus(false);
+}
+
+void AudioEgress::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) {
+ RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0);
+ RTC_DCHECK_LE(audio_frame->num_channels_, 8);
+
+ encoder_queue_.PostTask(
+ [this, audio_frame = std::move(audio_frame)]() mutable {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ if (!rtp_rtcp_->SendingMedia()) {
+ return;
+ }
+
+ double duration_seconds =
+ static_cast<double>(audio_frame->samples_per_channel_) /
+ audio_frame->sample_rate_hz_;
+
+ input_audio_level_.ComputeLevel(*audio_frame, duration_seconds);
+
+ AudioFrameOperations::Mute(audio_frame.get(),
+ encoder_context_.previously_muted_,
+ encoder_context_.mute_);
+ encoder_context_.previously_muted_ = encoder_context_.mute_;
+
+ audio_frame->timestamp_ = encoder_context_.frame_rtp_timestamp_;
+
+ // This call will trigger AudioPacketizationCallback::SendData if
+ // encoding is done and payload is ready for packetization and
+ // transmission. Otherwise, it will return without invoking the
+ // callback.
+ if (audio_coding_->Add10MsData(*audio_frame) < 0) {
+ RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
+ return;
+ }
+
+ encoder_context_.frame_rtp_timestamp_ +=
+ rtc::dchecked_cast<uint32_t>(audio_frame->samples_per_channel_);
+ });
+}
+
+int32_t AudioEgress::SendData(AudioFrameType frame_type,
+ uint8_t payload_type,
+ uint32_t timestamp,
+ const uint8_t* payload_data,
+ size_t payload_size) {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+
+ rtc::ArrayView<const uint8_t> payload(payload_data, payload_size);
+
+ // Currently we don't get a capture time from downstream modules (ADM,
+ // AudioTransportImpl).
+ // TODO(natim@webrtc.org): Integrate once it's ready.
+ constexpr uint32_t kUndefinedCaptureTime = -1;
+
+ // Push data from ACM to RTP/RTCP-module to deliver audio frame for
+ // packetization.
+ if (!rtp_rtcp_->OnSendingRtpFrame(timestamp, kUndefinedCaptureTime,
+ payload_type,
+ /*force_sender_report=*/false)) {
+ return -1;
+ }
+
+ const uint32_t rtp_timestamp = timestamp + rtp_rtcp_->StartTimestamp();
+
+ // This call will trigger Transport::SendPacket() from the RTP/RTCP module.
+ if (!rtp_sender_audio_.SendAudio({.type = frame_type,
+ .payload = payload,
+ .payload_id = payload_type,
+ .rtp_timestamp = rtp_timestamp})) {
+ RTC_DLOG(LS_ERROR)
+ << "AudioEgress::SendData() failed to send data to RTP/RTCP module";
+ return -1;
+ }
+
+ return 0;
+}
+
+void AudioEgress::RegisterTelephoneEventType(int rtp_payload_type,
+ int sample_rate_hz) {
+ RTC_DCHECK_GE(rtp_payload_type, 0);
+ RTC_DCHECK_LE(rtp_payload_type, 127);
+
+ rtp_rtcp_->RegisterSendPayloadFrequency(rtp_payload_type, sample_rate_hz);
+ rtp_sender_audio_.RegisterAudioPayload("telephone-event", rtp_payload_type,
+ sample_rate_hz, 0, 0);
+}
+
+bool AudioEgress::SendTelephoneEvent(int dtmf_event, int duration_ms) {
+ RTC_DCHECK_GE(dtmf_event, 0);
+ RTC_DCHECK_LE(dtmf_event, 255);
+ RTC_DCHECK_GE(duration_ms, 0);
+ RTC_DCHECK_LE(duration_ms, 65535);
+
+ if (!IsSending()) {
+ return false;
+ }
+
+ constexpr int kTelephoneEventAttenuationdB = 10;
+
+ if (rtp_sender_audio_.SendTelephoneEvent(dtmf_event, duration_ms,
+ kTelephoneEventAttenuationdB) != 0) {
+ RTC_DLOG(LS_ERROR) << "SendTelephoneEvent() failed to send event";
+ return false;
+ }
+ return true;
+}
+
+void AudioEgress::SetMute(bool mute) {
+ encoder_queue_.PostTask([this, mute] {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ encoder_context_.mute_ = mute;
+ });
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/audio_egress.h b/third_party/libwebrtc/audio/voip/audio_egress.h
new file mode 100644
index 0000000000..989e5bda59
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_egress.h
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_VOIP_AUDIO_EGRESS_H_
+#define AUDIO_VOIP_AUDIO_EGRESS_H_
+
+#include <memory>
+#include <string>
+
+#include "api/audio_codecs/audio_format.h"
+#include "api/sequence_checker.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "audio/audio_level.h"
+#include "audio/utility/audio_frame_operations.h"
+#include "call/audio_sender.h"
+#include "modules/audio_coding/include/audio_coding_module.h"
+#include "modules/rtp_rtcp/include/report_block_data.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
+#include "modules/rtp_rtcp/source/rtp_sender_audio.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/task_queue.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+// AudioEgress receives input samples from AudioDeviceModule via
+// AudioTransportImpl through AudioSender interface. Once it encodes the sample
+// via selected encoder through AudioPacketizationCallback interface, the
+// encoded payload will be packetized by the RTP stack, resulting in ready to
+// send RTP packet to remote endpoint.
+//
+// TaskQueue is used to encode and send RTP asynchrounously as some OS platform
+// uses the same thread for both audio input and output sample deliveries which
+// can affect audio quality.
+//
+// Note that this class is originally based on ChannelSend in
+// audio/channel_send.cc with non-audio related logic trimmed as aimed for
+// smaller footprint.
+class AudioEgress : public AudioSender, public AudioPacketizationCallback {
+ public:
+ AudioEgress(RtpRtcpInterface* rtp_rtcp,
+ Clock* clock,
+ TaskQueueFactory* task_queue_factory);
+ ~AudioEgress() override;
+
+ // Set the encoder format and payload type for AudioCodingModule.
+ // It's possible to change the encoder type during its active usage.
+ // `payload_type` must be the type that is negotiated with peer through
+ // offer/answer.
+ void SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
+ std::unique_ptr<AudioEncoder> encoder);
+
+ // Start or stop sending operation of AudioEgress. This will start/stop
+ // the RTP stack also causes encoder queue thread to start/stop
+ // processing input audio samples. StartSend will return false if
+ // a send codec has not been set.
+ bool StartSend();
+ void StopSend();
+
+ // Query the state of the RTP stack. This returns true if StartSend()
+ // called and false if StopSend() is called.
+ bool IsSending() const;
+
+ // Enable or disable Mute state.
+ void SetMute(bool mute);
+
+ // Retrieve current encoder format info. This returns encoder format set
+ // by SetEncoder() and if encoder is not set, this will return nullopt.
+ absl::optional<SdpAudioFormat> GetEncoderFormat() const {
+ MutexLock lock(&lock_);
+ return encoder_format_;
+ }
+
+ // Register the payload type and sample rate for DTMF (RFC 4733) payload.
+ void RegisterTelephoneEventType(int rtp_payload_type, int sample_rate_hz);
+
+ // Send DTMF named event as specified by
+ // https://tools.ietf.org/html/rfc4733#section-3.2
+ // `duration_ms` specifies the duration of DTMF packets that will be emitted
+ // in place of real RTP packets instead.
+ // This will return true when requested dtmf event is successfully scheduled
+ // otherwise false when the dtmf queue reached maximum of 20 events.
+ bool SendTelephoneEvent(int dtmf_event, int duration_ms);
+
+ // See comments on LevelFullRange, TotalEnergy, TotalDuration from
+ // audio/audio_level.h.
+ int GetInputAudioLevel() const { return input_audio_level_.LevelFullRange(); }
+ double GetInputTotalEnergy() const {
+ return input_audio_level_.TotalEnergy();
+ }
+ double GetInputTotalDuration() const {
+ return input_audio_level_.TotalDuration();
+ }
+
+ // Implementation of AudioSender interface.
+ void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override;
+
+ // Implementation of AudioPacketizationCallback interface.
+ int32_t SendData(AudioFrameType frame_type,
+ uint8_t payload_type,
+ uint32_t timestamp,
+ const uint8_t* payload_data,
+ size_t payload_size) override;
+
+ private:
+ void SetEncoderFormat(const SdpAudioFormat& encoder_format) {
+ MutexLock lock(&lock_);
+ encoder_format_ = encoder_format;
+ }
+
+ mutable Mutex lock_;
+
+ // Current encoder format selected by caller.
+ absl::optional<SdpAudioFormat> encoder_format_ RTC_GUARDED_BY(lock_);
+
+ // Synchronization is handled internally by RtpRtcp.
+ RtpRtcpInterface* const rtp_rtcp_;
+
+ // Synchronization is handled internally by RTPSenderAudio.
+ RTPSenderAudio rtp_sender_audio_;
+
+ // Synchronization is handled internally by AudioCodingModule.
+ const std::unique_ptr<AudioCodingModule> audio_coding_;
+
+ // Synchronization is handled internally by voe::AudioLevel.
+ voe::AudioLevel input_audio_level_;
+
+ // Struct that holds all variables used by encoder task queue.
+ struct EncoderContext {
+ // Offset used to mark rtp timestamp in sample rate unit in
+ // newly received audio frame from AudioTransport.
+ uint32_t frame_rtp_timestamp_ = 0;
+
+ // Flag to track mute state from caller. `previously_muted_` is used to
+ // track previous state as part of input to AudioFrameOperations::Mute
+ // to implement fading effect when (un)mute is invoked.
+ bool mute_ = false;
+ bool previously_muted_ = false;
+ };
+
+ EncoderContext encoder_context_ RTC_GUARDED_BY(encoder_queue_);
+
+ // Defined last to ensure that there are no running tasks when the other
+ // members are destroyed.
+ rtc::TaskQueue encoder_queue_;
+};
+
+} // namespace webrtc
+
+#endif // AUDIO_VOIP_AUDIO_EGRESS_H_
diff --git a/third_party/libwebrtc/audio/voip/audio_ingress.cc b/third_party/libwebrtc/audio/voip/audio_ingress.cc
new file mode 100644
index 0000000000..80f21152c0
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_ingress.cc
@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_ingress.h"
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include "api/audio_codecs/audio_format.h"
+#include "audio/utility/audio_frame_operations.h"
+#include "modules/audio_coding/include/audio_coding_module.h"
+#include "modules/rtp_rtcp/source/byte_io.h"
+#include "modules/rtp_rtcp/source/rtcp_packet/common_header.h"
+#include "modules/rtp_rtcp/source/rtcp_packet/receiver_report.h"
+#include "modules/rtp_rtcp/source/rtcp_packet/sender_report.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/safe_minmax.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+namespace {
+
+acm2::AcmReceiver::Config CreateAcmConfig(
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) {
+ acm2::AcmReceiver::Config acm_config;
+ acm_config.neteq_config.enable_muted_state = true;
+ acm_config.decoder_factory = decoder_factory;
+ return acm_config;
+}
+
+} // namespace
+
+AudioIngress::AudioIngress(
+ RtpRtcpInterface* rtp_rtcp,
+ Clock* clock,
+ ReceiveStatistics* receive_statistics,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
+ : playing_(false),
+ remote_ssrc_(0),
+ first_rtp_timestamp_(-1),
+ rtp_receive_statistics_(receive_statistics),
+ rtp_rtcp_(rtp_rtcp),
+ acm_receiver_(CreateAcmConfig(decoder_factory)),
+ ntp_estimator_(clock) {}
+
+AudioIngress::~AudioIngress() = default;
+
+AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo(
+ int sampling_rate,
+ AudioFrame* audio_frame) {
+ audio_frame->sample_rate_hz_ = sampling_rate;
+
+ // Get 10ms raw PCM data from the ACM.
+ bool muted = false;
+ if (acm_receiver_.GetAudio(sampling_rate, audio_frame, &muted) == -1) {
+ RTC_DLOG(LS_ERROR) << "GetAudio() failed!";
+ // In all likelihood, the audio in this frame is garbage. We return an
+ // error so that the audio mixer module doesn't add it to the mix. As
+ // a result, it won't be played out and the actions skipped here are
+ // irrelevant.
+ return AudioMixer::Source::AudioFrameInfo::kError;
+ }
+
+ if (muted) {
+ AudioFrameOperations::Mute(audio_frame);
+ }
+
+ // Measure audio level.
+ constexpr double kAudioSampleDurationSeconds = 0.01;
+ output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
+
+ // If caller invoked StopPlay(), then mute the frame.
+ if (!playing_) {
+ AudioFrameOperations::Mute(audio_frame);
+ muted = true;
+ }
+
+ // Set first rtp timestamp with first audio frame with valid timestamp.
+ if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) {
+ first_rtp_timestamp_ = audio_frame->timestamp_;
+ }
+
+ if (first_rtp_timestamp_ >= 0) {
+ // Compute elapsed and NTP times.
+ int64_t unwrap_timestamp;
+ {
+ MutexLock lock(&lock_);
+ unwrap_timestamp =
+ timestamp_wrap_handler_.Unwrap(audio_frame->timestamp_);
+ audio_frame->ntp_time_ms_ =
+ ntp_estimator_.Estimate(audio_frame->timestamp_);
+ }
+ // For clock rate, default to the playout sampling rate if we haven't
+ // received any packets yet.
+ absl::optional<std::pair<int, SdpAudioFormat>> decoder =
+ acm_receiver_.LastDecoder();
+ int clock_rate = decoder ? decoder->second.clockrate_hz
+ : acm_receiver_.last_output_sample_rate_hz();
+ RTC_DCHECK_GT(clock_rate, 0);
+ audio_frame->elapsed_time_ms_ =
+ (unwrap_timestamp - first_rtp_timestamp_) / (clock_rate / 1000);
+ }
+
+ return muted ? AudioMixer::Source::AudioFrameInfo::kMuted
+ : AudioMixer::Source::AudioFrameInfo::kNormal;
+}
+
+bool AudioIngress::StartPlay() {
+ {
+ MutexLock lock(&lock_);
+ if (receive_codec_info_.empty()) {
+ RTC_DLOG(LS_WARNING) << "Receive codecs have not been set yet";
+ return false;
+ }
+ }
+ playing_ = true;
+ return true;
+}
+
+void AudioIngress::SetReceiveCodecs(
+ const std::map<int, SdpAudioFormat>& codecs) {
+ {
+ MutexLock lock(&lock_);
+ for (const auto& kv : codecs) {
+ receive_codec_info_[kv.first] = kv.second.clockrate_hz;
+ }
+ }
+ acm_receiver_.SetCodecs(codecs);
+}
+
+void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) {
+ RtpPacketReceived rtp_packet_received;
+ rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size());
+
+ // Set payload type's sampling rate before we feed it into ReceiveStatistics.
+ {
+ MutexLock lock(&lock_);
+ const auto& it =
+ receive_codec_info_.find(rtp_packet_received.PayloadType());
+ // If sampling rate info is not available in our received codec set, it
+ // would mean that remote media endpoint is sending incorrect payload id
+ // which can't be processed correctly especially on payload type id in
+ // dynamic range.
+ if (it == receive_codec_info_.end()) {
+ RTC_DLOG(LS_WARNING) << "Unexpected payload id received: "
+ << rtp_packet_received.PayloadType();
+ return;
+ }
+ rtp_packet_received.set_payload_type_frequency(it->second);
+ }
+
+ // Track current remote SSRC.
+ if (rtp_packet_received.Ssrc() != remote_ssrc_) {
+ rtp_rtcp_->SetRemoteSSRC(rtp_packet_received.Ssrc());
+ remote_ssrc_.store(rtp_packet_received.Ssrc());
+ }
+
+ rtp_receive_statistics_->OnRtpPacket(rtp_packet_received);
+
+ RTPHeader header;
+ rtp_packet_received.GetHeader(&header);
+
+ size_t packet_length = rtp_packet_received.size();
+ if (packet_length < header.headerLength ||
+ (packet_length - header.headerLength) < header.paddingLength) {
+ RTC_DLOG(LS_ERROR) << "Packet length(" << packet_length << ") header("
+ << header.headerLength << ") padding("
+ << header.paddingLength << ")";
+ return;
+ }
+
+ const uint8_t* payload = rtp_packet_received.data() + header.headerLength;
+ size_t payload_length = packet_length - header.headerLength;
+ size_t payload_data_length = payload_length - header.paddingLength;
+ auto data_view = rtc::ArrayView<const uint8_t>(payload, payload_data_length);
+
+ // Push the incoming payload (parsed and ready for decoding) into the ACM.
+ if (acm_receiver_.InsertPacket(header, data_view) != 0) {
+ RTC_DLOG(LS_ERROR) << "AudioIngress::ReceivedRTPPacket() unable to "
+ "push data to the ACM";
+ }
+}
+
+void AudioIngress::ReceivedRTCPPacket(
+ rtc::ArrayView<const uint8_t> rtcp_packet) {
+ rtcp::CommonHeader rtcp_header;
+ if (rtcp_header.Parse(rtcp_packet.data(), rtcp_packet.size()) &&
+ (rtcp_header.type() == rtcp::SenderReport::kPacketType ||
+ rtcp_header.type() == rtcp::ReceiverReport::kPacketType)) {
+ RTC_DCHECK_GE(rtcp_packet.size(), 8);
+
+ uint32_t sender_ssrc =
+ ByteReader<uint32_t>::ReadBigEndian(rtcp_packet.data() + 4);
+
+ // If we don't have remote ssrc at this point, it's likely that remote
+ // endpoint is receive-only or it could have restarted the media.
+ if (sender_ssrc != remote_ssrc_) {
+ rtp_rtcp_->SetRemoteSSRC(sender_ssrc);
+ remote_ssrc_.store(sender_ssrc);
+ }
+ }
+
+ // Deliver RTCP packet to RTP/RTCP module for parsing and processing.
+ rtp_rtcp_->IncomingRtcpPacket(rtcp_packet);
+
+ absl::optional<TimeDelta> rtt = rtp_rtcp_->LastRtt();
+ if (!rtt.has_value()) {
+ // Waiting for valid RTT.
+ return;
+ }
+
+ absl::optional<RtpRtcpInterface::SenderReportStats> last_sr =
+ rtp_rtcp_->GetSenderReportStats();
+ if (!last_sr.has_value()) {
+ // Waiting for RTCP.
+ return;
+ }
+
+ {
+ MutexLock lock(&lock_);
+ ntp_estimator_.UpdateRtcpTimestamp(*rtt, last_sr->last_remote_timestamp,
+ last_sr->last_remote_rtp_timestamp);
+ }
+}
+
+ChannelStatistics AudioIngress::GetChannelStatistics() {
+ ChannelStatistics channel_stats;
+
+ // Get clockrate for current decoder ahead of jitter calculation.
+ uint32_t clockrate_hz = 0;
+ absl::optional<std::pair<int, SdpAudioFormat>> decoder =
+ acm_receiver_.LastDecoder();
+ if (decoder) {
+ clockrate_hz = decoder->second.clockrate_hz;
+ }
+
+ StreamStatistician* statistician =
+ rtp_receive_statistics_->GetStatistician(remote_ssrc_);
+ if (statistician) {
+ RtpReceiveStats stats = statistician->GetStats();
+ channel_stats.packets_lost = stats.packets_lost;
+ channel_stats.packets_received = stats.packet_counter.packets;
+ channel_stats.bytes_received = stats.packet_counter.payload_bytes;
+ channel_stats.remote_ssrc = remote_ssrc_;
+ if (clockrate_hz > 0) {
+ channel_stats.jitter = static_cast<double>(stats.jitter) / clockrate_hz;
+ }
+ }
+
+ // Get RTCP report using remote SSRC.
+ const std::vector<ReportBlockData>& report_data =
+ rtp_rtcp_->GetLatestReportBlockData();
+ for (const ReportBlockData& rtcp_report : report_data) {
+ if (rtp_rtcp_->SSRC() != rtcp_report.source_ssrc() ||
+ remote_ssrc_ != rtcp_report.sender_ssrc()) {
+ continue;
+ }
+ RemoteRtcpStatistics remote_stat;
+ remote_stat.packets_lost = rtcp_report.cumulative_lost();
+ remote_stat.fraction_lost = rtcp_report.fraction_lost();
+ if (clockrate_hz > 0) {
+ remote_stat.jitter = rtcp_report.jitter(clockrate_hz).seconds<double>();
+ }
+ if (rtcp_report.has_rtt()) {
+ remote_stat.round_trip_time = rtcp_report.last_rtt().seconds<double>();
+ }
+ remote_stat.last_report_received_timestamp_ms =
+ rtcp_report.report_block_timestamp_utc().ms();
+ channel_stats.remote_rtcp = remote_stat;
+
+ // Receive only channel won't send any RTP packets.
+ if (!channel_stats.remote_ssrc.has_value()) {
+ channel_stats.remote_ssrc = remote_ssrc_;
+ }
+ break;
+ }
+
+ return channel_stats;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/audio_ingress.h b/third_party/libwebrtc/audio/voip/audio_ingress.h
new file mode 100644
index 0000000000..11bde7ce28
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/audio_ingress.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_VOIP_AUDIO_INGRESS_H_
+#define AUDIO_VOIP_AUDIO_INGRESS_H_
+
+#include <algorithm>
+#include <atomic>
+#include <map>
+#include <memory>
+#include <utility>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/audio/audio_mixer.h"
+#include "api/rtp_headers.h"
+#include "api/scoped_refptr.h"
+#include "api/voip/voip_statistics.h"
+#include "audio/audio_level.h"
+#include "modules/audio_coding/acm2/acm_receiver.h"
+#include "modules/audio_coding/include/audio_coding_module.h"
+#include "modules/rtp_rtcp/include/receive_statistics.h"
+#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+
+// AudioIngress handles incoming RTP/RTCP packets from the remote
+// media endpoint. Received RTP packets are injected into AcmReceiver and
+// when audio output thread requests for audio samples to play through system
+// output such as speaker device, AudioIngress provides the samples via its
+// implementation on AudioMixer::Source interface.
+//
+// Note that this class is originally based on ChannelReceive in
+// audio/channel_receive.cc with non-audio related logic trimmed as aimed for
+// smaller footprint.
+class AudioIngress : public AudioMixer::Source {
+ public:
+ AudioIngress(RtpRtcpInterface* rtp_rtcp,
+ Clock* clock,
+ ReceiveStatistics* receive_statistics,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory);
+ ~AudioIngress() override;
+
+ // Start or stop receiving operation of AudioIngress.
+ bool StartPlay();
+ void StopPlay() {
+ playing_ = false;
+ output_audio_level_.ResetLevelFullRange();
+ }
+
+ // Query the state of the AudioIngress.
+ bool IsPlaying() const { return playing_; }
+
+ // Set the decoder formats and payload type for AcmReceiver where the
+ // key type (int) of the map is the payload type of SdpAudioFormat.
+ void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs);
+
+ // APIs to handle received RTP/RTCP packets from caller.
+ void ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet);
+ void ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet);
+
+ // See comments on LevelFullRange, TotalEnergy, TotalDuration from
+ // audio/audio_level.h.
+ int GetOutputAudioLevel() const {
+ return output_audio_level_.LevelFullRange();
+ }
+ double GetOutputTotalEnergy() { return output_audio_level_.TotalEnergy(); }
+ double GetOutputTotalDuration() {
+ return output_audio_level_.TotalDuration();
+ }
+
+ NetworkStatistics GetNetworkStatistics() const {
+ NetworkStatistics stats;
+ acm_receiver_.GetNetworkStatistics(&stats,
+ /*get_and_clear_legacy_stats=*/false);
+ return stats;
+ }
+
+ ChannelStatistics GetChannelStatistics();
+
+ // Implementation of AudioMixer::Source interface.
+ AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo(
+ int sampling_rate,
+ AudioFrame* audio_frame) override;
+ int Ssrc() const override {
+ return rtc::dchecked_cast<int>(remote_ssrc_.load());
+ }
+ int PreferredSampleRate() const override {
+ // If we haven't received any RTP packet from remote and thus
+ // last_packet_sampling_rate is not available then use NetEq's sampling
+ // rate as that would be what would be used for audio output sample.
+ return std::max(acm_receiver_.last_packet_sample_rate_hz().value_or(0),
+ acm_receiver_.last_output_sample_rate_hz());
+ }
+
+ private:
+ // Indicates AudioIngress status as caller invokes Start/StopPlaying.
+ // If not playing, incoming RTP data processing is skipped, thus
+ // producing no data to output device.
+ std::atomic<bool> playing_;
+
+ // Currently active remote ssrc from remote media endpoint.
+ std::atomic<uint32_t> remote_ssrc_;
+
+ // The first rtp timestamp of the output audio frame that is used to
+ // calculate elasped time for subsequent audio frames.
+ std::atomic<int64_t> first_rtp_timestamp_;
+
+ // Synchronizaton is handled internally by ReceiveStatistics.
+ ReceiveStatistics* const rtp_receive_statistics_;
+
+ // Synchronizaton is handled internally by RtpRtcpInterface.
+ RtpRtcpInterface* const rtp_rtcp_;
+
+ // Synchronizaton is handled internally by acm2::AcmReceiver.
+ acm2::AcmReceiver acm_receiver_;
+
+ // Synchronizaton is handled internally by voe::AudioLevel.
+ voe::AudioLevel output_audio_level_;
+
+ Mutex lock_;
+
+ RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(lock_);
+
+ // For receiving RTP statistics, this tracks the sampling rate value
+ // per payload type set when caller set via SetReceiveCodecs.
+ std::map<int, int> receive_codec_info_ RTC_GUARDED_BY(lock_);
+
+ RtpTimestampUnwrapper timestamp_wrap_handler_ RTC_GUARDED_BY(lock_);
+};
+
+} // namespace webrtc
+
+#endif // AUDIO_VOIP_AUDIO_INGRESS_H_
diff --git a/third_party/libwebrtc/audio/voip/test/BUILD.gn b/third_party/libwebrtc/audio/voip/test/BUILD.gn
new file mode 100644
index 0000000000..00e9bee622
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/BUILD.gn
@@ -0,0 +1,107 @@
+# Copyright(c) 2020 The WebRTC project authors.All Rights Reserved.
+#
+# Use of this source code is governed by a BSD - style license
+# that can be found in the LICENSE file in the root of the source
+# tree.An additional intellectual property rights grant can be found
+# in the file PATENTS.All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../webrtc.gni")
+
+if (rtc_include_tests) {
+ rtc_source_set("mock_task_queue") {
+ testonly = true
+ visibility = [ "*" ]
+ sources = [ "mock_task_queue.h" ]
+ deps = [
+ "../../../api/task_queue:task_queue",
+ "../../../api/task_queue/test:mock_task_queue_base",
+ "../../../test:test_support",
+ ]
+ }
+
+ if (!build_with_chromium) {
+ rtc_library("voip_core_unittests") {
+ testonly = true
+ sources = [ "voip_core_unittest.cc" ]
+ deps = [
+ "..:voip_core",
+ "../../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../../api/task_queue:default_task_queue_factory",
+ "../../../modules/audio_device:mock_audio_device",
+ "../../../modules/audio_processing:mocks",
+ "../../../test:audio_codec_mocks",
+ "../../../test:mock_transport",
+ "../../../test:run_loop",
+ "../../../test:test_support",
+ ]
+ }
+ }
+
+ rtc_library("audio_channel_unittests") {
+ testonly = true
+ sources = [ "audio_channel_unittest.cc" ]
+ deps = [
+ ":mock_task_queue",
+ "..:audio_channel",
+ "../../../api:transport_api",
+ "../../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../../api/task_queue:task_queue",
+ "../../../modules/audio_mixer:audio_mixer_impl",
+ "../../../modules/audio_mixer:audio_mixer_test_utils",
+ "../../../modules/rtp_rtcp:rtp_rtcp",
+ "../../../modules/rtp_rtcp:rtp_rtcp_format",
+ "../../../rtc_base:logging",
+ "../../../test:mock_transport",
+ "../../../test:test_support",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/functional:any_invocable" ]
+ }
+
+ rtc_library("audio_ingress_unittests") {
+ testonly = true
+ sources = [ "audio_ingress_unittest.cc" ]
+ deps = [
+ "..:audio_egress",
+ "..:audio_ingress",
+ "../../../api:transport_api",
+ "../../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../../api/task_queue:default_task_queue_factory",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../modules/audio_mixer:audio_mixer_test_utils",
+ "../../../modules/rtp_rtcp:rtp_rtcp",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rtc_event",
+ "../../../test:mock_transport",
+ "../../../test:run_loop",
+ "../../../test:test_support",
+ "../../../test/time_controller:time_controller",
+ ]
+ }
+
+ rtc_library("audio_egress_unittests") {
+ testonly = true
+ sources = [ "audio_egress_unittest.cc" ]
+ deps = [
+ "..:audio_egress",
+ "../../../api:transport_api",
+ "../../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../../api/task_queue:default_task_queue_factory",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../modules/audio_mixer:audio_mixer_test_utils",
+ "../../../modules/rtp_rtcp:rtp_rtcp",
+ "../../../modules/rtp_rtcp:rtp_rtcp_format",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rtc_event",
+ "../../../test:mock_transport",
+ "../../../test:run_loop",
+ "../../../test:test_support",
+ "../../../test/time_controller:time_controller",
+ ]
+ }
+}
diff --git a/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc
new file mode 100644
index 0000000000..0c8312b738
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/audio_channel_unittest.cc
@@ -0,0 +1,355 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_channel.h"
+
+#include "absl/functional/any_invocable.h"
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/call/transport.h"
+#include "api/task_queue/task_queue_base.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "audio/voip/test/mock_task_queue.h"
+#include "modules/audio_mixer/audio_mixer_impl.h"
+#include "modules/audio_mixer/sine_wave_generator.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "rtc_base/logging.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_transport.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Invoke;
+using ::testing::NiceMock;
+using ::testing::Return;
+using ::testing::Unused;
+using ::testing::WithArg;
+
+constexpr uint64_t kStartTime = 123456789;
+constexpr uint32_t kLocalSsrc = 0xdeadc0de;
+constexpr int16_t kAudioLevel = 3004; // used for sine wave level
+constexpr int kPcmuPayload = 0;
+
+class AudioChannelTest : public ::testing::Test {
+ public:
+ const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1};
+
+ AudioChannelTest()
+ : fake_clock_(kStartTime), wave_generator_(1000.0, kAudioLevel) {
+ task_queue_factory_ = std::make_unique<MockTaskQueueFactory>(&task_queue_);
+ audio_mixer_ = AudioMixerImpl::Create();
+ encoder_factory_ = CreateBuiltinAudioEncoderFactory();
+ decoder_factory_ = CreateBuiltinAudioDecoderFactory();
+
+ // By default, run the queued task immediately.
+ ON_CALL(task_queue_, PostTaskImpl)
+ .WillByDefault(WithArg<0>(
+ [](absl::AnyInvocable<void() &&> task) { std::move(task)(); }));
+ }
+
+ void SetUp() override { audio_channel_ = CreateAudioChannel(kLocalSsrc); }
+
+ void TearDown() override { audio_channel_ = nullptr; }
+
+ rtc::scoped_refptr<AudioChannel> CreateAudioChannel(uint32_t ssrc) {
+ // Use same audio mixer here for simplicity sake as we are not checking
+ // audio activity of RTP in our testcases. If we need to do test on audio
+ // signal activity then we need to assign audio mixer for each channel.
+ // Also this uses the same transport object for different audio channel to
+ // simplify network routing logic.
+ rtc::scoped_refptr<AudioChannel> audio_channel =
+ rtc::make_ref_counted<AudioChannel>(
+ &transport_, ssrc, task_queue_factory_.get(), audio_mixer_.get(),
+ decoder_factory_);
+ audio_channel->SetEncoder(kPcmuPayload, kPcmuFormat,
+ encoder_factory_->MakeAudioEncoder(
+ kPcmuPayload, kPcmuFormat, absl::nullopt));
+ audio_channel->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}});
+ audio_channel->StartSend();
+ audio_channel->StartPlay();
+ return audio_channel;
+ }
+
+ std::unique_ptr<AudioFrame> GetAudioFrame(int order) {
+ auto frame = std::make_unique<AudioFrame>();
+ frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz;
+ frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms.
+ frame->num_channels_ = kPcmuFormat.num_channels;
+ frame->timestamp_ = frame->samples_per_channel_ * order;
+ wave_generator_.GenerateNextFrame(frame.get());
+ return frame;
+ }
+
+ SimulatedClock fake_clock_;
+ SineWaveGenerator wave_generator_;
+ NiceMock<MockTransport> transport_;
+ NiceMock<MockTaskQueue> task_queue_;
+ std::unique_ptr<TaskQueueFactory> task_queue_factory_;
+ rtc::scoped_refptr<AudioMixer> audio_mixer_;
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+ rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_;
+ rtc::scoped_refptr<AudioChannel> audio_channel_;
+};
+
+// Validate RTP packet generation by feeding audio frames with sine wave.
+// Resulted RTP packet is looped back into AudioChannel and gets decoded into
+// audio frame to see if it has some signal to indicate its validity.
+TEST_F(AudioChannelTest, PlayRtpByLocalLoop) {
+ auto loop_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ audio_channel_->ReceivedRTPPacket(packet);
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp));
+
+ auto audio_sender = audio_channel_->GetAudioSender();
+ audio_sender->SendAudioData(GetAudioFrame(0));
+ audio_sender->SendAudioData(GetAudioFrame(1));
+
+ AudioFrame empty_frame, audio_frame;
+ empty_frame.Mute();
+ empty_frame.mutable_data(); // This will zero out the data.
+ audio_frame.CopyFrom(empty_frame);
+ audio_mixer_->Mix(/*number_of_channels*/ 1, &audio_frame);
+
+ // We expect now audio frame to pick up something.
+ EXPECT_NE(memcmp(empty_frame.data(), audio_frame.data(),
+ AudioFrame::kMaxDataSizeBytes),
+ 0);
+}
+
+// Validate assigned local SSRC is resulted in RTP packet.
+TEST_F(AudioChannelTest, VerifyLocalSsrcAsAssigned) {
+ RtpPacketReceived rtp;
+ auto loop_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ rtp.Parse(packet);
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillOnce(Invoke(loop_rtp));
+
+ auto audio_sender = audio_channel_->GetAudioSender();
+ audio_sender->SendAudioData(GetAudioFrame(0));
+ audio_sender->SendAudioData(GetAudioFrame(1));
+
+ EXPECT_EQ(rtp.Ssrc(), kLocalSsrc);
+}
+
+// Check metrics after processing an RTP packet.
+TEST_F(AudioChannelTest, TestIngressStatistics) {
+ auto loop_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ audio_channel_->ReceivedRTPPacket(packet);
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp));
+
+ auto audio_sender = audio_channel_->GetAudioSender();
+ audio_sender->SendAudioData(GetAudioFrame(0));
+ audio_sender->SendAudioData(GetAudioFrame(1));
+
+ AudioFrame audio_frame;
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+
+ absl::optional<IngressStatistics> ingress_stats =
+ audio_channel_->GetIngressStatistics();
+ EXPECT_TRUE(ingress_stats);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 160ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL);
+ // To extract the jitter buffer length in millisecond, jitter_buffer_delay_ms
+ // needs to be divided by jitter_buffer_emitted_count (number of samples).
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL);
+ EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0);
+ EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.02);
+
+ // Now without any RTP pending in jitter buffer pull more.
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+
+ // Send another RTP packet to intentionally break PLC.
+ audio_sender->SendAudioData(GetAudioFrame(2));
+ audio_sender->SendAudioData(GetAudioFrame(3));
+
+ ingress_stats = audio_channel_->GetIngressStatistics();
+ EXPECT_TRUE(ingress_stats);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 320ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 1600ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 160ULL);
+ EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0);
+ EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.04);
+
+ // Pull the last RTP packet.
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+
+ ingress_stats = audio_channel_->GetIngressStatistics();
+ EXPECT_TRUE(ingress_stats);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_samples_received, 480ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealed_samples, 168ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.concealment_events, 1ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.inserted_samples_for_deceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.removed_samples_for_acceleration, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.silent_concealed_samples, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_delay_ms, 3200ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.jitter_buffer_emitted_count, 320ULL);
+ EXPECT_GT(ingress_stats->neteq_stats.jitter_buffer_target_delay_ms, 0ULL);
+ EXPECT_EQ(ingress_stats->neteq_stats.interruption_count, 0);
+ EXPECT_EQ(ingress_stats->neteq_stats.total_interruption_duration_ms, 0);
+ EXPECT_DOUBLE_EQ(ingress_stats->total_duration, 0.06);
+}
+
+// Check ChannelStatistics metric after processing RTP and RTCP packets.
+TEST_F(AudioChannelTest, TestChannelStatistics) {
+ auto loop_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ audio_channel_->ReceivedRTPPacket(packet);
+ return true;
+ };
+ auto loop_rtcp = [&](rtc::ArrayView<const uint8_t> packet) {
+ audio_channel_->ReceivedRTCPPacket(packet);
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(loop_rtp));
+ EXPECT_CALL(transport_, SendRtcp).WillRepeatedly(Invoke(loop_rtcp));
+
+ // Simulate microphone giving audio frame (10 ms). This will trigger transport
+ // to send RTP as handled in loop_rtp above.
+ auto audio_sender = audio_channel_->GetAudioSender();
+ audio_sender->SendAudioData(GetAudioFrame(0));
+ audio_sender->SendAudioData(GetAudioFrame(1));
+
+ // Simulate speaker requesting audio frame (10 ms). This will trigger VoIP
+ // engine to fetch audio samples from RTP packets stored in jitter buffer.
+ AudioFrame audio_frame;
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+
+ // Force sending RTCP SR report in order to have remote_rtcp field available
+ // in channel statistics. This will trigger transport to send RTCP as handled
+ // in loop_rtcp above.
+ audio_channel_->SendRTCPReportForTesting(kRtcpSr);
+
+ absl::optional<ChannelStatistics> channel_stats =
+ audio_channel_->GetChannelStatistics();
+ EXPECT_TRUE(channel_stats);
+
+ EXPECT_EQ(channel_stats->packets_sent, 1ULL);
+ EXPECT_EQ(channel_stats->bytes_sent, 160ULL);
+
+ EXPECT_EQ(channel_stats->packets_received, 1ULL);
+ EXPECT_EQ(channel_stats->bytes_received, 160ULL);
+ EXPECT_EQ(channel_stats->jitter, 0);
+ EXPECT_EQ(channel_stats->packets_lost, 0);
+ EXPECT_EQ(channel_stats->remote_ssrc.value(), kLocalSsrc);
+
+ EXPECT_TRUE(channel_stats->remote_rtcp.has_value());
+
+ EXPECT_EQ(channel_stats->remote_rtcp->jitter, 0);
+ EXPECT_EQ(channel_stats->remote_rtcp->packets_lost, 0);
+ EXPECT_EQ(channel_stats->remote_rtcp->fraction_lost, 0);
+ EXPECT_GT(channel_stats->remote_rtcp->last_report_received_timestamp_ms, 0);
+ EXPECT_FALSE(channel_stats->remote_rtcp->round_trip_time.has_value());
+}
+
+// Check ChannelStatistics RTT metric after processing RTP and RTCP packets
+// using three audio channels where each represents media endpoint.
+//
+// 1) AC1 <- RTP/RTCP -> AC2
+// 2) AC1 <- RTP/RTCP -> AC3
+//
+// During step 1), AC1 should be able to check RTT from AC2's SSRC.
+// During step 2), AC1 should be able to check RTT from AC3's SSRC.
+TEST_F(AudioChannelTest, RttIsAvailableAfterChangeOfRemoteSsrc) {
+ // Create AC2 and AC3.
+ constexpr uint32_t kAc2Ssrc = 0xdeadbeef;
+ constexpr uint32_t kAc3Ssrc = 0xdeafbeef;
+
+ auto ac_2 = CreateAudioChannel(kAc2Ssrc);
+ auto ac_3 = CreateAudioChannel(kAc3Ssrc);
+
+ auto send_recv_rtp = [&](rtc::scoped_refptr<AudioChannel> rtp_sender,
+ rtc::scoped_refptr<AudioChannel> rtp_receiver) {
+ // Setup routing logic via transport_.
+ auto route_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ rtp_receiver->ReceivedRTPPacket(packet);
+ return true;
+ };
+ ON_CALL(transport_, SendRtp).WillByDefault(route_rtp);
+
+ // This will trigger route_rtp callback via transport_.
+ rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(0));
+ rtp_sender->GetAudioSender()->SendAudioData(GetAudioFrame(1));
+
+ // Process received RTP in receiver.
+ AudioFrame audio_frame;
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+ audio_mixer_->Mix(/*number_of_channels=*/1, &audio_frame);
+
+ // Revert to default to avoid using reference in route_rtp lambda.
+ ON_CALL(transport_, SendRtp).WillByDefault(Return(true));
+ };
+
+ auto send_recv_rtcp = [&](rtc::scoped_refptr<AudioChannel> rtcp_sender,
+ rtc::scoped_refptr<AudioChannel> rtcp_receiver) {
+ // Setup routing logic via transport_.
+ auto route_rtcp = [&](rtc::ArrayView<const uint8_t> packet) {
+ rtcp_receiver->ReceivedRTCPPacket(packet);
+ return true;
+ };
+ ON_CALL(transport_, SendRtcp).WillByDefault(route_rtcp);
+
+ // This will trigger route_rtcp callback via transport_.
+ rtcp_sender->SendRTCPReportForTesting(kRtcpSr);
+
+ // Revert to default to avoid using reference in route_rtcp lambda.
+ ON_CALL(transport_, SendRtcp).WillByDefault(Return(true));
+ };
+
+ // AC1 <-- RTP/RTCP --> AC2
+ send_recv_rtp(audio_channel_, ac_2);
+ send_recv_rtp(ac_2, audio_channel_);
+ send_recv_rtcp(audio_channel_, ac_2);
+ send_recv_rtcp(ac_2, audio_channel_);
+
+ absl::optional<ChannelStatistics> channel_stats =
+ audio_channel_->GetChannelStatistics();
+ ASSERT_TRUE(channel_stats);
+ EXPECT_EQ(channel_stats->remote_ssrc, kAc2Ssrc);
+ ASSERT_TRUE(channel_stats->remote_rtcp);
+ EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0);
+
+ // AC1 <-- RTP/RTCP --> AC3
+ send_recv_rtp(audio_channel_, ac_3);
+ send_recv_rtp(ac_3, audio_channel_);
+ send_recv_rtcp(audio_channel_, ac_3);
+ send_recv_rtcp(ac_3, audio_channel_);
+
+ channel_stats = audio_channel_->GetChannelStatistics();
+ ASSERT_TRUE(channel_stats);
+ EXPECT_EQ(channel_stats->remote_ssrc, kAc3Ssrc);
+ ASSERT_TRUE(channel_stats->remote_rtcp);
+ EXPECT_GT(channel_stats->remote_rtcp->round_trip_time, 0.0);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc
new file mode 100644
index 0000000000..83df26eef1
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/audio_egress_unittest.cc
@@ -0,0 +1,326 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_egress.h"
+
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/call/transport.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "modules/audio_mixer/sine_wave_generator.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
+#include "rtc_base/event.h"
+#include "rtc_base/logging.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_transport.h"
+#include "test/run_loop.h"
+#include "test/time_controller/simulated_time_controller.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Invoke;
+using ::testing::NiceMock;
+using ::testing::Unused;
+
+std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpStack(Clock* clock,
+ Transport* transport,
+ uint32_t remote_ssrc) {
+ RtpRtcpInterface::Configuration rtp_config;
+ rtp_config.clock = clock;
+ rtp_config.audio = true;
+ rtp_config.rtcp_report_interval_ms = 5000;
+ rtp_config.outgoing_transport = transport;
+ rtp_config.local_media_ssrc = remote_ssrc;
+ auto rtp_rtcp = ModuleRtpRtcpImpl2::Create(rtp_config);
+ rtp_rtcp->SetSendingMediaStatus(false);
+ rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
+ return rtp_rtcp;
+}
+
+constexpr int16_t kAudioLevel = 3004; // Used for sine wave level.
+
+// AudioEgressTest configures audio egress by using Rtp Stack, fake clock,
+// and task queue factory. Encoder factory is needed to create codec and
+// configure the RTP stack in audio egress.
+class AudioEgressTest : public ::testing::Test {
+ public:
+ static constexpr uint16_t kSeqNum = 12345;
+ static constexpr uint64_t kStartTime = 123456789;
+ static constexpr uint32_t kRemoteSsrc = 0xDEADBEEF;
+ const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1};
+
+ AudioEgressTest() : wave_generator_(1000.0, kAudioLevel) {
+ encoder_factory_ = CreateBuiltinAudioEncoderFactory();
+ }
+
+ // Prepare test on audio egress by using PCMu codec with specific
+ // sequence number and its status to be running.
+ void SetUp() override {
+ rtp_rtcp_ =
+ CreateRtpStack(time_controller_.GetClock(), &transport_, kRemoteSsrc);
+ egress_ = std::make_unique<AudioEgress>(
+ rtp_rtcp_.get(), time_controller_.GetClock(),
+ time_controller_.GetTaskQueueFactory());
+ constexpr int kPcmuPayload = 0;
+ egress_->SetEncoder(kPcmuPayload, kPcmuFormat,
+ encoder_factory_->MakeAudioEncoder(
+ kPcmuPayload, kPcmuFormat, absl::nullopt));
+ egress_->StartSend();
+ rtp_rtcp_->SetSequenceNumber(kSeqNum);
+ rtp_rtcp_->SetSendingStatus(true);
+ }
+
+ // Make sure we have shut down rtp stack and reset egress for each test.
+ void TearDown() override {
+ egress_->StopSend();
+ rtp_rtcp_->SetSendingStatus(false);
+ egress_.reset();
+ rtp_rtcp_.reset();
+ }
+
+ // Create an audio frame prepared for pcmu encoding. Timestamp is
+ // increased per RTP specification which is the number of samples it contains.
+ // Wave generator writes sine wave which has expected high level set
+ // by kAudioLevel.
+ std::unique_ptr<AudioFrame> GetAudioFrame(int order) {
+ auto frame = std::make_unique<AudioFrame>();
+ frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz;
+ frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms.
+ frame->num_channels_ = kPcmuFormat.num_channels;
+ frame->timestamp_ = frame->samples_per_channel_ * order;
+ wave_generator_.GenerateNextFrame(frame.get());
+ return frame;
+ }
+
+ GlobalSimulatedTimeController time_controller_{Timestamp::Micros(kStartTime)};
+ NiceMock<MockTransport> transport_;
+ SineWaveGenerator wave_generator_;
+ std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
+ rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_;
+ std::unique_ptr<AudioEgress> egress_;
+};
+
+TEST_F(AudioEgressTest, SendingStatusAfterStartAndStop) {
+ EXPECT_TRUE(egress_->IsSending());
+ egress_->StopSend();
+ EXPECT_FALSE(egress_->IsSending());
+}
+
+TEST_F(AudioEgressTest, ProcessAudioWithMute) {
+ constexpr int kExpected = 10;
+ rtc::Event event;
+ int rtp_count = 0;
+ RtpPacketReceived rtp;
+ auto rtp_sent = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ rtp.Parse(packet);
+ if (++rtp_count == kExpected) {
+ event.Set();
+ }
+ return true;
+ };
+
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent));
+
+ egress_->SetMute(true);
+
+ // Two 10 ms audio frames will result in rtp packet with ptime 20.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+
+ event.Wait(TimeDelta::Seconds(1));
+ EXPECT_EQ(rtp_count, kExpected);
+
+ // we expect on pcmu payload to result in 255 for silenced payload
+ RTPHeader header;
+ rtp.GetHeader(&header);
+ size_t packet_length = rtp.size();
+ size_t payload_length = packet_length - header.headerLength;
+ size_t payload_data_length = payload_length - header.paddingLength;
+ const uint8_t* payload = rtp.data() + header.headerLength;
+ for (size_t i = 0; i < payload_data_length; ++i) {
+ EXPECT_EQ(*payload++, 255);
+ }
+}
+
+TEST_F(AudioEgressTest, ProcessAudioWithSineWave) {
+ constexpr int kExpected = 10;
+ rtc::Event event;
+ int rtp_count = 0;
+ RtpPacketReceived rtp;
+ auto rtp_sent = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ rtp.Parse(packet);
+ if (++rtp_count == kExpected) {
+ event.Set();
+ }
+ return true;
+ };
+
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent));
+
+ // Two 10 ms audio frames will result in rtp packet with ptime 20.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+
+ event.Wait(TimeDelta::Seconds(1));
+ EXPECT_EQ(rtp_count, kExpected);
+
+ // we expect on pcmu to result in < 255 for payload with sine wave
+ RTPHeader header;
+ rtp.GetHeader(&header);
+ size_t packet_length = rtp.size();
+ size_t payload_length = packet_length - header.headerLength;
+ size_t payload_data_length = payload_length - header.paddingLength;
+ const uint8_t* payload = rtp.data() + header.headerLength;
+ for (size_t i = 0; i < payload_data_length; ++i) {
+ EXPECT_NE(*payload++, 255);
+ }
+}
+
+TEST_F(AudioEgressTest, SkipAudioEncodingAfterStopSend) {
+ constexpr int kExpected = 10;
+ rtc::Event event;
+ int rtp_count = 0;
+ auto rtp_sent = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ if (++rtp_count == kExpected) {
+ event.Set();
+ }
+ return true;
+ };
+
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent));
+
+ // Two 10 ms audio frames will result in rtp packet with ptime 20.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+
+ event.Wait(TimeDelta::Seconds(1));
+ EXPECT_EQ(rtp_count, kExpected);
+
+ // Now stop send and yet feed more data.
+ egress_->StopSend();
+
+ // It should be safe to exit the test case while encoder_queue_ has
+ // outstanding data to process. We are making sure that this doesn't
+ // result in crashes or sanitizer errors due to remaining data.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+}
+
+TEST_F(AudioEgressTest, ChangeEncoderFromPcmuToOpus) {
+ absl::optional<SdpAudioFormat> pcmu = egress_->GetEncoderFormat();
+ EXPECT_TRUE(pcmu);
+ EXPECT_EQ(pcmu->clockrate_hz, kPcmuFormat.clockrate_hz);
+ EXPECT_EQ(pcmu->num_channels, kPcmuFormat.num_channels);
+
+ constexpr int kOpusPayload = 120;
+ const SdpAudioFormat kOpusFormat = {"opus", 48000, 2};
+
+ egress_->SetEncoder(kOpusPayload, kOpusFormat,
+ encoder_factory_->MakeAudioEncoder(
+ kOpusPayload, kOpusFormat, absl::nullopt));
+
+ absl::optional<SdpAudioFormat> opus = egress_->GetEncoderFormat();
+ EXPECT_TRUE(opus);
+ EXPECT_EQ(opus->clockrate_hz, kOpusFormat.clockrate_hz);
+ EXPECT_EQ(opus->num_channels, kOpusFormat.num_channels);
+}
+
+TEST_F(AudioEgressTest, SendDTMF) {
+ constexpr int kExpected = 7;
+ constexpr int kPayloadType = 100;
+ constexpr int kDurationMs = 100;
+ constexpr int kSampleRate = 8000;
+ constexpr int kEvent = 3;
+
+ egress_->RegisterTelephoneEventType(kPayloadType, kSampleRate);
+ // 100 ms duration will produce total 7 DTMF
+ // 1 @ 20 ms, 2 @ 40 ms, 3 @ 60 ms, 4 @ 80 ms
+ // 5, 6, 7 @ 100 ms (last one sends 3 dtmf)
+ egress_->SendTelephoneEvent(kEvent, kDurationMs);
+
+ rtc::Event event;
+ int dtmf_count = 0;
+ auto is_dtmf = [&](RtpPacketReceived& rtp) {
+ return (rtp.PayloadType() == kPayloadType &&
+ rtp.SequenceNumber() == kSeqNum + dtmf_count &&
+ rtp.padding_size() == 0 && rtp.Marker() == (dtmf_count == 0) &&
+ rtp.Ssrc() == kRemoteSsrc);
+ };
+
+ // It's possible that we may have actual audio RTP packets along with
+ // DTMF packtets. We are only interested in the exact number of DTMF
+ // packets rtp stack is emitting.
+ auto rtp_sent = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ RtpPacketReceived rtp;
+ rtp.Parse(packet);
+ if (is_dtmf(rtp) && ++dtmf_count == kExpected) {
+ event.Set();
+ }
+ return true;
+ };
+
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent));
+
+ // Two 10 ms audio frames will result in rtp packet with ptime 20.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+
+ event.Wait(TimeDelta::Seconds(1));
+ EXPECT_EQ(dtmf_count, kExpected);
+}
+
+TEST_F(AudioEgressTest, TestAudioInputLevelAndEnergyDuration) {
+ // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to
+ // get audio level from input source.
+ constexpr int kExpected = 6;
+ rtc::Event event;
+ int rtp_count = 0;
+ auto rtp_sent = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ if (++rtp_count == kExpected) {
+ event.Set();
+ }
+ return true;
+ };
+
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(rtp_sent));
+
+ // Two 10 ms audio frames will result in rtp packet with ptime 20.
+ for (size_t i = 0; i < kExpected * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+
+ event.Wait(/*give_up_after=*/TimeDelta::Seconds(1));
+ EXPECT_EQ(rtp_count, kExpected);
+
+ constexpr double kExpectedEnergy = 0.00016809565587789564;
+ constexpr double kExpectedDuration = 0.11999999999999998;
+
+ EXPECT_EQ(egress_->GetInputAudioLevel(), kAudioLevel);
+ EXPECT_DOUBLE_EQ(egress_->GetInputTotalEnergy(), kExpectedEnergy);
+ EXPECT_DOUBLE_EQ(egress_->GetInputTotalDuration(), kExpectedDuration);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc b/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc
new file mode 100644
index 0000000000..c7736b247a
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/audio_ingress_unittest.cc
@@ -0,0 +1,240 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/audio_ingress.h"
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/call/transport.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/units/time_delta.h"
+#include "audio/voip/audio_egress.h"
+#include "modules/audio_mixer/sine_wave_generator.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
+#include "rtc_base/event.h"
+#include "rtc_base/logging.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_transport.h"
+#include "test/run_loop.h"
+#include "test/time_controller/simulated_time_controller.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Invoke;
+using ::testing::NiceMock;
+using ::testing::Unused;
+
+constexpr int16_t kAudioLevel = 3004; // Used for sine wave level.
+
+class AudioIngressTest : public ::testing::Test {
+ public:
+ const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1};
+
+ AudioIngressTest() : wave_generator_(1000.0, kAudioLevel) {
+ receive_statistics_ =
+ ReceiveStatistics::Create(time_controller_.GetClock());
+
+ RtpRtcpInterface::Configuration rtp_config;
+ rtp_config.clock = time_controller_.GetClock();
+ rtp_config.audio = true;
+ rtp_config.receive_statistics = receive_statistics_.get();
+ rtp_config.rtcp_report_interval_ms = 5000;
+ rtp_config.outgoing_transport = &transport_;
+ rtp_config.local_media_ssrc = 0xdeadc0de;
+ rtp_rtcp_ = ModuleRtpRtcpImpl2::Create(rtp_config);
+
+ rtp_rtcp_->SetSendingMediaStatus(false);
+ rtp_rtcp_->SetRTCPStatus(RtcpMode::kCompound);
+
+ encoder_factory_ = CreateBuiltinAudioEncoderFactory();
+ decoder_factory_ = CreateBuiltinAudioDecoderFactory();
+ }
+
+ void SetUp() override {
+ constexpr int kPcmuPayload = 0;
+ ingress_ = std::make_unique<AudioIngress>(
+ rtp_rtcp_.get(), time_controller_.GetClock(), receive_statistics_.get(),
+ decoder_factory_);
+ ingress_->SetReceiveCodecs({{kPcmuPayload, kPcmuFormat}});
+
+ egress_ = std::make_unique<AudioEgress>(
+ rtp_rtcp_.get(), time_controller_.GetClock(),
+ time_controller_.GetTaskQueueFactory());
+ egress_->SetEncoder(kPcmuPayload, kPcmuFormat,
+ encoder_factory_->MakeAudioEncoder(
+ kPcmuPayload, kPcmuFormat, absl::nullopt));
+ egress_->StartSend();
+ ingress_->StartPlay();
+ rtp_rtcp_->SetSendingStatus(true);
+ }
+
+ void TearDown() override {
+ rtp_rtcp_->SetSendingStatus(false);
+ ingress_->StopPlay();
+ egress_->StopSend();
+ egress_.reset();
+ ingress_.reset();
+ }
+
+ std::unique_ptr<AudioFrame> GetAudioFrame(int order) {
+ auto frame = std::make_unique<AudioFrame>();
+ frame->sample_rate_hz_ = kPcmuFormat.clockrate_hz;
+ frame->samples_per_channel_ = kPcmuFormat.clockrate_hz / 100; // 10 ms.
+ frame->num_channels_ = kPcmuFormat.num_channels;
+ frame->timestamp_ = frame->samples_per_channel_ * order;
+ wave_generator_.GenerateNextFrame(frame.get());
+ return frame;
+ }
+
+ GlobalSimulatedTimeController time_controller_{Timestamp::Micros(123456789)};
+ SineWaveGenerator wave_generator_;
+ NiceMock<MockTransport> transport_;
+ std::unique_ptr<ReceiveStatistics> receive_statistics_;
+ std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
+ rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_;
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+ std::unique_ptr<AudioIngress> ingress_;
+ std::unique_ptr<AudioEgress> egress_;
+};
+
+TEST_F(AudioIngressTest, PlayingAfterStartAndStop) {
+ EXPECT_EQ(ingress_->IsPlaying(), true);
+ ingress_->StopPlay();
+ EXPECT_EQ(ingress_->IsPlaying(), false);
+}
+
+TEST_F(AudioIngressTest, GetAudioFrameAfterRtpReceived) {
+ rtc::Event event;
+ auto handle_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ ingress_->ReceivedRTPPacket(packet);
+ event.Set();
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp));
+ egress_->SendAudioData(GetAudioFrame(0));
+ egress_->SendAudioData(GetAudioFrame(1));
+ time_controller_.AdvanceTime(TimeDelta::Zero());
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(1)));
+
+ AudioFrame audio_frame;
+ EXPECT_EQ(
+ ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame),
+ AudioMixer::Source::AudioFrameInfo::kNormal);
+ EXPECT_FALSE(audio_frame.muted());
+ EXPECT_EQ(audio_frame.num_channels_, 1u);
+ EXPECT_EQ(audio_frame.samples_per_channel_,
+ static_cast<size_t>(kPcmuFormat.clockrate_hz / 100));
+ EXPECT_EQ(audio_frame.sample_rate_hz_, kPcmuFormat.clockrate_hz);
+ EXPECT_NE(audio_frame.timestamp_, 0u);
+ EXPECT_EQ(audio_frame.elapsed_time_ms_, 0);
+}
+
+TEST_F(AudioIngressTest, TestSpeechOutputLevelAndEnergyDuration) {
+ // Per audio_level's kUpdateFrequency, we need more than 10 audio samples to
+ // get audio level from output source.
+ constexpr int kNumRtp = 6;
+ int rtp_count = 0;
+ rtc::Event event;
+ auto handle_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ ingress_->ReceivedRTPPacket(packet);
+ if (++rtp_count == kNumRtp) {
+ event.Set();
+ }
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp));
+ for (int i = 0; i < kNumRtp * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+ event.Wait(/*give_up_after=*/TimeDelta::Seconds(1));
+
+ for (int i = 0; i < kNumRtp * 2; ++i) {
+ AudioFrame audio_frame;
+ EXPECT_EQ(
+ ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame),
+ AudioMixer::Source::AudioFrameInfo::kNormal);
+ }
+ EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel);
+
+ constexpr double kExpectedEnergy = 0.00016809565587789564;
+ constexpr double kExpectedDuration = 0.11999999999999998;
+
+ EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalEnergy(), kExpectedEnergy);
+ EXPECT_DOUBLE_EQ(ingress_->GetOutputTotalDuration(), kExpectedDuration);
+}
+
+TEST_F(AudioIngressTest, PreferredSampleRate) {
+ rtc::Event event;
+ auto handle_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ ingress_->ReceivedRTPPacket(packet);
+ event.Set();
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp));
+ egress_->SendAudioData(GetAudioFrame(0));
+ egress_->SendAudioData(GetAudioFrame(1));
+ time_controller_.AdvanceTime(TimeDelta::Zero());
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(1)));
+
+ AudioFrame audio_frame;
+ EXPECT_EQ(
+ ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame),
+ AudioMixer::Source::AudioFrameInfo::kNormal);
+ EXPECT_EQ(ingress_->PreferredSampleRate(), kPcmuFormat.clockrate_hz);
+}
+
+// This test highlights the case where caller invokes StopPlay() which then
+// AudioIngress should play silence frame afterwards.
+TEST_F(AudioIngressTest, GetMutedAudioFrameAfterRtpReceivedAndStopPlay) {
+ // StopPlay before we start sending RTP packet with sine wave.
+ ingress_->StopPlay();
+
+ // Send 6 RTP packets to generate more than 100 ms audio sample to get
+ // valid speech level.
+ constexpr int kNumRtp = 6;
+ int rtp_count = 0;
+ rtc::Event event;
+ auto handle_rtp = [&](rtc::ArrayView<const uint8_t> packet, Unused) {
+ ingress_->ReceivedRTPPacket(packet);
+ if (++rtp_count == kNumRtp) {
+ event.Set();
+ }
+ return true;
+ };
+ EXPECT_CALL(transport_, SendRtp).WillRepeatedly(Invoke(handle_rtp));
+ for (int i = 0; i < kNumRtp * 2; i++) {
+ egress_->SendAudioData(GetAudioFrame(i));
+ time_controller_.AdvanceTime(TimeDelta::Millis(10));
+ }
+ event.Wait(/*give_up_after=*/TimeDelta::Seconds(1));
+
+ for (int i = 0; i < kNumRtp * 2; ++i) {
+ AudioFrame audio_frame;
+ EXPECT_EQ(
+ ingress_->GetAudioFrameWithInfo(kPcmuFormat.clockrate_hz, &audio_frame),
+ AudioMixer::Source::AudioFrameInfo::kMuted);
+ const int16_t* audio_data = audio_frame.data();
+ size_t length =
+ audio_frame.samples_per_channel_ * audio_frame.num_channels_;
+ for (size_t j = 0; j < length; ++j) {
+ EXPECT_EQ(audio_data[j], 0);
+ }
+ }
+
+ // Now we should still see valid speech output level as StopPlay won't affect
+ // the measurement.
+ EXPECT_EQ(ingress_->GetOutputAudioLevel(), kAudioLevel);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/test/mock_task_queue.h b/third_party/libwebrtc/audio/voip/test/mock_task_queue.h
new file mode 100644
index 0000000000..547b0d3f75
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/mock_task_queue.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_
+#define AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_
+
+#include <memory>
+
+#include "api/task_queue/task_queue_factory.h"
+#include "api/task_queue/test/mock_task_queue_base.h"
+#include "test/gmock.h"
+
+namespace webrtc {
+
+// MockTaskQueue enables immediate task run from global TaskQueueBase.
+// It's necessary for some tests depending on TaskQueueBase internally.
+class MockTaskQueue : public MockTaskQueueBase {
+ public:
+ MockTaskQueue() : current_(this) {}
+
+ // Delete is deliberately defined as no-op as MockTaskQueue is expected to
+ // hold onto current global TaskQueueBase throughout the testing.
+ void Delete() override {}
+
+ private:
+ CurrentTaskQueueSetter current_;
+};
+
+class MockTaskQueueFactory : public TaskQueueFactory {
+ public:
+ explicit MockTaskQueueFactory(MockTaskQueue* task_queue)
+ : task_queue_(task_queue) {}
+
+ std::unique_ptr<TaskQueueBase, TaskQueueDeleter> CreateTaskQueue(
+ absl::string_view name,
+ Priority priority) const override {
+ // Default MockTaskQueue::Delete is no-op, therefore it's safe to pass the
+ // raw pointer.
+ return std::unique_ptr<TaskQueueBase, TaskQueueDeleter>(task_queue_);
+ }
+
+ private:
+ MockTaskQueue* task_queue_;
+};
+
+} // namespace webrtc
+
+#endif // AUDIO_VOIP_TEST_MOCK_TASK_QUEUE_H_
diff --git a/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc b/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc
new file mode 100644
index 0000000000..b432506b12
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/test/voip_core_unittest.cc
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/voip_core.h"
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "modules/audio_device/include/mock_audio_device.h"
+#include "modules/audio_processing/include/mock_audio_processing.h"
+#include "test/gtest.h"
+#include "test/mock_transport.h"
+#include "test/run_loop.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::NiceMock;
+using ::testing::Return;
+
+constexpr int kPcmuPayload = 0;
+constexpr int kPcmuSampleRateHz = 8000;
+constexpr int kDtmfEventDurationMs = 1000;
+constexpr DtmfEvent kDtmfEventCode = DtmfEvent::kDigitZero;
+
+class VoipCoreTest : public ::testing::Test {
+ public:
+ const SdpAudioFormat kPcmuFormat = {"pcmu", 8000, 1};
+
+ VoipCoreTest() { audio_device_ = test::MockAudioDeviceModule::CreateNice(); }
+
+ void SetUp() override {
+ auto encoder_factory = CreateBuiltinAudioEncoderFactory();
+ auto decoder_factory = CreateBuiltinAudioDecoderFactory();
+ rtc::scoped_refptr<AudioProcessing> audio_processing =
+ rtc::make_ref_counted<NiceMock<test::MockAudioProcessing>>();
+
+ voip_core_ = std::make_unique<VoipCore>(
+ std::move(encoder_factory), std::move(decoder_factory),
+ CreateDefaultTaskQueueFactory(), audio_device_,
+ std::move(audio_processing));
+ }
+
+ test::RunLoop run_loop_;
+ std::unique_ptr<VoipCore> voip_core_;
+ NiceMock<MockTransport> transport_;
+ rtc::scoped_refptr<test::MockAudioDeviceModule> audio_device_;
+};
+
+// Validate expected API calls that involves with VoipCore. Some verification is
+// involved with checking mock audio device.
+TEST_F(VoipCoreTest, BasicVoipCoreOperation) {
+ // Program mock as non-operational and ready to start.
+ EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(false));
+ EXPECT_CALL(*audio_device_, Playing()).WillOnce(Return(false));
+ EXPECT_CALL(*audio_device_, InitRecording()).WillOnce(Return(0));
+ EXPECT_CALL(*audio_device_, InitPlayout()).WillOnce(Return(0));
+ EXPECT_CALL(*audio_device_, StartRecording()).WillOnce(Return(0));
+ EXPECT_CALL(*audio_device_, StartPlayout()).WillOnce(Return(0));
+
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat),
+ VoipResult::kOk);
+ EXPECT_EQ(
+ voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}),
+ VoipResult::kOk);
+
+ EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk);
+ EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kOk);
+
+ EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload,
+ kPcmuSampleRateHz),
+ VoipResult::kOk);
+
+ EXPECT_EQ(
+ voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs),
+ VoipResult::kOk);
+
+ // Program mock as operational that is ready to be stopped.
+ EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true));
+ EXPECT_CALL(*audio_device_, Playing()).WillOnce(Return(true));
+ EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0));
+ EXPECT_CALL(*audio_device_, StopPlayout()).WillOnce(Return(0));
+
+ EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk);
+ EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk);
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+}
+
+TEST_F(VoipCoreTest, ExpectFailToUseReleasedChannelId) {
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ // Release right after creation.
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+
+ // Now use released channel.
+
+ EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat),
+ VoipResult::kInvalidArgument);
+ EXPECT_EQ(
+ voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}),
+ VoipResult::kInvalidArgument);
+ EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload,
+ kPcmuSampleRateHz),
+ VoipResult::kInvalidArgument);
+ EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kInvalidArgument);
+ EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kInvalidArgument);
+ EXPECT_EQ(
+ voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs),
+ VoipResult::kInvalidArgument);
+}
+
+TEST_F(VoipCoreTest, SendDtmfEventWithoutRegistering) {
+ // Program mock as non-operational and ready to start send.
+ EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(false));
+ EXPECT_CALL(*audio_device_, InitRecording()).WillOnce(Return(0));
+ EXPECT_CALL(*audio_device_, StartRecording()).WillOnce(Return(0));
+
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat),
+ VoipResult::kOk);
+
+ EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kOk);
+ // Send Dtmf event without registering beforehand, thus payload
+ // type is not set and kFailedPrecondition is expected.
+ EXPECT_EQ(
+ voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs),
+ VoipResult::kFailedPrecondition);
+
+ // Program mock as sending and is ready to be stopped.
+ EXPECT_CALL(*audio_device_, Recording()).WillOnce(Return(true));
+ EXPECT_CALL(*audio_device_, StopRecording()).WillOnce(Return(0));
+
+ EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk);
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+}
+
+TEST_F(VoipCoreTest, SendDtmfEventWithoutStartSend) {
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ EXPECT_EQ(voip_core_->RegisterTelephoneEventType(channel, kPcmuPayload,
+ kPcmuSampleRateHz),
+ VoipResult::kOk);
+
+ // Send Dtmf event without calling StartSend beforehand, thus
+ // Dtmf events cannot be sent and kFailedPrecondition is expected.
+ EXPECT_EQ(
+ voip_core_->SendDtmfEvent(channel, kDtmfEventCode, kDtmfEventDurationMs),
+ VoipResult::kFailedPrecondition);
+
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+}
+
+TEST_F(VoipCoreTest, StartSendAndPlayoutWithoutSettingCodec) {
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ // Call StartSend and StartPlayout without setting send/receive
+ // codec. Code should see that codecs aren't set and return false.
+ EXPECT_EQ(voip_core_->StartSend(channel), VoipResult::kFailedPrecondition);
+ EXPECT_EQ(voip_core_->StartPlayout(channel), VoipResult::kFailedPrecondition);
+
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+}
+
+TEST_F(VoipCoreTest, StopSendAndPlayoutWithoutStarting) {
+ auto channel = voip_core_->CreateChannel(&transport_, 0xdeadc0de);
+
+ EXPECT_EQ(voip_core_->SetSendCodec(channel, kPcmuPayload, kPcmuFormat),
+ VoipResult::kOk);
+ EXPECT_EQ(
+ voip_core_->SetReceiveCodecs(channel, {{kPcmuPayload, kPcmuFormat}}),
+ VoipResult::kOk);
+
+ // Call StopSend and StopPlayout without starting them in
+ // the first place. Should see that it is already in the
+ // stopped state and return true.
+ EXPECT_EQ(voip_core_->StopSend(channel), VoipResult::kOk);
+ EXPECT_EQ(voip_core_->StopPlayout(channel), VoipResult::kOk);
+
+ EXPECT_EQ(voip_core_->ReleaseChannel(channel), VoipResult::kOk);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/voip_core.cc b/third_party/libwebrtc/audio/voip/voip_core.cc
new file mode 100644
index 0000000000..8df1c594aa
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/voip_core.cc
@@ -0,0 +1,500 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "audio/voip/voip_core.h"
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+
+#include "api/audio_codecs/audio_format.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+namespace {
+
+// For Windows, use specific enum type to initialize default audio device as
+// defined in AudioDeviceModule::WindowsDeviceType.
+#if defined(WEBRTC_WIN)
+constexpr AudioDeviceModule::WindowsDeviceType kAudioDeviceId =
+ AudioDeviceModule::WindowsDeviceType::kDefaultCommunicationDevice;
+#else
+constexpr uint16_t kAudioDeviceId = 0;
+#endif // defined(WEBRTC_WIN)
+
+// Maximum value range limit on ChannelId. This can be increased without any
+// side effect and only set at this moderate value for better readability for
+// logging.
+static constexpr int kMaxChannelId = 100000;
+
+} // namespace
+
+VoipCore::VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
+ std::unique_ptr<TaskQueueFactory> task_queue_factory,
+ rtc::scoped_refptr<AudioDeviceModule> audio_device_module,
+ rtc::scoped_refptr<AudioProcessing> audio_processing) {
+ encoder_factory_ = std::move(encoder_factory);
+ decoder_factory_ = std::move(decoder_factory);
+ task_queue_factory_ = std::move(task_queue_factory);
+ audio_device_module_ = std::move(audio_device_module);
+ audio_processing_ = std::move(audio_processing);
+ audio_mixer_ = AudioMixerImpl::Create();
+
+ // AudioTransportImpl depends on audio mixer and audio processing instances.
+ audio_transport_ = std::make_unique<AudioTransportImpl>(
+ audio_mixer_.get(), audio_processing_.get(), nullptr);
+}
+
+bool VoipCore::InitializeIfNeeded() {
+ // `audio_device_module_` internally owns a lock and the whole logic here
+ // needs to be executed atomically once using another lock in VoipCore.
+ // Further changes in this method will need to make sure that no deadlock is
+ // introduced in the future.
+ MutexLock lock(&lock_);
+
+ if (initialized_) {
+ return true;
+ }
+
+ // Initialize ADM.
+ if (audio_device_module_->Init() != 0) {
+ RTC_LOG(LS_ERROR) << "Failed to initialize the ADM.";
+ return false;
+ }
+
+ // Note that failures on initializing default recording/speaker devices are
+ // not considered to be fatal here. In certain case, caller may not care about
+ // recording device functioning (e.g webinar where only speaker is available).
+ // It's also possible that there are other audio devices available that may
+ // work.
+
+ // Initialize default speaker device.
+ if (audio_device_module_->SetPlayoutDevice(kAudioDeviceId) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to set playout device.";
+ }
+ if (audio_device_module_->InitSpeaker() != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to access speaker.";
+ }
+
+ // Initialize default recording device.
+ if (audio_device_module_->SetRecordingDevice(kAudioDeviceId) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to set recording device.";
+ }
+ if (audio_device_module_->InitMicrophone() != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to access microphone.";
+ }
+
+ // Set number of channels on speaker device.
+ bool available = false;
+ if (audio_device_module_->StereoPlayoutIsAvailable(&available) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to query stereo playout.";
+ }
+ if (audio_device_module_->SetStereoPlayout(available) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to set mono/stereo playout mode.";
+ }
+
+ // Set number of channels on recording device.
+ available = false;
+ if (audio_device_module_->StereoRecordingIsAvailable(&available) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to query stereo recording.";
+ }
+ if (audio_device_module_->SetStereoRecording(available) != 0) {
+ RTC_LOG(LS_WARNING) << "Unable to set stereo recording mode.";
+ }
+
+ if (audio_device_module_->RegisterAudioCallback(audio_transport_.get()) !=
+ 0) {
+ RTC_LOG(LS_WARNING) << "Unable to register audio callback.";
+ }
+
+ initialized_ = true;
+
+ return true;
+}
+
+ChannelId VoipCore::CreateChannel(Transport* transport,
+ absl::optional<uint32_t> local_ssrc) {
+ ChannelId channel_id;
+
+ // Set local ssrc to random if not set by caller.
+ if (!local_ssrc) {
+ Random random(rtc::TimeMicros());
+ local_ssrc = random.Rand<uint32_t>();
+ }
+
+ rtc::scoped_refptr<AudioChannel> channel =
+ rtc::make_ref_counted<AudioChannel>(transport, local_ssrc.value(),
+ task_queue_factory_.get(),
+ audio_mixer_.get(), decoder_factory_);
+
+ {
+ MutexLock lock(&lock_);
+
+ channel_id = static_cast<ChannelId>(next_channel_id_);
+ channels_[channel_id] = channel;
+ next_channel_id_++;
+ if (next_channel_id_ >= kMaxChannelId) {
+ next_channel_id_ = 0;
+ }
+ }
+
+ // Set ChannelId in audio channel for logging/debugging purpose.
+ channel->SetId(channel_id);
+
+ return channel_id;
+}
+
+VoipResult VoipCore::ReleaseChannel(ChannelId channel_id) {
+ // Destroy channel outside of the lock.
+ rtc::scoped_refptr<AudioChannel> channel;
+
+ bool no_channels_after_release = false;
+
+ {
+ MutexLock lock(&lock_);
+
+ auto iter = channels_.find(channel_id);
+ if (iter != channels_.end()) {
+ channel = std::move(iter->second);
+ channels_.erase(iter);
+ }
+
+ no_channels_after_release = channels_.empty();
+ }
+
+ VoipResult status_code = VoipResult::kOk;
+ if (!channel) {
+ RTC_LOG(LS_WARNING) << "Channel " << channel_id << " not found";
+ status_code = VoipResult::kInvalidArgument;
+ }
+
+ if (no_channels_after_release) {
+ // TODO(bugs.webrtc.org/11581): unclear if we still need to clear `channel`
+ // here.
+ channel = nullptr;
+
+ // Make sure to stop playout on ADM if it is playing.
+ if (audio_device_module_->Playing()) {
+ if (audio_device_module_->StopPlayout() != 0) {
+ RTC_LOG(LS_WARNING) << "StopPlayout failed";
+ status_code = VoipResult::kInternal;
+ }
+ }
+ }
+
+ return status_code;
+}
+
+rtc::scoped_refptr<AudioChannel> VoipCore::GetChannel(ChannelId channel_id) {
+ rtc::scoped_refptr<AudioChannel> channel;
+ {
+ MutexLock lock(&lock_);
+ auto iter = channels_.find(channel_id);
+ if (iter != channels_.end()) {
+ channel = iter->second;
+ }
+ }
+ if (!channel) {
+ RTC_LOG(LS_ERROR) << "Channel " << channel_id << " not found";
+ }
+ return channel;
+}
+
+bool VoipCore::UpdateAudioTransportWithSenders() {
+ std::vector<AudioSender*> audio_senders;
+
+ // Gather a list of audio channel that are currently sending along with
+ // highest sampling rate and channel numbers to configure into audio
+ // transport.
+ int max_sampling_rate = 8000;
+ size_t max_num_channels = 1;
+ {
+ MutexLock lock(&lock_);
+ // Reserve to prevent run time vector re-allocation.
+ audio_senders.reserve(channels_.size());
+ for (auto kv : channels_) {
+ rtc::scoped_refptr<AudioChannel>& channel = kv.second;
+ if (channel->IsSendingMedia()) {
+ auto encoder_format = channel->GetEncoderFormat();
+ if (!encoder_format) {
+ RTC_LOG(LS_ERROR)
+ << "channel " << channel->GetId() << " encoder is not set";
+ continue;
+ }
+ audio_senders.push_back(channel->GetAudioSender());
+ max_sampling_rate =
+ std::max(max_sampling_rate, encoder_format->clockrate_hz);
+ max_num_channels =
+ std::max(max_num_channels, encoder_format->num_channels);
+ }
+ }
+ }
+
+ audio_transport_->UpdateAudioSenders(audio_senders, max_sampling_rate,
+ max_num_channels);
+
+ // Depending on availability of senders, turn on or off ADM recording.
+ if (!audio_senders.empty()) {
+ // Initialize audio device module and default device if needed.
+ if (!InitializeIfNeeded()) {
+ return false;
+ }
+
+ if (!audio_device_module_->Recording()) {
+ if (audio_device_module_->InitRecording() != 0) {
+ RTC_LOG(LS_ERROR) << "InitRecording failed";
+ return false;
+ }
+ if (audio_device_module_->StartRecording() != 0) {
+ RTC_LOG(LS_ERROR) << "StartRecording failed";
+ return false;
+ }
+ }
+ } else {
+ if (audio_device_module_->Recording() &&
+ audio_device_module_->StopRecording() != 0) {
+ RTC_LOG(LS_ERROR) << "StopRecording failed";
+ return false;
+ }
+ }
+ return true;
+}
+
+VoipResult VoipCore::StartSend(ChannelId channel_id) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ if (!channel->StartSend()) {
+ return VoipResult::kFailedPrecondition;
+ }
+
+ return UpdateAudioTransportWithSenders() ? VoipResult::kOk
+ : VoipResult::kInternal;
+}
+
+VoipResult VoipCore::StopSend(ChannelId channel_id) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->StopSend();
+
+ return UpdateAudioTransportWithSenders() ? VoipResult::kOk
+ : VoipResult::kInternal;
+}
+
+VoipResult VoipCore::StartPlayout(ChannelId channel_id) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ if (channel->IsPlaying()) {
+ return VoipResult::kOk;
+ }
+
+ if (!channel->StartPlay()) {
+ return VoipResult::kFailedPrecondition;
+ }
+
+ // Initialize audio device module and default device if needed.
+ if (!InitializeIfNeeded()) {
+ return VoipResult::kInternal;
+ }
+
+ if (!audio_device_module_->Playing()) {
+ if (audio_device_module_->InitPlayout() != 0) {
+ RTC_LOG(LS_ERROR) << "InitPlayout failed";
+ return VoipResult::kInternal;
+ }
+ if (audio_device_module_->StartPlayout() != 0) {
+ RTC_LOG(LS_ERROR) << "StartPlayout failed";
+ return VoipResult::kInternal;
+ }
+ }
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::StopPlayout(ChannelId channel_id) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->StopPlay();
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::ReceivedRTPPacket(
+ ChannelId channel_id,
+ rtc::ArrayView<const uint8_t> rtp_packet) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->ReceivedRTPPacket(rtp_packet);
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::ReceivedRTCPPacket(
+ ChannelId channel_id,
+ rtc::ArrayView<const uint8_t> rtcp_packet) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->ReceivedRTCPPacket(rtcp_packet);
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::SetSendCodec(ChannelId channel_id,
+ int payload_type,
+ const SdpAudioFormat& encoder_format) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ auto encoder = encoder_factory_->MakeAudioEncoder(
+ payload_type, encoder_format, absl::nullopt);
+ channel->SetEncoder(payload_type, encoder_format, std::move(encoder));
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::SetReceiveCodecs(
+ ChannelId channel_id,
+ const std::map<int, SdpAudioFormat>& decoder_specs) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->SetReceiveCodecs(decoder_specs);
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::RegisterTelephoneEventType(ChannelId channel_id,
+ int rtp_payload_type,
+ int sample_rate_hz) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->RegisterTelephoneEventType(rtp_payload_type, sample_rate_hz);
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::SendDtmfEvent(ChannelId channel_id,
+ DtmfEvent dtmf_event,
+ int duration_ms) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ return (channel->SendTelephoneEvent(static_cast<int>(dtmf_event), duration_ms)
+ ? VoipResult::kOk
+ : VoipResult::kFailedPrecondition);
+}
+
+VoipResult VoipCore::GetIngressStatistics(ChannelId channel_id,
+ IngressStatistics& ingress_stats) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ ingress_stats = channel->GetIngressStatistics();
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::GetChannelStatistics(ChannelId channel_id,
+ ChannelStatistics& channel_stats) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel_stats = channel->GetChannelStatistics();
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::SetInputMuted(ChannelId channel_id, bool enable) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ channel->SetMute(enable);
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::GetInputVolumeInfo(ChannelId channel_id,
+ VolumeInfo& input_volume) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ input_volume.audio_level = channel->GetInputAudioLevel();
+ input_volume.total_energy = channel->GetInputTotalEnergy();
+ input_volume.total_duration = channel->GetInputTotalDuration();
+
+ return VoipResult::kOk;
+}
+
+VoipResult VoipCore::GetOutputVolumeInfo(ChannelId channel_id,
+ VolumeInfo& output_volume) {
+ rtc::scoped_refptr<AudioChannel> channel = GetChannel(channel_id);
+
+ if (!channel) {
+ return VoipResult::kInvalidArgument;
+ }
+
+ output_volume.audio_level = channel->GetOutputAudioLevel();
+ output_volume.total_energy = channel->GetOutputTotalEnergy();
+ output_volume.total_duration = channel->GetOutputTotalDuration();
+
+ return VoipResult::kOk;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/voip/voip_core.h b/third_party/libwebrtc/audio/voip/voip_core.h
new file mode 100644
index 0000000000..6c3aec6fa2
--- /dev/null
+++ b/third_party/libwebrtc/audio/voip/voip_core.h
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AUDIO_VOIP_VOIP_CORE_H_
+#define AUDIO_VOIP_VOIP_CORE_H_
+
+#include <map>
+#include <memory>
+#include <queue>
+#include <unordered_map>
+#include <vector>
+
+#include "api/audio_codecs/audio_decoder_factory.h"
+#include "api/audio_codecs/audio_encoder_factory.h"
+#include "api/scoped_refptr.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "api/voip/voip_base.h"
+#include "api/voip/voip_codec.h"
+#include "api/voip/voip_dtmf.h"
+#include "api/voip/voip_engine.h"
+#include "api/voip/voip_network.h"
+#include "api/voip/voip_statistics.h"
+#include "api/voip/voip_volume_control.h"
+#include "audio/audio_transport_impl.h"
+#include "audio/voip/audio_channel.h"
+#include "modules/audio_device/include/audio_device.h"
+#include "modules/audio_mixer/audio_mixer_impl.h"
+#include "modules/audio_processing/include/audio_processing.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+
+// VoipCore is the implementatino of VoIP APIs listed in api/voip directory.
+// It manages a vector of AudioChannel objects where each is mapped with a
+// ChannelId (int) type. ChannelId is the primary key to locate a specific
+// AudioChannel object to operate requested VoIP API from the caller.
+//
+// This class receives required audio components from caller at construction and
+// owns the life cycle of them to orchestrate the proper destruction sequence.
+class VoipCore : public VoipEngine,
+ public VoipBase,
+ public VoipNetwork,
+ public VoipCodec,
+ public VoipDtmf,
+ public VoipStatistics,
+ public VoipVolumeControl {
+ public:
+ // Construct VoipCore with provided arguments.
+ VoipCore(rtc::scoped_refptr<AudioEncoderFactory> encoder_factory,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
+ std::unique_ptr<TaskQueueFactory> task_queue_factory,
+ rtc::scoped_refptr<AudioDeviceModule> audio_device_module,
+ rtc::scoped_refptr<AudioProcessing> audio_processing);
+ ~VoipCore() override = default;
+
+ // Implements VoipEngine interfaces.
+ VoipBase& Base() override { return *this; }
+ VoipNetwork& Network() override { return *this; }
+ VoipCodec& Codec() override { return *this; }
+ VoipDtmf& Dtmf() override { return *this; }
+ VoipStatistics& Statistics() override { return *this; }
+ VoipVolumeControl& VolumeControl() override { return *this; }
+
+ // Implements VoipBase interfaces.
+ ChannelId CreateChannel(Transport* transport,
+ absl::optional<uint32_t> local_ssrc) override;
+ VoipResult ReleaseChannel(ChannelId channel_id) override;
+ VoipResult StartSend(ChannelId channel_id) override;
+ VoipResult StopSend(ChannelId channel_id) override;
+ VoipResult StartPlayout(ChannelId channel_id) override;
+ VoipResult StopPlayout(ChannelId channel_id) override;
+
+ // Implements VoipNetwork interfaces.
+ VoipResult ReceivedRTPPacket(
+ ChannelId channel_id,
+ rtc::ArrayView<const uint8_t> rtp_packet) override;
+ VoipResult ReceivedRTCPPacket(
+ ChannelId channel_id,
+ rtc::ArrayView<const uint8_t> rtcp_packet) override;
+
+ // Implements VoipCodec interfaces.
+ VoipResult SetSendCodec(ChannelId channel_id,
+ int payload_type,
+ const SdpAudioFormat& encoder_format) override;
+ VoipResult SetReceiveCodecs(
+ ChannelId channel_id,
+ const std::map<int, SdpAudioFormat>& decoder_specs) override;
+
+ // Implements VoipDtmf interfaces.
+ VoipResult RegisterTelephoneEventType(ChannelId channel_id,
+ int rtp_payload_type,
+ int sample_rate_hz) override;
+ VoipResult SendDtmfEvent(ChannelId channel_id,
+ DtmfEvent dtmf_event,
+ int duration_ms) override;
+
+ // Implements VoipStatistics interfaces.
+ VoipResult GetIngressStatistics(ChannelId channel_id,
+ IngressStatistics& ingress_stats) override;
+ VoipResult GetChannelStatistics(ChannelId channe_id,
+ ChannelStatistics& channel_stats) override;
+
+ // Implements VoipVolumeControl interfaces.
+ VoipResult SetInputMuted(ChannelId channel_id, bool enable) override;
+ VoipResult GetInputVolumeInfo(ChannelId channel_id,
+ VolumeInfo& volume_info) override;
+ VoipResult GetOutputVolumeInfo(ChannelId channel_id,
+ VolumeInfo& volume_info) override;
+
+ private:
+ // Initialize ADM and default audio device if needed.
+ // Returns true if ADM is successfully initialized or already in such state
+ // (e.g called more than once). Returns false when ADM fails to initialize
+ // which would presumably render further processing useless. Note that such
+ // failure won't necessarily succeed in next initialization attempt as it
+ // would mean changing the ADM implementation. From Android N and onwards, the
+ // mobile app may not be able to gain microphone access when in background
+ // mode. Therefore it would be better to delay the logic as late as possible.
+ bool InitializeIfNeeded();
+
+ // Fetches the corresponding AudioChannel assigned with given `channel`.
+ // Returns nullptr if not found.
+ rtc::scoped_refptr<AudioChannel> GetChannel(ChannelId channel_id);
+
+ // Updates AudioTransportImpl with a new set of actively sending AudioSender
+ // (AudioEgress). This needs to be invoked whenever StartSend/StopSend is
+ // involved by caller. Returns false when the selected audio device fails to
+ // initialize where it can't expect to deliver any audio input sample.
+ bool UpdateAudioTransportWithSenders();
+
+ // Synchronization for these are handled internally.
+ rtc::scoped_refptr<AudioEncoderFactory> encoder_factory_;
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+ std::unique_ptr<TaskQueueFactory> task_queue_factory_;
+
+ // Synchronization is handled internally by AudioProcessing.
+ // Must be placed before `audio_device_module_` for proper destruction.
+ rtc::scoped_refptr<AudioProcessing> audio_processing_;
+
+ // Synchronization is handled internally by AudioMixer.
+ // Must be placed before `audio_device_module_` for proper destruction.
+ rtc::scoped_refptr<AudioMixer> audio_mixer_;
+
+ // Synchronization is handled internally by AudioTransportImpl.
+ // Must be placed before `audio_device_module_` for proper destruction.
+ std::unique_ptr<AudioTransportImpl> audio_transport_;
+
+ // Synchronization is handled internally by AudioDeviceModule.
+ rtc::scoped_refptr<AudioDeviceModule> audio_device_module_;
+
+ Mutex lock_;
+
+ // Member to track a next ChannelId for new AudioChannel.
+ int next_channel_id_ RTC_GUARDED_BY(lock_) = 0;
+
+ // Container to track currently active AudioChannel objects mapped by
+ // ChannelId.
+ std::unordered_map<ChannelId, rtc::scoped_refptr<AudioChannel>> channels_
+ RTC_GUARDED_BY(lock_);
+
+ // Boolean flag to ensure initialization only occurs once.
+ bool initialized_ RTC_GUARDED_BY(lock_) = false;
+};
+
+} // namespace webrtc
+
+#endif // AUDIO_VOIP_VOIP_CORE_H_