summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/audio
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:14:29 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:14:29 +0000
commitfbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8 (patch)
tree4c1ccaf5486d4f2009f9a338a98a83e886e29c97 /third_party/libwebrtc/audio
parentReleasing progress-linux version 124.0.1-1~progress7.99u1. (diff)
downloadfirefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.tar.xz
firefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.zip
Merging upstream version 125.0.1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/audio')
-rw-r--r--third_party/libwebrtc/audio/BUILD.gn2
-rw-r--r--third_party/libwebrtc/audio/audio_gn/moz.build5
-rw-r--r--third_party/libwebrtc/audio/audio_send_stream.cc4
-rw-r--r--third_party/libwebrtc/audio/audio_send_stream_unittest.cc11
-rw-r--r--third_party/libwebrtc/audio/channel_receive.cc26
-rw-r--r--third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc36
-rw-r--r--third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h9
-rw-r--r--third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc28
-rw-r--r--third_party/libwebrtc/audio/channel_receive_unittest.cc36
-rw-r--r--third_party/libwebrtc/audio/channel_send.cc31
-rw-r--r--third_party/libwebrtc/audio/channel_send.h1
-rw-r--r--third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc44
-rw-r--r--third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h10
-rw-r--r--third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc116
-rw-r--r--third_party/libwebrtc/audio/channel_send_unittest.cc7
-rw-r--r--third_party/libwebrtc/audio/mock_voe_channel_proxy.h4
-rw-r--r--third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build5
17 files changed, 311 insertions, 64 deletions
diff --git a/third_party/libwebrtc/audio/BUILD.gn b/third_party/libwebrtc/audio/BUILD.gn
index ec09e5a350..09562b9131 100644
--- a/third_party/libwebrtc/audio/BUILD.gn
+++ b/third_party/libwebrtc/audio/BUILD.gn
@@ -223,6 +223,7 @@ if (rtc_include_tests) {
"utility:utility_tests",
"//testing/gtest",
]
+ absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
}
rtc_library("channel_receive_unittest") {
@@ -241,6 +242,7 @@ if (rtc_include_tests) {
"../rtc_base:logging",
"../rtc_base:threading",
"../test:audio_codec_mocks",
+ "../test:mock_frame_transformer",
"../test:mock_transport",
"../test:test_support",
"../test/time_controller",
diff --git a/third_party/libwebrtc/audio/audio_gn/moz.build b/third_party/libwebrtc/audio/audio_gn/moz.build
index e81a4f673b..da615157b2 100644
--- a/third_party/libwebrtc/audio/audio_gn/moz.build
+++ b/third_party/libwebrtc/audio/audio_gn/moz.build
@@ -212,7 +212,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm":
OS_LIBS += [
- "android_support",
"unwind"
]
@@ -222,10 +221,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86":
"-msse2"
]
- OS_LIBS += [
- "android_support"
- ]
-
if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64":
DEFINES["_GNU_SOURCE"] = True
diff --git a/third_party/libwebrtc/audio/audio_send_stream.cc b/third_party/libwebrtc/audio/audio_send_stream.cc
index bffb910832..e7ebb2bf4e 100644
--- a/third_party/libwebrtc/audio/audio_send_stream.cc
+++ b/third_party/libwebrtc/audio/audio_send_stream.cc
@@ -637,12 +637,14 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) {
}
// Wrap the encoder in a RED encoder, if RED is enabled.
+ SdpAudioFormat format = spec.format;
if (spec.red_payload_type) {
AudioEncoderCopyRed::Config red_config;
red_config.payload_type = *spec.red_payload_type;
red_config.speech_encoder = std::move(encoder);
encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config),
field_trials_);
+ format.name = cricket::kRedCodecName;
}
// Set currently known overhead (used in ANA, opus only).
@@ -656,7 +658,7 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) {
}
StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
- channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
+ channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, format,
std::move(encoder));
return true;
diff --git a/third_party/libwebrtc/audio/audio_send_stream_unittest.cc b/third_party/libwebrtc/audio/audio_send_stream_unittest.cc
index d842afdfe5..c854f734b5 100644
--- a/third_party/libwebrtc/audio/audio_send_stream_unittest.cc
+++ b/third_party/libwebrtc/audio/audio_send_stream_unittest.cc
@@ -242,11 +242,11 @@ struct ConfigHelper {
void SetupMockForSetupSendCodec(bool expect_set_encoder_call) {
if (expect_set_encoder_call) {
EXPECT_CALL(*channel_send_, SetEncoder)
- .WillOnce(
- [this](int payload_type, std::unique_ptr<AudioEncoder> encoder) {
- this->audio_encoder_ = std::move(encoder);
- return true;
- });
+ .WillOnce([this](int payload_type, const SdpAudioFormat& format,
+ std::unique_ptr<AudioEncoder> encoder) {
+ this->audio_encoder_ = std::move(encoder);
+ return true;
+ });
}
}
@@ -595,6 +595,7 @@ TEST(AudioSendStreamTest, SendCodecCanApplyVad) {
std::unique_ptr<AudioEncoder> stolen_encoder;
EXPECT_CALL(*helper.channel_send(), SetEncoder)
.WillOnce([&stolen_encoder](int payload_type,
+ const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder> encoder) {
stolen_encoder = std::move(encoder);
return true;
diff --git a/third_party/libwebrtc/audio/channel_receive.cc b/third_party/libwebrtc/audio/channel_receive.cc
index c714b1dd4d..aff21fa72a 100644
--- a/third_party/libwebrtc/audio/channel_receive.cc
+++ b/third_party/libwebrtc/audio/channel_receive.cc
@@ -47,6 +47,7 @@
#include "rtc_base/numerics/safe_minmax.h"
#include "rtc_base/numerics/sequence_number_unwrapper.h"
#include "rtc_base/race_checker.h"
+#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/time_utils.h"
@@ -313,6 +314,8 @@ class ChannelReceive : public ChannelReceiveInterface,
mutable Mutex rtcp_counter_mutex_;
RtcpPacketTypeCounter rtcp_packet_type_counter_
RTC_GUARDED_BY(rtcp_counter_mutex_);
+
+ std::map<int, SdpAudioFormat> payload_type_map_;
};
void ChannelReceive::OnReceivedPayloadData(
@@ -639,6 +642,7 @@ void ChannelReceive::SetReceiveCodecs(
RTC_DCHECK_GE(kv.second.clockrate_hz, 1000);
payload_type_frequencies_[kv.first] = kv.second.clockrate_hz;
}
+ payload_type_map_ = codecs;
acm_receiver_.SetCodecs(codecs);
}
@@ -725,7 +729,14 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet,
if (frame_transformer_delegate_) {
// Asynchronously transform the received payload. After the payload is
// transformed, the delegate will call OnReceivedPayloadData to handle it.
- frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_);
+ char buf[1024];
+ rtc::SimpleStringBuilder mime_type(buf);
+ auto it = payload_type_map_.find(header.payloadType);
+ mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/"
+ << (it != payload_type_map_.end() ? it->second.name
+ : "x-unknown");
+ frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_,
+ mime_type.str());
} else {
OnReceivedPayloadData(payload_data, header);
}
@@ -917,12 +928,19 @@ void ChannelReceive::SetAssociatedSendChannel(
void ChannelReceive::SetDepacketizerToDecoderFrameTransformer(
rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
- // Depending on when the channel is created, the transformer might be set
- // twice. Don't replace the delegate if it was already initialized.
- if (!frame_transformer || frame_transformer_delegate_) {
+ if (!frame_transformer) {
RTC_DCHECK_NOTREACHED() << "Not setting the transformer?";
return;
}
+ if (frame_transformer_delegate_) {
+ // Depending on when the channel is created, the transformer might be set
+ // twice. Don't replace the delegate if it was already initialized.
+ // TODO(crbug.com/webrtc/15674): Prevent multiple calls during
+ // reconfiguration.
+ RTC_CHECK_EQ(frame_transformer_delegate_->FrameTransformer(),
+ frame_transformer);
+ return;
+ }
InitFrameTransformerDelegate(std::move(frame_transformer));
}
diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc
index 2d2893b8f7..8a7dda826e 100644
--- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc
+++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc
@@ -10,6 +10,7 @@
#include "audio/channel_receive_frame_transformer_delegate.h"
+#include <string>
#include <utility>
#include "rtc_base/buffer.h"
@@ -22,10 +23,12 @@ class TransformableIncomingAudioFrame
public:
TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload,
const RTPHeader& header,
- uint32_t ssrc)
+ uint32_t ssrc,
+ const std::string& codec_mime_type)
: payload_(payload.data(), payload.size()),
header_(header),
- ssrc_(ssrc) {}
+ ssrc_(ssrc),
+ codec_mime_type_(codec_mime_type) {}
~TransformableIncomingAudioFrame() override = default;
rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
@@ -45,6 +48,7 @@ class TransformableIncomingAudioFrame
}
Direction GetDirection() const override { return Direction::kReceiver; }
+ std::string GetMimeType() const override { return codec_mime_type_; }
const absl::optional<uint16_t> SequenceNumber() const override {
return header_.sequenceNumber;
}
@@ -65,6 +69,7 @@ class TransformableIncomingAudioFrame
rtc::Buffer payload_;
RTPHeader header_;
uint32_t ssrc_;
+ std::string codec_mime_type_;
};
} // namespace
@@ -92,10 +97,16 @@ void ChannelReceiveFrameTransformerDelegate::Reset() {
void ChannelReceiveFrameTransformerDelegate::Transform(
rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
- uint32_t ssrc) {
+ uint32_t ssrc,
+ const std::string& codec_mime_type) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
- frame_transformer_->Transform(
- std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc));
+ if (short_circuit_) {
+ receive_frame_callback_(packet, header);
+ } else {
+ frame_transformer_->Transform(
+ std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
+ codec_mime_type));
+ }
}
void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
@@ -107,6 +118,14 @@ void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
});
}
+void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() {
+ rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this);
+ channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable {
+ RTC_DCHECK_RUN_ON(&delegate->sequence_checker_);
+ delegate->short_circuit_ = true;
+ });
+}
+
void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
std::unique_ptr<TransformableFrameInterface> frame) const {
RTC_DCHECK_RUN_ON(&sequence_checker_);
@@ -138,4 +157,11 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
// originally from this receiver.
receive_frame_callback_(frame->GetData(), header);
}
+
+rtc::scoped_refptr<FrameTransformerInterface>
+ChannelReceiveFrameTransformerDelegate::FrameTransformer() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ return frame_transformer_;
+}
+
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h
index 04ad7c4695..37ff75c2e9 100644
--- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h
+++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h
@@ -12,6 +12,7 @@
#define AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_
#include <memory>
+#include <string>
#include "api/frame_transformer_interface.h"
#include "api/sequence_checker.h"
@@ -48,16 +49,21 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
// the frame asynchronously.
void Transform(rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
- uint32_t ssrc);
+ uint32_t ssrc,
+ const std::string& codec_mime_type);
// Implements TransformedFrameCallback. Can be called on any thread.
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to ChannelReceive::OnReceivedPayloadData on the
// `channel_receive_thread_`, by calling `receive_frame_callback_`.
void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
+ rtc::scoped_refptr<FrameTransformerInterface> FrameTransformer();
+
protected:
~ChannelReceiveFrameTransformerDelegate() override = default;
@@ -68,6 +74,7 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
RTC_GUARDED_BY(sequence_checker_);
TaskQueueBase* const channel_receive_thread_;
+ bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false;
};
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc
index 38ceb6d96d..8bdf217d5a 100644
--- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc
+++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc
@@ -93,7 +93,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
[&callback](std::unique_ptr<TransformableFrameInterface> frame) {
callback->OnTransformedFrame(std::move(frame));
});
- delegate->Transform(packet, header, 1111 /*ssrc*/);
+ delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
@@ -126,7 +126,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
static_cast<TransformableAudioFrameInterface*>(frame.get());
callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame));
});
- delegate->Transform(packet, header, 1111 /*ssrc*/);
+ delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
@@ -150,5 +150,29 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
+TEST(ChannelReceiveFrameTransformerDelegateTest,
+ ShortCircuitingSkipsTransform) {
+ rtc::AutoThread main_thread;
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+ MockChannelReceive mock_channel;
+ rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
+ mock_channel.callback(), mock_frame_transformer,
+ rtc::Thread::Current());
+ const uint8_t data[] = {1, 2, 3, 4};
+ rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
+ RTPHeader header;
+
+ delegate->StartShortCircuiting();
+ rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+ // Will pass the frame straight to the channel.
+ EXPECT_CALL(mock_channel, ReceiveFrame);
+ delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
+}
+
} // namespace
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_receive_unittest.cc b/third_party/libwebrtc/audio/channel_receive_unittest.cc
index 4b7b7c0231..aab8a95d8b 100644
--- a/third_party/libwebrtc/audio/channel_receive_unittest.cc
+++ b/third_party/libwebrtc/audio/channel_receive_unittest.cc
@@ -29,6 +29,7 @@
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/mock_audio_decoder_factory.h"
+#include "test/mock_frame_transformer.h"
#include "test/mock_transport.h"
#include "test/time_controller/simulated_time_controller.h"
@@ -226,6 +227,41 @@ TEST_F(ChannelReceiveTest, CaptureStartTimeBecomesValid) {
EXPECT_NE(ProbeCaptureStartNtpTime(*channel), -1);
}
+TEST_F(ChannelReceiveTest, SettingFrameTransformer) {
+ auto channel = CreateTestChannelReceive();
+
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<MockFrameTransformer>();
+
+ EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback);
+ channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer);
+
+ // Must start playout, otherwise packet is discarded.
+ channel->StartPlayout();
+
+ RtpPacketReceived packet = CreateRtpPacket();
+
+ // Receive one RTP packet, this should be transformed.
+ EXPECT_CALL(*mock_frame_transformer, Transform);
+ channel->OnRtpPacket(packet);
+}
+
+TEST_F(ChannelReceiveTest, SettingFrameTransformerMultipleTimes) {
+ auto channel = CreateTestChannelReceive();
+
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<MockFrameTransformer>();
+
+ EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback);
+ channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer);
+
+ // Set the same transformer again, shouldn't cause any additional callback
+ // registration calls.
+ EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback)
+ .Times(0);
+ channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer);
+}
+
} // namespace
} // namespace voe
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_send.cc b/third_party/libwebrtc/audio/channel_send.cc
index ee94760b6f..3c59be52b4 100644
--- a/third_party/libwebrtc/audio/channel_send.cc
+++ b/third_party/libwebrtc/audio/channel_send.cc
@@ -37,6 +37,7 @@
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/rate_limiter.h"
+#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/time_utils.h"
@@ -104,6 +105,7 @@ class ChannelSend : public ChannelSendInterface,
// Send using this encoder, with this payload type.
void SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) override;
void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
modifier) override;
@@ -192,7 +194,8 @@ class ChannelSend : public ChannelSendInterface,
uint8_t payloadType,
uint32_t rtp_timestamp_without_offset,
rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms)
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs)
RTC_RUN_ON(encoder_queue_);
void OnReceivedRtt(int64_t rtt_ms);
@@ -264,6 +267,8 @@ class ChannelSend : public ChannelSendInterface,
// Defined last to ensure that there are no running tasks when the other
// members are destroyed.
rtc::TaskQueue encoder_queue_;
+
+ SdpAudioFormat encoder_format_;
};
const int kTelephoneEventAttenuationdB = 10;
@@ -310,21 +315,26 @@ int32_t ChannelSend::SendData(AudioFrameType frameType,
if (frame_transformer_delegate_) {
// Asynchronously transform the payload before sending it. After the payload
// is transformed, the delegate will call SendRtpAudio to send it.
+ char buf[1024];
+ rtc::SimpleStringBuilder mime_type(buf);
+ mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/"
+ << encoder_format_.name;
frame_transformer_delegate_->Transform(
frameType, payloadType, rtp_timestamp + rtp_rtcp_->StartTimestamp(),
payloadData, payloadSize, absolute_capture_timestamp_ms,
- rtp_rtcp_->SSRC());
+ rtp_rtcp_->SSRC(), mime_type.str());
return 0;
}
return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
- absolute_capture_timestamp_ms);
+ absolute_capture_timestamp_ms, /*csrcs=*/{});
}
int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
uint8_t payloadType,
uint32_t rtp_timestamp_without_offset,
rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms) {
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs) {
// E2EE Custom Audio Frame Encryption (This is optional).
// Keep this buffer around for the lifetime of the send call.
rtc::Buffer encrypted_audio_payload;
@@ -386,7 +396,8 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
.payload = payload,
.payload_id = payloadType,
.rtp_timestamp =
- rtp_timestamp_without_offset + rtp_rtcp_->StartTimestamp()};
+ rtp_timestamp_without_offset + rtp_rtcp_->StartTimestamp(),
+ .csrcs = csrcs};
if (absolute_capture_timestamp_ms > 0) {
frame.capture_time = Timestamp::Millis(absolute_capture_timestamp_ms);
}
@@ -426,7 +437,8 @@ ChannelSend::ChannelSend(
crypto_options_(crypto_options),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"AudioEncoder",
- TaskQueueFactory::Priority::NORMAL)) {
+ TaskQueueFactory::Priority::NORMAL)),
+ encoder_format_("x-unknown", 0, 0) {
audio_coding_ = AudioCodingModule::Create();
RtpRtcpInterface::Configuration configuration;
@@ -527,6 +539,7 @@ void ChannelSend::StopSend() {
}
void ChannelSend::SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_DCHECK_GE(payload_type, 0);
@@ -540,6 +553,7 @@ void ChannelSend::SetEncoder(int payload_type,
encoder->RtpTimestampRateHz(),
encoder->NumChannels(), 0);
+ encoder_format_ = encoder_format;
audio_coding_->SetEncoder(std::move(encoder));
}
@@ -881,12 +895,13 @@ void ChannelSend::InitFrameTransformerDelegate(
[this](AudioFrameType frameType, uint8_t payloadType,
uint32_t rtp_timestamp_with_offset,
rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms) {
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
return SendRtpAudio(
frameType, payloadType,
rtp_timestamp_with_offset - rtp_rtcp_->StartTimestamp(), payload,
- absolute_capture_timestamp_ms);
+ absolute_capture_timestamp_ms, csrcs);
};
frame_transformer_delegate_ =
rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>(
diff --git a/third_party/libwebrtc/audio/channel_send.h b/third_party/libwebrtc/audio/channel_send.h
index f0c9232296..f36085c1fa 100644
--- a/third_party/libwebrtc/audio/channel_send.h
+++ b/third_party/libwebrtc/audio/channel_send.h
@@ -63,6 +63,7 @@ class ChannelSendInterface {
virtual CallSendStatistics GetRTCPStatistics() const = 0;
virtual void SetEncoder(int payload_type,
+ const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) = 0;
virtual void ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) = 0;
diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc
index 0f85216e92..2eea0d2387 100644
--- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc
+++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc
@@ -11,6 +11,7 @@
#include "audio/channel_send_frame_transformer_delegate.h"
#include <utility>
+#include <vector>
namespace webrtc {
namespace {
@@ -55,13 +56,17 @@ class TransformableOutgoingAudioFrame
const uint8_t* payload_data,
size_t payload_size,
absl::optional<uint64_t> absolute_capture_timestamp_ms,
- uint32_t ssrc)
+ uint32_t ssrc,
+ std::vector<uint32_t> csrcs,
+ const std::string& codec_mime_type)
: frame_type_(frame_type),
payload_type_(payload_type),
rtp_timestamp_with_offset_(rtp_timestamp_with_offset),
payload_(payload_data, payload_size),
absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms),
- ssrc_(ssrc) {}
+ ssrc_(ssrc),
+ csrcs_(std::move(csrcs)),
+ codec_mime_type_(codec_mime_type) {}
~TransformableOutgoingAudioFrame() override = default;
rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
void SetData(rtc::ArrayView<const uint8_t> data) override {
@@ -76,9 +81,10 @@ class TransformableOutgoingAudioFrame
uint8_t GetPayloadType() const override { return payload_type_; }
Direction GetDirection() const override { return Direction::kSender; }
+ std::string GetMimeType() const override { return codec_mime_type_; }
rtc::ArrayView<const uint32_t> GetContributingSources() const override {
- return {};
+ return csrcs_;
}
const absl::optional<uint16_t> SequenceNumber() const override {
@@ -100,6 +106,8 @@ class TransformableOutgoingAudioFrame
rtc::Buffer payload_;
absl::optional<uint64_t> absolute_capture_timestamp_ms_;
uint32_t ssrc_;
+ std::vector<uint32_t> csrcs_;
+ std::string codec_mime_type_;
};
} // namespace
@@ -131,11 +139,23 @@ void ChannelSendFrameTransformerDelegate::Transform(
const uint8_t* payload_data,
size_t payload_size,
int64_t absolute_capture_timestamp_ms,
- uint32_t ssrc) {
+ uint32_t ssrc,
+ const std::string& codec_mimetype) {
+ {
+ MutexLock lock(&send_lock_);
+ if (short_circuit_) {
+ send_frame_callback_(
+ frame_type, payload_type, rtp_timestamp,
+ rtc::ArrayView<const uint8_t>(payload_data, payload_size),
+ absolute_capture_timestamp_ms, /*csrcs=*/{});
+ return;
+ }
+ }
frame_transformer_->Transform(
std::make_unique<TransformableOutgoingAudioFrame>(
frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
- absolute_capture_timestamp_ms, ssrc));
+ absolute_capture_timestamp_ms, ssrc,
+ /*csrcs=*/std::vector<uint32_t>(), codec_mimetype));
}
void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
@@ -150,6 +170,11 @@ void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
});
}
+void ChannelSendFrameTransformerDelegate::StartShortCircuiting() {
+ MutexLock lock(&send_lock_);
+ short_circuit_ = true;
+}
+
void ChannelSendFrameTransformerDelegate::SendFrame(
std::unique_ptr<TransformableFrameInterface> frame) const {
MutexLock lock(&send_lock_);
@@ -164,16 +189,21 @@ void ChannelSendFrameTransformerDelegate::SendFrame(
transformed_frame->GetData(),
transformed_frame->AbsoluteCaptureTimestamp()
? *transformed_frame->AbsoluteCaptureTimestamp()
- : 0);
+ : 0,
+ transformed_frame->GetContributingSources());
}
std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame(
TransformableAudioFrameInterface* original) {
+ std::vector<uint32_t> csrcs;
+ csrcs.assign(original->GetContributingSources().begin(),
+ original->GetContributingSources().end());
return std::make_unique<TransformableOutgoingAudioFrame>(
InterfaceFrameTypeToInternalFrameType(original->Type()),
original->GetPayloadType(), original->GetTimestamp(),
original->GetData().data(), original->GetData().size(),
- original->AbsoluteCaptureTimestamp(), original->GetSsrc());
+ original->AbsoluteCaptureTimestamp(), original->GetSsrc(),
+ std::move(csrcs), original->GetMimeType());
}
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h
index eb0027e4c8..97fc14f737 100644
--- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h
+++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h
@@ -12,6 +12,7 @@
#define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
#include <memory>
+#include <string>
#include "api/frame_transformer_interface.h"
#include "api/sequence_checker.h"
@@ -34,7 +35,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
uint8_t payloadType,
uint32_t rtp_timestamp_with_offset,
rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms)>;
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs)>;
ChannelSendFrameTransformerDelegate(
SendFrameCallback send_frame_callback,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
@@ -57,12 +59,15 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
const uint8_t* payload_data,
size_t payload_size,
int64_t absolute_capture_timestamp_ms,
- uint32_t ssrc);
+ uint32_t ssrc,
+ const std::string& codec_mime_type);
// Implements TransformedFrameCallback. Can be called on any thread.
void OnTransformedFrame(
std::unique_ptr<TransformableFrameInterface> frame) override;
+ void StartShortCircuiting() override;
+
// Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`,
// by calling `send_audio_callback_`.
void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const;
@@ -75,6 +80,7 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_);
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_);
+ bool short_circuit_ RTC_GUARDED_BY(send_lock_) = false;
};
std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame(
diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc
index f75d4a8ab7..4dcd15cd95 100644
--- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc
+++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc
@@ -12,7 +12,9 @@
#include <memory>
#include <utility>
+#include <vector>
+#include "absl/memory/memory.h"
#include "rtc_base/task_queue_for_test.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -24,10 +26,13 @@ namespace {
using ::testing::_;
using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
using ::testing::NiceMock;
using ::testing::Return;
using ::testing::SaveArg;
+const uint8_t mock_data[] = {1, 2, 3, 4};
+
class MockChannelSend {
public:
MockChannelSend() = default;
@@ -39,30 +44,56 @@ class MockChannelSend {
uint8_t payloadType,
uint32_t rtp_timestamp,
rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms));
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs));
ChannelSendFrameTransformerDelegate::SendFrameCallback callback() {
return [this](AudioFrameType frameType, uint8_t payloadType,
uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload,
- int64_t absolute_capture_timestamp_ms) {
+ int64_t absolute_capture_timestamp_ms,
+ rtc::ArrayView<const uint32_t> csrcs) {
return SendFrame(frameType, payloadType, rtp_timestamp, payload,
- absolute_capture_timestamp_ms);
+ absolute_capture_timestamp_ms, csrcs);
};
}
};
-std::unique_ptr<MockTransformableAudioFrame> CreateMockReceiverFrame() {
- const uint8_t mock_data[] = {1, 2, 3, 4};
+std::unique_ptr<TransformableAudioFrameInterface> CreateMockReceiverFrame(
+ std::vector<const uint32_t> csrcs) {
std::unique_ptr<MockTransformableAudioFrame> mock_frame =
- std::make_unique<MockTransformableAudioFrame>();
+ std::make_unique<NiceMock<MockTransformableAudioFrame>>();
rtc::ArrayView<const uint8_t> payload(mock_data);
ON_CALL(*mock_frame, GetData).WillByDefault(Return(payload));
ON_CALL(*mock_frame, GetPayloadType).WillByDefault(Return(0));
ON_CALL(*mock_frame, GetDirection)
.WillByDefault(Return(TransformableFrameInterface::Direction::kReceiver));
+ ON_CALL(*mock_frame, GetContributingSources).WillByDefault(Return(csrcs));
return mock_frame;
}
+std::unique_ptr<TransformableAudioFrameInterface> CreateFrame() {
+ TaskQueueForTest channel_queue("channel_queue");
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+ MockChannelSend mock_channel;
+ rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>(
+ mock_channel.callback(), mock_frame_transformer, &channel_queue);
+
+ std::unique_ptr<TransformableFrameInterface> frame;
+ ON_CALL(*mock_frame_transformer, Transform)
+ .WillByDefault(
+ [&frame](
+ std::unique_ptr<TransformableFrameInterface> transform_frame) {
+ frame = std::move(transform_frame);
+ });
+ delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, mock_data,
+ sizeof(mock_data), 0,
+ /*ssrc=*/0, /*mimeType=*/"audio/opus");
+ return absl::WrapUnique(
+ static_cast<webrtc::TransformableAudioFrameInterface*>(frame.release()));
+}
+
// Test that the delegate registers itself with the frame transformer on Init().
TEST(ChannelSendFrameTransformerDelegateTest,
RegisterTransformedFrameCallbackOnInit) {
@@ -115,7 +146,7 @@ TEST(ChannelSendFrameTransformerDelegateTest,
callback->OnTransformedFrame(std::move(frame));
});
delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
- 0);
+ /*ssrc=*/0, /*mimeType=*/"audio/opus");
channel_queue.WaitForPreviouslyPostedTasks();
}
@@ -136,16 +167,17 @@ TEST(ChannelSendFrameTransformerDelegateTest,
delegate->Init();
ASSERT_TRUE(callback);
- const uint8_t data[] = {1, 2, 3, 4};
+ std::vector<const uint32_t> csrcs = {123, 234, 345, 456};
EXPECT_CALL(mock_channel, SendFrame).Times(0);
- EXPECT_CALL(mock_channel, SendFrame(_, 0, 0, ElementsAre(1, 2, 3, 4), _));
+ EXPECT_CALL(mock_channel, SendFrame(_, 0, 0, ElementsAreArray(mock_data), _,
+ ElementsAreArray(csrcs)));
ON_CALL(*mock_frame_transformer, Transform)
- .WillByDefault(
- [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
- callback->OnTransformedFrame(CreateMockReceiverFrame());
- });
- delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
- 0);
+ .WillByDefault([&](std::unique_ptr<TransformableFrameInterface> frame) {
+ callback->OnTransformedFrame(CreateMockReceiverFrame(csrcs));
+ });
+ delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, mock_data,
+ sizeof(mock_data), 0,
+ /*ssrc=*/0, /*mimeType=*/"audio/opus");
channel_queue.WaitForPreviouslyPostedTasks();
}
@@ -168,5 +200,59 @@ TEST(ChannelSendFrameTransformerDelegateTest,
channel_queue.WaitForPreviouslyPostedTasks();
}
+TEST(ChannelSendFrameTransformerDelegateTest, ShortCircuitingSkipsTransform) {
+ TaskQueueForTest channel_queue("channel_queue");
+ rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
+ rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>();
+ MockChannelSend mock_channel;
+ rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate =
+ rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>(
+ mock_channel.callback(), mock_frame_transformer, &channel_queue);
+
+ delegate->StartShortCircuiting();
+
+ // Will not call the actual transformer.
+ EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
+ // Will pass the frame straight to the channel.
+ EXPECT_CALL(mock_channel, SendFrame);
+ const uint8_t data[] = {1, 2, 3, 4};
+ delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
+ /*ssrc=*/0, /*mimeType=*/"audio/opus");
+}
+
+TEST(ChannelSendFrameTransformerDelegateTest,
+ CloningSenderFramePreservesInformation) {
+ std::unique_ptr<TransformableAudioFrameInterface> frame = CreateFrame();
+ std::unique_ptr<TransformableAudioFrameInterface> cloned_frame =
+ CloneSenderAudioFrame(frame.get());
+
+ EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp());
+ EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc());
+ EXPECT_EQ(cloned_frame->Type(), frame->Type());
+ EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType());
+ EXPECT_EQ(cloned_frame->GetMimeType(), frame->GetMimeType());
+ EXPECT_THAT(cloned_frame->GetContributingSources(),
+ ElementsAreArray(frame->GetContributingSources()));
+}
+
+TEST(ChannelSendFrameTransformerDelegateTest, CloningReceiverFrameWithCsrcs) {
+ std::unique_ptr<TransformableAudioFrameInterface> frame =
+ CreateMockReceiverFrame(/*csrcs=*/{123, 234, 345});
+ std::unique_ptr<TransformableAudioFrameInterface> cloned_frame =
+ CloneSenderAudioFrame(frame.get());
+
+ EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp());
+ EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc());
+ EXPECT_EQ(cloned_frame->Type(), frame->Type());
+ EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType());
+ EXPECT_EQ(cloned_frame->GetMimeType(), frame->GetMimeType());
+ EXPECT_EQ(cloned_frame->AbsoluteCaptureTimestamp(),
+ frame->AbsoluteCaptureTimestamp());
+
+ ASSERT_NE(frame->GetContributingSources().size(), 0u);
+ EXPECT_THAT(cloned_frame->GetContributingSources(),
+ ElementsAreArray(frame->GetContributingSources()));
+}
+
} // namespace
} // namespace webrtc
diff --git a/third_party/libwebrtc/audio/channel_send_unittest.cc b/third_party/libwebrtc/audio/channel_send_unittest.cc
index b9406e1523..58d7c93c1e 100644
--- a/third_party/libwebrtc/audio/channel_send_unittest.cc
+++ b/third_party/libwebrtc/audio/channel_send_unittest.cc
@@ -66,9 +66,10 @@ class ChannelSendTest : public ::testing::Test {
&transport_, nullptr, &event_log_, nullptr, crypto_options_, false,
kRtcpIntervalMs, kSsrc, nullptr, &transport_controller_, field_trials_);
encoder_factory_ = CreateBuiltinAudioEncoderFactory();
- std::unique_ptr<AudioEncoder> encoder = encoder_factory_->MakeAudioEncoder(
- kPayloadType, SdpAudioFormat("opus", kRtpRateHz, 2), {});
- channel_->SetEncoder(kPayloadType, std::move(encoder));
+ SdpAudioFormat opus = SdpAudioFormat("opus", kRtpRateHz, 2);
+ std::unique_ptr<AudioEncoder> encoder =
+ encoder_factory_->MakeAudioEncoder(kPayloadType, opus, {});
+ channel_->SetEncoder(kPayloadType, opus, std::move(encoder));
transport_controller_.EnsureStarted();
channel_->RegisterSenderCongestionControlObjects(&transport_controller_);
ON_CALL(transport_, SendRtcp).WillByDefault(Return(true));
diff --git a/third_party/libwebrtc/audio/mock_voe_channel_proxy.h b/third_party/libwebrtc/audio/mock_voe_channel_proxy.h
index 29005173df..71ef5d12fb 100644
--- a/third_party/libwebrtc/audio/mock_voe_channel_proxy.h
+++ b/third_party/libwebrtc/audio/mock_voe_channel_proxy.h
@@ -113,7 +113,9 @@ class MockChannelSend : public voe::ChannelSendInterface {
public:
MOCK_METHOD(void,
SetEncoder,
- (int payload_type, std::unique_ptr<AudioEncoder> encoder),
+ (int payload_type,
+ const SdpAudioFormat& encoder_format,
+ std::unique_ptr<AudioEncoder> encoder),
(override));
MOCK_METHOD(
void,
diff --git a/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build b/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build
index e215792f64..6489497db4 100644
--- a/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build
+++ b/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build
@@ -201,7 +201,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm":
OS_LIBS += [
- "android_support",
"unwind"
]
@@ -211,10 +210,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86":
"-msse2"
]
- OS_LIBS += [
- "android_support"
- ]
-
if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64":
DEFINES["_GNU_SOURCE"] = True