diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:14:29 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:14:29 +0000 |
commit | fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8 (patch) | |
tree | 4c1ccaf5486d4f2009f9a338a98a83e886e29c97 /third_party/libwebrtc/audio | |
parent | Releasing progress-linux version 124.0.1-1~progress7.99u1. (diff) | |
download | firefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.tar.xz firefox-fbaf0bb26397aa498eb9156f06d5a6fe34dd7dd8.zip |
Merging upstream version 125.0.1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/audio')
17 files changed, 311 insertions, 64 deletions
diff --git a/third_party/libwebrtc/audio/BUILD.gn b/third_party/libwebrtc/audio/BUILD.gn index ec09e5a350..09562b9131 100644 --- a/third_party/libwebrtc/audio/BUILD.gn +++ b/third_party/libwebrtc/audio/BUILD.gn @@ -223,6 +223,7 @@ if (rtc_include_tests) { "utility:utility_tests", "//testing/gtest", ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] } rtc_library("channel_receive_unittest") { @@ -241,6 +242,7 @@ if (rtc_include_tests) { "../rtc_base:logging", "../rtc_base:threading", "../test:audio_codec_mocks", + "../test:mock_frame_transformer", "../test:mock_transport", "../test:test_support", "../test/time_controller", diff --git a/third_party/libwebrtc/audio/audio_gn/moz.build b/third_party/libwebrtc/audio/audio_gn/moz.build index e81a4f673b..da615157b2 100644 --- a/third_party/libwebrtc/audio/audio_gn/moz.build +++ b/third_party/libwebrtc/audio/audio_gn/moz.build @@ -212,7 +212,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -222,10 +221,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/audio/audio_send_stream.cc b/third_party/libwebrtc/audio/audio_send_stream.cc index bffb910832..e7ebb2bf4e 100644 --- a/third_party/libwebrtc/audio/audio_send_stream.cc +++ b/third_party/libwebrtc/audio/audio_send_stream.cc @@ -637,12 +637,14 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) { } // Wrap the encoder in a RED encoder, if RED is enabled. + SdpAudioFormat format = spec.format; if (spec.red_payload_type) { AudioEncoderCopyRed::Config red_config; red_config.payload_type = *spec.red_payload_type; red_config.speech_encoder = std::move(encoder); encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config), field_trials_); + format.name = cricket::kRedCodecName; } // Set currently known overhead (used in ANA, opus only). @@ -656,7 +658,7 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) { } StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); - channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, + channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, format, std::move(encoder)); return true; diff --git a/third_party/libwebrtc/audio/audio_send_stream_unittest.cc b/third_party/libwebrtc/audio/audio_send_stream_unittest.cc index d842afdfe5..c854f734b5 100644 --- a/third_party/libwebrtc/audio/audio_send_stream_unittest.cc +++ b/third_party/libwebrtc/audio/audio_send_stream_unittest.cc @@ -242,11 +242,11 @@ struct ConfigHelper { void SetupMockForSetupSendCodec(bool expect_set_encoder_call) { if (expect_set_encoder_call) { EXPECT_CALL(*channel_send_, SetEncoder) - .WillOnce( - [this](int payload_type, std::unique_ptr<AudioEncoder> encoder) { - this->audio_encoder_ = std::move(encoder); - return true; - }); + .WillOnce([this](int payload_type, const SdpAudioFormat& format, + std::unique_ptr<AudioEncoder> encoder) { + this->audio_encoder_ = std::move(encoder); + return true; + }); } } @@ -595,6 +595,7 @@ TEST(AudioSendStreamTest, SendCodecCanApplyVad) { std::unique_ptr<AudioEncoder> stolen_encoder; EXPECT_CALL(*helper.channel_send(), SetEncoder) .WillOnce([&stolen_encoder](int payload_type, + const SdpAudioFormat& format, std::unique_ptr<AudioEncoder> encoder) { stolen_encoder = std::move(encoder); return true; diff --git a/third_party/libwebrtc/audio/channel_receive.cc b/third_party/libwebrtc/audio/channel_receive.cc index c714b1dd4d..aff21fa72a 100644 --- a/third_party/libwebrtc/audio/channel_receive.cc +++ b/third_party/libwebrtc/audio/channel_receive.cc @@ -47,6 +47,7 @@ #include "rtc_base/numerics/safe_minmax.h" #include "rtc_base/numerics/sequence_number_unwrapper.h" #include "rtc_base/race_checker.h" +#include "rtc_base/strings/string_builder.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/time_utils.h" @@ -313,6 +314,8 @@ class ChannelReceive : public ChannelReceiveInterface, mutable Mutex rtcp_counter_mutex_; RtcpPacketTypeCounter rtcp_packet_type_counter_ RTC_GUARDED_BY(rtcp_counter_mutex_); + + std::map<int, SdpAudioFormat> payload_type_map_; }; void ChannelReceive::OnReceivedPayloadData( @@ -639,6 +642,7 @@ void ChannelReceive::SetReceiveCodecs( RTC_DCHECK_GE(kv.second.clockrate_hz, 1000); payload_type_frequencies_[kv.first] = kv.second.clockrate_hz; } + payload_type_map_ = codecs; acm_receiver_.SetCodecs(codecs); } @@ -725,7 +729,14 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet, if (frame_transformer_delegate_) { // Asynchronously transform the received payload. After the payload is // transformed, the delegate will call OnReceivedPayloadData to handle it. - frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_); + char buf[1024]; + rtc::SimpleStringBuilder mime_type(buf); + auto it = payload_type_map_.find(header.payloadType); + mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/" + << (it != payload_type_map_.end() ? it->second.name + : "x-unknown"); + frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_, + mime_type.str()); } else { OnReceivedPayloadData(payload_data, header); } @@ -917,12 +928,19 @@ void ChannelReceive::SetAssociatedSendChannel( void ChannelReceive::SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); - // Depending on when the channel is created, the transformer might be set - // twice. Don't replace the delegate if it was already initialized. - if (!frame_transformer || frame_transformer_delegate_) { + if (!frame_transformer) { RTC_DCHECK_NOTREACHED() << "Not setting the transformer?"; return; } + if (frame_transformer_delegate_) { + // Depending on when the channel is created, the transformer might be set + // twice. Don't replace the delegate if it was already initialized. + // TODO(crbug.com/webrtc/15674): Prevent multiple calls during + // reconfiguration. + RTC_CHECK_EQ(frame_transformer_delegate_->FrameTransformer(), + frame_transformer); + return; + } InitFrameTransformerDelegate(std::move(frame_transformer)); } diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc index 2d2893b8f7..8a7dda826e 100644 --- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc +++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.cc @@ -10,6 +10,7 @@ #include "audio/channel_receive_frame_transformer_delegate.h" +#include <string> #include <utility> #include "rtc_base/buffer.h" @@ -22,10 +23,12 @@ class TransformableIncomingAudioFrame public: TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload, const RTPHeader& header, - uint32_t ssrc) + uint32_t ssrc, + const std::string& codec_mime_type) : payload_(payload.data(), payload.size()), header_(header), - ssrc_(ssrc) {} + ssrc_(ssrc), + codec_mime_type_(codec_mime_type) {} ~TransformableIncomingAudioFrame() override = default; rtc::ArrayView<const uint8_t> GetData() const override { return payload_; } @@ -45,6 +48,7 @@ class TransformableIncomingAudioFrame } Direction GetDirection() const override { return Direction::kReceiver; } + std::string GetMimeType() const override { return codec_mime_type_; } const absl::optional<uint16_t> SequenceNumber() const override { return header_.sequenceNumber; } @@ -65,6 +69,7 @@ class TransformableIncomingAudioFrame rtc::Buffer payload_; RTPHeader header_; uint32_t ssrc_; + std::string codec_mime_type_; }; } // namespace @@ -92,10 +97,16 @@ void ChannelReceiveFrameTransformerDelegate::Reset() { void ChannelReceiveFrameTransformerDelegate::Transform( rtc::ArrayView<const uint8_t> packet, const RTPHeader& header, - uint32_t ssrc) { + uint32_t ssrc, + const std::string& codec_mime_type) { RTC_DCHECK_RUN_ON(&sequence_checker_); - frame_transformer_->Transform( - std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc)); + if (short_circuit_) { + receive_frame_callback_(packet, header); + } else { + frame_transformer_->Transform( + std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc, + codec_mime_type)); + } } void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( @@ -107,6 +118,14 @@ void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame( }); } +void ChannelReceiveFrameTransformerDelegate::StartShortCircuiting() { + rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate(this); + channel_receive_thread_->PostTask([delegate = std::move(delegate)]() mutable { + RTC_DCHECK_RUN_ON(&delegate->sequence_checker_); + delegate->short_circuit_ = true; + }); +} + void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( std::unique_ptr<TransformableFrameInterface> frame) const { RTC_DCHECK_RUN_ON(&sequence_checker_); @@ -138,4 +157,11 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame( // originally from this receiver. receive_frame_callback_(frame->GetData(), header); } + +rtc::scoped_refptr<FrameTransformerInterface> +ChannelReceiveFrameTransformerDelegate::FrameTransformer() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return frame_transformer_; +} + } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h index 04ad7c4695..37ff75c2e9 100644 --- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h +++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate.h @@ -12,6 +12,7 @@ #define AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_ #include <memory> +#include <string> #include "api/frame_transformer_interface.h" #include "api/sequence_checker.h" @@ -48,16 +49,21 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { // the frame asynchronously. void Transform(rtc::ArrayView<const uint8_t> packet, const RTPHeader& header, - uint32_t ssrc); + uint32_t ssrc, + const std::string& codec_mime_type); // Implements TransformedFrameCallback. Can be called on any thread. void OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) override; + void StartShortCircuiting() override; + // Delegates the call to ChannelReceive::OnReceivedPayloadData on the // `channel_receive_thread_`, by calling `receive_frame_callback_`. void ReceiveFrame(std::unique_ptr<TransformableFrameInterface> frame) const; + rtc::scoped_refptr<FrameTransformerInterface> FrameTransformer(); + protected: ~ChannelReceiveFrameTransformerDelegate() override = default; @@ -68,6 +74,7 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback { rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_ RTC_GUARDED_BY(sequence_checker_); TaskQueueBase* const channel_receive_thread_; + bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false; }; } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc index 38ceb6d96d..8bdf217d5a 100644 --- a/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc +++ b/third_party/libwebrtc/audio/channel_receive_frame_transformer_delegate_unittest.cc @@ -93,7 +93,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, [&callback](std::unique_ptr<TransformableFrameInterface> frame) { callback->OnTransformedFrame(std::move(frame)); }); - delegate->Transform(packet, header, 1111 /*ssrc*/); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } @@ -126,7 +126,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, static_cast<TransformableAudioFrameInterface*>(frame.get()); callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame)); }); - delegate->Transform(packet, header, 1111 /*ssrc*/); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } @@ -150,5 +150,29 @@ TEST(ChannelReceiveFrameTransformerDelegateTest, rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } +TEST(ChannelReceiveFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + rtc::AutoThread main_thread; + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>(); + MockChannelReceive mock_channel; + rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate> delegate = + rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>( + mock_channel.callback(), mock_frame_transformer, + rtc::Thread::Current()); + const uint8_t data[] = {1, 2, 3, 4}; + rtc::ArrayView<const uint8_t> packet(data, sizeof(data)); + RTPHeader header; + + delegate->StartShortCircuiting(); + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the channel. + EXPECT_CALL(mock_channel, ReceiveFrame); + delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus"); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_receive_unittest.cc b/third_party/libwebrtc/audio/channel_receive_unittest.cc index 4b7b7c0231..aab8a95d8b 100644 --- a/third_party/libwebrtc/audio/channel_receive_unittest.cc +++ b/third_party/libwebrtc/audio/channel_receive_unittest.cc @@ -29,6 +29,7 @@ #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_audio_decoder_factory.h" +#include "test/mock_frame_transformer.h" #include "test/mock_transport.h" #include "test/time_controller/simulated_time_controller.h" @@ -226,6 +227,41 @@ TEST_F(ChannelReceiveTest, CaptureStartTimeBecomesValid) { EXPECT_NE(ProbeCaptureStartNtpTime(*channel), -1); } +TEST_F(ChannelReceiveTest, SettingFrameTransformer) { + auto channel = CreateTestChannelReceive(); + + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + rtc::make_ref_counted<MockFrameTransformer>(); + + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback); + channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer); + + // Must start playout, otherwise packet is discarded. + channel->StartPlayout(); + + RtpPacketReceived packet = CreateRtpPacket(); + + // Receive one RTP packet, this should be transformed. + EXPECT_CALL(*mock_frame_transformer, Transform); + channel->OnRtpPacket(packet); +} + +TEST_F(ChannelReceiveTest, SettingFrameTransformerMultipleTimes) { + auto channel = CreateTestChannelReceive(); + + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + rtc::make_ref_counted<MockFrameTransformer>(); + + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback); + channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer); + + // Set the same transformer again, shouldn't cause any additional callback + // registration calls. + EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback) + .Times(0); + channel->SetDepacketizerToDecoderFrameTransformer(mock_frame_transformer); +} + } // namespace } // namespace voe } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_send.cc b/third_party/libwebrtc/audio/channel_send.cc index ee94760b6f..3c59be52b4 100644 --- a/third_party/libwebrtc/audio/channel_send.cc +++ b/third_party/libwebrtc/audio/channel_send.cc @@ -37,6 +37,7 @@ #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/race_checker.h" #include "rtc_base/rate_limiter.h" +#include "rtc_base/strings/string_builder.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/task_queue.h" #include "rtc_base/time_utils.h" @@ -104,6 +105,7 @@ class ChannelSend : public ChannelSendInterface, // Send using this encoder, with this payload type. void SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, std::unique_ptr<AudioEncoder> encoder) override; void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) override; @@ -192,7 +194,8 @@ class ChannelSend : public ChannelSendInterface, uint8_t payloadType, uint32_t rtp_timestamp_without_offset, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms) + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs) RTC_RUN_ON(encoder_queue_); void OnReceivedRtt(int64_t rtt_ms); @@ -264,6 +267,8 @@ class ChannelSend : public ChannelSendInterface, // Defined last to ensure that there are no running tasks when the other // members are destroyed. rtc::TaskQueue encoder_queue_; + + SdpAudioFormat encoder_format_; }; const int kTelephoneEventAttenuationdB = 10; @@ -310,21 +315,26 @@ int32_t ChannelSend::SendData(AudioFrameType frameType, if (frame_transformer_delegate_) { // Asynchronously transform the payload before sending it. After the payload // is transformed, the delegate will call SendRtpAudio to send it. + char buf[1024]; + rtc::SimpleStringBuilder mime_type(buf); + mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/" + << encoder_format_.name; frame_transformer_delegate_->Transform( frameType, payloadType, rtp_timestamp + rtp_rtcp_->StartTimestamp(), payloadData, payloadSize, absolute_capture_timestamp_ms, - rtp_rtcp_->SSRC()); + rtp_rtcp_->SSRC(), mime_type.str()); return 0; } return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload, - absolute_capture_timestamp_ms); + absolute_capture_timestamp_ms, /*csrcs=*/{}); } int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType, uint8_t payloadType, uint32_t rtp_timestamp_without_offset, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms) { + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs) { // E2EE Custom Audio Frame Encryption (This is optional). // Keep this buffer around for the lifetime of the send call. rtc::Buffer encrypted_audio_payload; @@ -386,7 +396,8 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType, .payload = payload, .payload_id = payloadType, .rtp_timestamp = - rtp_timestamp_without_offset + rtp_rtcp_->StartTimestamp()}; + rtp_timestamp_without_offset + rtp_rtcp_->StartTimestamp(), + .csrcs = csrcs}; if (absolute_capture_timestamp_ms > 0) { frame.capture_time = Timestamp::Millis(absolute_capture_timestamp_ms); } @@ -426,7 +437,8 @@ ChannelSend::ChannelSend( crypto_options_(crypto_options), encoder_queue_(task_queue_factory->CreateTaskQueue( "AudioEncoder", - TaskQueueFactory::Priority::NORMAL)) { + TaskQueueFactory::Priority::NORMAL)), + encoder_format_("x-unknown", 0, 0) { audio_coding_ = AudioCodingModule::Create(); RtpRtcpInterface::Configuration configuration; @@ -527,6 +539,7 @@ void ChannelSend::StopSend() { } void ChannelSend::SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, std::unique_ptr<AudioEncoder> encoder) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); RTC_DCHECK_GE(payload_type, 0); @@ -540,6 +553,7 @@ void ChannelSend::SetEncoder(int payload_type, encoder->RtpTimestampRateHz(), encoder->NumChannels(), 0); + encoder_format_ = encoder_format; audio_coding_->SetEncoder(std::move(encoder)); } @@ -881,12 +895,13 @@ void ChannelSend::InitFrameTransformerDelegate( [this](AudioFrameType frameType, uint8_t payloadType, uint32_t rtp_timestamp_with_offset, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms) { + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs) { RTC_DCHECK_RUN_ON(&encoder_queue_); return SendRtpAudio( frameType, payloadType, rtp_timestamp_with_offset - rtp_rtcp_->StartTimestamp(), payload, - absolute_capture_timestamp_ms); + absolute_capture_timestamp_ms, csrcs); }; frame_transformer_delegate_ = rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( diff --git a/third_party/libwebrtc/audio/channel_send.h b/third_party/libwebrtc/audio/channel_send.h index f0c9232296..f36085c1fa 100644 --- a/third_party/libwebrtc/audio/channel_send.h +++ b/third_party/libwebrtc/audio/channel_send.h @@ -63,6 +63,7 @@ class ChannelSendInterface { virtual CallSendStatistics GetRTCPStatistics() const = 0; virtual void SetEncoder(int payload_type, + const SdpAudioFormat& encoder_format, std::unique_ptr<AudioEncoder> encoder) = 0; virtual void ModifyEncoder( rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) = 0; diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc index 0f85216e92..2eea0d2387 100644 --- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc +++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.cc @@ -11,6 +11,7 @@ #include "audio/channel_send_frame_transformer_delegate.h" #include <utility> +#include <vector> namespace webrtc { namespace { @@ -55,13 +56,17 @@ class TransformableOutgoingAudioFrame const uint8_t* payload_data, size_t payload_size, absl::optional<uint64_t> absolute_capture_timestamp_ms, - uint32_t ssrc) + uint32_t ssrc, + std::vector<uint32_t> csrcs, + const std::string& codec_mime_type) : frame_type_(frame_type), payload_type_(payload_type), rtp_timestamp_with_offset_(rtp_timestamp_with_offset), payload_(payload_data, payload_size), absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms), - ssrc_(ssrc) {} + ssrc_(ssrc), + csrcs_(std::move(csrcs)), + codec_mime_type_(codec_mime_type) {} ~TransformableOutgoingAudioFrame() override = default; rtc::ArrayView<const uint8_t> GetData() const override { return payload_; } void SetData(rtc::ArrayView<const uint8_t> data) override { @@ -76,9 +81,10 @@ class TransformableOutgoingAudioFrame uint8_t GetPayloadType() const override { return payload_type_; } Direction GetDirection() const override { return Direction::kSender; } + std::string GetMimeType() const override { return codec_mime_type_; } rtc::ArrayView<const uint32_t> GetContributingSources() const override { - return {}; + return csrcs_; } const absl::optional<uint16_t> SequenceNumber() const override { @@ -100,6 +106,8 @@ class TransformableOutgoingAudioFrame rtc::Buffer payload_; absl::optional<uint64_t> absolute_capture_timestamp_ms_; uint32_t ssrc_; + std::vector<uint32_t> csrcs_; + std::string codec_mime_type_; }; } // namespace @@ -131,11 +139,23 @@ void ChannelSendFrameTransformerDelegate::Transform( const uint8_t* payload_data, size_t payload_size, int64_t absolute_capture_timestamp_ms, - uint32_t ssrc) { + uint32_t ssrc, + const std::string& codec_mimetype) { + { + MutexLock lock(&send_lock_); + if (short_circuit_) { + send_frame_callback_( + frame_type, payload_type, rtp_timestamp, + rtc::ArrayView<const uint8_t>(payload_data, payload_size), + absolute_capture_timestamp_ms, /*csrcs=*/{}); + return; + } + } frame_transformer_->Transform( std::make_unique<TransformableOutgoingAudioFrame>( frame_type, payload_type, rtp_timestamp, payload_data, payload_size, - absolute_capture_timestamp_ms, ssrc)); + absolute_capture_timestamp_ms, ssrc, + /*csrcs=*/std::vector<uint32_t>(), codec_mimetype)); } void ChannelSendFrameTransformerDelegate::OnTransformedFrame( @@ -150,6 +170,11 @@ void ChannelSendFrameTransformerDelegate::OnTransformedFrame( }); } +void ChannelSendFrameTransformerDelegate::StartShortCircuiting() { + MutexLock lock(&send_lock_); + short_circuit_ = true; +} + void ChannelSendFrameTransformerDelegate::SendFrame( std::unique_ptr<TransformableFrameInterface> frame) const { MutexLock lock(&send_lock_); @@ -164,16 +189,21 @@ void ChannelSendFrameTransformerDelegate::SendFrame( transformed_frame->GetData(), transformed_frame->AbsoluteCaptureTimestamp() ? *transformed_frame->AbsoluteCaptureTimestamp() - : 0); + : 0, + transformed_frame->GetContributingSources()); } std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame( TransformableAudioFrameInterface* original) { + std::vector<uint32_t> csrcs; + csrcs.assign(original->GetContributingSources().begin(), + original->GetContributingSources().end()); return std::make_unique<TransformableOutgoingAudioFrame>( InterfaceFrameTypeToInternalFrameType(original->Type()), original->GetPayloadType(), original->GetTimestamp(), original->GetData().data(), original->GetData().size(), - original->AbsoluteCaptureTimestamp(), original->GetSsrc()); + original->AbsoluteCaptureTimestamp(), original->GetSsrc(), + std::move(csrcs), original->GetMimeType()); } } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h index eb0027e4c8..97fc14f737 100644 --- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h +++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate.h @@ -12,6 +12,7 @@ #define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_ #include <memory> +#include <string> #include "api/frame_transformer_interface.h" #include "api/sequence_checker.h" @@ -34,7 +35,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { uint8_t payloadType, uint32_t rtp_timestamp_with_offset, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms)>; + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs)>; ChannelSendFrameTransformerDelegate( SendFrameCallback send_frame_callback, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, @@ -57,12 +59,15 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { const uint8_t* payload_data, size_t payload_size, int64_t absolute_capture_timestamp_ms, - uint32_t ssrc); + uint32_t ssrc, + const std::string& codec_mime_type); // Implements TransformedFrameCallback. Can be called on any thread. void OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) override; + void StartShortCircuiting() override; + // Delegates the call to ChannelSend::SendRtpAudio on the `encoder_queue_`, // by calling `send_audio_callback_`. void SendFrame(std::unique_ptr<TransformableFrameInterface> frame) const; @@ -75,6 +80,7 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { SendFrameCallback send_frame_callback_ RTC_GUARDED_BY(send_lock_); rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_; rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_); + bool short_circuit_ RTC_GUARDED_BY(send_lock_) = false; }; std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame( diff --git a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc index f75d4a8ab7..4dcd15cd95 100644 --- a/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc +++ b/third_party/libwebrtc/audio/channel_send_frame_transformer_delegate_unittest.cc @@ -12,7 +12,9 @@ #include <memory> #include <utility> +#include <vector> +#include "absl/memory/memory.h" #include "rtc_base/task_queue_for_test.h" #include "test/gmock.h" #include "test/gtest.h" @@ -24,10 +26,13 @@ namespace { using ::testing::_; using ::testing::ElementsAre; +using ::testing::ElementsAreArray; using ::testing::NiceMock; using ::testing::Return; using ::testing::SaveArg; +const uint8_t mock_data[] = {1, 2, 3, 4}; + class MockChannelSend { public: MockChannelSend() = default; @@ -39,30 +44,56 @@ class MockChannelSend { uint8_t payloadType, uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms)); + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs)); ChannelSendFrameTransformerDelegate::SendFrameCallback callback() { return [this](AudioFrameType frameType, uint8_t payloadType, uint32_t rtp_timestamp, rtc::ArrayView<const uint8_t> payload, - int64_t absolute_capture_timestamp_ms) { + int64_t absolute_capture_timestamp_ms, + rtc::ArrayView<const uint32_t> csrcs) { return SendFrame(frameType, payloadType, rtp_timestamp, payload, - absolute_capture_timestamp_ms); + absolute_capture_timestamp_ms, csrcs); }; } }; -std::unique_ptr<MockTransformableAudioFrame> CreateMockReceiverFrame() { - const uint8_t mock_data[] = {1, 2, 3, 4}; +std::unique_ptr<TransformableAudioFrameInterface> CreateMockReceiverFrame( + std::vector<const uint32_t> csrcs) { std::unique_ptr<MockTransformableAudioFrame> mock_frame = - std::make_unique<MockTransformableAudioFrame>(); + std::make_unique<NiceMock<MockTransformableAudioFrame>>(); rtc::ArrayView<const uint8_t> payload(mock_data); ON_CALL(*mock_frame, GetData).WillByDefault(Return(payload)); ON_CALL(*mock_frame, GetPayloadType).WillByDefault(Return(0)); ON_CALL(*mock_frame, GetDirection) .WillByDefault(Return(TransformableFrameInterface::Direction::kReceiver)); + ON_CALL(*mock_frame, GetContributingSources).WillByDefault(Return(csrcs)); return mock_frame; } +std::unique_ptr<TransformableAudioFrameInterface> CreateFrame() { + TaskQueueForTest channel_queue("channel_queue"); + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); + MockChannelSend mock_channel; + rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( + mock_channel.callback(), mock_frame_transformer, &channel_queue); + + std::unique_ptr<TransformableFrameInterface> frame; + ON_CALL(*mock_frame_transformer, Transform) + .WillByDefault( + [&frame]( + std::unique_ptr<TransformableFrameInterface> transform_frame) { + frame = std::move(transform_frame); + }); + delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, mock_data, + sizeof(mock_data), 0, + /*ssrc=*/0, /*mimeType=*/"audio/opus"); + return absl::WrapUnique( + static_cast<webrtc::TransformableAudioFrameInterface*>(frame.release())); +} + // Test that the delegate registers itself with the frame transformer on Init(). TEST(ChannelSendFrameTransformerDelegateTest, RegisterTransformedFrameCallbackOnInit) { @@ -115,7 +146,7 @@ TEST(ChannelSendFrameTransformerDelegateTest, callback->OnTransformedFrame(std::move(frame)); }); delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0, - 0); + /*ssrc=*/0, /*mimeType=*/"audio/opus"); channel_queue.WaitForPreviouslyPostedTasks(); } @@ -136,16 +167,17 @@ TEST(ChannelSendFrameTransformerDelegateTest, delegate->Init(); ASSERT_TRUE(callback); - const uint8_t data[] = {1, 2, 3, 4}; + std::vector<const uint32_t> csrcs = {123, 234, 345, 456}; EXPECT_CALL(mock_channel, SendFrame).Times(0); - EXPECT_CALL(mock_channel, SendFrame(_, 0, 0, ElementsAre(1, 2, 3, 4), _)); + EXPECT_CALL(mock_channel, SendFrame(_, 0, 0, ElementsAreArray(mock_data), _, + ElementsAreArray(csrcs))); ON_CALL(*mock_frame_transformer, Transform) - .WillByDefault( - [&callback](std::unique_ptr<TransformableFrameInterface> frame) { - callback->OnTransformedFrame(CreateMockReceiverFrame()); - }); - delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0, - 0); + .WillByDefault([&](std::unique_ptr<TransformableFrameInterface> frame) { + callback->OnTransformedFrame(CreateMockReceiverFrame(csrcs)); + }); + delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, mock_data, + sizeof(mock_data), 0, + /*ssrc=*/0, /*mimeType=*/"audio/opus"); channel_queue.WaitForPreviouslyPostedTasks(); } @@ -168,5 +200,59 @@ TEST(ChannelSendFrameTransformerDelegateTest, channel_queue.WaitForPreviouslyPostedTasks(); } +TEST(ChannelSendFrameTransformerDelegateTest, ShortCircuitingSkipsTransform) { + TaskQueueForTest channel_queue("channel_queue"); + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + rtc::make_ref_counted<testing::NiceMock<MockFrameTransformer>>(); + MockChannelSend mock_channel; + rtc::scoped_refptr<ChannelSendFrameTransformerDelegate> delegate = + rtc::make_ref_counted<ChannelSendFrameTransformerDelegate>( + mock_channel.callback(), mock_frame_transformer, &channel_queue); + + delegate->StartShortCircuiting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the channel. + EXPECT_CALL(mock_channel, SendFrame); + const uint8_t data[] = {1, 2, 3, 4}; + delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0, + /*ssrc=*/0, /*mimeType=*/"audio/opus"); +} + +TEST(ChannelSendFrameTransformerDelegateTest, + CloningSenderFramePreservesInformation) { + std::unique_ptr<TransformableAudioFrameInterface> frame = CreateFrame(); + std::unique_ptr<TransformableAudioFrameInterface> cloned_frame = + CloneSenderAudioFrame(frame.get()); + + EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp()); + EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc()); + EXPECT_EQ(cloned_frame->Type(), frame->Type()); + EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType()); + EXPECT_EQ(cloned_frame->GetMimeType(), frame->GetMimeType()); + EXPECT_THAT(cloned_frame->GetContributingSources(), + ElementsAreArray(frame->GetContributingSources())); +} + +TEST(ChannelSendFrameTransformerDelegateTest, CloningReceiverFrameWithCsrcs) { + std::unique_ptr<TransformableAudioFrameInterface> frame = + CreateMockReceiverFrame(/*csrcs=*/{123, 234, 345}); + std::unique_ptr<TransformableAudioFrameInterface> cloned_frame = + CloneSenderAudioFrame(frame.get()); + + EXPECT_EQ(cloned_frame->GetTimestamp(), frame->GetTimestamp()); + EXPECT_EQ(cloned_frame->GetSsrc(), frame->GetSsrc()); + EXPECT_EQ(cloned_frame->Type(), frame->Type()); + EXPECT_EQ(cloned_frame->GetPayloadType(), frame->GetPayloadType()); + EXPECT_EQ(cloned_frame->GetMimeType(), frame->GetMimeType()); + EXPECT_EQ(cloned_frame->AbsoluteCaptureTimestamp(), + frame->AbsoluteCaptureTimestamp()); + + ASSERT_NE(frame->GetContributingSources().size(), 0u); + EXPECT_THAT(cloned_frame->GetContributingSources(), + ElementsAreArray(frame->GetContributingSources())); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/audio/channel_send_unittest.cc b/third_party/libwebrtc/audio/channel_send_unittest.cc index b9406e1523..58d7c93c1e 100644 --- a/third_party/libwebrtc/audio/channel_send_unittest.cc +++ b/third_party/libwebrtc/audio/channel_send_unittest.cc @@ -66,9 +66,10 @@ class ChannelSendTest : public ::testing::Test { &transport_, nullptr, &event_log_, nullptr, crypto_options_, false, kRtcpIntervalMs, kSsrc, nullptr, &transport_controller_, field_trials_); encoder_factory_ = CreateBuiltinAudioEncoderFactory(); - std::unique_ptr<AudioEncoder> encoder = encoder_factory_->MakeAudioEncoder( - kPayloadType, SdpAudioFormat("opus", kRtpRateHz, 2), {}); - channel_->SetEncoder(kPayloadType, std::move(encoder)); + SdpAudioFormat opus = SdpAudioFormat("opus", kRtpRateHz, 2); + std::unique_ptr<AudioEncoder> encoder = + encoder_factory_->MakeAudioEncoder(kPayloadType, opus, {}); + channel_->SetEncoder(kPayloadType, opus, std::move(encoder)); transport_controller_.EnsureStarted(); channel_->RegisterSenderCongestionControlObjects(&transport_controller_); ON_CALL(transport_, SendRtcp).WillByDefault(Return(true)); diff --git a/third_party/libwebrtc/audio/mock_voe_channel_proxy.h b/third_party/libwebrtc/audio/mock_voe_channel_proxy.h index 29005173df..71ef5d12fb 100644 --- a/third_party/libwebrtc/audio/mock_voe_channel_proxy.h +++ b/third_party/libwebrtc/audio/mock_voe_channel_proxy.h @@ -113,7 +113,9 @@ class MockChannelSend : public voe::ChannelSendInterface { public: MOCK_METHOD(void, SetEncoder, - (int payload_type, std::unique_ptr<AudioEncoder> encoder), + (int payload_type, + const SdpAudioFormat& encoder_format, + std::unique_ptr<AudioEncoder> encoder), (override)); MOCK_METHOD( void, diff --git a/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build b/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build index e215792f64..6489497db4 100644 --- a/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build +++ b/third_party/libwebrtc/audio/utility/audio_frame_operations_gn/moz.build @@ -201,7 +201,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -211,10 +210,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True |