From: Michael Froman Date: Thu, 27 Jul 2023 12:42:44 -0500 Subject: Bug 1838080: Store the rid in TransformableVideoSenderFrame. r=ng,webrtc-reviewers This is necessary to reliably detect what rid a given keyframe is for, for the purposes of resolving promises from RTCRtpScriptTransformer.generateKeyFrame. Differential Revision: https://phabricator.services.mozilla.com/D180737 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/2f1a0ba74bf71cfa0bc4e77714b8a5276a70cc36 --- api/frame_transformer_interface.h | 1 + modules/rtp_rtcp/source/rtp_sender.h | 4 ++++ modules/rtp_rtcp/source/rtp_sender_video.cc | 1 + ...rtp_sender_video_frame_transformer_delegate.cc | 15 +++++++++++---- .../rtp_sender_video_frame_transformer_delegate.h | 2 ++ ..._stream_receiver_frame_transformer_delegate.cc | 5 +++++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 2bcc32d531..8ab8c13659 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -103,6 +103,7 @@ class TransformableVideoFrameInterface : public TransformableFrameInterface { RTC_EXPORT explicit TransformableVideoFrameInterface(Passkey passkey); virtual ~TransformableVideoFrameInterface() = default; virtual bool IsKeyFrame() const = 0; + virtual const std::string& GetRid() const = 0; virtual VideoFrameMetadata Metadata() const = 0; diff --git a/modules/rtp_rtcp/source/rtp_sender.h b/modules/rtp_rtcp/source/rtp_sender.h index 0a29968b0c..c23051401e 100644 --- a/modules/rtp_rtcp/source/rtp_sender.h +++ b/modules/rtp_rtcp/source/rtp_sender.h @@ -141,6 +141,10 @@ class RTPSender { uint32_t SSRC() const RTC_LOCKS_EXCLUDED(send_mutex_) { return ssrc_; } + const std::string& Rid() const RTC_LOCKS_EXCLUDED(send_mutex_) { + return rid_; + } + std::optional FlexfecSsrc() const RTC_LOCKS_EXCLUDED(send_mutex_) { return flexfec_ssrc_; } diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 7fbfe65099..ca2f0168c9 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -189,6 +189,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) this, config.frame_transformer, rtp_sender_->SSRC(), + rtp_sender_->Rid(), config.task_queue_factory) : nullptr) { if (frame_transformer_delegate_) diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index f8232202c6..86c69f7861 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -55,7 +55,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { uint32_t rtp_timestamp, TimeDelta expected_retransmission_time, uint32_t ssrc, - std::vector csrcs) + std::vector csrcs, + const std::string& rid) : TransformableVideoFrameInterface(Passkey()), encoded_data_(encoded_image.GetEncodedData()), pre_transform_payload_size_(encoded_image.size()), @@ -68,7 +69,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { presentation_timestamp_(encoded_image.PresentationTimestamp()), expected_retransmission_time_(expected_retransmission_time), ssrc_(ssrc), - csrcs_(csrcs) { + csrcs_(csrcs), + rid_(rid) { RTC_DCHECK_GE(payload_type_, 0); RTC_DCHECK_LE(payload_type_, 127); } @@ -143,6 +145,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { return std::nullopt; } + const std::string& GetRid() const override { return rid_; } + private: rtc::scoped_refptr encoded_data_; const size_t pre_transform_payload_size_; @@ -157,16 +161,19 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { uint32_t ssrc_; std::vector csrcs_; + const std::string rid_; }; RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( RTPVideoFrameSenderInterface* sender, rtc::scoped_refptr frame_transformer, uint32_t ssrc, + const std::string& rid, TaskQueueFactory* task_queue_factory) : sender_(sender), frame_transformer_(std::move(frame_transformer)), ssrc_(ssrc), + rid_(rid), transformation_queue_(task_queue_factory->CreateTaskQueue( "video_frame_transformer", TaskQueueFactory::Priority::NORMAL)) {} @@ -197,7 +204,7 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( frame_transformer_->Transform(std::make_unique( encoded_image, video_header, payload_type, codec_type, rtp_timestamp, expected_retransmission_time, ssrc_, - /*csrcs=*/std::vector())); + /*csrcs=*/std::vector(), rid_)); return true; } @@ -300,7 +307,7 @@ std::unique_ptr CloneSenderVideoFrame( return std::make_unique( encoded_image, new_header, original->GetPayloadType(), new_header.codec, original->GetTimestamp(), kDefaultRetransmissionsTime, - original->GetSsrc(), metadata.GetCsrcs()); + original->GetSsrc(), metadata.GetCsrcs(), original->GetRid()); } } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index e44a62685e..0f50f92108 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -67,6 +67,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { RTPVideoFrameSenderInterface* sender, rtc::scoped_refptr frame_transformer, uint32_t ssrc, + const std::string& rid, TaskQueueFactory* send_transport_queue); void Init(); @@ -115,6 +116,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { RTPVideoFrameSenderInterface* sender_ RTC_GUARDED_BY(sender_lock_); rtc::scoped_refptr frame_transformer_; const uint32_t ssrc_; + const std::string rid_; // Used when the encoded frames arrives without a current task queue. This can // happen if a hardware encoder was used. std::unique_ptr transformation_queue_; diff --git a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index c736cc647d..41f41219e2 100644 --- a/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -60,6 +60,11 @@ class TransformableVideoReceiverFrame return frame_->FrameType() == VideoFrameType::kVideoFrameKey; } + const std::string& GetRid() const override { + static const std::string empty; + return empty; + } + VideoFrameMetadata Metadata() const override { return metadata_; } void SetMetadata(const VideoFrameMetadata& metadata) override {