/* * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h" #include #include #include #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/checks.h" #include "rtc_base/thread.h" namespace webrtc { namespace { class TransformableVideoReceiverFrame : public TransformableVideoFrameInterface { public: TransformableVideoReceiverFrame(std::unique_ptr frame, uint32_t ssrc, RtpVideoFrameReceiver* receiver) : frame_(std::move(frame)), metadata_(frame_->GetRtpVideoHeader().GetAsMetadata()), receiver_(receiver) { metadata_.SetSsrc(ssrc); metadata_.SetCsrcs(frame_->Csrcs()); } ~TransformableVideoReceiverFrame() override = default; // Implements TransformableVideoFrameInterface. rtc::ArrayView GetData() const override { return *frame_->GetEncodedData(); } void SetData(rtc::ArrayView data) override { frame_->SetEncodedData( EncodedImageBuffer::Create(data.data(), data.size())); } uint8_t GetPayloadType() const override { return frame_->PayloadType(); } uint32_t GetSsrc() const override { return Metadata().GetSsrc(); } uint32_t GetTimestamp() const override { return frame_->RtpTimestamp(); } void SetRTPTimestamp(uint32_t timestamp) override { frame_->SetRtpTimestamp(timestamp); } bool IsKeyFrame() const override { return frame_->FrameType() == VideoFrameType::kVideoFrameKey; } const std::string& GetRid() const override { static const std::string empty; return empty; } VideoFrameMetadata Metadata() const override { return metadata_; } void SetMetadata(const VideoFrameMetadata& metadata) override { // Create |new_metadata| from existing metadata and change only frameId and // dependencies. VideoFrameMetadata new_metadata = Metadata(); new_metadata.SetFrameId(metadata.GetFrameId()); new_metadata.SetFrameDependencies(metadata.GetFrameDependencies()); RTC_DCHECK(new_metadata == metadata) << "TransformableVideoReceiverFrame::SetMetadata can be only used to " "change frameID and dependencies"; frame_->SetHeaderFromMetadata(new_metadata); } std::unique_ptr ExtractFrame() && { return std::move(frame_); } Direction GetDirection() const override { return Direction::kReceiver; } std::string GetMimeType() const override { std::string mime_type = "video/"; return mime_type + CodecTypeToPayloadString(frame_->codec_type()); } const RtpVideoFrameReceiver* Receiver() { return receiver_; } private: std::unique_ptr frame_; VideoFrameMetadata metadata_; RtpVideoFrameReceiver* receiver_; }; } // namespace RtpVideoStreamReceiverFrameTransformerDelegate:: RtpVideoStreamReceiverFrameTransformerDelegate( RtpVideoFrameReceiver* receiver, Clock* clock, rtc::scoped_refptr frame_transformer, TaskQueueBase* network_thread, uint32_t ssrc) : receiver_(receiver), frame_transformer_(std::move(frame_transformer)), network_thread_(network_thread), ssrc_(ssrc), clock_(clock) {} void RtpVideoStreamReceiverFrameTransformerDelegate::Init() { RTC_DCHECK_RUN_ON(&network_sequence_checker_); frame_transformer_->RegisterTransformedFrameSinkCallback( rtc::scoped_refptr(this), ssrc_); } void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() { RTC_DCHECK_RUN_ON(&network_sequence_checker_); frame_transformer_->UnregisterTransformedFrameSinkCallback(ssrc_); frame_transformer_ = nullptr; receiver_ = nullptr; } void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); frame_transformer_->Transform( std::make_unique(std::move(frame), ssrc_, receiver_)); } void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( std::unique_ptr frame) { rtc::scoped_refptr delegate( this); network_thread_->PostTask( [delegate = std::move(delegate), frame = std::move(frame)]() mutable { delegate->ManageFrame(std::move(frame)); }); } void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); if (!receiver_) return; if (frame->GetDirection() == TransformableFrameInterface::Direction::kReceiver) { auto transformed_frame = absl::WrapUnique( static_cast(frame.release())); auto frame_receiver = transformed_frame->Receiver(); std::unique_ptr frame_object = std::move(*transformed_frame).ExtractFrame(); if (frame_receiver != receiver_) { // This frame was received by a different RtpReceiver instance, so has // first and last sequence numbers which will be meaningless to our // receiver_. Work around this by using the frame id as a surrogate value, // same as when given a Sender frame below. // TODO(https://crbug.com/1250638): Change what happens after the encoded // insertable stream insertion to not require RTP data. frame_object->SetFirstSeqNum(frame_object->Id()); frame_object->SetLastSeqNum(frame_object->Id()); } receiver_->ManageFrame(std::move(frame_object)); } else { RTC_CHECK_EQ(frame->GetDirection(), TransformableFrameInterface::Direction::kSender); // This frame is actually an frame encoded locally, to be sent, but has been // fed back into this receiver's insertable stream writer. // Create a reasonable RtpFrameObject as if this frame had been received // over RTP, reusing the frameId as an analog for the RTP sequence number, // and handle it as if it had been received. // TODO(https://crbug.com/1250638): Rewrite the receiver's codepaths after // this transform to be transport-agnostic and not need a faked rtp // sequence number. auto transformed_frame = absl::WrapUnique( static_cast(frame.release())); VideoFrameMetadata metadata = transformed_frame->Metadata(); RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); VideoSendTiming timing; rtc::ArrayView data = transformed_frame->GetData(); int64_t receive_time = clock_->CurrentTime().ms(); receiver_->ManageFrame(std::make_unique( /*first_seq_num=*/metadata.GetFrameId().value_or(0), /*last_seq_num=*/metadata.GetFrameId().value_or(0), /*markerBit=*/video_header.is_last_frame_in_picture, /*times_nacked=*/0, /*first_packet_received_time=*/receive_time, /*last_packet_received_time=*/receive_time, /*rtp_timestamp=*/transformed_frame->GetTimestamp(), /*ntp_time_ms=*/0, timing, transformed_frame->GetPayloadType(), metadata.GetCodec(), metadata.GetRotation(), metadata.GetContentType(), video_header, video_header.color_space, RtpPacketInfos(), EncodedImageBuffer::Create(data.data(), data.size()))); } } } // namespace webrtc