/* * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h" #include #include #include #include #include "absl/memory/memory.h" #include "api/call/transport.h" #include "api/test/mock_transformable_video_frame.h" #include "api/units/timestamp.h" #include "call/video_receive_stream.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/event.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_frame_transformer.h" namespace webrtc { namespace { using ::testing::_; using ::testing::ElementsAre; using ::testing::NiceMock; using ::testing::NotNull; using ::testing::Return; using ::testing::SaveArg; const int kFirstSeqNum = 1; const int kLastSeqNum = 2; std::unique_ptr CreateRtpFrameObject( const RTPVideoHeader& video_header, std::vector csrcs) { RtpPacketInfo packet_info(/*ssrc=*/123, csrcs, /*rtc_timestamp=*/0, /*receive_time=*/Timestamp::Seconds(123456)); return std::make_unique( kFirstSeqNum, kLastSeqNum, /*markerBit=*/true, /*times_nacked=*/3, /*first_packet_received_time=*/4, /*last_packet_received_time=*/5, /*rtp_timestamp=*/6, /*ntp_time_ms=*/7, VideoSendTiming(), /*payload_type=*/8, video_header.codec, kVideoRotation_0, VideoContentType::UNSPECIFIED, video_header, absl::nullopt, RtpPacketInfos({packet_info}), EncodedImageBuffer::Create(0)); } std::unique_ptr CreateRtpFrameObject() { return CreateRtpFrameObject(RTPVideoHeader(), /*csrcs=*/{}); } class TestRtpVideoFrameReceiver : public RtpVideoFrameReceiver { public: TestRtpVideoFrameReceiver() {} ~TestRtpVideoFrameReceiver() override = default; MOCK_METHOD(void, ManageFrame, (std::unique_ptr frame), (override)); }; TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, RegisterTransformedFrameCallbackSinkOnInit) { TestRtpVideoFrameReceiver receiver; auto frame_transformer(rtc::make_ref_counted()); SimulatedClock clock(0); auto delegate( rtc::make_ref_counted( &receiver, &clock, frame_transformer, rtc::Thread::Current(), /*remote_ssrc*/ 1111)); EXPECT_CALL(*frame_transformer, RegisterTransformedFrameSinkCallback(testing::_, 1111)); delegate->Init(); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, UnregisterTransformedFrameSinkCallbackOnReset) { TestRtpVideoFrameReceiver receiver; auto frame_transformer(rtc::make_ref_counted()); SimulatedClock clock(0); auto delegate( rtc::make_ref_counted( &receiver, &clock, frame_transformer, rtc::Thread::Current(), /*remote_ssrc*/ 1111)); EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameSinkCallback(1111)); delegate->Reset(); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) { TestRtpVideoFrameReceiver receiver; auto frame_transformer( rtc::make_ref_counted>()); SimulatedClock clock(0); auto delegate( rtc::make_ref_counted( &receiver, &clock, frame_transformer, rtc::Thread::Current(), /*remote_ssrc*/ 1111)); auto frame = CreateRtpFrameObject(); EXPECT_CALL(*frame_transformer, Transform); delegate->TransformFrame(std::move(frame)); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, ManageFrameOnTransformedFrame) { rtc::AutoThread main_thread_; TestRtpVideoFrameReceiver receiver; auto mock_frame_transformer( rtc::make_ref_counted>()); SimulatedClock clock(0); std::vector csrcs = {234, 345, 456}; auto delegate = rtc::make_ref_counted( &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), /*remote_ssrc*/ 1111); rtc::scoped_refptr callback; EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback) .WillOnce(SaveArg<0>(&callback)); delegate->Init(); ASSERT_TRUE(callback); EXPECT_CALL(receiver, ManageFrame) .WillOnce([&](std::unique_ptr frame) { EXPECT_EQ(frame->Csrcs(), csrcs); EXPECT_EQ(frame->first_seq_num(), kFirstSeqNum); EXPECT_EQ(frame->last_seq_num(), kLastSeqNum); }); ON_CALL(*mock_frame_transformer, Transform) .WillByDefault( [&callback](std::unique_ptr frame) { EXPECT_STRCASEEQ("video/Generic", frame->GetMimeType().c_str()); callback->OnTransformedFrame(std::move(frame)); }); delegate->TransformFrame(CreateRtpFrameObject(RTPVideoHeader(), csrcs)); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformableFrameMetadataHasCorrectValue) { TestRtpVideoFrameReceiver receiver; auto mock_frame_transformer = rtc::make_ref_counted>(); SimulatedClock clock(0); auto delegate = rtc::make_ref_counted( &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), 1111); delegate->Init(); RTPVideoHeader video_header; video_header.width = 1280u; video_header.height = 720u; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); generic.frame_id = 10; generic.temporal_index = 3; generic.spatial_index = 2; generic.decode_target_indications = {DecodeTargetIndication::kSwitch}; generic.dependencies = {5}; std::vector csrcs = {234, 345, 456}; // Check that the transformable frame passed to the frame transformer has the // correct metadata. EXPECT_CALL(*mock_frame_transformer, Transform) .WillOnce([&](std::unique_ptr transformable_frame) { auto frame = absl::WrapUnique(static_cast( transformable_frame.release())); ASSERT_TRUE(frame); auto metadata = frame->Metadata(); EXPECT_EQ(metadata.GetWidth(), 1280u); EXPECT_EQ(metadata.GetHeight(), 720u); EXPECT_EQ(metadata.GetFrameId(), 10); EXPECT_EQ(metadata.GetTemporalIndex(), 3); EXPECT_EQ(metadata.GetSpatialIndex(), 2); EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5)); EXPECT_THAT(metadata.GetDecodeTargetIndications(), ElementsAre(DecodeTargetIndication::kSwitch)); EXPECT_EQ(metadata.GetCsrcs(), csrcs); }); // The delegate creates a transformable frame from the RtpFrameObject. delegate->TransformFrame(CreateRtpFrameObject(video_header, csrcs)); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformableFrameMetadataHasCorrectValueAfterSetMetadata) { rtc::AutoThread main_thread; TestRtpVideoFrameReceiver receiver; auto mock_frame_transformer = rtc::make_ref_counted>(); SimulatedClock clock(1000); auto delegate = rtc::make_ref_counted( &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), 1111); rtc::scoped_refptr callback; EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback) .WillOnce(SaveArg<0>(&callback)); delegate->Init(); ASSERT_TRUE(callback); RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); generic.frame_id = 10; generic.dependencies = {5}; std::vector csrcs = {234, 345, 456}; // Checks that the recieved RTPFrameObject has the new metadata. EXPECT_CALL(receiver, ManageFrame) .WillOnce([&](std::unique_ptr frame) { const absl::optional& descriptor = frame->GetRtpVideoHeader().generic; if (!descriptor.has_value()) { ADD_FAILURE() << "GenericDescriptorInfo in RTPVideoHeader doesn't " "have a value."; } else { EXPECT_EQ(descriptor->frame_id, 20); EXPECT_THAT(descriptor->dependencies, ElementsAre(15)); } EXPECT_EQ(frame->Csrcs(), csrcs); }); // Sets new metadata to the transformable frame. ON_CALL(*mock_frame_transformer, Transform) .WillByDefault([&](std::unique_ptr transformable_frame) { ASSERT_THAT(transformable_frame, NotNull()); auto& video_frame = static_cast( *transformable_frame); VideoFrameMetadata metadata = video_frame.Metadata(); EXPECT_EQ(metadata.GetFrameId(), 10); EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5)); EXPECT_EQ(metadata.GetCsrcs(), csrcs); metadata.SetFrameId(20); metadata.SetFrameDependencies(std::vector{15}); video_frame.SetMetadata(metadata); callback->OnTransformedFrame(std::move(transformable_frame)); }); // The delegate creates a transformable frame from the RtpFrameObject. delegate->TransformFrame(CreateRtpFrameObject(video_header, csrcs)); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, SenderFramesAreConvertedToReceiverFrames) { rtc::AutoThread main_thread_; TestRtpVideoFrameReceiver receiver; auto mock_frame_transformer = rtc::make_ref_counted>(); SimulatedClock clock(/*initial_timestamp_us=*/12345000); auto delegate = rtc::make_ref_counted( &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), /*remote_ssrc*/ 1111); auto mock_sender_frame = std::make_unique>(); ON_CALL(*mock_sender_frame, GetDirection) .WillByDefault(Return(TransformableFrameInterface::Direction::kSender)); VideoFrameMetadata metadata; metadata.SetCodec(kVideoCodecVP8); metadata.SetRTPVideoHeaderCodecSpecifics(RTPVideoHeaderVP8()); ON_CALL(*mock_sender_frame, Metadata).WillByDefault(Return(metadata)); rtc::scoped_refptr buffer = EncodedImageBuffer::Create(1); ON_CALL(*mock_sender_frame, GetData) .WillByDefault(Return(rtc::ArrayView(*buffer))); rtc::scoped_refptr callback; EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback) .WillOnce(SaveArg<0>(&callback)); delegate->Init(); ASSERT_TRUE(callback); EXPECT_CALL(receiver, ManageFrame) .WillOnce([&](std::unique_ptr frame) { EXPECT_EQ(frame->codec_type(), metadata.GetCodec()); EXPECT_EQ(frame->ReceivedTime(), 12345); }); callback->OnTransformedFrame(std::move(mock_sender_frame)); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, ManageFrameFromDifferentReceiver) { rtc::AutoThread main_thread_; std::vector csrcs = {234, 345, 456}; const int frame_id = 11; TestRtpVideoFrameReceiver receiver1; auto mock_frame_transformer1( rtc::make_ref_counted>()); SimulatedClock clock(0); auto delegate1 = rtc::make_ref_counted( &receiver1, &clock, mock_frame_transformer1, rtc::Thread::Current(), /*remote_ssrc*/ 1111); TestRtpVideoFrameReceiver receiver2; auto mock_frame_transformer2( rtc::make_ref_counted>()); auto delegate2 = rtc::make_ref_counted( &receiver2, &clock, mock_frame_transformer2, rtc::Thread::Current(), /*remote_ssrc*/ 1111); delegate1->Init(); rtc::scoped_refptr callback_for_2; EXPECT_CALL(*mock_frame_transformer2, RegisterTransformedFrameSinkCallback) .WillOnce(SaveArg<0>(&callback_for_2)); delegate2->Init(); ASSERT_TRUE(callback_for_2); // Expect a call on receiver2's ManageFrame with sequence numbers overwritten // with the frame's ID. EXPECT_CALL(receiver2, ManageFrame) .WillOnce([&](std::unique_ptr frame) { EXPECT_EQ(frame->Csrcs(), csrcs); EXPECT_EQ(frame->first_seq_num(), frame_id); EXPECT_EQ(frame->last_seq_num(), frame_id); }); // When the frame transformer for receiver 1 receives the frame to transform, // pipe it over to the callback for receiver 2. ON_CALL(*mock_frame_transformer1, Transform) .WillByDefault([&callback_for_2]( std::unique_ptr frame) { callback_for_2->OnTransformedFrame(std::move(frame)); }); std::unique_ptr untransformed_frame = CreateRtpFrameObject(RTPVideoHeader(), csrcs); untransformed_frame->SetId(frame_id); delegate1->TransformFrame(std::move(untransformed_frame)); rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } } // namespace } // namespace webrtc