/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "call/rtp_video_sender.h" #include #include #include #include #include "absl/functional/any_invocable.h" #include "call/rtp_transport_controller_send.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtcp_packet/nack.h" #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/video_coding/fec_controller_default.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/rate_limiter.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_frame_transformer.h" #include "test/mock_transport.h" #include "test/scenario/scenario.h" #include "test/scoped_key_value_config.h" #include "test/time_controller/simulated_time_controller.h" #include "video/send_delay_stats.h" #include "video/send_statistics_proxy.h" namespace webrtc { namespace { using ::testing::_; using ::testing::NiceMock; using ::testing::SaveArg; using ::testing::SizeIs; const int8_t kPayloadType = 96; const uint32_t kSsrc1 = 12345; const uint32_t kSsrc2 = 23456; const uint32_t kRtxSsrc1 = 34567; const uint32_t kRtxSsrc2 = 45678; const int16_t kInitialPictureId1 = 222; const int16_t kInitialPictureId2 = 44; const int16_t kInitialTl0PicIdx1 = 99; const int16_t kInitialTl0PicIdx2 = 199; const int64_t kRetransmitWindowSizeMs = 500; const int kTransportsSequenceExtensionId = 7; const int kDependencyDescriptorExtensionId = 8; class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { public: MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override)); }; RtpSenderObservers CreateObservers( RtcpRttStats* rtcp_rtt_stats, RtcpIntraFrameObserver* intra_frame_callback, ReportBlockDataObserver* report_block_data_observer, StreamDataCountersCallback* rtp_stats, BitrateStatisticsObserver* bitrate_observer, FrameCountObserver* frame_count_observer, RtcpPacketTypeCounterObserver* rtcp_type_observer, SendSideDelayObserver* send_delay_observer, SendPacketObserver* send_packet_observer) { RtpSenderObservers observers; observers.rtcp_rtt_stats = rtcp_rtt_stats; observers.intra_frame_callback = intra_frame_callback; observers.rtcp_loss_notification_observer = nullptr; observers.report_block_data_observer = report_block_data_observer; observers.rtp_stats = rtp_stats; observers.bitrate_observer = bitrate_observer; observers.frame_count_observer = frame_count_observer; observers.rtcp_type_observer = rtcp_type_observer; observers.send_delay_observer = send_delay_observer; observers.send_packet_observer = send_packet_observer; return observers; } BitrateConstraints GetBitrateConfig() { BitrateConstraints bitrate_config; bitrate_config.min_bitrate_bps = 30000; bitrate_config.start_bitrate_bps = 300000; bitrate_config.max_bitrate_bps = 3000000; return bitrate_config; } VideoSendStream::Config CreateVideoSendStreamConfig( Transport* transport, const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type) { VideoSendStream::Config config(transport); config.rtp.ssrcs = ssrcs; config.rtp.rtx.ssrcs = rtx_ssrcs; config.rtp.payload_type = payload_type; config.rtp.rtx.payload_type = payload_type + 1; config.rtp.nack.rtp_history_ms = 1000; config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri, kTransportsSequenceExtensionId); config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::Uri(), kDependencyDescriptorExtensionId); config.rtp.extmap_allow_mixed = true; return config; } class RtpVideoSenderTestFixture { public: RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states, FrameCountObserver* frame_count_observer, rtc::scoped_refptr frame_transformer, const FieldTrialsView* field_trials = nullptr) : time_controller_(Timestamp::Millis(1000000)), config_(CreateVideoSendStreamConfig(&transport_, ssrcs, rtx_ssrcs, payload_type)), send_delay_stats_(time_controller_.GetClock()), bitrate_config_(GetBitrateConfig()), transport_controller_( time_controller_.GetClock(), RtpTransportConfig{ .bitrate_config = bitrate_config_, .event_log = &event_log_, .task_queue_factory = time_controller_.GetTaskQueueFactory(), .trials = field_trials ? field_trials : &field_trials_, }), stats_proxy_(time_controller_.GetClock(), config_, VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials ? *field_trials : field_trials_), retransmission_rate_limiter_(time_controller_.GetClock(), kRetransmitWindowSizeMs) { transport_controller_.EnsureStarted(); std::map suspended_ssrcs; router_ = std::make_unique( time_controller_.GetClock(), suspended_ssrcs, suspended_payload_states, config_.rtp, config_.rtcp_report_interval_ms, &transport_, CreateObservers(nullptr, &encoder_feedback_, &stats_proxy_, &stats_proxy_, &stats_proxy_, frame_count_observer, &stats_proxy_, &stats_proxy_, &send_delay_stats_), &transport_controller_, &event_log_, &retransmission_rate_limiter_, std::make_unique(time_controller_.GetClock()), nullptr, CryptoOptions{}, frame_transformer, field_trials ? *field_trials : field_trials_, time_controller_.GetTaskQueueFactory()); } RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states, FrameCountObserver* frame_count_observer, const FieldTrialsView* field_trials = nullptr) : RtpVideoSenderTestFixture(ssrcs, rtx_ssrcs, payload_type, suspended_payload_states, frame_count_observer, /*frame_transformer=*/nullptr, field_trials) {} RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states, const FieldTrialsView* field_trials = nullptr) : RtpVideoSenderTestFixture(ssrcs, rtx_ssrcs, payload_type, suspended_payload_states, /*frame_count_observer=*/nullptr, /*frame_transformer=*/nullptr, field_trials) {} ~RtpVideoSenderTestFixture() { Stop(); } RtpVideoSender* router() { return router_.get(); } MockTransport& transport() { return transport_; } void AdvanceTime(TimeDelta delta) { time_controller_.AdvanceTime(delta); } void Stop() { RunOnTransportQueue([&]() { router_->Stop(); }); } void SetActiveModules(const std::vector& active_modules) { RunOnTransportQueue([&]() { router_->SetActiveModules(active_modules); }); } // Several RtpVideoSender methods expect to be called on the task queue as // owned by the send transport. While the SequenceChecker may pick up the // default thread as the transport queue, explicit checks for the transport // queue (not just using a SequenceChecker) aren't possible unless such a // queue is actually active. So RunOnTransportQueue is a convenience function // that allow for running a `task` on the transport queue, similar to // SendTask(). void RunOnTransportQueue(absl::AnyInvocable task) { transport_controller_.GetWorkerQueue()->RunOrPost(std::move(task)); AdvanceTime(TimeDelta::Zero()); } private: test::ScopedKeyValueConfig field_trials_; NiceMock transport_; NiceMock encoder_feedback_; GlobalSimulatedTimeController time_controller_; RtcEventLogNull event_log_; VideoSendStream::Config config_; SendDelayStats send_delay_stats_; BitrateConstraints bitrate_config_; RtpTransportControllerSend transport_controller_; SendStatisticsProxy stats_proxy_; RateLimiter retransmission_rate_limiter_; std::unique_ptr router_; }; BitrateAllocationUpdate CreateBitrateAllocationUpdate(int target_bitrate_bps) { BitrateAllocationUpdate update; update.target_bitrate = DataRate::BitsPerSec(target_bitrate_bps); update.round_trip_time = TimeDelta::Zero(); return update; } } // namespace TEST(RtpVideoSenderTest, SendOnOneModule) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); test.SetActiveModules({true}); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); test.SetActiveModules({false}); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); test.SetActiveModules({true}); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); } TEST(RtpVideoSenderTest, SendSimulcastSetActive) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image_1; encoded_image_1.SetTimestamp(1); encoded_image_1.capture_time_ms_ = 2; encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; test.SetActiveModules({true, true}); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); EncodedImage encoded_image_2(encoded_image_1); encoded_image_2.SetSpatialIndex(1); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); // Inactive. test.Stop(); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); } // Tests how setting individual rtp modules to active affects the overall // behavior of the payload router. First sets one module to active and checks // that outgoing data can be sent on this module, and checks that no data can // be sent if both modules are inactive. TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image_1; encoded_image_1.SetTimestamp(1); encoded_image_1.capture_time_ms_ = 2; encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); EncodedImage encoded_image_2(encoded_image_1); encoded_image_2.SetSpatialIndex(1); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; // Only setting one stream to active will still set the payload router to // active and allow sending data on the active stream. std::vector active_modules({true, false}); test.SetActiveModules(active_modules); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); // Setting both streams to inactive will turn the payload router to // inactive. active_modules = {false, false}; test.SetActiveModules(active_modules); // An incoming encoded image will not ask the module to send outgoing data // because the payload router is inactive. EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); } TEST( RtpVideoSenderTest, DiscardsHigherSpatialVideoFramesAfterLayerDisabledInVideoLayersAllocation) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image_1; encoded_image_1.SetTimestamp(1); encoded_image_1.capture_time_ms_ = 2; encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); EncodedImage encoded_image_2(encoded_image_1); encoded_image_2.SetSpatialIndex(1); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.SetActiveModules({true, true}); // A layer is sent on both rtp streams. test.router()->OnVideoLayersAllocationUpdated( {.active_spatial_layers = {{.rtp_stream_index = 0}, {.rtp_stream_index = 1}}}); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); // Only rtp stream index 0 is configured to send a stream. test.router()->OnVideoLayersAllocationUpdated( {.active_spatial_layers = {{.rtp_stream_index = 0}}}); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_1, &codec_info).error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image_2, &codec_info).error); } TEST(RtpVideoSenderTest, CreateWithNoPreviousStates) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.SetActiveModules({true, true}); std::map initial_states = test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_NE(initial_states.find(kSsrc1), initial_states.end()); EXPECT_NE(initial_states.find(kSsrc2), initial_states.end()); } TEST(RtpVideoSenderTest, CreateWithPreviousStates) { const int64_t kState1SharedFrameId = 123; const int64_t kState2SharedFrameId = 234; RtpPayloadState state1; state1.picture_id = kInitialPictureId1; state1.tl0_pic_idx = kInitialTl0PicIdx1; state1.shared_frame_id = kState1SharedFrameId; RtpPayloadState state2; state2.picture_id = kInitialPictureId2; state2.tl0_pic_idx = kInitialTl0PicIdx2; state2.shared_frame_id = kState2SharedFrameId; std::map states = {{kSsrc1, state1}, {kSsrc2, state2}}; RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, states); test.SetActiveModules({true, true}); std::map initial_states = test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id); EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx); EXPECT_EQ(kInitialPictureId2, initial_states[kSsrc2].picture_id); EXPECT_EQ(kInitialTl0PicIdx2, initial_states[kSsrc2].tl0_pic_idx); EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc1].shared_frame_id); EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc2].shared_frame_id); } TEST(RtpVideoSenderTest, FrameCountCallbacks) { class MockFrameCountObserver : public FrameCountObserver { public: MOCK_METHOD(void, FrameCountUpdated, (const FrameCounts& frame_counts, uint32_t ssrc), (override)); } callback; RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}, &callback); constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); encoded_image._frameType = VideoFrameType::kVideoFrameKey; // No callbacks when not active. EXPECT_CALL(callback, FrameCountUpdated).Times(0); EXPECT_NE(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); ::testing::Mock::VerifyAndClearExpectations(&callback); test.SetActiveModules({true}); FrameCounts frame_counts; EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) .WillOnce(SaveArg<0>(&frame_counts)); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); EXPECT_EQ(1, frame_counts.key_frames); EXPECT_EQ(0, frame_counts.delta_frames); ::testing::Mock::VerifyAndClearExpectations(&callback); encoded_image._frameType = VideoFrameType::kVideoFrameDelta; EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) .WillOnce(SaveArg<0>(&frame_counts)); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); EXPECT_EQ(1, frame_counts.key_frames); EXPECT_EQ(1, frame_counts.delta_frames); } // Integration test verifying that ack of packet via TransportFeedback means // that the packet is removed from RtpPacketHistory and won't be retransmitted // again. TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.SetActiveModules({true, true}); constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); // Send two tiny images, mapping to two RTP packets. Capture sequence numbers. std::vector rtp_sequence_numbers; std::vector transport_sequence_numbers; EXPECT_CALL(test.transport(), SendRtp) .Times(2) .WillRepeatedly([&rtp_sequence_numbers, &transport_sequence_numbers]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber()); transport_sequence_numbers.push_back(options.packet_id); return true; }); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); encoded_image.SetTimestamp(2); encoded_image.capture_time_ms_ = 3; EXPECT_EQ(EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr).error); test.AdvanceTime(TimeDelta::Millis(33)); // Construct a NACK message for requesting retransmission of both packet. rtcp::Nack nack; nack.SetMediaSsrc(kSsrc1); nack.SetPacketIds(rtp_sequence_numbers); rtc::Buffer nack_buffer = nack.Build(); std::vector retransmitted_rtp_sequence_numbers; EXPECT_CALL(test.transport(), SendRtp) .Times(2) .WillRepeatedly([&retransmitted_rtp_sequence_numbers]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Capture the retransmitted sequence number from the RTX header. rtc::ArrayView payload = rtp_packet.payload(); retransmitted_rtp_sequence_numbers.push_back( ByteReader::ReadBigEndian(payload.data())); return true; }); test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size()); test.AdvanceTime(TimeDelta::Millis(33)); // Verify that both packets were retransmitted. EXPECT_EQ(retransmitted_rtp_sequence_numbers, rtp_sequence_numbers); // Simulate transport feedback indicating fist packet received, next packet // lost (not other way around as that would trigger early retransmit). StreamFeedbackObserver::StreamPacketInfo lost_packet_feedback; lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0]; lost_packet_feedback.ssrc = kSsrc1; lost_packet_feedback.received = false; lost_packet_feedback.is_retransmission = false; StreamFeedbackObserver::StreamPacketInfo received_packet_feedback; received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1]; received_packet_feedback.ssrc = kSsrc1; received_packet_feedback.received = true; lost_packet_feedback.is_retransmission = false; test.router()->OnPacketFeedbackVector( {lost_packet_feedback, received_packet_feedback}); // Advance time to make sure retransmission would be allowed and try again. // This time the retransmission should not happen for the first packet since // the history has been notified of the ack and removed the packet. The // second packet, included in the feedback but not marked as received, should // still be retransmitted. test.AdvanceTime(TimeDelta::Millis(33)); EXPECT_CALL(test.transport(), SendRtp) .WillOnce([&lost_packet_feedback](const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Capture the retransmitted sequence number from the RTX header. rtc::ArrayView payload = rtp_packet.payload(); EXPECT_EQ(lost_packet_feedback.rtp_sequence_number, ByteReader::ReadBigEndian(payload.data())); return true; }); test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size()); test.AdvanceTime(TimeDelta::Millis(33)); } // This tests that we utilize transport wide feedback to retransmit lost // packets. This is tested by dropping all ordinary packets from a "lossy" // stream sent along with a secondary untouched stream. The transport wide // feedback packets from the secondary stream allows the sending side to // detect and retreansmit the lost packets from the lossy stream. TEST(RtpVideoSenderTest, RetransmitsOnTransportWideLossInfo) { int rtx_packets; test::Scenario s(test_info_); test::CallClientConfig call_conf; // Keeping the bitrate fixed to avoid RTX due to probing. call_conf.transport.rates.max_rate = DataRate::KilobitsPerSec(300); call_conf.transport.rates.start_rate = DataRate::KilobitsPerSec(300); test::NetworkSimulationConfig net_conf; net_conf.bandwidth = DataRate::KilobitsPerSec(300); auto send_node = s.CreateSimulationNode(net_conf); auto* callee = s.CreateClient("return", call_conf); auto* route = s.CreateRoutes(s.CreateClient("send", call_conf), {send_node}, callee, {s.CreateSimulationNode(net_conf)}); test::VideoStreamConfig lossy_config; lossy_config.source.framerate = 5; auto* lossy = s.CreateVideoStream(route->forward(), lossy_config); // The secondary stream acts a driver for transport feedback messages, // ensuring that lost packets on the lossy stream are retransmitted. s.CreateVideoStream(route->forward(), test::VideoStreamConfig()); send_node->router()->SetFilter([&](const EmulatedIpPacket& packet) { RtpPacket rtp; if (rtp.Parse(packet.data)) { // Drops all regular packets for the lossy stream and counts all RTX // packets. Since no packets are let trough, NACKs can't be triggered // by the receiving side. if (lossy->send()->UsingSsrc(rtp.Ssrc())) { return false; } else if (lossy->send()->UsingRtxSsrc(rtp.Ssrc())) { ++rtx_packets; } } return true; }); // Run for a short duration and reset counters to avoid counting RTX packets // from initial probing. s.RunFor(TimeDelta::Seconds(1)); rtx_packets = 0; int decoded_baseline = 0; callee->SendTask([&decoded_baseline, &lossy]() { decoded_baseline = lossy->receive()->GetStats().frames_decoded; }); s.RunFor(TimeDelta::Seconds(1)); // We expect both that RTX packets were sent and that an appropriate number of // frames were received. This is somewhat redundant but reduces the risk of // false positives in future regressions (e.g. RTX is send due to probing). EXPECT_GE(rtx_packets, 1); int frames_decoded = 0; callee->SendTask([&decoded_baseline, &frames_decoded, &lossy]() { frames_decoded = lossy->receive()->GetStats().frames_decoded - decoded_baseline; }); EXPECT_EQ(frames_decoded, 5); } // Integration test verifying that retransmissions are sent for packets which // can be detected as lost early, using transport wide feedback. TEST(RtpVideoSenderTest, EarlyRetransmits) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.SetActiveModules({true, true}); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); encoded_image.SetSpatialIndex(0); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; // Send two tiny images, mapping to single RTP packets. Capture sequence // numbers. uint16_t frame1_rtp_sequence_number = 0; uint16_t frame1_transport_sequence_number = 0; EXPECT_CALL(test.transport(), SendRtp) .WillOnce( [&frame1_rtp_sequence_number, &frame1_transport_sequence_number]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); frame1_rtp_sequence_number = rtp_packet.SequenceNumber(); frame1_transport_sequence_number = options.packet_id; EXPECT_EQ(rtp_packet.Ssrc(), kSsrc1); return true; }); EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); uint16_t frame2_rtp_sequence_number = 0; uint16_t frame2_transport_sequence_number = 0; encoded_image.SetSpatialIndex(1); EXPECT_CALL(test.transport(), SendRtp) .WillOnce( [&frame2_rtp_sequence_number, &frame2_transport_sequence_number]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); frame2_rtp_sequence_number = rtp_packet.SequenceNumber(); frame2_transport_sequence_number = options.packet_id; EXPECT_EQ(rtp_packet.Ssrc(), kSsrc2); return true; }); EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); EXPECT_NE(frame1_transport_sequence_number, frame2_transport_sequence_number); // Inject a transport feedback where the packet for the first frame is lost, // expect a retransmission for it. EXPECT_CALL(test.transport(), SendRtp) .WillOnce([&frame1_rtp_sequence_number](const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Retransmitted sequence number from the RTX header should match // the lost packet. rtc::ArrayView payload = rtp_packet.payload(); EXPECT_EQ(ByteReader::ReadBigEndian(payload.data()), frame1_rtp_sequence_number); return true; }); StreamFeedbackObserver::StreamPacketInfo first_packet_feedback; first_packet_feedback.rtp_sequence_number = frame1_rtp_sequence_number; first_packet_feedback.ssrc = kSsrc1; first_packet_feedback.received = false; first_packet_feedback.is_retransmission = false; StreamFeedbackObserver::StreamPacketInfo second_packet_feedback; second_packet_feedback.rtp_sequence_number = frame2_rtp_sequence_number; second_packet_feedback.ssrc = kSsrc2; second_packet_feedback.received = true; first_packet_feedback.is_retransmission = false; test.router()->OnPacketFeedbackVector( {first_packet_feedback, second_packet_feedback}); // Wait for pacer to run and send the RTX packet. test.AdvanceTime(TimeDelta::Millis(33)); } TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { FrameDependencyTemplate().T(0).Dtis("S"), FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. // Send in key frame. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(0).Dtis("S").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(1)); EXPECT_TRUE( sent_packets.back().HasExtension()); // Send in delta frame. encoded_image._frameType = VideoFrameType::kVideoFrameDelta; codec_specific.template_structure = absl::nullopt; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(1).Dtis("D").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}}; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE( sent_packets.back().HasExtension()); } TEST(RtpVideoSenderTest, SupportsDependencyDescriptorForVp8NotProvidedByEncoder) { constexpr uint8_t kPayload[1] = {'a'}; RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault( [&](const uint8_t* packet, size_t length, const PacketOptions&) { EXPECT_TRUE( sent_packets.emplace_back(&extensions).Parse(packet, length)); return true; }); test.SetActiveModules({true}); EncodedImage key_frame_image; key_frame_image._frameType = VideoFrameType::kVideoFrameKey; key_frame_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo key_frame_info; key_frame_info.codecType = VideoCodecType::kVideoCodecVP8; ASSERT_EQ( test.router()->OnEncodedImage(key_frame_image, &key_frame_info).error, EncodedImageCallback::Result::OK); EncodedImage delta_image; delta_image._frameType = VideoFrameType::kVideoFrameDelta; delta_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo delta_info; delta_info.codecType = VideoCodecType::kVideoCodecVP8; ASSERT_EQ(test.router()->OnEncodedImage(delta_image, &delta_info).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(123)); DependencyDescriptor key_frame_dd; DependencyDescriptor delta_dd; ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE(sent_packets[0].GetExtension( /*structure=*/nullptr, &key_frame_dd)); EXPECT_TRUE(sent_packets[1].GetExtension( key_frame_dd.attached_structure.get(), &delta_dd)); } TEST(RtpVideoSenderTest, SupportsDependencyDescriptorForVp9) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecVP9; codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 2; codec_specific.template_structure->templates = { FrameDependencyTemplate().S(0).Dtis("SS"), FrameDependencyTemplate().S(1).Dtis("-S").FrameDiffs({1}), }; // Send two tiny images, each mapping to single RTP packet. // Send in key frame for the base spatial layer. codec_specific.generic_frame_info = GenericFrameInfo::Builder().S(0).Dtis("SS").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); // Send in 2nd spatial layer. codec_specific.template_structure = absl::nullopt; codec_specific.generic_frame_info = GenericFrameInfo::Builder().S(1).Dtis("-S").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}, {1, false, true}}; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE(sent_packets[0].HasExtension()); EXPECT_TRUE(sent_packets[1].HasExtension()); } TEST(RtpVideoSenderTest, SupportsDependencyDescriptorForVp9NotProvidedByEncoder) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image._encodedWidth = 320; encoded_image._encodedHeight = 180; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecVP9; codec_specific.codecSpecific.VP9.num_spatial_layers = 1; codec_specific.codecSpecific.VP9.temporal_idx = kNoTemporalIdx; codec_specific.codecSpecific.VP9.first_frame_in_picture = true; codec_specific.end_of_picture = true; codec_specific.codecSpecific.VP9.inter_pic_predicted = false; // Send two tiny images, each mapping to single RTP packet. EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); // Send in 2nd picture. encoded_image._frameType = VideoFrameType::kVideoFrameDelta; encoded_image.SetTimestamp(3000); codec_specific.codecSpecific.VP9.inter_pic_predicted = true; codec_specific.codecSpecific.VP9.num_ref_pics = 1; codec_specific.codecSpecific.VP9.p_diff[0] = 1; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE(sent_packets[0].HasExtension()); EXPECT_TRUE(sent_packets[1].HasExtension()); } TEST(RtpVideoSenderTest, GenerateDependecyDescriptorForGenericCodecs) { test::ScopedKeyValueConfig field_trials( "WebRTC-GenericCodecDependencyDescriptor/Enabled/"); RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}, &field_trials); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image._encodedWidth = 320; encoded_image._encodedHeight = 180; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; codec_specific.end_of_picture = true; // Send two tiny images, each mapping to single RTP packet. EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); // Send in 2nd picture. encoded_image._frameType = VideoFrameType::kVideoFrameDelta; encoded_image.SetTimestamp(3000); EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE(sent_packets[0].HasExtension()); EXPECT_TRUE(sent_packets[1].HasExtension()); } TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { FrameDependencyTemplate().T(0).Dtis("S"), FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. // Send in a key frame. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(0).Dtis("S").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(1)); EXPECT_TRUE( sent_packets.back().HasExtension()); // Send in a new key frame without the support for the dependency descriptor. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.template_structure = absl::nullopt; EXPECT_EQ(test.router()->OnEncodedImage(encoded_image, &codec_specific).error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_FALSE( sent_packets.back().HasExtension()); } TEST(RtpVideoSenderTest, CanSetZeroBitrate) { RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(0), /*framerate*/ 0); } TEST(RtpVideoSenderTest, SimulcastSenderRegistersFrameTransformers) { rtc::scoped_refptr transformer = rtc::make_ref_counted(); EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc1)); EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc2)); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}, nullptr, transformer); EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc1)); EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc2)); } TEST(RtpVideoSenderTest, OverheadIsSubtractedFromTargetBitrate) { test::ScopedKeyValueConfig field_trials( "WebRTC-Video-UseFrameRateForOverhead/Enabled/"); // TODO(jakobi): RTP header size should not be hard coded. constexpr uint32_t kRtpHeaderSizeBytes = 20; constexpr uint32_t kTransportPacketOverheadBytes = 40; constexpr uint32_t kOverheadPerPacketBytes = kRtpHeaderSizeBytes + kTransportPacketOverheadBytes; RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}, &field_trials); test.router()->OnTransportOverheadChanged(kTransportPacketOverheadBytes); test.SetActiveModules({true}); { test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(300000), /*framerate*/ 15); // 1 packet per frame. EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 300000 - kOverheadPerPacketBytes * 8 * 30); } { test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(150000), /*framerate*/ 15); // 1 packet per frame. EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 150000 - kOverheadPerPacketBytes * 8 * 15); } { test.router()->OnBitrateUpdated(CreateBitrateAllocationUpdate(1000000), /*framerate*/ 30); // 3 packets per frame. EXPECT_EQ(test.router()->GetPayloadBitrateBps(), 1000000 - kOverheadPerPacketBytes * 8 * 30 * 3); } } TEST(RtpVideoSenderTest, ClearsPendingPacketsOnInactivation) { RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); test.SetActiveModules({true}); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); // Set a very low bitrate. test.router()->OnBitrateUpdated( CreateBitrateAllocationUpdate(/*rate_bps=*/30'000), /*framerate=*/30); // Create and send a large keyframe. const size_t kImageSizeBytes = 10000; constexpr uint8_t kPayload[kImageSizeBytes] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, /*codec_specific=*/nullptr) .error, EncodedImageCallback::Result::OK); // Advance time a small amount, check that sent data is only part of the // image. test.AdvanceTime(TimeDelta::Millis(5)); DataSize transmittedPayload = DataSize::Zero(); for (const RtpPacket& packet : sent_packets) { transmittedPayload += DataSize::Bytes(packet.payload_size()); // Make sure we don't see the end of the frame. EXPECT_FALSE(packet.Marker()); } EXPECT_GT(transmittedPayload, DataSize::Zero()); EXPECT_LT(transmittedPayload, DataSize::Bytes(kImageSizeBytes / 4)); // Record the RTP timestamp of the first frame. const uint32_t first_frame_timestamp = sent_packets[0].Timestamp(); sent_packets.clear(); // Disable the sending module and advance time slightly. No packets should be // sent. test.SetActiveModules({false}); test.AdvanceTime(TimeDelta::Millis(20)); EXPECT_TRUE(sent_packets.empty()); // Reactive the send module - any packets should have been removed, so nothing // should be transmitted. test.SetActiveModules({true}); test.AdvanceTime(TimeDelta::Millis(33)); EXPECT_TRUE(sent_packets.empty()); // Send a new frame. encoded_image.SetTimestamp(3); encoded_image.capture_time_ms_ = 4; EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, /*codec_specific=*/nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); // Advance time, check we get new packets - but only for the second frame. EXPECT_FALSE(sent_packets.empty()); EXPECT_NE(sent_packets[0].Timestamp(), first_frame_timestamp); } } // namespace webrtc