From: Andreas Pehrson Date: Tue, 23 Nov 2021 14:11:00 +0000 Subject: Bug 1742181 - libwebrtc: Implement packetsDiscarded bookkeeping for received video. r=ng Depends on D131707 Differential Revision: https://phabricator.services.mozilla.com/D131708 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/d0196a45a1f449874fc2a759e85e403c45c25575 Also includes: Bug 1804288 - (fix-de7ae5755b) reimplement Bug 1742181 - libwebrtc: Implement packetsDiscarded bookkeeping for received video. r=pehrsons Differential Revision: https://phabricator.services.mozilla.com/D163959 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/ee566d1bfb654d36e5d58dce637fb0580b989ac1 --- api/video/frame_buffer.cc | 25 ++++++++++++++++--- api/video/frame_buffer.h | 4 +++ call/video_receive_stream.h | 2 ++ .../include/video_coding_defines.h | 2 ++ modules/video_coding/packet_buffer.cc | 10 +++++--- modules/video_coding/packet_buffer.h | 5 +++- video/receive_statistics_proxy2.cc | 5 ++++ video/receive_statistics_proxy2.h | 1 + video/rtp_video_stream_receiver2.cc | 5 +++- video/rtp_video_stream_receiver2.h | 2 ++ video/video_receive_stream2.cc | 1 + video/video_stream_buffer_controller.cc | 12 +++++++++ video/video_stream_buffer_controller.h | 3 +++ 13 files changed, 69 insertions(+), 8 deletions(-) diff --git a/api/video/frame_buffer.cc b/api/video/frame_buffer.cc index 4cdf2212a6..8267b8e6cb 100644 --- a/api/video/frame_buffer.cc +++ b/api/video/frame_buffer.cc @@ -140,14 +140,29 @@ void FrameBuffer::DropNextDecodableTemporalUnit() { } auto end_it = std::next(next_decodable_temporal_unit_->last_frame); - num_dropped_frames_ += std::count_if( - frames_.begin(), end_it, - [](const auto& f) { return f.second.encoded_frame != nullptr; }); + + UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it); frames_.erase(frames_.begin(), end_it); FindNextAndLastDecodableTemporalUnit(); } +void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it, + FrameIterator end_it) { + unsigned int num_discarded_packets = 0; + unsigned int num_dropped_frames = + std::count_if(begin_it, end_it, [&](const auto& f) { + if (f.second.encoded_frame) { + const auto& packetInfos = f.second.encoded_frame->PacketInfos(); + num_discarded_packets += packetInfos.size(); + } + return f.second.encoded_frame != nullptr; + }); + + num_dropped_frames_ += num_dropped_frames; + num_discarded_packets_ += num_discarded_packets; +} + absl::optional FrameBuffer::LastContinuousFrameId() const { return last_continuous_frame_id_; } @@ -167,6 +182,9 @@ int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const { int FrameBuffer::GetTotalNumberOfDroppedFrames() const { return num_dropped_frames_; } +int FrameBuffer::GetTotalNumberOfDiscardedPackets() const { + return num_discarded_packets_; +} size_t FrameBuffer::CurrentSize() const { return frames_.size(); @@ -269,6 +287,7 @@ void FrameBuffer::FindNextAndLastDecodableTemporalUnit() { } void FrameBuffer::Clear() { + UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end()); frames_.clear(); next_decodable_temporal_unit_.reset(); decodable_temporal_units_info_.reset(); diff --git a/api/video/frame_buffer.h b/api/video/frame_buffer.h index 94edf64d5a..81fd12da58 100644 --- a/api/video/frame_buffer.h +++ b/api/video/frame_buffer.h @@ -66,6 +66,7 @@ class FrameBuffer { int GetTotalNumberOfContinuousTemporalUnits() const; int GetTotalNumberOfDroppedFrames() const; + int GetTotalNumberOfDiscardedPackets() const; size_t CurrentSize() const; private: @@ -87,6 +88,8 @@ class FrameBuffer { void PropagateContinuity(const FrameIterator& frame_it); void FindNextAndLastDecodableTemporalUnit(); void Clear(); + void UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it, + FrameIterator end_it); const bool legacy_frame_id_jump_behavior_; const size_t max_size_; @@ -99,6 +102,7 @@ class FrameBuffer { int num_continuous_temporal_units_ = 0; int num_dropped_frames_ = 0; + int num_discarded_packets_ = 0; }; } // namespace webrtc diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 25c294a2a6..1ab4a2a85b 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -106,6 +106,8 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface { // https://www.w3.org/TR/webrtc-stats/#dom-rtcvideoreceiverstats-framesdropped uint32_t frames_dropped = 0; uint32_t frames_decoded = 0; + // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats-packetsdiscarded + uint64_t packets_discarded = 0; // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime TimeDelta total_decode_time = TimeDelta::Zero(); // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h index 8f70e0298d..bf98d5e668 100644 --- a/modules/video_coding/include/video_coding_defines.h +++ b/modules/video_coding/include/video_coding_defines.h @@ -76,6 +76,8 @@ class VCMReceiveStatisticsCallback { virtual void OnDroppedFrames(uint32_t frames_dropped) = 0; + virtual void OnDiscardedPackets(uint32_t packets_discarded) = 0; + virtual void OnFrameBufferTimingsUpdated(int max_decode_ms, int current_delay_ms, int target_delay_ms, diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 3dcfc48213..04f02fce97 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -115,25 +115,27 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket( return result; } -void PacketBuffer::ClearTo(uint16_t seq_num) { +uint32_t PacketBuffer::ClearTo(uint16_t seq_num) { // We have already cleared past this sequence number, no need to do anything. if (is_cleared_to_first_seq_num_ && AheadOf(first_seq_num_, seq_num)) { - return; + return 0; } // If the packet buffer was cleared between a frame was created and returned. if (!first_packet_received_) - return; + return 0; // Avoid iterating over the buffer more than once by capping the number of // iterations to the `size_` of the buffer. ++seq_num; + uint32_t num_cleared_packets = 0; size_t diff = ForwardDiff(first_seq_num_, seq_num); size_t iterations = std::min(diff, buffer_.size()); for (size_t i = 0; i < iterations; ++i) { auto& stored = buffer_[first_seq_num_ % buffer_.size()]; if (stored != nullptr && AheadOf(seq_num, stored->seq_num)) { + ++num_cleared_packets; stored = nullptr; } ++first_seq_num_; @@ -149,6 +151,8 @@ void PacketBuffer::ClearTo(uint16_t seq_num) { received_padding_.erase(received_padding_.begin(), received_padding_.lower_bound(seq_num)); + + return num_cleared_packets; } void PacketBuffer::Clear() { diff --git a/modules/video_coding/packet_buffer.h b/modules/video_coding/packet_buffer.h index 53e08c95a1..47b2ffe199 100644 --- a/modules/video_coding/packet_buffer.h +++ b/modules/video_coding/packet_buffer.h @@ -78,7 +78,10 @@ class PacketBuffer { ABSL_MUST_USE_RESULT InsertResult InsertPacket(std::unique_ptr packet); ABSL_MUST_USE_RESULT InsertResult InsertPadding(uint16_t seq_num); - void ClearTo(uint16_t seq_num); + + // Clear all packets older than |seq_num|. Returns the number of packets + // cleared. + uint32_t ClearTo(uint16_t seq_num); void Clear(); void ForceSpsPpsIdrIsH264Keyframe(); diff --git a/video/receive_statistics_proxy2.cc b/video/receive_statistics_proxy2.cc index 4f208a1d5e..020e4bb0ae 100644 --- a/video/receive_statistics_proxy2.cc +++ b/video/receive_statistics_proxy2.cc @@ -959,6 +959,11 @@ void ReceiveStatisticsProxy::OnDroppedFrames(uint32_t frames_dropped) { })); } +void ReceiveStatisticsProxy::OnDiscardedPackets(uint32_t packets_discarded) { + RTC_DCHECK_RUN_ON(&main_thread_); + stats_.packets_discarded += packets_discarded; +} + void ReceiveStatisticsProxy::OnPreDecode(VideoCodecType codec_type, int qp) { RTC_DCHECK_RUN_ON(&main_thread_); last_codec_type_ = codec_type; diff --git a/video/receive_statistics_proxy2.h b/video/receive_statistics_proxy2.h index 1a2bb77fa6..20139b45e5 100644 --- a/video/receive_statistics_proxy2.h +++ b/video/receive_statistics_proxy2.h @@ -90,6 +90,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, size_t size_bytes, VideoContentType content_type) override; void OnDroppedFrames(uint32_t frames_dropped) override; + void OnDiscardedPackets(uint32_t packets_discarded) override; void OnFrameBufferTimingsUpdated(int max_decode_ms, int current_delay_ms, int target_delay_ms, diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index 094f8f4a54..46998b6d7c 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -244,6 +244,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RtcpCnameCallback* rtcp_cname_callback, NackPeriodicProcessor* nack_periodic_processor, + VCMReceiveStatisticsCallback* vcm_receive_statistics, OnCompleteFrameCallback* complete_frame_callback, rtc::scoped_refptr frame_decryptor, rtc::scoped_refptr frame_transformer, @@ -293,6 +294,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( &rtcp_feedback_buffer_, &rtcp_feedback_buffer_, field_trials_)), + vcm_receive_statistics_(vcm_receive_statistics), packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize(field_trials_)), reference_finder_(std::make_unique()), @@ -1219,7 +1221,8 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num); packet_infos_.erase(packet_infos_.begin(), packet_infos_.upper_bound(unwrapped_rtp_seq_num)); - packet_buffer_.ClearTo(seq_num); + uint32_t num_packets_cleared = packet_buffer_.ClearTo(seq_num); + vcm_receive_statistics_->OnDiscardedPackets(num_packets_cleared); reference_finder_->ClearTo(seq_num); } } diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h index 6bf4bf8453..931525a054 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h @@ -91,6 +91,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RtcpCnameCallback* rtcp_cname_callback, NackPeriodicProcessor* nack_periodic_processor, + VCMReceiveStatisticsCallback* vcm_receive_statistics, // The KeyFrameRequestSender is optional; if not provided, key frame // requests are sent via the internal RtpRtcp module. OnCompleteFrameCallback* complete_frame_callback, @@ -368,6 +369,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, std::unique_ptr loss_notification_controller_ RTC_GUARDED_BY(packet_sequence_checker_); + VCMReceiveStatisticsCallback* const vcm_receive_statistics_; video_coding::PacketBuffer packet_buffer_ RTC_GUARDED_BY(packet_sequence_checker_); UniqueTimestampCounter frame_counter_ diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 7cbd49d322..beb894e139 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -211,6 +211,7 @@ VideoReceiveStream2::VideoReceiveStream2( &stats_proxy_, &stats_proxy_, nack_periodic_processor, + &stats_proxy_, this, // OnCompleteFrameCallback std::move(config_.frame_decryptor), std::move(config_.frame_transformer), diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc index f7d3acdaf6..7e44eff39a 100644 --- a/video/video_stream_buffer_controller.cc +++ b/video/video_stream_buffer_controller.cc @@ -247,6 +247,7 @@ void VideoStreamBufferController::OnFrameReady( // Update stats. UpdateDroppedFrames(); + UpdateDiscardedPackets(); UpdateJitterDelay(); UpdateTimingFrameInfo(); @@ -312,6 +313,17 @@ void VideoStreamBufferController::UpdateDroppedFrames() buffer_->GetTotalNumberOfDroppedFrames(); } +void VideoStreamBufferController::UpdateDiscardedPackets() + RTC_RUN_ON(&worker_sequence_checker_) { + const int discarded_packets = buffer_->GetTotalNumberOfDiscardedPackets() - + packets_discarded_before_last_new_frame_; + if (discarded_packets > 0) { + stats_proxy_->OnDiscardedPackets(discarded_packets); + } + packets_discarded_before_last_new_frame_ = + buffer_->GetTotalNumberOfDiscardedPackets(); +} + void VideoStreamBufferController::UpdateJitterDelay() { auto timings = timing_->GetTimings(); if (timings.num_decoded_frames) { diff --git a/video/video_stream_buffer_controller.h b/video/video_stream_buffer_controller.h index ed79b0fa1f..7638c91471 100644 --- a/video/video_stream_buffer_controller.h +++ b/video/video_stream_buffer_controller.h @@ -67,6 +67,7 @@ class VideoStreamBufferController { void OnTimeout(TimeDelta delay); void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time); void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_); + void UpdateDiscardedPackets() RTC_RUN_ON(&worker_sequence_checker_); void UpdateJitterDelay(); void UpdateTimingFrameInfo(); bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_); @@ -94,6 +95,8 @@ class VideoStreamBufferController { RTC_GUARDED_BY(&worker_sequence_checker_); int frames_dropped_before_last_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = 0; + int packets_discarded_before_last_new_frame_ + RTC_GUARDED_BY(&worker_sequence_checker_) = 0; VCMVideoProtection protection_mode_ RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack; -- 2.34.1