diff options
Diffstat (limited to '')
| -rw-r--r-- | third_party/libwebrtc/api/video/frame_buffer.cc | 332 |
1 files changed, 332 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/video/frame_buffer.cc b/third_party/libwebrtc/api/video/frame_buffer.cc new file mode 100644 index 0000000000..f5d93f5f76 --- /dev/null +++ b/third_party/libwebrtc/api/video/frame_buffer.cc @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/frame_buffer.h" + +#include <algorithm> + +#include "absl/algorithm/container.h" +#include "absl/container/inlined_vector.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/trace_event.h" + +namespace webrtc { +namespace { +bool ValidReferences(const EncodedFrame& frame) { + // All references must point backwards, and duplicates are not allowed. + for (size_t i = 0; i < frame.num_references; ++i) { + if (frame.references[i] >= frame.Id()) + return false; + + for (size_t j = i + 1; j < frame.num_references; ++j) { + if (frame.references[i] == frame.references[j]) + return false; + } + } + + return true; +} + +// Since FrameBuffer::FrameInfo is private it can't be used in the function +// signature, hence the FrameIteratorT type. +template <typename FrameIteratorT> +rtc::ArrayView<const int64_t> GetReferences(const FrameIteratorT& it) { + return {it->second.encoded_frame->references, + std::min<size_t>(it->second.encoded_frame->num_references, + EncodedFrame::kMaxFrameReferences)}; +} + +template <typename FrameIteratorT> +int64_t GetFrameId(const FrameIteratorT& it) { + return it->first; +} + +template <typename FrameIteratorT> +uint32_t GetTimestamp(const FrameIteratorT& it) { + return it->second.encoded_frame->Timestamp(); +} + +template <typename FrameIteratorT> +bool IsLastFrameInTemporalUnit(const FrameIteratorT& it) { + return it->second.encoded_frame->is_last_spatial_layer; +} +} // namespace + +FrameBuffer::FrameBuffer(int max_size, + int max_decode_history, + const FieldTrialsView& field_trials) + : legacy_frame_id_jump_behavior_( + !field_trials.IsDisabled("WebRTC-LegacyFrameIdJumpBehavior")), + max_size_(max_size), + decoded_frame_history_(max_decode_history) {} + +bool FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) { + const uint32_t ssrc = + frame->PacketInfos().empty() ? 0 : frame->PacketInfos()[0].ssrc(); + if (!ValidReferences(*frame)) { + TRACE_EVENT2("webrtc", + "FrameBuffer::InsertFrame Frame dropped (Invalid references)", + "remote_ssrc", ssrc, "frame_id", frame->Id()); + RTC_DLOG(LS_WARNING) << "Frame " << frame->Id() + << " has invalid references, dropping frame."; + return false; + } + + if (frame->Id() <= decoded_frame_history_.GetLastDecodedFrameId()) { + if (legacy_frame_id_jump_behavior_ && frame->is_keyframe() && + AheadOf(frame->Timestamp(), + *decoded_frame_history_.GetLastDecodedFrameTimestamp())) { + TRACE_EVENT2("webrtc", + "FrameBuffer::InsertFrame Frames dropped (OOO + PicId jump)", + "remote_ssrc", ssrc, "frame_id", frame->Id()); + RTC_DLOG(LS_WARNING) + << "Keyframe " << frame->Id() + << " has newer timestamp but older picture id, clearing buffer."; + Clear(); + } else { + // Already decoded past this frame. + TRACE_EVENT2("webrtc", + "FrameBuffer::InsertFrame Frame dropped (Out of order)", + "remote_ssrc", ssrc, "frame_id", frame->Id()); + return false; + } + } + + if (frames_.size() == max_size_) { + if (frame->is_keyframe()) { + TRACE_EVENT2("webrtc", + "FrameBuffer::InsertFrame Frames dropped (KF + Full buffer)", + "remote_ssrc", ssrc, "frame_id", frame->Id()); + RTC_DLOG(LS_WARNING) << "Keyframe " << frame->Id() + << " inserted into full buffer, clearing buffer."; + Clear(); + } else { + // No space for this frame. + TRACE_EVENT2("webrtc", + "FrameBuffer::InsertFrame Frame dropped (Full buffer)", + "remote_ssrc", ssrc, "frame_id", frame->Id()); + return false; + } + } + + const int64_t frame_id = frame->Id(); + auto insert_res = frames_.emplace(frame_id, FrameInfo{std::move(frame)}); + if (!insert_res.second) { + // Frame has already been inserted. + return false; + } + + if (frames_.size() == max_size_) { + RTC_DLOG(LS_WARNING) << "Frame " << frame_id + << " inserted, buffer is now full."; + } + + PropagateContinuity(insert_res.first); + FindNextAndLastDecodableTemporalUnit(); + return true; +} + +absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> +FrameBuffer::ExtractNextDecodableTemporalUnit() { + absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> res; + if (!next_decodable_temporal_unit_) { + return res; + } + + auto end_it = std::next(next_decodable_temporal_unit_->last_frame); + for (auto it = next_decodable_temporal_unit_->first_frame; it != end_it; + ++it) { + decoded_frame_history_.InsertDecoded(GetFrameId(it), GetTimestamp(it)); + res.push_back(std::move(it->second.encoded_frame)); + } + + DropNextDecodableTemporalUnit(); + return res; +} + +void FrameBuffer::DropNextDecodableTemporalUnit() { + if (!next_decodable_temporal_unit_) { + return; + } + + auto end_it = std::next(next_decodable_temporal_unit_->last_frame); + + UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it); + + frames_.erase(frames_.begin(), end_it); + FindNextAndLastDecodableTemporalUnit(); +} + +void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it, + FrameIterator end_it) { + uint32_t dropped_ssrc = 0; + int64_t dropped_frame_id = 0; + unsigned int num_discarded_packets = 0; + unsigned int num_dropped_frames = + std::count_if(begin_it, end_it, [&](const auto& f) { + if (f.second.encoded_frame) { + const auto& packetInfos = f.second.encoded_frame->PacketInfos(); + dropped_frame_id = f.first; + if (!packetInfos.empty()) { + dropped_ssrc = packetInfos[0].ssrc(); + } + num_discarded_packets += packetInfos.size(); + } + return f.second.encoded_frame != nullptr; + }); + + if (num_dropped_frames > 0) { + TRACE_EVENT2("webrtc", "FrameBuffer Dropping Old Frames", "remote_ssrc", + dropped_ssrc, "frame_id", dropped_frame_id); + } + if (num_discarded_packets > 0) { + TRACE_EVENT2("webrtc", "FrameBuffer Discarding Old Packets", "remote_ssrc", + dropped_ssrc, "frame_id", dropped_frame_id); + } + + num_dropped_frames_ += num_dropped_frames; + num_discarded_packets_ += num_discarded_packets; +} + +absl::optional<int64_t> FrameBuffer::LastContinuousFrameId() const { + return last_continuous_frame_id_; +} + +absl::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const { + return last_continuous_temporal_unit_frame_id_; +} + +absl::optional<FrameBuffer::DecodabilityInfo> +FrameBuffer::DecodableTemporalUnitsInfo() const { + return decodable_temporal_units_info_; +} + +int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const { + return num_continuous_temporal_units_; +} +int FrameBuffer::GetTotalNumberOfDroppedFrames() const { + return num_dropped_frames_; +} +int FrameBuffer::GetTotalNumberOfDiscardedPackets() const { + return num_discarded_packets_; +} + +size_t FrameBuffer::CurrentSize() const { + return frames_.size(); +} + +bool FrameBuffer::IsContinuous(const FrameIterator& it) const { + for (int64_t reference : GetReferences(it)) { + if (decoded_frame_history_.WasDecoded(reference)) { + continue; + } + + auto reference_frame_it = frames_.find(reference); + if (reference_frame_it != frames_.end() && + reference_frame_it->second.continuous) { + continue; + } + + return false; + } + + return true; +} + +void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) { + for (auto it = frame_it; it != frames_.end(); ++it) { + if (!it->second.continuous) { + if (IsContinuous(it)) { + it->second.continuous = true; + if (last_continuous_frame_id_ < GetFrameId(it)) { + last_continuous_frame_id_ = GetFrameId(it); + } + if (IsLastFrameInTemporalUnit(it)) { + num_continuous_temporal_units_++; + if (last_continuous_temporal_unit_frame_id_ < GetFrameId(it)) { + last_continuous_temporal_unit_frame_id_ = GetFrameId(it); + } + } + } + } + } +} + +void FrameBuffer::FindNextAndLastDecodableTemporalUnit() { + next_decodable_temporal_unit_.reset(); + decodable_temporal_units_info_.reset(); + + if (!last_continuous_temporal_unit_frame_id_) { + return; + } + + FrameIterator first_frame_it = frames_.begin(); + FrameIterator last_frame_it = frames_.begin(); + absl::InlinedVector<int64_t, 4> frames_in_temporal_unit; + uint32_t last_decodable_temporal_unit_timestamp; + for (auto frame_it = frames_.begin(); frame_it != frames_.end();) { + if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) { + break; + } + + if (GetTimestamp(frame_it) != GetTimestamp(first_frame_it)) { + frames_in_temporal_unit.clear(); + first_frame_it = frame_it; + } + + frames_in_temporal_unit.push_back(GetFrameId(frame_it)); + + last_frame_it = frame_it++; + + if (IsLastFrameInTemporalUnit(last_frame_it)) { + bool temporal_unit_decodable = true; + for (auto it = first_frame_it; it != frame_it && temporal_unit_decodable; + ++it) { + for (int64_t reference : GetReferences(it)) { + if (!decoded_frame_history_.WasDecoded(reference) && + !absl::c_linear_search(frames_in_temporal_unit, reference)) { + // A frame in the temporal unit has a non-decoded reference outside + // the temporal unit, so it's not yet ready to be decoded. + temporal_unit_decodable = false; + break; + } + } + } + + if (temporal_unit_decodable) { + if (!next_decodable_temporal_unit_) { + next_decodable_temporal_unit_ = {first_frame_it, last_frame_it}; + } + + last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it); + } + } + } + + if (next_decodable_temporal_unit_) { + decodable_temporal_units_info_ = { + .next_rtp_timestamp = + GetTimestamp(next_decodable_temporal_unit_->first_frame), + .last_rtp_timestamp = last_decodable_temporal_unit_timestamp}; + } +} + +void FrameBuffer::Clear() { + UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end()); + frames_.clear(); + next_decodable_temporal_unit_.reset(); + decodable_temporal_units_info_.reset(); + last_continuous_frame_id_.reset(); + last_continuous_temporal_unit_frame_id_.reset(); + decoded_frame_history_.Clear(); +} + +} // namespace webrtc |