summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/api/video/frame_buffer.cc
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/libwebrtc/api/video/frame_buffer.cc')
-rw-r--r--third_party/libwebrtc/api/video/frame_buffer.cc332
1 files changed, 332 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/video/frame_buffer.cc b/third_party/libwebrtc/api/video/frame_buffer.cc
new file mode 100644
index 0000000000..f5d93f5f76
--- /dev/null
+++ b/third_party/libwebrtc/api/video/frame_buffer.cc
@@ -0,0 +1,332 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/frame_buffer.h"
+
+#include <algorithm>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/sequence_number_util.h"
+#include "rtc_base/trace_event.h"
+
+namespace webrtc {
+namespace {
+bool ValidReferences(const EncodedFrame& frame) {
+ // All references must point backwards, and duplicates are not allowed.
+ for (size_t i = 0; i < frame.num_references; ++i) {
+ if (frame.references[i] >= frame.Id())
+ return false;
+
+ for (size_t j = i + 1; j < frame.num_references; ++j) {
+ if (frame.references[i] == frame.references[j])
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Since FrameBuffer::FrameInfo is private it can't be used in the function
+// signature, hence the FrameIteratorT type.
+template <typename FrameIteratorT>
+rtc::ArrayView<const int64_t> GetReferences(const FrameIteratorT& it) {
+ return {it->second.encoded_frame->references,
+ std::min<size_t>(it->second.encoded_frame->num_references,
+ EncodedFrame::kMaxFrameReferences)};
+}
+
+template <typename FrameIteratorT>
+int64_t GetFrameId(const FrameIteratorT& it) {
+ return it->first;
+}
+
+template <typename FrameIteratorT>
+uint32_t GetTimestamp(const FrameIteratorT& it) {
+ return it->second.encoded_frame->Timestamp();
+}
+
+template <typename FrameIteratorT>
+bool IsLastFrameInTemporalUnit(const FrameIteratorT& it) {
+ return it->second.encoded_frame->is_last_spatial_layer;
+}
+} // namespace
+
+FrameBuffer::FrameBuffer(int max_size,
+ int max_decode_history,
+ const FieldTrialsView& field_trials)
+ : legacy_frame_id_jump_behavior_(
+ !field_trials.IsDisabled("WebRTC-LegacyFrameIdJumpBehavior")),
+ max_size_(max_size),
+ decoded_frame_history_(max_decode_history) {}
+
+bool FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
+ const uint32_t ssrc =
+ frame->PacketInfos().empty() ? 0 : frame->PacketInfos()[0].ssrc();
+ if (!ValidReferences(*frame)) {
+ TRACE_EVENT2("webrtc",
+ "FrameBuffer::InsertFrame Frame dropped (Invalid references)",
+ "remote_ssrc", ssrc, "frame_id", frame->Id());
+ RTC_DLOG(LS_WARNING) << "Frame " << frame->Id()
+ << " has invalid references, dropping frame.";
+ return false;
+ }
+
+ if (frame->Id() <= decoded_frame_history_.GetLastDecodedFrameId()) {
+ if (legacy_frame_id_jump_behavior_ && frame->is_keyframe() &&
+ AheadOf(frame->Timestamp(),
+ *decoded_frame_history_.GetLastDecodedFrameTimestamp())) {
+ TRACE_EVENT2("webrtc",
+ "FrameBuffer::InsertFrame Frames dropped (OOO + PicId jump)",
+ "remote_ssrc", ssrc, "frame_id", frame->Id());
+ RTC_DLOG(LS_WARNING)
+ << "Keyframe " << frame->Id()
+ << " has newer timestamp but older picture id, clearing buffer.";
+ Clear();
+ } else {
+ // Already decoded past this frame.
+ TRACE_EVENT2("webrtc",
+ "FrameBuffer::InsertFrame Frame dropped (Out of order)",
+ "remote_ssrc", ssrc, "frame_id", frame->Id());
+ return false;
+ }
+ }
+
+ if (frames_.size() == max_size_) {
+ if (frame->is_keyframe()) {
+ TRACE_EVENT2("webrtc",
+ "FrameBuffer::InsertFrame Frames dropped (KF + Full buffer)",
+ "remote_ssrc", ssrc, "frame_id", frame->Id());
+ RTC_DLOG(LS_WARNING) << "Keyframe " << frame->Id()
+ << " inserted into full buffer, clearing buffer.";
+ Clear();
+ } else {
+ // No space for this frame.
+ TRACE_EVENT2("webrtc",
+ "FrameBuffer::InsertFrame Frame dropped (Full buffer)",
+ "remote_ssrc", ssrc, "frame_id", frame->Id());
+ return false;
+ }
+ }
+
+ const int64_t frame_id = frame->Id();
+ auto insert_res = frames_.emplace(frame_id, FrameInfo{std::move(frame)});
+ if (!insert_res.second) {
+ // Frame has already been inserted.
+ return false;
+ }
+
+ if (frames_.size() == max_size_) {
+ RTC_DLOG(LS_WARNING) << "Frame " << frame_id
+ << " inserted, buffer is now full.";
+ }
+
+ PropagateContinuity(insert_res.first);
+ FindNextAndLastDecodableTemporalUnit();
+ return true;
+}
+
+absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4>
+FrameBuffer::ExtractNextDecodableTemporalUnit() {
+ absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> res;
+ if (!next_decodable_temporal_unit_) {
+ return res;
+ }
+
+ auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
+ for (auto it = next_decodable_temporal_unit_->first_frame; it != end_it;
+ ++it) {
+ decoded_frame_history_.InsertDecoded(GetFrameId(it), GetTimestamp(it));
+ res.push_back(std::move(it->second.encoded_frame));
+ }
+
+ DropNextDecodableTemporalUnit();
+ return res;
+}
+
+void FrameBuffer::DropNextDecodableTemporalUnit() {
+ if (!next_decodable_temporal_unit_) {
+ return;
+ }
+
+ auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
+
+ UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it);
+
+ frames_.erase(frames_.begin(), end_it);
+ FindNextAndLastDecodableTemporalUnit();
+}
+
+void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it,
+ FrameIterator end_it) {
+ uint32_t dropped_ssrc = 0;
+ int64_t dropped_frame_id = 0;
+ unsigned int num_discarded_packets = 0;
+ unsigned int num_dropped_frames =
+ std::count_if(begin_it, end_it, [&](const auto& f) {
+ if (f.second.encoded_frame) {
+ const auto& packetInfos = f.second.encoded_frame->PacketInfos();
+ dropped_frame_id = f.first;
+ if (!packetInfos.empty()) {
+ dropped_ssrc = packetInfos[0].ssrc();
+ }
+ num_discarded_packets += packetInfos.size();
+ }
+ return f.second.encoded_frame != nullptr;
+ });
+
+ if (num_dropped_frames > 0) {
+ TRACE_EVENT2("webrtc", "FrameBuffer Dropping Old Frames", "remote_ssrc",
+ dropped_ssrc, "frame_id", dropped_frame_id);
+ }
+ if (num_discarded_packets > 0) {
+ TRACE_EVENT2("webrtc", "FrameBuffer Discarding Old Packets", "remote_ssrc",
+ dropped_ssrc, "frame_id", dropped_frame_id);
+ }
+
+ num_dropped_frames_ += num_dropped_frames;
+ num_discarded_packets_ += num_discarded_packets;
+}
+
+absl::optional<int64_t> FrameBuffer::LastContinuousFrameId() const {
+ return last_continuous_frame_id_;
+}
+
+absl::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const {
+ return last_continuous_temporal_unit_frame_id_;
+}
+
+absl::optional<FrameBuffer::DecodabilityInfo>
+FrameBuffer::DecodableTemporalUnitsInfo() const {
+ return decodable_temporal_units_info_;
+}
+
+int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
+ return num_continuous_temporal_units_;
+}
+int FrameBuffer::GetTotalNumberOfDroppedFrames() const {
+ return num_dropped_frames_;
+}
+int FrameBuffer::GetTotalNumberOfDiscardedPackets() const {
+ return num_discarded_packets_;
+}
+
+size_t FrameBuffer::CurrentSize() const {
+ return frames_.size();
+}
+
+bool FrameBuffer::IsContinuous(const FrameIterator& it) const {
+ for (int64_t reference : GetReferences(it)) {
+ if (decoded_frame_history_.WasDecoded(reference)) {
+ continue;
+ }
+
+ auto reference_frame_it = frames_.find(reference);
+ if (reference_frame_it != frames_.end() &&
+ reference_frame_it->second.continuous) {
+ continue;
+ }
+
+ return false;
+ }
+
+ return true;
+}
+
+void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) {
+ for (auto it = frame_it; it != frames_.end(); ++it) {
+ if (!it->second.continuous) {
+ if (IsContinuous(it)) {
+ it->second.continuous = true;
+ if (last_continuous_frame_id_ < GetFrameId(it)) {
+ last_continuous_frame_id_ = GetFrameId(it);
+ }
+ if (IsLastFrameInTemporalUnit(it)) {
+ num_continuous_temporal_units_++;
+ if (last_continuous_temporal_unit_frame_id_ < GetFrameId(it)) {
+ last_continuous_temporal_unit_frame_id_ = GetFrameId(it);
+ }
+ }
+ }
+ }
+ }
+}
+
+void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
+ next_decodable_temporal_unit_.reset();
+ decodable_temporal_units_info_.reset();
+
+ if (!last_continuous_temporal_unit_frame_id_) {
+ return;
+ }
+
+ FrameIterator first_frame_it = frames_.begin();
+ FrameIterator last_frame_it = frames_.begin();
+ absl::InlinedVector<int64_t, 4> frames_in_temporal_unit;
+ uint32_t last_decodable_temporal_unit_timestamp;
+ for (auto frame_it = frames_.begin(); frame_it != frames_.end();) {
+ if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) {
+ break;
+ }
+
+ if (GetTimestamp(frame_it) != GetTimestamp(first_frame_it)) {
+ frames_in_temporal_unit.clear();
+ first_frame_it = frame_it;
+ }
+
+ frames_in_temporal_unit.push_back(GetFrameId(frame_it));
+
+ last_frame_it = frame_it++;
+
+ if (IsLastFrameInTemporalUnit(last_frame_it)) {
+ bool temporal_unit_decodable = true;
+ for (auto it = first_frame_it; it != frame_it && temporal_unit_decodable;
+ ++it) {
+ for (int64_t reference : GetReferences(it)) {
+ if (!decoded_frame_history_.WasDecoded(reference) &&
+ !absl::c_linear_search(frames_in_temporal_unit, reference)) {
+ // A frame in the temporal unit has a non-decoded reference outside
+ // the temporal unit, so it's not yet ready to be decoded.
+ temporal_unit_decodable = false;
+ break;
+ }
+ }
+ }
+
+ if (temporal_unit_decodable) {
+ if (!next_decodable_temporal_unit_) {
+ next_decodable_temporal_unit_ = {first_frame_it, last_frame_it};
+ }
+
+ last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it);
+ }
+ }
+ }
+
+ if (next_decodable_temporal_unit_) {
+ decodable_temporal_units_info_ = {
+ .next_rtp_timestamp =
+ GetTimestamp(next_decodable_temporal_unit_->first_frame),
+ .last_rtp_timestamp = last_decodable_temporal_unit_timestamp};
+ }
+}
+
+void FrameBuffer::Clear() {
+ UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end());
+ frames_.clear();
+ next_decodable_temporal_unit_.reset();
+ decodable_temporal_units_info_.reset();
+ last_continuous_frame_id_.reset();
+ last_continuous_temporal_unit_frame_id_.reset();
+ decoded_frame_history_.Clear();
+}
+
+} // namespace webrtc