Adding upstream version 115.7.0esr.upstream/115.7.0esr

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 332 insertions, 0 deletions

diff --git a/third_party/libwebrtc/api/video/frame_buffer.cc b/third_party/libwebrtc/api/video/frame_buffer.cc
new file mode 100644
index 0000000000..f5d93f5f76
--- /dev/null
+++ b/third_party/libwebrtc/api/video/frame_buffer.cc
@@ -0,0 +1,332 @@
+/*
+ *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/frame_buffer.h"
+
+#include <algorithm>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/sequence_number_util.h"
+#include "rtc_base/trace_event.h"
+
+namespace webrtc {
+namespace {
+bool ValidReferences(const EncodedFrame& frame) {
+  // All references must point backwards, and duplicates are not allowed.
+  for (size_t i = 0; i < frame.num_references; ++i) {
+    if (frame.references[i] >= frame.Id())
+      return false;
+
+    for (size_t j = i + 1; j < frame.num_references; ++j) {
+      if (frame.references[i] == frame.references[j])
+        return false;
+    }
+  }
+
+  return true;
+}
+
+// Since FrameBuffer::FrameInfo is private it can't be used in the function
+// signature, hence the FrameIteratorT type.
+template <typename FrameIteratorT>
+rtc::ArrayView<const int64_t> GetReferences(const FrameIteratorT& it) {
+  return {it->second.encoded_frame->references,
+          std::min<size_t>(it->second.encoded_frame->num_references,
+                           EncodedFrame::kMaxFrameReferences)};
+}
+
+template <typename FrameIteratorT>
+int64_t GetFrameId(const FrameIteratorT& it) {
+  return it->first;
+}
+
+template <typename FrameIteratorT>
+uint32_t GetTimestamp(const FrameIteratorT& it) {
+  return it->second.encoded_frame->Timestamp();
+}
+
+template <typename FrameIteratorT>
+bool IsLastFrameInTemporalUnit(const FrameIteratorT& it) {
+  return it->second.encoded_frame->is_last_spatial_layer;
+}
+}  // namespace
+
+FrameBuffer::FrameBuffer(int max_size,
+                         int max_decode_history,
+                         const FieldTrialsView& field_trials)
+    : legacy_frame_id_jump_behavior_(
+          !field_trials.IsDisabled("WebRTC-LegacyFrameIdJumpBehavior")),
+      max_size_(max_size),
+      decoded_frame_history_(max_decode_history) {}
+
+bool FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
+  const uint32_t ssrc =
+      frame->PacketInfos().empty() ? 0 : frame->PacketInfos()[0].ssrc();
+  if (!ValidReferences(*frame)) {
+    TRACE_EVENT2("webrtc",
+                 "FrameBuffer::InsertFrame Frame dropped (Invalid references)",
+                 "remote_ssrc", ssrc, "frame_id", frame->Id());
+    RTC_DLOG(LS_WARNING) << "Frame " << frame->Id()
+                         << " has invalid references, dropping frame.";
+    return false;
+  }
+
+  if (frame->Id() <= decoded_frame_history_.GetLastDecodedFrameId()) {
+    if (legacy_frame_id_jump_behavior_ && frame->is_keyframe() &&
+        AheadOf(frame->Timestamp(),
+                *decoded_frame_history_.GetLastDecodedFrameTimestamp())) {
+      TRACE_EVENT2("webrtc",
+                   "FrameBuffer::InsertFrame Frames dropped (OOO + PicId jump)",
+                   "remote_ssrc", ssrc, "frame_id", frame->Id());
+      RTC_DLOG(LS_WARNING)
+          << "Keyframe " << frame->Id()
+          << " has newer timestamp but older picture id, clearing buffer.";
+      Clear();
+    } else {
+      // Already decoded past this frame.
+      TRACE_EVENT2("webrtc",
+                   "FrameBuffer::InsertFrame Frame dropped (Out of order)",
+                   "remote_ssrc", ssrc, "frame_id", frame->Id());
+      return false;
+    }
+  }
+
+  if (frames_.size() == max_size_) {
+    if (frame->is_keyframe()) {
+      TRACE_EVENT2("webrtc",
+                   "FrameBuffer::InsertFrame Frames dropped (KF + Full buffer)",
+                   "remote_ssrc", ssrc, "frame_id", frame->Id());
+      RTC_DLOG(LS_WARNING) << "Keyframe " << frame->Id()
+                           << " inserted into full buffer, clearing buffer.";
+      Clear();
+    } else {
+      // No space for this frame.
+      TRACE_EVENT2("webrtc",
+                   "FrameBuffer::InsertFrame Frame dropped (Full buffer)",
+                   "remote_ssrc", ssrc, "frame_id", frame->Id());
+      return false;
+    }
+  }
+
+  const int64_t frame_id = frame->Id();
+  auto insert_res = frames_.emplace(frame_id, FrameInfo{std::move(frame)});
+  if (!insert_res.second) {
+    // Frame has already been inserted.
+    return false;
+  }
+
+  if (frames_.size() == max_size_) {
+    RTC_DLOG(LS_WARNING) << "Frame " << frame_id
+                         << " inserted, buffer is now full.";
+  }
+
+  PropagateContinuity(insert_res.first);
+  FindNextAndLastDecodableTemporalUnit();
+  return true;
+}
+
+absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4>
+FrameBuffer::ExtractNextDecodableTemporalUnit() {
+  absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> res;
+  if (!next_decodable_temporal_unit_) {
+    return res;
+  }
+
+  auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
+  for (auto it = next_decodable_temporal_unit_->first_frame; it != end_it;
+       ++it) {
+    decoded_frame_history_.InsertDecoded(GetFrameId(it), GetTimestamp(it));
+    res.push_back(std::move(it->second.encoded_frame));
+  }
+
+  DropNextDecodableTemporalUnit();
+  return res;
+}
+
+void FrameBuffer::DropNextDecodableTemporalUnit() {
+  if (!next_decodable_temporal_unit_) {
+    return;
+  }
+
+  auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
+
+  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it);
+
+  frames_.erase(frames_.begin(), end_it);
+  FindNextAndLastDecodableTemporalUnit();
+}
+
+void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it,
+                                                         FrameIterator end_it) {
+  uint32_t dropped_ssrc = 0;
+  int64_t dropped_frame_id = 0;
+  unsigned int num_discarded_packets = 0;
+  unsigned int num_dropped_frames =
+      std::count_if(begin_it, end_it, [&](const auto& f) {
+        if (f.second.encoded_frame) {
+          const auto& packetInfos = f.second.encoded_frame->PacketInfos();
+          dropped_frame_id = f.first;
+          if (!packetInfos.empty()) {
+            dropped_ssrc = packetInfos[0].ssrc();
+          }
+          num_discarded_packets += packetInfos.size();
+        }
+        return f.second.encoded_frame != nullptr;
+      });
+
+  if (num_dropped_frames > 0) {
+    TRACE_EVENT2("webrtc", "FrameBuffer Dropping Old Frames", "remote_ssrc",
+                 dropped_ssrc, "frame_id", dropped_frame_id);
+  }
+  if (num_discarded_packets > 0) {
+    TRACE_EVENT2("webrtc", "FrameBuffer Discarding Old Packets", "remote_ssrc",
+                 dropped_ssrc, "frame_id", dropped_frame_id);
+  }
+
+  num_dropped_frames_ += num_dropped_frames;
+  num_discarded_packets_ += num_discarded_packets;
+}
+
+absl::optional<int64_t> FrameBuffer::LastContinuousFrameId() const {
+  return last_continuous_frame_id_;
+}
+
+absl::optional<int64_t> FrameBuffer::LastContinuousTemporalUnitFrameId() const {
+  return last_continuous_temporal_unit_frame_id_;
+}
+
+absl::optional<FrameBuffer::DecodabilityInfo>
+FrameBuffer::DecodableTemporalUnitsInfo() const {
+  return decodable_temporal_units_info_;
+}
+
+int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
+  return num_continuous_temporal_units_;
+}
+int FrameBuffer::GetTotalNumberOfDroppedFrames() const {
+  return num_dropped_frames_;
+}
+int FrameBuffer::GetTotalNumberOfDiscardedPackets() const {
+  return num_discarded_packets_;
+}
+
+size_t FrameBuffer::CurrentSize() const {
+  return frames_.size();
+}
+
+bool FrameBuffer::IsContinuous(const FrameIterator& it) const {
+  for (int64_t reference : GetReferences(it)) {
+    if (decoded_frame_history_.WasDecoded(reference)) {
+      continue;
+    }
+
+    auto reference_frame_it = frames_.find(reference);
+    if (reference_frame_it != frames_.end() &&
+        reference_frame_it->second.continuous) {
+      continue;
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+void FrameBuffer::PropagateContinuity(const FrameIterator& frame_it) {
+  for (auto it = frame_it; it != frames_.end(); ++it) {
+    if (!it->second.continuous) {
+      if (IsContinuous(it)) {
+        it->second.continuous = true;
+        if (last_continuous_frame_id_ < GetFrameId(it)) {
+          last_continuous_frame_id_ = GetFrameId(it);
+        }
+        if (IsLastFrameInTemporalUnit(it)) {
+          num_continuous_temporal_units_++;
+          if (last_continuous_temporal_unit_frame_id_ < GetFrameId(it)) {
+            last_continuous_temporal_unit_frame_id_ = GetFrameId(it);
+          }
+        }
+      }
+    }
+  }
+}
+
+void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
+  next_decodable_temporal_unit_.reset();
+  decodable_temporal_units_info_.reset();
+
+  if (!last_continuous_temporal_unit_frame_id_) {
+    return;
+  }
+
+  FrameIterator first_frame_it = frames_.begin();
+  FrameIterator last_frame_it = frames_.begin();
+  absl::InlinedVector<int64_t, 4> frames_in_temporal_unit;
+  uint32_t last_decodable_temporal_unit_timestamp;
+  for (auto frame_it = frames_.begin(); frame_it != frames_.end();) {
+    if (GetFrameId(frame_it) > *last_continuous_temporal_unit_frame_id_) {
+      break;
+    }
+
+    if (GetTimestamp(frame_it) != GetTimestamp(first_frame_it)) {
+      frames_in_temporal_unit.clear();
+      first_frame_it = frame_it;
+    }
+
+    frames_in_temporal_unit.push_back(GetFrameId(frame_it));
+
+    last_frame_it = frame_it++;
+
+    if (IsLastFrameInTemporalUnit(last_frame_it)) {
+      bool temporal_unit_decodable = true;
+      for (auto it = first_frame_it; it != frame_it && temporal_unit_decodable;
+           ++it) {
+        for (int64_t reference : GetReferences(it)) {
+          if (!decoded_frame_history_.WasDecoded(reference) &&
+              !absl::c_linear_search(frames_in_temporal_unit, reference)) {
+            // A frame in the temporal unit has a non-decoded reference outside
+            // the temporal unit, so it's not yet ready to be decoded.
+            temporal_unit_decodable = false;
+            break;
+          }
+        }
+      }
+
+      if (temporal_unit_decodable) {
+        if (!next_decodable_temporal_unit_) {
+          next_decodable_temporal_unit_ = {first_frame_it, last_frame_it};
+        }
+
+        last_decodable_temporal_unit_timestamp = GetTimestamp(first_frame_it);
+      }
+    }
+  }
+
+  if (next_decodable_temporal_unit_) {
+    decodable_temporal_units_info_ = {
+        .next_rtp_timestamp =
+            GetTimestamp(next_decodable_temporal_unit_->first_frame),
+        .last_rtp_timestamp = last_decodable_temporal_unit_timestamp};
+  }
+}
+
+void FrameBuffer::Clear() {
+  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end());
+  frames_.clear();
+  next_decodable_temporal_unit_.reset();
+  decodable_temporal_units_info_.reset();
+  last_continuous_frame_id_.reset();
+  last_continuous_temporal_unit_frame_id_.reset();
+  decoded_frame_history_.Clear();
+}
+
+}  // namespace webrtc